#!/usr/bin/env bash
#
# pypi-mirror.sh
# =============================================================================
# Offline PyPI mirror + installer for air-gapped tiCrypt research VMs.
#
# This script is meant to live on an NFS share that both an internet-connected
# "staging" machine and internet-less researcher VMs can see. The staging
# machine runs `mirror` (and later `update`/`add`) to pull packages down into
# a `pypi-packages/` directory next to this script. Researcher VMs, which
# cannot reach the internet, run `install` to install from that local mirror
# using `pip install --no-index --find-links=...`.
#
# Usage:
#   ./pypi-mirror.sh mirror                 # (internet machine) download the curated set
#   ./pypi-mirror.sh update                 # re-run mirror to refresh to latest versions
#   ./pypi-mirror.sh install [pkg ...]      # (offline VM) install from the local mirror
#   ./pypi-mirror.sh search <term>          # search package filenames in the mirror
#   ./pypi-mirror.sh add <pkg> [pkg ...]    # add package(s) to the curated list + download
#   ./pypi-mirror.sh list                   # list all mirrored packages + versions
#   ./pypi-mirror.sh --help                 # show detailed usage
#
# On-disk layout (all relative to this script's directory):
#   pypi-packages/         downloaded wheels / sdists (the actual mirror)
#   logs/                  timestamped run logs
#   custom-packages.txt    user-added packages, persists across `update` runs
#   pip.conf               generated pip config researchers can copy to ~/.pip/pip.conf
#   .pypi-mirror.lock      lockfile preventing concurrent mirror/add runs
#
# Requirements: python3 + pip only. No other external dependencies.
# =============================================================================

set -euo pipefail
shopt -s nullglob

# -----------------------------------------------------------------------------
# Paths & constants
# -----------------------------------------------------------------------------
SCRIPT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"

PACKAGES_DIR="${SCRIPT_PATH}/pypi-packages"
LOG_DIR="${SCRIPT_PATH}/logs"
LOCK_FILE="${SCRIPT_PATH}/.pypi-mirror.lock"
CUSTOM_LIST="${SCRIPT_PATH}/custom-packages.txt"
PIP_CONF_OUT="${SCRIPT_PATH}/pip.conf"
MANIFEST_FILE="${SCRIPT_PATH}/manifest.json"

TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
LOG_FILE="${LOG_DIR}/pypi-mirror-${TIMESTAMP}.log"

# Python versions / platform to mirror wheels for. Adjust here if the fleet
# of researcher VMs changes.
PYTHON_VERSIONS=(3.9 3.10 3.11 3.12)
PLATFORM_TAG="manylinux2014_x86_64"

MAX_RETRIES=3
RETRY_DELAY=5

EXTRA_INDEX="https://pypi.nvidia.com"

# Run-level counters used by print_summary().
SUCCESS_COUNT=0
FAIL_COUNT=0
SKIP_COUNT=0
FAILED_PACKAGES=()
SDIST_FALLBACK=()

LOCK_ACQUIRED=0
LOCK_FD=200

# -----------------------------------------------------------------------------
# Curated package list, grouped by category for readability/maintenance.
# `mirror` (with no extra args) and `install` (with no extra args) both
# operate on this list plus anything in custom-packages.txt.
# -----------------------------------------------------------------------------
# Core numerical & data (34)
CATEGORY_CORE_NUMERICAL=(
  numpy scipy sympy mpmath numba cython numexpr bottleneck pandas polars
  pyarrow fastparquet dask distributed dask-ml xarray zarr h5py netCDF4
  tables cftime joblib cloudpickle tqdm more-itertools sortedcontainers
  intervaltree python-dateutil pytz tzdata pyyaml psutil py-cpuinfo duckdb
)
# Statistics & Bayesian (22)
CATEGORY_STATISTICS=(
  statsmodels patsy formulaic pingouin scikit-posthocs lifelines pymc arviz
  numpyro cmdstanpy bambi linearmodels factor-analyzer semopy hmmlearn
  filterpy arch uncertainties lmfit emcee dynesty corner
)
# Causal inference (6)
CATEGORY_CAUSAL=(econml dowhy causal-learn causalml dice-ml pgmpy)
# Classical ML (26)
CATEGORY_CLASSICAL_ML=(
  scikit-learn xgboost lightgbm catboost ngboost imbalanced-learn
  category-encoders feature-engine featuretools umap-learn hdbscan pynndescent
  openTSNE optuna hyperopt scikit-optimize bayesian-optimization nevergrad
  skorch mlxtend yellowbrick cleanlab river networkx igraph sdv
)
# Deep learning frameworks (32)
CATEGORY_DEEP_LEARNING=(
  torch torchvision torchaudio torchdata pytorch-lightning torchmetrics
  torchinfo tensorflow tensorflow-datasets tensorflow-probability keras
  keras-tuner jax jaxlib flax optax equinox chex orbax-checkpoint dm-haiku
  transformers datasets tokenizers accelerate safetensors huggingface-hub
  einops onnx onnxruntime-gpu torch-geometric dgl skl2onnx
)
# LLM & generative AI (28)
CATEGORY_LLM=(
  vllm sglang deepspeed fairscale xformers peft trl bitsandbytes optimum
  autoawq llama-cpp-python ctranslate2 sentencepiece tiktoken
  sentence-transformers diffusers evaluate langchain langchain-community
  langgraph llama-index haystack-ai dspy-ai litellm outlines lm-eval ragas
  openai
)
# Vector search & retrieval (12)
CATEGORY_VECTOR_SEARCH=(
  faiss-cpu chromadb qdrant-client weaviate-client pymilvus lancedb hnswlib
  annoy usearch txtai rank-bm25 pgvector
)
# GPU / RAPIDS (14)
CATEGORY_GPU_RAPIDS=(
  cupy-cuda12x cudf-cu12 cuml-cu12 cugraph-cu12 cuspatial-cu12 rmm-cu12
  dask-cudf-cu12 nx-cugraph-cu12 cuda-python pycuda pyopencl tensorrt
  nvidia-ml-py nvitop
)
# Distributed & workflow (13)
CATEGORY_DISTRIBUTED=(
  ray mpi4py dask-jobqueue dask-mpi submitit parsl prefect luigi metaflow
  loky multiprocess pathos snakemake
)
# Interpretability & robustness (11)
CATEGORY_INTERPRETABILITY=(
  shap lime eli5 captum interpret alibi alibi-detect fairlearn aif360
  adversarial-robustness-toolbox foolbox
)
# MLOps & serving (15)
CATEGORY_MLOPS=(
  mlflow wandb tensorboard tensorboardX comet-ml neptune aim clearml sacred
  dvc kedro bentoml fastapi uvicorn label-studio
)
# Reinforcement learning (7)
CATEGORY_RL=(gymnasium stable-baselines3 sb3-contrib pettingzoo tianshou minari shimmy)
# Time series (12)
CATEGORY_TIMESERIES=(
  statsforecast mlforecast prophet pmdarima sktime tsfresh tslearn stumpy
  ruptures neuralforecast darts gluonts
)
# Computer vision (14)
CATEGORY_CV=(
  timm ultralytics supervision segmentation-models-pytorch albumentations
  kornia opencv-python pillow scikit-image imageio tifffile imagecodecs
  mahotas av
)
# Bioimaging & microscopy (7)
CATEGORY_BIOIMAGING=(aicsimageio napari cellpose stardist trackpy nd2reader imutils)
# Audio & speech (12)
CATEGORY_AUDIO=(
  librosa soundfile audioread pydub noisereduce speechbrain openai-whisper
  faster-whisper whisperx pyannote-audio praat-parselmouth madmom
)
# NLP (20)
CATEGORY_NLP=(
  nltk spacy spacy-transformers gensim textblob textacy stanza flair bertopic
  pyLDAvis sacrebleu rouge-score bert-score langdetect ftfy rapidfuzz
  textstat vaderSentiment wordcloud pytesseract
)
# Genomics & bioinformatics (20)
CATEGORY_GENOMICS=(
  biopython pysam pybedtools pyranges bioframe cyvcf2 pyfaidx gffutils HTSeq
  cutadapt multiqc scanpy anndata scvi-tools squidpy muon scikit-bio gseapy
  ete3 scvelo
)
# Chemistry & materials (16)
CATEGORY_CHEMISTRY=(
  rdkit biotite MDAnalysis mdtraj prody openmm ase pymatgen spglib phonopy
  deepchem nglview py3Dmol biopandas cantera pint
)
# Neuroimaging & medical (13)
CATEGORY_NEURO_MEDICAL=(
  nibabel nilearn nipype dipy mne mne-bids pybids antspyx pydicom SimpleITK
  itk monai torchio
)
# Geospatial & climate (24)
CATEGORY_GEOSPATIAL=(
  geopandas shapely fiona pyproj rtree rasterio rioxarray cartopy folium
  contextily osmnx geopy pysal libpysal esda verde pystac pystac-client
  stackstac satpy metpy cfgrib xclim obspy
)
# Astronomy & physics (18)
CATEGORY_ASTRO_PHYSICS=(
  astropy astroquery photutils specutils sunpy healpy reproject qutip qiskit
  cirq pennylane openfermion uproot awkward hist particle iminuit pyhf
)
# Engineering & optimization (18)
CATEGORY_ENGINEERING=(
  meshio gmsh pyvista vtk trimesh open3d scikit-fem fipy pyomo cvxpy pulp
  highspy casadi control simpy nlopt pymoo deap
)
# Visualization (22)
CATEGORY_VISUALIZATION=(
  matplotlib seaborn plotly bokeh altair holoviews hvplot panel datashader
  param colorcet plotnine mplfinance adjustText SciencePlots cmcrameri
  graphviz pydot dash streamlit gradio ipywidgets
)
# Notebooks & docs (16)
CATEGORY_NOTEBOOKS=(
  jupyter jupyterlab notebook ipython ipykernel nbconvert nbformat jupytext
  papermill voila jupyter-book myst-parser sphinx sphinx-rtd-theme numpydoc
  mkdocs-material
)
# Dev, test & config (20)
CATEGORY_DEV=(
  pytest pytest-cov pytest-xdist pytest-benchmark hypothesis coverage tox nox
  pre-commit black ruff isort mypy pylint poetry python-dotenv pydantic
  hydra-core omegaconf click
)
# Profiling (6)
CATEGORY_PROFILING=(memory-profiler line-profiler scalene py-spy pyinstrument snakeviz)
# Data IO & documents (18)
CATEGORY_DATA_IO=(
  requests httpx aiohttp beautifulsoup4 lxml scrapy selenium openpyxl
  xlsxwriter python-docx python-pptx pypdf pdfplumber PyMuPDF orjson
  protobuf pyreadstat pyreadr
)
# Databases & cloud storage (16)
CATEGORY_DATABASES=(
  sqlalchemy alembic psycopg2-binary pymysql pymongo redis sqlite-utils
  ibis-framework fsspec s3fs gcsfs adlfs boto3 google-cloud-storage paramiko
  globus-sdk
)
# Data quality & misc (12)
CATEGORY_DATA_QUALITY=(
  pandera great-expectations ydata-profiling pyjanitor missingno rich loguru
  cryptography deepchecks faker attrs regex
)
# Signal processing & differential equations (10)
CATEGORY_SIGNAL_DIFFEQ=(
  PyWavelets torchdiffeq diffrax torchsde scipy-signal pywt-wavelets
  signalp findpeaks emd-signal stockwell
)
# Anomaly detection & outlier analysis (8)
CATEGORY_ANOMALY=(
  pyod alibi-detect suod combo pythresh pyodds luminol adtk
)
# Survival analysis (4)
CATEGORY_SURVIVAL=(scikit-survival lifelines2 auton-survival pycox)
# AutoML & hyperparameter (8)
CATEGORY_AUTOML=(
  auto-sklearn FLAML autogluon autokeras pycaret lazypredict
  neural-compressor torch-pruning
)
# Privacy & federated learning (8)
CATEGORY_PRIVACY=(
  opacus flower tenseal pysyft differential-privacy dp-accounting
  crypten syft
)
# Financial & econometrics (10)
CATEGORY_FINANCIAL=(
  quantlib zipline-reloaded bt alphalens empyrical pyfolio
  arch-model fredapi yfinance ta
)
# PDF, OCR & document processing (8)
CATEGORY_PDF_OCR=(
  pdf2image ocrmypdf camelot-py tabula-py pdfminer.six pikepdf
  img2pdf doctr
)
# SSH, automation & containers (8)
CATEGORY_AUTOMATION=(
  fabric invoke docker paramiko-expect plumbum sshtunnel rpyc ansible-core
)
# Graph & knowledge databases (8)
CATEGORY_GRAPH_DB=(
  neo4j py2neo arangodb python-arango rdflib sparqlwrapper owlready2 pykg2vec
)
# Web scraping & automation (6)
CATEGORY_WEB_SCRAPING=(
  playwright playwright-stealth httpcore selectolax parsel newspaper3k
)
# Large-scale data & streaming (10)
CATEGORY_LARGE_SCALE=(
  vaex modin koalas pyspark delta-spark kafka-python confluent-kafka
  faust-streaming great-tables fugue
)
# Symbolic & formal methods (6)
CATEGORY_SYMBOLIC=(z3-solver pysat python-constraint symengine galois gmpy2)
# Survey, social science & text annotation (8)
CATEGORY_SOCIAL_SCIENCE=(
  prodigy-recipes label-studio-sdk argilla rubrix spaCy-llm
  sklearn-crfsuite dedupe recordlinkage
)
# Recommender systems (6)
CATEGORY_RECOMMENDER=(surprise implicit cornac lenskit recbole lightfm)
# Geospatial extensions (6)
CATEGORY_GEO_EXTRA=(
  sentinelsat eolearn h3 movingpandas trajectools keplergl
)
# Reproducibility & environment (8)
CATEGORY_REPRODUCIBILITY=(
  pipreqs pip-tools conda-pack conda-lock virtualenv pipenv
  setuptools wheel
)
# Common transitive dependencies (16)
CATEGORY_COMMON_DEPS=(
  certifi charset-normalizer urllib3 idna packaging setuptools wheel
  six wrapt decorator filelock platformdirs appdirs typing-extensions
  importlib-metadata zipp pyparsing
)

# Top PyPI packages by download count (auto-generated 2026-08-31)
CATEGORY_TOP_PYPI=(
  botocore cffi pluggy pygments aiobotocore pycparser iniconfig anyio pydantic-core grpcio-status
  s3transfer h11 annotated-types markupsafe typing-inspection pathspec jinja2 pyjwt jmespath yarl
  pip markdown-it-py rpds-py starlette jsonschema multidict google-auth propcache pyasn1 aiohappyeyeballs
  frozenlist mdurl referencing websockets opentelemetry-semantic-conventions trove-classifiers jsonschema-specifications aiosignal opentelemetry-sdk googleapis-common-protos
  sniffio hatchling google-api-core greenlet opentelemetry-api pyasn1-modules annotated-doc pydantic-settings grpcio textual
  colorama tenacity soupsieve distro python-multipart opentelemetry-exporter-otlp-proto-http opentelemetry-proto tomli opentelemetry-instrumentation jiter
  shellingham requests-oauthlib typer watchfiles editables cachetools proto-plus et-xmlfile opentelemetry-exporter-otlp-proto-common awscli
  tomlkit oauthlib exceptiongroup dnspython distlib mcp mypy-extensions opentelemetry-exporter-otlp-proto-grpc sse-starlette wcwidth
  requests-toolbelt rsa google-genai pyopenssl python-discovery msgpack grpcio-tools pytest-asyncio docstring-parser pytest-json-ctrf
  werkzeug gitpython hf-xet uvloop google-cloud-bigquery httptools smmap azure-core azure-identity pynacl
  keyring defusedxml tabulate httpx-sse email-validator fonttools prompt-toolkit isodate websocket-client jaraco-classes
  jaraco-functools jeepney opentelemetry-instrumentation-requests ghapi gitdb secretstorage docutils tzlocal msal itsdangerous
  opentelemetry-util-http jaraco-context google-cloud-core bcrypt flask ydb dill kiwisolver ptyprocess threadpoolctl
  mako ruamel-yaml nodeenv pexpect chardet google-crc32c zstandard snowflake-connector-python google-resumable-media blinker
  google-api-python-client contourpy deprecated opentelemetry-exporter-otlp uv jsonpointer identify uritemplate anthropic httplib2
  hpack google-auth-httplib2 cfgv prometheus-client google-cloud-secret-manager xxhash cycler toml langchain-core google-auth-oauthlib
  h2 opentelemetry-instrumentation-threading google-cloud-aiplatform asn1crypto poetry-core pyproject-hooks execnet hyperframe sentry-sdk msal-extensions
  backoff async-timeout aiofiles google-cloud-batch build jsonpatch joserfc authlib pydantic-ai-slim fastjsonschema
  traitlets narwhals asgiref google-cloud-kms google-analytics-admin sqlparse librt google-cloud-compute webencodings executing
  babel azure-storage-blob jedi databricks-sdk gunicorn parso asttokens setuptools-scm grpc-google-iam-v1 google-cloud-dlp
  matplotlib-inline tornado types-requests cachecontrol google-cloud-texttospeech xmltodict marshmallow stack-data pure-eval vcs-versioning
  google-cloud-speech python-json-logger markdown dbt-adapters databricks-sql-connector google-cloud-pubsub psycopg langsmith asyncpg cfn-lint
  kubernetes dbt-core structlog tinycss2 watchdog py4j pydantic-graph psycopg-binary pyzmq pytokens
  pyperclip debugpy aioitertools termcolor pytest-mock slack-sdk lz4 importlib-resources nest-asyncio pytest-timeout
  pycryptodome jsonschema-path google-cloud-logging flatbuffers pandas-stubs google-cloud-tasks snowflake-snowpark-python jupyter-core typedload opentelemetry-exporter-prometheus
  deprecation dulwich beartype tomli-w griffelib google-cloud-monitoring cyclopts pathable jsonref jupyter-client
  fastuuid brotli llama-parse mdit-py-plugins opentelemetry-instrumentation-fastapi llama-cloud-services mccabe pkginfo installer croniter
  rich-rst semver dbt-common opentelemetry-instrumentation-asgi py-key-value-aio xlrd onnxruntime google-cloud-bigtable python-slugify pycodestyle
  wsproto text-unidecode linkify-it-py rich-toolkit fastmcp markdownify typing-inspect awswrangler pyee truststore
  ruamel-yaml-clib google-cloud-vision uc-micro-py poetry-plugin-export argcomplete graphql-core grpcio-health-checking sqlglot crashtest google-cloud-run
  msrest ast-serialize deepdiff fastmcp-slim pygithub rfc3339-validator google-cloud-videointelligence google-cloud-language google-cloud-workflows types-pyyaml
  arrow cleo uuid-utils mistune google-cloud-redis comm google-cloud-dataform sqlalchemy-bigquery lark snowflake-sqlalchemy
  google-cloud-os-login pytest-rerunfailures tree-sitter google-cloud-bigquery-datatransfer requests-file bleach durationpy ipython-pygments-lexers google-cloud-dataproc-metastore backports-zstd
  google-cloud-orchestration-airflow typeguard backports-tarfile google-cloud-memcache aiosqlite pypdfium2 reportlab uncalled-for google-cloud-automl pycryptodomex
  aiofile pendulum fastapi-cli argon2-cffi argon2-cffi-bindings cattrs mmh3 jsonpath-ng smart-open pdfminer-six
  caio triton humanize responses future google-cloud-dataflow-client datadog types-protobuf cbor2 pydantic-extra-types
  langchain-openai llvmlite pysocks google-cloud-resource-manager py types-toml dataclasses-json openapi-pydantic scramp zope-interface
  nbclient pyflakes colorlog azure-common portalocker pbs-installer jupyterlab-pygments toolz simplejson azure-keyvault-secrets
  pandocfilters freezegun ecdsa types-certifi flask-cors click-plugins grpclib orderly-set types-python-dateutil jupyter-server
  psycopg2 kombu modal langgraph-prebuilt flake8 celery types-awscrt absl-py fastapi-cloud-cli langgraph-sdk
  botocore-stubs findpython json5 prettytable polars-runtime-32 trio opentelemetry-instrumentation-httpx httpx2 langchain-protocol elasticsearch
  opensearch-py redshift-connector django astroid httpcore2 langgraph-checkpoint pymssql aws-sam-translator google-cloud-spanner humanfriendly
  nh3 vine types-s3transfer amqp synchronicity billiard google-cloud-appengine-logging semantic-version ijson browser-use
  outcome ormsgpack cssselect2 click-didyoumean nvidia-nccl-cu12 click-repl gevent mypy-boto3-s3 cuda-pathfinder antlr4-python3-runtime
  libcst peewee inflection webcolors apscheduler fakeredis setproctitle isoduration cuda-bindings opencv-python-headless
  boto3-stubs gcloud-aio-auth opentelemetry-instrumentation-urllib3 mistralai mysql-connector-python opentelemetry-instrumentation-dbapi google-cloud-translate posthog azure-monitor-opentelemetry-exporter rich-click
  pywin32 requests-aws4auth opentelemetry-instrumentation-wsgi langchain-text-splitters google-cloud-audit-log opentelemetry-instrumentation-django google-cloud-container opentelemetry-instrumentation-psycopg2 async-lru dateparser
  tableauserverclient moto ddtrace google-cloud-datacatalog openapi-spec-validator fqdn db-dtypes gcloud-aio-storage uri-template fastcore
  opentelemetry-instrumentation-flask bracex gspread opentelemetry-instrumentation-urllib apache-airflow-providers-common-sql python-jose fastavro filetype limits send2trash
  rfc3986 rfc3986-validator pg8000 thrift opentelemetry-instrumentation-logging jupyterlab-server widgetsnbextension google-cloud-storage-transfer adal pinotdb
  msgspec wcmatch ply jupyterlab-widgets mlflow-skinny lazy-object-proxy pyodbc nvidia-cublas stripe nvidia-cuda-nvrtc
  nvidia-nvjitlink nvidia-cudnn-cu13 jupyter-events nvidia-nccl-cu13 python-telegram-bot zeep unidiff nvidia-cusparselt-cu13 realtime terminado
  bytecode nvidia-nvshmem-cu13 pyright overrides nvidia-cusparse jupyter-server-terminals rignore graphql-relay nest-asyncio2 pyrsistent
  cuda-toolkit nvidia-cufft readme-renderer nvidia-cusolver graphene nvidia-curand pyarrow-hotfix google-cloud-firestore ordered-set trio-websocket
  google-ads aioboto3 cohere google-cloud-bigquery-storage envier yamllint strenum diskcache pycountry nvidia-cuda-runtime
  types-pytz aenum passlib weasyprint fastar coloredlogs mashumaro nvidia-cuda-cupti ty boolean-py
  license-expression pyroaring azure-mgmt-core llama-index-llms-openai notebook-shim nvidia-cufile pyiceberg jupyter-lsp nvidia-nvtx phonenumbers
  pyotp html5lib retrying llama-index-indices-managed-llama-cloud stevedore tldextract json-repair elastic-transport entrypoints supabase
  retry databricks-sqlalchemy flit-core rfc3987-syntax curl-cffi azure-storage-queue oauth2client aiohttp-retry azure-storage-file-datalake storage3
  postgrest eval-type-backport zopfli twine strands-agents lmnr ml-dtypes snowballstemmer pybreaker langchain-google-vertexai
  cyclonedx-python-lib pyphen semgrep mergedeep pytest-metadata packageurl-python openai-agents parsedatetime agate universal-pathlib
  ujson hyperlink cramjam agent-client-protocol pandas-gbq id py-serializable pytimeparse types-setuptools crc32c
  supabase-auth supabase-functions zope-event events lockfile simple-salesforce mock tblib unidecode psycopg-pool
  griffe testcontainers uv-build hvac pydyf dbt-protos lazy-loader imagesize pip-requirements-parser validators
  inflect strictyaml flask-sqlalchemy clickhouse-connect lupa tree-sitter-languages ninja click-option-group python-magic pbr
  holidays google-cloud-dataplex msrestazure sphinxcontrib-serializinghtml google-cloud-alloydb temporalio alabaster claude-agent-sdk google-pasta youtube-transcript-api
  bidict pybind11 deltalake hiredis socksio opentelemetry-instrumentation-sqlalchemy tinyhtml5 sqlalchemy-utils python-http-client swebench
  llama-cloud pydata-google-auth bandit leather opentelemetry-distro requests-mock python-engineio python-socketio schema djangorestframework
  dbt-extractor sendgrid pip-audit openapi-schema-validator simple-websocket azure-batch flask-login apache-airflow-providers-fab nexus-rpc pip-api
  qrcode sphinxcontrib-htmlhelp sphinxcontrib-qthelp sphinxcontrib-applehelp sphinxcontrib-devhelp nvidia-cublas-cu12 groq google-cloud-dataproc userpath ua-parser
  pydeck nvidia-cuda-nvrtc-cu12 gcloud-aio-bigquery asyncio vcrpy oracledb jsonpath-python nvidia-cusparse-cu12 nvidia-cudnn-cu12 logfire-api
  jpype1 nvidia-nvjitlink-cu12 types-paramiko pytest-runner sphinxcontrib-jsmath openlineage-python nvidia-cufft-cu12 pypdf2 azure-cosmos genai-prices
  dbt-semantic-interfaces logfire prek pymdown-extensions opentelemetry-instrumentation-aiohttp-client boltons grpc-interceptor tree-sitter-bash watchtower datadog-api-client
  olefile nvidia-cusolver-cu12 nvidia-cuda-cupti-cu12 aws-requests-auth nvidia-curand-cu12 natsort azure-keyvault-keys nvidia-cuda-runtime-cu12 langfuse opt-einsum
  gql texttable sounddevice daff cron-descriptor rich-argparse jira ua-parser-builtins hatch diff-cover
  cloudpathlib pyathena types-cachetools questionary giturlparse nvidia-nvtx-cu12 catalogue types-urllib3 jupyter-builder shortuuid
  cssselect pytest-env opencensus binaryornot opencensus-context thinc bitarray opentelemetry-instrumentation-redis dacite face
  trino protego glom langchain-anthropic immutabledict pymupdf4llm apache-airflow-providers-snowflake time-machine emoji pymupdf-layout
  weasel flask-limiter azure-servicebus apache-airflow-providers-databricks aws-xray-sdk mlflow-tracing cached-property pyspnego oscrypto multitasking
  azure-datalake-store apache-airflow-providers-http azure-mgmt-resource snowplow-tracker factory-boy pytest-httpx google-cloud-build backports-asyncio-runner partd locket
  jwcrypto apache-airflow argparse url-normalize aws-lambda-powertools apache-airflow-providers-google resolvelib murmurhash levenshtein bs4
  hatch-vcs opentelemetry-semantic-conventions-ai respx firebase-admin pybase64 langchain-classic configargparse blis types-redis pytest-html
  django-cors-headers preshed srsly slowapi pathlib-abc databricks-labs-blueprint mysqlclient pytest-django pkgutil-resolve-name cymem
  kubernetes-asyncio pycares tensorboard-data-server types-deprecated fire maxminddb checkov pytzdata parameterized python-daemon
  typer-slim huey wasabi sqlmodel python-gitlab requests-cache pytest-split blake3 sagemaker azure-monitor-opentelemetry
  py-partiql-parser jsonpickle types-cffi cairosvg spacy-legacy spacy-loggers pathvalidate confection aiodns lxml-html-clean
  apache-airflow-providers-common-compat pywin32-ctypes twilio azure-kusto-data feedparser python-utils flask-wtf gast looker-sdk yt-dlp
  astor nvidia-cusparselt-cu12 skops azure-mgmt-compute keyrings-google-artifactregistry-auth ollama instructor awscrt sqlalchemy-spanner types-pyopenssl
  arxiv aiosmtplib python-frontmatter amazon-ion wtforms azure-storage-file-share asyncssh maturin genson progressbar2
  cachelib sagemaker-studio aiohttp-cors wikipedia-api fpdf2 ldap3 opentelemetry-instrumentation-grpc google-adk docopt opentelemetry-resourcedetector-gcp
  pyproject-api ipdb geographiclib pox parse geoip2 uuid6 colorful docling statsd
  cmake meson ppft gremlinpython google-cloud-storage-control makefun jupyter-console requirements-parser incremental types-markdown
  langchain-google-genai python-gnupg frozendict azure-mgmt-storage mkdocs xattr jaydebeapi datamodel-code-generator fastapi-mcp aliyun-trace
  aliyun-semantic-conventions smdebug-rulesconfig pydantic-evals pyvespa types-aiofiles pyserial office365-rest-python-client django-filter ghp-import pyogrio
  flit pydocket cadwyn pyyaml-env-tag cairocffi restructuredtext-lint pooch inputimeout clickhouse-driver grpcio-gcp
  aiomysql datetime whitenoise griffecli avro boostedblob pfzy pydeequ pydantic-ai bashlex
  service-identity google-generativeai apache-airflow-providers-ssh llama-index-core inquirerpy kfp azure-core-tracing-opentelemetry mkdocs-get-deps segment-analytics-python memray
  google-re2 prometheus-fastapi-instrumentator mammoth twisted pyelftools opentelemetry-resource-detector-azure atlassian-python-api textual-speedups types-tabulate tld
  readchar google-ai-generativelanguage fuzzywuzzy apache-airflow-providers-mysql jsonlines python-snappy cobble flask-babel types-croniter fasteners
  pyrfc3339 pytest-repeat mini-swe-agent dunamai yandexcloud requests-ntlm meson-python microsoft-kiota-http pickleshare flask-session
  magika html2text tree-sitter-javascript apispec slack-bolt mutagen pytest-json-report chevron flask-appbuilder scp
  simpleeval nodejs-wheel-binaries fastf1 apache-airflow-providers-imap htmldate mkdocs-material-extensions slicer microsoft-kiota-authentication-azure backcall tyro
  databricks-cli cronsim oss2 std-uritemplate apache-airflow-providers-ftp uuid7 blessed cachebox opensearch-protobufs nvidia-cufile-cu12
  azure-data-tables tritonclient microsoft-kiota-serialization-json markitdown azure-mgmt-containerregistry paginate imageio-ffmpeg grimp microsoft-kiota-serialization-text automat
  django-extensions azure-mgmt-cosmosdb apache-airflow-providers-sqlite constantly junitparser libclang jsonconversion python-levenshtein pyproject-metadata locust
  astunparse monotonic codeowners pytablewriter azure-keyvault-certificates netaddr azure-eventhub drf-spectacular microsoft-kiota-abstractions pytest-socket
  pygtrie google-cloud-managedkafka pika patchelf msgraph-core apache-airflow-providers-smtp resend pypika flask-jwt-extended sseclient-py
  dataproperty oldest-supported-numpy backrefs mypy-boto3-sqs ansible import-linter apache-airflow-providers-cncf-kubernetes langchain-aws detect-installer types-pymysql
  trafilatura syrupy scantree dirhash opentelemetry-instrumentation-botocore azure-mgmt-datafactory pillow-heif courlan supervisor libtmux
  pprintpp opentelemetry-instrumentation-system-metrics toposort mypy-boto3-rds edge-tts types-docutils django-storages minio python-on-whales ansicolors
  junit-xml ratelimit azure-mgmt-containerinstance screeninfo roman-numerals iso8601 django-redis pdm langcodes smbprotocol
  aiolimiter oci uritools arro3-core rq msoffcrypto-tool a2a-sdk crewai altgraph harbor
  google-cloud-trace apprise contextlib2 databricks-connect a2wsgi sh llama-index-workflows openhands-sdk marshmallow-sqlalchemy lmnr-claude-code-proxy
  django-stubs-ext burner-redis inspect-ai geventhttpclient mkdocstrings-python xlwt pyinstaller launchdarkly-eventsource jellyfish xmlsec
  opentelemetry-propagator-aws-xray dep-logic unearth google-cloud-iam browser-use-sdk avro-python3 opentelemetry-exporter-gcp-trace azure-mgmt-datalake-store configparser sphinxcontrib-jquery
  pyclipper ciso8601 icalendar impyla habluetooth launchdarkly-server-sdk speechrecognition justext pyroscope-io dagster-postgres
  mypy-boto3-dynamodb mcp-types primp user-agents dockerfile-parse thrift-sasl webdriver-manager dirtyjson jsondiff azure-nspkg
  pysftp pyinstaller-hooks-contrib kaleido docker-pycreds types-jsonschema sqlfluff azure-ai-projects motor langchain-google-community pyxlsb
  gradio-client crewai-tools lightning-utilities dpath httpx-ws line-bot-sdk azure-storage-common polyfactory python-bidi bottle
  scikit-build-core cloudevents soxr granian bubus vulture asynctest methodtools wirerope django-stubs
  cdp-use aws-cdk-asset-awscli-v1 lance-namespace opencensus-ext-azure mypy-boto3-lambda pytest-unordered dask-expr tree-sitter-c mypy-boto3-ec2 lance-namespace-urllib3-client
  fake-useragent expiringdict opentelemetry-instrumentation-celery hypercorn tree-sitter-java circuitbreaker azure-synapse-artifacts dotenv azure-mgmt-containerservice quack-kernels
  mypy-boto3-sts tree-sitter-c-sharp asciinema azure-monitor-query autopep8 connexion apache-airflow-providers-slack aiokafka obstore asgi-lifespan
  basedpyright anytree pydash cookiecutter elevenlabs teradatasql azure-kusto-ingest openhands-tools sphinx-design icdiff
  reactivex types-boto3 azure-synapse-spark tree-sitter-rust swifter fastspec pyyaml-ft dbt-snowflake types-aiobotocore types-aiobotocore-s3
  optree types-html5lib pytest-base-url curlify xyzservices svix python-ulid amplitude-analytics cloudflare prison
  sgmllib3k ffmpeg-python tree-sitter-go azure-ai-documentintelligence thefuzz tensorflow-estimator flask-caching apache-airflow-providers-common-io pytest-randomly azure-keyvault
  nose applicationinsights grpcio-reflection opentelemetry-instrumentation-asyncpg deepmerge langgraph-api appnope pyhcl azure-mgmt-authorization azure-mgmt-keyvault
  markdown2 astronomer-cosmos dataclasses dagster apache-airflow-core typepy exa-py pytest-icdiff blobfile tree-sitter-python
  pytest-playwright schedule numcodecs py7zr greenback cligj apache-airflow-providers-amazon parse-type addict etils
  uvicorn-worker pi-heif pyppmd python-crontab sphinx-autodoc-typehints num2words zstd cssutils python3-saml pytest-homeassistant-custom-component
  kgb constructs dagster-pipes mypy-protobuf microsoft-kiota-serialization-multipart microsoft-kiota-serialization-form pep517 aniso8601 pyzipper audioop-lts
  aioresponses enum34 opencv-contrib-python dictdiffer pybcj w3lib types-python-slugify dbt-databricks tom-swe llama-index-readers-llama-parse
  yapf namex waitress cytoolz multivolumefile colorclass inflate64 mypy-boto3-cloudformation azure-appconfiguration mypy-boto3-secretsmanager
  pyhanko django-debug-toolbar jsii openlineage-sql mistral-common allure-python-commons elasticsearch-dsl elementpath daytona pytest-dependency
  daytona-api-client unstructured-client daytona-api-client-async restrictedpython geomet coolname aiocache uipath-langchain pipdeptree deptry
  azure-mgmt-monitor django-timezone-field daytona-toolbox-api-client-async pypandoc cassandra-driver python-decouple daytona-toolbox-api-client python-jenkins pypng simple-gcp-object-downloader
  uv-dynamic-versioning ip3country fastembed docx2txt types-psycopg2 django-environ facebook-business atpublic paho-mqtt openinference-semantic-conventions
  pyzstd pycrypto openlineage-integration-common singer-sdk pdbr dj-database-url xmlschema dagster-shared azure-mgmt-redis kaitaistruct
  pulumi cchardet unittest-xml-reporting backports-strenum mkdocstrings dependency-injector pefile pyshp banks sagemaker-core
  python3-openid djangorestframework-simplejwt openinference-instrumentation azure-mgmt-nspkg bedrock-agentcore publication asyncer llama-index-instrumentation azure-mgmt-web tabledata
  google-cloud-bigquery-biglake opentelemetry-instrumentation-lancedb databricks-agents crcmod statsig llama-index-embeddings-openai dlt pytest-sugar moviepy starkbank-ecdsa
  influxdb-client tox-uv mkdocs-autorefs allure-pytest azure-mgmt-datalake-nspkg click-default-group msgraph-sdk pinecone opentelemetry-instrumentation-haystack azure-mgmt-sql
  azure-mgmt-cognitiveservices farama-notifications protobuf3-to-dict alibabacloud-credentials sentinels langchain-mcp-adapters py-key-value-shared dagster-webserver rustworkx cloud-sql-python-connector
  functions-framework mixpanel mongomock azure-mgmt-msi langgraph-checkpoint-postgres pyrate-limiter toons types-psutil enum-compat python-keycloak
  pytest-custom-exit-code apache-tvm-ffi pgpy parver tree-sitter-typescript easygui pure-sasl braintrust pdfkit sphinx-copybutton
  dagster-graphql tokenize-rt harbor-rewardkit strawberry-graphql opentelemetry-exporter-gcp-logging azure-mgmt-servicebus azure-mgmt-rdbms apache-airflow-task-sdk azure-mgmt-loganalytics slackclient
  oletools pyreadline3 pcodedmp hatch-fancy-pypi-readme pyaml django-celery-beat pymsteams nvidia-nvshmem-cu12 azure-ai-agents publicsuffix2
  func-timeout azure-mgmt-applicationinsights aws-cdk-lib azure-mgmt-eventhub striprtf pastel puremagic types-retry opentelemetry-instrumentation-pinecone xgrammar
  eventlet stringcase google-cloud-discoveryengine opentelemetry-instrumentation-milvus diagrams opentelemetry-instrumentation-vertexai databricks-labs-dqx ebcdic knack mbstrdecoder
  azure-mgmt-recoveryservices svcs priority json-rpc ag-ui-protocol azure-mgmt-recoveryservicesbackup port-for detect-secrets marshmallow-enum azure-mgmt-cdn
  mistral-vibe firecrawl-py compressed-tensors azure-mgmt-managementgroups tree-sitter-php boto azure-mgmt-search pytest-instafail eth-account azure-mgmt-batch
  pytest-forked python-box asana openai-harmony exchange-calendars ibm-cloud-sdk-core azure-mgmt-eventgrid construct pipx funcy
  dynaconf extract-msg jinxed jsbeautifier aliyun-python-sdk-core configobj jaxtyping azure-mgmt-iothub tavily-python auth0-python
  autobahn opentelemetry-instrumentation-bedrock eth-utils tree-sitter-ruby pyhumps azure-mgmt-trafficmanager proglog dbt-postgres browser-harness azure-mgmt-marketplaceordering
  polib fastapi-sso ansible-compat jq opentelemetry-instrumentation-cohere webauthn azure-mgmt-advisor types-webencodings tinydb pywinrm
  parsimonious opentelemetry-instrumentation-llamaindex setuptools-rust c7n-org azure-mgmt-network pyhive ipython-genutils dependency-groups azure-mgmt-policyinsights gguf
  azure-cli-core asgi-correlation-id azure-cli voluptuous hexbytes opentelemetry-instrumentation-alephalpha opentelemetry-instrumentation-qdrant datasketch opentelemetry-instrumentation-ollama whenever
  eth-abi opentelemetry-instrumentation-chromadb azure-mgmt-servicefabric azure-mgmt-signalr opentelemetry-instrumentation-crewai opentelemetry-instrumentation-watsonx opentelemetry-instrumentation-replicate pdm-backend opentelemetry-instrumentation-transformers eth-hash
  discord-py opentelemetry-instrumentation-mistralai urwid opentelemetry-instrumentation-together eth-typing txaio opentelemetry-instrumentation-weaviate certbot-dns-cloudflare sql-metadata pyiceberg-core
  opentelemetry-instrumentation-sagemaker opentelemetry-instrumentation-marqo google-analytics-data azure-mgmt-billing sqlparams presidio-analyzer azure-mgmt-media azure-mgmt-maps azure-mgmt-iothubprovisioningservices azure-mgmt-datamigration
  ifaddr mangum azure-mgmt-iotcentral azure-mgmt-batchai backports-zoneinfo schemathesis ddgs fetch-use opentelemetry-instrumentation-mcp compressed-rtf
  cel-python janus korean-lunar-calendar cerberus pyfiglet mypy-boto3-glue azure-search-documents aws-cdk-cloud-assembly-schema pkgconfig pytest-postgresql
  marshmallow-oneofschema github3-py ortools gepa pyrefly litellm-proxy-extras aws-cdk-asset-node-proxy-agent-v6 apache-airflow-providers-standard pex rtfde
  django-phonenumber-field opentelemetry-instrumentation-langchain alibabacloud-tea-openapi dbt-spark opentelemetry-exporter-gcp-monitoring uipath ndg-httpsclient eth-rlp pathlib opentelemetry-instrumentation-asyncio
  deepagents nbclassic autoflake pamqp nanoid apache-airflow-microsoft-fabric-plugin tensorflow-io-gcs-filesystem python-socks deepeval ultralytics-thop
  dspy marko alibabacloud-adb20211201 litellm-enterprise webob choreographer timezonefinder pyhamcrest acme sphinxcontrib-mermaid
  lightning partial-json-parser jinja2-humanize-extension aiormq ckzg codespell pycomposefile djangorestframework-stubs python-crfsuite docling-core
  python-hcl2 google opsgenie-sdk aliyun-python-sdk-kms types-aioboto3 apache-beam marshmallow-dataclass pyhanko-certvalidator unicodecsv channels
  python-calamine clickclick win32-setctime nvidia-cutlass-dsl-libs-base asteval mitmproxy yaspin pympler terminaltables gprof2dot
  jsonschema-rs types-openpyxl social-auth-core hdfs opentelemetry-sdk-extension-aws py-vapid types-tqdm prance legacy-cgi python-stdnum
  azure-functions poethepoet ipaddress pytest-recording configupdater kazoo nvidia-cutlass-dsl tree-sitter-yaml editorconfig pytest-order
  safehttpx aio-pika daphne imagehash pytest-aiohttp rarfile async-property sqlalchemy-jsonfield asyncstdlib rlp
  furl orderedmultidict strands-agents-tools roboflow eth-keys subprocess-tee stanio commonmark pinecone-plugin-interface dbt-bigquery
  donfig pyahocorasick python-chess azure-eventgrid
)

CURATED_PACKAGES=(
  "${CATEGORY_CORE_NUMERICAL[@]}"
  "${CATEGORY_STATISTICS[@]}"
  "${CATEGORY_CAUSAL[@]}"
  "${CATEGORY_CLASSICAL_ML[@]}"
  "${CATEGORY_DEEP_LEARNING[@]}"
  "${CATEGORY_LLM[@]}"
  "${CATEGORY_VECTOR_SEARCH[@]}"
  "${CATEGORY_GPU_RAPIDS[@]}"
  "${CATEGORY_DISTRIBUTED[@]}"
  "${CATEGORY_INTERPRETABILITY[@]}"
  "${CATEGORY_MLOPS[@]}"
  "${CATEGORY_RL[@]}"
  "${CATEGORY_TIMESERIES[@]}"
  "${CATEGORY_CV[@]}"
  "${CATEGORY_BIOIMAGING[@]}"
  "${CATEGORY_AUDIO[@]}"
  "${CATEGORY_NLP[@]}"
  "${CATEGORY_GENOMICS[@]}"
  "${CATEGORY_CHEMISTRY[@]}"
  "${CATEGORY_NEURO_MEDICAL[@]}"
  "${CATEGORY_GEOSPATIAL[@]}"
  "${CATEGORY_ASTRO_PHYSICS[@]}"
  "${CATEGORY_ENGINEERING[@]}"
  "${CATEGORY_VISUALIZATION[@]}"
  "${CATEGORY_NOTEBOOKS[@]}"
  "${CATEGORY_DEV[@]}"
  "${CATEGORY_PROFILING[@]}"
  "${CATEGORY_DATA_IO[@]}"
  "${CATEGORY_DATABASES[@]}"
  "${CATEGORY_DATA_QUALITY[@]}"
  "${CATEGORY_SIGNAL_DIFFEQ[@]}"
  "${CATEGORY_ANOMALY[@]}"
  "${CATEGORY_SURVIVAL[@]}"
  "${CATEGORY_AUTOML[@]}"
  "${CATEGORY_PRIVACY[@]}"
  "${CATEGORY_FINANCIAL[@]}"
  "${CATEGORY_PDF_OCR[@]}"
  "${CATEGORY_AUTOMATION[@]}"
  "${CATEGORY_GRAPH_DB[@]}"
  "${CATEGORY_WEB_SCRAPING[@]}"
  "${CATEGORY_LARGE_SCALE[@]}"
  "${CATEGORY_SYMBOLIC[@]}"
  "${CATEGORY_SOCIAL_SCIENCE[@]}"
  "${CATEGORY_RECOMMENDER[@]}"
  "${CATEGORY_GEO_EXTRA[@]}"
  "${CATEGORY_REPRODUCIBILITY[@]}"
  "${CATEGORY_COMMON_DEPS[@]}"
  "${CATEGORY_TOP_PYPI[@]}"
)

# -----------------------------------------------------------------------------
# Logging
# -----------------------------------------------------------------------------
mkdir -p "$LOG_DIR" 2>/dev/null || true

log() {
  local line
  line="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
  echo "$line"
  echo "$line" >> "$LOG_FILE" 2>/dev/null || true
}

die() {
  log "ERROR: $*"
  exit 1
}

# -----------------------------------------------------------------------------
# Error handling / cleanup
# -----------------------------------------------------------------------------
error_handler() {
  local exit_code=$?
  local line_no="$1"
  log "ERROR: ${SCRIPT_NAME} failed at line ${line_no} (exit code ${exit_code})"
}

cleanup() {
  local exit_code=$?
  if [[ "$LOCK_ACQUIRED" -eq 1 ]]; then
    release_lock
  fi
  exit "$exit_code"
}

trap 'error_handler $LINENO' ERR
trap cleanup EXIT

# -----------------------------------------------------------------------------
# Lockfile — prevents two mirror/add runs from stomping on each other.
# Uses flock when available, falls back to a PID-checked lockfile otherwise
# (e.g. on systems without util-linux's flock).
# -----------------------------------------------------------------------------
acquire_lock() {
  if command -v flock >/dev/null 2>&1; then
    eval "exec ${LOCK_FD}>\"$LOCK_FILE\""
    if ! flock -n "$LOCK_FD"; then
      die "Another instance of ${SCRIPT_NAME} appears to be running (lock: ${LOCK_FILE}). If this is stale, remove the lockfile and retry."
    fi
    LOCK_ACQUIRED=1
  else
    if [[ -e "$LOCK_FILE" ]]; then
      local old_pid=""
      old_pid="$(cat "$LOCK_FILE" 2>/dev/null || true)"
      if [[ -n "$old_pid" ]] && kill -0 "$old_pid" 2>/dev/null; then
        die "Another instance of ${SCRIPT_NAME} appears to be running (pid ${old_pid})."
      fi
    fi
    echo $$ > "$LOCK_FILE"
    LOCK_ACQUIRED=1
  fi
  log "Lock acquired (${LOCK_FILE})."
}

release_lock() {
  if command -v flock >/dev/null 2>&1; then
    eval "exec ${LOCK_FD}>&-" 2>/dev/null || true
  fi
  rm -f "$LOCK_FILE" 2>/dev/null || true
  LOCK_ACQUIRED=0
}

# -----------------------------------------------------------------------------
# Prerequisite checks
# -----------------------------------------------------------------------------
check_prerequisites() {
  if ! command -v python3 >/dev/null 2>&1; then
    die "python3 was not found on PATH. Install Python 3 before running ${SCRIPT_NAME}."
  fi
  if ! python3 -m pip --version >/dev/null 2>&1; then
    die "pip is not available for python3 (try: python3 -m ensurepip --upgrade). Install pip before running ${SCRIPT_NAME}."
  fi
}

# -----------------------------------------------------------------------------
# Small string helper: trim leading/trailing whitespace.
# -----------------------------------------------------------------------------
trim() {
  local s="$1"
  s="${s#"${s%%[![:space:]]*}"}"
  s="${s%"${s##*[![:space:]]}"}"
  printf '%s' "$s"
}

# -----------------------------------------------------------------------------
# Read a package list file (one package per line, '#' comments and blank
# lines ignored) into stdout, one package per line.
# -----------------------------------------------------------------------------
read_package_file() {
  local file="$1"
  [[ -f "$file" ]] || return 0
  local line trimmed
  while IFS= read -r line || [[ -n "$line" ]]; do
    trimmed="$(trim "$line")"
    [[ -z "$trimmed" || "$trimmed" == \#* ]] && continue
    printf '%s\n' "$trimmed"
  done < "$file"
}

# -----------------------------------------------------------------------------
# Retry helper: runs "$@", retrying up to MAX_RETRIES times with a short
# delay between attempts. Full command output goes to the log file; a short
# status line goes to both log and stdout for each attempt.
# -----------------------------------------------------------------------------
retry_cmd() {
  local desc="$1"; shift
  local attempt
  for (( attempt=1; attempt<=MAX_RETRIES; attempt++ )); do
    log "  [attempt ${attempt}/${MAX_RETRIES}] ${desc}"
    if "$@" >>"$LOG_FILE" 2>&1; then
      return 0
    fi
    log "  attempt ${attempt} failed: ${desc}"
    if (( attempt < MAX_RETRIES )); then
      sleep "$RETRY_DELAY"
    fi
  done
  log "  giving up after ${MAX_RETRIES} attempts: ${desc}"
  return 1
}

# -----------------------------------------------------------------------------
# Download a single package's wheel (with dependencies) for one Python
# version, restricted to the target manylinux platform. Returns non-zero if
# no matching binary distribution could be found/downloaded.
# -----------------------------------------------------------------------------
download_wheel_for_version() {
  local pkg="$1" pyver="$2"
  local abi="cp${pyver//./}"
  retry_cmd "download ${pkg} (python ${pyver}, wheel, ${PLATFORM_TAG})" \
    python3 -m pip download \
      --dest "$PACKAGES_DIR" \
      --extra-index-url "$EXTRA_INDEX" \
      --platform "$PLATFORM_TAG" \
      --platform linux_x86_64 \
      --python-version "$pyver" \
      --implementation cp \
      --abi "$abi" \
      --abi none \
      --only-binary=:all: \
      "$pkg"
}

# -----------------------------------------------------------------------------
# Mirror a list of top-level packages (and their dependencies). For each
# package, tries prebuilt wheels for every configured Python version first;
# if none of those succeed, falls back to a source distribution download.
# -----------------------------------------------------------------------------
mirror_packages() {
  local packages=("$@")
  [[ ${#packages[@]} -gt 0 ]] || { log "Nothing to mirror."; return 0; }

  mkdir -p "$PACKAGES_DIR"
  log "Mirroring ${#packages[@]} package(s) for Python version(s): ${PYTHON_VERSIONS[*]} (platform: ${PLATFORM_TAG})"

  SDIST_FALLBACK=()

  local pkg pyver pkg_ok
  for pkg in "${packages[@]}"; do
    log "==> ${pkg}"
    pkg_ok=0
    for pyver in "${PYTHON_VERSIONS[@]}"; do
      if download_wheel_for_version "$pkg" "$pyver"; then
        pkg_ok=1
      fi
    done
    if [[ "$pkg_ok" -eq 1 ]]; then
      SUCCESS_COUNT=$(( SUCCESS_COUNT + 1 ))
    else
      SDIST_FALLBACK+=("$pkg")
    fi
  done

  if [[ ${#SDIST_FALLBACK[@]} -gt 0 ]]; then
    log "No wheels found for ${#SDIST_FALLBACK[@]} package(s) on ${PLATFORM_TAG}; falling back to source: ${SDIST_FALLBACK[*]}"
    for pkg in "${SDIST_FALLBACK[@]}"; do
      if retry_cmd "download ${pkg} (unrestricted fallback)" \
          python3 -m pip download --dest "$PACKAGES_DIR" --extra-index-url "$EXTRA_INDEX" "$pkg"; then
        SUCCESS_COUNT=$(( SUCCESS_COUNT + 1 ))
      else
        FAIL_COUNT=$(( FAIL_COUNT + 1 ))
        FAILED_PACKAGES+=("$pkg")
      fi
    done
  fi
}

# -----------------------------------------------------------------------------
# Build the full curated + custom package list.
# -----------------------------------------------------------------------------
full_package_list() {
  local -a all=("${CURATED_PACKAGES[@]}")
  local extra
  while IFS= read -r extra; do
    all+=("$extra")
  done < <(read_package_file "$CUSTOM_LIST")
  printf '%s\n' "${all[@]}"
}

# -----------------------------------------------------------------------------
# pip.conf generation — researchers copy this to ~/.pip/pip.conf so pip uses
# the offline mirror by default (no --no-index/--find-links needed by hand).
# -----------------------------------------------------------------------------
generate_pip_conf() {
  cat > "$PIP_CONF_OUT" <<EOF
# pip.conf generated by ${SCRIPT_NAME} on $(date '+%Y-%m-%d %H:%M:%S')
#
# This makes pip use the offline tiCrypt PyPI mirror by default, with no
# network access required. On a researcher VM, install it with:
#
#   mkdir -p ~/.pip
#   cp "${PIP_CONF_OUT}" ~/.pip/pip.conf
#
# (On some Linux distros pip instead reads ~/.config/pip/pip.conf — copy it
# there too, or symlink one to the other, if pip does not pick this up.)

[global]
no-index = true
find-links = file://${PACKAGES_DIR}
EOF
  log "Wrote pip.conf to ${PIP_CONF_OUT}"
}

# -----------------------------------------------------------------------------
# Manifest generation — scans pypi-packages/ and writes manifest.json with
# every package file, its size, and the timestamp of the run. Researchers and
# admins can inspect this to see exactly what is in the mirror.
# -----------------------------------------------------------------------------
generate_manifest() {
  [[ -d "$PACKAGES_DIR" ]] || return 0

  local tmpfile="${MANIFEST_FILE}.tmp"
  local run_date
  run_date="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"

  {
    printf '{\n'
    printf '  "generated": "%s",\n' "$run_date"
    printf '  "packages_dir": "%s",\n' "$PACKAGES_DIR"
    printf '  "platform": "%s",\n' "$PLATFORM_TAG"
    printf '  "python_versions": [%s],\n' "$(printf '"%s",' "${PYTHON_VERSIONS[@]}" | sed 's/,$//')"
    printf '  "total_files": %d,\n' "$(find "$PACKAGES_DIR" -maxdepth 1 -type f | wc -l | tr -d ' ')"
    printf '  "total_size_bytes": %d,\n' "$(du -sb "$PACKAGES_DIR" 2>/dev/null | cut -f1 || echo 0)"
    printf '  "files": [\n'

    local first=1
    while IFS= read -r filepath; do
      local filename size
      filename="$(basename "$filepath")"
      size="$(stat -f%z "$filepath" 2>/dev/null || stat -c%s "$filepath" 2>/dev/null || echo 0)"

      local pkg_name pkg_version pkg_type
      pkg_type="unknown"
      if [[ "$filename" == *.whl ]]; then
        pkg_type="wheel"
        pkg_name="${filename%%-*}"
        pkg_version="$(echo "$filename" | sed 's/^[^-]*-\([^-]*\)-.*/\1/')"
      elif [[ "$filename" == *.tar.gz ]]; then
        pkg_type="sdist"
        pkg_name="$(echo "$filename" | sed 's/-[0-9].*//')"
        pkg_version="$(echo "$filename" | sed 's/.*-\([0-9][^-]*\)\.tar\.gz/\1/')"
      elif [[ "$filename" == *.zip ]]; then
        pkg_type="sdist"
        pkg_name="$(echo "$filename" | sed 's/-[0-9].*//')"
        pkg_version="$(echo "$filename" | sed 's/.*-\([0-9][^-]*\)\.zip/\1/')"
      fi

      [[ $first -eq 1 ]] && first=0 || printf ',\n'
      printf '    {"file": "%s", "package": "%s", "version": "%s", "type": "%s", "size": %s}' \
        "$filename" "$pkg_name" "$pkg_version" "$pkg_type" "$size"
    done < <(find "$PACKAGES_DIR" -maxdepth 1 -type f \( -name '*.whl' -o -name '*.tar.gz' -o -name '*.zip' \) | sort)

    printf '\n  ]\n'
    printf '}\n'
  } > "$tmpfile"

  mv "$tmpfile" "$MANIFEST_FILE"
  log "Wrote manifest to ${MANIFEST_FILE} ($(find "$PACKAGES_DIR" -maxdepth 1 -type f | wc -l | tr -d ' ') files)"
}

# -----------------------------------------------------------------------------
# install: find or create a virtualenv, then install from the local mirror
# only (--no-index), never touching the network.
# -----------------------------------------------------------------------------
find_or_create_venv() {
  if [[ -n "${VIRTUAL_ENV:-}" ]]; then
    log "Using already-activated virtualenv: ${VIRTUAL_ENV}"
    return 0
  fi

  local candidate
  for candidate in "./venv" "./.venv" "${HOME}/venv"; do
    if [[ -f "${candidate}/bin/activate" ]]; then
      log "Found existing virtualenv at ${candidate}"
      # shellcheck disable=SC1091
      source "${candidate}/bin/activate"
      return 0
    fi
  done

  local new_venv="./venv"
  log "No virtualenv found; creating one at ${new_venv}"
  python3 -m venv "$new_venv" || die "Failed to create virtualenv at ${new_venv}"
  # shellcheck disable=SC1091
  source "${new_venv}/bin/activate"
  log "Activated new virtualenv at ${new_venv}"
}

# Check whether a package is already installed in the active Python environment.
# Returns 0 (true) if installed, 1 (false) otherwise.
is_installed() {
  python3 -m pip show "$1" &>/dev/null
}

# install_packages [args...]
# Parses args: .txt files are read line-by-line; everything else is a package name.
# Already-installed packages are skipped.
install_packages() {
  local -a packages=()
  local arg
  for arg in "$@"; do
    if [[ "$arg" == *.txt ]] && [[ -f "$arg" ]]; then
      log "Reading package list from ${arg}"
      while IFS= read -r line; do
        line="${line%%#*}"
        line="${line// /}"
        [[ -n "$line" ]] && packages+=("$line")
      done < "$arg"
    else
      packages+=("$arg")
    fi
  done

  [[ -d "$PACKAGES_DIR" ]] || die "Mirror directory not found at ${PACKAGES_DIR}. Run '${SCRIPT_NAME} mirror' on an internet-connected machine first."

  find_or_create_venv

  if [[ ${#packages[@]} -eq 0 ]]; then
    local -a full=()
    while IFS= read -r pkg; do
      full+=("$pkg")
    done < <(full_package_list)
    packages=("${full[@]}")
    log "No packages specified; installing full curated set (${#packages[@]} packages)."
  fi

  local pkg
  for pkg in "${packages[@]}"; do
    if is_installed "$pkg"; then
      SKIP_COUNT=$(( SKIP_COUNT + 1 ))
      continue
    fi
    if retry_cmd "install ${pkg}" \
        python3 -m pip install --no-index --find-links="file://${PACKAGES_DIR}" "$pkg"; then
      SUCCESS_COUNT=$(( SUCCESS_COUNT + 1 ))
    else
      FAIL_COUNT=$(( FAIL_COUNT + 1 ))
      FAILED_PACKAGES+=("$pkg")
    fi
  done

  if [[ $SKIP_COUNT -gt 0 ]]; then
    log "Skipped ${SKIP_COUNT} already-installed package(s)."
  fi
}

# -----------------------------------------------------------------------------
# add: append package(s) to custom-packages.txt (so they survive future
# `update` runs) and immediately mirror them.
# -----------------------------------------------------------------------------
add_packages() {
  local packages=("$@")
  [[ ${#packages[@]} -gt 0 ]] || die "Usage: ${SCRIPT_NAME} add <package> [package...]"

  if [[ ! -f "$CUSTOM_LIST" ]]; then
    cat > "$CUSTOM_LIST" <<EOF
# custom-packages.txt
# One package name per line. Lines starting with '#' are ignored.
# This file is preserved across 'update' runs and is combined with the
# built-in curated list whenever 'mirror', 'update' or 'install' run without
# explicit package arguments.
EOF
  fi

  local pkg
  for pkg in "${packages[@]}"; do
    if grep -qxF "$pkg" "$CUSTOM_LIST" 2>/dev/null; then
      log "  ${pkg} is already in ${CUSTOM_LIST}"
    else
      echo "$pkg" >> "$CUSTOM_LIST"
      log "  Added ${pkg} to ${CUSTOM_LIST}"
    fi
  done

  mirror_packages "${packages[@]}"
}

# -----------------------------------------------------------------------------
# search: case-insensitive substring search over mirrored filenames.
# -----------------------------------------------------------------------------
search_packages() {
  local term="$1"
  [[ -d "$PACKAGES_DIR" ]] || die "Mirror directory not found at ${PACKAGES_DIR}. Run '${SCRIPT_NAME} mirror' first."

  local term_lc
  term_lc="$(printf '%s' "$term" | tr '[:upper:]' '[:lower:]')"

  log "Searching mirror for '${term}'..."
  local found=0 f base base_lc
  for f in "$PACKAGES_DIR"/*; do
    [[ -f "$f" ]] || continue
    base="$(basename "$f")"
    base_lc="$(printf '%s' "$base" | tr '[:upper:]' '[:lower:]')"
    if [[ "$base_lc" == *"$term_lc"* ]]; then
      echo "  $base"
      found=$(( found + 1 ))
    fi
  done

  if [[ "$found" -eq 0 ]]; then
    log "No matches found for '${term}'."
  else
    log "Found ${found} matching file(s)."
  fi
}

# -----------------------------------------------------------------------------
# Parse a wheel/sdist filename into "name version". Wheel names/versions have
# their hyphens normalized to underscores per PEP 427, so splitting on '-'
# works there; for sdists we fall back to a heuristic: the version is the
# first '-'-delimited field that starts with a digit.
# -----------------------------------------------------------------------------
parse_name_version() {
  local fname="$1"
  local base="$fname"
  base="${base%.whl}"
  base="${base%.tar.gz}"
  base="${base%.tar.bz2}"
  base="${base%.tar.xz}"
  base="${base%.tgz}"
  base="${base%.zip}"

  local -a fields
  IFS='-' read -r -a fields <<< "$base"

  local i version_idx=-1
  for (( i=0; i<${#fields[@]}; i++ )); do
    if [[ "${fields[$i]}" =~ ^[0-9] ]]; then
      version_idx=$i
      break
    fi
  done

  if [[ "$version_idx" -lt 0 ]]; then
    printf '%s %s\n' "$base" "unknown"
    return 0
  fi

  local name="${fields[0]}" j
  for (( j=1; j<version_idx; j++ )); do
    name="${name}-${fields[$j]}"
  done
  printf '%s %s\n' "$name" "${fields[$version_idx]}"
}

# -----------------------------------------------------------------------------
# list: show every mirrored package + version, deduplicated and sorted.
# -----------------------------------------------------------------------------
list_packages() {
  [[ -d "$PACKAGES_DIR" ]] || die "Mirror directory not found at ${PACKAGES_DIR}. Run '${SCRIPT_NAME} mirror' first."

  log "Packages currently mirrored in ${PACKAGES_DIR}:"

  local tmpfile
  tmpfile="$(mktemp)"
  local f base
  for f in "$PACKAGES_DIR"/*; do
    [[ -f "$f" ]] || continue
    base="$(basename "$f")"
    parse_name_version "$base" >> "$tmpfile"
  done

  if [[ -s "$tmpfile" ]]; then
    sort -u "$tmpfile" | awk '{printf "  %-35s %s\n", $1, $2}'
  else
    echo "  (mirror is empty)"
  fi

  local total
  total="$(wc -l < "$tmpfile" | tr -d ' ')"
  rm -f "$tmpfile"
  log "Total mirrored artifacts: ${total}"
}

# -----------------------------------------------------------------------------
# Progress summary shown at the end of mirror/update/add/install runs.
# -----------------------------------------------------------------------------
print_summary() {
  local duration="$1"
  local total_size="unknown"
  if [[ -d "$PACKAGES_DIR" ]]; then
    total_size="$(du -sh "$PACKAGES_DIR" 2>/dev/null | cut -f1)"
    [[ -z "$total_size" ]] && total_size="unknown"
  fi

  log "----------------------------------------------------------------"
  log "Summary"
  log "  Succeeded    : ${SUCCESS_COUNT}"
  log "  Skipped      : ${SKIP_COUNT}"
  log "  Failed       : ${FAIL_COUNT}"
  if [[ ${#FAILED_PACKAGES[@]} -gt 0 ]]; then
    log "  Failed items : ${FAILED_PACKAGES[*]}"
  fi
  log "  Mirror size  : ${total_size} (${PACKAGES_DIR})"
  log "  Elapsed      : ${duration}s"
  log "  Log file     : ${LOG_FILE}"
  log "----------------------------------------------------------------"
}

# -----------------------------------------------------------------------------
# Command implementations
# -----------------------------------------------------------------------------
cmd_mirror() {
  check_prerequisites
  acquire_lock
  local start_ts=$SECONDS

  local -a packages=()
  while IFS= read -r pkg; do
    packages+=("$pkg")
  done < <(full_package_list)

  mirror_packages "${packages[@]}"
  generate_pip_conf
  generate_manifest
  print_summary "$(( SECONDS - start_ts ))"
}

cmd_update() {
  log "update: re-running mirror to refresh to the latest available versions."
  cmd_mirror
}

cmd_install() {
  check_prerequisites
  local start_ts=$SECONDS
  install_packages "$@"
  print_summary "$(( SECONDS - start_ts ))"
}

cmd_search() {
  [[ $# -ge 1 ]] || die "Usage: ${SCRIPT_NAME} search <term>"
  search_packages "$1"
}

cmd_add() {
  [[ $# -ge 1 ]] || die "Usage: ${SCRIPT_NAME} add <package> [package...]"
  check_prerequisites
  acquire_lock
  local start_ts=$SECONDS
  add_packages "$@"
  generate_pip_conf
  generate_manifest
  print_summary "$(( SECONDS - start_ts ))"
}

cmd_list() {
  list_packages
}

usage() {
  cat <<EOF
${SCRIPT_NAME} - offline PyPI mirror & installer for air-gapped tiCrypt research VMs

USAGE:
  ${SCRIPT_NAME} mirror                  Download all ~2,400 packages (+ dependencies)
                                          into ./pypi-packages/. Run this on the
                                          internet-connected staging machine.
  ${SCRIPT_NAME} update                  Alias for 'mirror' — refreshes everything to the
                                          latest available versions.
  ${SCRIPT_NAME} install                  Install the full curated + custom set from the
                                          local mirror (no network). Run on an offline VM.
                                          Skips packages that are already installed.
  ${SCRIPT_NAME} install <pkg> [pkg ...]  Install only the named packages. Skips duplicates.
  ${SCRIPT_NAME} install <file.txt>       Install packages listed in a text file (one per
                                          line; blank lines and #comments are ignored).
                                          You can mix .txt files and package names.
                                          All install modes auto-detect an active/existing
                                          virtualenv (\$VIRTUAL_ENV, ./venv, ./.venv,
                                          ~/venv) or create one at ./venv.
  ${SCRIPT_NAME} search <term>           Search mirrored package filenames for <term>.
  ${SCRIPT_NAME} add <pkg> [pkg ...]     Add package(s) to custom-packages.txt (persists
                                          across future updates) and download them now.
  ${SCRIPT_NAME} list                    List all mirrored packages with versions.
  ${SCRIPT_NAME} readme                  Print a detailed reference guide (for admins
                                          and researchers).
  ${SCRIPT_NAME} --help                  Show this help.

FILES (next to this script):
  pypi-packages/         The mirror itself (wheels/sdists).
  manifest.json          Package inventory: every file, package name, version,
                          type (wheel/sdist), and size. Regenerated after every
                          mirror/update/add run.
  logs/                  Timestamped logs for every run.
  custom-packages.txt    User-added packages; edit directly or use 'add'.
  pip.conf               Generated by 'mirror'/'add'/'update'. Copy to
                          ~/.pip/pip.conf on a researcher VM to make pip use
                          this mirror by default.
  .pypi-mirror.lock      Lockfile guarding mirror/add against concurrent runs.

NOTES:
  - The mirror includes ~2,400 packages: 643 curated research packages
    plus the top 1,744 PyPI packages by download count (hardcoded, sourced
    from PyPI stats as of 2026-08-31), plus custom-packages.txt.
  - Wheels are downloaded for Python ${PYTHON_VERSIONS[*]} on platform
    ${PLATFORM_TAG}. Packages without a matching wheel automatically fall
    back to a source distribution download.
  - Failed downloads/installs are retried up to ${MAX_RETRIES} times.
  - Requires only python3 + pip; no other external tools.
EOF
}

cmd_readme() {
  cat <<'READMEEOF'
================================================================================
  pypi-mirror.sh — Offline Python Package Mirror for tiCrypt Research VMs
================================================================================

  Full documentation: https://ticrypt.com/docs/admin-guide/operations/pypi-mirror
  Researcher guide:   https://ticrypt.com/docs/user-guide/virtual-machines/python-packages
  Download script:    https://ticrypt.com/pypi-mirror.sh

────────────────────────────────────────────────────────────────────────────────
  FOR SYSTEM ADMINISTRATORS
────────────────────────────────────────────────────────────────────────────────

  INITIAL SETUP (on an internet-connected staging machine):

    1. Download the script onto the NFS share:
       curl -o /mnt/nfs/pypi-mirror/pypi-mirror.sh https://ticrypt.com/pypi-mirror.sh
       chmod +x /mnt/nfs/pypi-mirror/pypi-mirror.sh

    2. Run the mirror:
       cd /mnt/nfs/pypi-mirror
       ./pypi-mirror.sh mirror

    This downloads ~2,400 packages into pypi-packages/ and generates pip.conf.
    First run: expect 60–100 GB. Requires python3 and pip, nothing else.

  KEEPING IT UPDATED:

    ./pypi-mirror.sh update

    Re-downloads the full package list with the latest versions. Schedule monthly:
    0 2 1 * * /mnt/nfs/pypi-mirror/pypi-mirror.sh update >> /var/log/pypi-mirror.log 2>&1

  ADDING PACKAGES ON DEMAND:

    ./pypi-mirror.sh add <package> [package...]

    Downloads immediately and adds to custom-packages.txt (persists across updates).

  DIRECTORY LAYOUT:

    pypi-mirror.sh          This script
    pypi-packages/          Downloaded wheels and source distributions (flat)
    manifest.json           Package inventory — every file, name, version, type, size
    pip.conf                Generated config — researchers copy to ~/.pip/pip.conf
    custom-packages.txt     Admin-added packages (survives updates)
    logs/                   Timestamped log for every run
    .pypi-mirror.lock       Prevents concurrent mirror/add/update

────────────────────────────────────────────────────────────────────────────────
  FOR RESEARCHERS
────────────────────────────────────────────────────────────────────────────────

  INSTALLING PACKAGES (from a VM terminal, no internet needed):

    # Install the full curated set (~2,400 packages), skip already-installed
    /mnt/nfs/pypi-mirror/pypi-mirror.sh install

    # Install specific packages only (skips duplicates)
    /mnt/nfs/pypi-mirror/pypi-mirror.sh install numpy pandas scikit-learn

    # Install from a requirements.txt file
    /mnt/nfs/pypi-mirror/pypi-mirror.sh install requirements.txt

    # Mix files and package names
    /mnt/nfs/pypi-mirror/pypi-mirror.sh install requirements.txt extra-package

    Already-installed packages are skipped automatically. To upgrade a
    package to a newer version in the mirror, use pip directly:
      pip install --upgrade numpy

  USING pip DIRECTLY:

    Copy the generated pip.conf so pip always uses the mirror:
      mkdir -p ~/.pip
      cp /mnt/nfs/pypi-mirror/pip.conf ~/.pip/pip.conf

    Then use pip normally:
      pip install numpy
      pip install -r requirements.txt
      pip install --upgrade pandas

    Or pass the path manually:
      pip install --no-index --find-links /mnt/nfs/pypi-mirror/pypi-packages numpy

  SEARCHING AND LISTING:

    /mnt/nfs/pypi-mirror/pypi-mirror.sh search torch
    /mnt/nfs/pypi-mirror/pypi-mirror.sh list

  WHERE PACKAGES INSTALL:

    The script installs into a per-user virtualenv, not system-wide:
    1. Uses your active venv if $VIRTUAL_ENV is set.
    2. Otherwise looks for ./venv, ./.venv, or ~/venv.
    3. If none exist, creates ./venv in your current directory.

    Benefits: per-user isolation, no sudo, no conflicts between users,
    version independence, easy cleanup (just delete the venv).

  TIPS:

    - Activate your venv each session:  source ~/venv/bin/activate
    - Auto-activate:                    add the above to ~/.bashrc
    - One venv per project:             python3 -m venv ~/projects/myproject/venv
    - Pin dependencies:                 pip freeze > requirements.txt
    - Check your Python:                which python3  (should be in your venv)
    - Package not found?                Ask your admin to run: ./pypi-mirror.sh add <pkg>

  CONDA:

    conda activate myenv
    pip install --no-index --find-links /mnt/nfs/pypi-mirror/pypi-packages numpy

    Install conda packages first, then pip packages. Do not run conda install
    after pip install in the same environment.

================================================================================
READMEEOF
}

# -----------------------------------------------------------------------------
# Entry point
# -----------------------------------------------------------------------------
main() {
  local cmd="${1:-}"
  [[ $# -gt 0 ]] && shift

  case "$cmd" in
    -h|--help|help|"")
      usage
      ;;
    mirror)
      cmd_mirror "$@"
      ;;
    update)
      cmd_update "$@"
      ;;
    install)
      cmd_install "$@"
      ;;
    search)
      cmd_search "$@"
      ;;
    add)
      cmd_add "$@"
      ;;
    list)
      cmd_list "$@"
      ;;
    readme)
      cmd_readme
      ;;
    *)
      log "Unknown command: '${cmd}'"
      usage
      exit 1
      ;;
  esac
}

log "=== ${SCRIPT_NAME} invoked: $* ==="
main "$@"