diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 899f0431f..014f7a8fd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,10 +13,20 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - - name: Build image and run tests - run: | - docker build --tag seunglab/pychunkedgraph:$GITHUB_SHA . - docker run --rm seunglab/pychunkedgraph:$GITHUB_SHA /bin/sh -c "pytest --cov-config .coveragerc --cov=pychunkedgraph ./pychunkedgraph/tests && codecov" + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build image + uses: docker/build-push-action@v6 + with: + context: . + load: true + tags: seunglab/pychunkedgraph:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Run tests + run: | + docker run --rm seunglab/pychunkedgraph:${{ github.sha }} /bin/sh -c "pytest --cov-config .coveragerc --cov=pychunkedgraph ./pychunkedgraph/tests && codecov" diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a5e33242d..000000000 --- a/.travis.yml +++ /dev/null @@ -1,60 +0,0 @@ -sudo: true -services: - docker - -env: - global: - - CLOUDSDK_CORE_DISABLE_PROMPTS=1 - -stages: - - test - - name: merge-deploy -python: 3.6 -notifications: - email: - on_success: change - on_failure: always - -jobs: - include: - - stage: test - name: "Running Tests" - language: minimal - before_script: - # request codecov to detect CI environment to pass through to docker - - ci_env=`bash <(curl -s https://codecov.io/env)` - - script: - - openssl aes-256-cbc -K $encrypted_506e835c2891_key -iv $encrypted_506e835c2891_iv -in key.json.enc -out key.json -d - - curl https://sdk.cloud.google.com | bash > /dev/null - - source "$HOME/google-cloud-sdk/path.bash.inc" - - gcloud auth activate-service-account --key-file=key.json - - gcloud auth configure-docker - - docker build --tag seunglab/pychunkedgraph:$TRAVIS_BRANCH . || travis_terminate 1 - - docker run $ci_env --rm seunglab/pychunkedgraph:$TRAVIS_BRANCH /bin/sh -c "tox -v -- --cov-config .coveragerc --cov=pychunkedgraph && codecov" - - - stage: merge-deploy - name: "version bump and merge into master" - language: python - install: - - pip install bumpversion - - before_script: - - "git clone https://gist.github.com/2c04596a45ccac57fe8dde0718ad58ee.git /tmp/travis-automerge" - - "chmod a+x /tmp/travis-automerge/auto_merge_travis_with_bumpversion.sh" - - script: - - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master /tmp/travis-automerge/auto_merge_travis_with_bumpversion.sh" - - - stage: merge-deploy - name: "deploy to pypi" - language: python - install: - - pip install twine - - before_script: - - "git clone https://gist.github.com/cf9b261f26a1bf3fae6b59e7047f007a.git /tmp/travis-autodist" - - "chmod a+x /tmp/travis-autodist/pypi_dist.sh" - - script: - - "BRANCHES_TO_DIST='develop' /tmp/travis-autodist/pypi_dist.sh" diff --git a/Dockerfile b/Dockerfile index 2b7eeb151..1b623a8c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,86 @@ -FROM caveconnectome/pychunkedgraph:base_042124 +# syntax=docker/dockerfile:1 +ARG PYTHON_VERSION=3.11 +ARG BASE_IMAGE=python:${PYTHON_VERSION}-slim + + +###################################################### +# Stage 1: Conda environment +###################################################### +FROM ${BASE_IMAGE} AS conda-deps +ENV PATH="/root/miniconda3/bin:${PATH}" + +RUN apt-get update && apt-get install build-essential wget -y \ + && wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && bash Miniconda3-latest-Linux-x86_64.sh -b \ + && rm Miniconda3-latest-Linux-x86_64.sh \ + && conda config --add channels conda-forge \ + && conda update -y --override-channels -c conda-forge conda \ + && conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main \ + && conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r \ + && conda install -y --override-channels -c conda-forge conda-pack + +COPY requirements.yml requirements.txt requirements-dev.txt ./ + +RUN --mount=type=cache,target=/opt/conda/pkgs \ + conda env create -n pcg -f requirements.yml + +RUN conda-pack -n pcg --ignore-missing-files -o /tmp/env.tar \ + && mkdir -p /app/venv && cd /app/venv \ + && tar xf /tmp/env.tar && rm /tmp/env.tar \ + && /app/venv/bin/conda-unpack + + +###################################################### +# Stage 2: Bigtable emulator +###################################################### +FROM golang:bullseye AS bigtable-emulator +ARG GOOGLE_CLOUD_GO_VERSION=bigtable/v1.19.0 +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + git clone --depth=1 --branch="$GOOGLE_CLOUD_GO_VERSION" \ + https://github.com/googleapis/google-cloud-go.git /usr/src \ + && cd /usr/src/bigtable && go install -v ./cmd/emulator + + +###################################################### +# Stage 3: Production +###################################################### +FROM ${BASE_IMAGE} ENV VIRTUAL_ENV=/app/venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# Force ld to resolve libpython3.x.so to the conda venv's copy. The slim base +# ships its own /usr/local/lib/libpython, which the loader would otherwise pair +# with conda-built C extensions, segfaulting in PyObject_Hash on first import. +ENV LD_LIBRARY_PATH="$VIRTUAL_ENV/lib" +RUN apt-get update && apt-get install -y --no-install-recommends \ + nginx supervisor redis-tools procps \ + && (id nginx >/dev/null 2>&1 || useradd -r -d /home/nginx -s /bin/bash nginx) \ + && mkdir -p /etc/uwsgi /home/nginx/.cloudvolume/secrets \ + && chown -R nginx /home/nginx \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=conda-deps /app/venv /app/venv +COPY --from=bigtable-emulator /go/bin/emulator /app/venv/bin/cbtemulator COPY override/gcloud /app/venv/bin/gcloud COPY override/timeout.conf /etc/nginx/conf.d/timeout.conf +COPY override/nginx.conf /etc/nginx/nginx.conf COPY override/supervisord.conf /etc/supervisor/conf.d/supervisord.conf +COPY uwsgi.ini /etc/uwsgi/uwsgi.ini + +# PyPI wheel bundles the zstd C source; conda-forge's system-linked build lacks +# multi_decompress_to_buffer, used by io/edges.py. +RUN pip install --no-cache-dir --no-deps --force-reinstall zstandard==0.21.0 COPY requirements.txt . -RUN pip install --upgrade -r requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --upgrade -r requirements.txt + COPY . /app +WORKDIR /app + +# --no-deps: graph-tool/cloudvolume etc. are already in the conda venv; setup.py +# reads __version__ from pychunkedgraph/__init__.py. +RUN pip install --no-deps -e . + +CMD ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/base.Dockerfile b/base.Dockerfile deleted file mode 100644 index b5123e137..000000000 --- a/base.Dockerfile +++ /dev/null @@ -1,70 +0,0 @@ -ARG PYTHON_VERSION=3.11 -ARG BASE_IMAGE=tiangolo/uwsgi-nginx-flask:python${PYTHON_VERSION} - - -###################################################### -# Build Image - PCG dependencies -###################################################### -FROM ${BASE_IMAGE} AS pcg-build -ENV PATH="/root/miniconda3/bin:${PATH}" -ENV CONDA_ENV="pychunkedgraph" - -# Setup Miniconda -RUN apt-get update && apt-get install build-essential wget -y -RUN wget \ - https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - && mkdir /root/.conda \ - && bash Miniconda3-latest-Linux-x86_64.sh -b \ - && rm -f Miniconda3-latest-Linux-x86_64.sh \ - && conda update conda - -# Install PCG dependencies - especially graph-tool -# Note: uwsgi has trouble with pip and python3.11, so adding this with conda, too -COPY requirements.txt . -COPY requirements.yml . -COPY requirements-dev.txt . -RUN conda env create -n ${CONDA_ENV} -f requirements.yml - -# Shrink conda environment into portable non-conda env -RUN conda install conda-pack -c conda-forge - -RUN conda-pack -n ${CONDA_ENV} --ignore-missing-files -o /tmp/env.tar \ - && mkdir -p /app/venv \ - && cd /app/venv \ - && tar xf /tmp/env.tar \ - && rm /tmp/env.tar -RUN /app/venv/bin/conda-unpack - - -###################################################### -# Build Image - Bigtable Emulator (without Google SDK) -###################################################### -FROM golang:bullseye as bigtable-emulator-build -RUN mkdir -p /usr/src -WORKDIR /usr/src -ENV GOOGLE_CLOUD_GO_VERSION bigtable/v1.19.0 -RUN apt-get update && apt-get install git -y -RUN git clone --depth=1 --branch="$GOOGLE_CLOUD_GO_VERSION" https://github.com/googleapis/google-cloud-go.git . \ - && cd bigtable \ - && go install -v ./cmd/emulator - - -###################################################### -# Production Image -###################################################### -FROM ${BASE_IMAGE} -ENV VIRTUAL_ENV=/app/venv -ENV PATH="$VIRTUAL_ENV/bin:$PATH" - -COPY --from=pcg-build /app/venv /app/venv -COPY --from=bigtable-emulator-build /go/bin/emulator /app/venv/bin/cbtemulator -COPY override/gcloud /app/venv/bin/gcloud -COPY override/timeout.conf /etc/nginx/conf.d/timeout.conf -COPY override/supervisord.conf /etc/supervisor/conf.d/supervisord.conf -# Hack to get zstandard from PyPI - remove if conda-forge linked lib issue is resolved -RUN pip install --no-cache-dir --no-deps --force-reinstall zstandard==0.21.0 -COPY . /app - -RUN mkdir -p /home/nginx/.cloudvolume/secrets \ - && chown -R nginx /home/nginx \ - && usermod -d /home/nginx -s /bin/bash nginx diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 21f4cc58d..734aca0b4 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -5,19 +5,24 @@ steps: args: ["-c", "docker login --username=$$USERNAME --password=$$PASSWORD"] secretEnv: ["USERNAME", "PASSWORD"] + # Create a buildx builder; the docker-container driver is required to export a + # registry-backed layer cache (the default docker driver cannot). - name: "gcr.io/cloud-builders/docker" entrypoint: "bash" - args: - - "-c" - - | - docker build -t $$USERNAME/pychunkedgraph:$TAG_NAME . - timeout: 600s - secretEnv: ["USERNAME"] + args: ["-c", "docker buildx create --name pcgbuilder --driver docker-container --use"] - # Push the final image to Dockerhub + # Build with a registry layer cache and push. mode=max caches intermediate + # stages (conda env, cbtemulator) so tagged rebuilds reuse them. - name: "gcr.io/cloud-builders/docker" entrypoint: "bash" - args: ["-c", "docker push $$USERNAME/pychunkedgraph:$TAG_NAME"] + args: + - "-c" + - | + docker buildx build \ + --cache-from type=registry,ref=$$USERNAME/pychunkedgraph:buildcache \ + --cache-to type=registry,ref=$$USERNAME/pychunkedgraph:buildcache,mode=max \ + --tag $$USERNAME/pychunkedgraph:$TAG_NAME \ + --push . secretEnv: ["USERNAME"] availableSecrets: @@ -26,3 +31,7 @@ availableSecrets: env: "PASSWORD" - versionName: projects/$PROJECT_ID/secrets/docker-username/versions/1 env: "USERNAME" + +timeout: 3600s +options: + machineType: "E2_HIGHCPU_8" diff --git a/override/nginx.conf b/override/nginx.conf new file mode 100644 index 000000000..3659d6977 --- /dev/null +++ b/override/nginx.conf @@ -0,0 +1,43 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +daemon off; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + map $http_user_agent $loggable { + default 1; + ~*kube-probe 0; + } + + access_log /var/log/nginx/access.log main if=$loggable; + + sendfile on; + + keepalive_timeout 65; + + client_max_body_size 0; + + include /etc/nginx/conf.d/*.conf; + + server { + listen 80; + location / { + include uwsgi_params; + uwsgi_pass unix:///tmp/uwsgi.sock; + } + } +} diff --git a/pychunkedgraph/pipeline/ingest/setup.py b/pychunkedgraph/pipeline/ingest/setup.py index 7f635bbc8..3b683af1c 100644 --- a/pychunkedgraph/pipeline/ingest/setup.py +++ b/pychunkedgraph/pipeline/ingest/setup.py @@ -1,13 +1,13 @@ """One-shot ingest setup — create the Bigtable table and write graph meta. Run once per graph, inside the image, before any layer Jobs: - python -m pychunkedgraph.pipeline.ingest.setup [--raw] + python -m pychunkedgraph.pipeline.ingest.setup [--raw] [--exist-ok] Reads the dataset yaml from its mounted location (no path passed); this is the only step that touches the yaml. Folds the agglomeration source into ``meta.custom_data["agg"] = {"path": str, "raw": bool}`` so workers read everything from Bigtable at run time — no yaml, no Redis. Errors out if the -table already exists (the operator runs setup explicitly once). +table already exists, unless ``--exist-ok`` (idempotent re-run for converge/resume). """ import argparse @@ -25,7 +25,12 @@ DATASET_PATH = environ.get("PCG_DATASET", "/app/datasets/dataset.yml") -def setup(graph_id: str, raw: bool = False, dataset_path: str = DATASET_PATH) -> None: +def setup( + graph_id: str, + raw: bool = False, + exist_ok: bool = False, + dataset_path: str = DATASET_PATH, +) -> None: with open(dataset_path) as stream: config = yaml.safe_load(stream) client_config = BigTableConfig(**config["backend_client"]["CONFIG"]) @@ -35,6 +40,8 @@ def setup(graph_id: str, raw: bool = False, dataset_path: str = DATASET_PATH) -> agg = {"path": config.get("ingest_config", {}).get("AGGLOMERATION"), "raw": raw} meta = ChunkedGraphMeta(graph_config, data_source, custom_data={"agg": agg}) cg = ChunkedGraph(meta=meta, client_info=client_info) + if exist_ok and cg.client._table.exists(): + return # idempotent re-run: the graph is already set up cg.create() @@ -46,8 +53,13 @@ def main() -> None: action="store_true", help="raw agglomeration input; the L2 workers convert it to processed data", ) + parser.add_argument( + "--exist-ok", + action="store_true", + help="idempotent re-run: skip creation if the graph table already exists", + ) args = parser.parse_args() - setup(args.graph_id, raw=args.raw) + setup(args.graph_id, raw=args.raw, exist_ok=args.exist_ok) if __name__ == "__main__": diff --git a/pychunkedgraph/tests/test_uncategorized.py b/pychunkedgraph/tests/test_uncategorized.py index 93c41158d..ddbb2cf74 100644 --- a/pychunkedgraph/tests/test_uncategorized.py +++ b/pychunkedgraph/tests/test_uncategorized.py @@ -2,6 +2,7 @@ import os import subprocess import sys +import uuid from time import sleep from datetime import datetime, timedelta from functools import partial @@ -37,6 +38,7 @@ from ..graph.utils.serializers import serialize_uint64 from ..graph.utils.serializers import deserialize_uint64 from ..ingest.create.abstract_layers import add_layer +from ..pipeline.ingest import setup as setup_mod class TestGraphNodeConversion: @@ -3542,3 +3544,45 @@ def test_indefinite_lock_with_normal_lock_expiration(self, gen_graph): # assert cg.get_root(to_label(cg, 1, 0, 0, 0, 0)) == cg.get_root( # to_label(cg, 1, 0, 0, 0, 0) # ) + + +def test_setup_exist_ok_is_idempotent(bigtable_emulator, monkeypatch, tmp_path): + """`--exist-ok` makes ingest setup a no-op when the graph already exists.""" + # setup only pickles the meta (the watershed is fetched lazily, never here), + # so the data source paths are placeholders; the real config holds non-YAML + # objects (credentials, timedelta), so feed the dict directly. + config = { + "data_source": { + "EDGES": "gs://test/edges", + "COMPONENTS": "gs://test/components", + "WATERSHED": "gs://test/watershed", + }, + "graph_config": { + "CHUNK_SIZE": [512, 512, 64], + "FANOUT": 2, + "SPATIAL_BITS": 10, + "ID_PREFIX": "", + "ROOT_LOCK_EXPIRY": timedelta(seconds=5), + }, + "backend_client": { + "TYPE": "bigtable", + "CONFIG": { + "ADMIN": True, + "READ_ONLY": False, + "PROJECT": "IGNORE_ENVIRONMENT_PROJECT", + "INSTANCE": "emulated_instance", + "CREDENTIALS": credentials.AnonymousCredentials(), + "MAX_ROW_KEY_COUNT": 1000, + }, + }, + "ingest_config": {}, + } + monkeypatch.setattr(setup_mod.yaml, "safe_load", lambda stream: config) + dataset = tmp_path / "dataset.yml" + dataset.write_text("{}") + gid = f"test_{uuid.uuid4().hex[:8]}" + + setup_mod.setup(gid, dataset_path=str(dataset)) # first run creates the table + meta + setup_mod.setup(gid, exist_ok=True, dataset_path=str(dataset)) # re-run: no-op, no raise + with pytest.raises(ValueError): + setup_mod.setup(gid, dataset_path=str(dataset)) # without exist_ok, still errors diff --git a/requirements.yml b/requirements.yml index 0bfa5b227..59ff911cd 100644 --- a/requirements.yml +++ b/requirements.yml @@ -4,7 +4,6 @@ channels: dependencies: - python==3.11.4 - pip - - tox - uwsgi==2.0.21 - graph-tool-base==2.58 - zstandard==0.19.0 # ugly hack to force PyPi install 0.21.0 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 5398564e6..000000000 --- a/tox.ini +++ /dev/null @@ -1,13 +0,0 @@ -[tox] -envlist = py311 -requires = tox-conda - -[testenv] -setenv = HOME = {env:HOME} - -deps = pytest - pytest-cov - pytest-mock - pytest-timeout -conda_env = requirements.yml -commands = python -m pytest -v {posargs} ./pychunkedgraph/tests/