pcgv1 - fix(manifest): allow overriding start layer using query parameter

.bumpversion.cfg

@@ -1,11 +1,6 @@
 [bumpversion]
-current_version = 0.1.113
+current_version = 2.19.2
 commit = True
 tag = True
 
 [bumpversion:file:pychunkedgraph/__init__.py]
-
-[bumpversion:file:pychunkedgraph/app/cg_app_blueprint.py]
-
-[bumpversion:file:pychunkedgraph/app/meshing_app_blueprint.py]
-

.coveragerc

@@ -1,7 +1,6 @@
 # .coveragerc to control coverage.py
 [run]
 branch = True
-concurrency = "multiprocessing"
 source = pychunkedgraph
 omit =
     *test*

.dockerignore

@@ -1,13 +1,124 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
 
-build
-dist
-__pycache__
-.pytest_cache
-.tox
-*.egg-info
-*.egg/
-*.pyc
-*.swp
+# C extensions
+*.so
 
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+.idea/*
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
 .coverage
-.vscode
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# Visual Code
+.vscode/
+
+# terraform
+.terraform/
+*.lock.hcl
+*.tfstate
+*.tfstate.*
+
+
+# local dev stuff
+.devcontainer/
+*.ipynb
+*.rdb
+/protobuf*
+
+# Git
+.git/

.github/workflows/main.yml

@@ -0,0 +1,22 @@
+name: PyChunkedGraph
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+    branches:
+      - "main"
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Build image and run tests
+        run: |
+          docker build --tag seunglab/pychunkedgraph:$GITHUB_SHA .
+          docker run --rm seunglab/pychunkedgraph:$GITHUB_SHA /bin/sh -c "pytest --cov-config .coveragerc --cov=pychunkedgraph ./pychunkedgraph/tests && codecov"
+

.gitignore

@@ -23,7 +23,6 @@ wheels/
 *.egg-info/
 .installed.cfg
 *.egg
-MANIFEST
 .idea/*
 
 # PyInstaller
@@ -108,7 +107,16 @@ venv.bak/
 # Visual Code
 .vscode/
 
+# terraform
+.terraform/
+*.lock.hcl
+*.tfstate
+*.tfstate.*
+
 
 # local dev stuff
-output.txt
-src/
+.devcontainer/
+*.ipynb
+*.rdb
+/protobuf*
+.DS_Store

.travis.yml

@@ -2,6 +2,10 @@ sudo: true
 services:
   docker
 
+env:
+  global:
+  - CLOUDSDK_CORE_DISABLE_PROMPTS=1
+
 stages:
   - test
   - name: merge-deploy
@@ -21,6 +25,11 @@ jobs:
         - ci_env=`bash <(curl -s https://codecov.io/env)`
 
       script:
+        - openssl aes-256-cbc -K $encrypted_506e835c2891_key -iv $encrypted_506e835c2891_iv -in key.json.enc -out key.json -d
+        - curl https://sdk.cloud.google.com | bash > /dev/null
+        - source "$HOME/google-cloud-sdk/path.bash.inc"
+        - gcloud auth activate-service-account --key-file=key.json
+        - gcloud auth configure-docker
         - docker build --tag seunglab/pychunkedgraph:$TRAVIS_BRANCH . || travis_terminate 1
         - docker run $ci_env --rm seunglab/pychunkedgraph:$TRAVIS_BRANCH /bin/sh -c "tox -v -- --cov-config .coveragerc --cov=pychunkedgraph && codecov"
 

Dockerfile

@@ -1,62 +1,11 @@
-FROM tiangolo/uwsgi-nginx-flask:python3.6
-
+FROM caveconnectome/pychunkedgraph:base_042124
+ENV VIRTUAL_ENV=/app/venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
+COPY override/gcloud /app/venv/bin/gcloud
 COPY override/timeout.conf /etc/nginx/conf.d/timeout.conf
 COPY override/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
-COPY requirements.txt /app
-RUN mkdir -p /home/nginx/.cloudvolume/secrets \
-  && chown -R nginx /home/nginx \
-  && usermod -d /home/nginx -s /bin/bash nginx \
-  && apt-get update \
-  && apt-get install -y \
-      # Boost and g++ for compiling DracoPy and graph_tool
-      build-essential \
-      libboost-dev \
-      # Required for adding graph-tools and cloud-sdk to the apt source list
-      lsb-release \
-      curl \
-      apt-transport-https \
-  # GOOGLE-CLOUD-SDK
-  && pip install --no-cache-dir --upgrade crcmod \
-  && echo "deb https://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" > /etc/apt/sources.list.d/google-cloud-sdk.list \
-  && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
-  && apt-get update \
-  && apt-get install -y google-cloud-sdk google-cloud-sdk-bigtable-emulator \
-  # PYTHON-GRAPH-TOOL
-  # WARNING: This is ugly, graph-tools will use Debian's Python version and install as dist-util,
-  #          but all our packages use the site-util Python version - we just create a sym_link,
-  #          because it _seems_ to work and saves 80 minutes compilation time ...
-  && echo "deb http://downloads.skewed.de/apt/$(lsb_release -s -c) $(lsb_release -s -c) main" > /etc/apt/sources.list.d/graph-tool.list \
-  && echo "deb-src http://downloads.skewed.de/apt/$(lsb_release -s -c) $(lsb_release -s -c) main" >> /etc/apt/sources.list.d/graph-tool.list \
-  && apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com --recv-key 612DEFB798507F25 \
-  && apt-get update \
-  && apt-get install -y python3-graph-tool \
-  && ln -s /usr/lib/python3/dist-packages/graph_tool /usr/local/lib/python3.6/site-packages/graph_tool \
-  && pip install --no-cache-dir --upgrade scipy \
-  # PYCHUNKEDGRAPH
-  #   Need pip 18.1 for process-dependency-links flag support
-  && pip install --no-cache-dir pip==18.1 \
-  #   Need numpy to prevent install issue with cloud-volume / fpzip
-  && pip install --no-cache-dir --upgrade numpy \
-  && pip install --no-cache-dir --upgrade --process-dependency-links -r requirements.txt \
-  #   Tests
-  && pip install tox codecov \
-  # CLEANUP
-  #   libboost-dev and build-essentials will be required by tox to build python dependencies
-  && apt-get remove --purge -y lsb-release curl \
-  && apt-get autoremove --purge -y \
-  && rm -rf /var/lib/apt/lists/* \
-  && find /usr/local/lib/python3* -depth \
-      \( \
-        \( -type d -a \( -name __pycache__ \) \) \
-        -o \
-        \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
-      \) -exec rm -rf '{}' + \
-  && find /usr/lib/python3* -depth \
-      \( \
-        \( -type d -a \( -name __pycache__ \) \) \
-        -o \
-        \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
-      \) -exec rm -rf '{}' +
 
-COPY . /app
+COPY requirements.txt .
+RUN pip install --upgrade -r requirements.txt
+COPY . /app

README.md

@@ -4,27 +4,54 @@
 [![Build Status](https://travis-ci.org/seung-lab/PyChunkedGraph.svg?branch=master)](https://travis-ci.org/seung-lab/PyChunkedGraph)
 [![codecov](https://codecov.io/gh/seung-lab/PyChunkedGraph/branch/master/graph/badge.svg)](https://codecov.io/gh/seung-lab/PyChunkedGraph)
 
-The PyChunkedGraph is a proofreading and segmentation data management backend with (not limited to) the following features:
-- Concurrent proofreading by multiple users without restrictions on the workflow
-- Continuous versioning of proofreading edits
-- Making changes visible to all users immediately
-- Local mincut computations
+The PyChunkedGraph is a proofreading and segmentation data management backend powering FlyWire and other proofreading platforms. It builds on an initial agglomeration of supervoxels and facilitates fast and parallel editing of connected components in the agglomeration graph by many users.
 
-## Scaling to large datasets
+## PyChunkedGraph versions
 
+The main branch represents the second version (v2) of the PyChunkedGraph implementation. The first version (v1) is still maintained and can be found under `pcgv1`. The v2 implementation resolved data storage concerns and removed some scaling bottlenecks in the implementation. Any new dataset should use the v2. 
 
-## Deployment 
+## Using the PyChunkedGraph
 
-While the PyChunkedGraph can be deployed as a stand-alone entity, we deploy it within our [annotation infrastructure](https://github.com/seung-lab/AnnotationPipelineOverview) which uses a CI Kubernetes deployment. We will make instructions and scripts for a mostly automated deployment to Google Cloud available soon.
+The ChunkedGraph is built on Google Cloud BigTable. A BigTable instance is required to use this ChunkedGraph implementation. 
 
-## Building your own PyChunkedGraph
+### Environmental Variables
+There are three environmental variables that need to be set
+to connect to a chunkgraph:
 
+- `GOOGLE_APPLICATION_CREDENTIALS`: Location of the google-secret.json file.
+- `BIGTABLE_PROJECT`: Name of the Google Cloud project name.
+- `BIGTABLE_INSTANCE`: Name of the Bigtable Instance ID. (Default is 'pychunkedgraph')
 
+### Ingest 
+
+`/ingest` provides examples for ingest scripts. The ingestion pipeline designed to use the output of the seunglab's agglomeration pipeline but can be adjusted to use alternative data sources. 
+
+### Deployment / Import
+
+The PyChunkedGraph can be locally deployed (`run_dev.py`), imported in a python script (`from pychunkedgraph.backend import chunkedgraph`) or deployed on a kubernetes server. Deployment code for a kubernetes server on Google Cloud is not included in this repository. Please feel free to reach out if you are interested in that. 
 
 ## System Design
 
 As a backend the PyChunkedGraph can be combined with any frontend that adheres to its API. We use an adapted version of [neuroglancer](https://github.com/seung-lab/neuroglancer/tree/nkem-multicut) which is publicly available.
 
 
-[system_design]: https://github.com/seung-lab/PyChunkedGraph/blob/master/ProofreadingDiagram.png "System Design"
-![alt text][system_design]
+## Publication 
+
+When using or referencing the PyChunkedGraph, please use the citation below. The FlyWire paper described and published the PyChunkedGraph v1.
+
+[FlyWire: Online community for whole-brain connectomics](https://www.nature.com/articles/s41592-021-01330-0)
+```
+@article{FlyWire2021,
+  doi = {10.1038/s41592-021-01330-0},
+  url = {https://doi.org/10.1038/s41592-021-01330-0},
+  year = {2021},
+  month = dec,
+  publisher = {Springer Science and Business Media {LLC}},
+  volume = {19},
+  number = {1},
+  pages = {119--128},
+  author = {Sven Dorkenwald and Claire E. McKellar and Thomas Macrina and Nico Kemnitz and Kisuk Lee and Ran Lu and Jingpeng Wu and Sergiy Popovych and Eric Mitchell and Barak Nehoran and Zhen Jia and J. Alexander Bae and Shang Mu and Dodam Ih and Manuel Castro and Oluwaseun Ogedengbe and Akhilesh Halageri and Kai Kuehner and Amy R. Sterling and Zoe Ashwood and Jonathan Zung and Derrick Brittain and Forrest Collman and Casey Schneider-Mizell and Chris Jordan and William Silversmith and Christa Baker and David Deutsch and Lucas Encarnacion-Rivera and Sandeep Kumar and Austin Burke and Doug Bland and Jay Gager and James Hebditch and Selden Koolman and Merlin Moore and Sarah Morejohn and Ben Silverman and Kyle Willie and Ryan Willie and Szi-chieh Yu and Mala Murthy and H. Sebastian Seung},
+  title = {{FlyWire}: online community for whole-brain connectomics},
+  journal = {Nature Methods}
+}
+```