Merge pull request #173 from kushalbakshi/codespace

Add DevContainers + Codespaces tutorial

.devcontainer/Dockerfile

@@ -0,0 +1,48 @@
+FROM python:3.9-slim@sha256:5f0192a4f58a6ce99f732fe05e3b3d00f12ae62e183886bca3ebe3d202686c7f
+
+ENV PATH /usr/local/bin:$PATH
+ENV PYTHON_VERSION 3.9.17
+
+RUN \
+    adduser --system --disabled-password --shell /bin/bash vscode && \
+    # install docker
+    apt-get update && \
+    apt-get install ca-certificates curl gnupg lsb-release -y && \
+    mkdir -m 0755 -p /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
+    echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \
+    apt-get update && \
+    apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y && \
+    usermod -aG docker vscode && \
+    apt-get clean
+
+RUN \
+    # dev setup
+    apt update && \
+    apt-get install sudo git bash-completion graphviz default-mysql-client s3fs procps -y && \
+    usermod -aG sudo vscode && \
+    echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+    pip install --no-cache-dir --upgrade black pip nbconvert && \
+    echo '. /etc/bash_completion' >> /home/vscode/.bashrc && \
+    echo 'export PS1="\[\e[32;1m\]\u\[\e[m\]@\[\e[34;1m\]\H\[\e[m\]:\[\e[33;1m\]\w\[\e[m\]$ "' >> /home/vscode/.bashrc && \
+    apt-get clean
+
+COPY ./ /tmp/element-array-ephys/
+
+RUN \
+    # pipeline dependencies
+    apt-get install gcc g++ ffmpeg libsm6 libxext6 -y && \
+    pip install --no-cache-dir -e /tmp/element-array-ephys[elements,tests] && \
+    # clean up
+    rm -rf /tmp/element-array-ephys && \
+    apt-get clean
+
+ENV DJ_HOST fakeservices.datajoint.io
+ENV DJ_USER root
+ENV DJ_PASS simple
+
+ENV EPHYS_ROOT_DATA_DIR /workspaces/element-array-ephys/example_data
+ENV DATABASE_PREFIX neuro_
+
+USER vscode
+CMD bash -c "sudo rm /var/run/docker.pid; sudo dockerd"

.devcontainer/devcontainer.json

@@ -0,0 +1,30 @@
+{
+	"name": "Environment + Data",
+	"dockerComposeFile": "docker-compose.yaml",
+	"service": "app",
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+	"remoteEnv": {
+		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
+	},
+	"onCreateCommand": "mkdir -p ${EPHYS_ROOT_DATA_DIR} && pip install -e .",
+	"postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${EPHYS_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1",
+	"hostRequirements": {
+		"cpus": 4,
+		"memory": "8gb",
+		"storage": "32gb"
+	},
+	"forwardPorts": [
+		3306
+	],
+	"customizations": {
+		"settings": {
+			"python.pythonPath": "/usr/local/bin/python"
+		},
+		"vscode": {
+			"extensions": [
+				"[email protected]",
+				"[email protected]"
+			]
+		}
+	}
+}

.devcontainer/docker-compose.yaml

@@ -0,0 +1,25 @@
+version: "3"
+services:
+  app:
+    cpus: 4
+    mem_limit: 8g
+    build:
+      context: ..
+      dockerfile: ./.devcontainer/Dockerfile
+    # image: datajoint/element_array_ephys:latest
+    extra_hosts:
+      - fakeservices.datajoint.io:127.0.0.1
+    environment:
+      - DJ_PUBLIC_S3_LOCATION=djhub.vathes.datapub.elements:/workflow-array-ephys-benchmark/v2
+    devices:
+      - /dev/fuse
+    cap_add:
+      - SYS_ADMIN
+    security_opt:
+      - apparmor:unconfined
+    volumes:
+      - ..:/workspaces/element-array-ephys:cached
+      - docker_data:/var/lib/docker # persist docker images
+    privileged: true # only because of dind
+volumes:
+  docker_data:

.github/workflows/release.yaml

@@ -16,12 +16,12 @@ jobs:
     uses: datajoint/.github/.github/workflows/mkdocs_release.yaml@main
     permissions: 
       contents: write
-  # devcontainer-build:
-  #   uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
-  # devcontainer-publish:
-  #   needs: 
-  #     - devcontainer-build
-  #   uses: datajoint/.github/.github/workflows/devcontainer-publish.yaml@main
-  #   secrets:
-  #     DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}}
-  #     DOCKERHUB_TOKEN: ${{secrets.DOCKERHUB_TOKEN_FOR_ELEMENTS}}
+  devcontainer-build:
+    uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
+  devcontainer-publish:
+    needs: 
+      - devcontainer-build
+    uses: datajoint/.github/.github/workflows/devcontainer-publish.yaml@main
+    secrets:
+      DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}}
+      DOCKERHUB_TOKEN: ${{secrets.DOCKERHUB_TOKEN_FOR_ELEMENTS}}

.github/workflows/test.yaml

@@ -4,8 +4,8 @@ on:
   pull_request:
   workflow_dispatch:
 jobs:
-  # devcontainer-build:
-  #   uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
+  devcontainer-build:
+    uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main
   tests:
     runs-on: ubuntu-latest
     strategy:

.gitignore

@@ -116,14 +116,15 @@ dj_local_conf_old.json
 **/#*#
 **/.#*
 
-docker-compose.y*ml
 
 # include
 !docs/docker-compose.yaml
 
 # vscode settings
-.vscode
 *.code-workspace
 
 # exports/notes
 temp*
+
+# Codespaces
+example_data/

.markdownlint.yaml

@@ -11,6 +11,8 @@ MD030: false # Number of spaces after a list
 MD033: # HTML elements allowed
     allowed_elements:
         - "br"
+        - "figure"
+        - "figcaption"
 MD034: false # Permit bare URLs
 MD031: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling
 MD046: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": false
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,15 @@
+{
+    "editor.formatOnPaste": false,
+    "editor.formatOnSave": true,
+    "editor.rulers": [
+        88
+    ],
+    "python.formatting.provider": "black",
+    "[python]": {
+        "editor.defaultFormatter": null
+    },
+    "[markdown]": {
+        "editor.defaultFormatter": "disable"
+    },
+    "files.autoSave": "off"
+}

CHANGELOG.md

@@ -3,6 +3,13 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
  [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
+## [0.3.0] - 2023-10-25
+
++ Add - DevContainer for codespaces
++ Add - `tutorial_pipeline.py`
++ Add - 60 min tutorial using Jupyter Notebooks, short demo notebook
++ Update - General improvements to increase consistency with other DataJoint Elements
+
 ## [0.2.11] - 2023-06-29
 
 + Update - Improve kilosort triggering routine - better logging, remove temporary files, robust resumable processing
@@ -126,6 +133,8 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 + Add - Probe table supporting: Neuropixels probes 1.0 - 3A, 1.0 - 3B, 2.0 - SS,
   2.0 - MS
 
+[0.3.0]: https://github.com/datajoint/element-array-ephys/releases/tag/0.3.0
+[0.2.11]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.11
 [0.2.10]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.10
 [0.2.9]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.9
 [0.2.8]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.8

CONTRIBUTING.md

@@ -1,5 +1,5 @@
 # Contribution Guidelines
 
 This project follows the
-[DataJoint Contribution Guidelines](https://datajoint.com/docs/community/contribute/).
+[DataJoint Contribution Guidelines](https://datajoint.com/docs/about/contribute/).
 Please reference the link for more full details.

README.md

@@ -1,5 +1,3 @@
-[![PyPI version](https://badge.fury.io/py/element-array-ephys.svg)](http://badge.fury.io/py/element-array-ephys)
-
 # DataJoint Element for Extracellular Electrophysiology
 
 DataJoint Element for extracellular array electrophysiology that processes data 
@@ -12,7 +10,8 @@ Kilosort](https://github.com/MouseLand/pykilosort) spike sorting software. DataJ
 Elements collectively standardize and automate data collection and analysis for 
 neuroscience experiments. Each Element is a modular pipeline for data storage and 
 processing with corresponding database tables that can be combined with other Elements 
-to assemble a fully functional pipeline.
+to assemble a fully functional pipeline. This repository also provides a tutorial
+environment and notebooks to learn the pipeline.
 
 ## Experiment flowchart
 
@@ -22,19 +21,66 @@ to assemble a fully functional pipeline.
 
 ![datajoint](https://raw.githubusercontent.com/datajoint/element-array-ephys/main/images/attached_array_ephys_element_acute.svg)
 
++ We have designed variations of the pipeline to handle different use cases. Displayed
+  above is the `ephys_acute` schema for acute electrophysiology recordings. Details on
+  all `ephys` schemas can be found in the documentation.
 
 ## Getting Started
 
-+ Install from PyPI
++ Please fork this repository.
++ Clone the repository to your computer.
+```bash
+     git clone https://github.com/<enter_github_username>/element-array-ephys.git
+```
+
++ Install with `pip`:
 
      ```bash
-     pip install element-array-ephys
+     pip install -e .
      ```
-
-+ [Interactive tutorial on GitHub Codespaces](https://github.com/datajoint/workflow-array-ephys#interactive-tutorial)
++ [Interactive tutorial on GitHub
+  Codespaces](https://github.com/datajoint/element-array-ephys#interactive-tutorial)
 
 + [Documentation](https://datajoint.com/docs/elements/element-array-ephys)
 
 ## Support
 
-+ If you need help getting started or run into any errors, please contact our team by email at [email protected].
++ If you need help getting started or run into any errors, please contact our team by
+  email at [email protected].
+
+## Interactive Tutorial
+
++ The easiest way to learn about DataJoint Elements is to use the tutorial notebooks within the included interactive environment configured using [Dev Container](https://containers.dev/).
+
+### Launch Environment
+
+Here are some options that provide a great experience:
+
+- (*recommended*) Cloud-based Environment
+  - Launch using [GitHub Codespaces](https://github.com/features/codespaces) using the `+` option which will `Create codespace on main` in the codebase repository on your fork with default options. For more control, see the `...` where you may create `New with options...`.
+  - Build time for a codespace is a few minutes. This is done infrequently and cached for convenience.
+  - Start time for a codespace is less than 1 minute. This will pull the built codespace from cache when you need it.
+  - *Tip*: Each month, GitHub renews a [free-tier](https://docs.github.com/en/billing/managing-billing-for-github-codespaces/about-billing-for-github-codespaces#monthly-included-storage-and-core-hours-for-personal-accounts) quota of compute and storage. Typically we run into the storage limits before anything else since Codespaces consume storage while stopped. It is best to delete Codespaces when not actively in use and recreate when needed. We'll soon be creating prebuilds to avoid larger build times. Once any portion of your quota is reached, you will need to wait for it to be reset at the end of your cycle or add billing info to your GitHub account to handle overages.
+  - *Tip*: GitHub auto names the codespace but you can rename the codespace so that it is easier to identify later.
+
+- Local Environment
+  > *Note: Access to example data is currently limited to MacOS and Linux due to the s3fs utility. Windows users are recommended to use the above environment.*
+  - Install [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
+  - Install [Docker](https://docs.docker.com/get-docker/)
+  - Install [VSCode](https://code.visualstudio.com/)
+  - Install the VSCode [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers)
+  - `git clone` the codebase repository and open it in VSCode
+  - Use the `Dev Containers extension` to `Reopen in Container` (More info is in the `Getting started` included with the extension.)
+
+You will know your environment has finished loading once you either see a terminal open related to `Running postStartCommand` with a final message of `Done` or the `README.md` is opened in `Preview`.
+
+Once the environment has launched, please run the following command in the terminal:
+```
+MYSQL_VER=8.0 docker compose -f docker-compose-db.yaml up --build -d
+```
+
+### Instructions
+
+1. We recommend you start by navigating to the `notebooks` directory on the left panel and go through the `tutorial.ipynb` Jupyter notebook. Execute the cells in the notebook to begin your walk through of the tutorial.
+
+2. Once you are done, see the options available to you in the menu in the bottom-left corner. For example, in Codespace you will have an option to `Stop Current Codespace` but when running Dev Container on your own machine the equivalent option is `Reopen folder locally`. By default, GitHub will also automatically stop the Codespace after 30 minutes of inactivity.  Once the Codespace is no longer being used, we recommend deleting the Codespace.