Merge pull request #1881 from PrincetonUniversity/devel

Devel

.github/actions/cleanup-pip-cache/action.yml

@@ -0,0 +1,18 @@
+name: 'Cleanup pip wheels cache'
+description: 'Purge pip wheels cache, keeping wheels for the installed pacakges'
+
+runs:
+  using: "composite"
+  steps:
+   - name: "Cleanup old wheels"
+     shell: bash
+     run: |
+        pip cache info
+        INSTALLED=`pip list | sed 's/-/_/g' | sed 's/  */-/' | tail -n+3`
+        CACHED=`pip cache list | cut -f 2,3 -d- | tail -n+3`
+        for P in $CACHED; do
+          # Remove cached and not installed
+          if [ `echo $INSTALLED | grep -o $P | wc -l` == "0" ] ; then
+            pip cache remove -v $P
+          fi
+        done

.github/workflows/pnl-ci-docs.yml

@@ -0,0 +1,78 @@
+name: PsyNeuLink Docs CI
+
+on: push
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.6, 3.7] # Doesn't work in 3.8 or 3.9
+        python-architecture: ['x64']
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    steps:
+    - name: Checkout sources
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 10
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/[email protected]
+      with:
+        python-version: ${{ matrix.python-version }}
+        architecture: ${{ matrix.python-architecture }}
+
+    - name: Get pip cache location
+      shell: bash
+      id: pip_cache
+      run: |
+        python -m pip install -U pip
+        python -m pip --version
+        echo ::set-output name=pip_cache_dir::$(python -m pip cache dir)
+
+    - name: Wheels cache
+      uses: actions/[email protected]
+      with:
+        path: ${{ steps.pip_cache.outputs.pip_cache_dir }}/wheels
+        key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2-${{ github.sha }}
+        restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2
+
+    - name: MacOS dependencies
+      run: HOMEBREW_NO_AUTO_UPDATE=1 brew install graphviz
+      if: startsWith(runner.os, 'macOS')
+
+    - name: Linux dependencies
+      run: sudo apt-get install -y graphviz
+      if: startsWith(runner.os, 'Linux')
+
+    - name: Windows dependencies
+      run: choco install --no-progress -y graphviz --version=2.38.0.20190211
+      if: startsWith(runner.os, 'Windows')
+
+    - name: Shared dependencies
+      shell: bash
+      run: |
+        # explicitly install numpy (https://github.com/pypa/pip/issues/9239)
+        python -m pip install --upgrade pip wheel $(grep numpy requirements.txt)
+        pip install -e .[doc]
+
+    - name: Windows pytorch
+      shell: bash
+      run: |
+        pip install $(grep -o 'torch[0-9<=\.]*' requirements.txt) -f https://download.pytorch.org/whl/cpu/torch_stable.html
+      if: startsWith(runner.os, 'Windows') && matrix.python-architecture != 'x86'
+
+    - name: Cleanup old wheels
+      uses: ./.github/actions/cleanup-pip-cache
+
+    - name: Build Documentation
+      run: sphinx-build -b html -aE docs/source pnl-html
+
+    - name: Upload Documentation
+      uses: actions/[email protected]
+      with:
+        name: Documentation-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.python-architecture }}
+        retention-days: 1
+        path: pnl-html

.github/workflows/pnl-ci.yml

@@ -1,6 +1,14 @@
 name: PsyNeuLink CI
 
-on: [push, pull_request]
+on:
+  push:
+    paths-ignore:
+      - 'docs/**'
+      - 'doc_requirements.txt'
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - 'doc_requirements.txt'
 
 jobs:
   build:
@@ -18,40 +26,32 @@ jobs:
             os: windows-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - name: Checkout sources
+      uses: actions/checkout@v2
       with:
         fetch-depth: 10
 
-    - name: Linux wheels cache
-      uses: actions/[email protected]
-      if: startsWith(runner.os, 'Linux')
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/[email protected]
       with:
-        path: ~/.cache/pip/wheels
-        key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2-${{ github.sha }}
-        restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2
+        python-version: ${{ matrix.python-version }}
+        architecture: ${{ matrix.python-architecture }}
 
-    - name: MacOS wheels cache
-      uses: actions/[email protected]
-      if: startsWith(runner.os, 'macOS')
-      with:
-        path: ~/Library/Caches/pip/wheels
-        key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2-${{ github.sha }}
-        restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2
+    - name: Get pip cache location
+      shell: bash
+      id: pip_cache
+      run: |
+        python -m pip install -U pip
+        python -m pip --version
+        echo ::set-output name=pip_cache_dir::$(python -m pip cache dir)
 
-    - name: Windows wheels cache
+    - name: Wheels cache
       uses: actions/[email protected]
-      if: startsWith(runner.os, 'Windows')
       with:
-        path: ~\AppData\Local\pip\Cache\wheels
+        path: ${{ steps.pip_cache.outputs.pip_cache_dir }}/wheels
         key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2-${{ github.sha }}
         restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-v2
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/[email protected]
-      with:
-        python-version: ${{ matrix.python-version }}
-        architecture: ${{ matrix.python-architecture }}
-
     - name: MacOS dependencies
       run: HOMEBREW_NO_AUTO_UPDATE=1 brew install graphviz
       if: startsWith(runner.os, 'macOS')
@@ -78,17 +78,7 @@ jobs:
       if: startsWith(runner.os, 'Windows') && matrix.python-architecture != 'x86'
 
     - name: Cleanup old wheels
-      shell: bash
-      run: |
-        pip cache info
-        INSTALLED=`pip list | sed 's/-/_/g' | sed 's/  */-/' | tail -n+3`
-        CACHED=`pip cache list | cut -f 2,3 -d- | tail -n+3`
-        for P in $CACHED; do
-          # Remove cached and not installed
-          if [ `echo $INSTALLED | grep -o $P | wc -l` == "0" ] ; then
-            pip cache remove -v $P
-          fi
-        done
+      uses: ./.github/actions/cleanup-pip-cache
 
     - name: Lint with flake8
       shell: bash

.github/workflows/pnl-docs.yml

@@ -0,0 +1,116 @@
+name: PsyNeuLink Docs Compare
+
+on: pull_request
+
+jobs:
+  docs-build:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.7]
+        os: [ubuntu-latest]
+        pnl-version: [ 'base', 'merge']
+
+    runs-on: ${{ matrix.os }}
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+    - name: Checkout merge commit
+      uses: actions/checkout@v2
+      if: ${{ matrix.pnl-version == 'merge' }}
+      with:
+        fetch-depth: 10
+        ref: ${{ github.ref }}
+
+    - name: Checkout pull base
+      uses: actions/checkout@v2
+      if: ${{ matrix.pnl-version == 'base' }}
+      with:
+        fetch-depth: 10
+        ref: ${{ github.base_ref }}
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/[email protected]
+      with:
+        python-version: ${{ matrix.python-version }}
+        architecture: ${{ matrix.python-architecture }}
+
+    - name: Docs dependencies
+      run: |
+        # Install numpy first
+        python -m pip install --upgrade pip wheel $(grep numpy requirements.txt)
+        # We need to install all PNL deps since docs config imports psyneulink module
+        pip install -e .[doc]
+
+    - name: Add git tag
+      # The generated docs include PNL version,
+      # set it to a fixed value to prevent polluting the diff
+      run: git tag 'v999.999.999.999'
+
+    - name: Build docs
+      run: sphinx-build -b html -aE docs/source pnl-html
+
+    - name: Upload generated docs
+      uses: actions/upload-artifact@v2
+      with:
+        name: docs-${{ matrix.pnl-version }}-${{ matrix.os }}-${{ matrix.python-version }}
+        path: pnl-html
+        retention-days: 1
+
+  docs-compare:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.7]
+        os: [ubuntu-latest]
+
+    runs-on: ${{ matrix.os }}
+    needs: [docs-build]
+
+    steps:
+
+    - name: Download generated base docs
+      uses: actions/download-artifact@v2
+      with:
+        name: docs-base-${{ matrix.os }}-${{ matrix.python-version }}
+        path: docs-base
+
+    - name: Download generated merge docs
+      uses: actions/download-artifact@v2
+      with:
+        name: docs-merge-${{ matrix.os }}-${{ matrix.python-version }}
+        path: docs-merge
+
+    - name: Compare
+      shell: bash
+      run: |
+        # Store the resulting diff, or 'No differences!' to and output file
+        # The 'or true' part is needed to workaourd 'pipefail' flag used by github-actions
+        (diff -r docs-base docs-merge && echo 'No differences!' || true) | tee result.diff
+
+    - name: Post comment
+      uses: actions/github-script@v3
+      # Post comment only if not PR across repos
+#      if: ${{ github.event.base.full_name }} == ${{ github.event.head.repo.full_name }}
+      with:
+        script: |
+          // Post comment only if not PR across repos
+          console.log(context.payload.pull_request.base.repo.full_name)
+          console.log(context.payload.pull_request.head.repo.full_name)
+          var base_repo_name = context.payload.pull_request.base.repo.full_name
+          var head_repo_name = context.payload.pull_request.head.repo.full_name
+
+          if (base_repo_name != head_repo_name) return ;
+
+          var fs = require("fs");
+          var text = fs.readFileSync("./result.diff").slice(0,16384);
+
+          github.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: 'This PR causes the following changes to the html docs (${{ matrix.os }}, python-${{ matrix.python-version }}):\n```\n' + text + '\n...\n```\nSee CI logs for the full diff.'
+          })

dev_requirements.txt

@@ -1,5 +1,4 @@
 jupyter<=1.0.0
-psyneulink-sphinx-theme<=1.2.1.7
 pytest<6.2.2
 pytest-benchmark<=3.2.3
 pytest-cov<=2.10.1
@@ -8,5 +7,3 @@ pytest-profiling<=1.7.0
 pytest-pycodestyle<=2.2.0
 pytest-pydocstyle<=2.2.0
 pytest-xdist<2.3.0
-sphinx<3.3.2
-sphinx_autodoc_typehints<1.12.0

doc_requirements.txt

@@ -0,0 +1,3 @@
+psyneulink-sphinx-theme<=1.2.1.7
+sphinx<3.3.2
+sphinx_autodoc_typehints<1.12.0

docs/source/Compilation.rst

@@ -0,0 +1,39 @@
+Compilation
+===========
+
+PsyNeulink includes a runtime compiler to improve performance of executed models.
+This section describes the overview of the compiler design and its use.
+The performance improvements varies, but it has been observed to be between one and three orders of magnitude depending on the model.
+
+
+Overview
+--------
+
+The PsyNeuLink runtime compiler works in several steps when invoked via `run` or `execute`:
+*Compilation*:
+ #. The model is initialized. This is step is identical to non-compiled execution.
+ #. Data structures (input/output/parameters) are flattened and converted to LLVM IR form.
+ #. LLVM IR code is generated to match to semantics of individual components and the used scheduling rules.
+ #. Host CPU compatible binary code is generated
+ #. The resulting function is saved as `ctypes` function and the parameter types are converted to `ctypes` binary structures.
+
+*Execution*:
+ #. parameter structures are populated with the data from `Composition` based on the provided `execution_id`. These structures are preserved between invocations so executions with the same `execution_id` will reuse the same binary structures.
+ #. `ctype` function from step 5. is executed
+ #. Results are extracted from the binary structures and converted to Python format.
+
+
+Use
+---
+
+Compiled form of a model can be invoked by passing one of the following values to the `bin_execute` parameter of `Composition.run`, or `Composition.exec`:
+
+  * `False` or `Python`: Normal python execution
+  * `LLVM`: Compile and execute individual nodes. The scheduling loop still runs in Python. If any of the nodes fails to compile, an error is raised. *NOTE:* Schedules that require access to node data will not work correctly.
+  * `LLVMExec`: Execution of `Composition.exec` is replaced by a compiled equivalent. If the `Composition` fails to compile, an error is raised.
+  * `LLVMRun`: Execution of `Composition.run` is replaced by a compiled equivalent. If the `Composition` fails to compiler, an error is raised.
+  * `True`: This option attempts all three above mentioned granularities, and gracefully falls back to lower granularity. Warnings are raised in place of errors. This is the recommended way to invoke compiled execution as the final fallback is the Python baseline.
+
+Note that data other than `Composition.run` outputs are not synchronized between Python and compiled execution.
+
+ It is possible to invoke compiled version of `FUnction` s and `Mechanism` s. This functionality is provided for testing purposes only, because of the lack of data synchronization it is not recommended for general use.

docs/source/Core.rst

@@ -17,3 +17,4 @@ Core
    - `Registry`
    - `Preferences`
    - `json`
+   - `Compilation`

docs/source/Services.rst

@@ -9,3 +9,4 @@ Services
    Registry
    Preferences
    json
+   Compilation