WIP: Detect Python version from poetry-managed pyproject.toml

.travis.yml

@@ -9,12 +9,8 @@ before_script:
   - bundle exec hatchet ci:setup
 
 script:
-  - docker build --pull --tag travis-build-cedar-14 --file $(pwd)/builds/cedar-14.Dockerfile .
-  - docker run --rm -e "STACK=cedar-14" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-cedar-14 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-16 --file $(pwd)/builds/heroku-16.Dockerfile .
-  - docker run --rm -e "STACK=heroku-16" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-heroku-16 bash $TESTFOLDER
-  - docker build --pull --tag travis-build-heroku-18 --file $(pwd)/builds/heroku-18.Dockerfile .
-  - docker run --rm -e "STACK=heroku-18" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-heroku-18 bash $TESTFOLDER
+  - docker build --pull --tag travis-build-$STACK --file $(pwd)/builds/$STACK.Dockerfile .
+  - docker run --rm -e "STACK=$STACK" -e "USE_STAGING_BINARIES=$USE_STAGING_BINARIES" travis-build-$STACK bash $TESTFOLDER
 
 jobs:
   include:
@@ -27,16 +23,28 @@ jobs:
     script: "bundle exec rspec"
 env:
   matrix:
-    - TESTFOLDER=test/run-deps
-    - TESTFOLDER=test/run-deps USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging
-    - TESTFOLDER=test/run-versions
-    - TESTFOLDER=test/run-versions USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging
-    - TESTFOLDER=test/run-features
-    - TESTFOLDER=test/run-features USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging
+    - STACK=cedar-14 TESTFOLDER=test/run-deps
+    - STACK=cedar-14 TESTFOLDER=test/run-deps USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging
+    - STACK=cedar-14 TESTFOLDER=test/run-versions
+    - STACK=cedar-14 TESTFOLDER=test/run-versions USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging
+    - STACK=cedar-14 TESTFOLDER=test/run-features
+    - STACK=cedar-14 TESTFOLDER=test/run-features USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging
+    - STACK=heroku-16 TESTFOLDER=test/run-deps
+    - STACK=heroku-16 TESTFOLDER=test/run-deps USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging
+    - STACK=heroku-16 TESTFOLDER=test/run-versions
+    - STACK=heroku-16 TESTFOLDER=test/run-versions USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging
+    - STACK=heroku-16 TESTFOLDER=test/run-features
+    - STACK=heroku-16 TESTFOLDER=test/run-features USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging
+    - STACK=heroku-18 TESTFOLDER=test/run-deps
+    - STACK=heroku-18 TESTFOLDER=test/run-deps USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging
+    - STACK=heroku-18 TESTFOLDER=test/run-versions
+    - STACK=heroku-18 TESTFOLDER=test/run-versions USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging
+    - STACK=heroku-18 TESTFOLDER=test/run-features
+    - STACK=heroku-18 TESTFOLDER=test/run-features USE_STAGING_BINARIES=https://lang-python.s3.amazonaws.com/staging
   global:
     - HATCHET_RETRIES=3
     - IS_RUNNING_ON_CI=true
     - HATCHET_APP_LIMIT=5
     - HATCHET_DEPLOY_STRATEGY=git
-    - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
-    - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=
+    - secure: aICPFKFCmu+OWynYd0QRJVU37Rw3GTCxy3PePj5DSDx3/2S2i0BXXLhn/LECdMY98h24O0xL8k860+gy646Mjq8Cpaf16BUNJhWH5HWS50paymREhnrYd/twSh0i6M2dnlbCjXDHWc8n+64/vu4FmS1UhLw2nQBMCrcIRcxHBpPPHIy3F4pctGBb1LEaExmA3JQRDUoX6uzgt/68gjMDPfvUwM5lzBwRLOdpUwNOvIlYxKioxcqjhMAFuHNo3na50QzPlNE2dqt84iDRj4N7/7v6NzWhG/9O1TxkcVCjn8ZhRIsFQoIEW32X0tqGCsSMpKiR+h6PjGrt+y07xjiRKQPNoFdhjpASO7sGpPrENsOxN263XJPmea9iGJrAvLBYePHdhblzwrXfcM0285RxRJTZg6o7BFW16+PkurpuS7dGvX/7SYAv3IXCksWBbDk+RtpS5KVmYrv6MI+fuGdSFVRq+RS1zmWwaXD1zpc2N4hpV2ZnwI/uwuDwrTd60ou2pXLxttrIw9dqq44CmY5vAu69Fu4dw2BkmUhGUf8ApG6TTgqrt5rdO2W11D4qr/YNZ9Jydh3m+Fa2//JRAGSq0SSW+eqIjCU7D1kqR2lgQD5TdC7mYRFyWGF4YQ8V573Yo0tlL/zQzt07YuyfwIWkb2+AQyZh90cFs+O7T8vg99s=
+    - secure: cu0TCp7HFTCOJTOwQRm1ZhWL86JawLeaUR0/Jg9ThutphfGWNV8nOfgHUG/PEwgb0jwJz34c0DdfpwCdVuDmMsqptmA9I2Lu59BZidi87dqi+PuvfaUvSM69mPABPxaTE34iFtzpWtab+KsnDd/TG87KVk/1AmK1OjA4LKBkJOqo2XkkSFgE94QM04JRSaRiRD8qiLrXTlzkRWsMh1gGnj5kOP0jozBBWbjIMU42XCd9liei7eHoh3ZmFDLOatfIwhCkadG8vBMD5nV0UDCHt2+U6+YGC8cVI+nyjXNAMuOUXZ3GowQRCtIxokVgzRINVAcmDOc40i/gfy9p7HbU3G2FC9wNo98EKecF2GjYtbQdjEgBfijN4nFYi1R/D5CMl4Qo5GtGDkRPO1E626AoHTTtfpIP6MDtjLWTrriiobW8ZuV852jyl/lWJc/O2wYho/34kSER47r5+4398F+9BUbySLzJ9f3/8eNtPIsQunrgO93/PyfkS/2O+7Amov+NPMQ0JoUQuVJt/aHsrFCgbcHR+tOeT7eYYi7mxhLHsteMbRLfCHIIkg3FDEfPITOLgArKj6l9faTJiawLMG9xMMp3Wi6SYPvosDz6gwYDr+8CHeEn0UoUltYM/msMLiGZ0t8oi2DWjvRdLQe9qfC5N77mtcsI1pWEcyc9SOwrIBs=

Makefile

@@ -10,17 +10,17 @@ check:
 
 test-cedar-14:
 	@echo "Running tests in docker (cedar-14)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" -e TESTS=$(TESTS) heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
 	@echo ""
 
 test-heroku-16:
 	@echo "Running tests in docker (heroku-16)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" -e TESTS=$(TESTS) heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
 	@echo ""
 
 test-heroku-18:
 	@echo "Running tests in docker (heroku-18)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-18" -e TESTS=$(TESTS) heroku/heroku:18-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run-deps; test/run-features; test/run-versions;'
 	@echo ""
 
 buildenv-heroku-16:

README.md

@@ -83,6 +83,12 @@ make test-heroku-18
 make test-heroku-16
 ```
 
+You can also specify which tests to run:
+
+```
+make test TESTS="testPython3_7 testGitEgg"
+```
+
 The tests are run via the vendored
 [shunit2](https://github.com/kward/shunit2)
 test framework.

bin/compile

@@ -62,7 +62,25 @@ PY27="python-2.7"
 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
 # If pip doesn't match this version (the version we install), run the installer.
-PIP_UPDATE="9.0.2"
+PIP_UPDATE="19.2.3"
+
+for file in "$BUILD_DIR/runtime.txt" "$CACHE_DIR/.heroku/python-version" ; do
+    [ -f "$file" ] || continue
+
+    version=$(tr -d '[:space:]' < "$file")
+
+    case "$version" in "$PY34"*)
+        # Python 3.4 support was dropped in pip >= 19.2.
+        PIP_UPDATE="19.1.1"
+        break
+        ;;
+    esac
+done
+
+if [[ -f "$BUILD_DIR/Pipfile" ]]; then
+    # Do not force pipenv users to re-install pipenv locally.
+    PIP_UPDATE="9.0.2"
+fi
 
 export DEFAULT_PYTHON_STACK PIP_UPDATE
 export PY37 PY36 PY35 PY27 PY34
@@ -206,6 +224,13 @@ export CACHED_PYTHON_STACK
 # shellcheck source=bin/steps/pipenv-python-version
 source "$BIN_DIR/steps/pipenv-python-version"
 
+# Poetry Python version support.
+# Detect the version of Python requested from a pyproject.toml.
+# Convert it to a runtime.txt file.
+
+# shellcheck source=bin/steps/poetry-python-version
+source "$BIN_DIR/steps/poetry-python-version"
+
 # If no runtime was provided by the user, assume the default Python runtime version.
 if [ ! -f runtime.txt ]; then
   echo "$DEFAULT_PYTHON_VERSION" > runtime.txt
@@ -239,6 +264,10 @@ mtime "python.install.time" "${start}"
 # shellcheck source=bin/steps/pipenv
 source "$BIN_DIR/steps/pipenv"
 
+# Export requirements.txt from poetry.lock, if present.
+# shellcheck source=bin/steps/poetry
+source "$BIN_DIR/steps/poetry"
+
 # Uninstall removed dependencies with Pip.
 # The buildpack will automatically remove any declared dependencies (in requirements.txt)
 # that were explicitly removed. This machinery is a bit complex, but it is not complicated.
@@ -250,9 +279,8 @@ mtime "pip.uninstall.time" "${start}"
 # If no requirements.txt file given, assume `setup.py develop` is intended.
 # This allows for people to ship a setup.py application to Heroku
 # (which is rare, but I vouch that it should work!)
-
-if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then
-  echo "-e ." > requirements.txt
+if [ ! -f requirements.txt ] && [ ! -f Pipfile ] && [ ! -f pyproject.toml ]; then
+    echo "-e ." > requirements.txt
 fi
 
 # Fix egg-links.

bin/detect

@@ -15,7 +15,7 @@
 BUILD_DIR=$1
 
 # Exit early if app is clearly not Python.
-if [ ! -f "$BUILD_DIR/requirements.txt" ] && [ ! -f "$BUILD_DIR/setup.py" ] && [ ! -f "$BUILD_DIR/Pipfile" ]; then
+if [ ! -f "$BUILD_DIR/requirements.txt" ] && [ ! -f "$BUILD_DIR/setup.py" ] && [ ! -f "$BUILD_DIR/Pipfile" ] && [ ! -f "$BUILD_DIR/pyproject.toml" ]; then
   exit 1
 fi
 

bin/steps/pip-install

@@ -34,15 +34,26 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     mcount "tool.pip"
 
     # Count expected build failures.
-    if grep -q '==0.0.0' requirements.txt; then
+    if [ -f requirements.txt ] && grep -q '==0.0.0' requirements.txt; then
         mcount "failure.none-version"
     fi
 
     if [ ! -f "$BUILD_DIR/.heroku/python/bin/pip" ]; then
         exit 1
     fi
-    /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
-    PIP_STATUS="${PIPESTATUS[0]}"
+
+    if [ -f requirements.txt ]; then
+        /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent
+        PIP_STATUS="${PIPESTATUS[0]}"
+    else
+        PIP_STATUS=0
+    fi
+
+    if [ "$PIP_STATUS" -eq 0 ] && [ -f pyproject.toml ]; then
+        /app/.heroku/python/bin/pip install . --exists-action=w --disable-pip-version-check --no-cache-dir 2>&1 | tee -a "$WARNINGS_LOG" | cleanup | indent
+        PIP_STATUS="${PIPESTATUS[0]}"
+    fi
+
     set -e
 
     show-warnings
@@ -53,8 +64,10 @@ if [ ! "$SKIP_PIP_INSTALL" ]; then
     fi
 
     # Smart Requirements handling
-    cp requirements.txt .heroku/python/requirements-declared.txt
-    /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
+    if [ -f requirements.txt ]; then
+        cp requirements.txt .heroku/python/requirements-declared.txt
+        /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
+    fi
 
     echo
 

bin/steps/pip-uninstall

@@ -5,7 +5,7 @@ set +e
 # shellcheck source=bin/utils
 source "$BIN_DIR/utils"
 
-if [ ! "$SKIP_PIP_INSTALL" ]; then
+if [ ! "$SKIP_PIP_INSTALL" ] && [ -f requirements.txt ]; then
 
   if [[ -f .heroku/python/requirements-declared.txt ]]; then
 

bin/steps/poetry

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+set -e
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/utils"
+
+if [ ! -f requirements.txt ] && [ -f pyproject.toml ] && [ -f poetry.lock ]; then
+    # Measure that we're using Poetry.
+    mcount "tool.poetry"
+
+    # Hash poetry.lock to detect changes.
+    POETRY_LOCK_SHA=$(openssl dgst -sha256 poetry.lock)
+
+    # Use cached requirements.txt if poetry.lock is unchanged.
+    CACHED_REQUIREMENTS=$CACHE_DIR/requirements.txt
+    CACHED_POETRY_LOCK_SHA=$CACHE_DIR/poetry.lock.sha256
+
+    if [ -f "$CACHED_REQUIREMENTS" ] && [ -f "$CACHED_POETRY_LOCK_SHA" ] &&
+           [ "$POETRY_LOCK_SHA" == "$(cat "$CACHED_POETRY_LOCK_SHA")" ]; then
+        echo "Skipping requirements export, as poetry.lock hasn't changed since last deploy." | indent
+        cp "$CACHED_REQUIREMENTS" requirements.txt
+    else
+        # Set environment variables for pip
+        # This reads certain environment variables set on the Heroku app config
+        # and makes them accessible to the pip install process.
+        #
+        # PIP_EXTRA_INDEX_URL allows for an alternate pypi URL to be used.
+        if [[ -r "$ENV_DIR/PIP_EXTRA_INDEX_URL" ]]; then
+            PIP_EXTRA_INDEX_URL="$(cat "$ENV_DIR/PIP_EXTRA_INDEX_URL")"
+            export PIP_EXTRA_INDEX_URL
+            mcount "buildvar.PIP_EXTRA_INDEX_URL"
+        fi
+
+        # Set SLUGIFY_USES_TEXT_UNIDECODE, required for Airflow versions >=1.10
+        if [[ -r "$ENV_DIR/SLUGIFY_USES_TEXT_UNIDECODE" ]]; then
+            SLUGIFY_USES_TEXT_UNIDECODE="$(cat "$ENV_DIR/SLUGIFY_USES_TEXT_UNIDECODE")"
+            export SLUGIFY_USES_TEXT_UNIDECODE
+            mcount "buildvar.SLUGIFY_USES_TEXT_UNIDECODE"
+        fi
+
+        # Install Poetry.
+        #
+        # Poetry is not used to install the project because it does not clean up
+        # stale requirements (see sdispater/poetry#648), so we need to export
+        # requirements.txt anyway for the pip-uninstall step.
+        #
+        # Since we only use Poetry to export a requirements.txt file, ignore the
+        # Poetry version specified in pyproject.toml. Install a pre-release of
+        # 1.0.0 because the export command is not available before 1.0.0a0.
+        export POETRY_VERSION="1.0.0b7"
+        puts-step "Exporting requirements with Poetry $POETRY_VERSION…"
+        /app/.heroku/python/bin/pip install "poetry==$POETRY_VERSION" \
+            --disable-pip-version-check &> /dev/null
+
+        # Export requirements.
+        /app/.heroku/python/bin/poetry export -f requirements.txt -o requirements.txt
+
+        # Write SHA and requirements.txt to cache dir.
+        echo "$POETRY_LOCK_SHA" > "$CACHED_POETRY_LOCK_SHA"
+        cp requirements.txt "$CACHED_REQUIREMENTS"
+    fi
+fi

bin/steps/poetry-python-version

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Determine Python version from pyproject.toml, if no runtime.txt is present.
+if [ ! -f $BUILD_DIR/runtime.txt ] &&
+       [ -f $BUILD_DIR/pyproject.toml ] &&
+       [ -f $BUILD_DIR/poetry.lock ]; then
+    # The Python version is specified in the section [tool.poetry.dependencies]
+    # under the `python` key.
+    beg='^ *\[tool.poetry.dependencies\]'
+    end='^ *\['
+    pat='^ *python *= *"\([^"]*\)"'
+
+    PYTHON=$(sed -n "/$beg/,/$end/s/$pat/\\1/p" pyproject.toml)
+
+    # We only handle a few simple cases here. If the Python version constraints
+    # are any more complicated, the user needs to provide an explicit
+    # runtime.txt.
+    case $PYTHON in
+        '^2.7' | '^2.7.'* | '~2.7' | '~2.7.'* | '2.7.*' | "${LATEST_27#python-}")
+            echo "$LATEST_27" > "$BUILD_DIR/runtime.txt"
+            ;;
+
+        '^3.4' | '^3.4.'* | '~3.4' | '~3.4.'* | '3.4.*' | "${LATEST_34#python-}")
+            echo "$LATEST_34" > "$BUILD_DIR/runtime.txt"
+            ;;
+
+        '^3.5' | '^3.5.'* | '~3.5' | '~3.5.'* | '3.5.*' | "${LATEST_35#python-}")
+            echo "$LATEST_35" > "$BUILD_DIR/runtime.txt"
+            ;;
+
+        '^3.6' | '^3.6.'* | '~3.6' | '~3.6.'* | '3.6.*' | "${LATEST_36#python-}")
+            echo "$LATEST_36" > "$BUILD_DIR/runtime.txt"
+            ;;
+
+        '^3.7' | '^3.7.'* | '~3.7' | '~3.7.'* | '3.7.*' | "${LATEST_37#python-}")
+            echo "$LATEST_37" > "$BUILD_DIR/runtime.txt"
+            ;;
+    esac
+fi

test/fixtures/flit-requires/foobar.py

@@ -0,0 +1,3 @@
+"""An amazing sample package!"""
+
+__version__ = '0.1'

test/fixtures/flit-requires/pyproject.toml

@@ -0,0 +1,10 @@
+[build-system]
+requires = ["flit"]
+build-backend = "flit.buildapi"
+
+[tool.flit.metadata]
+module = "foobar"
+author = "Sir Robin"
+author-email = "[email protected]"
+home-page = "https://github.com/sirrobin/foobar"
+requires = ["attrs >=19.1.0"]

test/fixtures/flit/foobar.py

@@ -0,0 +1,3 @@
+"""An amazing sample package!"""
+
+__version__ = '0.1'

test/fixtures/flit/pyproject.toml

@@ -0,0 +1,9 @@
+[build-system]
+requires = ["flit"]
+build-backend = "flit.buildapi"
+
+[tool.flit.metadata]
+module = "foobar"
+author = "Sir Robin"
+author-email = "[email protected]"
+home-page = "https://github.com/sirrobin/foobar"

test/fixtures/poetry-lock/pyproject.toml

@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "foobar"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <[email protected]>"]
+
+[tool.poetry.dependencies]
+python = "^3.6"
+marshmallow = "^3.0.0"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"

test/fixtures/poetry/pyproject.toml

@@ -0,0 +1,14 @@
+[tool.poetry]
+name = "foobar"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <[email protected]>"]
+
+[tool.poetry.dependencies]
+python = "^3.6"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"