Merge pull request #1 from jotompki/travis-job

Travis job

.gitignore

@@ -125,4 +125,8 @@ dmypy.json
 
 .DS_Store
 .idea/
+.vscode/
 *.tar.gz
+
+# We want to allow the tests/lib folder
+!tests/lib

.pre-commit-config.yaml

@@ -1,10 +1,13 @@
+exclude: ^(buildspec.yml|.pre-commit-config.yaml)$
+fail_fast: true
 repos:
 - repo: https://github.com/pre-commit/mirrors-isort
   rev: v4.3.17
   hooks:
   - id: isort
+    # language_version: python3.6
 - repo: https://github.com/ambv/black
-  rev: stable
+  rev: 20.8b1
   hooks:
   - id: black
     exclude: templates/
@@ -72,6 +75,7 @@ repos:
       --cov-report=html
       --cov="rpdk.python"
       --cov="cloudformation_cli_python_lib"
+      --durations=5
       "tests/"
     language: system
     # ignore all files, run on hard-coded modules instead

.pylintrc

@@ -16,7 +16,7 @@ disable=
 
 [BASIC]
 
-good-names=e,ex,f,fp,i,j,k,n,_
+good-names=e,ex,f,fp,i,j,k,v,n,_
 
 [FORMAT]
 

.travis.yml

@@ -0,0 +1,55 @@
+language: python
+dist: bionic
+sudo: false
+cache:
+  pip: true
+  directories:
+    - $HOME/.cache/pre-commit
+before_install:
+- >
+  pip install --upgrade pip mypy 'attrs==19.2.0'
+  -r https://raw.githubusercontent.com/aws-cloudformation/aws-cloudformation-rpdk/master/requirements.txt
+install:
+- pip install . src/
+env:
+  global:
+  - AWS_DEFAULT_REGION="us-east-1"
+script:
+- pre-commit run --all-files
+jobs:
+  include:
+  - python: "3.6"
+  - python: "3.7"
+  - python: "3.8"
+  - stage: "integ python 3.6"
+    language: python
+    python: "3.6"
+    script:
+      - ls -la
+      - DIR=$(mktemp -d)
+      - cd "$DIR"
+      - ls -la
+      - printf "AWS::Foo::Bar\n1\ny" | cfn init -vv
+      - ls -la
+      - mypy src/aws_foo_bar/ --strict --implicit-reexport
+  - stage: "integ python 3.7"
+    language: python
+    python: "3.7"
+    script:
+      - DIR=$(mktemp -d)
+      - cd "$DIR"
+      - ls -la
+      - printf "AWS::Foo::Bar\n2\ny" | cfn init -vv
+      - ls -la
+      - mypy src/aws_foo_bar/ --strict --implicit-reexport
+  - stage: deploy
+    python: "3.7"
+    script: cd $(bash before_deploy.sh)
+    deploy:
+      provider: pypi
+      user: aws-cloudformation-developers
+      password:
+        secure: "KDSSnOhDMO3sHi4eeOSrsRcs3be5C1cYBdtnmTMOUz6npf39wsssBM6iJfkSdRdpWCr8k1cKodVhE3fcz+Z0vq33oPSWOvakynMrcRQk5Xe7Fzc53kesDEc562smPMiERtFfse0oO+InzIPjsfbsBzLqKlWWQGMqUxrshPmfexsOwKDo+JzT4lVflz6AGQPI0smXa9gHkAu11ne7mIlrmR7f8+mWgqzLTExIJFqYjNECOrT/gDo3zzySO13h5CXf7AM1i0o5p02b9hZ41blkS2OgBeDMSS9qN6QFPT+Erl6Q6y579/vM+knXlPzWBdbqJ2uWaeBfcZZlP7jNp6TkW1WPu4jPL/VnJ/3Eihy4rMkRBuer5zPHj0KBJZoU4jjZx5ctnsYPSZrH7Xo3CHnk1QNckXb+4GZVgz6EWAMGgRmDzJUWTzzu7Dw5EwFQZwESTETqqd+53Ht9yDeJgzA6OneZ4MsWq0OzjUFiiAKMS8BO/uiQrTv3/pJo75JJCLW8wrwTaBTZt6gTuYl+UNeuogITVCdStiH1ECZZ001Bv7tKDhcD4rVB/lJ/I8qIx9QXdWDiRhqqt1+WUl6tlA6sX2vFrTD2wqw9XNwNpIbHF8IoBRI9Cp5wO4m0CreAD6TYbPwEXKMyU5mCEQAv1zSJVag3hf/lhmAR3T7eLUPId2c="
+      server: https://test.pypi.org/legacy/
+      on:
+        tags: true

README.md

@@ -1,28 +1,27 @@
-# DEVELOPER PREVIEW
+# AWS CloudFormation Resource Provider Python Plugin
 
-We're excited to share our progress with adding new languages to the CloudFormation CLI! This plugin is an early preview, and not ready for production use.
+The CloudFormation CLI (cfn) allows you to author your own resource providers that can be used by CloudFormation.
+
+This plugin library helps to provide Python runtime bindings for the execution of your providers by CloudFormation.
 
 ## AWS CloudFormation Resource Provider Python Plugin
 
 The CloudFormation Resource Provider Development Kit (RPDK) allows you to author your own resource providers that can be used by CloudFormation.
 
 This plugin library helps to provide runtime bindings for the execution of your providers by CloudFormation.
 
+[![Build Status](https://travis-ci.com/aws-cloudformation/cloudformation-cli-python-plugin.svg?branch=master)](https://travis-ci.com/aws-cloudformation/cloudformation-cli-python-plugin)
+
 Installation
 ------------
 
-Because this is a developer preview, you can't install it from pypi (the version there will not work)
-You can still install the plugin using [pip](https://pypi.org/project/pip/) from GitHub.
-
 ```bash
-pip install git+https://github.com/aws-cloudformation/aws-cloudformation-rpdk-python-plugin.git#egg=cloudformation-cli-python-plugin
+pip install cloudformation-cli-python-plugin
 ```
 
 Howto
 -----
 
-The support library, `cloudformation-cli-python-lib` must be packaged and present in the project's directory. Packaging can be done by running the `package_lib.sh` script. Example run:
-
 ```
 $ cfn init
 Initializing new project
@@ -41,9 +40,8 @@ This is highly recommended unless you are experienced
 with cross-platform Python packaging.
 >> y
 Initialized a new project in <>
-$ cp ../cloudformation-cli-python-lib-0.0.1.tar.gz .
 $ cfn submit --dry-run
-$ cat test.json
+$ cat <<EOT > test.json
 {
   "credentials": {
     "accessKeyId": "",
@@ -54,14 +52,15 @@ $ cat test.json
   "request": {
     "clientRequestToken": "ecba020e-b2e6-4742-a7d0-8a06ae7c4b2b",
     "desiredResourceState": {
-      "Title": "foo",
-      "Description": "bar"
+      "Title": "This_Is_The_Title_For_My_Example",
+      "TestCode": "NOT_STARTED"
     },
     "previousResourceState": null,
     "logicalResourceIdentifier": null
   },
   "callbackContext": null
 }
+EOT
 $ sam local invoke TestEntrypoint --event test.json
 ```
 

before_deploy.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+if [[ "$TRAVIS_TAG" =~ lib ]]; then
+    echo .
+elif [[ "$TRAVIS_TAG" =~ plugin ]]; then
+    echo src
+fi

python/rpdk/python/__init__.py

@@ -1,5 +1,5 @@
 import logging
 
-__version__ = "0.1"
+__version__ = "2.1.2"
 
 logging.getLogger(__name__).addHandler(logging.NullHandler())

python/rpdk/python/codegen.py

@@ -1,19 +1,21 @@
 import logging
+import os
 import shutil
 import zipfile
 from pathlib import PurePosixPath
-from subprocess import CalledProcessError, run as subprocess_run  # nosec
+from subprocess import PIPE, CalledProcessError, run as subprocess_run  # nosec
 from tempfile import TemporaryFile
 
 import docker
 from docker.errors import APIError, ContainerError, ImageLoadError
+from requests.exceptions import ConnectionError as RequestsConnectionError
 from rpdk.core.data_loaders import resource_stream
 from rpdk.core.exceptions import DownstreamError, SysExitRecommendedError
 from rpdk.core.init import input_with_validation
 from rpdk.core.jsonutils.resolver import ContainerType, resolve_models
 from rpdk.core.plugin_base import LanguagePlugin
 
-from .resolver import models_in_properties, translate_type
+from .resolver import contains_model, translate_type
 
 LOG = logging.getLogger(__name__)
 
@@ -42,33 +44,38 @@ def __init__(self):
             trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True
         )
         self.env.filters["translate_type"] = translate_type
-        self.env.filters["models_in_properties"] = models_in_properties
+        self.env.filters["contains_model"] = contains_model
         self.env.globals["ContainerType"] = ContainerType
         self.namespace = None
         self.package_name = None
         self.package_root = None
-        self._use_docker = True
+        self._use_docker = None
+        self._protocol_version = "2.0.0"
 
     def _init_from_project(self, project):
         self.namespace = tuple(s.lower() for s in project.type_info)
         self.package_name = "_".join(self.namespace)
-        self._use_docker = project.settings.get("use_docker", True)
+        self._use_docker = project.settings.get("use_docker")
         self.package_root = project.root / "src"
 
-    def _prompt_for_use_docker(self, project):
-        self._use_docker = input_with_validation(
+    def _init_settings(self, project):
+        LOG.debug("Writing settings")
+
+        self._use_docker = self._use_docker or input_with_validation(
             "Use docker for platform-independent packaging (Y/n)?\n",
             validate_no,
             "This is highly recommended unless you are experienced \n"
             "with cross-platform Python packaging.",
         )
+
         project.settings["use_docker"] = self._use_docker
+        project.settings["protocolVersion"] = self._protocol_version
 
     def init(self, project):
         LOG.debug("Init started")
 
         self._init_from_project(project)
-        self._prompt_for_use_docker(project)
+        self._init_settings(project)
 
         project.runtime = self.RUNTIME
         project.entrypoint = self.ENTRY_POINT.format(self.package_name)
@@ -95,7 +102,9 @@ def _copy_resource(path, resource_name=None):
         handler_package_path.mkdir(parents=True, exist_ok=True)
         _copy_resource(handler_package_path / "__init__.py")
         _render_template(
-            handler_package_path / "handlers.py", support_lib_pkg=SUPPORT_LIB_PKG
+            handler_package_path / "handlers.py",
+            support_lib_pkg=SUPPORT_LIB_PKG,
+            type_name=project.type_name,
         )
         # models.py produced by generate
 
@@ -197,17 +206,6 @@ def _build(self, base_path):
             self._pip_build(base_path)
         LOG.debug("Dependencies build finished")
 
-    @staticmethod
-    def _check_for_support_lib_sdist(base_path):
-        # TODO: remove this check (and exception) when published to PyPI
-        sdist = base_path / f"{SUPPORT_LIB_NAME}-0.0.1.tar.gz"
-        try:
-            sdist.resolve(strict=True)
-        except FileNotFoundError:
-            raise StandardDistNotFoundError(
-                f"Could not find packaged CloudFormation support library: {sdist}\n"
-            )
-
     @staticmethod
     def _make_pip_command(base_path):
         return [
@@ -217,9 +215,6 @@ def _make_pip_command(base_path):
             "--no-color",
             "--disable-pip-version-check",
             "--upgrade",
-            # TODO: remove find-links when published to PyPI
-            "--find-links",
-            str(base_path),
             "--requirement",
             str(base_path / "requirements.txt"),
             "--target",
@@ -228,8 +223,6 @@ def _make_pip_command(base_path):
 
     @classmethod
     def _docker_build(cls, external_path):
-        cls._check_for_support_lib_sdist(external_path)
-
         internal_path = PurePosixPath("/project")
         command = " ".join(cls._make_pip_command(internal_path))
         LOG.debug("command is '%s'", command)
@@ -249,7 +242,17 @@ def _docker_build(cls, external_path):
                 auto_remove=True,
                 volumes=volumes,
                 stream=True,
+                user=f"{os.geteuid()}:{os.getgid()}",
+            )
+        except RequestsConnectionError as e:
+            # it seems quite hard to reliably extract the cause from
+            # ConnectionError. we replace it with a friendlier error message
+            # and preserve the cause for debug traceback
+            cause = RequestsConnectionError(
+                "Could not connect to docker - is it running?"
             )
+            cause.__cause__ = e
+            raise DownstreamError("Error running docker build") from cause
         except (ContainerError, ImageLoadError, APIError) as e:
             raise DownstreamError("Error running docker build") from e
         LOG.debug("Build running. Output:")
@@ -258,14 +261,13 @@ def _docker_build(cls, external_path):
 
     @classmethod
     def _pip_build(cls, base_path):
-        cls._check_for_support_lib_sdist(base_path)
         command = cls._make_pip_command(base_path)
         LOG.debug("command is '%s'", command)
 
         LOG.warning("Starting pip build.")
         try:
             completed_proc = subprocess_run(  # nosec
-                command, capture_output=True, cwd=base_path, check=True
+                command, stdout=PIPE, stderr=PIPE, cwd=base_path, check=True
             )
         except (FileNotFoundError, CalledProcessError) as e:
             raise DownstreamError("pip build failed") from e

python/rpdk/python/data/Python.gitignore

@@ -122,3 +122,8 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# contains credentials
+sam-tests/
+
+rpdk.log*