Fixes towards python3 support

 * update heronpy release scripts for python3
 * update dist Dockerfiles to only use python3
 * remove python2 from docker images

Additionally:
 * fix Ubuntu 16.04 images

There is an issue with encapsulation in the builds where the global python3 environment is used
while PEX installs a nested transitive dependency of pylint: `pylint>astroid>wrapt`. This seems
to be because of logic in its setup.py which can be disabled with `WRAPT_INSTALL_EXTENSIONS=false`

.travis.yml

@@ -19,7 +19,7 @@ addons:
       - python3
       - pkg-config
       - python-dev
-      - python-wheel
+      - python3-wheel
       - wget
       - zip
       - zlib1g-dev

docker/base/Dockerfile.base.debian9

@@ -19,7 +19,7 @@ FROM openjdk:11-jdk-slim-stretch
 
 RUN apt-get -y update && apt-get -y install \
     netcat-openbsd \
-    python \
+    python3 \
     unzip \
     curl \
     supervisor && \

docker/compile/Dockerfile.centos7

@@ -35,8 +35,7 @@ RUN yum -y install \
       libtool \
       make \
       patch \
-      python-devel \
-      cppunit-devel \
+      python3-devel \
       zip \
       unzip \
       wget \

docker/compile/Dockerfile.debian10

@@ -32,10 +32,10 @@ RUN apt-get update && apt-get -y install \
       libtool-bin \
       libcppunit-dev \
       pkg-config \
-      python \
-      python-dev \
+      python3 \
+      python3-dev \
       software-properties-common \
-      python-setuptools \
+      python3-setuptools \
       tree \
       zip \
       unzip \

docker/compile/Dockerfile.debian9

@@ -32,9 +32,8 @@ RUN apt-get update && apt-get -y install \
       libtool-bin \
       libcppunit-dev \
       pkg-config \
-      python-dev \
-      python3-dev \
       software-properties-common \
+      python3-dev \
       python3-setuptools \
       tree \
       zip \

docker/compile/Dockerfile.ubuntu14.04

@@ -33,7 +33,7 @@ RUN apt-get update && apt-get -y install \
       libssl-dev \
       git \
       libtool \
-      python-dev \
+      python3-dev \
       libcppunit-dev \
       zip \
       unzip \

docker/compile/Dockerfile.ubuntu16.04

@@ -21,23 +21,26 @@ FROM ubuntu:16.04
 ENV TARGET_PLATFORM ubuntu
 ENV bazelVersion 3.0.0
 
-RUN apt-get update && apt-get -y install \
+
+RUN apt-get -y update && apt-get -y install \
       automake \
       build-essential \
       cmake \
       curl \
-      libssl-dev \
       git \
+      libssl-dev \
       libtool-bin \
-      libunwind8 \
       libunwind-setjmp0-dev \
-      python-dev \
+      python3-dev \
       libcppunit-dev \
+      software-properties-common \
       tree \
-      zip \
       unzip \
-      wget \
-      openjdk-11-jdk-headless
+      zip \
+      wget
+
+RUN add-apt-repository ppa:openjdk-r/ppa && apt-get -y update && \
+    apt-get -y install openjdk-11-jdk-headless
 
 ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
 

docker/compile/Dockerfile.ubuntu18.04

@@ -29,7 +29,7 @@ RUN apt-get update && apt-get -y install \
       libunwind8 \
       libcppunit-dev \
       patch \
-      python-dev \
+      python3-dev \
       pkg-config \
       wget \
       zip \

docker/compile/Dockerfile.ubuntu20.04

@@ -31,7 +31,7 @@ RUN apt-get update && apt-get -y install \
       libunwind8 \
       libcppunit-dev \
       patch \
-      python-dev \
+      python3-dev \
       pkg-config \
       wget \
       zip \

docker/dist/Dockerfile.dist.centos7

@@ -18,14 +18,16 @@
 FROM centos:centos7
 
 RUN yum -y upgrade
-RUN yum -y install python; yum clean all
 RUN yum -y install unzip; yum clean all
 RUN yum -y install which; yum clean all
 RUN yum -y install curl; yum clean all
 RUN yum -y install nmap-ncat; yum clean all
-RUN yum -y install python python3-setuptools; yum clean all
-RUN easy_install supervisor
-RUN echo_supervisord_conf > /etc/supervisord.conf
+RUN yum -y install python3 python3-setuptools; yum clean all
+RUN yum -y install epel-release \
+    && yum -y update \
+    && yum -y install supervisor \
+    && yum clean all \
+    && echo_supervisord_conf > /etc/supervisord.conf
 
 RUN yum -y install java-11-openjdk java-11-openjdk-devel; yum clean all
 

docker/dist/Dockerfile.dist.debian10

@@ -20,8 +20,8 @@ FROM openjdk:11.0.6-slim-buster
 RUN apt-get update && apt-get -y install \
       netcat-openbsd \
       curl \
-      python \
-      python2.7-dev \
+      python3 \
+      python3-dev \
       supervisor \
       unzip
 

docker/dist/Dockerfile.dist.debian9

@@ -19,8 +19,8 @@ FROM openjdk:11-jdk-slim-stretch
 
 RUN apt-get -y update && apt-get -y install \
     netcat-openbsd \
-    python \
-    python-dev \
+    python3 \
+    python3-dev \
     unzip \
     curl \
     vim \

docker/dist/Dockerfile.dist.ubuntu14.04

@@ -18,7 +18,6 @@
 FROM ubuntu:14.04
 
 RUN apt-get -y update && apt-get -y install \
-    python \
     python3 \
     unzip \
     software-properties-common \

docker/dist/Dockerfile.dist.ubuntu16.04

@@ -20,13 +20,15 @@ FROM ubuntu:16.04
 RUN apt-get update
 
 RUN apt-get -y install \
-    python \
+    python3 \
     python3 \
     unzip \
     software-properties-common \
     curl \
-    supervisor \
-    openjdk-11-jdk-headless
+    supervisor
+
+RUN add-apt-repository ppa:openjdk-r/ppa && apt-get -y update && \
+    apt-get -y install openjdk-11-jdk-headless
 
 ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
 RUN update-ca-certificates -f

docker/dist/Dockerfile.dist.ubuntu18.04

@@ -19,7 +19,7 @@ FROM ubuntu:18.04
 
 RUN apt-get update
 RUN apt-get -y install \
-    unzip software-properties-common curl python python3 supervisor openjdk-11-jdk-headless
+    unzip software-properties-common curl python3 supervisor openjdk-11-jdk-headless
 
 ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
 RUN update-ca-certificates -f

docker/dist/Dockerfile.dist.ubuntu20.04

@@ -21,7 +21,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update
 RUN apt-get -y install \
-    unzip software-properties-common curl python python3 supervisor openjdk-11-jdk-headless
+    unzip software-properties-common curl python3 supervisor openjdk-11-jdk-headless
 
 ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
 RUN update-ca-certificates -f

docker/test/Dockerfile.centos7

@@ -36,7 +36,7 @@ RUN yum -y install \
       cppunit-devel \
       make \
       patch \
-      python-devel \
+      python3-devel \
       python3-devel \
       python3-setuptools \
       zip \

docker/test/Dockerfile.ubuntu18.04

@@ -29,7 +29,6 @@ RUN apt-get update && apt-get -y install \
       libunwind8 \
       libcppunit-dev \
       patch \
-      python-dev \
       python3-dev \
       wget \
       zip \

scripts/packages/BUILD

@@ -644,12 +644,12 @@ genrule(
         'find heronpy -type f -name "*.bak" -delete',
         "rm setup.py.template",
         "tree $$HERONPY_DIR",
-        "/usr/bin/env python2.7 setup.py sdist",
-        "/usr/bin/env python2.7 setup.py bdist_wheel --universal",
+        "/usr/bin/env python3 setup.py sdist",
+        "/usr/bin/env python3 setup.py bdist_wheel",
         "cd -",
         "ls -l $$HERONPY_DIR/dist",
-        "cp $$HERONPY_DIR/dist/heronpy-*-py2.py3-*.whl $$OUTPUT_DIR",
-        'cp $$HERONPY_DIR/dist/heronpy-*-py2.py3-*.whl "$@"',
+        "cp $$HERONPY_DIR/dist/heronpy-*-py3-*.whl $$OUTPUT_DIR",
+        'cp $$HERONPY_DIR/dist/heronpy-*-py3-*.whl "$@"',
         "cp $$HERONPY_DIR/dist/heronpy-*.tar.gz $$OUTPUT_DIR",
         "touch $$OUTPUT_DIR/heronpy.whl",
         "rm -rf $$TMP_DIR",

scripts/packages/heronpy/setup.py.template

@@ -29,7 +29,7 @@ with open(requirement_file, 'r') as f:
   raw_requirements = f.read().strip()
 
 requirements = raw_requirements.split('\n')
-print "Requirements: %s" % requirements
+print("Requirements: %s" % requirements)
 
 long_description = "The heronpy package enables you to write Heron topologies in Python. " \
                    "Topologies can be run using a variety of schedulers, including Mesos/Aurora," \
@@ -51,14 +51,16 @@ setup(
 
     'Intended Audience :: Developers',
 
-    'Programming Language :: Python :: 2.7',
     'Programming Language :: Python :: 3.4',
     'Programming Language :: Python :: 3.5',
     'Programming Language :: Python :: 3.6',
+    'Programming Language :: Python :: 3.7',
+    'Programming Language :: Python :: 3.8',
   ],
 
   keywords='heron topology python',
   packages=find_packages(),
 
-  install_requires=requirements
+  install_requires=requirements,
+  python_requires='~=3.4',
 )

scripts/travis/build.sh

@@ -36,9 +36,8 @@ if [ "$JARS" ]; then
 fi
 
 # verify that eggs have not been added to the repo
-# ./third_party/pex/wheel-0.23.0-py2.7.egg should be the only one
 set +e
-EGGS=`find . -name "*.egg" | grep -v "third_party/pex/wheel"`
+EGGS=`find . -name "*.egg"`
 set -e
 if [ "$EGGS" ]; then
   echo 'ERROR: The following eggs were found in the repo, '\
@@ -49,9 +48,8 @@ if [ "$EGGS" ]; then
 fi
 
 # verify that wheels have not been added to the repo
-# ./third_party/pex/setuptools-18.0.1-py2.py3-none-any.whl should be the only one
 set +e
-WHEELS=`find . -name "*.whl" | grep -v "third_party/pex/setuptools"`
+WHEELS=`find . -name "*.whl"`
 set -e
 if [ "$WHEELS" ]; then
   echo 'ERROR: The following wheels were found in the repo, '\

third_party/python/pylint/BUILD

@@ -6,5 +6,5 @@ pex_binary(
     name = "pylint",
     srcs = ["main.py"],
     main = "main.py",
-    reqs = ["pylint==1.5.5"],
+    reqs = ["pylint==2.5.0"],
 )

tools/rules/pex/BUILD

@@ -26,9 +26,11 @@ POST_EXECUTE = [
     'ln -sf "$$OUTDIR" "$$TMPF"',
     'VENV="$${TMPF}/venv"',
     '$(location @virtualenv//:virtualenv) --no-download --quiet --clear "$$VENV"',
-    'PYTHON="$$VENV/bin/python"',
+    '# this is just to keep the activate script happy',
+    'PS1=',
+    'source "$$VENV/bin/activate"',
 
-    '$$VENV/bin/pip install pex \
+    'pip install pex \
             --quiet --no-cache-dir --no-index --build $(@D)/pexbuild \
             --find-links $$(dirname $(location @pex_src//file)) \
             --find-links $$(dirname $(location @wheel_src//file)) \
@@ -39,8 +41,10 @@ POST_EXECUTE = [
     'cp -r $$(dirname $(location wrapper/setup.py)) $(@D)/.pex_wrapper',
 
     '# Use the bootstrapped pex to build pex_wrapper.pex',
-    '$$VENV/bin/pex $(@D)/.pex_wrapper \
-            --disable-cache --no-index -m pex_wrapper -o $@ \
+    'pex $(@D)/.pex_wrapper setuptools wheel requests \
+            --disable-cache --no-index \
+            --entry-point=pex_wrapper \
+            --output-file=$@ \
             --find-links $$(dirname $(location @pex_src//file)) \
             --find-links $$(dirname $(location @setuptools_src//file)) \
             --find-links $$(dirname $(location @requests_src//file)) \

tools/rules/pex/wrapper/setup.py

@@ -27,6 +27,7 @@
     version="0.1",
     install_requires=[
         "pex",
+        "setuptools",
         "wheel",
         # Not strictly required, but requests makes SSL more likely to work
         "requests",

website2/docs/schedulers-nomad.md

@@ -39,7 +39,7 @@ The advantages of this mode is that it is incredibly lightweight and likely do n
 When setting up your Nomad cluster, the following are required:
 
 * The [Heron CLI tool](user-manuals-heron-cli) must be installed on each machine used to deploy Heron topologies
-* Python 2.7, Java 7 or 8, and [curl](https://curl.haxx.se/) must be installed on every machine in the cluster
+* Python 3, Java 7 or 8, and [curl](https://curl.haxx.se/) must be installed on every machine in the cluster
 * A [ZooKeeper cluster](https://zookeeper.apache.org)
 
 ## Configuring Heron settings

website2/website/versioned_docs/version-0.20.0-incubating/compiling-docker.md

@@ -205,7 +205,7 @@ RUN apt-get update && apt-get -y install \
          libunwind8 \
          libunwind-setjmp0-dev \
          python \
-         python2.7-dev \
+         python3-dev \
          python-software-properties \
          software-properties-common \
          python-setuptools \