From d9c9dbf2a102805b7164bebb18dbab59682cb8ef Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sat, 2 May 2020 15:50:31 +0100 Subject: [PATCH] Fixes towards python3 support * update heronpy release scripts for python3 * update dist Dockerfiles to only use python3 * remove python2 from docker images Additionally: * fix Ubuntu 16.04 images --- docker/base/Dockerfile.base.debian9 | 2 +- docker/compile/Dockerfile.centos7 | 2 +- docker/compile/Dockerfile.debian10 | 6 +++--- docker/compile/Dockerfile.debian9 | 2 +- docker/compile/Dockerfile.ubuntu14.04 | 2 +- docker/compile/Dockerfile.ubuntu16.04 | 18 +++++++++++------- docker/compile/Dockerfile.ubuntu18.04 | 2 +- docker/dist/Dockerfile.dist.centos7 | 10 ++++++---- docker/dist/Dockerfile.dist.debian10 | 4 ++-- docker/dist/Dockerfile.dist.debian9 | 4 ++-- docker/dist/Dockerfile.dist.ubuntu14.04 | 1 - docker/dist/Dockerfile.dist.ubuntu16.04 | 8 +++++--- docker/dist/Dockerfile.dist.ubuntu18.04 | 2 +- docker/test/Dockerfile.centos7 | 2 +- docker/test/Dockerfile.ubuntu18.04 | 1 - scripts/packages/BUILD | 8 ++++---- scripts/packages/heronpy/setup.py.template | 8 +++++--- website2/docs/schedulers-nomad.md | 2 +- .../compiling-docker.md | 2 +- 19 files changed, 47 insertions(+), 39 deletions(-) diff --git a/docker/base/Dockerfile.base.debian9 b/docker/base/Dockerfile.base.debian9 index 27892fce64a..58b1f295764 100644 --- a/docker/base/Dockerfile.base.debian9 +++ b/docker/base/Dockerfile.base.debian9 @@ -19,7 +19,7 @@ FROM openjdk:8-jdk-slim RUN apt-get -y update && apt-get -y install \ netcat-openbsd \ - python \ + python3 \ unzip \ curl \ supervisor && \ diff --git a/docker/compile/Dockerfile.centos7 b/docker/compile/Dockerfile.centos7 index 5e3bc49194a..6d1bb0cad40 100644 --- a/docker/compile/Dockerfile.centos7 +++ b/docker/compile/Dockerfile.centos7 @@ -35,7 +35,7 @@ RUN yum -y install \ libtool \ make \ patch \ - python-devel \ + python3-devel \ zip \ unzip \ wget \ diff --git a/docker/compile/Dockerfile.debian10 b/docker/compile/Dockerfile.debian10 index 2731995f536..fab07045a6c 100644 --- a/docker/compile/Dockerfile.debian10 +++ b/docker/compile/Dockerfile.debian10 @@ -32,10 +32,10 @@ RUN apt-get update && apt-get -y install \ libtool-bin \ libcppunit-dev \ pkg-config \ - python \ - python-dev \ + python3 \ + python3-dev \ software-properties-common \ - python-setuptools \ + python3-setuptools \ tree \ zip \ unzip \ diff --git a/docker/compile/Dockerfile.debian9 b/docker/compile/Dockerfile.debian9 index 95726522d85..b68a4fc62e5 100644 --- a/docker/compile/Dockerfile.debian9 +++ b/docker/compile/Dockerfile.debian9 @@ -30,7 +30,7 @@ RUN apt-get update && apt-get -y install \ git \ libtool \ libtool-bin \ - python-dev \ + python3-dev \ python3-dev \ software-properties-common \ python3-setuptools \ diff --git a/docker/compile/Dockerfile.ubuntu14.04 b/docker/compile/Dockerfile.ubuntu14.04 index 8ba7bea7b8d..24870be1f4a 100644 --- a/docker/compile/Dockerfile.ubuntu14.04 +++ b/docker/compile/Dockerfile.ubuntu14.04 @@ -33,7 +33,7 @@ RUN apt-get update && apt-get -y install \ libssl-dev \ git \ libtool \ - python-dev \ + python3-dev \ zip \ unzip \ wget \ diff --git a/docker/compile/Dockerfile.ubuntu16.04 b/docker/compile/Dockerfile.ubuntu16.04 index 0befaaebf83..edbab6f0497 100644 --- a/docker/compile/Dockerfile.ubuntu16.04 +++ b/docker/compile/Dockerfile.ubuntu16.04 @@ -21,22 +21,26 @@ FROM ubuntu:16.04 ENV TARGET_PLATFORM ubuntu ENV bazelVersion 3.0.0 -RUN apt-get update && apt-get -y install \ + +RUN apt-get -y update && apt-get -y install \ automake \ build-essential \ cmake \ curl \ - libssl-dev \ git \ + libssl-dev \ libtool-bin \ - libunwind8 \ libunwind-setjmp0-dev \ - python-dev \ + libunwind8 \ + python3-dev \ + software-properties-common \ tree \ - zip \ unzip \ - wget \ - openjdk-11-jdk-headless + zip \ + wget + +RUN add-apt-repository ppa:openjdk-r/ppa && apt-get -y update && \ + apt-get -y install openjdk-11-jdk-headless ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 diff --git a/docker/compile/Dockerfile.ubuntu18.04 b/docker/compile/Dockerfile.ubuntu18.04 index 2f115cf25ea..e276c7b461a 100644 --- a/docker/compile/Dockerfile.ubuntu18.04 +++ b/docker/compile/Dockerfile.ubuntu18.04 @@ -29,7 +29,7 @@ RUN apt-get update && apt-get -y install \ libunwind8 \ libcppunit-dev \ patch \ - python-dev \ + python3-dev \ pkg-config \ wget \ zip \ diff --git a/docker/dist/Dockerfile.dist.centos7 b/docker/dist/Dockerfile.dist.centos7 index d548ff5c6e4..6a047ea1b4f 100644 --- a/docker/dist/Dockerfile.dist.centos7 +++ b/docker/dist/Dockerfile.dist.centos7 @@ -18,14 +18,16 @@ FROM centos:centos7 RUN yum -y upgrade -RUN yum -y install python; yum clean all RUN yum -y install unzip; yum clean all RUN yum -y install which; yum clean all RUN yum -y install curl; yum clean all RUN yum -y install nmap-ncat; yum clean all -RUN yum -y install python python3-setuptools; yum clean all -RUN easy_install supervisor -RUN echo_supervisord_conf > /etc/supervisord.conf +RUN yum -y install python3 python3-setuptools; yum clean all +RUN yum -y install epel-release \ + && yum -y update \ + && yum -y install supervisor \ + && yum clean all \ + && echo_supervisord_conf > /etc/supervisord.conf RUN yum -y install java-11-openjdk java-11-openjdk-devel; yum clean all diff --git a/docker/dist/Dockerfile.dist.debian10 b/docker/dist/Dockerfile.dist.debian10 index a78d6f94625..0a12bc25017 100644 --- a/docker/dist/Dockerfile.dist.debian10 +++ b/docker/dist/Dockerfile.dist.debian10 @@ -20,8 +20,8 @@ FROM openjdk:11.0.6-slim-buster RUN apt-get update && apt-get -y install \ netcat-openbsd \ curl \ - python \ - python2.7-dev \ + python3 \ + python3-dev \ supervisor \ unzip diff --git a/docker/dist/Dockerfile.dist.debian9 b/docker/dist/Dockerfile.dist.debian9 index 5d56c1ec2af..7d532f48af7 100644 --- a/docker/dist/Dockerfile.dist.debian9 +++ b/docker/dist/Dockerfile.dist.debian9 @@ -19,8 +19,8 @@ FROM openjdk:11-jdk-slim-stretch RUN apt-get -y update && apt-get -y install \ netcat-openbsd \ - python \ - python-dev \ + python3 \ + python3-dev \ unzip \ curl \ vim \ diff --git a/docker/dist/Dockerfile.dist.ubuntu14.04 b/docker/dist/Dockerfile.dist.ubuntu14.04 index df807aa4e9b..fb1194ecc7f 100644 --- a/docker/dist/Dockerfile.dist.ubuntu14.04 +++ b/docker/dist/Dockerfile.dist.ubuntu14.04 @@ -18,7 +18,6 @@ FROM ubuntu:14.04 RUN apt-get -y update && apt-get -y install \ - python \ python3 \ unzip \ software-properties-common \ diff --git a/docker/dist/Dockerfile.dist.ubuntu16.04 b/docker/dist/Dockerfile.dist.ubuntu16.04 index 0fe59f955a0..dd4c48a00fc 100644 --- a/docker/dist/Dockerfile.dist.ubuntu16.04 +++ b/docker/dist/Dockerfile.dist.ubuntu16.04 @@ -20,13 +20,15 @@ FROM ubuntu:16.04 RUN apt-get update RUN apt-get -y install \ - python \ + python3 \ python3 \ unzip \ software-properties-common \ curl \ - supervisor \ - openjdk-11-jdk-headless + supervisor + +RUN add-apt-repository ppa:openjdk-r/ppa && apt-get -y update && \ + apt-get -y install openjdk-11-jdk-headless ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 RUN update-ca-certificates -f diff --git a/docker/dist/Dockerfile.dist.ubuntu18.04 b/docker/dist/Dockerfile.dist.ubuntu18.04 index 89d4ad7bbbb..893c171afc7 100644 --- a/docker/dist/Dockerfile.dist.ubuntu18.04 +++ b/docker/dist/Dockerfile.dist.ubuntu18.04 @@ -19,7 +19,7 @@ FROM ubuntu:18.04 RUN apt-get update RUN apt-get -y install \ - unzip software-properties-common curl python python3 supervisor openjdk-11-jdk-headless + unzip software-properties-common curl python3 supervisor openjdk-11-jdk-headless ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 RUN update-ca-certificates -f diff --git a/docker/test/Dockerfile.centos7 b/docker/test/Dockerfile.centos7 index 8122b732dff..e7eaad5643e 100644 --- a/docker/test/Dockerfile.centos7 +++ b/docker/test/Dockerfile.centos7 @@ -35,7 +35,7 @@ RUN yum -y install \ libtool \ make \ patch \ - python-devel \ + python3-devel \ python3-devel \ python3-setuptools \ zip \ diff --git a/docker/test/Dockerfile.ubuntu18.04 b/docker/test/Dockerfile.ubuntu18.04 index e9c2e820868..96425e3644b 100644 --- a/docker/test/Dockerfile.ubuntu18.04 +++ b/docker/test/Dockerfile.ubuntu18.04 @@ -28,7 +28,6 @@ RUN apt-get update && apt-get -y install \ libtool-bin \ libunwind8 \ patch \ - python-dev \ python3-dev \ wget \ zip \ diff --git a/scripts/packages/BUILD b/scripts/packages/BUILD index 4f0f7140f96..1cdf4cca229 100644 --- a/scripts/packages/BUILD +++ b/scripts/packages/BUILD @@ -644,12 +644,12 @@ genrule( 'find heronpy -type f -name "*.bak" -delete', "rm setup.py.template", "tree $$HERONPY_DIR", - "/usr/bin/env python2.7 setup.py sdist", - "/usr/bin/env python2.7 setup.py bdist_wheel --universal", + "/usr/bin/env python3 setup.py sdist", + "/usr/bin/env python3 setup.py bdist_wheel", "cd -", "ls -l $$HERONPY_DIR/dist", - "cp $$HERONPY_DIR/dist/heronpy-*-py2.py3-*.whl $$OUTPUT_DIR", - 'cp $$HERONPY_DIR/dist/heronpy-*-py2.py3-*.whl "$@"', + "cp $$HERONPY_DIR/dist/heronpy-*-py3-*.whl $$OUTPUT_DIR", + 'cp $$HERONPY_DIR/dist/heronpy-*-py3-*.whl "$@"', "cp $$HERONPY_DIR/dist/heronpy-*.tar.gz $$OUTPUT_DIR", "touch $$OUTPUT_DIR/heronpy.whl", "rm -rf $$TMP_DIR", diff --git a/scripts/packages/heronpy/setup.py.template b/scripts/packages/heronpy/setup.py.template index 6c7f5219acd..93581407040 100644 --- a/scripts/packages/heronpy/setup.py.template +++ b/scripts/packages/heronpy/setup.py.template @@ -29,7 +29,7 @@ with open(requirement_file, 'r') as f: raw_requirements = f.read().strip() requirements = raw_requirements.split('\n') -print "Requirements: %s" % requirements +print("Requirements: %s" % requirements) long_description = "The heronpy package enables you to write Heron topologies in Python. " \ "Topologies can be run using a variety of schedulers, including Mesos/Aurora," \ @@ -51,14 +51,16 @@ setup( 'Intended Audience :: Developers', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', ], keywords='heron topology python', packages=find_packages(), - install_requires=requirements + install_requires=requirements, + python_requires='~=3.4', ) diff --git a/website2/docs/schedulers-nomad.md b/website2/docs/schedulers-nomad.md index 3c411c692da..94025a494c7 100644 --- a/website2/docs/schedulers-nomad.md +++ b/website2/docs/schedulers-nomad.md @@ -39,7 +39,7 @@ The advantages of this mode is that it is incredibly lightweight and likely do n When setting up your Nomad cluster, the following are required: * The [Heron CLI tool](user-manuals-heron-cli) must be installed on each machine used to deploy Heron topologies -* Python 2.7, Java 7 or 8, and [curl](https://curl.haxx.se/) must be installed on every machine in the cluster +* Python 3, Java 7 or 8, and [curl](https://curl.haxx.se/) must be installed on every machine in the cluster * A [ZooKeeper cluster](https://zookeeper.apache.org) ## Configuring Heron settings diff --git a/website2/website/versioned_docs/version-0.20.0-incubating/compiling-docker.md b/website2/website/versioned_docs/version-0.20.0-incubating/compiling-docker.md index dc714487627..a3eb0b6a96d 100644 --- a/website2/website/versioned_docs/version-0.20.0-incubating/compiling-docker.md +++ b/website2/website/versioned_docs/version-0.20.0-incubating/compiling-docker.md @@ -205,7 +205,7 @@ RUN apt-get update && apt-get -y install \ libunwind8 \ libunwind-setjmp0-dev \ python \ - python2.7-dev \ + python3-dev \ python-software-properties \ software-properties-common \ python-setuptools \