diff --git a/infrabox/generator/Dockerfile b/infrabox/generator/Dockerfile index 070185447..4e56689ea 100644 --- a/infrabox/generator/Dockerfile +++ b/infrabox/generator/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:3.6 -RUN apk add --no-cache python2 +RUN apk add --no-cache python3 COPY infrabox/generator /generator -CMD python /generator/generator.py +CMD python3 /generator/generator.py diff --git a/infrabox/test-registry/Dockerfile b/infrabox/test-registry/Dockerfile index c157bf034..b1d2923a1 100644 --- a/infrabox/test-registry/Dockerfile +++ b/infrabox/test-registry/Dockerfile @@ -3,6 +3,9 @@ FROM quay.io/infrabox/images-test:build_$INFRABOX_BUILD_NUMBER ENV PYTHONPATH=/ +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 + COPY infrabox/test-registry/ /test/ COPY infrabox/test/utils/id_rsa /var/run/secrets/infrabox.net/rsa/id_rsa COPY src/utils/wait-for-webserver.sh /wait-for-webserver.sh diff --git a/infrabox/test-registry/Dockerfile_auth b/infrabox/test-registry/Dockerfile_auth index 7a0be5240..fad992d49 100644 --- a/infrabox/test-registry/Dockerfile_auth +++ b/infrabox/test-registry/Dockerfile_auth @@ -8,4 +8,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils COPY src/pyinfrabox /pyinfrabox COPY infrabox/test/utils/id_rsa.pub /var/run/secrets/infrabox.net/rsa/id_rsa.pub -CMD python /server.py +CMD python3 /server.py diff --git a/infrabox/test-registry/test.py b/infrabox/test-registry/test.py index 857ac395f..4ef34e52d 100644 --- a/infrabox/test-registry/test.py +++ b/infrabox/test-registry/test.py @@ -18,7 +18,8 @@ class InputTests(TestCase): def get(self, url, password='b514af82-3c4f-4bb5-b1da-a89a0ced5e6f'): - auth = base64.b64encode('infrabox:%s' % encode_project_token(password, '2514af82-3c4f-4bb5-b1da-a89a0ced5e6f', 'myproject')) + basic = 'infrabox:%s' % encode_project_token(password, '2514af82-3c4f-4bb5-b1da-a89a0ced5e6f', 'myproject') + auth = base64.b64encode(basic.encode('utf-8')).decode('utf-8') headers = {'authorization': "Basic " + auth} return requests.get(url, headers=headers) @@ -34,7 +35,7 @@ class Test(TestCase): image_path = project_id + '/image_name' def _get_headers(self): - auth = base64.b64encode('infrabox:%s' % encode_project_token(self.token, self.project_id, 'myproject')) + auth = base64.b64encode(('infrabox:%s' % encode_project_token(self.token, self.project_id, 'myproject')).encode('utf-8')).decode('utf-8') headers = {'authorization': "Basic " + auth} return headers diff --git a/infrabox/test/collector-api/temp_tools.py b/infrabox/test/collector-api/temp_tools.py index 8c86ef8ee..845298ac5 100644 --- a/infrabox/test/collector-api/temp_tools.py +++ b/infrabox/test/collector-api/temp_tools.py @@ -37,7 +37,7 @@ def get(url, headers): # pragma: no cover r = TestClient.app.get(url, headers=headers) if r.mimetype == 'application/json': - j = json.loads(r.data) + j = json.loads(r.data.decode('utf-8')) return j return r diff --git a/infrabox/test/e2e/Dockerfile b/infrabox/test/e2e/Dockerfile index 3efe56087..28a73fe40 100644 --- a/infrabox/test/e2e/Dockerfile +++ b/infrabox/test/e2e/Dockerfile @@ -1,17 +1,23 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive ARG HELM_VERSION=2.12.3 RUN apt-get update -y && apt-get install -y \ curl \ - python-pip \ + pkg-config \ + libpq-dev \ + python3-dev \ + python3-pip \ postgresql-client \ - python-xmlrunner \ - python-requests \ - python-psycopg2 \ - python-jwt \ - python-crypto \ - python-cryptography \ - git + python3-crypto \ + python3-cryptography \ + git \ + && pip3 install \ + xmlrunner \ + requests==2.18.4 \ + psycopg2 \ + PyJWT RUN mkdir /project WORKDIR /project diff --git a/infrabox/test/e2e/entrypoint.sh b/infrabox/test/e2e/entrypoint.sh index 43e213015..70eaff643 100755 --- a/infrabox/test/e2e/entrypoint.sh +++ b/infrabox/test/e2e/entrypoint.sh @@ -44,7 +44,7 @@ _getDependencies() { # pip install infraboxcli git clone https://github.com/SAP/InfraBox-cli.git /cli pushd /cli - pip install -e . + pip3 install -e . infrabox version git rev-parse HEAD popd @@ -139,12 +139,15 @@ _installNginxIngress() { _installInfrabox() { cd /infrabox/context/deploy/infrabox - ssh-keygen -N '' -t rsa -f id_rsa + ssh-keygen -N '' -t rsa -m pem -f id_rsa ssh-keygen -f id_rsa.pub -e -m pem > id_rsa.pem mkdir -p /var/run/secrets/infrabox.net/rsa/ cp id_rsa* /var/run/secrets/infrabox.net/rsa/ + export INFRABOX_RSA_PRIVATE_KEY_PATH=/var/run/secrets/infrabox.net/rsa/id_rsa + export INFRABOX_RSA_PUBLIC_KEY_PATH=/var/run/secrets/infrabox.net/rsa/id_rsa.pem + echo "## Install infrabox" PW=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) @@ -198,7 +201,7 @@ _runTests() { pushd /infrabox/context/infrabox/test/e2e set +e - python test.py + python3 test.py rc=$? cp results.xml /infrabox/upload/testresult diff --git a/infrabox/test/github-review/test.py b/infrabox/test/github-review/test.py index a8c284827..0bea7f6a4 100644 --- a/infrabox/test/github-review/test.py +++ b/infrabox/test/github-review/test.py @@ -123,7 +123,12 @@ def side_effect(_1, _2, _3): requests_post.assert_called_with( 'status_url', - data='{"state": "pending", "target_url": "GITHUB_URL/dashboard/#/project/projectname/build/123/123/job/jobname", "description": "%s", "context": "Job: jobname"}' % ("project_id:%s job_id:%s" % (self.job_data['project_id'], self.job_data['id']) ), + json={ + "target_url": "GITHUB_URL/dashboard/#/project/projectname/build/123/123/job/jobname", + "state": "pending", + "context": "Job: jobname", + "description": "project_id:%s job_id:%s" % (self.job_data['project_id'], self.job_data['id']) + }, headers={'Authorization': 'token token', 'User-Agent': 'InfraBox'}, timeout=10, verify=False) @@ -149,10 +154,14 @@ def side_effect(_1, _2, _3): execute_sql.side_effect = side_effect handle_job_update(None, self.event) - data = '{"state": "%s", "target_url": "GITHUB_URL/dashboard/#/project/projectname/build/123/123/job/jobname", "description": "%s", "context": "Job: jobname"}' % (self.expected_github_status, "project_id:%s job_id:%s" % (self.job_data['project_id'], self.job_data['id'])) requests_post.assert_called_with( 'status_url', - data=data, + json={ + "target_url": "GITHUB_URL/dashboard/#/project/projectname/build/123/123/job/jobname", + "context": "Job: jobname", + "state": self.expected_github_status, + "description": "project_id:%s job_id:%s" % (self.job_data['project_id'], self.job_data['id']) + }, headers={'Authorization': 'token token', 'User-Agent': 'InfraBox'}, timeout=10, diff --git a/infrabox/test/registry-auth/test.py b/infrabox/test/registry-auth/test.py index 9b4b80aa9..d46b3f12d 100644 --- a/infrabox/test/registry-auth/test.py +++ b/infrabox/test/registry-auth/test.py @@ -58,12 +58,12 @@ def test_header_invalid_token(self): self.assertEqual(r['status'], 401) def test_header_no_password(self): - h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox')} + h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox'.encode("utf-8"))} r = self.get('/v2', h) self.assertEqual(r['status'], 401) def test_header_no_password_2(self): - h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox:2')} + h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox:2'.encode("utf-8"))} r = self.get('/v2', h) self.assertEqual(r['status'], 401) @@ -121,7 +121,7 @@ def get_project_headers(self, project_token=None): # pragma: no cover project_token = self.project_token token = encode_project_token(project_token, self.project_id, 'myproject') - h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox:%s' % token)} + h = {'Authorization': 'Basic %s' % base64.b64encode(('infrabox:%s' % token).encode("utf-8")).decode('utf-8')} return h def get(self, url, headers=None, method='GET'): # pragma: no cover @@ -131,7 +131,7 @@ def get(self, url, headers=None, method='GET'): # pragma: no cover r = self.app.get(url, headers=headers) if r.mimetype == 'application/json': - return json.loads(r.data) + return json.loads(r.data.decode('utf-8')) return r diff --git a/src/api/Dockerfile b/src/api/Dockerfile index a3db93875..7e497e41c 100644 --- a/src/api/Dockerfile +++ b/src/api/Dockerfile @@ -8,4 +8,4 @@ COPY src/openpolicyagent /infrabox/openpolicyagent COPY src/pyinfraboxutils /infrabox/pyinfraboxutils COPY src/pyinfrabox /infrabox/pyinfrabox -CMD python /infrabox/api/server.py +CMD python3 /infrabox/api/server.py diff --git a/src/api/handlers/account/saml.py b/src/api/handlers/account/saml.py index 6043ca421..173af5f94 100644 --- a/src/api/handlers/account/saml.py +++ b/src/api/handlers/account/saml.py @@ -1,4 +1,4 @@ -from urlparse import urlparse +from urllib.parse import urlparse from flask import g, request, abort, redirect, make_response @@ -36,7 +36,7 @@ def init_saml_auth(): def get_attribute_dict(saml_auth): attributes = {} nested_attribute_dict = saml_auth.get_attributes() - for attribute_name, nested_attribute in nested_attribute_dict.items(): + for attribute_name, nested_attribute in list(nested_attribute_dict.items()): if len(nested_attribute) > 0: attributes[attribute_name] = nested_attribute[0] attributes["NameID"] = saml_auth.get_nameid() diff --git a/src/api/handlers/admin/projects.py b/src/api/handlers/admin/projects.py index 92e71abf1..516f5f9da 100644 --- a/src/api/handlers/admin/projects.py +++ b/src/api/handlers/admin/projects.py @@ -52,7 +52,7 @@ def get(self): result[j['name']]['jobs'] = j['cnt'] response = [] - for _, v in result.iteritems(): + for _, v in result.items(): response.append(v) return response diff --git a/src/api/handlers/job_api.py b/src/api/handlers/job_api.py index e3550a92f..f5204d0f5 100644 --- a/src/api/handlers/job_api.py +++ b/src/api/handlers/job_api.py @@ -3,7 +3,7 @@ import json import uuid import copy -import urllib +import urllib.request, urllib.parse, urllib.error import random from datetime import datetime from io import BytesIO @@ -344,8 +344,8 @@ def get_secret(name): root_url = get_root_url("global") # Default env vars - project_name = urllib.quote_plus(data['project']['name']).replace('+', '%20') - job_name = urllib.quote_plus(data['job']['name']).replace('+', '%20') + project_name = urllib.parse.quote_plus(data['project']['name']).replace('+', '%20') + job_name = urllib.parse.quote_plus(data['job']['name']).replace('+', '%20') build_url = "%s/dashboard/#/project/%s/build/%s/%s" % (root_url, project_name, data['build']['build_number'], @@ -390,14 +390,14 @@ def get_secret(name): data['env_vars']['INFRABOX_GITHUB_PULL_REQUEST'] = "true" if env_vars: - for name, value in env_vars.iteritems(): + for name, value in env_vars.items(): try: data['env_vars'][name] = str(value) except UnicodeEncodeError: data['env_vars'][name] = value.encode('utf-8') if env_var_refs: - for name, value in env_var_refs.iteritems(): + for name, value in env_var_refs.items(): secret = get_secret(value) if secret is None: @@ -467,7 +467,7 @@ def post(self): project_id = g.token['project']['id'] job_name = g.token['job']['name'] - for f, _ in request.files.items(): + for f, _ in list(request.files.items()): template = 'project_%s_job_%s_%s' key = template % (project_id, job_name, f) key = key.replace('/', '_') @@ -520,7 +520,7 @@ class Output(Resource): def post(self): job_id = g.token['job']['id'] - for f, _ in request.files.items(): + for f, _ in list(request.files.items()): key = "%s/%s" % (job_id, f) stream = request.files[f].stream @@ -942,7 +942,7 @@ def post(self): stats = request.json['stats'] s = 0 c = 0 - for _, values in stats.items(): + for _, values in list(stats.items()): for v in values: c += 1 s += v['cpu'] @@ -1000,7 +1000,7 @@ def _remove_file(response): delete_file(path) return response - for name, f in request.files.iteritems(): + for name, f in request.files.items(): try: if not allowed_file(f.filename, ("json",)): abort(400, "Filetype not allowed") @@ -1049,7 +1049,7 @@ def _remove_file(response): delete_file(path) return response - for _, f in request.files.iteritems(): + for _, f in request.files.items(): if not allowed_file(f.filename, ("json")): abort(400, "Filetype not allowed") diff --git a/src/api/handlers/project.py b/src/api/handlers/project.py index 756e55220..ce0e5fa35 100644 --- a/src/api/handlers/project.py +++ b/src/api/handlers/project.py @@ -1,5 +1,5 @@ import uuid -import urllib +import urllib.request, urllib.parse, urllib.error import os import json import mimetypes @@ -49,16 +49,16 @@ def no_cache(*args, **kwargs): return update_wrapper(no_cache, view) def get_badge(subject, status, color): - subject = urllib.quote(subject) - status = urllib.quote(status) - color = urllib.quote(color) + subject = urllib.parse.quote(subject) + status = urllib.parse.quote(status) + color = urllib.parse.quote(color) url = 'https://img.shields.io/static/v1.svg?label=%s&message=%s&color=%s' % (subject, status, color) resp = requests.get(url) excluded_headers = ['content-encoding', 'content-length', 'transfer-encoding', 'connection'] - headers = [(name, value) for (name, value) in resp.raw.headers.items() + headers = [(name, value) for (name, value) in list(resp.raw.headers.items()) if name.lower() not in excluded_headers] return Response(resp.content, resp.status_code, headers) diff --git a/src/api/run_dev.sh b/src/api/run_dev.sh index 4b4e0b141..b473627e9 100755 --- a/src/api/run_dev.sh +++ b/src/api/run_dev.sh @@ -14,7 +14,7 @@ sigint_handler() trap sigint_handler SIGINT while true; do - python $DIR/server.py & + python3 $DIR/server.py & PID=$! inotifywait -e modify -e move -e create --exclude='.*__pycache__.*' -r $watch kill $PID diff --git a/src/api/run_with_dummy.sh b/src/api/run_with_dummy.sh index b38d10555..f085a1d7b 100755 --- a/src/api/run_with_dummy.sh +++ b/src/api/run_with_dummy.sh @@ -67,4 +67,4 @@ export INFRABOX_LEGAL_PRIVACY_URL=https://www.sap.com/about/legal/privacy.html export INFRABOX_LEGAL_TERMS_OF_USE_URL=https://www.sap.com/corporate/en/legal/terms-of-use.html -python server.py +python3 server.py diff --git a/src/checker/Dockerfile b/src/checker/Dockerfile index 233f39808..bc38923e4 100644 --- a/src/checker/Dockerfile +++ b/src/checker/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -ENTRYPOINT ["python", "checker/checker.py"] +ENTRYPOINT ["python3", "checker/checker.py"] diff --git a/src/cluster-status/Dockerfile b/src/cluster-status/Dockerfile index 586d3e65d..9f2c166c0 100644 --- a/src/cluster-status/Dockerfile +++ b/src/cluster-status/Dockerfile @@ -7,4 +7,4 @@ COPY src/pyinfrabox /pyinfrabox ENV PYTHONPATH=/ -CMD python /server.py +CMD python3 /server.py diff --git a/src/collector-api/Dockerfile b/src/collector-api/Dockerfile index 66bae7f84..f5b5afa68 100644 --- a/src/collector-api/Dockerfile +++ b/src/collector-api/Dockerfile @@ -7,4 +7,4 @@ COPY src/collector-api /collector COPY src/pyinfraboxutils /pyinfraboxutils COPY src/pyinfrabox /pyinfrabox -CMD python /collector/server.py +CMD python3 /collector/server.py diff --git a/src/db/Dockerfile b/src/db/Dockerfile index f0631baee..f2c85a1c1 100644 --- a/src/db/Dockerfile +++ b/src/db/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -CMD python db/migrate.py +CMD python3 db/migrate.py diff --git a/src/docker-registry/auth/Dockerfile b/src/docker-registry/auth/Dockerfile index f5cfa5245..33f801823 100644 --- a/src/docker-registry/auth/Dockerfile +++ b/src/docker-registry/auth/Dockerfile @@ -7,4 +7,4 @@ COPY src/pyinfrabox /pyinfrabox ENV PYTHONPATH=/ -CMD python /server.py +CMD python3 /server.py diff --git a/src/docker-registry/nginx/Dockerfile b/src/docker-registry/nginx/Dockerfile index 98250562d..2e26284ec 100644 --- a/src/docker-registry/nginx/Dockerfile +++ b/src/docker-registry/nginx/Dockerfile @@ -1,6 +1,6 @@ FROM nginx:1.15-alpine -RUN apk add --no-cache curl python py-pip openssl && \ - pip install htpasswd && \ +RUN apk add --no-cache curl python3 py-pip openssl && \ + pip3 install htpasswd && \ apk del py-pip COPY src/docker-registry/nginx/nginx.conf /etc/nginx/nginx.conf diff --git a/src/docker-registry/nginx/entrypoint.sh b/src/docker-registry/nginx/entrypoint.sh index e493ec0fe..fc733681c 100755 --- a/src/docker-registry/nginx/entrypoint.sh +++ b/src/docker-registry/nginx/entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/sh -e -HTPASSWD_PATH=/home/nginx/nginx.htpasswd python get_admin_pw.py +HTPASSWD_PATH=/home/nginx/nginx.htpasswd python3 get_admin_pw.py sed -i -e "s/auth-host-placeholder/$INFRABOX_AUTH_HOST/g" /etc/nginx/nginx.conf sed -i -e "s/registry-host-placeholder/$INFRABOX_REGISTRY_HOST/g" /etc/nginx/nginx.conf diff --git a/src/docker-registry/nginx/get_admin_pw.py b/src/docker-registry/nginx/get_admin_pw.py index bdfa5551f..0853a0a1b 100644 --- a/src/docker-registry/nginx/get_admin_pw.py +++ b/src/docker-registry/nginx/get_admin_pw.py @@ -21,5 +21,5 @@ with htpasswd.Basic(os.environ['HTPASSWD_PATH'], mode="md5") as userdb: try: userdb.add(admin_user, admin_password) - except htpasswd.basic.UserExists, e: + except htpasswd.basic.UserExists as e: print(e) diff --git a/src/gc/Dockerfile b/src/gc/Dockerfile index 215318a81..aa82500a5 100644 --- a/src/gc/Dockerfile +++ b/src/gc/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -CMD python gc/gc.py +CMD python3 gc/gc.py diff --git a/src/gerrit/api/entrypoint.sh b/src/gerrit/api/entrypoint.sh index 762cae851..653daae94 100755 --- a/src/gerrit/api/entrypoint.sh +++ b/src/gerrit/api/entrypoint.sh @@ -5,4 +5,4 @@ cp /tmp/gerrit/id_rsa ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa ssh-keyscan -p $INFRABOX_GERRIT_PORT $INFRABOX_GERRIT_HOSTNAME >> ~/.ssh/known_hosts -python /api.py +python3 /api.py diff --git a/src/gerrit/review/entrypoint.sh b/src/gerrit/review/entrypoint.sh index 24ca95f21..d1f2f8b2f 100755 --- a/src/gerrit/review/entrypoint.sh +++ b/src/gerrit/review/entrypoint.sh @@ -5,4 +5,4 @@ cp /tmp/gerrit/id_rsa ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa ssh-keyscan -p $INFRABOX_GERRIT_PORT $INFRABOX_GERRIT_HOSTNAME >> ~/.ssh/known_hosts -python /review.py +python3 /review.py diff --git a/src/gerrit/review/review.py b/src/gerrit/review/review.py index c701fdef8..258fdb161 100755 --- a/src/gerrit/review/review.py +++ b/src/gerrit/review/review.py @@ -1,6 +1,6 @@ import json import select -import urllib +import urllib.request, urllib.parse, urllib.error import time import os @@ -60,7 +60,7 @@ def main(): time.sleep(3) def handle_job_update_retry(conn, update): - for _ in xrange(0, 3): + for _ in range(0, 3): try: handle_job_update(conn, update) return @@ -163,7 +163,7 @@ def handle_job_update(conn, event): key_filename=gerrit_key_filename) client.get_transport().set_keepalive(60) - project_name_quote = urllib.quote_plus(project_name).replace('+', '%20') + project_name_quote = urllib.parse.quote_plus(project_name).replace('+', '%20') build_url = "%s/dashboard/#/project/%s/build/%s/%s" % (dashboard_url, project_name_quote, build_number, diff --git a/src/gerrit/trigger/entrypoint.sh b/src/gerrit/trigger/entrypoint.sh index 809c70af3..4c82f330b 100755 --- a/src/gerrit/trigger/entrypoint.sh +++ b/src/gerrit/trigger/entrypoint.sh @@ -5,4 +5,4 @@ cp /tmp/gerrit/id_rsa ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa ssh-keyscan -p $INFRABOX_GERRIT_PORT $INFRABOX_GERRIT_HOSTNAME >> ~/.ssh/known_hosts -python /trigger.py +python3 /trigger.py diff --git a/src/github/review/Dockerfile b/src/github/review/Dockerfile index b28690758..a18ff9bf6 100644 --- a/src/github/review/Dockerfile +++ b/src/github/review/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -CMD python review.py +CMD python3 review.py diff --git a/src/github/review/review.py b/src/github/review/review.py index 0116f26c2..ad218e7cd 100755 --- a/src/github/review/review.py +++ b/src/github/review/review.py @@ -2,7 +2,7 @@ import select import os -import urllib +import urllib.request, urllib.parse, urllib.error import requests import psycopg2 @@ -150,10 +150,10 @@ def handle_job_update(conn, event): ''', [])[0]['root_url'] target_url = '%s/dashboard/#/project/%s/build/%s/%s/job/%s' % (dashboard_url, - urllib.quote(project_name, safe=''), + urllib.parse.quote(project_name, safe=''), build_number, build_restartCounter, - urllib.quote_plus(job_name).replace('+', '%20')) + urllib.parse.quote_plus(job_name).replace('+', '%20')) job_name = job_name.split(".")[0] payload = { @@ -171,7 +171,7 @@ def handle_job_update(conn, event): # TODO(ib-steffen): support ca bundles try: r = requests.post(github_status_url, - data=json.dumps(payload), + json=payload, headers=headers, timeout=10, verify=False) diff --git a/src/github/trigger/Dockerfile b/src/github/trigger/Dockerfile index a0f33536f..69aa3d7ea 100644 --- a/src/github/trigger/Dockerfile +++ b/src/github/trigger/Dockerfile @@ -7,4 +7,4 @@ COPY src/pyinfrabox /pyinfrabox ENV PYTHONPATH=/ -CMD python /trigger.py +CMD python3 /trigger.py diff --git a/src/images/base/Dockerfile b/src/images/base/Dockerfile index ad08ea1c6..0169a175d 100644 --- a/src/images/base/Dockerfile +++ b/src/images/base/Dockerfile @@ -1,23 +1,28 @@ -FROM debian:9.6-slim +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y \ && apt-get install -y \ - libpq-dev \ - python \ - python-pip \ - python-six \ - python-cryptography \ - python-crypto \ - python-pyasn1 \ - python-ldap \ + pkg-config \ + libpq-dev \ + python3-dev \ + python3-pip \ + python3-cryptography \ + python3-crypto \ + python3-pyasn1 \ libxml2-dev \ libxmlsec1-dev \ - python-paramiko \ + python3-paramiko \ openssh-client \ inotify-tools \ - && pip install \ - pyrsistent==0.15.7 \ - six==1.14.0 \ + libsasl2-dev \ + libldap2-dev \ + libssl-dev \ + libxmlsec1-openssl \ + && pip3 install \ + six \ + python-ldap \ psycopg2 \ flask \ flask-restx \ @@ -37,11 +42,9 @@ RUN apt-get update -y \ keystoneauth1==3.7.0 \ python-swiftclient \ python-cachetclient \ - python-saml \ - croniter \ - && apt-get remove -y python-pip \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* + xmlsec \ + python3-saml \ + croniter RUN useradd -ms /bin/bash infrabox USER infrabox diff --git a/src/images/test/Dockerfile b/src/images/test/Dockerfile index 5f0015270..1887142c0 100644 --- a/src/images/test/Dockerfile +++ b/src/images/test/Dockerfile @@ -1,20 +1,30 @@ -FROM debian:9.6-slim +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y \ && apt-get install -y \ - libpq-dev \ - python \ - python-pip \ - python-six \ - python-cryptography \ - python-pyasn1 \ - python-crypto \ - python-ldap \ + pkg-config \ + libpq-dev \ + python3-dev \ + python3-pip \ + python3-cryptography \ + python3-pyasn1 \ + python3-crypto \ + libxml2-dev \ + libxmlsec1-dev \ + python3-paramiko \ + openssh-client \ + inotify-tools \ + libsasl2-dev \ + libldap2-dev \ + libssl-dev \ + libxmlsec1-openssl \ curl \ git \ - && pip install \ - pyrsistent==0.15.7 \ - six==1.14.0 \ + && pip3 install \ + six \ + python-ldap \ psycopg2 \ nose \ pyyaml \ @@ -38,10 +48,7 @@ RUN apt-get update -y \ keystoneauth1==3.7.0 \ python-swiftclient \ python-cachetclient \ - croniter \ - && apt-get remove -y python-pip \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* + croniter RUN useradd -ms /bin/bash infrabox USER infrabox diff --git a/src/job/infrabox_job/process.py b/src/job/infrabox_job/process.py index 56c37d27c..e2cd14ba0 100644 --- a/src/job/infrabox_job/process.py +++ b/src/job/infrabox_job/process.py @@ -23,7 +23,7 @@ def collect(self, line, show=False): try: print(line) except UnicodeEncodeError: - print(line.encode('utf-8')) + print((line.encode('utf-8'))) sys.stdout.flush() def execute_mask(self, command, cwd=None, shell=False, show=False, env=None, ignore_error=False, show_cmd=True, retry=False, mask=None): diff --git a/src/job/job.py b/src/job/job.py index 1c092d0d0..13ee70fb7 100755 --- a/src/job/job.py +++ b/src/job/job.py @@ -330,7 +330,7 @@ def main(self): # Show environment self.console.collect("Environment:", show=True) - for name, value in self.env_vars.items(): + for name, value in list(self.env_vars.items()): self.console.collect("%s=%s" % (name, value), show=True) self.console.collect("", show=True) @@ -338,7 +338,7 @@ def main(self): # Show secrets if self.secrets: self.console.collect("Secrets:", show=True) - for name, _ in self.secrets.items(): + for name, _ in list(self.secrets.items()): self.console.collect("%s=*****" % name, show=True) self.console.collect("", show=True) @@ -857,7 +857,7 @@ def run_docker_container(self, image_name): cmd += ['-v', "/local-cache:/infrabox/local-cache"] # add env vars - for name, value in self.environment.items(): + for name, value in list(self.environment.items()): cmd += ['-e', '%s=%s' % (name, value)] # add resource env vars @@ -968,7 +968,7 @@ def build_docker_image(self, image_name, cache_image, target=None): cmd += ['--build-arg', 'INFRABOX_BUILD_NUMBER=%s' % self.build['build_number']] if 'build_arguments' in self.job and self.job['build_arguments']: - for name, value in self.job['build_arguments'].items(): + for name, value in list(self.job['build_arguments'].items()): cmd += ['--build-arg', '%s=%s' % (name, value)] for arg in BUILD_ARGS: @@ -1304,7 +1304,7 @@ def get_job_list(self, data, c, repo, parent_name="", # overwrite env vars if set if 'environment' in job: - for n, v in job['environment'].items(): + for n, v in list(job['environment'].items()): if 'environment' not in s: s['environment'] = {} diff --git a/src/metrics/Dockerfile b/src/metrics/Dockerfile index 94c8a58ce..109c7cc9a 100644 --- a/src/metrics/Dockerfile +++ b/src/metrics/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -CMD python /metrics/server.py +CMD python3 /metrics/server.py diff --git a/src/metrics/server.py b/src/metrics/server.py index b59adc00b..828f40b88 100755 --- a/src/metrics/server.py +++ b/src/metrics/server.py @@ -101,8 +101,8 @@ def _reset_combination_dict(self): Set all the occurrences of the combination dict to True in order to detect which values to set to 0 because they are missing in the request result """ - for project_dict in self._possible_combination.values(): - for project_name in project_dict.keys(): + for project_dict in list(self._possible_combination.values()): + for project_name in list(project_dict.keys()): project_dict[project_name] = True @staticmethod @@ -148,9 +148,9 @@ def _set_values(self, per_cluster, total): self._gauge.labels(rsc="mem", cluster="'%'", project=row[0]).set(row[1]) self._gauge.labels(rsc="cpu", cluster="'%'", project=row[0]).set(row[2]) - for cluster, project_dict in self._possible_combination.items(): + for cluster, project_dict in list(self._possible_combination.items()): to_delete = [] - for project, not_used in project_dict.items(): + for project, not_used in list(project_dict.items()): if not_used: to_delete.append(project) self._gauge.labels(rsc="mem", cluster=cluster, project=project).set(0) @@ -214,8 +214,8 @@ def _set_values(self, per_node): node_dict[row[1]] = False self._gauge.labels(node=row[0], state=row[1]).set(row[2]) - for node, state_dict in self._possible_combination.items(): - for state, to_set in state_dict.items(): + for node, state_dict in list(self._possible_combination.items()): + for state, to_set in list(state_dict.items()): if to_set: self._gauge.labels(node=node, state=state).set(0) diff --git a/src/pyinfrabox/docker_compose/__init__.py b/src/pyinfrabox/docker_compose/__init__.py index 58e29c402..e398d7325 100644 --- a/src/pyinfrabox/docker_compose/__init__.py +++ b/src/pyinfrabox/docker_compose/__init__.py @@ -11,7 +11,7 @@ def handle_version(d, r): def handle_service(name, d, r): r['services'][name] = {} - for key, value in d[name].items(): + for key, value in list(d[name].items()): allowed_fields = [ 'links', 'environment', @@ -38,7 +38,7 @@ def handle_service(name, d, r): def handle_services(d, r): d = d['services'] r['services'] = {} - for key in d.keys(): + for key in list(d.keys()): handle_service(key, d, r) @@ -54,7 +54,7 @@ def parse(d): if "services" not in d: raise Exception("services not found") - for key in d.keys(): + for key in list(d.keys()): if key == "version": handle_version(d, r) elif key == "services": @@ -68,5 +68,5 @@ def parse(d): def create_from(path): with open(path) as f: - d = yaml.load(f.read()) + d = yaml.load(f.read(), yaml.SafeLoader) return parse(d) diff --git a/src/pyinfrabox/infrabox/__init__.py b/src/pyinfrabox/infrabox/__init__.py index ad5ca80b6..8dd4406ed 100644 --- a/src/pyinfrabox/infrabox/__init__.py +++ b/src/pyinfrabox/infrabox/__init__.py @@ -550,12 +550,12 @@ def validate_json(d): if deps: all_deps[job_name] = deps - for job_name, deps in all_deps.items(): + for job_name, deps in list(all_deps.items()): queue = list(deps.keys()) for dep_job in queue: if dep_job == job_name: raise ValidationError("Jobs", "Circular dependency detected.") if dep_job in all_deps: - queue.extend(all_deps[dep_job].keys()) + queue.extend(list(all_deps[dep_job].keys())) return True diff --git a/src/pyinfrabox/tests/test_docker_compose.py b/src/pyinfrabox/tests/test_docker_compose.py index 2810d498b..d3dc28269 100644 --- a/src/pyinfrabox/tests/test_docker_compose.py +++ b/src/pyinfrabox/tests/test_docker_compose.py @@ -12,7 +12,7 @@ def run_exception(self, path, message): assert False except Exception as e: print(e) - self.assertEqual(e.message, message) + self.assertEqual(str(e), message) def test_empty(self): self.run_exception('./test/empty.yml', 'invalid file') diff --git a/src/pyinfrabox/tests/test_json.py b/src/pyinfrabox/tests/test_json.py index 2abef1daf..0c9e75964 100644 --- a/src/pyinfrabox/tests/test_json.py +++ b/src/pyinfrabox/tests/test_json.py @@ -9,7 +9,7 @@ def raises_expect(self, data, expected): validate_json(data) assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(str(e), expected) def test_version(self): self.raises_expect({}, "#: property 'version' is required") diff --git a/src/pyinfrabox/tests/test_markup.py b/src/pyinfrabox/tests/test_markup.py index 52bc993c5..44c0cb44f 100644 --- a/src/pyinfrabox/tests/test_markup.py +++ b/src/pyinfrabox/tests/test_markup.py @@ -13,7 +13,7 @@ def raises_expect(self, data, expected): parse_document(data) assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(str(e), expected) def test_version(self): @@ -65,7 +65,7 @@ def raises_expect_text(data, expected): parse_text(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(str(e), expected) raises_expect_text({}, "#: property 'type' is required") raises_expect_text({"type": "text"}, "#: property 'text' is required") @@ -86,7 +86,7 @@ def raises_expect_list(data, expected): parse_ordered_list(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(str(e), expected) raises_expect_list({}, "#: property 'type' is required") raises_expect_list({"type": "ordered_list"}, "#: property 'elements' is required") @@ -104,7 +104,7 @@ def raises_expect_list(data, expected): parse_unordered_list(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(str(e), expected) raises_expect_list({}, "#: property 'type' is required") raises_expect_list({"type": "unordered_list"}, "#: property 'elements' is required") @@ -122,7 +122,7 @@ def raises_expect_group(data, expected): parse_unordered_list(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(str(e), expected) raises_expect_group({}, "#: property 'type' is required") raises_expect_group({"type": "group"}, "#: property 'elements' is required") @@ -140,7 +140,7 @@ def raises_expect_p(data, expected): parse_unordered_list(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(str(e), expected) raises_expect_p({}, "#: property 'type' is required") raises_expect_p({"type": "paragraph"}, "#: property 'elements' is required") @@ -158,7 +158,7 @@ def raises_expect_grid(data, expected): parse_grid(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(str(e), expected) raises_expect_grid({}, "#: property 'type' is required") raises_expect_grid({"type": "grid"}, "#: property 'rows' is required") @@ -178,7 +178,7 @@ def raises_expect_table(data, expected): parse_table(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(str(e), expected) raises_expect_table({}, "#: property 'type' is required") raises_expect_table({"type": "table"}, "#: property 'rows' is required") diff --git a/src/pyinfrabox/tests/test_testresult.py b/src/pyinfrabox/tests/test_testresult.py index 6e9dd05a8..d751762e0 100644 --- a/src/pyinfrabox/tests/test_testresult.py +++ b/src/pyinfrabox/tests/test_testresult.py @@ -9,7 +9,7 @@ def raises_expect(self, data, expected): validate_result(data) assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(str(e), expected) def test_version(self): self.raises_expect({}, "#: property 'version' is required") diff --git a/src/pyinfrabox/utils.py b/src/pyinfrabox/utils.py index 2aa5f9d4d..dd7465324 100644 --- a/src/pyinfrabox/utils.py +++ b/src/pyinfrabox/utils.py @@ -1,22 +1,13 @@ -import os -import errno import uuid -from builtins import int, range, str -from past.builtins import basestring from pyinfrabox import ValidationError -try: - #python2 - from urlparse import urlparse -except: - #python3 - from urllib.parse import urlparse +from urllib.parse import urlparse def check_text(t, path, allowEmpty=False): - if not isinstance(t, basestring): + if not isinstance(t, str): raise ValidationError(path, "is not a string") if not allowEmpty and not t: diff --git a/src/pyinfraboxutils/coverage.py b/src/pyinfraboxutils/coverage.py index 64b2984fa..cbaf0e6e0 100644 --- a/src/pyinfraboxutils/coverage.py +++ b/src/pyinfraboxutils/coverage.py @@ -72,7 +72,7 @@ def __convert_jacoco_xml(self): f = File(sourcefile.attrib["name"]) for elt in sourcefile: - if 'type' in elt.attrib.keys(): + if 'type' in list(elt.attrib.keys()): if elt.attrib["type"] == "METHOD": f.functions_found = int(elt.attrib["covered"]) + int(elt.attrib["missed"]) diff --git a/src/pyinfraboxutils/ibbottle.py b/src/pyinfraboxutils/ibbottle.py index 09c95561f..2d4ec4a6b 100644 --- a/src/pyinfraboxutils/ibbottle.py +++ b/src/pyinfraboxutils/ibbottle.py @@ -30,7 +30,7 @@ def wrapper(*args, **kwargs): conn = None try: conn = self.pool.getconn() - except HTTPResponse, e: + except HTTPResponse as e: raise HTTPError(500, "Database Error", e) # Add the connection handle as a keyword argument. @@ -39,9 +39,9 @@ def wrapper(*args, **kwargs): try: rv = callback(*args, **kwargs) return rv - except HTTPError, e: + except HTTPError as e: raise - except HTTPResponse, e: + except HTTPResponse as e: raise except psycopg2.OperationalError: print("Operational Error. Retrying.") diff --git a/src/pyinfraboxutils/ibflask.py b/src/pyinfraboxutils/ibflask.py index 4c885aa42..5391128f6 100644 --- a/src/pyinfraboxutils/ibflask.py +++ b/src/pyinfraboxutils/ibflask.py @@ -80,7 +80,7 @@ def get_token(): auth = auth.split(" ")[1] try: - decoded = base64.b64decode(auth) + decoded = base64.b64decode(auth).decode('utf-8') except: logger.warn('could not base64 decode auth header %s', auth) return None diff --git a/src/pyinfraboxutils/token.py b/src/pyinfraboxutils/token.py index 1b6480619..597ecaa94 100644 --- a/src/pyinfraboxutils/token.py +++ b/src/pyinfraboxutils/token.py @@ -14,7 +14,7 @@ def encode_user_token(user_id): 'type': 'user' } - return jwt.encode(data, key=s.read(), algorithm='RS256') + return jwt.encode(data, key=s.read(), algorithm='RS256').decode('utf-8') def encode_project_token(token_id, project_id, name): with open(private_key_path) as s: @@ -27,7 +27,7 @@ def encode_project_token(token_id, project_id, name): 'type': 'project' } - return jwt.encode(data, key=s.read(), algorithm='RS256') + return jwt.encode(data, key=s.read(), algorithm='RS256').decode('utf-8') def encode_job_token(job_id): with open(private_key_path) as s: @@ -38,7 +38,7 @@ def encode_job_token(job_id): 'type': 'job' } - return jwt.encode(data, key=s.read(), algorithm='RS256') + return jwt.encode(data, key=s.read(), algorithm='RS256').decode('utf-8') def decode(encoded): with open(public_key_path) as s: diff --git a/src/scheduler/kubernetes/Dockerfile b/src/scheduler/kubernetes/Dockerfile index 88799ab44..ea0b2d77b 100644 --- a/src/scheduler/kubernetes/Dockerfile +++ b/src/scheduler/kubernetes/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -ENTRYPOINT ["python", "scheduler/scheduler.py"] +ENTRYPOINT ["python3", "scheduler/scheduler.py"] diff --git a/src/services/gardener/Dockerfile b/src/services/gardener/Dockerfile index 4c4a17485..04aac5873 100644 --- a/src/services/gardener/Dockerfile +++ b/src/services/gardener/Dockerfile @@ -17,7 +17,7 @@ ENV PATH /google-cloud-sdk/bin:$PATH RUN apk --no-cache add \ curl \ - python \ + python3 \ py-crcmod \ bash \ libc6-compat \ diff --git a/src/services/gardener/tmp/build/Dockerfile b/src/services/gardener/tmp/build/Dockerfile index 38ccbd317..060bb13ef 100644 --- a/src/services/gardener/tmp/build/Dockerfile +++ b/src/services/gardener/tmp/build/Dockerfile @@ -5,7 +5,7 @@ ENV PATH /google-cloud-sdk/bin:$PATH RUN apk --no-cache add \ curl \ - python \ + python3 \ py-crcmod \ bash \ libc6-compat \ diff --git a/src/services/gcp/Dockerfile b/src/services/gcp/Dockerfile index 9d54b5d89..89e230372 100644 --- a/src/services/gcp/Dockerfile +++ b/src/services/gcp/Dockerfile @@ -17,7 +17,7 @@ ENV PATH /google-cloud-sdk/bin:$PATH RUN apk --no-cache add \ curl \ - python \ + python3 \ py-crcmod \ bash \ libc6-compat \ diff --git a/src/services/gcp/tmp/build/Dockerfile b/src/services/gcp/tmp/build/Dockerfile index ada0a64d4..15b089061 100644 --- a/src/services/gcp/tmp/build/Dockerfile +++ b/src/services/gcp/tmp/build/Dockerfile @@ -5,7 +5,7 @@ ENV PATH /google-cloud-sdk/bin:$PATH RUN apk --no-cache add \ curl \ - python \ + python3 \ py-crcmod \ bash \ libc6-compat \ diff --git a/src/status/cachet/Dockerfile b/src/status/cachet/Dockerfile index 8e51e556d..97e711276 100644 --- a/src/status/cachet/Dockerfile +++ b/src/status/cachet/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -CMD python /cachet.py +CMD python3 /cachet.py