From d69689a47c152a5ec4f225c4a034b27bad04f60c Mon Sep 17 00:00:00 2001 From: I517992 Date: Wed, 11 Nov 2020 17:34:01 +0800 Subject: [PATCH 01/40] enable redirect to origin page after login --- src/dashboard-client/src/App.vue | 5 ++++- src/dashboard-client/src/components/account/Login.vue | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/dashboard-client/src/App.vue b/src/dashboard-client/src/App.vue index 2842a74f2..9617a2ee8 100644 --- a/src/dashboard-client/src/App.vue +++ b/src/dashboard-client/src/App.vue @@ -157,7 +157,10 @@ export default { this.$refs.leftSidenav.toggle() }, login () { - router.push('/login') + router.push({ + path: '/login', + query: {redirect: location.href} + }) }, logout () { UserService.logout() diff --git a/src/dashboard-client/src/components/account/Login.vue b/src/dashboard-client/src/components/account/Login.vue index 83a95f2a6..3b601a096 100644 --- a/src/dashboard-client/src/components/account/Login.vue +++ b/src/dashboard-client/src/components/account/Login.vue @@ -100,7 +100,7 @@ export default { password: this.password }).then(() => { UserService.login() - location.reload() + router.push(router.query.redirect || '/') }).catch((err) => { NotificationService.$emit('NOTIFICATION', new Notification(err)) }) From 267b402e6bc367be6c2320eed3d4cdc506aa4fba Mon Sep 17 00:00:00 2001 From: I517992 Date: Wed, 11 Nov 2020 18:09:51 +0800 Subject: [PATCH 02/40] bug fixed --- src/dashboard-client/src/components/account/Login.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dashboard-client/src/components/account/Login.vue b/src/dashboard-client/src/components/account/Login.vue index 3b601a096..cafc2ddef 100644 --- a/src/dashboard-client/src/components/account/Login.vue +++ b/src/dashboard-client/src/components/account/Login.vue @@ -100,7 +100,7 @@ export default { password: this.password }).then(() => { UserService.login() - router.push(router.query.redirect || '/') + location.href = this.$route.query.redirect }).catch((err) => { NotificationService.$emit('NOTIFICATION', new Notification(err)) }) From 8038339b1c3dc0190acffd2cd18f5b5d686cfb1b Mon Sep 17 00:00:00 2001 From: I517992 Date: Wed, 11 Nov 2020 18:33:13 +0800 Subject: [PATCH 03/40] bug fixed --- src/dashboard-client/src/components/account/Login.vue | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/dashboard-client/src/components/account/Login.vue b/src/dashboard-client/src/components/account/Login.vue index cafc2ddef..0bfb48b08 100644 --- a/src/dashboard-client/src/components/account/Login.vue +++ b/src/dashboard-client/src/components/account/Login.vue @@ -100,7 +100,11 @@ export default { password: this.password }).then(() => { UserService.login() - location.href = this.$route.query.redirect + if (this.$route.query.redirect !== undefined) { + location.href = this.$route.query.redirect + } else { + router.reload() + } }).catch((err) => { NotificationService.$emit('NOTIFICATION', new Notification(err)) }) From 72da589c5cafee21460a9275b30cc8dff7d42d7e Mon Sep 17 00:00:00 2001 From: I517992 Date: Wed, 11 Nov 2020 18:43:07 +0800 Subject: [PATCH 04/40] bug fixed --- src/dashboard-client/src/components/account/Login.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dashboard-client/src/components/account/Login.vue b/src/dashboard-client/src/components/account/Login.vue index 0bfb48b08..8acf30efb 100644 --- a/src/dashboard-client/src/components/account/Login.vue +++ b/src/dashboard-client/src/components/account/Login.vue @@ -103,7 +103,7 @@ export default { if (this.$route.query.redirect !== undefined) { location.href = this.$route.query.redirect } else { - router.reload() + location.reload() } }).catch((err) => { NotificationService.$emit('NOTIFICATION', new Notification(err)) From 3beedf9c3d823e337294fa3515c0b547ef523a7d Mon Sep 17 00:00:00 2001 From: I517992 Date: Mon, 16 Nov 2020 13:34:51 +0800 Subject: [PATCH 05/40] python2 to python3 --- ib.py | 2 +- infrabox/test/api/job_api_test.py | 6 +++--- infrabox/test/e2e/test.py | 6 +++--- .../tests/docker_compose_job/tests/tests.py | 2 +- src/api/handlers/account/saml.py | 4 ++-- src/api/handlers/admin/projects.py | 2 +- src/api/handlers/job_api.py | 20 +++++++++---------- src/api/handlers/project.py | 10 +++++----- src/docker-registry/nginx/get_admin_pw.py | 2 +- src/gerrit/review/review.py | 6 +++--- src/github/review/review.py | 6 +++--- src/job/infrabox_job/process.py | 2 +- src/job/job.py | 10 +++++----- src/metrics/server.py | 12 +++++------ src/pyinfrabox/docker_compose/__init__.py | 6 +++--- src/pyinfrabox/infrabox/__init__.py | 4 ++-- src/pyinfrabox/utils.py | 6 +++--- src/pyinfraboxutils/coverage.py | 2 +- src/pyinfraboxutils/ibbottle.py | 6 +++--- 19 files changed, 57 insertions(+), 57 deletions(-) diff --git a/ib.py b/ib.py index bd706691a..4c5ae5871 100755 --- a/ib.py +++ b/ib.py @@ -65,7 +65,7 @@ def execute(command, cwd=None, env=None, ignore_error=False, ignore_output=False if ignore_output: continue - print(line.rstrip()) + print((line.rstrip())) process.wait() diff --git a/infrabox/test/api/job_api_test.py b/infrabox/test/api/job_api_test.py index 0df65e325..88d343023 100644 --- a/infrabox/test/api/job_api_test.py +++ b/infrabox/test/api/job_api_test.py @@ -237,6 +237,6 @@ def test_testresult(self): keys = ['status', 'duration', 'message', 'stack'] for i, received_row in enumerate(r): # create dictionary from the list to compare it easier - row_dictionary = dict(zip(keys, received_row)) - self.assertTrue(all(item in testresult_data["tests"][i].items() - for item in row_dictionary.items())) + row_dictionary = dict(list(zip(keys, received_row))) + self.assertTrue(all(item in list(testresult_data["tests"][i].items()) + for item in list(row_dictionary.items()))) diff --git a/infrabox/test/e2e/test.py b/infrabox/test/e2e/test.py index 2f45d86b3..bcf06e8fa 100644 --- a/infrabox/test/e2e/test.py +++ b/infrabox/test/e2e/test.py @@ -73,7 +73,7 @@ def _get_build(self): return result.json()[0] except: print("Get build failed: ") - print(result.text) + print((result.text)) raise def _get_jobs(self): @@ -322,7 +322,7 @@ def main(): print("Connecting to DB") connect_db() # Wait for DB - print("ROOT_URL: %s" % root_url) + print(("ROOT_URL: %s" % root_url)) while True: time.sleep(1) r = None @@ -332,7 +332,7 @@ def main(): if r.status_code in (200, 404): break - print(r.text) + print((r.text)) except Exception as e: print(e) diff --git a/infrabox/test/e2e/tests/docker_compose_job/tests/tests.py b/infrabox/test/e2e/tests/docker_compose_job/tests/tests.py index 77b2c7cb0..e7929b831 100644 --- a/infrabox/test/e2e/tests/docker_compose_job/tests/tests.py +++ b/infrabox/test/e2e/tests/docker_compose_job/tests/tests.py @@ -3,5 +3,5 @@ def test_get(): print("get") r = requests.get("http://test-server:3000") - print(r.json()) + print((r.json())) assert True diff --git a/src/api/handlers/account/saml.py b/src/api/handlers/account/saml.py index 6043ca421..173af5f94 100644 --- a/src/api/handlers/account/saml.py +++ b/src/api/handlers/account/saml.py @@ -1,4 +1,4 @@ -from urlparse import urlparse +from urllib.parse import urlparse from flask import g, request, abort, redirect, make_response @@ -36,7 +36,7 @@ def init_saml_auth(): def get_attribute_dict(saml_auth): attributes = {} nested_attribute_dict = saml_auth.get_attributes() - for attribute_name, nested_attribute in nested_attribute_dict.items(): + for attribute_name, nested_attribute in list(nested_attribute_dict.items()): if len(nested_attribute) > 0: attributes[attribute_name] = nested_attribute[0] attributes["NameID"] = saml_auth.get_nameid() diff --git a/src/api/handlers/admin/projects.py b/src/api/handlers/admin/projects.py index 92e71abf1..516f5f9da 100644 --- a/src/api/handlers/admin/projects.py +++ b/src/api/handlers/admin/projects.py @@ -52,7 +52,7 @@ def get(self): result[j['name']]['jobs'] = j['cnt'] response = [] - for _, v in result.iteritems(): + for _, v in result.items(): response.append(v) return response diff --git a/src/api/handlers/job_api.py b/src/api/handlers/job_api.py index e3550a92f..f5204d0f5 100644 --- a/src/api/handlers/job_api.py +++ b/src/api/handlers/job_api.py @@ -3,7 +3,7 @@ import json import uuid import copy -import urllib +import urllib.request, urllib.parse, urllib.error import random from datetime import datetime from io import BytesIO @@ -344,8 +344,8 @@ def get_secret(name): root_url = get_root_url("global") # Default env vars - project_name = urllib.quote_plus(data['project']['name']).replace('+', '%20') - job_name = urllib.quote_plus(data['job']['name']).replace('+', '%20') + project_name = urllib.parse.quote_plus(data['project']['name']).replace('+', '%20') + job_name = urllib.parse.quote_plus(data['job']['name']).replace('+', '%20') build_url = "%s/dashboard/#/project/%s/build/%s/%s" % (root_url, project_name, data['build']['build_number'], @@ -390,14 +390,14 @@ def get_secret(name): data['env_vars']['INFRABOX_GITHUB_PULL_REQUEST'] = "true" if env_vars: - for name, value in env_vars.iteritems(): + for name, value in env_vars.items(): try: data['env_vars'][name] = str(value) except UnicodeEncodeError: data['env_vars'][name] = value.encode('utf-8') if env_var_refs: - for name, value in env_var_refs.iteritems(): + for name, value in env_var_refs.items(): secret = get_secret(value) if secret is None: @@ -467,7 +467,7 @@ def post(self): project_id = g.token['project']['id'] job_name = g.token['job']['name'] - for f, _ in request.files.items(): + for f, _ in list(request.files.items()): template = 'project_%s_job_%s_%s' key = template % (project_id, job_name, f) key = key.replace('/', '_') @@ -520,7 +520,7 @@ class Output(Resource): def post(self): job_id = g.token['job']['id'] - for f, _ in request.files.items(): + for f, _ in list(request.files.items()): key = "%s/%s" % (job_id, f) stream = request.files[f].stream @@ -942,7 +942,7 @@ def post(self): stats = request.json['stats'] s = 0 c = 0 - for _, values in stats.items(): + for _, values in list(stats.items()): for v in values: c += 1 s += v['cpu'] @@ -1000,7 +1000,7 @@ def _remove_file(response): delete_file(path) return response - for name, f in request.files.iteritems(): + for name, f in request.files.items(): try: if not allowed_file(f.filename, ("json",)): abort(400, "Filetype not allowed") @@ -1049,7 +1049,7 @@ def _remove_file(response): delete_file(path) return response - for _, f in request.files.iteritems(): + for _, f in request.files.items(): if not allowed_file(f.filename, ("json")): abort(400, "Filetype not allowed") diff --git a/src/api/handlers/project.py b/src/api/handlers/project.py index 756e55220..ce0e5fa35 100644 --- a/src/api/handlers/project.py +++ b/src/api/handlers/project.py @@ -1,5 +1,5 @@ import uuid -import urllib +import urllib.request, urllib.parse, urllib.error import os import json import mimetypes @@ -49,16 +49,16 @@ def no_cache(*args, **kwargs): return update_wrapper(no_cache, view) def get_badge(subject, status, color): - subject = urllib.quote(subject) - status = urllib.quote(status) - color = urllib.quote(color) + subject = urllib.parse.quote(subject) + status = urllib.parse.quote(status) + color = urllib.parse.quote(color) url = 'https://img.shields.io/static/v1.svg?label=%s&message=%s&color=%s' % (subject, status, color) resp = requests.get(url) excluded_headers = ['content-encoding', 'content-length', 'transfer-encoding', 'connection'] - headers = [(name, value) for (name, value) in resp.raw.headers.items() + headers = [(name, value) for (name, value) in list(resp.raw.headers.items()) if name.lower() not in excluded_headers] return Response(resp.content, resp.status_code, headers) diff --git a/src/docker-registry/nginx/get_admin_pw.py b/src/docker-registry/nginx/get_admin_pw.py index bdfa5551f..0853a0a1b 100644 --- a/src/docker-registry/nginx/get_admin_pw.py +++ b/src/docker-registry/nginx/get_admin_pw.py @@ -21,5 +21,5 @@ with htpasswd.Basic(os.environ['HTPASSWD_PATH'], mode="md5") as userdb: try: userdb.add(admin_user, admin_password) - except htpasswd.basic.UserExists, e: + except htpasswd.basic.UserExists as e: print(e) diff --git a/src/gerrit/review/review.py b/src/gerrit/review/review.py index c701fdef8..258fdb161 100755 --- a/src/gerrit/review/review.py +++ b/src/gerrit/review/review.py @@ -1,6 +1,6 @@ import json import select -import urllib +import urllib.request, urllib.parse, urllib.error import time import os @@ -60,7 +60,7 @@ def main(): time.sleep(3) def handle_job_update_retry(conn, update): - for _ in xrange(0, 3): + for _ in range(0, 3): try: handle_job_update(conn, update) return @@ -163,7 +163,7 @@ def handle_job_update(conn, event): key_filename=gerrit_key_filename) client.get_transport().set_keepalive(60) - project_name_quote = urllib.quote_plus(project_name).replace('+', '%20') + project_name_quote = urllib.parse.quote_plus(project_name).replace('+', '%20') build_url = "%s/dashboard/#/project/%s/build/%s/%s" % (dashboard_url, project_name_quote, build_number, diff --git a/src/github/review/review.py b/src/github/review/review.py index 0116f26c2..7fbac6f5c 100755 --- a/src/github/review/review.py +++ b/src/github/review/review.py @@ -2,7 +2,7 @@ import select import os -import urllib +import urllib.request, urllib.parse, urllib.error import requests import psycopg2 @@ -150,10 +150,10 @@ def handle_job_update(conn, event): ''', [])[0]['root_url'] target_url = '%s/dashboard/#/project/%s/build/%s/%s/job/%s' % (dashboard_url, - urllib.quote(project_name, safe=''), + urllib.parse.quote(project_name, safe=''), build_number, build_restartCounter, - urllib.quote_plus(job_name).replace('+', '%20')) + urllib.parse.quote_plus(job_name).replace('+', '%20')) job_name = job_name.split(".")[0] payload = { diff --git a/src/job/infrabox_job/process.py b/src/job/infrabox_job/process.py index 56c37d27c..e2cd14ba0 100644 --- a/src/job/infrabox_job/process.py +++ b/src/job/infrabox_job/process.py @@ -23,7 +23,7 @@ def collect(self, line, show=False): try: print(line) except UnicodeEncodeError: - print(line.encode('utf-8')) + print((line.encode('utf-8'))) sys.stdout.flush() def execute_mask(self, command, cwd=None, shell=False, show=False, env=None, ignore_error=False, show_cmd=True, retry=False, mask=None): diff --git a/src/job/job.py b/src/job/job.py index 48b00134a..aaa2b336d 100755 --- a/src/job/job.py +++ b/src/job/job.py @@ -329,7 +329,7 @@ def main(self): # Show environment self.console.collect("Environment:", show=True) - for name, value in self.env_vars.items(): + for name, value in list(self.env_vars.items()): self.console.collect("%s=%s" % (name, value), show=True) self.console.collect("", show=True) @@ -337,7 +337,7 @@ def main(self): # Show secrets if self.secrets: self.console.collect("Secrets:", show=True) - for name, _ in self.secrets.items(): + for name, _ in list(self.secrets.items()): self.console.collect("%s=*****" % name, show=True) self.console.collect("", show=True) @@ -843,7 +843,7 @@ def run_docker_container(self, image_name): cmd += ['-v', "/local-cache:/infrabox/local-cache"] # add env vars - for name, value in self.environment.items(): + for name, value in list(self.environment.items()): cmd += ['-e', '%s=%s' % (name, value)] # add resource env vars @@ -954,7 +954,7 @@ def build_docker_image(self, image_name, cache_image, target=None): cmd += ['--build-arg', 'INFRABOX_BUILD_NUMBER=%s' % self.build['build_number']] if 'build_arguments' in self.job and self.job['build_arguments']: - for name, value in self.job['build_arguments'].items(): + for name, value in list(self.job['build_arguments'].items()): cmd += ['--build-arg', '%s=%s' % (name, value)] for arg in BUILD_ARGS: @@ -1290,7 +1290,7 @@ def get_job_list(self, data, c, repo, parent_name="", # overwrite env vars if set if 'environment' in job: - for n, v in job['environment'].items(): + for n, v in list(job['environment'].items()): if 'environment' not in s: s['environment'] = {} diff --git a/src/metrics/server.py b/src/metrics/server.py index b59adc00b..828f40b88 100755 --- a/src/metrics/server.py +++ b/src/metrics/server.py @@ -101,8 +101,8 @@ def _reset_combination_dict(self): Set all the occurrences of the combination dict to True in order to detect which values to set to 0 because they are missing in the request result """ - for project_dict in self._possible_combination.values(): - for project_name in project_dict.keys(): + for project_dict in list(self._possible_combination.values()): + for project_name in list(project_dict.keys()): project_dict[project_name] = True @staticmethod @@ -148,9 +148,9 @@ def _set_values(self, per_cluster, total): self._gauge.labels(rsc="mem", cluster="'%'", project=row[0]).set(row[1]) self._gauge.labels(rsc="cpu", cluster="'%'", project=row[0]).set(row[2]) - for cluster, project_dict in self._possible_combination.items(): + for cluster, project_dict in list(self._possible_combination.items()): to_delete = [] - for project, not_used in project_dict.items(): + for project, not_used in list(project_dict.items()): if not_used: to_delete.append(project) self._gauge.labels(rsc="mem", cluster=cluster, project=project).set(0) @@ -214,8 +214,8 @@ def _set_values(self, per_node): node_dict[row[1]] = False self._gauge.labels(node=row[0], state=row[1]).set(row[2]) - for node, state_dict in self._possible_combination.items(): - for state, to_set in state_dict.items(): + for node, state_dict in list(self._possible_combination.items()): + for state, to_set in list(state_dict.items()): if to_set: self._gauge.labels(node=node, state=state).set(0) diff --git a/src/pyinfrabox/docker_compose/__init__.py b/src/pyinfrabox/docker_compose/__init__.py index 58e29c402..ebc766d14 100644 --- a/src/pyinfrabox/docker_compose/__init__.py +++ b/src/pyinfrabox/docker_compose/__init__.py @@ -11,7 +11,7 @@ def handle_version(d, r): def handle_service(name, d, r): r['services'][name] = {} - for key, value in d[name].items(): + for key, value in list(d[name].items()): allowed_fields = [ 'links', 'environment', @@ -38,7 +38,7 @@ def handle_service(name, d, r): def handle_services(d, r): d = d['services'] r['services'] = {} - for key in d.keys(): + for key in list(d.keys()): handle_service(key, d, r) @@ -54,7 +54,7 @@ def parse(d): if "services" not in d: raise Exception("services not found") - for key in d.keys(): + for key in list(d.keys()): if key == "version": handle_version(d, r) elif key == "services": diff --git a/src/pyinfrabox/infrabox/__init__.py b/src/pyinfrabox/infrabox/__init__.py index ad5ca80b6..8dd4406ed 100644 --- a/src/pyinfrabox/infrabox/__init__.py +++ b/src/pyinfrabox/infrabox/__init__.py @@ -550,12 +550,12 @@ def validate_json(d): if deps: all_deps[job_name] = deps - for job_name, deps in all_deps.items(): + for job_name, deps in list(all_deps.items()): queue = list(deps.keys()) for dep_job in queue: if dep_job == job_name: raise ValidationError("Jobs", "Circular dependency detected.") if dep_job in all_deps: - queue.extend(all_deps[dep_job].keys()) + queue.extend(list(all_deps[dep_job].keys())) return True diff --git a/src/pyinfrabox/utils.py b/src/pyinfrabox/utils.py index 2aa5f9d4d..e6a8395e1 100644 --- a/src/pyinfrabox/utils.py +++ b/src/pyinfrabox/utils.py @@ -3,20 +3,20 @@ import uuid from builtins import int, range, str -from past.builtins import basestring +from past.builtins import str from pyinfrabox import ValidationError try: #python2 - from urlparse import urlparse + from urllib.parse import urlparse except: #python3 from urllib.parse import urlparse def check_text(t, path, allowEmpty=False): - if not isinstance(t, basestring): + if not isinstance(t, str): raise ValidationError(path, "is not a string") if not allowEmpty and not t: diff --git a/src/pyinfraboxutils/coverage.py b/src/pyinfraboxutils/coverage.py index 64b2984fa..cbaf0e6e0 100644 --- a/src/pyinfraboxutils/coverage.py +++ b/src/pyinfraboxutils/coverage.py @@ -72,7 +72,7 @@ def __convert_jacoco_xml(self): f = File(sourcefile.attrib["name"]) for elt in sourcefile: - if 'type' in elt.attrib.keys(): + if 'type' in list(elt.attrib.keys()): if elt.attrib["type"] == "METHOD": f.functions_found = int(elt.attrib["covered"]) + int(elt.attrib["missed"]) diff --git a/src/pyinfraboxutils/ibbottle.py b/src/pyinfraboxutils/ibbottle.py index 09c95561f..2d4ec4a6b 100644 --- a/src/pyinfraboxutils/ibbottle.py +++ b/src/pyinfraboxutils/ibbottle.py @@ -30,7 +30,7 @@ def wrapper(*args, **kwargs): conn = None try: conn = self.pool.getconn() - except HTTPResponse, e: + except HTTPResponse as e: raise HTTPError(500, "Database Error", e) # Add the connection handle as a keyword argument. @@ -39,9 +39,9 @@ def wrapper(*args, **kwargs): try: rv = callback(*args, **kwargs) return rv - except HTTPError, e: + except HTTPError as e: raise - except HTTPResponse, e: + except HTTPResponse as e: raise except psycopg2.OperationalError: print("Operational Error. Retrying.") From 65993266034edeb11f59774286cdbf04f930e899 Mon Sep 17 00:00:00 2001 From: I517992 Date: Mon, 16 Nov 2020 15:43:16 +0800 Subject: [PATCH 06/40] python2 to python3 dockerfile change --- src/images/base/Dockerfile | 21 +++++++++++---------- src/images/test/Dockerfile | 16 ++++++++-------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/images/base/Dockerfile b/src/images/base/Dockerfile index 2d8a4abac..ee781bfbb 100644 --- a/src/images/base/Dockerfile +++ b/src/images/base/Dockerfile @@ -3,19 +3,19 @@ FROM debian:9.6-slim RUN apt-get update -y \ && apt-get install -y \ libpq-dev \ - python \ - python-pip \ - python-six \ - python-cryptography \ - python-crypto \ - python-pyasn1 \ + python3 \ + python3-pip \ + python3-six \ + python3-cryptography \ + python3-crypto \ + python3-pyasn1 \ python-ldap \ libxml2-dev \ libxmlsec1-dev \ - python-paramiko \ + python3-paramiko \ openssh-client \ inotify-tools \ - && pip install \ + && pip3 install \ psycopg2 \ flask \ flask-restx \ @@ -35,9 +35,10 @@ RUN apt-get update -y \ keystoneauth1==3.7.0 \ python-swiftclient \ python-cachetclient \ - python-saml \ + xmlsec \ + python3-saml \ croniter \ - && apt-get remove -y python-pip \ + && apt-get remove -y python3-pip \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* diff --git a/src/images/test/Dockerfile b/src/images/test/Dockerfile index 175994667..e46a00598 100644 --- a/src/images/test/Dockerfile +++ b/src/images/test/Dockerfile @@ -3,16 +3,16 @@ FROM debian:9.6-slim RUN apt-get update -y \ && apt-get install -y \ libpq-dev \ - python \ - python-pip \ - python-six \ - python-cryptography \ - python-pyasn1 \ - python-crypto \ + python3 \ + python3-pip \ + python3-six \ + python3-cryptography \ + python3-pyasn1 \ + python3-crypto \ python-ldap \ curl \ git \ - && pip install \ + && pip3 install \ psycopg2 \ nose \ pyyaml \ @@ -37,7 +37,7 @@ RUN apt-get update -y \ python-swiftclient \ python-cachetclient \ croniter \ - && apt-get remove -y python-pip \ + && apt-get remove -y python3-pip \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* From cdf9b8a4a662e38facebd388eac2ee6051b6a961 Mon Sep 17 00:00:00 2001 From: I517992 Date: Mon, 16 Nov 2020 16:22:58 +0800 Subject: [PATCH 07/40] add decode and encode in registry-auth --- infrabox/test/registry-auth/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrabox/test/registry-auth/test.py b/infrabox/test/registry-auth/test.py index 9b4b80aa9..e2b7b8e98 100644 --- a/infrabox/test/registry-auth/test.py +++ b/infrabox/test/registry-auth/test.py @@ -120,7 +120,7 @@ def get_project_headers(self, project_token=None): # pragma: no cover if not project_token: project_token = self.project_token - token = encode_project_token(project_token, self.project_id, 'myproject') + token = encode_project_token(project_token, self.project_id, 'myproject').encode('utf-8') h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox:%s' % token)} return h @@ -131,7 +131,7 @@ def get(self, url, headers=None, method='GET'): # pragma: no cover r = self.app.get(url, headers=headers) if r.mimetype == 'application/json': - return json.loads(r.data) + return json.loads(r.data.decode('utf-8')) return r From c29fbcae1123057cb4561fc08f1cc6ef59cb2d3f Mon Sep 17 00:00:00 2001 From: I517992 Date: Mon, 16 Nov 2020 16:34:14 +0800 Subject: [PATCH 08/40] bug fixed --- infrabox/test/registry-auth/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infrabox/test/registry-auth/test.py b/infrabox/test/registry-auth/test.py index e2b7b8e98..b7953916d 100644 --- a/infrabox/test/registry-auth/test.py +++ b/infrabox/test/registry-auth/test.py @@ -58,12 +58,12 @@ def test_header_invalid_token(self): self.assertEqual(r['status'], 401) def test_header_no_password(self): - h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox')} + h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox'.encode('utf-8'))} r = self.get('/v2', h) self.assertEqual(r['status'], 401) def test_header_no_password_2(self): - h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox:2')} + h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox:2'.encode('utf-8'))} r = self.get('/v2', h) self.assertEqual(r['status'], 401) @@ -120,7 +120,7 @@ def get_project_headers(self, project_token=None): # pragma: no cover if not project_token: project_token = self.project_token - token = encode_project_token(project_token, self.project_id, 'myproject').encode('utf-8') + token = encode_project_token(project_token, self.project_id, 'myproject').decode('utf-8') h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox:%s' % token)} return h From 47861147c5379f2324b2c3e54cc2e847fa5bd244 Mon Sep 17 00:00:00 2001 From: I517992 Date: Mon, 16 Nov 2020 17:08:37 +0800 Subject: [PATCH 09/40] bug fixed --- infrabox/test/registry-auth/test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/infrabox/test/registry-auth/test.py b/infrabox/test/registry-auth/test.py index b7953916d..bb82fbdbb 100644 --- a/infrabox/test/registry-auth/test.py +++ b/infrabox/test/registry-auth/test.py @@ -3,7 +3,6 @@ import unittest import json -import base64 from auth import server import xmlrunner import psycopg2 @@ -58,12 +57,12 @@ def test_header_invalid_token(self): self.assertEqual(r['status'], 401) def test_header_no_password(self): - h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox'.encode('utf-8'))} + h = {'Authorization': 'Basic %s' % 'infrabox'} r = self.get('/v2', h) self.assertEqual(r['status'], 401) def test_header_no_password_2(self): - h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox:2'.encode('utf-8'))} + h = {'Authorization': 'Basic %s' % 'infrabox:2'} r = self.get('/v2', h) self.assertEqual(r['status'], 401) @@ -121,7 +120,7 @@ def get_project_headers(self, project_token=None): # pragma: no cover project_token = self.project_token token = encode_project_token(project_token, self.project_id, 'myproject').decode('utf-8') - h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox:%s' % token)} + h = {'Authorization': 'Basic %s' % ('infrabox:%s' % token)} return h def get(self, url, headers=None, method='GET'): # pragma: no cover From 03cd52888fdf012b328ec0b761724eb047248053 Mon Sep 17 00:00:00 2001 From: I517992 Date: Mon, 16 Nov 2020 17:20:03 +0800 Subject: [PATCH 10/40] bug fixed --- infrabox/test/registry-auth/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrabox/test/registry-auth/test.py b/infrabox/test/registry-auth/test.py index bb82fbdbb..73c41578e 100644 --- a/infrabox/test/registry-auth/test.py +++ b/infrabox/test/registry-auth/test.py @@ -57,7 +57,7 @@ def test_header_invalid_token(self): self.assertEqual(r['status'], 401) def test_header_no_password(self): - h = {'Authorization': 'Basic %s' % 'infrabox'} + h = {'Authorization': 'Basic %s' % 'infrabox'.encode('utf-8')} r = self.get('/v2', h) self.assertEqual(r['status'], 401) From 79d2a421652b33421616ccb4b54dfc5a81c57b5b Mon Sep 17 00:00:00 2001 From: I517992 Date: Mon, 16 Nov 2020 17:27:20 +0800 Subject: [PATCH 11/40] trigger job --- infrabox/test/registry-auth/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/infrabox/test/registry-auth/test.py b/infrabox/test/registry-auth/test.py index 73c41578e..9b3ee1194 100644 --- a/infrabox/test/registry-auth/test.py +++ b/infrabox/test/registry-auth/test.py @@ -134,6 +134,7 @@ def get(self, url, headers=None, method='GET'): # pragma: no cover return r + if __name__ == '__main__': with open('results.xml', 'wb') as output: unittest.main(testRunner=xmlrunner.XMLTestRunner(output=output)) From b50a601028e8d13f477901fdf6fa6249f3663e07 Mon Sep 17 00:00:00 2001 From: I517992 Date: Mon, 16 Nov 2020 17:57:12 +0800 Subject: [PATCH 12/40] try --- infrabox/test/registry-auth/test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/infrabox/test/registry-auth/test.py b/infrabox/test/registry-auth/test.py index 9b3ee1194..56107c8b4 100644 --- a/infrabox/test/registry-auth/test.py +++ b/infrabox/test/registry-auth/test.py @@ -3,6 +3,7 @@ import unittest import json +import base64 from auth import server import xmlrunner import psycopg2 @@ -57,12 +58,12 @@ def test_header_invalid_token(self): self.assertEqual(r['status'], 401) def test_header_no_password(self): - h = {'Authorization': 'Basic %s' % 'infrabox'.encode('utf-8')} + h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox'.encode("utf-8"))} r = self.get('/v2', h) self.assertEqual(r['status'], 401) def test_header_no_password_2(self): - h = {'Authorization': 'Basic %s' % 'infrabox:2'} + h = {'Authorization': 'Basic %s' % base64.b64encode('infrabox:2'.encode("utf-8"))} r = self.get('/v2', h) self.assertEqual(r['status'], 401) @@ -119,8 +120,8 @@ def get_project_headers(self, project_token=None): # pragma: no cover if not project_token: project_token = self.project_token - token = encode_project_token(project_token, self.project_id, 'myproject').decode('utf-8') - h = {'Authorization': 'Basic %s' % ('infrabox:%s' % token)} + token = encode_project_token(project_token, self.project_id, 'myproject') + h = {'Authorization': 'Basic %s' % base64.b64encode(('infrabox:%s' % token).encode("utf-8"))} return h def get(self, url, headers=None, method='GET'): # pragma: no cover @@ -134,7 +135,6 @@ def get(self, url, headers=None, method='GET'): # pragma: no cover return r - if __name__ == '__main__': with open('results.xml', 'wb') as output: unittest.main(testRunner=xmlrunner.XMLTestRunner(output=output)) From 10ee16eab613ddffaa13fb2ebb36371539cc52d9 Mon Sep 17 00:00:00 2001 From: I517992 Date: Mon, 16 Nov 2020 18:17:51 +0800 Subject: [PATCH 13/40] add decode --- infrabox/generator/Dockerfile | 2 +- src/pyinfraboxutils/ibflask.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infrabox/generator/Dockerfile b/infrabox/generator/Dockerfile index 070185447..50d90566f 100644 --- a/infrabox/generator/Dockerfile +++ b/infrabox/generator/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:3.6 -RUN apk add --no-cache python2 +RUN apk add --no-cache python3 COPY infrabox/generator /generator diff --git a/src/pyinfraboxutils/ibflask.py b/src/pyinfraboxutils/ibflask.py index 4c885aa42..5391128f6 100644 --- a/src/pyinfraboxutils/ibflask.py +++ b/src/pyinfraboxutils/ibflask.py @@ -80,7 +80,7 @@ def get_token(): auth = auth.split(" ")[1] try: - decoded = base64.b64decode(auth) + decoded = base64.b64decode(auth).decode('utf-8') except: logger.warn('could not base64 decode auth header %s', auth) return None From 8d2f4ee71ed31c9174c819b46b7ba720df6bd706 Mon Sep 17 00:00:00 2001 From: I517992 Date: Mon, 16 Nov 2020 18:22:29 +0800 Subject: [PATCH 14/40] add python3 --- infrabox/generator/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrabox/generator/Dockerfile b/infrabox/generator/Dockerfile index 50d90566f..4e56689ea 100644 --- a/infrabox/generator/Dockerfile +++ b/infrabox/generator/Dockerfile @@ -4,4 +4,4 @@ RUN apk add --no-cache python3 COPY infrabox/generator /generator -CMD python /generator/generator.py +CMD python3 /generator/generator.py From 9b6d8d5afe66e1a9051ab42646958168a1bba07e Mon Sep 17 00:00:00 2001 From: I517992 Date: Wed, 18 Nov 2020 15:18:42 +0800 Subject: [PATCH 15/40] fixed issue --- infrabox/test/collector-api/temp_tools.py | 2 +- infrabox/test/registry-auth/test.py | 2 +- src/images/base/Dockerfile | 19 +++++++++++++------ src/pyinfraboxutils/token.py | 6 +++--- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/infrabox/test/collector-api/temp_tools.py b/infrabox/test/collector-api/temp_tools.py index 8c86ef8ee..845298ac5 100644 --- a/infrabox/test/collector-api/temp_tools.py +++ b/infrabox/test/collector-api/temp_tools.py @@ -37,7 +37,7 @@ def get(url, headers): # pragma: no cover r = TestClient.app.get(url, headers=headers) if r.mimetype == 'application/json': - j = json.loads(r.data) + j = json.loads(r.data.decode('utf-8')) return j return r diff --git a/infrabox/test/registry-auth/test.py b/infrabox/test/registry-auth/test.py index 56107c8b4..d46b3f12d 100644 --- a/infrabox/test/registry-auth/test.py +++ b/infrabox/test/registry-auth/test.py @@ -121,7 +121,7 @@ def get_project_headers(self, project_token=None): # pragma: no cover project_token = self.project_token token = encode_project_token(project_token, self.project_id, 'myproject') - h = {'Authorization': 'Basic %s' % base64.b64encode(('infrabox:%s' % token).encode("utf-8"))} + h = {'Authorization': 'Basic %s' % base64.b64encode(('infrabox:%s' % token).encode("utf-8")).decode('utf-8')} return h def get(self, url, headers=None, method='GET'): # pragma: no cover diff --git a/src/images/base/Dockerfile b/src/images/base/Dockerfile index ee781bfbb..0ac53b6c1 100644 --- a/src/images/base/Dockerfile +++ b/src/images/base/Dockerfile @@ -1,21 +1,28 @@ -FROM debian:9.6-slim +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y \ && apt-get install -y \ - libpq-dev \ - python3 \ - python3-pip \ - python3-six \ + pkg-config \ + libpq-dev \ + python3-dev \ + python3-pip \ python3-cryptography \ python3-crypto \ python3-pyasn1 \ - python-ldap \ libxml2-dev \ libxmlsec1-dev \ python3-paramiko \ openssh-client \ inotify-tools \ + libsasl2-dev \ + libldap2-dev \ + libssl-dev \ + libxmlsec1-openssl \ && pip3 install \ + six \ + python-ldap \ psycopg2 \ flask \ flask-restx \ diff --git a/src/pyinfraboxutils/token.py b/src/pyinfraboxutils/token.py index 1b6480619..597ecaa94 100644 --- a/src/pyinfraboxutils/token.py +++ b/src/pyinfraboxutils/token.py @@ -14,7 +14,7 @@ def encode_user_token(user_id): 'type': 'user' } - return jwt.encode(data, key=s.read(), algorithm='RS256') + return jwt.encode(data, key=s.read(), algorithm='RS256').decode('utf-8') def encode_project_token(token_id, project_id, name): with open(private_key_path) as s: @@ -27,7 +27,7 @@ def encode_project_token(token_id, project_id, name): 'type': 'project' } - return jwt.encode(data, key=s.read(), algorithm='RS256') + return jwt.encode(data, key=s.read(), algorithm='RS256').decode('utf-8') def encode_job_token(job_id): with open(private_key_path) as s: @@ -38,7 +38,7 @@ def encode_job_token(job_id): 'type': 'job' } - return jwt.encode(data, key=s.read(), algorithm='RS256') + return jwt.encode(data, key=s.read(), algorithm='RS256').decode('utf-8') def decode(encoded): with open(public_key_path) as s: From 6d2ead0a59e1828c9f3c0654c4d2ff8a8a9d0249 Mon Sep 17 00:00:00 2001 From: I517992 Date: Wed, 18 Nov 2020 16:35:38 +0800 Subject: [PATCH 16/40] fixed issues --- infrabox/test-registry/Dockerfile | 3 +++ infrabox/test/github-review/test.py | 4 ++-- src/pyinfrabox/tests/test_markup.py | 16 ++++++++-------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/infrabox/test-registry/Dockerfile b/infrabox/test-registry/Dockerfile index c157bf034..b1d2923a1 100644 --- a/infrabox/test-registry/Dockerfile +++ b/infrabox/test-registry/Dockerfile @@ -3,6 +3,9 @@ FROM quay.io/infrabox/images-test:build_$INFRABOX_BUILD_NUMBER ENV PYTHONPATH=/ +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 + COPY infrabox/test-registry/ /test/ COPY infrabox/test/utils/id_rsa /var/run/secrets/infrabox.net/rsa/id_rsa COPY src/utils/wait-for-webserver.sh /wait-for-webserver.sh diff --git a/infrabox/test/github-review/test.py b/infrabox/test/github-review/test.py index a8c284827..c6ad1bf1e 100644 --- a/infrabox/test/github-review/test.py +++ b/infrabox/test/github-review/test.py @@ -123,7 +123,7 @@ def side_effect(_1, _2, _3): requests_post.assert_called_with( 'status_url', - data='{"state": "pending", "target_url": "GITHUB_URL/dashboard/#/project/projectname/build/123/123/job/jobname", "description": "%s", "context": "Job: jobname"}' % ("project_id:%s job_id:%s" % (self.job_data['project_id'], self.job_data['id']) ), + data='{"target_url": "GITHUB_URL/dashboard/#/project/projectname/build/123/123/job/jobname", "state": "pending", "context": "Job: jobname", "description": "%s"}' % ("project_id:%s job_id:%s" % (self.job_data['project_id'], self.job_data['id']) ), headers={'Authorization': 'token token', 'User-Agent': 'InfraBox'}, timeout=10, verify=False) @@ -149,7 +149,7 @@ def side_effect(_1, _2, _3): execute_sql.side_effect = side_effect handle_job_update(None, self.event) - data = '{"state": "%s", "target_url": "GITHUB_URL/dashboard/#/project/projectname/build/123/123/job/jobname", "description": "%s", "context": "Job: jobname"}' % (self.expected_github_status, "project_id:%s job_id:%s" % (self.job_data['project_id'], self.job_data['id'])) + data = '{"target_url": "GITHUB_URL/dashboard/#/project/projectname/build/123/123/job/jobname", "context": "Job: jobname", "state": "%s", "description": "%s"}' % (self.expected_github_status, "project_id:%s job_id:%s" % (self.job_data['project_id'], self.job_data['id'])) requests_post.assert_called_with( 'status_url', data=data, diff --git a/src/pyinfrabox/tests/test_markup.py b/src/pyinfrabox/tests/test_markup.py index 52bc993c5..0f1e19963 100644 --- a/src/pyinfrabox/tests/test_markup.py +++ b/src/pyinfrabox/tests/test_markup.py @@ -13,7 +13,7 @@ def raises_expect(self, data, expected): parse_document(data) assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(e.msg, expected) def test_version(self): @@ -65,7 +65,7 @@ def raises_expect_text(data, expected): parse_text(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(e.msg, expected) raises_expect_text({}, "#: property 'type' is required") raises_expect_text({"type": "text"}, "#: property 'text' is required") @@ -86,7 +86,7 @@ def raises_expect_list(data, expected): parse_ordered_list(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(e.msg, expected) raises_expect_list({}, "#: property 'type' is required") raises_expect_list({"type": "ordered_list"}, "#: property 'elements' is required") @@ -104,7 +104,7 @@ def raises_expect_list(data, expected): parse_unordered_list(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(e.msg, expected) raises_expect_list({}, "#: property 'type' is required") raises_expect_list({"type": "unordered_list"}, "#: property 'elements' is required") @@ -122,7 +122,7 @@ def raises_expect_group(data, expected): parse_unordered_list(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(e.msg, expected) raises_expect_group({}, "#: property 'type' is required") raises_expect_group({"type": "group"}, "#: property 'elements' is required") @@ -140,7 +140,7 @@ def raises_expect_p(data, expected): parse_unordered_list(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(e.msg, expected) raises_expect_p({}, "#: property 'type' is required") raises_expect_p({"type": "paragraph"}, "#: property 'elements' is required") @@ -158,7 +158,7 @@ def raises_expect_grid(data, expected): parse_grid(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(e.msg, expected) raises_expect_grid({}, "#: property 'type' is required") raises_expect_grid({"type": "grid"}, "#: property 'rows' is required") @@ -178,7 +178,7 @@ def raises_expect_table(data, expected): parse_table(data, "#") assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(e.msg, expected) raises_expect_table({}, "#: property 'type' is required") raises_expect_table({"type": "table"}, "#: property 'rows' is required") From b6b073d980af03037ee33be52e29d66701235645 Mon Sep 17 00:00:00 2001 From: I517992 Date: Wed, 18 Nov 2020 17:25:01 +0800 Subject: [PATCH 17/40] fixed problems --- src/images/test/Dockerfile | 22 ++++++++++++++++----- src/pyinfrabox/docker_compose/__init__.py | 2 +- src/pyinfrabox/tests/test_docker_compose.py | 2 +- src/pyinfrabox/tests/test_markup.py | 16 +++++++-------- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/images/test/Dockerfile b/src/images/test/Dockerfile index e46a00598..1deb8cafb 100644 --- a/src/images/test/Dockerfile +++ b/src/images/test/Dockerfile @@ -1,18 +1,30 @@ -FROM debian:9.6-slim +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y \ && apt-get install -y \ - libpq-dev \ - python3 \ + pkg-config \ + libpq-dev \ + python3-dev \ python3-pip \ - python3-six \ python3-cryptography \ python3-pyasn1 \ python3-crypto \ - python-ldap \ + libxml2-dev \ + libxmlsec1-dev \ + python3-paramiko \ + openssh-client \ + inotify-tools \ + libsasl2-dev \ + libldap2-dev \ + libssl-dev \ + libxmlsec1-openssl \ curl \ git \ && pip3 install \ + six \ + python-ldap \ psycopg2 \ nose \ pyyaml \ diff --git a/src/pyinfrabox/docker_compose/__init__.py b/src/pyinfrabox/docker_compose/__init__.py index ebc766d14..e398d7325 100644 --- a/src/pyinfrabox/docker_compose/__init__.py +++ b/src/pyinfrabox/docker_compose/__init__.py @@ -68,5 +68,5 @@ def parse(d): def create_from(path): with open(path) as f: - d = yaml.load(f.read()) + d = yaml.load(f.read(), yaml.SafeLoader) return parse(d) diff --git a/src/pyinfrabox/tests/test_docker_compose.py b/src/pyinfrabox/tests/test_docker_compose.py index 2810d498b..d3dc28269 100644 --- a/src/pyinfrabox/tests/test_docker_compose.py +++ b/src/pyinfrabox/tests/test_docker_compose.py @@ -12,7 +12,7 @@ def run_exception(self, path, message): assert False except Exception as e: print(e) - self.assertEqual(e.message, message) + self.assertEqual(str(e), message) def test_empty(self): self.run_exception('./test/empty.yml', 'invalid file') diff --git a/src/pyinfrabox/tests/test_markup.py b/src/pyinfrabox/tests/test_markup.py index 0f1e19963..44c0cb44f 100644 --- a/src/pyinfrabox/tests/test_markup.py +++ b/src/pyinfrabox/tests/test_markup.py @@ -13,7 +13,7 @@ def raises_expect(self, data, expected): parse_document(data) assert False except ValidationError as e: - self.assertEqual(e.msg, expected) + self.assertEqual(str(e), expected) def test_version(self): @@ -65,7 +65,7 @@ def raises_expect_text(data, expected): parse_text(data, "#") assert False except ValidationError as e: - self.assertEqual(e.msg, expected) + self.assertEqual(str(e), expected) raises_expect_text({}, "#: property 'type' is required") raises_expect_text({"type": "text"}, "#: property 'text' is required") @@ -86,7 +86,7 @@ def raises_expect_list(data, expected): parse_ordered_list(data, "#") assert False except ValidationError as e: - self.assertEqual(e.msg, expected) + self.assertEqual(str(e), expected) raises_expect_list({}, "#: property 'type' is required") raises_expect_list({"type": "ordered_list"}, "#: property 'elements' is required") @@ -104,7 +104,7 @@ def raises_expect_list(data, expected): parse_unordered_list(data, "#") assert False except ValidationError as e: - self.assertEqual(e.msg, expected) + self.assertEqual(str(e), expected) raises_expect_list({}, "#: property 'type' is required") raises_expect_list({"type": "unordered_list"}, "#: property 'elements' is required") @@ -122,7 +122,7 @@ def raises_expect_group(data, expected): parse_unordered_list(data, "#") assert False except ValidationError as e: - self.assertEqual(e.msg, expected) + self.assertEqual(str(e), expected) raises_expect_group({}, "#: property 'type' is required") raises_expect_group({"type": "group"}, "#: property 'elements' is required") @@ -140,7 +140,7 @@ def raises_expect_p(data, expected): parse_unordered_list(data, "#") assert False except ValidationError as e: - self.assertEqual(e.msg, expected) + self.assertEqual(str(e), expected) raises_expect_p({}, "#: property 'type' is required") raises_expect_p({"type": "paragraph"}, "#: property 'elements' is required") @@ -158,7 +158,7 @@ def raises_expect_grid(data, expected): parse_grid(data, "#") assert False except ValidationError as e: - self.assertEqual(e.msg, expected) + self.assertEqual(str(e), expected) raises_expect_grid({}, "#: property 'type' is required") raises_expect_grid({"type": "grid"}, "#: property 'rows' is required") @@ -178,7 +178,7 @@ def raises_expect_table(data, expected): parse_table(data, "#") assert False except ValidationError as e: - self.assertEqual(e.msg, expected) + self.assertEqual(str(e), expected) raises_expect_table({}, "#: property 'type' is required") raises_expect_table({"type": "table"}, "#: property 'rows' is required") From 22894015594fb2c60186f7ce31be9f601bfd36c9 Mon Sep 17 00:00:00 2001 From: I517992 Date: Wed, 18 Nov 2020 18:03:33 +0800 Subject: [PATCH 18/40] fixed problems --- src/pyinfrabox/tests/test_json.py | 2 +- src/pyinfrabox/utils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pyinfrabox/tests/test_json.py b/src/pyinfrabox/tests/test_json.py index 2abef1daf..0c9e75964 100644 --- a/src/pyinfrabox/tests/test_json.py +++ b/src/pyinfrabox/tests/test_json.py @@ -9,7 +9,7 @@ def raises_expect(self, data, expected): validate_json(data) assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(str(e), expected) def test_version(self): self.raises_expect({}, "#: property 'version' is required") diff --git a/src/pyinfrabox/utils.py b/src/pyinfrabox/utils.py index e6a8395e1..0c3d2cf6c 100644 --- a/src/pyinfrabox/utils.py +++ b/src/pyinfrabox/utils.py @@ -17,6 +17,7 @@ def check_text(t, path, allowEmpty=False): if not isinstance(t, str): + print(t) raise ValidationError(path, "is not a string") if not allowEmpty and not t: From 5ebc373ba9bf14bb39c5f46de135a4ce97d6c044 Mon Sep 17 00:00:00 2001 From: I517992 Date: Wed, 18 Nov 2020 18:18:55 +0800 Subject: [PATCH 19/40] fixed problems --- src/pyinfrabox/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pyinfrabox/utils.py b/src/pyinfrabox/utils.py index 0c3d2cf6c..118087b93 100644 --- a/src/pyinfrabox/utils.py +++ b/src/pyinfrabox/utils.py @@ -18,6 +18,7 @@ def check_text(t, path, allowEmpty=False): if not isinstance(t, str): print(t) + print(type(t)) raise ValidationError(path, "is not a string") if not allowEmpty and not t: From 13d5f15b57bf4ad83ff5f4bcd68f49b9fb86233c Mon Sep 17 00:00:00 2001 From: I517992 Date: Wed, 18 Nov 2020 18:20:48 +0800 Subject: [PATCH 20/40] fixed problems --- infrabox/test-registry/Dockerfile_auth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrabox/test-registry/Dockerfile_auth b/infrabox/test-registry/Dockerfile_auth index 7a0be5240..fad992d49 100644 --- a/infrabox/test-registry/Dockerfile_auth +++ b/infrabox/test-registry/Dockerfile_auth @@ -8,4 +8,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils COPY src/pyinfrabox /pyinfrabox COPY infrabox/test/utils/id_rsa.pub /var/run/secrets/infrabox.net/rsa/id_rsa.pub -CMD python /server.py +CMD python3 /server.py From 70fc6d9f388243a2738ee8b69e091c5a6eea6ac8 Mon Sep 17 00:00:00 2001 From: berry Date: Thu, 19 Nov 2020 17:27:47 +0800 Subject: [PATCH 21/40] fix test utils --- src/pyinfrabox/utils.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/pyinfrabox/utils.py b/src/pyinfrabox/utils.py index 118087b93..dd7465324 100644 --- a/src/pyinfrabox/utils.py +++ b/src/pyinfrabox/utils.py @@ -1,24 +1,13 @@ -import os -import errno import uuid -from builtins import int, range, str -from past.builtins import str from pyinfrabox import ValidationError -try: - #python2 - from urllib.parse import urlparse -except: - #python3 - from urllib.parse import urlparse +from urllib.parse import urlparse def check_text(t, path, allowEmpty=False): if not isinstance(t, str): - print(t) - print(type(t)) raise ValidationError(path, "is not a string") if not allowEmpty and not t: From 9fc165325c009c7f963f50c922e192870d843919 Mon Sep 17 00:00:00 2001 From: berry Date: Fri, 20 Nov 2020 10:01:57 +0800 Subject: [PATCH 22/40] fix assert exception --- src/pyinfrabox/tests/test_testresult.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyinfrabox/tests/test_testresult.py b/src/pyinfrabox/tests/test_testresult.py index 6e9dd05a8..d751762e0 100644 --- a/src/pyinfrabox/tests/test_testresult.py +++ b/src/pyinfrabox/tests/test_testresult.py @@ -9,7 +9,7 @@ def raises_expect(self, data, expected): validate_result(data) assert False except ValidationError as e: - self.assertEqual(e.message, expected) + self.assertEqual(str(e), expected) def test_version(self): self.raises_expect({}, "#: property 'version' is required") From 85b8816d09550cf505526eb75306a185e2175c75 Mon Sep 17 00:00:00 2001 From: berry Date: Fri, 20 Nov 2020 10:08:15 +0800 Subject: [PATCH 23/40] fix test get headers --- infrabox/test-registry/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrabox/test-registry/test.py b/infrabox/test-registry/test.py index 857ac395f..c44ab5c84 100644 --- a/infrabox/test-registry/test.py +++ b/infrabox/test-registry/test.py @@ -34,7 +34,7 @@ class Test(TestCase): image_path = project_id + '/image_name' def _get_headers(self): - auth = base64.b64encode('infrabox:%s' % encode_project_token(self.token, self.project_id, 'myproject')) + auth = base64.b64encode(('infrabox:%s' % encode_project_token(self.token, self.project_id, 'myproject')).encode('utf-8')).decode('utf-8') headers = {'authorization': "Basic " + auth} return headers From a31b04bc8aa4aa77ece675a9e592525470482883 Mon Sep 17 00:00:00 2001 From: berry Date: Fri, 20 Nov 2020 10:19:30 +0800 Subject: [PATCH 24/40] fix base64 encode --- infrabox/test-registry/test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infrabox/test-registry/test.py b/infrabox/test-registry/test.py index c44ab5c84..4ef34e52d 100644 --- a/infrabox/test-registry/test.py +++ b/infrabox/test-registry/test.py @@ -18,7 +18,8 @@ class InputTests(TestCase): def get(self, url, password='b514af82-3c4f-4bb5-b1da-a89a0ced5e6f'): - auth = base64.b64encode('infrabox:%s' % encode_project_token(password, '2514af82-3c4f-4bb5-b1da-a89a0ced5e6f', 'myproject')) + basic = 'infrabox:%s' % encode_project_token(password, '2514af82-3c4f-4bb5-b1da-a89a0ced5e6f', 'myproject') + auth = base64.b64encode(basic.encode('utf-8')).decode('utf-8') headers = {'authorization': "Basic " + auth} return requests.get(url, headers=headers) From 07baa46c776383cd37c89c9bb4b4748720a8324a Mon Sep 17 00:00:00 2001 From: berry Date: Fri, 20 Nov 2020 10:31:25 +0800 Subject: [PATCH 25/40] fix github-review test --- infrabox/test/github-review/test.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/infrabox/test/github-review/test.py b/infrabox/test/github-review/test.py index c6ad1bf1e..0bea7f6a4 100644 --- a/infrabox/test/github-review/test.py +++ b/infrabox/test/github-review/test.py @@ -123,7 +123,12 @@ def side_effect(_1, _2, _3): requests_post.assert_called_with( 'status_url', - data='{"target_url": "GITHUB_URL/dashboard/#/project/projectname/build/123/123/job/jobname", "state": "pending", "context": "Job: jobname", "description": "%s"}' % ("project_id:%s job_id:%s" % (self.job_data['project_id'], self.job_data['id']) ), + json={ + "target_url": "GITHUB_URL/dashboard/#/project/projectname/build/123/123/job/jobname", + "state": "pending", + "context": "Job: jobname", + "description": "project_id:%s job_id:%s" % (self.job_data['project_id'], self.job_data['id']) + }, headers={'Authorization': 'token token', 'User-Agent': 'InfraBox'}, timeout=10, verify=False) @@ -149,10 +154,14 @@ def side_effect(_1, _2, _3): execute_sql.side_effect = side_effect handle_job_update(None, self.event) - data = '{"target_url": "GITHUB_URL/dashboard/#/project/projectname/build/123/123/job/jobname", "context": "Job: jobname", "state": "%s", "description": "%s"}' % (self.expected_github_status, "project_id:%s job_id:%s" % (self.job_data['project_id'], self.job_data['id'])) requests_post.assert_called_with( 'status_url', - data=data, + json={ + "target_url": "GITHUB_URL/dashboard/#/project/projectname/build/123/123/job/jobname", + "context": "Job: jobname", + "state": self.expected_github_status, + "description": "project_id:%s job_id:%s" % (self.job_data['project_id'], self.job_data['id']) + }, headers={'Authorization': 'token token', 'User-Agent': 'InfraBox'}, timeout=10, From 83c6982dd4c31b0f5b5823ed96f4a2bff23c99e9 Mon Sep 17 00:00:00 2001 From: berry Date: Fri, 20 Nov 2020 10:39:33 +0800 Subject: [PATCH 26/40] fix github-review test --- src/github/review/review.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/github/review/review.py b/src/github/review/review.py index 7fbac6f5c..ad218e7cd 100755 --- a/src/github/review/review.py +++ b/src/github/review/review.py @@ -171,7 +171,7 @@ def handle_job_update(conn, event): # TODO(ib-steffen): support ca bundles try: r = requests.post(github_status_url, - data=json.dumps(payload), + json=payload, headers=headers, timeout=10, verify=False) From 467fff3b88064f5fd5276c19fa3d30f8a9f456e3 Mon Sep 17 00:00:00 2001 From: I517992 Date: Fri, 20 Nov 2020 11:00:22 +0800 Subject: [PATCH 27/40] remove unused code --- ib.py | 2 +- infrabox/test/api/job_api_test.py | 6 +++--- infrabox/test/e2e/test.py | 4 ++-- infrabox/test/e2e/tests/docker_compose_job/tests/tests.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ib.py b/ib.py index 4c5ae5871..bd706691a 100755 --- a/ib.py +++ b/ib.py @@ -65,7 +65,7 @@ def execute(command, cwd=None, env=None, ignore_error=False, ignore_output=False if ignore_output: continue - print((line.rstrip())) + print(line.rstrip()) process.wait() diff --git a/infrabox/test/api/job_api_test.py b/infrabox/test/api/job_api_test.py index 88d343023..0df65e325 100644 --- a/infrabox/test/api/job_api_test.py +++ b/infrabox/test/api/job_api_test.py @@ -237,6 +237,6 @@ def test_testresult(self): keys = ['status', 'duration', 'message', 'stack'] for i, received_row in enumerate(r): # create dictionary from the list to compare it easier - row_dictionary = dict(list(zip(keys, received_row))) - self.assertTrue(all(item in list(testresult_data["tests"][i].items()) - for item in list(row_dictionary.items()))) + row_dictionary = dict(zip(keys, received_row)) + self.assertTrue(all(item in testresult_data["tests"][i].items() + for item in row_dictionary.items())) diff --git a/infrabox/test/e2e/test.py b/infrabox/test/e2e/test.py index bcf06e8fa..2f7454603 100644 --- a/infrabox/test/e2e/test.py +++ b/infrabox/test/e2e/test.py @@ -73,7 +73,7 @@ def _get_build(self): return result.json()[0] except: print("Get build failed: ") - print((result.text)) + print(result.text) raise def _get_jobs(self): @@ -322,7 +322,7 @@ def main(): print("Connecting to DB") connect_db() # Wait for DB - print(("ROOT_URL: %s" % root_url)) + print("ROOT_URL: %s" % root_url) while True: time.sleep(1) r = None diff --git a/infrabox/test/e2e/tests/docker_compose_job/tests/tests.py b/infrabox/test/e2e/tests/docker_compose_job/tests/tests.py index e7929b831..77b2c7cb0 100644 --- a/infrabox/test/e2e/tests/docker_compose_job/tests/tests.py +++ b/infrabox/test/e2e/tests/docker_compose_job/tests/tests.py @@ -3,5 +3,5 @@ def test_get(): print("get") r = requests.get("http://test-server:3000") - print((r.json())) + print(r.json()) assert True From 1b02f340d079f3b44bbc9028a65a24bb731c3206 Mon Sep 17 00:00:00 2001 From: I517992 Date: Fri, 20 Nov 2020 13:31:12 +0800 Subject: [PATCH 28/40] change base image for e2e test --- infrabox/test/e2e/Dockerfile | 24 +++++++++++++++--------- infrabox/test/e2e/entrypoint.sh | 2 +- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/infrabox/test/e2e/Dockerfile b/infrabox/test/e2e/Dockerfile index 3efe56087..176b182bb 100644 --- a/infrabox/test/e2e/Dockerfile +++ b/infrabox/test/e2e/Dockerfile @@ -1,17 +1,23 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive ARG HELM_VERSION=2.12.3 RUN apt-get update -y && apt-get install -y \ curl \ - python-pip \ + pkg-config \ + libpq-dev \ + python3-dev \ + python3-pip \ postgresql-client \ - python-xmlrunner \ - python-requests \ - python-psycopg2 \ - python-jwt \ - python-crypto \ - python-cryptography \ - git + python3-crypto \ + python3-cryptography \ + git \ + && pip3 install \ + xmlrunner \ + requests==2.18.4 \ + psycopg2 \ + jwt RUN mkdir /project WORKDIR /project diff --git a/infrabox/test/e2e/entrypoint.sh b/infrabox/test/e2e/entrypoint.sh index 4ebc35c16..f4ddd71e9 100755 --- a/infrabox/test/e2e/entrypoint.sh +++ b/infrabox/test/e2e/entrypoint.sh @@ -197,7 +197,7 @@ _runTests() { pushd /infrabox/context/infrabox/test/e2e set +e - python test.py + python3 test.py rc=$? cp results.xml /infrabox/upload/testresult From bd455dd67089d64fe898531a1b6b230d131143ed Mon Sep 17 00:00:00 2001 From: I517992 Date: Fri, 20 Nov 2020 14:14:53 +0800 Subject: [PATCH 29/40] set pip as pip3 --- infrabox/test/e2e/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrabox/test/e2e/entrypoint.sh b/infrabox/test/e2e/entrypoint.sh index f4ddd71e9..d5e6f8542 100755 --- a/infrabox/test/e2e/entrypoint.sh +++ b/infrabox/test/e2e/entrypoint.sh @@ -41,10 +41,10 @@ _prepareKubectl() { _getDependencies() { echo "## install infraboxcli" - # pip install infraboxcli + # pip3 install infraboxcli git clone https://github.com/SAP/InfraBox-cli.git /cli pushd /cli - pip install -e . + pip3 install -e . infrabox version git rev-parse HEAD popd From 37b2c5f88ec891cf04f303e69dc398bc37f15bc6 Mon Sep 17 00:00:00 2001 From: I517992 Date: Fri, 20 Nov 2020 14:55:01 +0800 Subject: [PATCH 30/40] add `helm repo update` --- infrabox/test/e2e/entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/infrabox/test/e2e/entrypoint.sh b/infrabox/test/e2e/entrypoint.sh index d5e6f8542..9701ff76e 100755 --- a/infrabox/test/e2e/entrypoint.sh +++ b/infrabox/test/e2e/entrypoint.sh @@ -67,6 +67,7 @@ _initHelm() { --clusterrole cluster-admin \ --serviceaccount=kube-system:tiller helm init --service-account tiller --wait + helm repo update } _getPodNameImpl() { From 3c653fe27aae68c4d3de48a77ff2f8a5a06c328b Mon Sep 17 00:00:00 2001 From: I517992 Date: Fri, 20 Nov 2020 15:50:49 +0800 Subject: [PATCH 31/40] remove `helm repo update` --- infrabox/test/e2e/entrypoint.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/infrabox/test/e2e/entrypoint.sh b/infrabox/test/e2e/entrypoint.sh index 9501bce2c..ef8e69cf1 100755 --- a/infrabox/test/e2e/entrypoint.sh +++ b/infrabox/test/e2e/entrypoint.sh @@ -67,7 +67,6 @@ _initHelm() { --clusterrole cluster-admin \ --serviceaccount=kube-system:tiller helm init --service-account tiller --wait - helm repo update } _getPodNameImpl() { From 3a3cf570dcc9a764aae52396a17d237494f5af25 Mon Sep 17 00:00:00 2001 From: I517992 Date: Fri, 20 Nov 2020 16:45:59 +0800 Subject: [PATCH 32/40] set python as python3 --- infrabox/test/e2e/entrypoint.sh | 2 +- src/api/Dockerfile | 2 +- src/api/run_dev.sh | 2 +- src/api/run_with_dummy.sh | 2 +- src/cluster-status/Dockerfile | 2 +- src/collector-api/Dockerfile | 2 +- src/db/Dockerfile | 2 +- src/docker-registry/auth/Dockerfile | 2 +- src/docker-registry/nginx/Dockerfile | 2 +- src/docker-registry/nginx/entrypoint.sh | 2 +- src/gc/Dockerfile | 2 +- src/gerrit/api/entrypoint.sh | 2 +- src/gerrit/review/entrypoint.sh | 2 +- src/gerrit/trigger/entrypoint.sh | 2 +- src/github/review/Dockerfile | 2 +- src/github/trigger/Dockerfile | 2 +- src/metrics/Dockerfile | 2 +- src/services/gardener/Dockerfile | 2 +- src/services/gardener/tmp/build/Dockerfile | 2 +- src/services/gcp/Dockerfile | 2 +- src/services/gcp/tmp/build/Dockerfile | 2 +- src/status/cachet/Dockerfile | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/infrabox/test/e2e/entrypoint.sh b/infrabox/test/e2e/entrypoint.sh index ef8e69cf1..ca82ca71e 100755 --- a/infrabox/test/e2e/entrypoint.sh +++ b/infrabox/test/e2e/entrypoint.sh @@ -41,7 +41,7 @@ _prepareKubectl() { _getDependencies() { echo "## install infraboxcli" - # pip3 install infraboxcli + # pip install infraboxcli git clone https://github.com/SAP/InfraBox-cli.git /cli pushd /cli pip3 install -e . diff --git a/src/api/Dockerfile b/src/api/Dockerfile index a3db93875..7e497e41c 100644 --- a/src/api/Dockerfile +++ b/src/api/Dockerfile @@ -8,4 +8,4 @@ COPY src/openpolicyagent /infrabox/openpolicyagent COPY src/pyinfraboxutils /infrabox/pyinfraboxutils COPY src/pyinfrabox /infrabox/pyinfrabox -CMD python /infrabox/api/server.py +CMD python3 /infrabox/api/server.py diff --git a/src/api/run_dev.sh b/src/api/run_dev.sh index 4b4e0b141..b473627e9 100755 --- a/src/api/run_dev.sh +++ b/src/api/run_dev.sh @@ -14,7 +14,7 @@ sigint_handler() trap sigint_handler SIGINT while true; do - python $DIR/server.py & + python3 $DIR/server.py & PID=$! inotifywait -e modify -e move -e create --exclude='.*__pycache__.*' -r $watch kill $PID diff --git a/src/api/run_with_dummy.sh b/src/api/run_with_dummy.sh index b38d10555..f085a1d7b 100755 --- a/src/api/run_with_dummy.sh +++ b/src/api/run_with_dummy.sh @@ -67,4 +67,4 @@ export INFRABOX_LEGAL_PRIVACY_URL=https://www.sap.com/about/legal/privacy.html export INFRABOX_LEGAL_TERMS_OF_USE_URL=https://www.sap.com/corporate/en/legal/terms-of-use.html -python server.py +python3 server.py diff --git a/src/cluster-status/Dockerfile b/src/cluster-status/Dockerfile index 586d3e65d..9f2c166c0 100644 --- a/src/cluster-status/Dockerfile +++ b/src/cluster-status/Dockerfile @@ -7,4 +7,4 @@ COPY src/pyinfrabox /pyinfrabox ENV PYTHONPATH=/ -CMD python /server.py +CMD python3 /server.py diff --git a/src/collector-api/Dockerfile b/src/collector-api/Dockerfile index 66bae7f84..f5b5afa68 100644 --- a/src/collector-api/Dockerfile +++ b/src/collector-api/Dockerfile @@ -7,4 +7,4 @@ COPY src/collector-api /collector COPY src/pyinfraboxutils /pyinfraboxutils COPY src/pyinfrabox /pyinfrabox -CMD python /collector/server.py +CMD python3 /collector/server.py diff --git a/src/db/Dockerfile b/src/db/Dockerfile index f0631baee..f2c85a1c1 100644 --- a/src/db/Dockerfile +++ b/src/db/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -CMD python db/migrate.py +CMD python3 db/migrate.py diff --git a/src/docker-registry/auth/Dockerfile b/src/docker-registry/auth/Dockerfile index f5cfa5245..33f801823 100644 --- a/src/docker-registry/auth/Dockerfile +++ b/src/docker-registry/auth/Dockerfile @@ -7,4 +7,4 @@ COPY src/pyinfrabox /pyinfrabox ENV PYTHONPATH=/ -CMD python /server.py +CMD python3 /server.py diff --git a/src/docker-registry/nginx/Dockerfile b/src/docker-registry/nginx/Dockerfile index 98250562d..d57724a96 100644 --- a/src/docker-registry/nginx/Dockerfile +++ b/src/docker-registry/nginx/Dockerfile @@ -1,5 +1,5 @@ FROM nginx:1.15-alpine -RUN apk add --no-cache curl python py-pip openssl && \ +RUN apk add --no-cache curl python3 py-pip openssl && \ pip install htpasswd && \ apk del py-pip diff --git a/src/docker-registry/nginx/entrypoint.sh b/src/docker-registry/nginx/entrypoint.sh index e493ec0fe..fc733681c 100755 --- a/src/docker-registry/nginx/entrypoint.sh +++ b/src/docker-registry/nginx/entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/sh -e -HTPASSWD_PATH=/home/nginx/nginx.htpasswd python get_admin_pw.py +HTPASSWD_PATH=/home/nginx/nginx.htpasswd python3 get_admin_pw.py sed -i -e "s/auth-host-placeholder/$INFRABOX_AUTH_HOST/g" /etc/nginx/nginx.conf sed -i -e "s/registry-host-placeholder/$INFRABOX_REGISTRY_HOST/g" /etc/nginx/nginx.conf diff --git a/src/gc/Dockerfile b/src/gc/Dockerfile index 215318a81..aa82500a5 100644 --- a/src/gc/Dockerfile +++ b/src/gc/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -CMD python gc/gc.py +CMD python3 gc/gc.py diff --git a/src/gerrit/api/entrypoint.sh b/src/gerrit/api/entrypoint.sh index 762cae851..653daae94 100755 --- a/src/gerrit/api/entrypoint.sh +++ b/src/gerrit/api/entrypoint.sh @@ -5,4 +5,4 @@ cp /tmp/gerrit/id_rsa ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa ssh-keyscan -p $INFRABOX_GERRIT_PORT $INFRABOX_GERRIT_HOSTNAME >> ~/.ssh/known_hosts -python /api.py +python3 /api.py diff --git a/src/gerrit/review/entrypoint.sh b/src/gerrit/review/entrypoint.sh index 24ca95f21..d1f2f8b2f 100755 --- a/src/gerrit/review/entrypoint.sh +++ b/src/gerrit/review/entrypoint.sh @@ -5,4 +5,4 @@ cp /tmp/gerrit/id_rsa ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa ssh-keyscan -p $INFRABOX_GERRIT_PORT $INFRABOX_GERRIT_HOSTNAME >> ~/.ssh/known_hosts -python /review.py +python3 /review.py diff --git a/src/gerrit/trigger/entrypoint.sh b/src/gerrit/trigger/entrypoint.sh index 809c70af3..4c82f330b 100755 --- a/src/gerrit/trigger/entrypoint.sh +++ b/src/gerrit/trigger/entrypoint.sh @@ -5,4 +5,4 @@ cp /tmp/gerrit/id_rsa ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa ssh-keyscan -p $INFRABOX_GERRIT_PORT $INFRABOX_GERRIT_HOSTNAME >> ~/.ssh/known_hosts -python /trigger.py +python3 /trigger.py diff --git a/src/github/review/Dockerfile b/src/github/review/Dockerfile index b28690758..a18ff9bf6 100644 --- a/src/github/review/Dockerfile +++ b/src/github/review/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -CMD python review.py +CMD python3 review.py diff --git a/src/github/trigger/Dockerfile b/src/github/trigger/Dockerfile index a0f33536f..69aa3d7ea 100644 --- a/src/github/trigger/Dockerfile +++ b/src/github/trigger/Dockerfile @@ -7,4 +7,4 @@ COPY src/pyinfrabox /pyinfrabox ENV PYTHONPATH=/ -CMD python /trigger.py +CMD python3 /trigger.py diff --git a/src/metrics/Dockerfile b/src/metrics/Dockerfile index 94c8a58ce..109c7cc9a 100644 --- a/src/metrics/Dockerfile +++ b/src/metrics/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -CMD python /metrics/server.py +CMD python3 /metrics/server.py diff --git a/src/services/gardener/Dockerfile b/src/services/gardener/Dockerfile index 4c4a17485..04aac5873 100644 --- a/src/services/gardener/Dockerfile +++ b/src/services/gardener/Dockerfile @@ -17,7 +17,7 @@ ENV PATH /google-cloud-sdk/bin:$PATH RUN apk --no-cache add \ curl \ - python \ + python3 \ py-crcmod \ bash \ libc6-compat \ diff --git a/src/services/gardener/tmp/build/Dockerfile b/src/services/gardener/tmp/build/Dockerfile index 38ccbd317..060bb13ef 100644 --- a/src/services/gardener/tmp/build/Dockerfile +++ b/src/services/gardener/tmp/build/Dockerfile @@ -5,7 +5,7 @@ ENV PATH /google-cloud-sdk/bin:$PATH RUN apk --no-cache add \ curl \ - python \ + python3 \ py-crcmod \ bash \ libc6-compat \ diff --git a/src/services/gcp/Dockerfile b/src/services/gcp/Dockerfile index 9d54b5d89..89e230372 100644 --- a/src/services/gcp/Dockerfile +++ b/src/services/gcp/Dockerfile @@ -17,7 +17,7 @@ ENV PATH /google-cloud-sdk/bin:$PATH RUN apk --no-cache add \ curl \ - python \ + python3 \ py-crcmod \ bash \ libc6-compat \ diff --git a/src/services/gcp/tmp/build/Dockerfile b/src/services/gcp/tmp/build/Dockerfile index ada0a64d4..15b089061 100644 --- a/src/services/gcp/tmp/build/Dockerfile +++ b/src/services/gcp/tmp/build/Dockerfile @@ -5,7 +5,7 @@ ENV PATH /google-cloud-sdk/bin:$PATH RUN apk --no-cache add \ curl \ - python \ + python3 \ py-crcmod \ bash \ libc6-compat \ diff --git a/src/status/cachet/Dockerfile b/src/status/cachet/Dockerfile index 8e51e556d..97e711276 100644 --- a/src/status/cachet/Dockerfile +++ b/src/status/cachet/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -CMD python /cachet.py +CMD python3 /cachet.py From 6092bc50a9720f7f6b9cc63d6695ac567e0bab52 Mon Sep 17 00:00:00 2001 From: I517992 Date: Fri, 20 Nov 2020 16:57:53 +0800 Subject: [PATCH 33/40] set pip as pip3 --- src/docker-registry/nginx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docker-registry/nginx/Dockerfile b/src/docker-registry/nginx/Dockerfile index d57724a96..2e26284ec 100644 --- a/src/docker-registry/nginx/Dockerfile +++ b/src/docker-registry/nginx/Dockerfile @@ -1,6 +1,6 @@ FROM nginx:1.15-alpine RUN apk add --no-cache curl python3 py-pip openssl && \ - pip install htpasswd && \ + pip3 install htpasswd && \ apk del py-pip COPY src/docker-registry/nginx/nginx.conf /etc/nginx/nginx.conf From cd5b7ee5eda25fbd294ba4f5f2776053c1f8fa2e Mon Sep 17 00:00:00 2001 From: I517992 Date: Mon, 23 Nov 2020 16:27:37 +0800 Subject: [PATCH 34/40] replace jwt with pyjwt --- infrabox/test/e2e/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrabox/test/e2e/Dockerfile b/infrabox/test/e2e/Dockerfile index 176b182bb..9d73f419f 100644 --- a/infrabox/test/e2e/Dockerfile +++ b/infrabox/test/e2e/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update -y && apt-get install -y \ xmlrunner \ requests==2.18.4 \ psycopg2 \ - jwt + PyJWT RUN mkdir /project WORKDIR /project From 01fb5c8ef5327fc0a9d8c052d72f2577b05d6741 Mon Sep 17 00:00:00 2001 From: I517992 Date: Wed, 25 Nov 2020 10:48:45 +0800 Subject: [PATCH 35/40] remove tab --- infrabox/test/e2e/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infrabox/test/e2e/Dockerfile b/infrabox/test/e2e/Dockerfile index 9d73f419f..28a73fe40 100644 --- a/infrabox/test/e2e/Dockerfile +++ b/infrabox/test/e2e/Dockerfile @@ -6,9 +6,9 @@ ARG HELM_VERSION=2.12.3 RUN apt-get update -y && apt-get install -y \ curl \ pkg-config \ - libpq-dev \ - python3-dev \ - python3-pip \ + libpq-dev \ + python3-dev \ + python3-pip \ postgresql-client \ python3-crypto \ python3-cryptography \ From 699d52c85d075d7cbe60470aa4c0325333e2bdfa Mon Sep 17 00:00:00 2001 From: I517992 Date: Wed, 25 Nov 2020 10:56:33 +0800 Subject: [PATCH 36/40] remove useless parenthesis --- infrabox/test/e2e/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrabox/test/e2e/test.py b/infrabox/test/e2e/test.py index 2f7454603..2f45d86b3 100644 --- a/infrabox/test/e2e/test.py +++ b/infrabox/test/e2e/test.py @@ -332,7 +332,7 @@ def main(): if r.status_code in (200, 404): break - print((r.text)) + print(r.text) except Exception as e: print(e) From 517c6cff795831112ba0277662e9bc67d763a1e2 Mon Sep 17 00:00:00 2001 From: berry Date: Thu, 26 Nov 2020 10:50:10 +0800 Subject: [PATCH 37/40] fix(integration-test): fix rsa public key path --- infrabox/test/e2e/entrypoint.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/infrabox/test/e2e/entrypoint.sh b/infrabox/test/e2e/entrypoint.sh index ca82ca71e..56c5c761c 100755 --- a/infrabox/test/e2e/entrypoint.sh +++ b/infrabox/test/e2e/entrypoint.sh @@ -145,6 +145,9 @@ _installInfrabox() { mkdir -p /var/run/secrets/infrabox.net/rsa/ cp id_rsa* /var/run/secrets/infrabox.net/rsa/ + export INFRABOX_RSA_PRIVATE_KEY_PATH=/var/run/secrets/infrabox.net/rsa/id_rsa + export INFRABOX_RSA_PUBLIC_KEY_PATH=/var/run/secrets/infrabox.net/rsa/id_rsa.pem + echo "## Install infrabox" PW=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) From f17bba78c686851f6751a9f9b9a052d624f9f384 Mon Sep 17 00:00:00 2001 From: berry Date: Thu, 26 Nov 2020 10:50:10 +0800 Subject: [PATCH 38/40] fix(integration-test): fix rsa public key path --- infrabox/test/e2e/entrypoint.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/infrabox/test/e2e/entrypoint.sh b/infrabox/test/e2e/entrypoint.sh index ca82ca71e..70eaff643 100755 --- a/infrabox/test/e2e/entrypoint.sh +++ b/infrabox/test/e2e/entrypoint.sh @@ -139,12 +139,15 @@ _installNginxIngress() { _installInfrabox() { cd /infrabox/context/deploy/infrabox - ssh-keygen -N '' -t rsa -f id_rsa + ssh-keygen -N '' -t rsa -m pem -f id_rsa ssh-keygen -f id_rsa.pub -e -m pem > id_rsa.pem mkdir -p /var/run/secrets/infrabox.net/rsa/ cp id_rsa* /var/run/secrets/infrabox.net/rsa/ + export INFRABOX_RSA_PRIVATE_KEY_PATH=/var/run/secrets/infrabox.net/rsa/id_rsa + export INFRABOX_RSA_PUBLIC_KEY_PATH=/var/run/secrets/infrabox.net/rsa/id_rsa.pem + echo "## Install infrabox" PW=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) From 4742792c9e3fe390fb43f4bdf064adb7f2ef1960 Mon Sep 17 00:00:00 2001 From: I517992 Date: Thu, 26 Nov 2020 15:12:53 +0800 Subject: [PATCH 39/40] remove `apt-get remove/autoremove` & `rm -rf` --- src/images/base/Dockerfile | 5 +---- src/images/test/Dockerfile | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/images/base/Dockerfile b/src/images/base/Dockerfile index 0ac53b6c1..0169a175d 100644 --- a/src/images/base/Dockerfile +++ b/src/images/base/Dockerfile @@ -44,10 +44,7 @@ RUN apt-get update -y \ python-cachetclient \ xmlsec \ python3-saml \ - croniter \ - && apt-get remove -y python3-pip \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* + croniter RUN useradd -ms /bin/bash infrabox USER infrabox diff --git a/src/images/test/Dockerfile b/src/images/test/Dockerfile index 1deb8cafb..1887142c0 100644 --- a/src/images/test/Dockerfile +++ b/src/images/test/Dockerfile @@ -48,10 +48,7 @@ RUN apt-get update -y \ keystoneauth1==3.7.0 \ python-swiftclient \ python-cachetclient \ - croniter \ - && apt-get remove -y python3-pip \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* + croniter RUN useradd -ms /bin/bash infrabox USER infrabox From 310b59ba50e081a0c15c022b2d76592a23feb011 Mon Sep 17 00:00:00 2001 From: I517992 Date: Fri, 27 Nov 2020 15:10:33 +0800 Subject: [PATCH 40/40] rename python to python3 --- src/checker/Dockerfile | 2 +- src/scheduler/kubernetes/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/checker/Dockerfile b/src/checker/Dockerfile index 233f39808..bc38923e4 100644 --- a/src/checker/Dockerfile +++ b/src/checker/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -ENTRYPOINT ["python", "checker/checker.py"] +ENTRYPOINT ["python3", "checker/checker.py"] diff --git a/src/scheduler/kubernetes/Dockerfile b/src/scheduler/kubernetes/Dockerfile index 88799ab44..ea0b2d77b 100644 --- a/src/scheduler/kubernetes/Dockerfile +++ b/src/scheduler/kubernetes/Dockerfile @@ -6,4 +6,4 @@ COPY src/pyinfraboxutils /pyinfraboxutils ENV PYTHONPATH=/ -ENTRYPOINT ["python", "scheduler/scheduler.py"] +ENTRYPOINT ["python3", "scheduler/scheduler.py"]