From ffb33c73092d783abb34e50d6ae3daf7e28efcf1 Mon Sep 17 00:00:00 2001 From: trent-codecov Date: Wed, 17 Nov 2021 11:50:54 -0600 Subject: [PATCH] Upgrade to python 3.9 and cleanup --- .deep-dive.yaml | 2 +- Dockerfile.circle-enterprise | 17 ++++++----------- Dockerfile.enterprise | 22 ++++++++++++---------- docker-compose.yml | 10 ++++++---- enterprise/generate_pyinstaller_args.py | 13 +++++++++---- enterprise/package.sh | 8 ++++---- graphql_api/types/enums/__init__.py | 2 ++ prod.sh | 4 ++-- staging.sh | 7 ++----- 9 files changed, 44 insertions(+), 41 deletions(-) diff --git a/.deep-dive.yaml b/.deep-dive.yaml index 42661720de..b232e532f9 100644 --- a/.deep-dive.yaml +++ b/.deep-dive.yaml @@ -1,6 +1,6 @@ search: exclude_files: - python3libs: '/usr/lib/python3\.8.*' + python3libs: '/usr/lib/python3\.9.*' files: python: '.*\.pyc?' json: '.*\.json' \ No newline at end of file diff --git a/Dockerfile.circle-enterprise b/Dockerfile.circle-enterprise index a6ad4e5fb0..c5cc45b4ad 100644 --- a/Dockerfile.circle-enterprise +++ b/Dockerfile.circle-enterprise @@ -18,16 +18,11 @@ RUN apk --update --no-cache add \ COPY . /app WORKDIR /app -# Build bootloader for alpine -RUN git clone --depth 1 --single-branch --branch v3.5 https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller \ - && cd /tmp/pyinstaller/bootloader \ - && CFLAGS="-Wno-stringop-overflow" python ./waf configure --no-lsb all \ - && pip install .. \ - && rm -Rf /tmp/pyinstaller -# install cython -RUN pip install cython==3.0a1 -RUN pip install PyCrypto==2.6.1 +RUN pip install pyinstaller +# install cython +RUN pip install cython==3.0a9 +RUN pip install tinyaes # set settings module ENV DJANGO_SETTINGS_MODULE "codecov.settings_enterprise" @@ -55,7 +50,6 @@ COPY enterprise/ldd /pyinstaller/ldd COPY enterprise/setup.py setup.py COPY enterprise/cythonize.sh /pyinstaller/cythonize_everything.sh -COPY enterprise/package.sh /pyinstaller/package.sh COPY enterprise/pyi_rth_django.py /pyinstaller/pyi_rth_django.py COPY enterprise/generate_pyinstaller_args.py /pyinstaller/generate_pyinstaller_args.py @@ -66,6 +60,7 @@ COPY enterprise/hooks /hooks RUN chmod a+x /pyinstaller/* RUN ["/pyinstaller/cythonize_everything.sh"] +COPY enterprise/package.sh /pyinstaller/package.sh RUN chmod a+x /pyinstaller/* RUN ["/pyinstaller/package.sh"] ENV RUN_ENV=ENTERPRISE @@ -82,7 +77,7 @@ RUN chown codecov:application /home/enterprise.sh && \ chown codecov:application /home/api -FROM alpine:3.13 +FROM alpine:3.14 RUN addgroup -S application && adduser -S codecov -G application && rm -rf /home/codecov USER codecov diff --git a/Dockerfile.enterprise b/Dockerfile.enterprise index 696d45f867..a1ad6b877b 100644 --- a/Dockerfile.enterprise +++ b/Dockerfile.enterprise @@ -14,15 +14,15 @@ RUN apk --update --no-cache add \ && pip install --upgrade pip # Build bootloader for alpine -RUN git clone --depth 1 --single-branch --branch v3.5 https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller \ - && cd /tmp/pyinstaller/bootloader \ - && CFLAGS="-Wno-stringop-overflow" python ./waf configure --no-lsb all \ - && pip install .. \ - && rm -Rf /tmp/pyinstaller - +#RUN git clone --depth 1 --single-branch --branch v3.5 https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller \ +# && cd /tmp/pyinstaller/bootloader \ +# && CFLAGS="-Wno-stringop-overflow" python ./waf configure --no-lsb all \ +# && pip install .. \ +# && rm -Rf /tmp/pyinstaller +RUN pip install pyinstaller # install cython -RUN pip install cython==3.0a1 -RUN pip install PyCrypto==2.6.1 +RUN pip install cython==3.0a9 +RUN pip install tinyaes # set settings module ENV DJANGO_SETTINGS_MODULE "codecov.settings_enterprise" @@ -50,7 +50,7 @@ COPY enterprise/ldd /pyinstaller/ldd COPY enterprise/setup.py setup.py COPY enterprise/cythonize.sh /pyinstaller/cythonize_everything.sh -COPY enterprise/package.sh /pyinstaller/package.sh + COPY enterprise/pyi_rth_django.py /pyinstaller/pyi_rth_django.py COPY enterprise/generate_pyinstaller_args.py /pyinstaller/generate_pyinstaller_args.py @@ -61,6 +61,8 @@ COPY enterprise/hooks /hooks RUN chmod a+x /pyinstaller/* RUN ["/pyinstaller/cythonize_everything.sh"] +RUN pip install tinyaes +COPY enterprise/package.sh /pyinstaller/package.sh RUN chmod a+x /pyinstaller/* RUN ["/pyinstaller/package.sh"] ENV RUN_ENV=ENTERPRISE @@ -76,7 +78,7 @@ RUN chown codecov:application /home/enterprise.sh && \ chmod +x /home/enterprise.sh && \ chown codecov:application /home/api -FROM alpine:3.13 +FROM alpine:3.14 RUN addgroup -S application && adduser -S codecov -G application RUN rm -rf /home/codecov diff --git a/docker-compose.yml b/docker-compose.yml index f40f11a646..98beb0ae93 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: api: - image: codecov/api:latest + image: codecov/enterprise-api:v4.6.2 networks: - codecovapi ports: @@ -14,8 +14,8 @@ services: depends_on: - postgres - redis - entrypoint: - - ./dev.sh +# entrypoint: +# - ./api volumes: - ./:/app environment: @@ -27,7 +27,9 @@ services: networks: - codecovapi ports: - - "5432:5432" + - "5436:5432" + environment: + - POSTGRES_PASSWORD=postgres redis: image: redis:4.0-alpine volumes: diff --git a/enterprise/generate_pyinstaller_args.py b/enterprise/generate_pyinstaller_args.py index c48788b13d..ee1066beab 100644 --- a/enterprise/generate_pyinstaller_args.py +++ b/enterprise/generate_pyinstaller_args.py @@ -7,7 +7,7 @@ finder = ModuleFinder() -so_extension = ".cpython-37m-x86_64-linux-gnu.so" +so_extension = ".cpython-39-x86_64-linux-gnu.so" def get_relevant_paths(path): @@ -31,9 +31,13 @@ def get_relevant_dirs(path): def find_imported_modules(filename): - finder.run_script(filename) - for name, mod in finder.modules.items(): - yield name + try: + finder.run_script(filename) + for name, mod in finder.modules.items(): + yield name + except AttributeError: + pass + def generate_files_to_be_cythonized(): @@ -85,6 +89,7 @@ def main(): "whitenoise", "whitenoise.middleware", "graphql_api", + "graphql_api.types.enums", "legacy_migrations", "legacy_migrations.migrations", "shared.celery_config", diff --git a/enterprise/package.sh b/enterprise/package.sh index c9a321f12c..b33c9143e4 100644 --- a/enterprise/package.sh +++ b/enterprise/package.sh @@ -1,7 +1,7 @@ #!/bin/sh # Simple wrapper around pyinstaller set -e -set -x +#set -x # Generate a random key for encryption random_key=$(pwgen -s 16 1) @@ -10,13 +10,13 @@ random_key=$(pwgen -s 16 1) PATH="/pyinstaller:$PATH" args_to_use=$(cat /pyinstaller/installer_args.txt) -echo $(find . -name "*.so") +#echo $(find . -name "*.so") mkdir src echo 'true' > src/is_enterprise # # Exclude pycrypto and PyInstaller from built packages -pyinstaller -F --exclude-module pycrypto --exclude-module PyInstaller --add-data src:/src --name api --clean --key $random_key ${args_to_use} ${pyinstaller_args} /app/manage.py +pyinstaller -F --exclude-module PyInstaller --add-data src:/src --name api --clean --key ${random_key} ${args_to_use} ${pyinstaller_args} /app/manage.py cat api.spec @@ -28,4 +28,4 @@ rm -rf /hooks mv -v app/dist/* /home rm -rf /app rm -rf /pyinstaller -rm -rf /usr/local/lib/python3.8/site-packages +rm -rf /usr/local/lib/python3.9/site-packages diff --git a/graphql_api/types/enums/__init__.py b/graphql_api/types/enums/__init__.py index 29883f4391..296d7eec88 100644 --- a/graphql_api/types/enums/__init__.py +++ b/graphql_api/types/enums/__init__.py @@ -4,4 +4,6 @@ OrderingDirection, PullRequestState, RepositoryOrdering, + TypeProjectOnboarding, + GoalOnboarding ) diff --git a/prod.sh b/prod.sh index 82ca75c30b..946838612e 100755 --- a/prod.sh +++ b/prod.sh @@ -8,7 +8,7 @@ if [ -f "/usr/local/bin/berglas" ]; then fi if [ $ELASTIC_APM_ENABLED ]; then - $prefix gunicorn codecov.wsgi:application --workers=2 --bind 0.0.0.0:8000 --access-logfile '-' --statsd-host ${STATSD_HOST}:${STATSD_PORT} + $prefix gunicorn codecov.wsgi:application --workers=${GUNICORN_WORKERS:-3} --threads=${GUNICORN_THREADS:-6} --bind 0.0.0.0:8000 --access-logfile '-' --statsd-host ${STATSD_HOST}:${STATSD_PORT} else - $prefix ddtrace-run gunicorn codecov.wsgi:application --workers=2 --bind 0.0.0.0:8000 --access-logfile '-' --statsd-host ${STATSD_HOST}:${STATSD_PORT} + $prefix ddtrace-run gunicorn codecov.wsgi:application --workers=${GUNICORN_WORKERS:-3} --threads=${GUNICORN_THREADS:-6} --bind 0.0.0.0:8000 --access-logfile '-' --statsd-host ${STATSD_HOST}:${STATSD_PORT} fi diff --git a/staging.sh b/staging.sh index 1f41ef24e3..4293110850 100644 --- a/staging.sh +++ b/staging.sh @@ -12,8 +12,5 @@ suffix="" if [[ "$STATSD_HOST" ]]; then suffix="--statsd-host ${STATSD_HOST}:${STATSD_PORT}" fi -if [ $ELASTIC_APM_ENABLED ]; then - $prefix gunicorn codecov.wsgi:application --reload --bind 0.0.0.0:8000 --access-logfile '-' $suffix -else - $prefix ddtrace-run gunicorn codecov.wsgi:application --reload --bind 0.0.0.0:8000 --access-logfile '-' $suffix -fi \ No newline at end of file + +$prefix gunicorn codecov.wsgi:application --reload --workers=${GUNICORN_WORKERS:-3} --threads=${GUNICORN_THREADS:-6} --worker-class=gthread --bind 0.0.0.0:8000 --access-logfile '-' $suffix