diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 71273aaa2a..c1bc950eaf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -97,6 +97,19 @@ deploy: paths: - terraform/plan.json +import: + extends: .base_on_push + stage: deploy + # The 1000G snapshot on `anvildev` takes about 3.5 minutes to import. There + # are currently 257 snapshots on `anvilprod`. 257 * 3.5 / 60 = 14.99 hours, + # which we choose to double. + timeout: 30h + needs: + - build_image + - deploy + script: + - make import + deploy_browser: extends: .base_on_push stage: deploy diff --git a/Makefile b/Makefile index 717574bc3e..1804fdf3fd 100644 --- a/Makefile +++ b/Makefile @@ -102,12 +102,16 @@ $(1)terraform: lambdas .PHONY: $(1)deploy $(1)deploy: check_python $(1)terraform - python $(project_root)/scripts/post_deploy_tdr.py endef $(eval $(call deploy,)) $(eval $(call deploy,auto_)) +.PHONY: import +import: check_python + python $(project_root)/scripts/reindex.py --import --sources "tdr:parquet:gcp:${GOOGLE_PROJECT}:*" + python $(project_root)/scripts/verify_tdr_sources.py + .PHONY: destroy destroy: $(MAKE) -C terraform destroy diff --git a/bin/wheels/runtime/idna-3.10-py3-none-any.whl b/bin/wheels/runtime/idna-3.10-py3-none-any.whl new file mode 100644 index 0000000000..52759bdd23 Binary files /dev/null and b/bin/wheels/runtime/idna-3.10-py3-none-any.whl differ diff --git a/bin/wheels/runtime/idna-3.8-py3-none-any.whl b/bin/wheels/runtime/idna-3.8-py3-none-any.whl deleted file mode 100644 index 84d04cdf9f..0000000000 Binary files a/bin/wheels/runtime/idna-3.8-py3-none-any.whl and /dev/null differ diff --git a/bin/wheels/runtime/pyasn1-0.6.0-py2.py3-none-any.whl b/bin/wheels/runtime/pyasn1-0.6.0-py2.py3-none-any.whl deleted file mode 100644 index a6a9380b06..0000000000 Binary files a/bin/wheels/runtime/pyasn1-0.6.0-py2.py3-none-any.whl and /dev/null differ diff --git a/bin/wheels/runtime/pyasn1-0.6.1-py3-none-any.whl b/bin/wheels/runtime/pyasn1-0.6.1-py3-none-any.whl new file mode 100644 index 0000000000..eef3fa5340 Binary files /dev/null and b/bin/wheels/runtime/pyasn1-0.6.1-py3-none-any.whl differ diff --git a/bin/wheels/runtime/pyasn1_modules-0.4.0-py3-none-any.whl b/bin/wheels/runtime/pyasn1_modules-0.4.0-py3-none-any.whl deleted file mode 100644 index c719fed167..0000000000 Binary files a/bin/wheels/runtime/pyasn1_modules-0.4.0-py3-none-any.whl and /dev/null differ diff --git a/bin/wheels/runtime/pyasn1_modules-0.4.1-py3-none-any.whl b/bin/wheels/runtime/pyasn1_modules-0.4.1-py3-none-any.whl new file mode 100644 index 0000000000..cddf2d6e2c Binary files /dev/null and b/bin/wheels/runtime/pyasn1_modules-0.4.1-py3-none-any.whl differ diff --git a/bin/wheels/runtime/pytz-2024.1-py2.py3-none-any.whl b/bin/wheels/runtime/pytz-2024.1-py2.py3-none-any.whl deleted file mode 100644 index 571f586a0d..0000000000 Binary files a/bin/wheels/runtime/pytz-2024.1-py2.py3-none-any.whl and /dev/null differ diff --git a/bin/wheels/runtime/pytz-2024.2-py2.py3-none-any.whl b/bin/wheels/runtime/pytz-2024.2-py2.py3-none-any.whl new file mode 100644 index 0000000000..b4644561e8 Binary files /dev/null and b/bin/wheels/runtime/pytz-2024.2-py2.py3-none-any.whl differ diff --git a/requirements.all.txt b/requirements.all.txt index faca0ab3cc..3156b73bf4 100644 --- a/requirements.all.txt +++ b/requirements.all.txt @@ -10,7 +10,7 @@ blinker==1.8.2 boto3==1.28.63 boto3-stubs==1.28.63 botocore==1.31.63 -botocore-stubs==1.35.15 +botocore-stubs==1.35.22 brotli==1.1.0 cachetools==5.5.0 certifi==2024.8.30 @@ -50,13 +50,13 @@ google-cloud-storage==2.12.0 google-crc32c==1.6.0 google-resumable-media==2.7.2 googleapis-common-protos==1.65.0 -greenlet==3.0.3 +greenlet==3.1.0 grpcio==1.66.1 grpcio-status==1.62.3 http-message-signatures==0.4.4 http_sfv==0.9.9 httplib2==0.22.0 -idna==3.8 +idna==3.10 importlib-resources==5.13.0 inquirer==2.10.1 itsdangerous==2.2.0 @@ -94,8 +94,8 @@ proto-plus==1.24.0 protobuf==4.25.4 psutil==6.0.0 py-partiql-parser==0.3.3 -pyasn1==0.6.0 -pyasn1_modules==0.4.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.1 pycodestyle==2.9.1 pycparser==2.22 pyflakes==2.5.0 @@ -109,7 +109,7 @@ python-dateutil==2.9.0.post0 python-dxf==11.4.0 python-editor==1.0.4 python-gitlab==3.13.0 -pytz==2024.1 +pytz==2024.2 pyyaml==6.0.1 pyzmq==26.2.0 readchar==4.2.0 diff --git a/requirements.dev.trans.txt b/requirements.dev.trans.txt index 23de37512c..b83c416189 100644 --- a/requirements.dev.trans.txt +++ b/requirements.dev.trans.txt @@ -1,6 +1,6 @@ blessed==1.20.0 blinker==1.8.2 -botocore-stubs==1.35.15 +botocore-stubs==1.35.22 brotli==1.1.0 click==8.1.7 colorama==0.4.4 @@ -13,7 +13,7 @@ flask-cors==5.0.0 geventhttpclient==2.3.1 gitdb==4.0.11 google-auth-httplib2==0.2.0 -greenlet==3.0.3 +greenlet==3.1.0 httplib2==0.22.0 importlib-resources==5.13.0 inquirer==2.10.1 diff --git a/requirements.trans.txt b/requirements.trans.txt index 69273cf90f..f2f8a1db1a 100644 --- a/requirements.trans.txt +++ b/requirements.trans.txt @@ -12,18 +12,18 @@ googleapis-common-protos==1.65.0 grpcio==1.66.1 grpcio-status==1.62.3 http_sfv==0.9.9 -idna==3.8 +idna==3.10 markupsafe==2.1.5 orderedmultidict==1.0.1 packaging==24.1 proto-plus==1.24.0 protobuf==4.25.4 -pyasn1==0.6.0 -pyasn1_modules==0.4.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.1 pycparser==2.22 pyopenssl==24.2.1 python-dateutil==2.9.0.post0 -pytz==2024.1 +pytz==2024.2 s3transfer==0.7.0 setuptools-scm==5.0.2 six==1.16.0 diff --git a/scripts/post_deploy_tdr.py b/scripts/verify_tdr_sources.py similarity index 100% rename from scripts/post_deploy_tdr.py rename to scripts/verify_tdr_sources.py