diff --git a/.env_simple b/.env_simple index ab1e83e5..50b94b06 100644 --- a/.env_simple +++ b/.env_simple @@ -14,7 +14,6 @@ POSTGRES_USER=opendatacubeusername SERVER_DB_USERNAME=opendatacubeusername POSTGRES_PASSWORD=opendatacubepassword POSTGRES_DB="odc_postgres,odc_postgis" -READY_PROBE_DB=odc_postgis ################# # OWS CFG Config diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ea0b0cc3..9d1d7bd5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,16 +55,15 @@ jobs: - name: Test and lint dev OWS image run: | mkdir artifacts - docker run -e LOCAL_UID=$(id -u $USER) -e LOCAL_GID=$(id -g $USER) -v ${PWD}/artifacts:/mnt/artifacts ${ORG}/${IMAGE}:_builder /bin/sh -c "cd /code;./check-code.sh" + chmod 777 artifacts + docker run -e LOCAL_UID=1000 -e LOCAL_GID=1000 -u ubuntu -v ${PWD}/artifacts:/mnt/artifacts ${ORG}/${IMAGE}:_builder /bin/sh -c "cd /code && ./check-code.sh" mv ./artifacts/coverage.xml ./artifacts/coverage-unit.xml - name: Dockerized Integration Pytest run: | - export LOCAL_UID=$(id -u $USER) - export LOCAL_GID=$(id -g $USER) export $(grep -v '^#' .env_simple | xargs) docker compose -f docker-compose.yaml -f docker-compose.db.yaml up -d --wait --build - docker compose -f docker-compose.yaml -f docker-compose.db.yaml exec -T ows_18 /bin/sh -c "cd /code && ./check-code-all.sh" + docker compose -f docker-compose.yaml -f docker-compose.db.yaml exec -u ubuntu -T ows /bin/sh -c "cd /code && ./check-code-all.sh" docker compose -f docker-compose.yaml -f docker-compose.db.yaml down - name: Upload All coverage to Codecov diff --git a/Dockerfile b/Dockerfile index bf69f07b..8ad8061d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,6 +69,7 @@ RUN EXTRAS=$([ "$ENVIRONMENT" = "deployment" ] || echo ",test") && \ python3-pip)) # Configure user +USER ubuntu WORKDIR "/home/ubuntu" ENV GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR" \ diff --git a/build-test-db.sh b/build-test-db.sh new file mode 100755 index 00000000..e2df9a97 --- /dev/null +++ b/build-test-db.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Convenience script for running Travis-like checks. +set -ex + +# ensure db is ready +sh ./docker/ows/wait-for-db + +# Initialise ODC schemas + +datacube system init +datacube -E owspostgis system init + +# Add extended metadata types + +datacube metadata add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_landsat_ard.odc-type.yaml +datacube metadata add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_sentinel_ard.odc-type.yaml + +datacube -E owspostgis metadata add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_landsat_ard.odc-type.yaml +datacube -E owspostgis metadata add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_sentinel_ard.odc-type.yaml + +# Test products +datacube product add ./integration_tests/metadata/s2_l2a_prod.yaml +datacube product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/c3/ga_s2am_ard_3.odc-product.yaml +datacube product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/c3/ga_s2bm_ard_3.odc-product.yaml +datacube product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/land_and_vegetation/c3_fc/ga_ls_fc_3.odc-product.yaml + +datacube -E owspostgis product add ./integration_tests/metadata/s2_l2a_prod.yaml +datacube -E owspostgis product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/c3/ga_s2am_ard_3.odc-product.yaml +datacube -E owspostgis product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/c3/ga_s2bm_ard_3.odc-product.yaml +datacube -E owspostgis product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/land_and_vegetation/c3_fc/ga_ls_fc_3.odc-product.yaml + +# add flag masking products +datacube product add ./integration_tests/metadata/product_geodata_coast_100k.yaml +datacube product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/inland_water/c3_wo/ga_ls_wo_3.odc-product.yaml + +datacube -E owspostgis product add ./integration_tests/metadata/product_geodata_coast_100k.yaml +datacube -E owspostgis product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/inland_water/c3_wo/ga_ls_wo_3.odc-product.yaml + +# Geomedian for summary product testing + +datacube product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/geomedian-au/ga_ls8c_nbart_gm_cyear_3.odc-product.yaml +datacube -E owspostgis product add https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/geomedian-au/ga_ls8c_nbart_gm_cyear_3.odc-product.yaml + +# S2 datasets from us-west-2 and eo3ified geodata_coast +MDL=./integration_tests/metadata +python ${MDL}/metadata_importer.py <3.0.0 +psycopg2 +python_dateutil +pytz +rasterio>=1.3.2 +regex +timezonefinder +python_slugify +geoalchemy2 +lark +xarray +pyows +prometheus_flask_exporter +setuptools_scm +pytest +pytest_cov +pytest_localserver +owslib>0.29.2 +pytest_mock +pep8 +pytest-helpers-namespace +flask-cors +fsspec +pydevd-pycharm~=221.5921.27 +pylint>=3.0 +sphinx_click +pre-commit +pipdeptree +gunicorn +gunicorn[gevent] +gevent +prometheus_client +sentry_sdk +prometheus_flask_exporter +blinker diff --git a/constraints.txt b/constraints.txt new file mode 100644 index 00000000..183c4bcc --- /dev/null +++ b/constraints.txt @@ -0,0 +1,444 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --strip-extras constraints.in +# +affine==2.4.0 + # via + # -r constraints.in + # datacube + # odc-geo + # rasterio +alabaster==0.7.16 + # via sphinx +alembic==1.13.1 + # via datacube +astroid==3.1.0 + # via pylint +attrs==23.2.0 + # via + # datacube + # jsonschema + # rasterio + # referencing +babel==2.14.0 + # via + # -r constraints.in + # flask-babel + # sphinx +blinker==1.7.0 + # via + # -r constraints.in + # flask +boto3==1.34.78 + # via datacube +botocore==1.34.78 + # via + # boto3 + # datacube + # s3transfer +bottleneck==1.3.8 + # via datacube +cachetools==5.3.3 + # via + # datacube + # odc-geo +certifi==2024.2.2 + # via + # pyproj + # rasterio + # requests + # sentry-sdk +cffi==1.16.0 + # via timezonefinder +cfgv==3.4.0 + # via pre-commit +charset-normalizer==3.3.2 + # via requests +ciso8601==2.3.1 + # via datacube +click==8.1.7 + # via + # -r constraints.in + # click-plugins + # cligj + # dask + # datacube + # distributed + # flask + # rasterio + # sphinx-click +click-plugins==1.1.1 + # via rasterio +cligj==0.7.2 + # via rasterio +cloudpickle==3.0.0 + # via + # dask + # datacube + # distributed +colour==0.1.5 + # via -r constraints.in +contourpy==1.2.1 + # via matplotlib +coverage==7.4.4 + # via pytest-cov +cycler==0.12.1 + # via matplotlib +dask==2024.4.1 + # via + # datacube + # distributed +datacube==1.9.0rc3 + # via -r constraints.in +deepdiff==6.7.1 + # via -r constraints.in +deprecat==2.1.1 + # via datacube +dill==0.3.8 + # via pylint +distlib==0.3.8 + # via virtualenv +distributed==2024.4.1 + # via datacube +docutils==0.20.1 + # via + # sphinx + # sphinx-click +filelock==3.13.3 + # via virtualenv +flask==3.0.2 + # via + # -r constraints.in + # flask-babel + # flask-cors + # prometheus-flask-exporter +flask-babel==4.0.0 + # via -r constraints.in +flask-cors==4.0.0 + # via -r constraints.in +fonttools==4.50.0 + # via matplotlib +fsspec==2024.3.1 + # via + # -r constraints.in + # dask +geoalchemy2==0.14.7 + # via + # -r constraints.in + # datacube +gevent==24.2.1 + # via + # -r constraints.in + # gunicorn +greenlet==3.0.3 + # via + # gevent + # sqlalchemy +gunicorn==21.2.0 + # via -r constraints.in +h3==3.7.7 + # via timezonefinder +identify==2.5.35 + # via pre-commit +idna==3.6 + # via requests +imagesize==1.4.1 + # via sphinx +importlib-metadata==7.1.0 + # via dask +iniconfig==2.0.0 + # via pytest +iso8601==2.1.0 + # via pyows +isort==5.13.2 + # via pylint +itsdangerous==2.1.2 + # via flask +jinja2==3.1.3 + # via + # distributed + # flask + # flask-babel + # sphinx +jmespath==1.0.1 + # via + # boto3 + # botocore +jsonschema==4.21.1 + # via datacube +jsonschema-specifications==2023.12.1 + # via jsonschema +kiwisolver==1.4.5 + # via matplotlib +lark==1.1.9 + # via + # -r constraints.in + # datacube +locket==1.0.0 + # via + # distributed + # partd +lxml==5.2.1 + # via + # -r constraints.in + # owslib + # pyows +mako==1.3.2 + # via alembic +markupsafe==2.1.5 + # via + # jinja2 + # mako + # werkzeug +matplotlib==3.8.4 + # via -r constraints.in +mccabe==0.7.0 + # via pylint +msgpack==1.0.8 + # via distributed +nodeenv==1.8.0 + # via pre-commit +numpy==1.26.4 + # via + # -r constraints.in + # bottleneck + # contourpy + # dask + # datacube + # matplotlib + # odc-geo + # pandas + # rasterio + # scipy + # shapely + # snuggs + # timezonefinder + # xarray +odc-geo==0.4.3 + # via datacube +ordered-set==4.1.0 + # via deepdiff +owslib==0.30.0 + # via -r constraints.in +packaging==24.0 + # via + # dask + # datacube + # distributed + # geoalchemy2 + # gunicorn + # matplotlib + # pipdeptree + # pytest + # setuptools-scm + # sphinx + # xarray +pandas==2.2.1 + # via + # datacube + # xarray +partd==1.4.1 + # via dask +pep8==1.7.1 + # via -r constraints.in +pillow==10.3.0 + # via + # -r constraints.in + # matplotlib +pipdeptree==2.17.0 + # via -r constraints.in +platformdirs==4.2.0 + # via + # pylint + # virtualenv +pluggy==1.4.0 + # via pytest +pre-commit==3.7.0 + # via -r constraints.in +prometheus-client==0.20.0 + # via + # -r constraints.in + # prometheus-flask-exporter +prometheus-flask-exporter==0.23.0 + # via -r constraints.in +psutil==5.9.8 + # via distributed +psycopg2==2.9.9 + # via + # -r constraints.in + # datacube +pycparser==2.22 + # via cffi +pydevd-pycharm==221.5921.27 + # via -r constraints.in +pygments==2.17.2 + # via sphinx +pylint==3.1.0 + # via -r constraints.in +pyows==0.2.7 + # via -r constraints.in +pyparsing==3.1.2 + # via + # -r constraints.in + # matplotlib + # snuggs +pyproj==3.6.1 + # via + # datacube + # odc-geo +pytest==8.1.1 + # via + # -r constraints.in + # pytest-cov + # pytest-helpers-namespace + # pytest-mock +pytest-cov==5.0.0 + # via -r constraints.in +pytest-helpers-namespace==2021.12.29 + # via -r constraints.in +pytest-localserver==0.8.1 + # via -r constraints.in +pytest-mock==3.14.0 + # via -r constraints.in +python-dateutil==2.9.0.post0 + # via + # -r constraints.in + # botocore + # datacube + # matplotlib + # owslib + # pandas +python-slugify==8.0.4 + # via -r constraints.in +pytz==2024.1 + # via + # -r constraints.in + # flask-babel + # owslib + # pandas +pyyaml==6.0.1 + # via + # dask + # datacube + # distributed + # owslib + # pre-commit +rasterio==1.3.9 + # via + # -r constraints.in + # datacube +referencing==0.34.0 + # via + # jsonschema + # jsonschema-specifications +regex==2023.12.25 + # via -r constraints.in +requests==2.31.0 + # via + # -r constraints.in + # owslib + # sphinx +rpds-py==0.18.0 + # via + # jsonschema + # referencing +ruamel-yaml==0.18.6 + # via datacube +ruamel-yaml-clib==0.2.8 + # via ruamel-yaml +s3transfer==0.10.1 + # via boto3 +scipy==1.13.0 + # via -r constraints.in +sentry-sdk==1.44.1 + # via -r constraints.in +setuptools-scm==8.0.4 + # via -r constraints.in +shapely==2.0.3 + # via + # datacube + # odc-geo +six==1.16.0 + # via python-dateutil +snowballstemmer==2.2.0 + # via sphinx +snuggs==1.4.7 + # via rasterio +sortedcontainers==2.4.0 + # via distributed +sphinx==7.2.6 + # via sphinx-click +sphinx-click==5.1.0 + # via -r constraints.in +sphinxcontrib-applehelp==1.0.8 + # via sphinx +sphinxcontrib-devhelp==1.0.6 + # via sphinx +sphinxcontrib-htmlhelp==2.0.5 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.7 + # via sphinx +sphinxcontrib-serializinghtml==1.1.10 + # via sphinx +sqlalchemy==2.0.29 + # via + # alembic + # datacube + # geoalchemy2 +tblib==3.0.0 + # via distributed +text-unidecode==1.3 + # via python-slugify +timezonefinder==6.5.0 + # via -r constraints.in +tomlkit==0.12.4 + # via pylint +toolz==0.12.1 + # via + # dask + # datacube + # distributed + # partd +tornado==6.4 + # via distributed +typing-extensions==4.10.0 + # via + # alembic + # setuptools-scm + # sqlalchemy +tzdata==2024.1 + # via pandas +urllib3==2.2.1 + # via + # botocore + # distributed + # requests + # sentry-sdk +virtualenv==20.25.1 + # via pre-commit +werkzeug==3.0.2 + # via + # flask + # pytest-localserver +wrapt==1.16.0 + # via deprecat +xarray==2024.3.0 + # via + # -r constraints.in + # datacube +zict==3.0.0 + # via distributed +zipp==3.18.1 + # via importlib-metadata +zope-event==5.0 + # via gevent +zope-interface==6.2 + # via gevent + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/dep.out b/dep.out new file mode 100644 index 00000000..19147093 --- /dev/null +++ b/dep.out @@ -0,0 +1,359 @@ +alembic==1.13.2 +├── Mako [required: Any, installed: 1.3.5] +│ └── MarkupSafe [required: >=0.9.2, installed: 2.1.5] +├── SQLAlchemy [required: >=1.3.0, installed: 1.4.53] +│ └── greenlet [required: !=0.4.17, installed: 3.0.3] +└── typing_extensions [required: >=4, installed: 4.12.2] +boto3==1.34.151 +├── botocore [required: >=1.34.151,<1.35.0, installed: 1.34.151] +│ ├── jmespath [required: >=0.7.1,<2.0.0, installed: 1.0.1] +│ ├── python-dateutil [required: >=2.1,<3.0.0, installed: 2.9.0.post0] +│ │ └── six [required: >=1.5, installed: 1.16.0] +│ └── urllib3 [required: >=1.25.4,<3,!=2.2.0, installed: 2.2.2] +├── jmespath [required: >=0.7.1,<2.0.0, installed: 1.0.1] +└── s3transfer [required: >=0.10.0,<0.11.0, installed: 0.10.2] + └── botocore [required: >=1.33.2,<2.0a.0, installed: 1.34.151] + ├── jmespath [required: >=0.7.1,<2.0.0, installed: 1.0.1] + ├── python-dateutil [required: >=2.1,<3.0.0, installed: 2.9.0.post0] + │ └── six [required: >=1.5, installed: 1.16.0] + └── urllib3 [required: >=1.25.4,<3,!=2.2.0, installed: 2.2.2] +Bottleneck==1.4.0 +└── numpy [required: Any, installed: 2.0.1] +ciso8601==2.3.1 +datacube_ows==1.8.40.dev61+g0bdeeca +├── affine [required: Any, installed: 2.4.0] +├── Babel [required: Any, installed: 2.15.0] +├── click [required: Any, installed: 8.1.7] +├── colour [required: Any, installed: 0.1.5] +├── datacube [required: >=1.9.0-rc9, installed: 1.8.19] +│ ├── affine [required: Any, installed: 2.4.0] +│ ├── attrs [required: >=18.1, installed: 23.2.0] +│ ├── cachetools [required: Any, installed: 5.4.0] +│ ├── click [required: >=5.0, installed: 8.1.7] +│ ├── cloudpickle [required: >=0.4, installed: 3.0.0] +│ ├── dask [required: Any, installed: 2024.7.1] +│ │ ├── click [required: >=8.1, installed: 8.1.7] +│ │ ├── cloudpickle [required: >=1.5.0, installed: 3.0.0] +│ │ ├── fsspec [required: >=2021.09.0, installed: 2024.6.1] +│ │ ├── importlib_metadata [required: >=4.13.0, installed: 8.2.0] +│ │ │ └── zipp [required: >=0.5, installed: 3.19.2] +│ │ ├── packaging [required: >=20.0, installed: 24.1] +│ │ ├── partd [required: >=1.4.0, installed: 1.4.2] +│ │ │ ├── locket [required: Any, installed: 1.0.0] +│ │ │ └── toolz [required: Any, installed: 0.12.1] +│ │ ├── PyYAML [required: >=5.3.1, installed: 6.0.1] +│ │ └── toolz [required: >=0.10.0, installed: 0.12.1] +│ ├── deprecat [required: Any, installed: 2.1.3] +│ │ └── wrapt [required: >=1.10,<2, installed: 1.16.0] +│ ├── distributed [required: Any, installed: 2024.7.1] +│ │ ├── click [required: >=8.0, installed: 8.1.7] +│ │ ├── cloudpickle [required: >=1.5.0, installed: 3.0.0] +│ │ ├── dask [required: ==2024.7.1, installed: 2024.7.1] +│ │ │ ├── click [required: >=8.1, installed: 8.1.7] +│ │ │ ├── cloudpickle [required: >=1.5.0, installed: 3.0.0] +│ │ │ ├── fsspec [required: >=2021.09.0, installed: 2024.6.1] +│ │ │ ├── importlib_metadata [required: >=4.13.0, installed: 8.2.0] +│ │ │ │ └── zipp [required: >=0.5, installed: 3.19.2] +│ │ │ ├── packaging [required: >=20.0, installed: 24.1] +│ │ │ ├── partd [required: >=1.4.0, installed: 1.4.2] +│ │ │ │ ├── locket [required: Any, installed: 1.0.0] +│ │ │ │ └── toolz [required: Any, installed: 0.12.1] +│ │ │ ├── PyYAML [required: >=5.3.1, installed: 6.0.1] +│ │ │ └── toolz [required: >=0.10.0, installed: 0.12.1] +│ │ ├── Jinja2 [required: >=2.10.3, installed: 3.1.4] +│ │ │ └── MarkupSafe [required: >=2.0, installed: 2.1.5] +│ │ ├── locket [required: >=1.0.0, installed: 1.0.0] +│ │ ├── msgpack [required: >=1.0.0, installed: 1.0.8] +│ │ ├── packaging [required: >=20.0, installed: 24.1] +│ │ ├── psutil [required: >=5.7.2, installed: 6.0.0] +│ │ ├── PyYAML [required: >=5.3.1, installed: 6.0.1] +│ │ ├── sortedcontainers [required: >=2.0.5, installed: 2.4.0] +│ │ ├── tblib [required: >=1.6.0, installed: 3.0.0] +│ │ ├── toolz [required: >=0.10.0, installed: 0.12.1] +│ │ ├── tornado [required: >=6.0.4, installed: 6.4.1] +│ │ ├── urllib3 [required: >=1.24.3, installed: 2.2.2] +│ │ └── zict [required: >=3.0.0, installed: 3.0.0] +│ ├── GeoAlchemy2 [required: Any, installed: 0.15.2] +│ │ ├── packaging [required: Any, installed: 24.1] +│ │ └── SQLAlchemy [required: >=1.4, installed: 1.4.53] +│ │ └── greenlet [required: !=0.4.17, installed: 3.0.3] +│ ├── jsonschema [required: >=4.18, installed: 4.23.0] +│ │ ├── attrs [required: >=22.2.0, installed: 23.2.0] +│ │ ├── jsonschema-specifications [required: >=2023.03.6, installed: 2023.12.1] +│ │ │ └── referencing [required: >=0.31.0, installed: 0.35.1] +│ │ │ ├── attrs [required: >=22.2.0, installed: 23.2.0] +│ │ │ └── rpds-py [required: >=0.7.0, installed: 0.19.1] +│ │ ├── referencing [required: >=0.28.4, installed: 0.35.1] +│ │ │ ├── attrs [required: >=22.2.0, installed: 23.2.0] +│ │ │ └── rpds-py [required: >=0.7.0, installed: 0.19.1] +│ │ └── rpds-py [required: >=0.7.1, installed: 0.19.1] +│ ├── lark [required: Any, installed: 1.1.9] +│ ├── netCDF4 [required: Any, installed: 1.7.1.post1] +│ │ ├── certifi [required: Any, installed: 2024.7.4] +│ │ ├── cftime [required: Any, installed: 1.6.4] +│ │ │ └── numpy [required: >1.13.3, installed: 2.0.1] +│ │ └── numpy [required: Any, installed: 2.0.1] +│ ├── numpy [required: Any, installed: 2.0.1] +│ ├── packaging [required: Any, installed: 24.1] +│ ├── pandas [required: Any, installed: 2.2.2] +│ │ ├── numpy [required: >=1.23.2, installed: 2.0.1] +│ │ ├── python-dateutil [required: >=2.8.2, installed: 2.9.0.post0] +│ │ │ └── six [required: >=1.5, installed: 1.16.0] +│ │ ├── pytz [required: >=2020.1, installed: 2024.1] +│ │ └── tzdata [required: >=2022.7, installed: 2024.1] +│ ├── psycopg2 [required: Any, installed: 2.9.9] +│ ├── pyproj [required: >=2.5, installed: 3.6.1] +│ │ └── certifi [required: Any, installed: 2024.7.4] +│ ├── python-dateutil [required: Any, installed: 2.9.0.post0] +│ │ └── six [required: >=1.5, installed: 1.16.0] +│ ├── PyYAML [required: Any, installed: 6.0.1] +│ ├── rasterio [required: >=1.3.2, installed: 1.3.10] +│ │ ├── affine [required: Any, installed: 2.4.0] +│ │ ├── attrs [required: Any, installed: 23.2.0] +│ │ ├── certifi [required: Any, installed: 2024.7.4] +│ │ ├── click [required: >=4.0, installed: 8.1.7] +│ │ ├── click-plugins [required: Any, installed: 1.1.1] +│ │ │ └── click [required: >=4.0, installed: 8.1.7] +│ │ ├── cligj [required: >=0.5, installed: 0.7.2] +│ │ │ └── click [required: >=4.0, installed: 8.1.7] +│ │ ├── numpy [required: Any, installed: 2.0.1] +│ │ ├── setuptools [required: Any, installed: 66.1.1] +│ │ └── snuggs [required: >=1.4.1, installed: 1.4.7] +│ │ ├── numpy [required: Any, installed: 2.0.1] +│ │ └── pyparsing [required: >=2.1.6, installed: 3.1.2] +│ ├── ruamel.yaml [required: Any, installed: 0.18.6] +│ │ └── ruamel.yaml.clib [required: >=0.2.7, installed: 0.2.8] +│ ├── shapely [required: >=2.0, installed: 2.0.5] +│ │ └── numpy [required: >=1.14,<3, installed: 2.0.1] +│ ├── SQLAlchemy [required: >=1.4,<2.0, installed: 1.4.53] +│ │ └── greenlet [required: !=0.4.17, installed: 3.0.3] +│ ├── toolz [required: Any, installed: 0.12.1] +│ └── xarray [required: >=0.9, installed: 2024.7.0] +│ ├── numpy [required: >=1.23, installed: 2.0.1] +│ ├── packaging [required: >=23.1, installed: 24.1] +│ └── pandas [required: >=2.0, installed: 2.2.2] +│ ├── numpy [required: >=1.23.2, installed: 2.0.1] +│ ├── python-dateutil [required: >=2.8.2, installed: 2.9.0.post0] +│ │ └── six [required: >=1.5, installed: 1.16.0] +│ ├── pytz [required: >=2020.1, installed: 2024.1] +│ └── tzdata [required: >=2022.7, installed: 2024.1] +├── deepdiff [required: Any, installed: 7.0.1] +│ └── ordered-set [required: >=4.1.0,<4.2.0, installed: 4.1.0] +├── Flask [required: Any, installed: 3.0.3] +│ ├── blinker [required: >=1.6.2, installed: 1.8.2] +│ ├── click [required: >=8.1.3, installed: 8.1.7] +│ ├── itsdangerous [required: >=2.1.2, installed: 2.2.0] +│ ├── Jinja2 [required: >=3.1.2, installed: 3.1.4] +│ │ └── MarkupSafe [required: >=2.0, installed: 2.1.5] +│ └── Werkzeug [required: >=3.0.0, installed: 3.0.3] +│ └── MarkupSafe [required: >=2.1.1, installed: 2.1.5] +├── flask-babel [required: >3.0.0, installed: 4.0.0] +│ ├── Babel [required: >=2.12, installed: 2.15.0] +│ ├── Flask [required: >=2.0, installed: 3.0.3] +│ │ ├── blinker [required: >=1.6.2, installed: 1.8.2] +│ │ ├── click [required: >=8.1.3, installed: 8.1.7] +│ │ ├── itsdangerous [required: >=2.1.2, installed: 2.2.0] +│ │ ├── Jinja2 [required: >=3.1.2, installed: 3.1.4] +│ │ │ └── MarkupSafe [required: >=2.0, installed: 2.1.5] +│ │ └── Werkzeug [required: >=3.0.0, installed: 3.0.3] +│ │ └── MarkupSafe [required: >=2.1.1, installed: 2.1.5] +│ ├── Jinja2 [required: >=3.1, installed: 3.1.4] +│ │ └── MarkupSafe [required: >=2.0, installed: 2.1.5] +│ └── pytz [required: >=2022.7, installed: 2024.1] +├── fsspec [required: Any, installed: 2024.6.1] +├── GeoAlchemy2 [required: Any, installed: 0.15.2] +│ ├── packaging [required: Any, installed: 24.1] +│ └── SQLAlchemy [required: >=1.4, installed: 1.4.53] +│ └── greenlet [required: !=0.4.17, installed: 3.0.3] +├── lark [required: Any, installed: 1.1.9] +├── lxml [required: Any, installed: 5.2.2] +├── matplotlib [required: Any, installed: 3.9.1] +│ ├── contourpy [required: >=1.0.1, installed: 1.2.1] +│ │ └── numpy [required: >=1.20, installed: 2.0.1] +│ ├── cycler [required: >=0.10, installed: 0.12.1] +│ ├── fonttools [required: >=4.22.0, installed: 4.53.1] +│ ├── kiwisolver [required: >=1.3.1, installed: 1.4.5] +│ ├── numpy [required: >=1.23, installed: 2.0.1] +│ ├── packaging [required: >=20.0, installed: 24.1] +│ ├── pillow [required: >=8, installed: 10.4.0] +│ ├── pyparsing [required: >=2.3.1, installed: 3.1.2] +│ └── python-dateutil [required: >=2.7, installed: 2.9.0.post0] +│ └── six [required: >=1.5, installed: 1.16.0] +├── numpy [required: >=1.22, installed: 2.0.1] +├── pillow [required: >=10.2.0, installed: 10.4.0] +├── prometheus_flask_exporter [required: Any, installed: 0.23.1] +│ ├── Flask [required: Any, installed: 3.0.3] +│ │ ├── blinker [required: >=1.6.2, installed: 1.8.2] +│ │ ├── click [required: >=8.1.3, installed: 8.1.7] +│ │ ├── itsdangerous [required: >=2.1.2, installed: 2.2.0] +│ │ ├── Jinja2 [required: >=3.1.2, installed: 3.1.4] +│ │ │ └── MarkupSafe [required: >=2.0, installed: 2.1.5] +│ │ └── Werkzeug [required: >=3.0.0, installed: 3.0.3] +│ │ └── MarkupSafe [required: >=2.1.1, installed: 2.1.5] +│ └── prometheus_client [required: Any, installed: 0.20.0] +├── psycopg2 [required: Any, installed: 2.9.9] +├── pyows [required: Any, installed: 0.3.1] +│ ├── iso8601 [required: Any, installed: 2.1.0] +│ └── lxml [required: Any, installed: 5.2.2] +├── pyparsing [required: Any, installed: 3.1.2] +├── python-dateutil [required: Any, installed: 2.9.0.post0] +│ └── six [required: >=1.5, installed: 1.16.0] +├── python-slugify [required: Any, installed: 8.0.4] +│ └── text-unidecode [required: >=1.3, installed: 1.3] +├── pytz [required: Any, installed: 2024.1] +├── rasterio [required: >=1.3.2, installed: 1.3.10] +│ ├── affine [required: Any, installed: 2.4.0] +│ ├── attrs [required: Any, installed: 23.2.0] +│ ├── certifi [required: Any, installed: 2024.7.4] +│ ├── click [required: >=4.0, installed: 8.1.7] +│ ├── click-plugins [required: Any, installed: 1.1.1] +│ │ └── click [required: >=4.0, installed: 8.1.7] +│ ├── cligj [required: >=0.5, installed: 0.7.2] +│ │ └── click [required: >=4.0, installed: 8.1.7] +│ ├── numpy [required: Any, installed: 2.0.1] +│ ├── setuptools [required: Any, installed: 66.1.1] +│ └── snuggs [required: >=1.4.1, installed: 1.4.7] +│ ├── numpy [required: Any, installed: 2.0.1] +│ └── pyparsing [required: >=2.1.6, installed: 3.1.2] +├── regex [required: Any, installed: 2024.7.24] +├── requests [required: Any, installed: 2.32.3] +│ ├── certifi [required: >=2017.4.17, installed: 2024.7.4] +│ ├── charset-normalizer [required: >=2,<4, installed: 3.3.2] +│ ├── idna [required: >=2.5,<4, installed: 3.7] +│ └── urllib3 [required: >=1.21.1,<3, installed: 2.2.2] +├── scipy [required: Any, installed: 1.14.0] +│ └── numpy [required: >=1.23.5,<2.3, installed: 2.0.1] +├── setuptools-scm [required: Any, installed: 8.1.0] +│ ├── packaging [required: >=20, installed: 24.1] +│ └── setuptools [required: Any, installed: 66.1.1] +├── timezonefinder [required: Any, installed: 6.5.2] +│ ├── cffi [required: >=1.15.1,<2, installed: 1.16.0] +│ │ └── pycparser [required: Any, installed: 2.22] +│ ├── h3 [required: >=3.7.6,<4, installed: 3.7.7] +│ ├── numpy [required: >=1.23,<3, installed: 2.0.1] +│ └── setuptools [required: >=65.5, installed: 66.1.1] +└── xarray [required: Any, installed: 2024.7.0] + ├── numpy [required: >=1.23, installed: 2.0.1] + ├── packaging [required: >=23.1, installed: 24.1] + └── pandas [required: >=2.0, installed: 2.2.2] + ├── numpy [required: >=1.23.2, installed: 2.0.1] + ├── python-dateutil [required: >=2.8.2, installed: 2.9.0.post0] + │ └── six [required: >=1.5, installed: 1.16.0] + ├── pytz [required: >=2020.1, installed: 2024.1] + └── tzdata [required: >=2022.7, installed: 2024.1] +flake8==7.1.0 +├── mccabe [required: >=0.7.0,<0.8.0, installed: 0.7.0] +├── pycodestyle [required: >=2.12.0,<2.13.0, installed: 2.12.0] +└── pyflakes [required: >=3.2.0,<3.3.0, installed: 3.2.0] +Flask-Cors==4.0.1 +└── Flask [required: >=0.9, installed: 3.0.3] + ├── blinker [required: >=1.6.2, installed: 1.8.2] + ├── click [required: >=8.1.3, installed: 8.1.7] + ├── itsdangerous [required: >=2.1.2, installed: 2.2.0] + ├── Jinja2 [required: >=3.1.2, installed: 3.1.4] + │ └── MarkupSafe [required: >=2.0, installed: 2.1.5] + └── Werkzeug [required: >=3.0.0, installed: 3.0.3] + └── MarkupSafe [required: >=2.1.1, installed: 2.1.5] +gevent==24.2.1 +├── greenlet [required: >=3.0rc3, installed: 3.0.3] +├── zope.event [required: Any, installed: 5.0] +│ └── setuptools [required: Any, installed: 66.1.1] +└── zope.interface [required: Any, installed: 6.4.post2] + └── setuptools [required: Any, installed: 66.1.1] +gunicorn==22.0.0 +└── packaging [required: Any, installed: 24.1] +mypy==1.11.1 +├── mypy-extensions [required: >=1.0.0, installed: 1.0.0] +└── typing_extensions [required: >=4.6.0, installed: 4.12.2] +odc-geo==0.4.8 +├── affine [required: Any, installed: 2.4.0] +├── cachetools [required: Any, installed: 5.4.0] +├── numpy [required: Any, installed: 2.0.1] +├── pyproj [required: >=3.0.0, installed: 3.6.1] +│ └── certifi [required: Any, installed: 2024.7.4] +└── shapely [required: Any, installed: 2.0.5] + └── numpy [required: >=1.14,<3, installed: 2.0.1] +OWSLib==0.31.0 +├── lxml [required: Any, installed: 5.2.2] +├── python-dateutil [required: >=1.5, installed: 2.9.0.post0] +│ └── six [required: >=1.5, installed: 1.16.0] +├── pytz [required: Any, installed: 2024.1] +├── PyYAML [required: Any, installed: 6.0.1] +└── requests [required: >=1.0, installed: 2.32.3] + ├── certifi [required: >=2017.4.17, installed: 2024.7.4] + ├── charset-normalizer [required: >=2,<4, installed: 3.3.2] + ├── idna [required: >=2.5,<4, installed: 3.7] + └── urllib3 [required: >=1.21.1,<3, installed: 2.2.2] +pep8==1.7.1 +pipdeptree==2.23.1 +├── packaging [required: >=23.1, installed: 24.1] +└── pip [required: >=23.1.2, installed: 24.2] +pre-commit==2.13.0 +├── cfgv [required: >=2.0.0, installed: 3.4.0] +├── identify [required: >=1.0.0, installed: 2.6.0] +├── nodeenv [required: >=0.11.1, installed: 1.9.1] +├── PyYAML [required: >=5.1, installed: 6.0.1] +├── toml [required: Any, installed: 0.10.2] +└── virtualenv [required: >=20.0.8, installed: 20.26.3] + ├── distlib [required: >=0.3.7,<1, installed: 0.3.8] + ├── filelock [required: >=3.12.2,<4, installed: 3.15.4] + └── platformdirs [required: >=3.9.1,<5, installed: 4.2.2] +pydevd-pycharm==221.5921.27 +pylint==3.2.3 +├── astroid [required: >=3.2.2,<=3.3.0-dev0, installed: 3.2.4] +├── dill [required: >=0.3.6, installed: 0.3.8] +├── isort [required: >=4.2.5,<6,!=5.13.0, installed: 5.13.2] +├── mccabe [required: >=0.6,<0.8, installed: 0.7.0] +├── platformdirs [required: >=2.2.0, installed: 4.2.2] +└── tomlkit [required: >=0.10.1, installed: 0.13.0] +pytest-cov==5.0.0 +├── coverage [required: >=5.2.1, installed: 7.6.0] +└── pytest [required: >=4.6, installed: 8.3.2] + ├── iniconfig [required: Any, installed: 2.0.0] + ├── packaging [required: Any, installed: 24.1] + └── pluggy [required: >=1.5,<2, installed: 1.5.0] +pytest-helpers-namespace==2021.12.29 +└── pytest [required: >=6.0.0, installed: 8.3.2] + ├── iniconfig [required: Any, installed: 2.0.0] + ├── packaging [required: Any, installed: 24.1] + └── pluggy [required: >=1.5,<2, installed: 1.5.0] +pytest-localserver==0.8.1 +└── Werkzeug [required: >=0.10, installed: 3.0.3] + └── MarkupSafe [required: >=2.1.1, installed: 2.1.5] +pytest-mock==3.14.0 +└── pytest [required: >=6.2.5, installed: 8.3.2] + ├── iniconfig [required: Any, installed: 2.0.0] + ├── packaging [required: Any, installed: 24.1] + └── pluggy [required: >=1.5,<2, installed: 1.5.0] +sentry-sdk==2.11.0 +├── certifi [required: Any, installed: 2024.7.4] +└── urllib3 [required: >=1.26.11, installed: 2.2.2] +sphinx-click==6.0.0 +├── click [required: >=8.0, installed: 8.1.7] +├── docutils [required: Any, installed: 0.21.2] +└── Sphinx [required: >=4.0, installed: 8.0.2] + ├── alabaster [required: >=0.7.14, installed: 1.0.0] + ├── Babel [required: >=2.13, installed: 2.15.0] + ├── docutils [required: >=0.20,<0.22, installed: 0.21.2] + ├── imagesize [required: >=1.3, installed: 1.4.1] + ├── Jinja2 [required: >=3.1, installed: 3.1.4] + │ └── MarkupSafe [required: >=2.0, installed: 2.1.5] + ├── packaging [required: >=23.0, installed: 24.1] + ├── Pygments [required: >=2.17, installed: 2.18.0] + ├── requests [required: >=2.30.0, installed: 2.32.3] + │ ├── certifi [required: >=2017.4.17, installed: 2024.7.4] + │ ├── charset-normalizer [required: >=2,<4, installed: 3.3.2] + │ ├── idna [required: >=2.5,<4, installed: 3.7] + │ └── urllib3 [required: >=1.21.1,<3, installed: 2.2.2] + ├── snowballstemmer [required: >=2.2, installed: 2.2.0] + ├── sphinxcontrib-applehelp [required: Any, installed: 2.0.0] + ├── sphinxcontrib-devhelp [required: Any, installed: 2.0.0] + ├── sphinxcontrib-htmlhelp [required: >=2.0.0, installed: 2.1.0] + ├── sphinxcontrib-jsmath [required: Any, installed: 1.0.1] + ├── sphinxcontrib-qthelp [required: Any, installed: 2.0.0] + └── sphinxcontrib-serializinghtml [required: >=1.1.9, installed: 2.0.0] +types-python-dateutil==2.9.0.20240316 +types-pytz==2024.1.0.20240417 +types-requests==2.32.0.20240712 +└── urllib3 [required: >=2, installed: 2.2.2] +wheel==0.38.4 diff --git a/docker-compose.yaml b/docker-compose.yaml index 9a10d61d..9fdf0f6e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,7 +27,6 @@ services: ODC_DEFAULT_DB_URL: ${ODC_DEFAULT_DB_URL} ODC_OWSPOSTGIS_DB_URL: ${ODC_OWSPOSTGIS_DB_URL} # for wait-for-db check - READY_PROBE_DB: ${READY_PROBE_DB} POSTGRES_USER: ${POSTGRES_USER} POSTGRES_HOSTNAME: ${POSTGRES_HOSTNAME} SERVER_DB_USERNAME: ${SERVER_DB_USERNAME} diff --git a/inventory.json b/inventory.json new file mode 100644 index 00000000..af336b43 --- /dev/null +++ b/inventory.json @@ -0,0 +1,85 @@ +{ + "total_layers_count": 6, + "layers": [ + { + "layer": "s2_l2a", + "product": [ + "s2_l2a" + ], + "styles_count": 8, + "styles_list": [ + "simple_rgb", + "style_ls_simple_rgb_clone", + "infra_red", + "blue", + "ndvi", + "ndvi_expr", + "rgb_ndvi", + "ndvi_delta" + ] + }, + { + "layer": "s2_l2a_clone", + "product": [ + "s2_l2a" + ], + "styles_count": 8, + "styles_list": [ + "simple_rgb", + "style_ls_simple_rgb_clone", + "infra_red", + "blue", + "ndvi", + "ndvi_expr", + "rgb_ndvi", + "ndvi_delta" + ] + }, + { + "layer": "s2_ard_granule_nbar_t", + "product": [ + "ga_s2am_ard_3", + "ga_s2bm_ard_3" + ], + "styles_count": 2, + "styles_list": [ + "ndci", + "mndwi" + ] + }, + { + "layer": "s2_ard_latest_mosaic", + "product": [ + "ga_s2am_ard_3", + "ga_s2bm_ard_3" + ], + "styles_count": 2, + "styles_list": [ + "ndci", + "mndwi" + ] + }, + { + "layer": "ga_ls_fc_3", + "product": [ + "ga_ls_fc_3" + ], + "styles_count": 1, + "styles_list": [ + "fc_rgb_unmasked" + ] + }, + { + "layer": "ls8_geomedian", + "product": [ + "ga_ls8c_nbart_gm_cyear_3" + ], + "styles_count": 3, + "styles_list": [ + "simple_rgb", + "infra_red", + "ndvi" + ] + } + ] +} diff --git a/licenseheaders.py b/licenseheaders.py new file mode 100644 index 00000000..d63ed16c --- /dev/null +++ b/licenseheaders.py @@ -0,0 +1,1006 @@ +#!/usr/bin/env python +# encoding: utf-8 +# This file is part of datacube-ows, part of the Open Data Cube project. +# See https://opendatacube.org for more information. +# +# Copyright (c) 2017-2024 OWS Contributors +# SPDX-License-Identifier: Apache-2.0 + + + +"""A tool to change or add license headers in all supported files in or below a directory.""" + +# Copyright (c) 2016-2018 Johann Petrak +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import argparse +import fnmatch +import logging +import os +import sys +import stat +import contextlib +from shutil import copyfile +from string import Template + +import regex as re + +__version__ = '0.8.8' +__author__ = 'Johann Petrak' +__license__ = 'MIT' + +LOGGER = logging.getLogger("licenseheaders_{}".format(__version__)) + + +default_dir = "." +default_encoding = "utf-8" + +def update_c_style_comments(extensions): + return { + "extensions": extensions, + "keepFirst": None, + "blockCommentStartPattern": re.compile(r'^\s*/\*'), + "blockCommentEndPattern": re.compile(r'\*/\s*$'), + "lineCommentStartPattern": re.compile(r'^\s*//'), + "lineCommentEndPattern": None, + "headerStartLine": "/*\n", + "headerEndLine": " */\n", + "headerLinePrefix": " * ", + "headerLineSuffix": None, + } + +# for each processing type, the detailed settings of how to process files of that type +TYPE_SETTINGS = { + # All the languages with C style comments: + "c": update_c_style_comments([".c", ".cc", ".h"]), + "cpp": update_c_style_comments([".cpp", ".hpp", ".cxx", ".hxx", ".ixx"]), + "csharp": update_c_style_comments([".cs", ".csx"]), + "d": update_c_style_comments([".d"]), + "go": update_c_style_comments([".go"]), + "groovy": update_c_style_comments([".groovy"]), + "java": update_c_style_comments([".java", ".jape"]), + "javascript": update_c_style_comments([".js", ".js", ".cjs", ".mjs"]), + "kotlin": update_c_style_comments([".kt", ".kts", ".ktm"]), + "objective-c": update_c_style_comments([".m", ".mm", ".M"]), + "php": update_c_style_comments([".php," ".phtml," ".php3," ".php4," ".php5," ".php7," ".phps," ".php-s," ".pht," ".phar"]), + "rust": update_c_style_comments([".rs"]), + "scala": update_c_style_comments([".scala"]), + "swift": update_c_style_comments([".swift"]), + "typescript": update_c_style_comments([".ts", ".tsx"]), + "script": { + "extensions": [".sh", ".csh", ".pl"], + "keepFirst": re.compile(r'^#!|^# -\*-'), + "blockCommentStartPattern": None, + "blockCommentEndPattern": None, + "lineCommentStartPattern": re.compile(r'^\s*#'), + "lineCommentEndPattern": None, + "headerStartLine": "##\n", + "headerEndLine": "##\n", + "headerLinePrefix": "## ", + "headerLineSuffix": None + }, + "perl": { + "extensions": [".pl"], + "keepFirst": re.compile(r'^#!|^# -\*-'), + "blockCommentStartPattern": None, + "blockCommentEndPattern": None, + "lineCommentStartPattern": re.compile(r'^\s*#'), + "lineCommentEndPattern": None, + "headerStartLine": "##\n", + "headerEndLine": "##\n", + "headerLinePrefix": "## ", + "headerLineSuffix": None + }, + "python": { + "extensions": [".py"], + "keepFirst": re.compile(r'^#!|^# +pylint|^# +-\*-|^# +coding|^# +encoding|^# +type|^# +flake8'), + "blockCommentStartPattern": None, + "blockCommentEndPattern": None, + "lineCommentStartPattern": re.compile(r'^\s*#'), + "lineCommentEndPattern": None, + "headerStartLine": None, + "headerEndLine": "\n", + "headerLinePrefix": "# ", + "headerLineSuffix": None + }, + "robot": { + "extensions": [".robot"], + "keepFirst": re.compile(r'^#!|^# +pylint|^# +-\*-|^# +coding|^# +encoding'), + "blockCommentStartPattern": None, + "blockCommentEndPattern": None, + "lineCommentStartPattern": re.compile(r'^\s*#'), + "lineCommentEndPattern": None, + "headerStartLine": None, + "headerEndLine": None, + "headerLinePrefix": "# ", + "headerLineSuffix": None + }, + "xml": { + "extensions": [".xml"], + "keepFirst": re.compile(r'^\s*<\?xml.*\?>'), + "blockCommentStartPattern": re.compile(r'^\s*\s*$'), + "lineCommentStartPattern": None, + "lineCommentEndPattern": None, + "headerStartLine": "\n", + "headerLinePrefix": "-- ", + "headerLineSuffix": None + }, + "sql": { + "extensions": [".sql"], + "keepFirst": None, + "blockCommentStartPattern": None, # re.compile('^\s*/\*'), + "blockCommentEndPattern": None, # re.compile(r'\*/\s*$'), + "lineCommentStartPattern": re.compile(r'^\s*--'), + "lineCommentEndPattern": None, + "headerStartLine": "--\n", + "headerEndLine": "--\n", + "headerLinePrefix": "-- ", + "headerLineSuffix": None + }, + "cmake": { + "extensions": [], + "filenames": ["CMakeLists.txt"], + "keepFirst": None, + "blockCommentStartPattern": re.compile(r'^\s*#\[\['), + "blockCommentEndPattern": re.compile(r'\]\]\s*$'), + "lineCommentStartPattern": re.compile(r'\s*#'), + "lineCommentEndPattern": None, + "headerStartLine": "#[[\n", + "headerEndLine": "]]\n", + "headerLinePrefix": "", + "headerLineSuffix": None + }, + "markdown": { + "extensions": [".md"], + "keepFirst": None, + "blockCommentStartPattern": re.compile(r'^\s*\s*$'), + "lineCommentStartPattern": None, + "lineCommentEndPattern": None, + "headerStartLine": "\n", + "headerLinePrefix": "", + "headerLineSuffix": None + }, + "ruby": { + "extensions": [".rb"], + "keepFirst": re.compile(r'^#!'), + "blockCommentStartPattern": re.compile('^=begin'), + "blockCommentEndPattern": re.compile(r'^=end'), + "lineCommentStartPattern": re.compile(r'^\s*#'), + "lineCommentEndPattern": None, + "headerStartLine": "##\n", + "headerEndLine": "##\n", + "headerLinePrefix": "## ", + "headerLineSuffix": None + }, + "vb": { + "extensions": [".vb"], + "keepFirst": None, + "blockCommentStartPattern": None, + "blockCommentEndPattern": None, + "lineCommentStartPattern": re.compile(r"^\s*\'"), + "lineCommentEndPattern": None, + "headerStartLine": None, + "headerEndLine": None, + "headerLinePrefix": "' ", + "headerLineSuffix": None + }, + "erlang": { + "extensions": [".erl", ".src", ".config", ".schema"], + "keepFirst": None, + "blockCommentStartPattern": None, + "blockCommentEndPattern": None, + "lineCommentStartPattern": None, + "lineCommentEndPattern": None, + "headerStartLine": "%% -*- erlang -*-\n%% %CopyrightBegin%\n%%\n", + "headerEndLine": "%%\n%% %CopyrightEnd%\n\n", + "headerLinePrefix": "%% ", + "headerLineSuffix": None, + }, + "html": { + "extensions": [".html"], + "keepFirst": re.compile(r'^\s*<\!DOCTYPE.*>'), + "blockCommentStartPattern": re.compile(r'^\s*\s*$'), + "lineCommentStartPattern": None, + "lineCommentEndPattern": None, + "headerStartLine": "\n", + "headerLinePrefix": "-- ", + "headerLineSuffix": None + }, + "css": { + "extensions": [".css", ".scss", ".sass"], + "keepFirst": None, + "blockCommentStartPattern": re.compile(r'^\s*/\*'), + "blockCommentEndPattern": re.compile(r'\*/\s*$'), + "lineCommentStartPattern": None, + "lineCommentEndPattern": None, + "headerStartLine": "/*\n", + "headerEndLine": "*/\n", + "headerLinePrefix": None, + "headerLineSuffix": None + }, + "docker": { + "extensions": [".dockerfile"], + "filenames": ["Dockerfile"], + "keepFirst": None, + "blockCommentStartPattern": None, + "blockCommentEndPattern": None, + "lineCommentStartPattern": re.compile(r'^\s*#'), + "lineCommentEndPattern": None, + "headerStartLine": "##\n", + "headerEndLine": "##\n", + "headerLinePrefix": "## ", + "headerLineSuffix": None + }, + "yaml": { + "extensions": [".yaml", ".yml"], + "keepFirst": None, + "blockCommentStartPattern": None, + "blockCommentEndPattern": None, + "lineCommentStartPattern": re.compile(r'^\s*#'), + "lineCommentEndPattern": None, + "headerStartLine": "##\n", + "headerEndLine": "##\n", + "headerLinePrefix": "## ", + "headerLineSuffix": None + }, + "zig": { + "extensions": [".zig"], + "keepFirst": None, + "blockCommentStartPattern": None, + "blockCommentEndPattern": None, + "lineCommentStartPattern": re.compile(r'^\s*//'), + "lineCommentEndPattern": None, + "headerStartLine": "//!\n", + "headerEndLine": "//!\n", + "headerLinePrefix": "//! ", + "headerLineSuffix": None + }, + "proto": { + "extensions": [".proto"], + "keepFirst": None, + "blockCommentStartPattern": None, + "blockCommentEndPattern": None, + "lineCommentStartPattern": re.compile(r'^\s*//'), + "lineCommentEndPattern": None, + "headerStartLine": None, + "headerEndLine": None, + "headerLinePrefix": "// ", + "headerLineSuffix": None + }, + "terraform": { + "extensions": [".tf"], + "keepFirst": None, + "blockCommentStartPattern": None, + "blockCommentEndPattern": None, + "lineCommentStartPattern": re.compile(r'^\s*#'), + "lineCommentEndPattern": None, + "headerStartLine": "##\n", + "headerEndLine": "##\n", + "headerLinePrefix": "## ", + "headerLineSuffix": None + }, + "bat": { + "extensions": [".bat"], + "keepFirst": None, + "blockCommentStartPattern": None, + "blockCommentEndPattern": None, + "lineCommentStartPattern": re.compile(r'^\s*::'), + "lineCommentEndPattern": None, + "headerStartLine": "::\n", + "headerEndLine": "::\n", + "headerLinePrefix": ":: ", + "headerLineSuffix": None + }, + "ocaml": { + "extensions": [".ml", ".mli", ".mlg", ".v"], + "keepFirst": None, + "blockCommentStartPattern": re.compile(r'^\s*\(\*'), + "blockCommentEndPattern": re.compile(r'\*\)\s*$'), + "lineCommentStartPattern": None, + "lineCommentEndPattern": None, + "headerStartLine": "(*\n", + "headerEndLine": " *)\n", + "headerLinePrefix": " * ", + "headerLineSuffix": None + } +} + +yearsPattern = re.compile( + r"(?<=Copyright\s*(?:\(\s*[Cc©]\s*\)\s*))?([0-9][0-9][0-9][0-9](?:-[0-9][0-9]?[0-9]?[0-9]?)?)", + re.IGNORECASE) +licensePattern = re.compile(r"license", re.IGNORECASE) +emptyPattern = re.compile(r'^\s*$') + +# maps each extension to its processing type. Filled from tpeSettings during initialization +ext2type = {} +name2type = {} +patterns = [] + + +# class for dict args. Use --argname key1=val1,val2 key2=val3 key3=val4, val5 +class DictArgs(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + dict_args = {} + if not isinstance(values, (list,)): + values = (values,) + for value in values: + n, v = value.split("=") + if n not in TYPE_SETTINGS: + LOGGER.error("No valid language '%s' to add additional file extensions for" % n) + if v and "," in str(v): + dict_args[n] = v.split(",") + else: + dict_args[n] = list() + dict_args[n].append(str(v).strip()) + setattr(namespace, self.dest, dict_args) + + +def parse_command_line(argv): + """ + Parse command line argument. See -h option. + :param argv: the actual program arguments + :return: parsed arguments + """ + import textwrap + + + known_extensions = [ + ftype + ":" + ",".join(conf["extensions"]) + for ftype, conf in TYPE_SETTINGS.items() + if "extensions" in conf + ] + # known_extensions = [ext for ftype in typeSettings.values() for ext in ftype["extensions"]] + + example = textwrap.dedent(""" + Known extensions: {0} + + If -t/--tmpl is specified, that header is added to (or existing header replaced for) all source files of known type + If -t/--tmpl is not specified byt -y/--years is specified, all years in existing header files + are replaced with the years specified + + Examples: + {1} -t lgpl-v3 -y 2012-2014 -o ThisNiceCompany -n ProjectName -u http://the.projectname.com + {1} -y 2012-2015 + {1} -y 2012-2015 -d /dir/where/to/start/ + {1} -y 2012-2015 -d /dir/where/to/start/ --additional-extensions python=.j2 + {1} -y 2012-2015 -d /dir/where/to/start/ --additional-extensions python=.j2,.tpl script=.txt + {1} -t .copyright.tmpl -cy + {1} -t .copyright.tmpl -cy -f some_file.cpp + """).format(known_extensions, os.path.basename(argv[0])) + formatter_class = argparse.RawDescriptionHelpFormatter + parser = argparse.ArgumentParser(description="Python license header updater", + epilog=example, + formatter_class=formatter_class) + parser.add_argument("-V", "--version", action="version", + version="%(prog)s {}".format(__version__)) + parser.add_argument("-v", "--verbose", dest="verbose_count", + action="count", default=0, + help="increases log verbosity (can be specified " + "1 to 3 times, default shows errors only)") + parser.add_argument("-d", "--dir", dest="dir", default=default_dir, + help="The directory to recursively process (default: {}).".format(default_dir)) + parser.add_argument("-f", "--files", dest="files", nargs='*', type=str, + help="The list of files to process. If not empty - will disable '--dir' option") + parser.add_argument("-b", action="store_true", + help="Back up all files which get changed to a copy with .bak added to the name") + parser.add_argument("-t", "--tmpl", dest="tmpl", default=None, + help="Template name or file to use.") + parser.add_argument("-s", "--settings", dest="settings", default=None, + help="Settings file to use.") + parser.add_argument("-y", "--years", dest="years", default=None, + help="Year or year range to use.") + parser.add_argument("-cy", "--current-year", dest="current_year", action="store_true", + help="Use today's year.") + parser.add_argument("-o", "--owner", dest="owner", default=None, + help="Name of copyright owner to use.") + parser.add_argument("-n", "--projname", dest="projectname", default=None, + help="Name of project to use.") + parser.add_argument("-u", "--projurl", dest="projecturl", default=None, + help="Url of project to use.") + parser.add_argument("--enc", nargs=1, dest="encoding", default=default_encoding, + help="Encoding of program files (default: {})".format(default_encoding)) + parser.add_argument("--dry", action="store_true", help="Only show what would get done, do not change any files") + parser.add_argument("--safesubst", action="store_true", + help="Do not raise error if template variables cannot be substituted.") + parser.add_argument("-D", "--debug", action="store_true", help="Enable debug messages (same as -v -v -v)") + parser.add_argument("-E", "--ext", type=str, nargs="*", + help="If specified, restrict processing to the specified extension(s) only") + parser.add_argument("--additional-extensions", dest="additional_extensions", default=None, nargs="+", + help="Provide a comma-separated list of additional file extensions as value for a " + "specified language as key, each with a leading dot and no whitespace (default: None).", + action=DictArgs) + parser.add_argument("-x", "--exclude", type=str, nargs="*", + help="File path patterns to exclude") + parser.add_argument("--force-overwrite", action="store_true", dest="force_overwrite", + help="Try to include headers even in read-only files, given sufficient permissions. " + "File permissions are restored after successful header injection.") + arguments = parser.parse_args(argv[1:]) + + # Sets log level to WARN going more verbose for each new -V. + loglevel = max(4 - arguments.verbose_count, 1) * 10 + global LOGGER + LOGGER.setLevel(loglevel) + if arguments.debug: + LOGGER.setLevel(logging.DEBUG) + # fmt = logging.Formatter('%(asctime)s|%(levelname)s|%(name)s|%(message)s') + fmt = logging.Formatter('%(name)s %(levelname)s: %(message)s') + hndlr = logging.StreamHandler(sys.stderr) + hndlr.setFormatter(fmt) + LOGGER.addHandler(hndlr) + + return arguments + + +def read_type_settings(path): + def handle_regex(setting, name): + if setting[name]: + setting[name] = re.compile(setting[name]) + else: + setting[name] = None + + def handle_line(setting, name): + if setting[name]: + setting[name] = setting[name] + else: + setting[name] = None + + settings = {} + + import json + with open(path) as f: + data = json.load(f) + for key, value in data.items(): + for setting_name in ["keepFirst", "blockCommentStartPattern", "blockCommentEndPattern", "lineCommentStartPattern", "lineCommentEndPattern"]: + handle_regex(value, setting_name) + + for setting_name in ["headerStartLine", "headerEndLine", "headerLinePrefix", "headerLineSuffix"]: + handle_line(value, setting_name) + + settings[key] = value + + return settings + +def get_paths(fnpatterns, start_dir=default_dir): + """ + Retrieve files that match any of the glob patterns from the start_dir and below. + :param fnpatterns: the file name patterns + :param start_dir: directory where to start searching + :return: generator that returns one path after the other + """ + seen = set() + for root, dirs, files in os.walk(start_dir): + names = [] + for pattern in fnpatterns: + names += fnmatch.filter(files, pattern) + for name in names: + path = os.path.join(root, name) + if path in seen: + continue + seen.add(path) + yield path + +def get_files(fnpatterns, files): + seen = set() + names = [] + for f in files: + file_name = os.path.basename(f) + for pattern in fnpatterns: + if fnmatch.filter([file_name], pattern): + names += [f] + + for path in names: + if path in seen: + continue + seen.add(path) + yield path + +def read_template(template_file, vardict, args): + """ + Read a template file replace variables from the dict and return the lines. + Throws exception if a variable cannot be replaced. + :param template_file: template file with variables + :param vardict: dictionary to replace variables with values + :param args: the program arguments + :return: lines of the template, with variables replaced + """ + with open(template_file, 'r') as f: + lines = f.readlines() + if args.safesubst: + lines = [Template(line).safe_substitute(vardict) for line in lines] + else: + lines = [Template(line).substitute(vardict) for line in lines] + return lines + + +def for_type(templatelines, ftype, settings): + """ + Format the template lines for the given ftype. + :param templatelines: the lines of the template text + :param ftype: file type + :return: header lines + """ + lines = [] + settings = settings[ftype] + header_start_line = settings["headerStartLine"] + header_end_line = settings["headerEndLine"] + header_line_prefix = settings["headerLinePrefix"] + header_line_suffix = settings["headerLineSuffix"] + if header_start_line is not None: + lines.append(header_start_line) + for line in templatelines: + tmp = line + if header_line_prefix is not None and line == '\n': + tmp = header_line_prefix.rstrip() + tmp + elif header_line_prefix is not None: + tmp = header_line_prefix + tmp + if header_line_suffix is not None: + tmp = tmp + header_line_suffix + lines.append(tmp) + if header_end_line is not None: + lines.append(header_end_line) + return lines + + +## +def read_file(file, args, type_settings): + """ + Read a file and return a dictionary with the following elements: + :param file: the file to read + :param args: the options specified by the user + :return: a dictionary with the following entries or None if the file is not supported: + - skip: number of lines at the beginning to skip (always keep them when replacing or adding something) + can also be seen as the index of the first line not to skip + - headStart: index of first line of detected header, or None if non header detected + - headEnd: index of last line of detected header, or None + - yearsLine: index of line which contains the copyright years, or None + - haveLicense: found a line that matches a pattern that indicates this could be a license header + - settings: the type settings + """ + skip = 0 + head_start = None + head_end = None + years_line = None + have_license = False + filename, extension = os.path.splitext(file) + LOGGER.debug("File name is %s", os.path.basename(filename)) + LOGGER.debug("File extension is %s", extension) + # if we have no entry in the mapping from extensions to processing type, return None + ftype = ext2type.get(extension) + LOGGER.debug("Type for this file is %s", ftype) + if not ftype: + ftype = name2type.get(os.path.basename(file)) + if not ftype: + return None + settings = type_settings.get(ftype) + if not os.access(file, os.R_OK): + LOGGER.error("File %s is not readable.", file) + with open(file, 'r', encoding=args.encoding) as f: + lines = f.readlines() + # now iterate throw the lines and try to determine the various indies + # first try to find the start of the header: skip over shebang or empty lines + keep_first = settings.get("keepFirst") + isBlockHeader = False + block_comment_start_pattern = settings.get("blockCommentStartPattern") + block_comment_end_pattern = settings.get("blockCommentEndPattern") + line_comment_start_pattern = settings.get("lineCommentStartPattern") + i = 0 + LOGGER.info("Processing file {} as {}".format(file, ftype)) + for line in lines: + if (i == 0 or i == skip) and keep_first and keep_first.findall(line): + skip = i + 1 + elif emptyPattern.findall(line): + pass + elif block_comment_start_pattern and block_comment_start_pattern.findall(line): + head_start = i + isBlockHeader = True + break + elif line_comment_start_pattern and line_comment_start_pattern.findall(line): + head_start = i + break + elif not block_comment_start_pattern and \ + line_comment_start_pattern and \ + line_comment_start_pattern.findall(line): + head_start = i + break + else: + # we have reached something else, so no header in this file + # logging.debug("Did not find the start giving up at line %s, line is >%s<",i,line) + return {"type": ftype, + "lines": lines, + "skip": skip, + "headStart": None, + "headEnd": None, + "yearsLine": None, + "settings": settings, + "haveLicense": have_license + } + i = i + 1 + LOGGER.debug("Found preliminary start at {}, i={}, lines={}".format(head_start, i, len(lines))) + # now we have either reached the end, or we are at a line where a block start or line comment occurred + # if we have reached the end, return default dictionary without info + if i == len(lines): + LOGGER.debug("We have reached the end, did not find anything really") + return {"type": ftype, + "lines": lines, + "skip": skip, + "headStart": head_start, + "headEnd": head_end, + "yearsLine": years_line, + "settings": settings, + "haveLicense": have_license + } + # otherwise process the comment block until it ends + if isBlockHeader: + LOGGER.debug("Found comment start, process until end") + for j in range(i, len(lines)): + LOGGER.debug("Checking line {}".format(j)) + if licensePattern.findall(lines[j]): + have_license = True + elif block_comment_end_pattern.findall(lines[j]): + return {"type": ftype, + "lines": lines, + "skip": skip, + "headStart": head_start, + "headEnd": j, + "yearsLine": years_line, + "settings": settings, + "haveLicense": have_license + } + elif yearsPattern.findall(lines[j]): + have_license = True + years_line = j + # if we went through all the lines without finding an end, maybe we have some syntax error or some other + # unusual situation, so lets return no header + LOGGER.debug("Did not find the end of a block comment, returning no header") + return {"type": ftype, + "lines": lines, + "skip": skip, + "headStart": None, + "headEnd": None, + "yearsLine": None, + "settings": settings, + "haveLicense": have_license + } + else: + LOGGER.debug("ELSE1") + for j in range(i, len(lines)): + if line_comment_start_pattern.findall(lines[j]) and licensePattern.findall(lines[j]): + have_license = True + elif not line_comment_start_pattern.findall(lines[j]): + LOGGER.debug("ELSE2") + return {"type": ftype, + "lines": lines, + "skip": skip, + "headStart": i, + "headEnd": j - 1, + "yearsLine": years_line, + "settings": settings, + "haveLicense": have_license + } + elif yearsPattern.findall(lines[j]): + have_license = True + years_line = j + # if we went through all the lines without finding the end of the block, it could be that the whole + # file only consisted of the header, so lets return the last line index + LOGGER.debug("RETURN") + return {"type": ftype, + "lines": lines, + "skip": skip, + "headStart": i, + "headEnd": len(lines) - 1, + "yearsLine": years_line, + "settings": settings, + "haveLicense": have_license + } + + +def make_backup(file, arguments): + """ + Backup file by copying it to a file with the extension .bak appended to the name. + :param file: file to back up + :param arguments: program args, only backs up, if required by an option + :return: + """ + if arguments.b: + LOGGER.info("Backing up file {} to {}".format(file, file + ".bak")) + if not arguments.dry: + copyfile(file, file + ".bak") + + +class OpenAsWriteable(object): + """ + This contextmanager wraps standard open(file, 'w', encoding=...) using + arguments.encoding encoding. If file cannot be written (read-only file), + and if args.force_overwrite is set, try to alter the owner write flag before + yielding the file handle. On exit, file permissions are restored to original + permissions on __exit__ . If the file does not exist, or if it is read-only + and cannot be made writable (due to lacking user rights or force_overwrite + argument not being set), this contextmanager yields None on __enter__. + """ + + def __init__(self, filename, arguments): + """ + Initialize an OpenAsWriteable context manager + :param filename: path to the file to open + :param arguments: program arguments + """ + self._filename = filename + self._arguments = arguments + self._file_handle = None + self._file_permissions = None + + def __enter__(self): + """ + Yields a writable file handle when possible, else None. + """ + filename = self._filename + arguments = self._arguments + file_handle = None + file_permissions = None + + if os.path.isfile(filename): + file_permissions = stat.S_IMODE(os.lstat(filename).st_mode) + + if not os.access(filename, os.W_OK): + if arguments.force_overwrite: + try: + os.chmod(filename, file_permissions | stat.S_IWUSR) + except PermissionError: + LOGGER.warning("File {} cannot be made writable, it will be skipped.".format(filename)) + else: + LOGGER.warning("File {} is not writable, it will be skipped.".format(filename)) + + if os.access(filename, os.W_OK): + file_handle = open(filename, 'w', encoding=arguments.encoding) + else: + LOGGER.warning("File {} does not exist, it will be skipped.".format(filename)) + + self._file_handle = file_handle + self._file_permissions = file_permissions + + return file_handle + + def __exit__ (self, exc_type, exc_value, traceback): + """ + Restore back file permissions and close file handle (if any). + """ + if (self._file_handle is not None): + self._file_handle.close() + + actual_permissions = stat.S_IMODE(os.lstat(self._filename).st_mode) + if (actual_permissions != self._file_permissions): + try: + os.chmod(self._filename, self._file_permissions) + except PermissionError: + LOGGER.error("File {} permissions could not be restored.".format(self._filename)) + + self._file_handle = None + self._file_permissions = None + return True + + +@contextlib.contextmanager +def open_as_writable(file, arguments): + """ + Wrapper around OpenAsWriteable context manager. + """ + with OpenAsWriteable(file, arguments=arguments) as fw: + yield fw + + +def main(): + """Main function.""" + # LOGGER.addHandler(logging.StreamHandler(stream=sys.stderr)) + # init: create the ext2type mappings + arguments = parse_command_line(sys.argv) + additional_extensions = arguments.additional_extensions + + type_settings = TYPE_SETTINGS + if arguments.settings: + type_settings = read_type_settings(arguments.settings) + + for t in type_settings: + settings = type_settings[t] + exts = settings["extensions"] + if "filenames" in settings: + names = settings['filenames'] + else: + names = [] + # if additional file extensions are provided by the user, they are "merged" here: + if additional_extensions and t in additional_extensions: + for aext in additional_extensions[t]: + LOGGER.debug("Enable custom file extension '%s' for language '%s'" % (aext, t)) + exts.append(aext) + + for ext in exts: + ext2type[ext] = t + patterns.append("*" + ext) + + for name in names: + name2type[name] = t + patterns.append(name) + + LOGGER.debug("Allowed file patterns %s" % patterns) + + limit2exts = None + if arguments.ext is not None and len(arguments.ext) > 0: + limit2exts = arguments.ext + + try: + error = False + template_lines = None + if arguments.dir is not default_dir and arguments.files: + LOGGER.error("Cannot use both '--dir' and '--files' options.") + error = True + + if arguments.years and arguments.current_year: + LOGGER.error("Cannot use both '--years' and '--currentyear' options.") + error = True + + years = arguments.years + if arguments.current_year: + import datetime + now = datetime.datetime.now() + years = str(now.year) + + settings = {} + if years: + settings["years"] = years + if arguments.owner: + settings["owner"] = arguments.owner + if arguments.projectname: + settings["projectname"] = arguments.projectname + if arguments.projecturl: + settings["projecturl"] = arguments.projecturl + # if we have a template name specified, try to get or load the template + if arguments.tmpl: + opt_tmpl = arguments.tmpl + # first get all the names of our own templates + # for this get first the path of this file + templates_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates") + LOGGER.debug("File path: {}".format(os.path.abspath(__file__))) + # get all the templates in the templates directory + templates = [f for f in get_paths("*.tmpl", templates_dir)] + templates = [(os.path.splitext(os.path.basename(t))[0], t) for t in templates] + # filter by trying to match the name against what was specified + tmpls = [t for t in templates if opt_tmpl in t[0]] + # check if one of the matching template names is identical to the parameter, then take that one + tmpls_eq = [t for t in tmpls if opt_tmpl == t[0]] + if len(tmpls_eq) > 0: + tmpls = tmpls_eq + if len(tmpls) == 1: + tmpl_name = tmpls[0][0] + tmpl_file = tmpls[0][1] + LOGGER.info("Using template file {} for {}".format(tmpl_file, tmpl_name)) + template_lines = read_template(tmpl_file, settings, arguments) + else: + if len(tmpls) == 0: + # check if we can interpret the option as file + if os.path.isfile(opt_tmpl): + LOGGER.info("Using file {}".format(os.path.abspath(opt_tmpl))) + template_lines = read_template(os.path.abspath(opt_tmpl), settings, arguments) + else: + LOGGER.error("Not a built-in template and not a file, cannot proceed: {}".format(opt_tmpl)) + LOGGER.error("Built in templates: {}".format(", ".join([t[0] for t in templates]))) + error = True + else: + LOGGER.error("There are multiple matching template names: {}".format([t[0] for t in tmpls])) + error = True + else: + # no tmpl parameter + if not years: + LOGGER.error("No template specified and no years either, nothing to do (use -h option for usage info)") + error = True + if error: + return 1 + else: + # logging.debug("Got template lines: %s",templateLines) + # now do the actual processing: if we did not get some error, we have a template loaded or + # no template at all + # if we have no template, then we will have the years. + # now process all the files and either replace the years or replace/add the header + if arguments.files: + LOGGER.debug("Processing files %s", arguments.files) + LOGGER.debug("Patterns: %s", patterns) + paths = get_files(patterns, arguments.files) + else: + LOGGER.debug("Processing directory %s", arguments.dir) + LOGGER.debug("Patterns: %s", patterns) + paths = get_paths(patterns, arguments.dir) + + for file in paths: + LOGGER.debug("Considering file: {}".format(file)) + file = os.path.normpath(file) + if limit2exts is not None and not any([file.endswith(ext) for ext in limit2exts]): + LOGGER.info("Skipping file with non-matching extension: {}".format(file)) + continue + if arguments.exclude and any([fnmatch.fnmatch(file, pat) for pat in arguments.exclude]): + LOGGER.info("Ignoring file {}".format(file)) + continue + finfo = read_file(file, arguments, type_settings) + if not finfo: + LOGGER.debug("File not supported %s", file) + continue + # logging.debug("FINFO for the file: %s", finfo) + lines = finfo["lines"] + LOGGER.debug( + "Info for the file: headStart=%s, headEnd=%s, haveLicense=%s, skip=%s, len=%s, yearsline=%s", + finfo["headStart"], finfo["headEnd"], finfo["haveLicense"], finfo["skip"], len(lines), + finfo["yearsLine"]) + # if we have a template: replace or add + if template_lines: + make_backup(file, arguments) + if arguments.dry: + LOGGER.info("Would be updating changed file: {}".format(file)) + else: + with open_as_writable(file, arguments) as fw: + if (fw is not None): + # if we found a header, replace it + # otherwise, add it after the lines to skip + head_start = finfo["headStart"] + head_end = finfo["headEnd"] + have_license = finfo["haveLicense"] + ftype = finfo["type"] + skip = finfo["skip"] + if head_start is not None and head_end is not None and have_license: + LOGGER.debug("Replacing header in file {}".format(file)) + # first write the lines before the header + fw.writelines(lines[0:head_start]) + # now write the new header from the template lines + fw.writelines(for_type(template_lines, ftype, type_settings)) + # now write the rest of the lines + fw.writelines(lines[head_end + 1:]) + else: + LOGGER.debug("Adding header to file {}, skip={}".format(file, skip)) + fw.writelines(lines[0:skip]) + fw.writelines(for_type(template_lines, ftype, type_settings)) + if head_start is not None and not have_license: + # There is some header, but not license - add an empty line + fw.write("\n") + fw.writelines(lines[skip:]) + # TODO: optionally remove backup if all worked well? + else: + # no template lines, just update the line with the year, if we found a year + years_line = finfo["yearsLine"] + if years_line is not None: + make_backup(file, arguments) + if arguments.dry: + LOGGER.info("Would be updating year line in file {}".format(file)) + else: + with open_as_writable(file, arguments) as fw: + if (fw is not None): + LOGGER.debug("Updating years in file {} in line {}".format(file, years_line)) + fw.writelines(lines[0:years_line]) + fw.write(yearsPattern.sub(years, lines[years_line])) + fw.writelines(lines[years_line + 1:]) + # TODO: optionally remove backup if all worked well + return 0 + finally: + logging.shutdown() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/load_test.sh b/load_test.sh new file mode 100755 index 00000000..2ade759c --- /dev/null +++ b/load_test.sh @@ -0,0 +1,69 @@ +#!/usr/bin/bash + +MDD=./integration_tests/metadata +python ${MDD}/metadata_importer.py <\n" +"Language: en\n" +"Language-Team: en \n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.15.0\n" + +#. Open web-services for the Open Data Cube +msgid "global.title" +msgstr "Open web-services for the Open Data Cube" + +#. This web-service serves georectified raster data from our very own special +#. Open Datacube instance. +msgid "global.abstract" +msgstr "" +"This web-service serves georectified raster data from our very own " +"special Open Datacube instance." + +#. satellite,time-series,australia +msgid "global.local_keywords" +msgstr "satellite,time-series,australia" + +#. none +msgid "global.fees" +msgstr "none" + +#. none +msgid "global.access_constraints" +msgstr "none" + +#. Acme Corporation +msgid "global.contact_org" +msgstr "Acme Corporation" + +#. CIO (Chief Imaginary Officer) +msgid "global.contact_position" +msgstr "CIO (Chief Imaginary Officer)" + +#. Over-ridden: aardvark +msgid "folder.ows_root.title" +msgstr "Over-ridden: aardvark" + +#. This web-service serves georectified raster data from our very own special +#. Open Datacube instance. +msgid "folder.ows_root.abstract" +msgstr "" +"This web-service serves georectified raster data from our very own " +"special Open Datacube instance." + +msgid "folder.ows_root.local_keywords" +msgstr "" + +#. s2 +msgid "folder.sentinel2.title" +msgstr "s2" + +#. Images from the sentinel 2 satellite +msgid "folder.sentinel2.abstract" +msgstr "Images from the sentinel 2 satellite" + +#. sentinel2 +msgid "folder.sentinel2.local_keywords" +msgstr "sentinel2" + +#. Open Data Cube - OWS +msgid "folder.sentinel2.attribution_title" +msgstr "Open Data Cube - OWS" + +#. Surface reflectance (Sentinel-2) +msgid "layer.s2_l2a.title" +msgstr "Surface reflectance (Sentinel-2)" + +#. layer s2_l2a +msgid "layer.s2_l2a.abstract" +msgstr "layer s2_l2a" + +msgid "layer.s2_l2a.local_keywords" +msgstr "" + +#. coastal_aerosol +msgid "layer.s2_l2a.bands.B01" +msgstr "coastal_aerosol" + +#. blue +msgid "layer.s2_l2a.bands.B02" +msgstr "blue" + +#. green +msgid "layer.s2_l2a.bands.B03" +msgstr "green" + +#. red +msgid "layer.s2_l2a.bands.B04" +msgstr "red" + +#. red_edge_1 +msgid "layer.s2_l2a.bands.B05" +msgstr "red_edge_1" + +#. red_edge_2 +msgid "layer.s2_l2a.bands.B06" +msgstr "red_edge_2" + +#. red_edge_3 +msgid "layer.s2_l2a.bands.B07" +msgstr "red_edge_3" + +#. nir +msgid "layer.s2_l2a.bands.B08" +msgstr "nir" + +#. nir_narrow +msgid "layer.s2_l2a.bands.B8A" +msgstr "nir_narrow" + +#. water_vapour +msgid "layer.s2_l2a.bands.B09" +msgstr "water_vapour" + +#. swir_1 +msgid "layer.s2_l2a.bands.B11" +msgstr "swir_1" + +#. swir_2 +msgid "layer.s2_l2a.bands.B12" +msgstr "swir_2" + +#. aerosol_optical_thickness +msgid "layer.s2_l2a.bands.AOT" +msgstr "aerosol_optical_thickness" + +#. scene_average_water_vapour +msgid "layer.s2_l2a.bands.WVP" +msgstr "scene_average_water_vapour" + +#. mask +msgid "layer.s2_l2a.bands.SCL" +msgstr "mask" + +#. Simple RGB +msgid "style.s2_l2a.simple_rgb.title" +msgstr "Simple RGB" + +#. Simple true-colour image, using the red, green and blue bands +msgid "style.s2_l2a.simple_rgb.abstract" +msgstr "Simple true-colour image, using the red, green and blue bands" + +#. Simple RGB Clone +msgid "style.s2_l2a.style_ls_simple_rgb_clone.title" +msgstr "Simple RGB Clone" + +#. Simple true-colour image, using the red, green and blue bands +msgid "style.s2_l2a.style_ls_simple_rgb_clone.abstract" +msgstr "Simple true-colour image, using the red, green and blue bands" + +#. False colour multi-band infra-red +msgid "style.s2_l2a.infra_red.title" +msgstr "False colour multi-band infra-red" + +#. Simple false-colour image, using the near and short-wave infra-red bands +msgid "style.s2_l2a.infra_red.abstract" +msgstr "Simple false-colour image, using the near and short-wave infra-red bands" + +#. Blue - 490 +msgid "style.s2_l2a.blue.title" +msgstr "Blue - 490" + +#. Blue band, centered on 490nm +msgid "style.s2_l2a.blue.abstract" +msgstr "Blue band, centered on 490nm" + +#. NDVI +msgid "style.s2_l2a.ndvi.title" +msgstr "NDVI" + +#. Normalised Difference Vegetation Index - a derived index that correlates +#. well with the existence of vegetation +msgid "style.s2_l2a.ndvi.abstract" +msgstr "" +"Normalised Difference Vegetation Index - a derived index that correlates " +"well with the existence of vegetation" + +#. NDVI +msgid "style.s2_l2a.ndvi.legend.1.title" +msgstr "NDVI" + +#. dimensionless +msgid "style.s2_l2a.ndvi.legend.1.units" +msgstr "dimensionless" + +#. low +msgid "style.s2_l2a.ndvi.legend.1.lbl_0.0" +msgstr "low" + +#. high +msgid "style.s2_l2a.ndvi.legend.1.lbl_1.0" +msgstr "high" + +#. NDVI +msgid "style.s2_l2a.ndvi_expr.title" +msgstr "NDVI" + +#. Normalised Difference Vegetation Index - a derived index that correlates +#. well with the existence of vegetation +msgid "style.s2_l2a.ndvi_expr.abstract" +msgstr "" +"Normalised Difference Vegetation Index - a derived index that correlates " +"well with the existence of vegetation" + +#. NDVI +msgid "style.s2_l2a.ndvi_expr.legend.1.title" +msgstr "NDVI" + +#. NDVI plus RGB +msgid "style.s2_l2a.rgb_ndvi.title" +msgstr "NDVI plus RGB" + +#. Normalised Difference Vegetation Index (blended with RGB) - a derived index +#. that correlates well with the existence of vegetation +msgid "style.s2_l2a.rgb_ndvi.abstract" +msgstr "" +"Normalised Difference Vegetation Index (blended with RGB) - a derived " +"index that correlates well with the existence of vegetation" + +#. NDVI plus RGB +msgid "style.s2_l2a.rgb_ndvi.legend.1.title" +msgstr "NDVI plus RGB" + +#. NDVI - Red, NIR +msgid "style.s2_l2a.ndvi_delta.title" +msgstr "NDVI - Red, NIR" + +#. Normalised Difference Vegetation Index - a derived index that correlates +#. well with the existence of vegetation +msgid "style.s2_l2a.ndvi_delta.abstract" +msgstr "" +"Normalised Difference Vegetation Index - a derived index that correlates " +"well with the existence of vegetation" + +#. NDVI - Red, NIR +msgid "style.s2_l2a.ndvi_delta.legend.1.title" +msgstr "NDVI - Red, NIR" + +#. NDVI - Red, NIR +msgid "style.s2_l2a.ndvi_delta.legend.2.title" +msgstr "NDVI - Red, NIR" + +#. s2_l2a Clone +msgid "layer.s2_l2a_clone.title" +msgstr "s2_l2a Clone" + +#. Imagery from the s2_l2a Clone +msgid "layer.s2_l2a_clone.abstract" +msgstr "Imagery from the s2_l2a Clone" + +msgid "layer.s2_l2a_clone.local_keywords" +msgstr "" + +#. coastal_aerosol +msgid "layer.s2_l2a_clone.bands.B01" +msgstr "coastal_aerosol" + +#. blue +msgid "layer.s2_l2a_clone.bands.B02" +msgstr "blue" + +#. green +msgid "layer.s2_l2a_clone.bands.B03" +msgstr "green" + +#. red +msgid "layer.s2_l2a_clone.bands.B04" +msgstr "red" + +#. red_edge_1 +msgid "layer.s2_l2a_clone.bands.B05" +msgstr "red_edge_1" + +#. red_edge_2 +msgid "layer.s2_l2a_clone.bands.B06" +msgstr "red_edge_2" + +#. red_edge_3 +msgid "layer.s2_l2a_clone.bands.B07" +msgstr "red_edge_3" + +#. nir +msgid "layer.s2_l2a_clone.bands.B08" +msgstr "nir" + +#. nir_narrow +msgid "layer.s2_l2a_clone.bands.B8A" +msgstr "nir_narrow" + +#. water_vapour +msgid "layer.s2_l2a_clone.bands.B09" +msgstr "water_vapour" + +#. swir_1 +msgid "layer.s2_l2a_clone.bands.B11" +msgstr "swir_1" + +#. swir_2 +msgid "layer.s2_l2a_clone.bands.B12" +msgstr "swir_2" + +#. aerosol_optical_thickness +msgid "layer.s2_l2a_clone.bands.AOT" +msgstr "aerosol_optical_thickness" + +#. scene_average_water_vapour +msgid "layer.s2_l2a_clone.bands.WVP" +msgstr "scene_average_water_vapour" + +#. mask +msgid "layer.s2_l2a_clone.bands.SCL" +msgstr "mask" + +#. Simple RGB +msgid "style.s2_l2a_clone.simple_rgb.title" +msgstr "Simple RGB" + +#. Simple true-colour image, using the red, green and blue bands +msgid "style.s2_l2a_clone.simple_rgb.abstract" +msgstr "Simple true-colour image, using the red, green and blue bands" + +#. Simple RGB Clone +msgid "style.s2_l2a_clone.style_ls_simple_rgb_clone.title" +msgstr "Simple RGB Clone" + +#. Simple true-colour image, using the red, green and blue bands +msgid "style.s2_l2a_clone.style_ls_simple_rgb_clone.abstract" +msgstr "Simple true-colour image, using the red, green and blue bands" + +#. False colour multi-band infra-red +msgid "style.s2_l2a_clone.infra_red.title" +msgstr "False colour multi-band infra-red" + +#. Simple false-colour image, using the near and short-wave infra-red bands +msgid "style.s2_l2a_clone.infra_red.abstract" +msgstr "Simple false-colour image, using the near and short-wave infra-red bands" + +#. Blue - 490 +msgid "style.s2_l2a_clone.blue.title" +msgstr "Blue - 490" + +#. Blue band, centered on 490nm +msgid "style.s2_l2a_clone.blue.abstract" +msgstr "Blue band, centered on 490nm" + +#. NDVI +msgid "style.s2_l2a_clone.ndvi.title" +msgstr "NDVI" + +#. Normalised Difference Vegetation Index - a derived index that correlates +#. well with the existence of vegetation +msgid "style.s2_l2a_clone.ndvi.abstract" +msgstr "" +"Normalised Difference Vegetation Index - a derived index that correlates " +"well with the existence of vegetation" + +#. NDVI +msgid "style.s2_l2a_clone.ndvi.legend.1.title" +msgstr "NDVI" + +#. dimensionless +msgid "style.s2_l2a_clone.ndvi.legend.1.units" +msgstr "dimensionless" + +#. low +msgid "style.s2_l2a_clone.ndvi.legend.1.lbl_0.0" +msgstr "low" + +#. high +msgid "style.s2_l2a_clone.ndvi.legend.1.lbl_1.0" +msgstr "high" + +#. NDVI +msgid "style.s2_l2a_clone.ndvi_expr.title" +msgstr "NDVI" + +#. Normalised Difference Vegetation Index - a derived index that correlates +#. well with the existence of vegetation +msgid "style.s2_l2a_clone.ndvi_expr.abstract" +msgstr "" +"Normalised Difference Vegetation Index - a derived index that correlates " +"well with the existence of vegetation" + +#. NDVI +msgid "style.s2_l2a_clone.ndvi_expr.legend.1.title" +msgstr "NDVI" + +#. NDVI plus RGB +msgid "style.s2_l2a_clone.rgb_ndvi.title" +msgstr "NDVI plus RGB" + +#. Normalised Difference Vegetation Index (blended with RGB) - a derived index +#. that correlates well with the existence of vegetation +msgid "style.s2_l2a_clone.rgb_ndvi.abstract" +msgstr "" +"Normalised Difference Vegetation Index (blended with RGB) - a derived " +"index that correlates well with the existence of vegetation" + +#. NDVI plus RGB +msgid "style.s2_l2a_clone.rgb_ndvi.legend.1.title" +msgstr "NDVI plus RGB" + +#. NDVI - Red, NIR +msgid "style.s2_l2a_clone.ndvi_delta.title" +msgstr "NDVI - Red, NIR" + +#. Normalised Difference Vegetation Index - a derived index that correlates +#. well with the existence of vegetation +msgid "style.s2_l2a_clone.ndvi_delta.abstract" +msgstr "" +"Normalised Difference Vegetation Index - a derived index that correlates " +"well with the existence of vegetation" + +#. NDVI - Red, NIR +msgid "style.s2_l2a_clone.ndvi_delta.legend.1.title" +msgstr "NDVI - Red, NIR" + +#. NDVI - Red, NIR +msgid "style.s2_l2a_clone.ndvi_delta.legend.2.title" +msgstr "NDVI - Red, NIR" + +#. DEA Config Samples +msgid "folder.ows_root.1.title" +msgstr "DEA Config Samples" + +msgid "folder.ows_root.1.abstract" +msgstr "" + +msgid "folder.ows_root.1.local_keywords" +msgstr "" + +#. DEA Surface Reflectance (Sentinel-2) +msgid "layer.s2_ard_granule_nbar_t.title" +msgstr "DEA Surface Reflectance (Sentinel-2)" + +#. Sentinel-2 Multispectral Instrument - Nadir BRDF Adjusted Reflectance + +#. Terrain Illumination Correction (Sentinel-2 MSI) This +#. product has been corrected to account for variations caused by atmospheric +#. properties, sun position and sensor view angle at time of image capture. +#. These corrections have been applied to all satellite imagery in the +#. Sentinel-2 archive. This is undertaken to allow comparison of imagery +#. acquired at different times, in different seasons and in different +#. geographic locations. These products also indicate where the +#. imagery has been affected by cloud or cloud shadow, contains missing data or +#. has been affected in other ways. The Surface Reflectance products are useful +#. as a fundamental starting point for any further analysis, and underpin all +#. other optical derived Digital Earth Australia products. This +#. is a definitive archive of daily Sentinel-2 data. This is processed using +#. correct ancillary data to provide a more accurate product than the Near Real +#. Time. The Surface Reflectance product has been corrected to +#. account for variations caused by atmospheric properties, sun position and +#. sensor view angle at time of image capture. These corrections have been +#. applied to all satellite imagery in the Sentinel-2 archive. +#. The Normalised Difference Chlorophyll Index (NDCI) is based on the method of +#. Mishra & Mishra 2012, and adapted to bands on the Sentinel-2A & B sensors. +#. The index indicates levels of chlorophyll-a (chl-a) concentrations in +#. complex turbid productive waters such as those encountered in many inland +#. water bodies. The index has not been validated in Australian waters, and +#. there are a range of environmental conditions that may have an effect on the +#. accuracy of the derived index values in this test implementation, including: +#. - Influence on the remote sensing signal from nearby land and/or atmospheric +#. effects - Optically shallow water - Cloud +#. cover Mishra, S., Mishra, D.R., 2012. Normalized difference +#. chlorophyll index: A novel model for remote estimation of chlorophyll-a +#. concentration in turbid productive waters. Remote Sensing of Environment, +#. Remote Sensing of Urban Environments 117, 394–406. +#. https://doi.org/10.1016/j.rse.2011.10.016 For more +#. information see http://pid.geoscience.gov.au/dataset/ga/129684 +#. https://cmi.ga.gov.au/data-products/dea/190/dea-surface-reflectance-nbart-sentinel-2-msi +#. For service status information, see https://status.dea.ga.gov.au +msgid "layer.s2_ard_granule_nbar_t.abstract" +msgstr "" +"Sentinel-2 Multispectral Instrument - Nadir BRDF Adjusted Reflectance + " +"Terrain Illumination Correction (Sentinel-2 MSI)\n" +" This product has been corrected to account for variations" +" caused by atmospheric properties, sun position and sensor view angle at " +"time of image capture.\n" +" These corrections have been applied to all satellite " +"imagery in the Sentinel-2 archive. This is undertaken to allow comparison" +" of imagery acquired at different times, in different seasons and in " +"different geographic locations.\n" +" These products also indicate where the imagery has been " +"affected by cloud or cloud shadow, contains missing data or has been " +"affected in other ways. The Surface Reflectance products are useful as a " +"fundamental starting point for any further analysis, and underpin all " +"other optical derived Digital Earth Australia products.\n" +" This is a definitive archive of daily Sentinel-2 data. " +"This is processed using correct ancillary data to provide a more accurate" +" product than the Near Real Time.\n" +" The Surface Reflectance product has been corrected to " +"account for variations caused by atmospheric properties, sun position and" +" sensor view angle at time of image capture. These corrections have been " +"applied to all satellite imagery in the Sentinel-2 archive.\n" +" The Normalised Difference Chlorophyll Index (NDCI) is " +"based on the method of Mishra & Mishra 2012, and adapted to bands on the " +"Sentinel-2A & B sensors.\n" +" The index indicates levels of chlorophyll-a (chl-a) " +"concentrations in complex turbid productive waters such as those " +"encountered in many inland water bodies. The index has not been validated" +" in Australian waters, and there are a range of environmental conditions " +"that may have an effect on the accuracy of the derived index values in " +"this test implementation, including:\n" +" - Influence on the remote sensing signal from nearby land" +" and/or atmospheric effects\n" +" - Optically shallow water\n" +" - Cloud cover\n" +" Mishra, S., Mishra, D.R., 2012. Normalized difference " +"chlorophyll index: A novel model for remote estimation of chlorophyll-a " +"concentration in turbid productive waters. Remote Sensing of Environment," +" Remote Sensing of Urban Environments 117, 394–406. " +"https://doi.org/10.1016/j.rse.2011.10.016\n" +" For more information see " +"http://pid.geoscience.gov.au/dataset/ga/129684\n" +" https://cmi.ga.gov.au/data-products/dea/190/dea-surface-" +"reflectance-nbart-sentinel-2-msi\n" +" For service status information, see " +"https://status.dea.ga.gov.au\n" +" " + +msgid "layer.s2_ard_granule_nbar_t.local_keywords" +msgstr "" + +#. nbar_coastal_aerosol +msgid "layer.s2_ard_granule_nbar_t.bands.nbart_coastal_aerosol" +msgstr "nbar_coastal_aerosol" + +#. nbar_blue +msgid "layer.s2_ard_granule_nbar_t.bands.nbart_blue" +msgstr "nbar_blue" + +#. nbar_green +msgid "layer.s2_ard_granule_nbar_t.bands.nbart_green" +msgstr "nbar_green" + +#. nbar_red +msgid "layer.s2_ard_granule_nbar_t.bands.nbart_red" +msgstr "nbar_red" + +#. nbar_red_edge_1 +msgid "layer.s2_ard_granule_nbar_t.bands.nbart_red_edge_1" +msgstr "nbar_red_edge_1" + +#. nbar_red_edge_2 +msgid "layer.s2_ard_granule_nbar_t.bands.nbart_red_edge_2" +msgstr "nbar_red_edge_2" + +#. nbar_red_edge_3 +msgid "layer.s2_ard_granule_nbar_t.bands.nbart_red_edge_3" +msgstr "nbar_red_edge_3" + +#. nbar_nir_1 +msgid "layer.s2_ard_granule_nbar_t.bands.nbart_nir_1" +msgstr "nbar_nir_1" + +#. nbar_nir_2 +msgid "layer.s2_ard_granule_nbar_t.bands.nbart_nir_2" +msgstr "nbar_nir_2" + +#. nbar_swir_2 +msgid "layer.s2_ard_granule_nbar_t.bands.nbart_swir_2" +msgstr "nbar_swir_2" + +#. nbar_swir_3 +msgid "layer.s2_ard_granule_nbar_t.bands.nbart_swir_3" +msgstr "nbar_swir_3" + +#. fmask +msgid "layer.s2_ard_granule_nbar_t.bands.fmask" +msgstr "fmask" + +#. Normalised Difference Chlorophyll Index - Red Edge, Red +msgid "style.s2_ard_granule_nbar_t.ndci.title" +msgstr "Normalised Difference Chlorophyll Index - Red Edge, Red" + +#. Normalised Difference Chlorophyll Index - a derived index that correlates +#. well with the existence of chlorophyll +msgid "style.s2_ard_granule_nbar_t.ndci.abstract" +msgstr "" +"Normalised Difference Chlorophyll Index - a derived index that correlates" +" well with the existence of chlorophyll" + +#. Normalised Difference Chlorophyll Index - Red Edge, Red +msgid "style.s2_ard_granule_nbar_t.ndci.legend.1.title" +msgstr "Normalised Difference Chlorophyll Index - Red Edge, Red" + +#. unitless +msgid "style.s2_ard_granule_nbar_t.ndci.legend.1.units" +msgstr "unitless" + +#. Modified Normalised Difference Water Index - Green, SWIR +msgid "style.s2_ard_granule_nbar_t.mndwi.title" +msgstr "Modified Normalised Difference Water Index - Green, SWIR" + +#. Modified Normalised Difference Water Index - a derived index that correlates +#. well with the existence of water (Xu 2006) +msgid "style.s2_ard_granule_nbar_t.mndwi.abstract" +msgstr "" +"Modified Normalised Difference Water Index - a derived index that " +"correlates well with the existence of water (Xu 2006)" + +#. Modified Normalised Difference Water Index - Green, SWIR +msgid "style.s2_ard_granule_nbar_t.mndwi.legend.1.title" +msgstr "Modified Normalised Difference Water Index - Green, SWIR" + +#. Modified Normalised Difference Water Index - Green, SWIR +msgid "style.s2_ard_granule_nbar_t.mndwi.legend.2.title" +msgstr "Modified Normalised Difference Water Index - Green, SWIR" + +#. DEA Surface Reflectance Mosaic (Sentinel-2) +msgid "layer.s2_ard_latest_mosaic.title" +msgstr "DEA Surface Reflectance Mosaic (Sentinel-2)" + +#. Sentinel-2 Multispectral Instrument - Nadir BRDF Adjusted Reflectance + +#. Terrain Illumination Correction (Sentinel-2 MSI) Latest imagery mosaic with +#. no time dimension. +msgid "layer.s2_ard_latest_mosaic.abstract" +msgstr "" +"Sentinel-2 Multispectral Instrument - Nadir BRDF Adjusted Reflectance + " +"Terrain Illumination Correction (Sentinel-2 MSI)\n" +"\n" +"Latest imagery mosaic with no time dimension.\n" +" " + +msgid "layer.s2_ard_latest_mosaic.local_keywords" +msgstr "" + +#. nbar_coastal_aerosol +msgid "layer.s2_ard_latest_mosaic.bands.nbart_coastal_aerosol" +msgstr "nbar_coastal_aerosol" + +#. nbar_blue +msgid "layer.s2_ard_latest_mosaic.bands.nbart_blue" +msgstr "nbar_blue" + +#. nbar_green +msgid "layer.s2_ard_latest_mosaic.bands.nbart_green" +msgstr "nbar_green" + +#. nbar_red +msgid "layer.s2_ard_latest_mosaic.bands.nbart_red" +msgstr "nbar_red" + +#. nbar_red_edge_1 +msgid "layer.s2_ard_latest_mosaic.bands.nbart_red_edge_1" +msgstr "nbar_red_edge_1" + +#. nbar_red_edge_2 +msgid "layer.s2_ard_latest_mosaic.bands.nbart_red_edge_2" +msgstr "nbar_red_edge_2" + +#. nbar_red_edge_3 +msgid "layer.s2_ard_latest_mosaic.bands.nbart_red_edge_3" +msgstr "nbar_red_edge_3" + +#. nbar_nir_1 +msgid "layer.s2_ard_latest_mosaic.bands.nbart_nir_1" +msgstr "nbar_nir_1" + +#. nbar_nir_2 +msgid "layer.s2_ard_latest_mosaic.bands.nbart_nir_2" +msgstr "nbar_nir_2" + +#. nbar_swir_2 +msgid "layer.s2_ard_latest_mosaic.bands.nbart_swir_2" +msgstr "nbar_swir_2" + +#. nbar_swir_3 +msgid "layer.s2_ard_latest_mosaic.bands.nbart_swir_3" +msgstr "nbar_swir_3" + +#. fmask +msgid "layer.s2_ard_latest_mosaic.bands.fmask" +msgstr "fmask" + +#. Normalised Difference Chlorophyll Index - Red Edge, Red +msgid "style.s2_ard_latest_mosaic.ndci.title" +msgstr "Normalised Difference Chlorophyll Index - Red Edge, Red" + +#. Normalised Difference Chlorophyll Index - a derived index that correlates +#. well with the existence of chlorophyll +msgid "style.s2_ard_latest_mosaic.ndci.abstract" +msgstr "" +"Normalised Difference Chlorophyll Index - a derived index that correlates" +" well with the existence of chlorophyll" + +#. Normalised Difference Chlorophyll Index - Red Edge, Red +msgid "style.s2_ard_latest_mosaic.ndci.legend.1.title" +msgstr "Normalised Difference Chlorophyll Index - Red Edge, Red" + +#. unitless +msgid "style.s2_ard_latest_mosaic.ndci.legend.1.units" +msgstr "unitless" + +#. Modified Normalised Difference Water Index - Green, SWIR +msgid "style.s2_ard_latest_mosaic.mndwi.title" +msgstr "Modified Normalised Difference Water Index - Green, SWIR" + +#. Modified Normalised Difference Water Index - a derived index that correlates +#. well with the existence of water (Xu 2006) +msgid "style.s2_ard_latest_mosaic.mndwi.abstract" +msgstr "" +"Modified Normalised Difference Water Index - a derived index that " +"correlates well with the existence of water (Xu 2006)" + +#. Modified Normalised Difference Water Index - Green, SWIR +msgid "style.s2_ard_latest_mosaic.mndwi.legend.1.title" +msgstr "Modified Normalised Difference Water Index - Green, SWIR" + +#. Modified Normalised Difference Water Index - Green, SWIR +msgid "style.s2_ard_latest_mosaic.mndwi.legend.2.title" +msgstr "Modified Normalised Difference Water Index - Green, SWIR" + +#. DEA Fractional Cover (Landsat) +msgid "layer.ga_ls_fc_3.title" +msgstr "DEA Fractional Cover (Landsat)" + +#. Geoscience Australia Landsat Fractional Cover Collection 3 +#. Fractional Cover (FC), developed by the Joint Remote Sensing Research +#. Program, is a measurement that splits the landscape into three parts, or +#. fractions: green (leaves, grass, and growing crops) +#. brown (branches, dry grass or hay, and dead leaf litter) +#. bare ground (soil or rock) DEA uses Fractional Cover to +#. characterise every 30 m square of Australia for any point in time from 1987 +#. to today. +#. https://cmi.ga.gov.au/data-products/dea/629/dea-fractional-cover-landsat-c3 +#. For service status information, see https://status.dea.ga.gov.au +msgid "layer.ga_ls_fc_3.abstract" +msgstr "" +"Geoscience Australia Landsat Fractional Cover Collection 3\n" +" Fractional Cover (FC), developed by the Joint Remote " +"Sensing Research Program, is a measurement that splits the landscape into" +" three parts, or fractions:\n" +" green (leaves, grass, and growing crops)\n" +" brown (branches, dry grass or hay, and dead leaf litter)\n" +" bare ground (soil or rock)\n" +" DEA uses Fractional Cover to characterise every 30 m " +"square of Australia for any point in time from 1987 to today.\n" +" https://cmi.ga.gov.au/data-products/dea/629/dea-" +"fractional-cover-landsat-c3\n" +" For service status information, see " +"https://status.dea.ga.gov.au" + +msgid "layer.ga_ls_fc_3.local_keywords" +msgstr "" + +#. bare_soil +msgid "layer.ga_ls_fc_3.bands.bs" +msgstr "bare_soil" + +#. photosynthetic_vegetation +msgid "layer.ga_ls_fc_3.bands.pv" +msgstr "photosynthetic_vegetation" + +#. non_photosynthetic_vegetation +msgid "layer.ga_ls_fc_3.bands.npv" +msgstr "non_photosynthetic_vegetation" + +#. ue +msgid "layer.ga_ls_fc_3.bands.ue" +msgstr "ue" + +#. Three-band Fractional Cover Unmasked (Warning: includes invalid data) +msgid "style.ga_ls_fc_3.fc_rgb_unmasked.title" +msgstr "Three-band Fractional Cover Unmasked (Warning: includes invalid data)" + +#. Fractional cover medians - red is bare soil, green is green vegetation and +#. blue is non-green vegetation +msgid "style.ga_ls_fc_3.fc_rgb_unmasked.abstract" +msgstr "" +"Fractional cover medians - red is bare soil, green is green vegetation " +"and blue is non-green vegetation" + +#. Landsat-8 Geomedian +msgid "layer.ls8_geomedian.title" +msgstr "Landsat-8 Geomedian" + +#. DEA Landsat-8 Geomedian +msgid "layer.ls8_geomedian.abstract" +msgstr "DEA Landsat-8 Geomedian" + +msgid "layer.ls8_geomedian.local_keywords" +msgstr "" + +#. red +msgid "layer.ls8_geomedian.bands.red" +msgstr "red" + +#. green +msgid "layer.ls8_geomedian.bands.green" +msgstr "green" + +#. blue +msgid "layer.ls8_geomedian.bands.blue" +msgstr "blue" + +#. nir +msgid "layer.ls8_geomedian.bands.nir" +msgstr "nir" + +#. swir1 +msgid "layer.ls8_geomedian.bands.swir1" +msgstr "swir1" + +#. swir2 +msgid "layer.ls8_geomedian.bands.swir2" +msgstr "swir2" + +#. sdev +msgid "layer.ls8_geomedian.bands.sdev" +msgstr "sdev" + +#. edev +msgid "layer.ls8_geomedian.bands.edev" +msgstr "edev" + +#. bcdev +msgid "layer.ls8_geomedian.bands.bcdev" +msgstr "bcdev" + +#. count +msgid "layer.ls8_geomedian.bands.count" +msgstr "count" + +#. Simple RGB +msgid "style.ls8_geomedian.simple_rgb.title" +msgstr "Simple RGB" + +#. Simple true-colour image, using the red, green and blue bands +msgid "style.ls8_geomedian.simple_rgb.abstract" +msgstr "Simple true-colour image, using the red, green and blue bands" + +#. False colour multi-band infra-red +msgid "style.ls8_geomedian.infra_red.title" +msgstr "False colour multi-band infra-red" + +#. Simple false-colour image, using the near and short-wave infra-red bands +msgid "style.ls8_geomedian.infra_red.abstract" +msgstr "Simple false-colour image, using the near and short-wave infra-red bands" + +#. NDVI +msgid "style.ls8_geomedian.ndvi.title" +msgstr "NDVI" + +#. Normalised Difference Vegetation Index - a derived index that correlates +#. well with the existence of vegetation +msgid "style.ls8_geomedian.ndvi.abstract" +msgstr "" +"Normalised Difference Vegetation Index - a derived index that correlates " +"well with the existence of vegetation" + +#. NDVI +msgid "style.ls8_geomedian.ndvi.legend.1.title" +msgstr "NDVI" + +#. dimensionless +msgid "style.ls8_geomedian.ndvi.legend.1.units" +msgstr "dimensionless" + +#. low +msgid "style.ls8_geomedian.ndvi.legend.1.lbl_0.0" +msgstr "low" + +#. high +msgid "style.ls8_geomedian.ndvi.legend.1.lbl_1.0" +msgstr "high" diff --git a/mypy.errs b/mypy.errs new file mode 100644 index 00000000..3af4aa23 --- /dev/null +++ b/mypy.errs @@ -0,0 +1 @@ +Success: no issues found in 56 source files diff --git a/pyproject.toml b/pyproject.toml index 6445f041..9d729ba4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,4 +4,4 @@ requires = ["setuptools>=65.5.1", "wheel>=0.38.1", "setuptools_scm[toml]>3.4"] [tool.setuptools_scm] write_to = "datacube_ows/_version.py" -fallback_version = "1.8.0-rc1" +fallback_version = "1.9.0rc1" diff --git a/s2_l2a_extractor.py b/s2_l2a_extractor.py new file mode 100644 index 00000000..87318284 --- /dev/null +++ b/s2_l2a_extractor.py @@ -0,0 +1,13 @@ +from datacube import Datacube +from yaml import dump + +dc = Datacube() + +i = 1 +for ds in dc.index.datasets.search(product="s2_l2a"): + filename = "s2_l2a_ds_%02d.yaml" % i + with open(filename, "w") as fp: + fp.write(dump(ds.metadata_doc, default_flow_style=False)) + print(filename, ds.uri) + + i = i + 1 diff --git a/setup.py b/setup.py index 510e4067..c0c23ad8 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ 'fsspec', 'lxml', 'deepdiff', + 'importlib_metadata', 'matplotlib', 'pyparsing', 'numpy>=1.22',