diff --git a/.bumpversion.cfg b/.bumpversion.cfg index a04430cd3..4bff7e4a6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,16 +1,15 @@ [bumpversion] -current_version = 1.1.4-dev10 +current_version = 1.3.3 commit = True tag = True -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+).(?P\d+))? serialize = - {major}.{minor}.{patch}-{release}{build} + {major}.{minor}.{patch}-{release}.{build} {major}.{minor}.{patch} tag_name = release-{new_version} [bumpversion:part:release] -optional_value = final -first_value = dev +optional_value = dev values = dev alpha diff --git a/.dockerignore b/.dockerignore index e38888922..5e37f5a31 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,8 @@ .git +.venv autotest build dist documentation -schemas \ No newline at end of file +schemas +debian/ diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..94ddb9046 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,31 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..bbcbbe7d6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e729b526..c2e61a4c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,127 +1,200 @@ name: CI -on: push +run-name: CI pipeline triggered by @${{ github.actor }} +on: + push: + paths: + - '.github/workflows/**' + - 'setup.py' + - 'setup.cfg' + - 'MANIFEST.in' + - 'pyproject.toml' + - 'eoxserver/**' + - 'autotest/**' + - 'requirements.txt' jobs: - run: + build-docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build the eoxserver docker image + run: | + docker pull eoxa/eoxserver:latest || true + docker build --cache-from eoxa/eoxserver:latest -t eoxserver . + docker save eoxserver | gzip > eoxserver.tar.gz + - name: Save image to cache + uses: actions/cache/save@v3 + with: + path: eoxserver.tar.gz + key: eoxserver.tar.gz-${{ github.run_id }}-${{ github.run_number }} + - name: Slack Notify + uses: 8398a7/action-slack@v3.8.0 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + if: failure() + + test: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - include: - # TODO: deactivated as django 1.11 does not seem to work with the GEOS version supplied by Ubuntu 20.04 - # - os: ubuntu - # python: py3 - # db: postgis - # django: "1.11.26" - # python_bin: python3 - # pip_bin: pip3 - - os: ubuntu - python: py3 - db: postgis - django: "3.2.12" - python_bin: python3 - pip_bin: pip3 - - os: ubuntu - python: py3 - db: spatialite - django: "3.2.12" - python_bin: python3 - pip_bin: pip3 - latest: true + command: + - "-m eoxserver.services.ows.wps.test_data_types" + - "-m eoxserver.services.ows.wps.test_allowed_values" + - "manage.py test --pythonpath=./eoxserver/ eoxserver.core -v2" + - "manage.py test --pythonpath=./eoxserver/ eoxserver.backends -v2" + - "manage.py test --pythonpath=./eoxserver/ eoxserver.services -v2" + - "manage.py test --pythonpath=./eoxserver/ eoxserver.resources.coverages -v2" + - "manage.py test autotest_services --tag wcs20 -v2" + - "manage.py test autotest_services --tag wcs11 -v2" + - "manage.py test autotest_services --tag wcs10 -v2" + - "manage.py test autotest_services --tag wms -v2" + - "manage.py test autotest_services --tag wps -v2" + - "manage.py test autotest_services --tag opensearch -v2" + needs: build-docker steps: - - uses: actions/checkout@v2 - - name: Build the eoxserver docker image + - uses: actions/checkout@v3 + - uses: actions/cache/restore@v3 + id: cache + with: + path: eoxserver.tar.gz + key: eoxserver.tar.gz-${{ github.run_id }}-${{ github.run_number }} + - name: Import docker image and name it autotest run: | - docker build -t eoxserver --build-arg DJANGO=${{ matrix.django }} -f docker/${{ matrix.os }}/${{ matrix.python }}/Dockerfile . + docker load --input eoxserver.tar.gz + docker tag eoxserver:latest eoxserver:autotest + - name: Start the services and install test dependencies + run: | + echo "DB=spatialite" >> sample.env + docker compose config + docker compose up -d + docker compose ps + docker exec -i eoxserver-autotest-1 pip3 install scipy - name: Run the tests env: COMPOSE_INTERACTIVE_NO_CLI: 1 run: | - cd autotest - echo "DB=${{ matrix.db }}" >> eoxserver.env - docker-compose config - docker-compose up -d - docker-compose ps - docker exec -i autotest_autotest_1 ${{ matrix.pip_bin }} install scipy - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} -m eoxserver.services.ows.wps.test_data_types - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} -m eoxserver.services.ows.wps.test_allowed_values - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=../eoxserver/ eoxserver.core -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=../eoxserver/ eoxserver.backends -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=../eoxserver/ eoxserver.services -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test --pythonpath=../eoxserver/ eoxserver.resources.coverages -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wcs20 -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wcs11 -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wcs10 -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wms -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag wps -v2 - docker exec -i autotest_autotest_1 ${{ matrix.python_bin }} manage.py test autotest_services --tag opensearch -v2 - cd .. + docker exec -i eoxserver-autotest-1 python3 ${{ matrix.command }} - name: Upload logs and outputs of failed tests - uses: 'actions/upload-artifact@v2' + uses: 'actions/upload-artifact@v3' with: - name: logs ${{ matrix.python }} ${{ matrix.db }} ${{ matrix.django }} + name: logs ${{ matrix.command }} path: | autotest/autotest/logs/*.log autotest/autotest/responses/* retention-days: 5 if: failure() + - name: Slack Notify + uses: 8398a7/action-slack@v3.8.0 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + if: failure() - # get branch/tag name for later stages + publish-docker: + runs-on: ubuntu-latest + needs: test + if: contains(github.ref, 'refs/tags/') || github.ref == 'refs/heads/master' + steps: + - uses: actions/checkout@v3 + - uses: actions/cache/restore@v3 + id: cache + with: + path: eoxserver.tar.gz + key: eoxserver.tar.gz-${{ github.run_id }}-${{ github.run_number }} - name: Branch name id: branch_name run: | - echo ::set-output name=SOURCE_BRANCH:: $([[ $GITHUB_REF == refs/heads/* ]] && echo ${GITHUB_REF#refs/heads/} || echo "") echo ::set-output name=SOURCE_TAG::$([[ $GITHUB_REF == refs/tags/* ]] && echo ${GITHUB_REF#refs/tags/} || echo "") - - # docker image tagging/publishing - name: Login to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} if: success() - - # conditionally tag docker images and push them to dockerhub - - name: Tag docker latest master image + - name: Import docker image run: | - docker tag eoxserver eoxa/eoxserver:master - if: success() && steps.branch_name.outputs.SOURCE_BRANCH == 'master' && matrix.latest + docker load --input eoxserver.tar.gz - name: Tag docker latest image run: | docker tag eoxserver eoxa/eoxserver:latest - if: success() && matrix.latest - - name: Tag docker latest release image + if: github.ref == 'refs/heads/master' + - name: Tag docker release image run: | docker tag eoxserver eoxa/eoxserver:${{ steps.branch_name.outputs.SOURCE_TAG }} - if: success() && steps.branch_name.outputs.SOURCE_TAG && matrix.latest - - name: Tag docker release image with OS and Python/Django versions - run: | - docker tag eoxserver eoxa/eoxserver:${{ steps.branch_name.outputs.SOURCE_TAG }}-${{ matrix.os }}-${{ matrix.python }}-django${{ matrix.django }} - if: success() && startsWith(steps.branch_name.outputs.SOURCE_TAG, 'release-') + if: steps.branch_name.outputs.SOURCE_TAG - name: Push docker images run: | # TODO: --all-tags does not seem to work with the version on github-actions # docker push --all-tags eoxa/eoxserver for tag in $(docker image ls --format "{{.Tag}}" eoxa/eoxserver) ; do docker push "eoxa/eoxserver:$tag" ; done if: success() + - name: Slack Notify + uses: 8398a7/action-slack@v3.8.0 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + if: failure() - # build a Python package and publish it on pypi + publish-pypi: + runs-on: ubuntu-latest + needs: test + if: contains(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v3 + - name: Branch name + id: branch_name + run: | + echo ::set-output name=SOURCE_TAG::$([[ $GITHUB_REF == refs/tags/* ]] && echo ${GITHUB_REF#refs/tags/} || echo "") - name: Build Python package id: build_python_release run: | python -m pip install --upgrade pip pip install setuptools wheel + sudo apt-get install -y gdal-bin python setup.py sdist bdist_wheel - echo ::set-output name=WHEEL_FILE::$(ls dist/*.whl) - echo ::set-output name=SRC_DIST_FILE::$(ls dist/*.tar.gz) + - uses: actions/upload-artifact@v3 + with: + name: eoxserver-dist + path: ./dist/ + retention-days: 2 - name: Push package to pypi uses: pypa/gh-action-pypi-publish@master with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} - if: success() && matrix.latest && startsWith(steps.branch_name.outputs.SOURCE_TAG, 'release-') + if: success() + - name: Slack Notify + uses: 8398a7/action-slack@v3.8.0 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + if: failure() - # draft a github release and add files - - name: Create Release + publish-github: + runs-on: ubuntu-latest + needs: publish-pypi + if: contains(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3.0.1 + with: + name: eoxserver-dist + - name: Branch name + id: branch_name + run: | + echo ::set-output name=SOURCE_TAG::$([[ $GITHUB_REF == refs/tags/* ]] && echo ${GITHUB_REF#refs/tags/} || echo "") + echo ::set-output name=WHEEL_FILE::$(ls *.whl) + echo ::set-output name=SRC_DIST_FILE::$(ls *.tar.gz) + - name: Draft Release id: create_release uses: actions/create-release@v1 env: @@ -130,38 +203,50 @@ jobs: tag_name: ${{ github.ref }} release_name: ${{ steps.branch_name.outputs.SOURCE_TAG }} draft: true - if: success() && matrix.latest && startsWith(steps.branch_name.outputs.SOURCE_TAG, 'release-') - name: Upload Release Asset Wheel uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ steps.build_python_release.outputs.WHEEL_FILE }} - asset_name: ${{ steps.build_python_release.outputs.WHEEL_FILE }} + asset_path: ${{ steps.branch_name.outputs.WHEEL_FILE }} + asset_name: ${{ steps.branch_name.outputs.WHEEL_FILE }} asset_content_type: application/x-wheel+zip - if: success() && matrix.latest && startsWith(steps.branch_name.outputs.SOURCE_TAG, 'release-') + if: success() - name: Upload Release Asset Source Dist uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps - asset_path: ${{ steps.build_python_release.outputs.SRC_DIST_FILE }} - asset_name: ${{ steps.build_python_release.outputs.SRC_DIST_FILE }} + asset_path: ${{ steps.branch_name.outputs.SRC_DIST_FILE }} + asset_name: ${{ steps.branch_name.outputs.SRC_DIST_FILE }} asset_content_type: application/tar+gzip - if: success() && matrix.latest && startsWith(steps.branch_name.outputs.SOURCE_TAG, 'release-') + if: success() + - name: Slack Notify + uses: 8398a7/action-slack@v3.8.0 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + if: failure() - note: + notify: runs-on: ubuntu-20.04 - needs: run + needs: [publish-github, publish-docker] steps: - # send Slack notifications to the eox organization - - name: action-slack + - name: Slack Notify uses: 8398a7/action-slack@v3.8.0 with: - status: ${{ needs.run.result }} + status: ${{ job.status }} fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + custom_payload: | + { + attachments: [{ + text: `Publish tag finished`, + }] + } env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} if: always() diff --git a/.gitignore b/.gitignore index fe21e32f2..3719d12cd 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ .devcontainer/ instances/ +.minio.sys/ diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..6baa2a96e --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +office@eox.at. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..7eb3b150d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,144 @@ +# Contributing + +## EOxServer code style guide + +This section tries to establish a set of rules to help harmonizing the source +code written by many contributors. The goal is to improve compatibility and +maintainability. + +Above all rules, adhere the rules defined in the [Python PEP 8](https://www.python.org/dev/peps/pep-0008/). Please try to adhere the +mentioned code styles. You can check if you compliant to the style guide with +the `flake8` command line utilities. + +Then: +``` +>>> import this +The Zen of Python, by Tim Peters + +Beautiful is better than ugly. +Explicit is better than implicit. +Simple is better than complex. +Complex is better than complicated. +Flat is better than nested. +Sparse is better than dense. +Readability counts. +Special cases aren't special enough to break the rules. +Although practicality beats purity. +Errors should never pass silently. +Unless explicitly silenced. +In the face of ambiguity, refuse the temptation to guess. +There should be one-- and preferably only one --obvious way to do it. +Although that way may not be obvious at first unless you're Dutch. +Now is better than never. +Although never is often better than *right* now. +If the implementation is hard to explain, it's a bad idea. +If the implementation is easy to explain, it may be a good idea. +Namespaces are one honking great idea -- let's do more of those! +``` + +### Package layout and namespacing + +Use Python package structures to enable hierarchical namespaces. Do not encode +the namespace in function or class names. + +E.g: don't do this: +```python +# somemodule.py + +def myNS_FunctionA(): + pass + +class myNS_ClassB(): + pass +``` +Instead, do this: +```python +# somemodule/myNS.py + +def functionA(): + pass + +class ClassB(): + pass +``` + +A developer using these functions can choose to use the namespace explicitly: +``` +from somepackage import myNS + +myNS.functionA() +c = myNS.ClassB() +``` + +# Import rules + +As defined in Python PEP 8, place all imports in the top of the file. This makes +it easier to trace dependencies and allows to see and resolve importing issues. + +Try to use the following importing order: + +1. Standard library imports or libraries that can be seen as industry standard (like numpy). +2. Third party libraries (or libraries that are not directly associated with the current project). E.g: GDAL, Django, etc. +3. Imports that are directly associated with the current project. In case of EOxServer, everything that is under the :mod:`eoxserver` package root. + +Use single empty lines to separate these import groups. + +## Coding guidelines + +### Minimizing pitfalls + +Don't use mutable types as default arguments + +As default arguments are evaluated at the time the module is imported and not +when the function/method is called, default arguments are a sort of global +variable and calling the function can have unintended side effects. Consider the +following example: +```python + def add_one(arg=[]): + arg.append(1) + print arg +``` +When called with no arguments, the function will print different results every +time it is invoked. Also, since the list will never be released, this is also a +memory leak, as the list grows with the time. + + +Don't put code in the `__init__.py` of a package + +When importing a package or a module from a package, the packages +`__init__.py` will first be imported. If there is production code included +(which will likely be accompanied by imports) this can lead to unintended +circular imports. Try to put all production code in modules instead, and use the +`__init__.py` only for really necessary stuff. + + +Use abbreviations sparingly + +Try not to use abbreviations, unless the meaning is commonly known. Examples +are HTTP, URL, WCS, BBox or the like. + +Don't use leading double underscores to specify 'private' fields or methods or +module functions, unless *really* necessary (which it isn't, usually). Using +double underscores makes it unnecessarily hard to debug methods/fields and is +still not really private, as compared to other languages like C++ or Java. Use +single leading underscores instead. The meaning is clear to any programmer and +it does not impose any unnecessary comlications during debugging. + + +## Improving tests + +### General rules + +Implementing new features shall *always* incorporate writing new tests! Try to +find corner/special cases and also try to find cases that shall provoke +exceptions. + +Where to add the tests? + +Try to let tests *fail* by calling the correct assertion or the +`fail` functions. Don't use exceptions (apart from `AssertionError`), +because when running the tests, this will be visible as "Error" and not a simple +failure. Test errors should indicate that something completely unexpected +happened that broke the testing code. + +For more information on development check out [the docs](https://docs.eoxserver.org/en/stable/developers/index.html) diff --git a/docker/ubuntu/py3/Dockerfile b/Dockerfile similarity index 63% rename from docker/ubuntu/py3/Dockerfile rename to Dockerfile index 89468b369..a132323bd 100644 --- a/docker/ubuntu/py3/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ FROM ubuntu:22.04 -ARG DJANGO=3.2.12 -ARG TZ=UTC - ENV INSTANCE_NAME=instance +ENV TZ=UTC +ENV PYTHONPATH='/opt/eoxserver' +ENV PYTHONUNBUFFERED="1" # possible values are "postgis" and "spatialite" ENV DB=spatialite @@ -24,7 +24,6 @@ ENV INIT_SCRIPTS='' ENV GUNICORN_CMD_ARGS "--config /opt/eoxserver/gunicorn.conf.py ${INSTANCE_NAME}.wsgi:application" # install OS dependency packages -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN apt-get update \ && apt-get install -y \ python3 \ @@ -40,24 +39,22 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/partial/* /tmp/* /var/tmp/* -# install EOxServer -RUN mkdir /opt/eoxserver/ - -ADD eoxserver /opt/eoxserver/eoxserver -ADD tools /opt/eoxserver/tools -ADD setup.cfg setup.py MANIFEST.in README.rst requirements.txt /opt/eoxserver/ -ADD docker/eoxserver-entrypoint.sh /opt/eoxserver/ -ADD docker/gunicorn.conf.py /opt/eoxserver/ +ENV PROMETHEUS_MULTIPROC_DIR /var/tmp/prometheus_multiproc_dir +RUN mkdir $PROMETHEUS_MULTIPROC_DIR # make sure this is writable by webserver user -# install EOxServer and its dependencies +RUN mkdir /opt/eoxserver/ WORKDIR /opt/eoxserver -RUN pip3 install -r requirements.txt \ - && pip3 install "django==$DJANGO" \ - && pip3 install . +# install dependencies +COPY requirements.txt . +RUN python3 -m pip install -U pip \ + && python3 -m pip install --no-cache-dir -r requirements.txt + +# install EOxServer +COPY . . EXPOSE 8000 -ENTRYPOINT ["/opt/eoxserver/eoxserver-entrypoint.sh"] +ENTRYPOINT ["/opt/eoxserver/entrypoint.sh"] -CMD ["gunicorn", "$GUNICORN_CMD_ARGS"] +CMD "gunicorn" $GUNICORN_CMD_ARGS diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..5f1b46848 --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +EOxServer Open License +Version 1, 8 June 2011 + +Copyright (C) 2011 EOX IT Services GmbH + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the “Software”), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies +of this Software or works derived from this Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in index de8fb00be..bbdc47152 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include README.rst +include README.md include MANIFEST.in graft eoxserver recursive-exclude eoxserver *.pyc diff --git a/README.md b/README.md new file mode 100644 index 000000000..fcc9204cb --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +# EOxServer + +EOxServer is a Python application and library for presenting Earth +Observation (EO) data and metadata. + +![build](https://github.com/EOxServer/eoxserver/actions/workflows/ci.yml/badge.svg) +[![PyPi](https://badge.fury.io/py/EOxServer.svg)](https://pypi.org/project/EOxServer/) +[![ReadTheDocs](https://readthedocs.org/projects/eoxserver/badge/?version=master)](http://docs.eoxserver.org/en/master) + +EOxServer implements the [OGC](http://www.opengeospatial.org) +Implementation Specifications EO-WCS and EO-WMS on top of +[MapServer's](http://mapserver.org) [WCS](http://www.opengeospatial.org/standards/wcs) and +[WMS](http://www.opengeospatial.org/standards/wms) implementations. +EOxServer is released under the +[EOxServer Open License](https://docs.eoxserver.org/en/stable/copyright.html) an MIT-style +license and written in python and entirely based on open source software including: + +- [MapServer](http://mapserver.org) +- [Django/GeoDjango](https://www.djangoproject.com) +- [GDAL](http://www.gdal.org>) +- [SpatiaLite](http://www.gaia-gis.it/spatialite) +- [PostGIS](http://postgis.refractions.net/>) +- [PROJ.4](http://trac.osgeo.org/proj/>) + +More information is available at [https://eoxserver.org](https://eoxserver.org). Documentation +is available at [readthedocs](https://docs.eoxserver.org/en/stable/) + +## Docker + +To run with SpatiaLite database simply run: + +```sh +docker run -it --rm -p 8080:8000 eoxa/eoxserver +``` + +EOxServer is now accessible at [http://localhost:8080/](http://localhost:8080/). +And you can login to the `Admin Client` using: + +- username: admin +- password: admin + +The following environment variables control configuration: + +- `DB`: Specify the used database type. either `spatialite` or `postgis` +- `DB_PW`, `DB_NAME`, `DB_HOST`, `DB_USER`: these credentials will be used to establish a + connection to the postgres database when DB is set to `postgis` in order to wait + for it to come online +- `INSTANCE_NAME`: the name of the instance passed to `eoxserver-instance.py` - defaults + to `instance` +- `INSTANCE_DIR`: the directory of the instance. Defaults to `/opt/instance` +- `DJANGO_USER`, `DJANGO_MAIL`, `DJANGO_PASSWORD`: when set, these credentials will be + used to create a superuser to be used for the Django Admin. By default, no user is + created +- `COLLECT_STATIC`: if set to "true" (the default), static files will be collected + upon initialization +- `PREINIT_SCRIPTS`: the list of commands that will be executed before + the instance is initialized +- `INIT_SCRIPTS`: the list of commands that will be executed once + when the instance is initialized +- `STARTUP_SCRIPTS`: the list of commands that will be executed before + the command is run +- `GUNICORN_CMD_ARGS`: gunicorn command arguments. Defaults to + `--config /opt/eoxserver/gunicorn.conf.py ${INSTANCE_NAME}.wsgi:application` + +## Development + +The autotest instance can be used for development and testing. +More information in `./autotest/README.md` diff --git a/README.rst b/README.rst deleted file mode 100644 index afb24338d..000000000 --- a/README.rst +++ /dev/null @@ -1,34 +0,0 @@ -EOxServer -========= - -.. image:: https://travis-ci.org/EOxServer/eoxserver.svg?branch=master - :target: https://travis-ci.org/EOxServer/eoxserver - -.. image:: https://badge.fury.io/py/EOxServer.svg - :target: https://pypi.org/project/EOxServer/ - -.. image:: https://readthedocs.org/projects/eoxserver/badge/?version=master - :alt: Documentation Status - :scale: 100% - :target: http://docs.eoxserver.org/en/master - -EOxServer is a Python application and framework for presenting Earth -Observation (EO) data and metadata. - -EOxServer implements the `OGC `_ -Implementation Specifications EO-WCS and EO-WMS on top of -`MapServer's `_ -`WCS `_ and -`WMS `_ implementations. - -EOxServer is released under the `EOxServer Open License -`_ a MIT-style -license and written in `Python `_ and entirely based on -Open Source software including `MapServer `_, -`Django/GeoDjango `_, -`GDAL `_, -`SpatiaLite `_, or -`PostGIS `_, and -`PROJ.4 `_. - -More information is available at `http://eoxserver.org `_. diff --git a/autotest/HOWTO b/autotest/HOWTO deleted file mode 100644 index fdbdeb912..000000000 --- a/autotest/HOWTO +++ /dev/null @@ -1,247 +0,0 @@ -#------------------------------------------------------------------------------- -# -# Project: EOxServer -# Authors: Stephan Krause -# Stephan Meissl -# Martin Paces -# -#------------------------------------------------------------------------------- -# Copyright (C) 2011 EOX IT Services GmbH -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies of this Software or works derived from this Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -#------------------------------------------------------------------------------- - - -################################################################################ -# Autotest HowTos # -################################################################################ - - -+------------------------------------------------------------------------------+ -| 1. How to configure the autotest instance using vagrant (recommended) | -+------------------------------------------------------------------------------+ - -# See HOWTO in vagrant directory of EOxServer repository -# https://github.com/EOxServer/eoxserver.git -# https://github.com/EOxServer/eoxserver/tree/master/vagrant/HOWTO - - -+------------------------------------------------------------------------------+ -| 2. How to configure the autotest instance without vagrant (not recommended) | -+------------------------------------------------------------------------------+ - -# Clone EOxServer -git clone git@github.com:EOxServer/eoxserver.git -cd eoxserver/ -cd autotest/ - -# Configure database -vi settings.py - -python manage.py syncdb --noinput -python manage.py loaddata auth_data.json initial_rangetypes.json - - -+------------------------------------------------------------------------------+ -| 3. How to generate a custom EOxServer instance | -+------------------------------------------------------------------------------+ - -# See also prepare_instance.sh in jenkins directory of EOxServer repository -# https://github.com/EOxServer/eoxserver.git -# https://github.com/EOxServer/eoxserver/tree/master/jenkins/prepare_instance.sh - -eoxserver-admin.py create_instance --init_spatialite -cd / -python manage.py syncdb - - -+------------------------------------------------------------------------------+ -| 4. How to run tests | -+------------------------------------------------------------------------------+ - -# See also run_tests.sh in jenkins directory of EOxServer repository -# https://github.com/EOxServer/eoxserver.git -# https://github.com/EOxServer/eoxserver/tree/master/jenkins/run_tests.sh - -# Perform steps in "1. How to configure the autotest instance using vagrant" - -vagrant ssh -cd /var/eoxserver/autotest/ -export XML_CATALOG_FILES="../schemas/catalog.xml" -python manage.py test - -# autotest_services only -python manage.py test autotest_services -v2 - -# all modules -python manage.py test autotest_services services coverages backends processes core - -# or simply -python manage.py test -# this also runs some django tests. - - -# Running single tests -python manage.py test autotest_services. -# e.g. -python manage.py test autotest_services.WCS20GetCapabilities - - -+------------------------------------------------------------------------------+ -| 5. How to load test data | -+------------------------------------------------------------------------------+ - -# Perform steps in "1. How to configure the autotest instance using vagrant" - -vagrant ssh -cd /var/eoxserver/autotest/ -python manage.py loaddata data/fixtures/some_fixture.json ... - -# To load all test fixtures: -python manage.py loaddata auth_data.json range_types.json \ - testing_base.json testing_coverages.json \ - testing_asar_base.json testing_asar.json \ - testing_reprojected_coverages.json - -List of fixtures: - * initial_data.json - Base data to enable components. Loaded with syncdb. - * auth_data.json - An administration account. - * range_types.json - Range types for RGB and gray-scale coverages. - * testing_base.json - Range type for the 15 band uint16 test data. - * testing_coverages.json - Metadata for the MERIS test data. - * testing_asar_base.json - Range type for the ASAR test data. - * testing_asar.json - Metadata for the ASAR test data. - * testing_rasdaman_coverages.json - Use this fixtures in addition when - rasdaman is installed and configured. - * testing_backends.json - This fixtures are used for testing the backend - layer only and shouldn't be loaded in the test - instance. - * testing_reprojected_coverages.json - Metadata for the reprojected - MERIS test data. - - -+------------------------------------------------------------------------------+ -| 6. How to run development server | -+------------------------------------------------------------------------------+ - -# Perform steps in "1. How to configure the autotest instance using vagrant" -# Optionally perform steps in "5. How to load test data" - -vagrant ssh -cd /var/eoxserver/autotest/ -python manage.py runserver 0.0.0.0:8000 - -# Access server -http://localhost:8001/ - - -+------------------------------------------------------------------------------+ -| 7. How to update fixtures | -+------------------------------------------------------------------------------+ - -vagrant ssh -cd /var/eoxserver/autotest/ -python manage.py dumpdata --format=json --indent=4 > tmp.json -# Inspect file e.g. with: meld tmp.json data/fixtures/initial_data.json -mv tmp.json data/fixtures/ - - -+------------------------------------------------------------------------------+ -| 8. How to add expected results | -+------------------------------------------------------------------------------+ - -# To format XML files in a pretty way use the following command -xmllint --format > -mv - - -+------------------------------------------------------------------------------+ -| 9. How to compare XML documents | -+------------------------------------------------------------------------------+ - -# To compare an expected XML document with the actual XML response -# use the following command -vagrant ssh -cd /var/eoxserver/autotest/ -../tools/xcomp.py responses/ expected/ - -#The XML comparator parses the XML documents and compares -#the documents' trees, therefore the tool is able to cope with -#different formatting (including different order of elements' -#attributes and various name-space prefixes). - - -+------------------------------------------------------------------------------+ -| 10. How to validate XML documents | -+------------------------------------------------------------------------------+ - -export XML_CATALOG_FILES="/schemas/catalog.xml" - -xmllint --noout --schema http://schemas.opengis.net/wcseo/1.0/wcsEOAll.xsd - - -+------------------------------------------------------------------------------+ -| 11. How to run schematron tests | -+------------------------------------------------------------------------------+ - -export XML_CATALOG_FILES="/schemas/catalog.xml" - -cd /schemas/ -xsltproc schematron_xslt1/iso_dsdl_include.xsl wcseo/1.0/wcsEOSchematron.sch | xsltproc schematron_xslt1/iso_abstract_expand.xsl - | xsltproc schematron_xslt1/iso_svrl_for_xslt1.xsl - | xsltproc - - - -+------------------------------------------------------------------------------+ -| 12. How to run all tests | -+------------------------------------------------------------------------------+ - -# See also run_tests.sh in jenkins directory of EOxServer repository -# https://github.com/EOxServer/eoxserver.git -# https://github.com/EOxServer/eoxserver/tree/master/jenkins/run_tests.sh - -1. Run tests (see above) -2. Run Selenium tests -3. Run test instance, load data via CLI commands, run some requests, and - compare responses with unit tests results -4. If libxml is built with schematron support run schematron tests (see - above) - - -+------------------------------------------------------------------------------+ -| 13. How to reset the autotest instance | -+------------------------------------------------------------------------------+ - -# Perform steps in "1. How to configure the autotest instance using vagrant" - -vagrant ssh -cd /var/eoxserver/autotest/ - -sudo service httpd stop - -# Reset DB with PostgreSQL: -sudo su postgres -c "dropdb eoxserver_testing" -sudo su postgres -c "createdb -O eoxserver -T template_postgis eoxserver_testing" - -python manage.py syncdb --noinput --traceback -python manage.py loaddata auth_data.json initial_rangetypes.json --traceback - -# Reset EOxServer -rm -f autotest/logs/eoxserver.log -touch autotest/logs/eoxserver.log - -sudo service httpd start diff --git a/autotest/MANIFEST.in b/autotest/MANIFEST.in index 4b9da0dac..5e55e167e 100644 --- a/autotest/MANIFEST.in +++ b/autotest/MANIFEST.in @@ -1,6 +1,5 @@ -include README.rst +include README.md include MANIFEST.in -include HOWTO graft autotest graft selenium graft scripts diff --git a/autotest/README.md b/autotest/README.md new file mode 100644 index 000000000..7f65c37b2 --- /dev/null +++ b/autotest/README.md @@ -0,0 +1,184 @@ +# autotest + +Autotest is an instance of eoxserver used for running tests. + +## How to configure autotest + +Configure database + +```sh +cd autotest/ +vi settings.py + +python manage.py syncdb --noinput +python manage.py loaddata auth_data.json initial_rangetypes.json +``` + +## How to generate a custom EOxServer instance + +```sh +eoxserver-admin.py create_instance --init_spatialite +cd / +python manage.py syncdb +``` + +## How to run tests + +Perform steps outlined in `How to configure autotest` + +```sh +cd /var/eoxserver/autotest/ +export XML_CATALOG_FILES="../schemas/catalog.xml" +python manage.py test +``` + +Autotest_services only + +```sh +python manage.py test autotest_services -v2 +``` + +Test all modules + +```sh +python manage.py test autotest_services services coverages backends processes core +``` + +or simply + +```sh +python manage.py test +``` + +Running single tests + +```sh +python manage.py test autotest_services.WCS20GetCapabilities +``` + +## How to load test data + +Perform steps mentioned in `How to configure autotest` + +```sh +cd /var/eoxserver/autotest/ +python manage.py loaddata data/fixtures/some_fixture.json ... + +# To load all test fixtures: +python manage.py loaddata auth_data.json range_types.json \ + testing_base.json testing_coverages.json \ + testing_asar_base.json testing_asar.json \ + testing_reprojected_coverages.json +``` + +List of fixtures: + +* initial_data.json - Base data to enable components. Loaded with syncdb. +* auth_data.json - An administration account. +* range_types.json - Range types for RGB and gray-scale coverages. +* testing_base.json - Range type for the 15 band uint16 test data. +* testing_coverages.json - Metadata for the MERIS test data. +* testing_asar_base.json - Range type for the ASAR test data. +* testing_asar.json - Metadata for the ASAR test data. +* testing_rasdaman_coverages.json - Use this fixtures in addition when + rasdaman is installed and configured. +* testing_backends.json - This fixtures are used for testing the backend + layer only and shouldn't be loaded in the test + instance. +* testing_reprojected_coverages.json - Metadata for the reprojected + MERIS test data. + +## How to run development server + +Perform steps mentioned in `How to configure autotest` + +Optionally perform steps mentioned in `How to load test data` + +```sh +cd /var/eoxserver/autotest/ +python manage.py runserver 0.0.0.0:8000 +``` + +The server is running on [http://localhost:8000/] + +## How to update fixtures + +```sh +cd /var/eoxserver/autotest/ +python manage.py dumpdata --format=json --indent=4 > tmp.json +# Inspect file e.g. with: meld tmp.json data/fixtures/initial_data.json +mv tmp.json data/fixtures/ +``` + +## How to add expected results + +To format XML files in a pretty way use the following command + +```sh +xmllint --format > +mv +``` + +## How to compare XML documents + +To compare an expected XML document with the actual XML response use the following command + +```sh +cd /var/eoxserver/autotest/ +../tools/xcomp.py responses/ expected/ +``` + +The XML comparator parses the XML documents and compares the documents' trees, therefore the tool is able to cope with different formatting (including different order of elements attributes and various name-space prefixes). + +## How to validate XML documents + +```sh +export XML_CATALOG_FILES="/schemas/catalog.xml" +xmllint --noout --schema http://schemas.opengis.net/wcseo/1.0/wcsEOAll.xsd +``` + +## How to run schematron tests + +```sh +export XML_CATALOG_FILES="/schemas/catalog.xml" + +cd /schemas/ +xsltproc schematron_xslt1/iso_dsdl_include.xsl wcseo/1.0/wcsEOSchematron.sch | xsltproc schematron_xslt1/iso_abstract_expand.xsl - | xsltproc schematron_xslt1/iso_svrl_for_xslt1.xsl - | xsltproc - +``` + +## How to run all tests + +1. Run tests (see above) +2. Run Selenium tests +3. Run test instance, load data via CLI commands, run some requests, and + compare responses with unit tests results +4. If libxml is built with schematron support run schematron tests (see + above) + +## How to reset the autotest instance + +Perform steps outlined in `How to configure autotest` + +```sh +cd /var/eoxserver/autotest/ +sudo service httpd stop +``` + +Reset DB with PostgreSQL: + +```sh +sudo su postgres -c "dropdb eoxserver_testing" +sudo su postgres -c "createdb -O eoxserver -T template_postgis eoxserver_testing" + +python manage.py syncdb --noinput --traceback +python manage.py loaddata auth_data.json initial_rangetypes.json --traceback +``` + +Reset EOxServer + +```sh +rm -f autotest/logs/eoxserver.log +touch autotest/logs/eoxserver.log + +sudo service httpd start +``` diff --git a/autotest/README.rst b/autotest/README.rst deleted file mode 100644 index d8d4e16ac..000000000 --- a/autotest/README.rst +++ /dev/null @@ -1,25 +0,0 @@ -autotest instance for EOxServer -=============================== - -autotest instance for EOxServer used for unit and integration tests. - -EOxServer is a Python application and framework for presenting Earth -Observation (EO) data and metadata. - -EOxServer implements the `OGC `_ -Implementation Specifications EO-WCS and EO-WMS on top of -`MapServer's `_ -`WCS `_ and -`WMS `_ implementations. - -EOxServer is released under the `EOxServer Open License -`_ a MIT-style -license and written in `Python `_ and entirely based on -Open Source software including `MapServer `_, -`Django/GeoDjango `_, -`GDAL `_, -`SpatiaLite `_, or -`PostGIS `_, and -`PROJ.4 `_. - -More information is available at `http://eoxserver.org `_. diff --git a/autotest/autotest/data/CLM_clipped.tif b/autotest/autotest/data/CLM_clipped.tif new file mode 100644 index 000000000..1d943bf6b Binary files /dev/null and b/autotest/autotest/data/CLM_clipped.tif differ diff --git a/autotest/autotest/data/SCL/SCL_clipped.tif b/autotest/autotest/data/SCL/SCL_clipped.tif new file mode 100644 index 000000000..1ee374349 Binary files /dev/null and b/autotest/autotest/data/SCL/SCL_clipped.tif differ diff --git a/autotest/autotest/data/SCL/fixtures.json b/autotest/autotest/data/SCL/fixtures.json new file mode 100644 index 000000000..92f556dfc --- /dev/null +++ b/autotest/autotest/data/SCL/fixtures.json @@ -0,0 +1,376 @@ +[ +{ + "model": "coverages.fieldtype", + "pk": 1, + "fields": { + "coverage_type": 1, + "index": 0, + "identifier": "scl", + "description": "SCL", + "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", + "unit_of_measure": "nil", + "wavelength": null, + "significant_figures": 2, + "numbits": 8, + "signed": false, + "is_float": false + } +}, +{ + "model": "coverages.allowedvaluerange", + "pk": 1, + "fields": { + "field_type": 1, + "start": 0.0, + "end": 11.0 + } +}, +{ + "model": "coverages.nilvalue", + "pk": 1, + "fields": { + "value": "0", + "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", + "field_types": [ + 1 + ] + } +}, +{ + "model": "coverages.coveragetype", + "pk": 1, + "fields": { + "name": "SCL" + } +}, +{ + "model": "coverages.producttype", + "pk": 1, + "fields": { + "name": "SCL", + "allowed_coverage_types": [ + 1 + ] + } +}, +{ + "model": "coverages.browsetype", + "pk": 1, + "fields": { + "product_type": 1, + "name": "SCL", + "red_or_grey_expression": "scl", + "green_expression": null, + "blue_expression": null, + "alpha_expression": null, + "red_or_grey_nodata_value": null, + "green_nodata_value": null, + "blue_nodata_value": null, + "alpha_nodata_value": null, + "red_or_grey_range_min": null, + "green_range_min": null, + "blue_range_min": null, + "alpha_range_min": null, + "red_or_grey_range_max": null, + "green_range_max": null, + "blue_range_max": null, + "alpha_range_max": null, + "show_out_of_bounds_data": false + } +}, +{ + "model": "coverages.rasterstyle", + "pk": 1, + "fields": { + "name": "SCL", + "type": "values", + "title": null, + "abstract": null + } +}, +{ + "model": "coverages.rasterstyletobrowsetypethrough", + "pk": 1, + "fields": { + "raster_style": 1, + "browse_type": 1, + "style_name": "color" + } +}, +{ + "model": "coverages.rasterstylecolorentry", + "pk": 1, + "fields": { + "raster_style": 1, + "value": 0.0, + "color": "#000000", + "opacity": 1.0, + "label": "NO_DATA" + } +}, +{ + "model": "coverages.rasterstylecolorentry", + "pk": 2, + "fields": { + "raster_style": 1, + "value": 1.0, + "color": "#ff0000", + "opacity": 1.0, + "label": "SATURATED_OR_DEFECTIVE" + } +}, +{ + "model": "coverages.rasterstylecolorentry", + "pk": 3, + "fields": { + "raster_style": 1, + "value": 2.0, + "color": "#2e2e2e", + "opacity": 1.0, + "label": "DARK_AREA_PIXELS" + } +}, +{ + "model": "coverages.rasterstylecolorentry", + "pk": 4, + "fields": { + "raster_style": 1, + "value": 3.0, + "color": "#541800", + "opacity": 1.0, + "label": "CLOUD_SHADOWS" + } +}, +{ + "model": "coverages.rasterstylecolorentry", + "pk": 5, + "fields": { + "raster_style": 1, + "value": 4.0, + "color": "#46e800", + "opacity": 1.0, + "label": "VEGETATION" + } +}, +{ + "model": "coverages.rasterstylecolorentry", + "pk": 6, + "fields": { + "raster_style": 1, + "value": 5.0, + "color": "#ffff00", + "opacity": 1.0, + "label": "NOT_VEGETATED" + } +}, +{ + "model": "coverages.rasterstylecolorentry", + "pk": 7, + "fields": { + "raster_style": 1, + "value": 6.0, + "color": "#0000ff", + "opacity": 1.0, + "label": "WATER" + } +}, +{ + "model": "coverages.rasterstylecolorentry", + "pk": 8, + "fields": { + "raster_style": 1, + "value": 7.0, + "color": "#525252", + "opacity": 1.0, + "label": "UNCLASSIFIED" + } +}, +{ + "model": "coverages.rasterstylecolorentry", + "pk": 9, + "fields": { + "raster_style": 1, + "value": 8.0, + "color": "#787878", + "opacity": 1.0, + "label": "CLOUD_MEDIUM_PROBABILITY" + } +}, +{ + "model": "coverages.rasterstylecolorentry", + "pk": 10, + "fields": { + "raster_style": 1, + "value": 9.0, + "color": "#b5b5b5", + "opacity": 1.0, + "label": "CLOUD_HIGH_PROBABILITY" + } +}, +{ + "model": "coverages.rasterstylecolorentry", + "pk": 11, + "fields": { + "raster_style": 1, + "value": 10.0, + "color": "#00b6bf", + "opacity": 1.0, + "label": "THIN_CIRRUS" + } +}, +{ + "model": "coverages.rasterstylecolorentry", + "pk": 12, + "fields": { + "raster_style": 1, + "value": 11.0, + "color": "#da00f2", + "opacity": 1.0, + "label": "SNOW" + } +}, +{ + "model": "coverages.grid", + "pk": 1, + "fields": { + "name": null, + "coordinate_reference_system": "EPSG:32630", + "axis_1_name": "x", + "axis_2_name": "y", + "axis_3_name": null, + "axis_4_name": null, + "axis_1_type": 0, + "axis_2_type": 0, + "axis_3_type": null, + "axis_4_type": null, + "axis_1_offset": "200.0", + "axis_2_offset": "-200.0", + "axis_3_offset": null, + "axis_4_offset": null, + "resolution": 200 + } +}, +{ + "model": "coverages.eoobject", + "pk": 1, + "fields": { + "identifier": "S2B_30UUG_20221226_0_L2A", + "begin_time": null, + "end_time": null, + "footprint": "SRID=4326;POLYGON ((-6.1994886 55.9041676, -6.1207799 54.9190265, -4.4083987 54.9509423, -4.4439779 55.9372733, -6.1994886 55.9041676))", + "inserted": "2023-09-18T08:57:08.231Z", + "updated": "2023-09-18T11:44:34.789Z" + } +}, +{ + "model": "coverages.eoobject", + "pk": 2, + "fields": { + "identifier": "S2B_30UUG_20221226_0_L2A_scl", + "begin_time": null, + "end_time": null, + "footprint": "SRID=4326;POLYGON ((-6.120779882947349 54.919026538369536, -6.1994885612119175 55.904167624663266, -4.443977851096375 55.937273345275685, -4.40839870205061 54.95094226131747, -6.120779882947349 54.919026538369536))", + "inserted": "2023-09-18T08:57:08.757Z", + "updated": "2023-09-18T08:57:08.765Z" + } +}, +{ + "model": "coverages.product", + "pk": 1, + "fields": { + "product_type": 1, + "package": null, + "collections": [] + } +}, +{ + "model": "coverages.coverage", + "pk": 2, + "fields": { + "grid": 1, + "axis_1_origin": "300000.0", + "axis_2_origin": "6200040.0", + "axis_3_origin": null, + "axis_4_origin": null, + "axis_1_size": 549, + "axis_2_size": 549, + "axis_3_size": null, + "axis_4_size": null, + "coverage_type": 1, + "parent_product": 1, + "collections": [], + "mosaics": [] + } +}, +{ + "model": "coverages.arraydataitem", + "pk": 1, + "fields": { + "storage": null, + "location": "autotest/data/SCL/scl_small.tif", + "format": "image/tiff", + "coverage": 2, + "field_index": 0, + "band_count": 1, + "subdataset_type": null, + "subdataset_locator": null, + "bands_interpretation": 0 + } +}, +{ + "model": "coverages.productmetadata", + "pk": 1, + "fields": { + "product": 1, + "parent_identifier": null, + "production_status": null, + "acquisition_type": null, + "orbit_number": null, + "orbit_direction": null, + "track": null, + "frame": null, + "swath_identifier": null, + "product_version": null, + "product_quality_status": null, + "product_quality_degradation_tag": null, + "processor_name": null, + "processing_center": null, + "creation_date": null, + "modification_date": null, + "processing_date": null, + "sensor_mode": null, + "archiving_center": null, + "processing_mode": null, + "availability_time": null, + "acquisition_station": null, + "acquisition_sub_type": null, + "start_time_from_ascending_node": null, + "completion_time_from_ascending_node": null, + "illumination_azimuth_angle": null, + "illumination_zenith_angle": null, + "illumination_elevation_angle": null, + "polarisation_mode": null, + "polarization_channels": null, + "antenna_look_direction": null, + "minimum_incidence_angle": null, + "maximum_incidence_angle": null, + "across_track_incidence_angle": null, + "along_track_incidence_angle": null, + "doppler_frequency": null, + "incidence_angle_variation": null, + "cloud_cover": null, + "snow_cover": null, + "lowest_location": null, + "highest_location": null + } +}, +{ + "model": "services.servicevisibility", + "pk": 1, + "fields": { + "eo_object": 1, + "service": "wms", + "visibility": true + } +} +] diff --git a/autotest/autotest/data/SCL/scl.sld b/autotest/autotest/data/SCL/scl.sld new file mode 100644 index 000000000..b7e965b64 --- /dev/null +++ b/autotest/autotest/data/SCL/scl.sld @@ -0,0 +1,36 @@ + + + + + + + + S2B_30UUG_20221226_0_L2A_scl + + + + + + 1 + + + + + + + + + + + + + + + + + + + + + + diff --git a/autotest/autotest/data/SCL/scl_coverage_type.json b/autotest/autotest/data/SCL/scl_coverage_type.json new file mode 100644 index 000000000..f2882ebe2 --- /dev/null +++ b/autotest/autotest/data/SCL/scl_coverage_type.json @@ -0,0 +1,24 @@ +[{ + "bands": [ + { + "definition": "http://www.opengis.net/def/property/OGC/0/Radiance", + "description": "SCL", + "gdal_interpretation": "gray", + "identifier": "scl", + "name": "scl", + "nil_values": [ + { + "reason": "http://www.opengis.net/def/nil/OGC/0/unknown", + "value": 0 + } + ], + "uom": "nil", + "significant_figures": 2, + "allowed_value_ranges": [ + [0, 11] + ] + } + ], + "data_type": "Byte", + "name": "SCL" +}] diff --git a/autotest/autotest/data/SCL/scl_small.tif b/autotest/autotest/data/SCL/scl_small.tif new file mode 100644 index 000000000..bd349ccfc Binary files /dev/null and b/autotest/autotest/data/SCL/scl_small.tif differ diff --git a/autotest/autotest/data/SCL/setup_data.sh b/autotest/autotest/data/SCL/setup_data.sh new file mode 100755 index 000000000..705467347 --- /dev/null +++ b/autotest/autotest/data/SCL/setup_data.sh @@ -0,0 +1,14 @@ +python3 manage.py coveragetype import autotest/data/SCL/scl_coverage_type.json +python3 manage.py producttype create SCL -c SCL +python3 manage.py browsetype create SCL SCL --grey scl + +python3 manage.py coveragetype import + +python3 manage.py rasterstyle import ./autotest/data/SCL/scl.sld --rename S2B_30UUG_20221226_0_L2A_scl SCL + +python3 manage.py rasterstyle link SCL SCL SCL color + +python3 manage.py product register -i S2B_30UUG_20221226_0_L2A -t SCL --footprint "POLYGON ((-6.1994886 55.9041676, -6.1207799 54.9190265, -4.4083987 54.9509423, -4.4439779 55.9372733, -6.1994886 55.9041676))" --replace +python3 manage.py coverage register -t SCL -r -d autotest/data/SCL/scl_small.tif --footprint-from-extent -i S2B_30UUG_20221226_0_L2A_scl -p S2B_30UUG_20221226_0_L2A + +python3 manage.py visibility S2B_30UUG_20221226_0_L2A --wms diff --git a/autotest/autotest/data/fixtures/clm_cloud_coverages.json b/autotest/autotest/data/fixtures/clm_cloud_coverages.json new file mode 100644 index 000000000..f2e1b6a8c --- /dev/null +++ b/autotest/autotest/data/fixtures/clm_cloud_coverages.json @@ -0,0 +1,75 @@ +[ +{ + "model": "coverages.coveragetype", + "pk": 5, + "fields": { + "name": "CLM" + } +}, +{ + "model": "coverages.eoobject", + "pk": 22, + "fields": { + "identifier": "CLM_CLOUD_COVERAGE_PRODUCT", + "begin_time": "2020-02-15T07:59:36.409Z", + "end_time": "2020-02-20T08:00:36.342Z", + "footprint": "SRID=4326;POLYGON ((16.6276837970076 47.466520638378,16.7499627020568 47.4647191364739,16.7516507329145 47.5154960582897,16.6292540083727 47.5173007469159,16.6276837970076 47.466520638378))", + "inserted": "2019-01-01T00:00:00.000Z", + "updated": "2019-01-01T00:00:00.000Z" + } +}, +{ + "model": "coverages.product", + "pk": 22, + "fields": { + "product_type": null, + "package": null + } +}, +{ + "model": "coverages.eoobject", + "pk": 23, + "fields": { + "identifier": "CLM_CLOUD_COVERAGE", + "begin_time": null, + "end_time": null, + "footprint": "SRID=4326;POLYGON ((16.6276837970076 47.466520638378,16.7499627020568 47.4647191364739,16.7516507329145 47.5154960582897,16.6292540083727 47.5173007469159,16.6276837970076 47.466520638378))", + "inserted": "2019-01-01T00:00:00.000Z", + "updated": "2019-01-01T00:00:00.000Z" + } +}, +{ + "model": "coverages.coverage", + "pk": 23, + "fields": { + "grid": 1, + "axis_1_origin": null, + "axis_2_origin": null, + "axis_3_origin": null, + "axis_4_origin": null, + "axis_1_size": 569, + "axis_2_size": 486, + "axis_3_size": null, + "axis_4_size": null, + "coverage_type": 5, + "parent_product": 22, + "collections": [], + "mosaics": [] + } +}, +{ + "model": "coverages.arraydataitem", + "pk": 21, + "fields": { + "storage": null, + "location": "autotest/data/CLM_clipped.tif", + "format": "image/tiff", + "coverage": 23, + "field_index": 0, + "band_count": 1, + "subdataset_type": null, + "subdataset_locator": null, + "bands_interpretation": 0 + } +} +] diff --git a/autotest/autotest/data/fixtures/scl_cloud_coverages.json b/autotest/autotest/data/fixtures/scl_cloud_coverages.json new file mode 100644 index 000000000..66493fa89 --- /dev/null +++ b/autotest/autotest/data/fixtures/scl_cloud_coverages.json @@ -0,0 +1,75 @@ +[ +{ + "model": "coverages.coveragetype", + "pk": 4, + "fields": { + "name": "SCL" + } +}, +{ + "model": "coverages.eoobject", + "pk": 21, + "fields": { + "identifier": "CLOUD_COVERAGE_PRODUCT", + "begin_time": "2020-02-15T07:59:36.409Z", + "end_time": "2020-02-20T08:00:36.342Z", + "footprint": "SRID=4326;MULTIPOLYGON (((69.1714578 80.1407449, 69.1714578 80.1333736, 69.2069740 80.1333736, 69.2069740 80.1407449, 69.1714578 80.1407449)))", + "inserted": "2019-01-01T00:00:00.000Z", + "updated": "2019-01-01T00:00:00.000Z" + } +}, +{ + "model": "coverages.product", + "pk": 21, + "fields": { + "product_type": null, + "package": null + } +}, +{ + "model": "coverages.eoobject", + "pk": 20, + "fields": { + "identifier": "CLOUD_COVERAGE", + "begin_time": null, + "end_time": null, + "footprint": "SRID=4326;MULTIPOLYGON (((69.1714578 80.1407449, 69.1714578 80.1333736, 69.2069740 80.1333736, 69.2069740 80.1407449, 69.1714578 80.1407449)))", + "inserted": "2019-01-01T00:00:00.000Z", + "updated": "2019-01-01T00:00:00.000Z" + } +}, +{ + "model": "coverages.coverage", + "pk": 20, + "fields": { + "grid": 1, + "axis_1_origin": null, + "axis_2_origin": null, + "axis_3_origin": null, + "axis_4_origin": null, + "axis_1_size": 569, + "axis_2_size": 486, + "axis_3_size": null, + "axis_4_size": null, + "coverage_type": 4, + "parent_product": 21, + "collections": [], + "mosaics": [] + } +}, +{ + "model": "coverages.arraydataitem", + "pk": 20, + "fields": { + "storage": null, + "location": "autotest/data/SCL/SCL_clipped.tif", + "format": "image/tiff", + "coverage": 20, + "field_index": 0, + "band_count": 1, + "subdataset_type": null, + "subdataset_locator": null, + "bands_interpretation": 0 + } +} +] diff --git a/autotest/autotest/data/fixtures/scl_styled.json b/autotest/autotest/data/fixtures/scl_styled.json new file mode 120000 index 000000000..a5d97c0f0 --- /dev/null +++ b/autotest/autotest/data/fixtures/scl_styled.json @@ -0,0 +1 @@ +../SCL/fixtures.json \ No newline at end of file diff --git a/autotest/autotest/data/meris/meris_products_rgb.json b/autotest/autotest/data/meris/meris_products_rgb.json index 404004cfd..d2a78dc85 100644 --- a/autotest/autotest/data/meris/meris_products_rgb.json +++ b/autotest/autotest/data/meris/meris_products_rgb.json @@ -14,6 +14,30 @@ { "model": "coverages.eoobject", "pk": 15, + "fields": { + "identifier": "product_mosaic_MER_FRS_1PNPDE20060822_092058_000001972050_00308_23408_0077_RGB_reduced", + "begin_time": "2006-08-22T09:20:58Z", + "end_time": "2006-08-22T09:24:15Z", + "footprint": "SRID=4326;MULTIPOLYGON (((11.45216499930154 46.21538203826643, 12.56460056524387 46.07713899233252, 13.88709473239655 45.89816054555529, 15.27428923693798 45.68087382785627, 16.73263457380611 45.44978576355449, 18.91093599972647 45.05191679046387, 20.40860552393306 44.74801169950303, 21.96696659827764 44.41194128767518, 23.51904328524226 44.04201602215931, 25.09349524830671 43.64372208846083, 24.69762334871068 42.82058014584347, 24.09572551417862 41.52681585719382, 23.44873586198113 40.08086776323874, 22.98946525457247 39.02515181830724, 22.50232224402901 37.85145204973027, 22.01522231099019 36.64248172093542, 21.49885247842534 35.30701482357061, 20.94415866775533 33.81513728290902, 20.38639122031948 32.26692667976634, 19.56420468222143 32.47301341272738, 18.2080921891265 32.79995702514136, 16.63597483093452 33.15675496605559, 15.04583039593244 33.49010596857972, 13.46867305396488 33.80242022189355, 11.78865627996702 34.11762771456879, 10.10198645739054 34.40475932684974, 8.778925655898064 34.61206991018219, 9.065762626406386 35.93773536210887, 9.314689072633191 37.04263450498039, 9.571219329906695 38.21169286827537, 9.910891562982675 39.70883235016088, 10.25693665372459 41.23523865932673, 10.57063808257736 42.57173906512232, 10.80473516394502 43.56727382751684, 11.06940217936636 44.65704022384523, 11.28225456261711 45.55003941502519, 11.45216499930154 46.21538203826643)))", + "inserted": "2019-01-01T00:00:00.000Z", + "updated": "2019-01-01T00:00:00.000Z" + } +}, +{ + "model": "coverages.eoobject", + "pk": 16, + "fields": { + "identifier": "mosaic_MER_FRS_1PNPDE20060830_100949_000001972050_00423_23523_0079_RGB_reduced", + "begin_time": "2006-08-30T10:09:49Z", + "end_time": "2006-08-30T10:13:06Z", + "footprint": "SRID=4326;MULTIPOLYGON (((-0.7697718236173799 46.21844540418552, 0.288547363821689 46.08290644415141, 2.733419759721718 45.73636263672618, 4.815657039292408 45.39947037754385, 6.894775798282314 45.00948175187284, 8.154913571318612 44.75018471574041, 9.634245839133294 44.41952395308687, 11.00618247148604 44.10606584164272, 12.27883282764158 43.79980935386448, 12.87473445741455 43.64035642239579, 12.22874811272635 42.2970241539363, 11.42842470078386 40.54055904761669, 10.66098787948851 38.74418401722271, 9.927539260629768 36.9630879853842, 9.239587579469651 35.18226041011476, 8.728287480275119 33.80781071010245, 8.174463354403226 32.26454057758526, 7.346658308388547 32.47445746391359, 5.48900097684443 32.9147209085525, 3.594912710998026 33.32929471736243, 1.679561673218221 33.71894896552116, -0.256213475006078 34.08288220106373, -1.744083689584682 34.34057243249224, -3.43798101398678 34.6029479595267, -3.312138031931722 35.20050022128398, -2.96861379128801 36.75709370137025, -2.682010254826003 38.05579573474809, -2.350758514252345 39.51733579731436, -1.942069312500661 41.30092067756908, -1.553859522839208 42.94621606148156, -1.166995203885614 44.59251101422208, -0.7697718236173799 46.21844540418552)))", + "inserted": "2019-01-01T00:00:00.000Z", + "updated": "2019-01-01T00:00:00.000Z" + } +}, +{ + "model": "coverages.eoobject", + "pk": 17, "fields": { "identifier": "MER_FRS_1P_reduced_products_RGB", "begin_time": "2006-08-16T09:09:29Z", @@ -30,7 +54,29 @@ "product_type": 1, "package": null, "collections": [ - 15 + 17 + ] + } +}, +{ + "model": "coverages.product", + "pk": 15, + "fields": { + "product_type": 1, + "package": null, + "collections": [ + 17 + ] + } +}, +{ + "model": "coverages.product", + "pk": 16, + "fields": { + "product_type": 1, + "package": null, + "collections": [ + 17 ] } }, @@ -44,7 +90,7 @@ }, { "model": "coverages.collection", - "pk": 15, + "pk": 17, "fields": { "collection_type": 1, "grid": null @@ -80,6 +126,44 @@ "height": 449 } }, +{ + "model": "coverages.browse", + "pk": 2, + "fields": { + "storage": null, + "location": "/opt/instance/autotest/data/meris/mosaic_MER_FRS_1P_reduced_RGB/mosaic_ENVISAT-MER_FRS_1PNPDE20060822_092058_000001972050_00308_23408_0077_RGB_reduced.tif", + "format": null, + "product": 15, + "browse_type": null, + "style": null, + "coordinate_reference_system": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433],AUTHORITY[\"EPSG\",\"4326\"]]", + "min_x": 8.47845, + "min_y": 32.19025, + "max_x": 25.4101500, + "max_y": 46.268645, + "width": 540, + "height": 449 + } +}, +{ + "model": "coverages.browse", + "pk": 3, + "fields": { + "storage": null, + "location": "/opt/instance/autotest/data/meris/mosaic_MER_FRS_1P_reduced_RGB/mosaic_ENVISAT-MER_FRS_1PNPDE20060830_100949_000001972050_00423_23523_0079_RGB_reduced.tif", + "format": null, + "product": 16, + "browse_type": null, + "style": null, + "coordinate_reference_system": "GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433],AUTHORITY[\"EPSG\",\"4326\"]]", + "min_x": -3.75, + "min_y": 32.19025, + "max_x": 13.213055, + "max_y": 46.268645, + "width": 541, + "height": 449 + } +}, { "model": "coverages.masktype", "pk": 1, diff --git a/autotest/autotest/expected/WMS13GetMapCollectionHeatmapTestCase.png b/autotest/autotest/expected/WMS13GetMapCollectionHeatmapTestCase.png new file mode 100644 index 000000000..cf262da5c Binary files /dev/null and b/autotest/autotest/expected/WMS13GetMapCollectionHeatmapTestCase.png differ diff --git a/autotest/autotest/expected/WMS13GetMapCrossesDatelineDatasetTestCase.jpg b/autotest/autotest/expected/WMS13GetMapCrossesDatelineDatasetTestCase.jpg index 9de44a98b..25a93e20e 100644 Binary files a/autotest/autotest/expected/WMS13GetMapCrossesDatelineDatasetTestCase.jpg and b/autotest/autotest/expected/WMS13GetMapCrossesDatelineDatasetTestCase.jpg differ diff --git a/autotest/autotest/expected/WMS13GetMapDatasetStyledTestCase.png b/autotest/autotest/expected/WMS13GetMapDatasetStyledTestCase.png new file mode 100644 index 000000000..4da8217a0 Binary files /dev/null and b/autotest/autotest/expected/WMS13GetMapDatasetStyledTestCase.png differ diff --git a/autotest/autotest/expected/WPS20ExecuteCloudCoverageCloudyGeometry.json b/autotest/autotest/expected/WPS20ExecuteCloudCoverageCloudyGeometry.json new file mode 100644 index 000000000..0bfa5cf43 --- /dev/null +++ b/autotest/autotest/expected/WPS20ExecuteCloudCoverageCloudyGeometry.json @@ -0,0 +1 @@ +{"result": {"2020-02-15T07:59:36.409000+00:00": 0.0}} \ No newline at end of file diff --git a/autotest/autotest/expected/WPS20ExecuteCloudCoverageCustomMask.json b/autotest/autotest/expected/WPS20ExecuteCloudCoverageCustomMask.json new file mode 100644 index 000000000..09cd12ef1 --- /dev/null +++ b/autotest/autotest/expected/WPS20ExecuteCloudCoverageCustomMask.json @@ -0,0 +1 @@ +{"result": {"2020-02-15T07:59:36.409000+00:00": 0.33}} \ No newline at end of file diff --git a/autotest/autotest/expected/WPS20ExecuteCloudCoverageNonCloudy.json b/autotest/autotest/expected/WPS20ExecuteCloudCoverageNonCloudy.json new file mode 100644 index 000000000..20c0baeef --- /dev/null +++ b/autotest/autotest/expected/WPS20ExecuteCloudCoverageNonCloudy.json @@ -0,0 +1 @@ +{"result": {"2020-02-15T07:59:36.409000+00:00": 0.55}} \ No newline at end of file diff --git a/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLM.json b/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLM.json new file mode 100644 index 000000000..7c797adc3 --- /dev/null +++ b/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLM.json @@ -0,0 +1 @@ +{"result": {"2020-02-15T07:59:36.409000+00:00": 0.0671425858274402}} \ No newline at end of file diff --git a/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMCustomMask.json b/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMCustomMask.json new file mode 100644 index 000000000..b4da905b2 --- /dev/null +++ b/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMCustomMask.json @@ -0,0 +1 @@ +{"result": {"2020-02-15T07:59:36.409000+00:00": 0.2569736339319832}} \ No newline at end of file diff --git a/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMFewClouds.json b/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMFewClouds.json new file mode 100644 index 000000000..b4da905b2 --- /dev/null +++ b/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMFewClouds.json @@ -0,0 +1 @@ +{"result": {"2020-02-15T07:59:36.409000+00:00": 0.2569736339319832}} \ No newline at end of file diff --git a/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMMoreCloudy.json b/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMMoreCloudy.json new file mode 100644 index 000000000..0bfa5cf43 --- /dev/null +++ b/autotest/autotest/expected/WPS20ExecuteCloudCoverageOnCLMMoreCloudy.json @@ -0,0 +1 @@ +{"result": {"2020-02-15T07:59:36.409000+00:00": 0.0}} \ No newline at end of file diff --git a/autotest/autotest/expected/WPS20ExecuteCloudCoverageTinyGeometry.json b/autotest/autotest/expected/WPS20ExecuteCloudCoverageTinyGeometry.json new file mode 100644 index 000000000..0bfa5cf43 --- /dev/null +++ b/autotest/autotest/expected/WPS20ExecuteCloudCoverageTinyGeometry.json @@ -0,0 +1 @@ +{"result": {"2020-02-15T07:59:36.409000+00:00": 0.0}} \ No newline at end of file diff --git a/autotest/autotest/expected/WPS20GetCapabilitiesValidTestCase.xml b/autotest/autotest/expected/WPS20GetCapabilitiesValidTestCase.xml index 655d79223..dbf150ef2 100644 --- a/autotest/autotest/expected/WPS20GetCapabilitiesValidTestCase.xml +++ b/autotest/autotest/expected/WPS20GetCapabilitiesValidTestCase.xml @@ -94,6 +94,10 @@ Copyright (C) European Space Agency - ESA provides processed results of all the coverages whithin a provided bounding box. The processes returns hillshade, aspect/ratio, slope and contour. DemProcessing + + Cloud coverage information about images of an AOI/TOI + CloudCoverage + Test Case 00: Literal data identity. Test identity process (the outputs are copies of the inputs) diff --git a/autotest/autotest/settings.py b/autotest/autotest/settings.py index 01c4dad5b..86f1d85a5 100644 --- a/autotest/autotest/settings.py +++ b/autotest/autotest/settings.py @@ -1,10 +1,10 @@ -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # # Project: EOxServer # Authors: Stephan Krause # Stephan Meissl # -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- # Copyright (C) 2011 EOX IT Services GmbH # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -24,7 +24,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------- """ Django settings for EOxServer's autotest instance. @@ -176,6 +176,7 @@ ] MIDDLEWARE = [ + 'django_prometheus.middleware.PrometheusBeforeMiddleware', 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -186,6 +187,7 @@ 'django.middleware.clickjacking.XFrameOptionsMiddleware', # # For management of the per/request cache system. # 'eoxserver.backends.middleware.BackendsCacheMiddleware', + 'django_prometheus.middleware.PrometheusAfterMiddleware' ] MIDDLEWARE_CLASSES = ( @@ -219,6 +221,7 @@ # Enable for better schema and data-migrations # Enable for debugging # 'django_extensions', + 'django_prometheus', # Enable EOxServer: 'eoxserver.core', 'eoxserver.services', @@ -241,11 +244,6 @@ # search will be done. COMPONENTS = () - -# -# -# -# EOXS_PROCESSES = DEFAULT_EOXS_PROCESSES + [ 'autotest_services.processes.test00_identity_literal.TestProcess00', 'autotest_services.processes.test01_identity_bbox.TestProcess01', @@ -301,9 +299,9 @@ 'filters': [], }, 'console': { - 'level': 'DEBUG', - 'filters': ['require_debug_true'], 'class': 'logging.StreamHandler', + 'formatter': 'verbose', + 'filters': [], } }, 'loggers': { @@ -331,3 +329,4 @@ # Set this variable if the path to the instance cannot be resolved # automatically, e.g. in case of redirects #FORCE_SCRIPT_NAME="/path/to/instance/" +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" diff --git a/autotest/autotest/urls.py b/autotest/autotest/urls.py index f63efe4f7..f6e3331af 100644 --- a/autotest/autotest/urls.py +++ b/autotest/autotest/urls.py @@ -6,6 +6,7 @@ from django.urls import include, re_path from django.contrib import admin from django.conf.urls.static import static +import django_prometheus.exports from eoxserver.services.opensearch.urls import urlpatterns as opensearch from eoxserver.webclient.urls import urlpatterns as webclient @@ -30,4 +31,9 @@ re_path(r'^admin/doc/', include('django.contrib.admindocs.urls')), # Enable the admin: re_path(r'^admin/', admin.site.urls), + re_path( + r'^metrics$', + django_prometheus.exports.ExportToDjangoView, + name="prometheus-django-metrics", + ), ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/autotest/autotest_services/tests/wms/base.py b/autotest/autotest_services/tests/wms/base.py index 691783835..468a196d9 100644 --- a/autotest/autotest_services/tests/wms/base.py +++ b/autotest/autotest_services/tests/wms/base.py @@ -97,6 +97,7 @@ class WMS13GetMapTestCase(testbase.RasterTestCase): frmt = "image/jpeg" time = None dim_bands = None + dim_range = None swap_axes = True @@ -128,6 +129,9 @@ def getRequest(self): if self.dim_bands: params += "&dim_bands=%s" % self.dim_bands + if self.dim_range: + params += "&dim_range=%s" % self.dim_range + if self.httpHeaders is None: return (params, "kvp") else: diff --git a/autotest/autotest_services/tests/wms/test_v13.py b/autotest/autotest_services/tests/wms/test_v13.py index 6f9ab2ed2..1b7d911a1 100644 --- a/autotest/autotest_services/tests/wms/test_v13.py +++ b/autotest/autotest_services/tests/wms/test_v13.py @@ -340,23 +340,39 @@ class WMS13GetMapCollectionMaskedOutlinesTestCase(wmsbase.WMS13GetMapTestCase): layers = ("MER_FRS_1P_reduced_products_RGB__outlines_masked_clouds",) bbox = (11, 32, 28, 46) -#=============================================================================== +# ============================================================================== # Styled Coverages -#=============================================================================== +# ============================================================================== -# currently disabled because of segfaults in MapServer -''' class WMS13GetMapDatasetStyledTestCase(wmsbase.WMS13GetMapTestCase): """ Test a GetMap request a dataset with an associated style. """ - fixtures = wmsbase.WMS13GetMapTestCase.fixtures + [ - "cryo_range_type.json", "cryo_coverages.json" - ] - - layers = ("FSC_0.0025deg_201303030930_201303031110_MOD_Alps_ENVEOV2.1.00",) - bbox = (6, 44.5, 16, 48) + fixtures = ["scl_styled.json"] + layers = ("S2B_30UUG_20221226_0_L2A__SCL",) + swap_axes = True + bbox = (-6.282089176104, 54.89235272910, -4.3728695585011, 55.962341471504) + crs = "EPSG:4326" width = 200 -''' + height = 200 + styles = ("color",) + frmt = "image/png" + + +# ============================================================================== +# Heatmap +# ============================================================================== + +class WMS13GetMapCollectionHeatmapTestCase(wmsbase.WMS13GetMapTestCase): + fixtures = MASK_FIXTURES + + layers = ["MER_FRS_1P_reduced_products_RGB__heatmap"] + height = 50 + width = 100 + bbox = [-3.75, 32.158895, 28.326165, 46.3] + dim_range = "0 5" + frmt = "image/png" + + #=============================================================================== # Feature Info #=============================================================================== @@ -438,3 +454,6 @@ def getRequest(self): def getFileExtension(self, file_type): return "png" + + + diff --git a/autotest/autotest_services/tests/wps/test_v20_cloud_coverage.py b/autotest/autotest_services/tests/wps/test_v20_cloud_coverage.py new file mode 100644 index 000000000..d40c35738 --- /dev/null +++ b/autotest/autotest_services/tests/wps/test_v20_cloud_coverage.py @@ -0,0 +1,309 @@ +# ------------------------------------------------------------------------------- +# +# Project: EOxServer +# Authors: Bernhard Mallinger +# +# ------------------------------------------------------------------------------- +# Copyright (C) 2022 EOX IT Services GmbH +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies of this Software or works derived from this Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# ------------------------------------------------------------------------------- +# pylint: disable=missing-docstring,line-too-long,too-many-ancestors + +from autotest_services import base as testbase +from autotest_services.tests.wps.base import ( + ContentTypeCheckMixIn, +) + + +class WPS20ExecuteCloudCoverageNonCloudy(ContentTypeCheckMixIn, testbase.JSONTestCase): + fixtures = testbase.JSONTestCase.fixtures + ["scl_cloud_coverages.json"] + + expectedContentType = "application/json; charset=utf-8" + + def getRequest(self): + params = """ + CloudCoverage + 2020-01-01 + 2020-05-31 + + + POLYGON ((69.19913354922439908 80.1406125504016984, 69.19921132386413376 80.13719046625288911, 69.20360559100976161 80.13719046625288911, 69.20364447832963606 80.14065143772157285, 69.20364447832963606 80.14065143772157285, 69.19913354922439908 80.1406125504016984)) + + + + + + """ + return (params, "xml") + + +class WPS20ExecuteCloudCoverageCloudyGeometry( + ContentTypeCheckMixIn, testbase.JSONTestCase +): + fixtures = testbase.JSONTestCase.fixtures + ["scl_cloud_coverages.json"] + + expectedContentType = "application/json; charset=utf-8" + + def getRequest(self): + params = """ + CloudCoverage + 2020-01-01 + 2020-05-31 + + + POLYGON ((69.19753916910960356 80.14057366308182395, 69.19757805642947801 80.13960148008500539, 69.20484998524568709 80.13956259276513094, 69.20481109792581265 80.1405347757619495, 69.20481109792581265 80.1405347757619495, 69.19753916910960356 80.14057366308182395)) + + + + + + """ + return (params, "xml") + + +class WPS20ExecuteCloudCoverageTinyGeometry( + ContentTypeCheckMixIn, testbase.JSONTestCase +): + fixtures = testbase.JSONTestCase.fixtures + ["scl_cloud_coverages.json"] + + expectedContentType = "application/json; charset=utf-8" + + def getRequest(self): + params = """ + CloudCoverage + 2020-01-01 + 2020-05-31 + + + POLYGON((69.1714578 80.1407449,69.17148193988112 80.14073878013805,69.17149910356903 80.1407419147403,69.1714578 80.1407449)) + + + + + + """ + return (params, "xml") + + +class WPS20ExecuteCloudCoverageCustomMask( + ContentTypeCheckMixIn, testbase.JSONTestCase +): + fixtures = testbase.JSONTestCase.fixtures + ["scl_cloud_coverages.json"] + + expectedContentType = "application/json; charset=utf-8" + + def getRequest(self): + params = """ + CloudCoverage + 2020-01-01 + 2020-05-31 + + + POLYGON ((69.19913354922439908 80.1406125504016984, 69.19921132386413376 80.13719046625288911, 69.20360559100976161 80.13719046625288911, 69.20364447832963606 80.14065143772157285, 69.20364447832963606 80.14065143772157285, 69.19913354922439908 80.1406125504016984)) + + + + + [1, 2, 3, 8] + + + + + + """ + return (params, "xml") + + + +class WPS20ExecuteCloudCoverageEmptyResponse( + ContentTypeCheckMixIn, testbase.JSONTestCase +): + # Don't add cloud coverage fixture to provoke empty response + fixtures = testbase.JSONTestCase.fixtures + [] + + expectedContentType = "application/json; charset=utf-8" + + def getRequest(self): + params = """ + CloudCoverage + 2020-01-01 + 2020-05-31 + + + POLYGON((69.1714578 80.1407449,69.17148193988112 80.14073878013805,69.17149910356903 80.1407419147403,69.1714578 80.1407449)) + + + + + + """ + return (params, "xml") + + +class WPS20ExecuteCloudCoverageOnCLM(ContentTypeCheckMixIn, testbase.JSONTestCase): + fixtures = testbase.JSONTestCase.fixtures + ["clm_cloud_coverages.json"] + + expectedContentType = "application/json; charset=utf-8" + + def getRequest(self): + params = """ + CloudCoverage + 2020-01-01 + 2020-05-31 + + + POLYGON((16.69737831605056 47.47325091482521,16.711481970707467 47.50227740341751,16.665977994941493 47.4984370585647,16.65892616761306 47.47649970533886,16.69737831605056 47.47325091482521)) + + + + + + """ + return (params, "xml") + + +class WPS20ExecuteCloudCoverageOnCLMMoreCloudy( + ContentTypeCheckMixIn, testbase.JSONTestCase +): + fixtures = testbase.JSONTestCase.fixtures + ["clm_cloud_coverages.json"] + + expectedContentType = "application/json; charset=utf-8" + + def getRequest(self): + params = """ + CloudCoverage + 2020-01-01 + 2020-05-31 + + + POLYGON((16.689481892710717 47.49088479184637,16.685847177764813 47.49102703651446,16.685862090779757 47.49375416408526,16.689267315989525 47.49363955046273,16.689481892710717 47.49088479184637)) + + + + + + """ + return (params, "xml") + + +class WPS20ExecuteCloudCoverageOnCLMFewClouds( + ContentTypeCheckMixIn, testbase.JSONTestCase +): + fixtures = testbase.JSONTestCase.fixtures + ["clm_cloud_coverages.json"] + + expectedContentType = "application/json; charset=utf-8" + + def getRequest(self): + params = """ + CloudCoverage + 2020-01-01 + 2020-05-31 + + + POLYGON((16.689481892710717 47.49088479184637,16.685862090779757 47.49375416408526,16.717934765940697 47.50045333252222,16.689481892710717 47.49088479184637)) + + + + + + """ + return (params, "xml") + +class WPS20ExecuteCloudCoverageOnCLMCustomMask( + ContentTypeCheckMixIn, testbase.JSONTestCase +): + fixtures = testbase.JSONTestCase.fixtures + ["clm_cloud_coverages.json"] + + expectedContentType = "application/json; charset=utf-8" + + def getRequest(self): + params = """ + CloudCoverage + 2020-01-01 + 2020-05-31 + + + POLYGON((16.689481892710717 47.49088479184637,16.685862090779757 47.49375416408526,16.717934765940697 47.50045333252222,16.689481892710717 47.49088479184637)) + + + + + 7 + + + + + + """ + return (params, "xml") diff --git a/autotest/setup.py b/autotest/setup.py deleted file mode 100644 index d86315c7d..000000000 --- a/autotest/setup.py +++ /dev/null @@ -1,76 +0,0 @@ -#------------------------------------------------------------------------------- -# -# Project: EOxServer -# Authors: Stephan Krause -# Stephan Meissl -# Fabian Schindler -# -#------------------------------------------------------------------------------- -# Copyright (C) 2011 EOX IT Services GmbH -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies of this Software or works derived from this Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -#------------------------------------------------------------------------------- - -import os - -# Hack to remove setuptools "feature" which resulted in -# ignoring MANIFEST.in when code is in an svn repository. -# TODO find a nicer solution -from setuptools.command import sdist -del sdist.finders[:] - -from setuptools import setup -from setuptools.command.install import install as _install - -from eoxserver import get_version - -class install(_install): - def run(self): - _install.run(self) - - self.prefix -version = get_version() - -data_files = [] -for dirpath, dirnames, filenames in os.walk('autotest/data'): - data_files.append([dirpath, [os.path.join(dirpath, f) for f in filenames]]) - -setup( - name='EOxServer_autotest', - version=version.replace(' ', '-'), - # TODO: packages - data_files=data_files, - - install_requires=['eoxserver'], - - # Metadata - author="EOX IT Services GmbH", - author_email="office@eox.at", - maintainer="EOX IT Services GmbH", - maintainer_email="packages@eox.at", - - description="Autotest instance for EOxServer", - long_description="", - - license="EOxServer Open License (MIT-style)", - keywords="Earth Observation, EO, OGC, WCS, WMS", - url="http://eoxserver.org/", - - cmdclass={'install': install}, -) diff --git a/autotest/docker-compose.yml b/docker-compose.yml similarity index 80% rename from autotest/docker-compose.yml rename to docker-compose.yml index 9411dc34e..fe37da317 100644 --- a/autotest/docker-compose.yml +++ b/docker-compose.yml @@ -8,29 +8,31 @@ services: POSTGRES_USER: "user" POSTGRES_PASSWORD: "pw" POSTGRES_DB: "dbms" + autotest: - image: eoxserver + image: eoxserver:autotest + build: . env_file: - - ./eoxserver.env + - ./sample.env volumes: - type: bind - source: ./../eoxserver + source: ./eoxserver target: /usr/local/lib/python2.7/dist-packages/eoxserver/ - type: bind - source: ./../eoxserver + source: ./eoxserver target: /usr/local/lib/python3.10/dist-packages/eoxserver/ - type: bind - source: ./../eoxserver + source: ./eoxserver target: /opt/eoxserver/eoxserver/ - type: bind - source: ./ + source: ./autotest target: /opt/instance - type: bind - source: ./../schemas + source: ./schemas target: /opt/schemas working_dir: /opt/instance command: - ["gunicorn", "--reload", "--bind=0.0.0.0:8000", "--chdir=/opt/instance", "autotest.wsgi:application"] + ["gunicorn", "--reload", "--bind=0.0.0.0:8000", "--chdir=/opt/instance", "autotest.wsgi:application", "--workers=3"] ports: - "8800:8000" diff --git a/docker/Readme.md b/docker/Readme.md deleted file mode 100644 index 37ead6061..000000000 --- a/docker/Readme.md +++ /dev/null @@ -1,53 +0,0 @@ -# Start EOXServer using docker - -Install [docker](https://www.docker.com/) for your system. - -The next steps are: - -```sh -git clone git@github.com:EOxServer/eoxserver.git -cd eoxserver/ -docker build -t eoxserver -f ./docker///Dockerfile . -``` -where `OS` and `py_version` are folders in the `docker` folder. - -This will give you a docker image ID as the last output e.g `55453fbf21c9`. The -image ID can also be retrieved using the `docker images` command. The last image -you built should be at the top of the list. - -**Note**: If you want to include static files for debugging create a debug_static.sh script: - -debug_static.sh -```bash -#!/bin/bash - -echo "Running debug_static.sh!" -cp /opt/scripts/urls.py /opt/instance/instance -``` - -and add the following line to be executed as a startup script to the sample.env: -```env -STARTUP_SCRIPTS=/opt/scripts/debug_static.sh -``` - -To run the image with static files do: - -```sh -mkdir docker/scripts -cp autotest/autotest/urls.py docker/scripts -mv debug_static.sh docker/scripts -docker run -it --rm -p 8080:8000 --mount type=bind,source=$PWD/docker/scripts,target=/opt/scripts --env-file docker/sample.env -``` - -where `` is the image ID from before e.g. `55453fbf21c9` - -The first few unique characters also work: - -```sh -docker run -i -t --rm -p 8080:8000 5545 -``` - -EOxServer is now accessible at [http://localhost:8080/](http://localhost:8080/). -And you can login to the `Admin Client` using: -- username: admin -- password: admin diff --git a/docker/centos/py2/Dockerfile b/docker/centos/py2/Dockerfile deleted file mode 100644 index 77ac82476..000000000 --- a/docker/centos/py2/Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -FROM centos:7 - -ARG DJANGO=1.11.26 - -# install OS dependency packages -RUN rpm -Uvh http://yum.packages.eox.at/el/eox-release-7-0.noarch.rpm \ - && yum update -y \ - && yum install -y epel-release \ - && yum install -y \ - python \ - postgresql \ - postgis \ - python-psycopg2 \ - gdal \ - gdal-python \ - mapserver \ - mapserver-python \ - libxml2 \ - libxml2-python \ - python-lxml \ - python-pip \ - && yum clean all - -# install EOxServer -RUN mkdir /opt/eoxserver/ - -ADD eoxserver /opt/eoxserver/eoxserver -ADD tools /opt/eoxserver/tools -ADD setup.cfg setup.py MANIFEST.in README.rst requirements.txt /opt/eoxserver/ - -# install EOxServer and its dependencies -WORKDIR /opt/eoxserver - -RUN pip install -r requirements.txt \ - && pip install -U "django==$DJANGO" \ - && pip install -U "gunicorn<19" \ - && pip install . - -# Create an EOxServer instance -RUN mkdir /opt/instance \ - && eoxserver-instance.py instance /opt/instance - -WORKDIR /opt/instance - -EXPOSE 8000 - -CMD ["gunicorn", "--chdir", "/opt/instance", "--bind", ":8000", "instance.wsgi:application", "--workers", "10", "--worker-class", "eventlet", "--timeout", "600"] diff --git a/docker/sample.env b/docker/sample.env deleted file mode 100644 index 0543707ce..000000000 --- a/docker/sample.env +++ /dev/null @@ -1,9 +0,0 @@ -DB_USER=user -DB_PW=pw -DB_HOST=database -DB_PORT=5432 -DB_NAME=dbms -DJANGO_USER=admin -DJANGO_PASSWORD=admin -DJANGO_MAIL=admin@sample.com -XML_CATALOG_FILES=/opt/schemas/catalog.xml diff --git a/docker/ubuntu/py2/Dockerfile b/docker/ubuntu/py2/Dockerfile deleted file mode 100644 index 7ef71e8d3..000000000 --- a/docker/ubuntu/py2/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -FROM ubuntu:18.04 - -ARG DJANGO=1.11.26 - -ENV INSTANCE_NAME=instance - -# possible values are "postgis" and "spatialite" -ENV DB=spatialite -ENV DB_HOST '' -ENV DB_NAME '' -ENV DB_USER '' -ENV DB_PW '' - -# set these variables to add a django user upon instance initialization -ENV DJANGO_USER '' -ENV DJANGO_MAIL '' -ENV DJANGO_PASSWORD '' - -# set this to a glob or filename in order to run after initialization -ENV INIT_SCRIPTS='' - -# override this or specify additional options in the config file -ENV GUNICORN_CMD_ARGS "--config /opt/eoxserver/gunicorn.conf.py ${INSTANCE_NAME}.wsgi:application" - -# install OS dependency packages -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - python \ - python-pip \ - python-psycopg2 \ - python-mapscript \ - python-gdal \ - gdal-bin \ - libsqlite3-mod-spatialite \ - postgresql-client \ - python-tk \ - && apt-get autoremove -y \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/partial/* /tmp/* /var/tmp/* - -# install EOxServer -RUN mkdir /opt/eoxserver/ - -ADD eoxserver /opt/eoxserver/eoxserver -ADD tools /opt/eoxserver/tools -ADD setup.cfg setup.py MANIFEST.in README.rst requirements.txt /opt/eoxserver/ -ADD docker/eoxserver-entrypoint.sh /opt/eoxserver/ -ADD docker/gunicorn.conf.py /opt/eoxserver/ - -# install EOxServer and its dependencies -WORKDIR /opt/eoxserver - -RUN pip install -r requirements.txt \ - && pip install -U "django==$DJANGO" \ - && pip install . - -EXPOSE 8000 - -ENTRYPOINT ["/opt/eoxserver/eoxserver-entrypoint.sh"] - -CMD ["gunicorn"] diff --git a/documentation/Makefile b/documentation/Makefile index 3bdaa36ad..9f3366cf4 100644 --- a/documentation/Makefile +++ b/documentation/Makefile @@ -8,8 +8,6 @@ PAPER = a4 BUILDDIR = _build # Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html html-new dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest diff --git a/documentation/Readme.md b/documentation/Readme.md deleted file mode 100644 index 59755164f..000000000 --- a/documentation/Readme.md +++ /dev/null @@ -1,64 +0,0 @@ -# Documentation HowTo - -## Rebuilding API-Documentation - -```sh -rm apidoc/*rst -sphinx-apidoc -o apidoc/ -M -f -a .. eoxserver -``` - -## Generate Sphinx documentation - -### UNIX - -```sh -cd /sphinx -export PYTHONPATH="" -export DJANGO_SETTINGS_MODULE="autotest.settings" # Use a valid EOxServer settings file here. Note that the configured database needs to be synced. -make html -``` - -The documentation is generated in *_build/html*. - -```sh -# For pdf output run: -make latex -cd _build/latex/ -make all-pdf -``` - -The pdf documentation is generated as *EOxServer.pdf*. - -### Windows - -```sh -cd /sphinx -set PYTHONPATH="" -set DJANGO_SETTINGS_MODULE="autotest.settings" # Use a valid EOxServer settings file here. Note that the configured database needs to be synced. -sphinx-build.exe -b html -d _build\doctrees -D latex_paper_size=a4 . _build/html #Note that the "sphinx-build.exe" has to be set in your path -```` - -The documentation is generated in *_build/html*. - -## Generate epydoc documentation - -```sh -cd -export DJANGO_SETTINGS_MODULE="autotest.settings" -epydoc --name=EOxServer --output=../docs/epydoc/ --html --docformat=javadoc --graph=all . -``` - -## Update model graphs - -```sh -cd -python manage.py graph_models --output=../docs/sphinx/en/developers/images/model_core.png core -python manage.py graph_models --output=../docs/sphinx/en/developers/images/model_services.png services -python manage.py graph_models --output=../docs/sphinx/en/developers/images/model_coverages.png coverages -python manage.py graph_models --output=../docs/sphinx/en/developers/images/model_backends.png backends - -python manage.py graph_models coverages --output=./tmp.dot -# Edit tmp.dot and remove top lines up to "digraph name {". -dot -Tsvg ./tmp.dot -o./tmp.svg -# Edit tmp.svg e.g. with Inkscape -``` diff --git a/documentation/_static/ESA_logo.png b/documentation/_static/ESA_logo.png new file mode 100644 index 000000000..7891d6c65 Binary files /dev/null and b/documentation/_static/ESA_logo.png differ diff --git a/documentation/_static/HMA_Logo.jpg b/documentation/_static/HMA_Logo.jpg deleted file mode 100644 index ddfc9b463..000000000 Binary files a/documentation/_static/HMA_Logo.jpg and /dev/null differ diff --git a/documentation/conf.py b/documentation/conf.py index 16412ba20..25f2f4e24 100644 --- a/documentation/conf.py +++ b/documentation/conf.py @@ -12,10 +12,12 @@ # serve to show the default. import sys, os +import django -from django.conf import settings - -settings.configure(DEBUG=True, ) +sys.path.insert(0, os.path.abspath('../autotest/')) +os.environ['DJANGO_SETTINGS_MODULE'] = 'autotest.settings' +os.environ['DB'] = 'spatialite' +django.setup() # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -29,7 +31,7 @@ # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.intersphinx'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.intersphinx', 'myst_parser'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -179,8 +181,8 @@ # -- Options for LaTeX output -------------------------------------------------- -# The paper size ('letter' or 'a4'). -latex_paper_size = 'a4' +# The paper size ('letter' or 'a4paper'). +latex_elements = {'papersize': 'a4paper'} # The font size ('10pt', '11pt' or '12pt'). #latex_font_size = '10pt' diff --git a/documentation/copyright.rst b/documentation/copyright.rst index 2c97a69ec..dd1fe7a7c 100644 --- a/documentation/copyright.rst +++ b/documentation/copyright.rst @@ -1,65 +1,8 @@ -.. EOxServer Open License - #----------------------------------------------------------------------------- - # - # Project: EOxServer - # Authors: Stephan Krause - # Stephan Meissl - # - #----------------------------------------------------------------------------- - # Copyright (C) 2011 EOX IT Services GmbH - # - # Permission is hereby granted, free of charge, to any person obtaining a copy - # of this software and associated documentation files (the "Software"), to - # deal in the Software without restriction, including without limitation the - # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - # sell copies of the Software, and to permit persons to whom the Software is - # furnished to do so, subject to the following conditions: - # - # The above copyright notice and this permission notice shall be included in - # all copies of this Software or works derived from this Software. - # - # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - # IN THE SOFTWARE. - #----------------------------------------------------------------------------- - -.. index:: - single: License - single: EOxServer Open License - single: EOxServer-SoapProxy Open License - single: Credits - -.. _EOxServer Open License: +.. _license: +======= License ======= -EOxServer Open License ----------------------- - -| EOxServer Open License -| Version 1, 8 June 2011 - -Copyright (C) 2011 EOX IT Services GmbH - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies of this Software or works derived from this Software. +.. include:: ../LICENSE -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/documentation/credits.rst b/documentation/credits.rst index a08d88aa0..321a06ba5 100644 --- a/documentation/credits.rst +++ b/documentation/credits.rst @@ -35,12 +35,9 @@ Credits ======= -.. figure:: ./_static/HMA_Logo.jpg - :target: http://rssportal.esa.int/tiki-index.php?page=Open%20Software +.. figure:: ./_static/ESA_logo.png + :width: 400 -Work on EOxServer has been partly funded by the `European Space Agency (ESA)`_ -in the frame of the HMA-FO_ and O3S_ projects. +Work on EOxServer has been funded by the `European Space Agency (ESA)`_ -.. _European Space Agency (ESA): http://www.esa.int/esaMI/ESRIN_SITE/ -.. _HMA-FO: http://wiki.services.eoportal.org/tiki-index.php?page=HMA-FO -.. _O3S: http://wiki.services.eoportal.org/tiki-index.php?page=O3S +.. _European Space Agency (ESA): http://www.esa.int/ diff --git a/documentation/developers/index.rst b/documentation/developers/index.rst index ba6bbe3bf..644e6c852 100644 --- a/documentation/developers/index.rst +++ b/documentation/developers/index.rst @@ -44,9 +44,6 @@ configuring the software stack and operators registering the available *EO Data* on the *Provider* side to end users consuming the registered *EO Data* on the *User* side. -.. figure:: ../users/images/Global_Use_Case.png - :align: center - .. toctree:: :maxdepth: 3 @@ -65,6 +62,3 @@ on the *User* side. atp_dev_guide testing code-style-guide - -.. TODO - processes diff --git a/documentation/index.rst b/documentation/index.rst index e34238d24..d5e25ab66 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -33,33 +33,11 @@ EOxServer's Documentation EOxServer is a Python application and framework for presenting Earth Observation (EO) data and metadata. -.. only:: html -.. raw:: html - -

- -EOxServer implements the `OGC `_ -Implementation Specifications EO-WCS and EO-WMS on top of -`MapServer's `_ -`WCS `_ and -`WMS `_ implementations. - -EOxServer is released under the -:ref:`EOxServer Open License ` a MIT-style -license and written in `Python `_ and entirely based on -Open Source software including `MapServer `_, -`Django/GeoDjango `_, -`GDAL `_, -`SpatiaLite `_, or -`PostGIS `_, and -`PROJ.4 `_. - -Here you find the documentation for users and developers of EOxServer -written in English. .. toctree:: :maxdepth: 1 + readme users/index developers/index release_notes/index diff --git a/documentation/readme.md b/documentation/readme.md new file mode 100644 index 000000000..451bedaec --- /dev/null +++ b/documentation/readme.md @@ -0,0 +1,2 @@ +```{include} ../README.md +``` diff --git a/documentation/release_notes/1.0.0.rst b/documentation/release_notes/1.0.0.rst new file mode 100644 index 000000000..6e3f5e98d --- /dev/null +++ b/documentation/release_notes/1.0.0.rst @@ -0,0 +1,49 @@ +.. _1-0-0-release-notes: + #----------------------------------------------------------------------------- + # $Id$ + # + # Project: EOxServer + # Authors: Fabian Schindler + # + #----------------------------------------------------------------------------- + # Copyright (C) 2014 EOX IT Services GmbH + # + # Permission is hereby granted, free of charge, to any person obtaining a copy + # of this software and associated documentation files (the "Software"), to + # deal in the Software without restriction, including without limitation the + # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + # sell copies of the Software, and to permit persons to whom the Software is + # furnished to do so, subject to the following conditions: + # + # The above copyright notice and this permission notice shall be included in + # all copies of this Software or works derived from this Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + # IN THE SOFTWARE. + #----------------------------------------------------------------------------- + +EOxServer 1.0.0 +=============== + +Updated data model +------------------ + +- New model Product to combine multiple Coverages, Browses, and Masks +- New models CollectionType, ProductType, CoverageType, BrowseType, and MaskType to allow grouping of said objects and store meta information. + +New render pipeline +------------------- + +- Allowing custom expressions to generate browse images from raw data +- Allowing to specify mask types to either mask in or mask out data. + +Service improvements +-------------------- + +- Implementing EO-WCS GetEOCoverageSet operation according to specification +- Adding non-standard cql parameter to several services such as OpenSearch diff --git a/documentation/release_notes/1.2.0.rst b/documentation/release_notes/1.2.0.rst new file mode 100644 index 000000000..1911109e0 --- /dev/null +++ b/documentation/release_notes/1.2.0.rst @@ -0,0 +1,48 @@ +.. _1-2-0-release-notes: + #----------------------------------------------------------------------------- + # $Id$ + # + # Project: EOxServer + # Authors: Fabian Schindler + # + #----------------------------------------------------------------------------- + # Copyright (C) 2014 EOX IT Services GmbH + # + # Permission is hereby granted, free of charge, to any person obtaining a copy + # of this software and associated documentation files (the "Software"), to + # deal in the Software without restriction, including without limitation the + # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + # sell copies of the Software, and to permit persons to whom the Software is + # furnished to do so, subject to the following conditions: + # + # The above copyright notice and this permission notice shall be included in + # all copies of this Software or works derived from this Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + # IN THE SOFTWARE. + #----------------------------------------------------------------------------- + +EOxServer 1.2.0 +=============== + +- Improved data access for multidimensional datasets +- Implemented Streaming VSI File response +- Added numpy wrapper +- Improved interpolate function +- Implemented dtype conversion utility +- Improved mapserver factories +- Added out of bounds argument for browsetype +- Implemented LUT rendering +- Added timeseries registration +- Improved OWS and Opensearch services +- Added cloud coverage WPS process +- Fixed bugs +- Added tests +- Improved typing +- Added docs +- Improved project structure & CI/CD diff --git a/documentation/release_notes/index.rst b/documentation/release_notes/index.rst index 418b64686..ad85b20d4 100644 --- a/documentation/release_notes/index.rst +++ b/documentation/release_notes/index.rst @@ -27,7 +27,6 @@ # IN THE SOFTWARE. #----------------------------------------------------------------------------- - Release Notes ============= @@ -36,6 +35,8 @@ Release notes from various versions of EOxServer. .. toctree:: :maxdepth: 1 - 0.3.1 + 1.2.0 + 1.0.0 + 0.4 0.3.2 - 0.4 \ No newline at end of file + 0.3.1 diff --git a/documentation/users/backends.rst b/documentation/users/backends.rst index 212bb9ee3..22fc1eef9 100644 --- a/documentation/users/backends.rst +++ b/documentation/users/backends.rst @@ -38,7 +38,7 @@ and other files that either reside on a local or remote storage. The backends have a static representation in the database (i.e the data models) and a dynamic behavioral implementation: the handlers. -The combintation of both allows the registration of storages, +The combination of both allows the registration of storages, backend authorization and data items and the access at runtime. @@ -83,6 +83,9 @@ for a ZIP file storage the URL is the path to the ZIP file. Each storage can be given a name, which helps with management. +Storage can have `streaming` property set to `true` which wherever possible uses +respective `streaming` version of `/vsi` file accessor, like `/vsicurl_streaming`. + A Storage can be linked to a `Storage Auth`_ model, which allows to specify authorization credentials. diff --git a/documentation/users/coverages.rst b/documentation/users/coverages.rst index eefc71157..0af9e650b 100644 --- a/documentation/users/coverages.rst +++ b/documentation/users/coverages.rst @@ -197,6 +197,35 @@ When a collection is linked to a `Collection Type`_ only Products and Coverages whose types are of the set of allowed coverage/product types can be inserted. +.. _RasterStyle Model: + +Raster Style +~~~~~~~~~~~~ + +A raster style is an instruction on ow to colorize a raster at the last step of +a rendering process of single band outputs to generate an RGB(A) image. + +A raster style has a name, title, abstract, type and a number of color entries. +Name, title, abstract are metadata displayed in the service capabilities. +Each color entry maps a value to a color, and has an optional label. The +type defines how the raster style colors are applied. The following types are +possible: + +* "ramp": this is the default. The colors are linearly interpolated for the + values. +* "values": only the colors specified in the color entries are rendered if they + exactly match the value. All other values are not rendered. +* "intervals": all values are mapped to the color of their next lower color + scale entry. + +Raster styles are linked to browse types using a distinct style name, so that +such styles can be re-used in multiple browse types. + +There are a number of default raster styles available, for when no raster +styles are configured. As soon as at least one raster style is configured, it +replaces all default raster styles. + + Command Line Interfaces ----------------------- @@ -233,7 +262,7 @@ coveragetype --in, -i read from ``stdin`` instead from a file - TODO: show definitition, example + TODO: show definition, example delete deletes a Coverage Type @@ -876,3 +905,87 @@ stac read the STAC Item from stdin instead from a file. --type TYPE_NAME, --product-type TYPE_NAME, -t TYPE_NAME the name of the new product type. Optional. + + +.. _cmd-rasterstyle: + +rasterstyle + this command allows to manage `Raster Style Model`_ instances and link them + with Browse Types. + + create + this sub-command creates a new raster style from a given set of values. + + name + The raster style name. Mandatory. + + import + this imports a raster style from an SLD file. + + filename + The SLD file name. Mandatory. + + --select + Only select the named styles. Can be specified multiple times. + + --rename + Rename a style from a name to another name + + delete + this sub-command deletes a raster style. + + name + The raster style name. Mandatory. + + link + this sub-command links a raster style to a browse type. + + name + The raster style name. Mandatory. + + product_type_name + The product type name containing the browse type. Mandatory. + + browse_type_name + The browse type name. Mandatory. + + style_name + The assigned style name. Optional. + + +.. _cmd-timeseries: + +timeseries + this command manages Time series instances (e.g zarr), Time series are registered as multiple instances of `Product Model`_, + for each time step (slice), each product item would have every dimension represented by a Coverage instance. + + register + this sub-command registers a timeseries Item as multiple Products and each Product dimensions as + Coverages. + + --collection, -c, --collection-identifier + Register timeseries for this collection. + --storage + The storage to use. + --path + Path to timeseries file. + --product-type-name + The product type name. + --coverage-type-mapping + Which dimension to map to which coverage type. + Use : as separator, e.g. --coverage-type-mapping "/Band1:b1" + --x-dim-name + Name of the array/band which represents X dimension. + --y-dim-name + Name of the array/band which represents Y dimension. + --time-dim-name + Name of the array/band which represents Time dimension. + --product-template + Format string for product identifier. Can use the following template variables: + collection_identifier, file_identifier, index, + product_type, begin_time, end_time. + --replace, -r + Optional. If the time series with the given identifier already + exists, replace it. Without this flag, this would result in + an error. + diff --git a/documentation/users/instance.rst b/documentation/users/instance.rst index 694455585..9b2c05728 100644 --- a/documentation/users/instance.rst +++ b/documentation/users/instance.rst @@ -152,8 +152,19 @@ EOXS_MAPSERVER_LAYER_FACTORIES 'eoxserver.render.mapserver.factories.MaskLayerFactory', 'eoxserver.render.mapserver.factories.MaskedBrowseLayerFactory', 'eoxserver.render.mapserver.factories.OutlinesLayerFactory', + 'eoxserver.render.mapserver.factories.HeatmapLayerFactory', ] + +DEFAULT_EOXS_MAPSERVER_HEATMAP_RANGE_DEFAULT = (0, 10) + The default range for heatmap layers when none are provided via ``dim_range``. + + Default: + + .. code-block:: python + + (0, 10) + EOXS_COVERAGE_METADATA_FORMAT_READERS The list of coverage metadata readers that will be employed to read metadata when a new coverage is registered. diff --git a/documentation/users/operations/images/dataset_diagraml.png b/documentation/users/operations/images/dataset_diagraml.png new file mode 100644 index 000000000..ebe617228 Binary files /dev/null and b/documentation/users/operations/images/dataset_diagraml.png differ diff --git a/documentation/users/operations/images/product_heatmap.png b/documentation/users/operations/images/product_heatmap.png new file mode 100644 index 000000000..645b6ef81 Binary files /dev/null and b/documentation/users/operations/images/product_heatmap.png differ diff --git a/documentation/users/operations/management.rst b/documentation/users/operations/management.rst index 632129a7f..de23604cb 100644 --- a/documentation/users/operations/management.rst +++ b/documentation/users/operations/management.rst @@ -344,6 +344,40 @@ The next step is to register a Coverage and associate it with the Product. For the data access let us define that the Product identifier is ``Product-A`` this the Coverages identifier is ``Product-A_coverage``. +Time Series registration +------------------------ + +Time series rasters (e.g zarr) are structured differently than other regular +raster data. +When registering time series data -for example the one shown in the figure +below- eoxserver computes the time series spatial extent (x & y) from the +latitude & longitude arrays, and using time array eoxserver creates +one product for each time period (slice), each product(of a specific time) +will have n added coverages where each one represents a slice of a band +from the data (e.g if we have temperature and precipitation +bands - as shown in the figure-, each product will have 2 +coverages -temperature & precipitation- ) + +.. figure:: ./images/dataset_diagraml.png + + +The special `timeseries` registration command can be used to +handle time series registration e.g: + +.. code-block:: bash + + python3 manage.py timeseries register -c \ + --storage \ + --path \ + --product-type-name \ + --x-dim-name "/latitude" --y-dim-name "/longitude" --time-dim-name "/time" \ + --product-template "{collection_identifier}_{file_identifier}_{index}" \ + --coverage-type-mapping "/temperature:temperature" \ + --coverage-type-mapping "/precipitation:precipitation" + + + + Data access ----------- @@ -374,6 +408,7 @@ This results in a catalog of the following available layers: geometries. - ``Collection__outlined``: this is a combination of the previous two layers: each Product is rendered in ``TRUE_COLOR`` with its outlines highlighted. + - ``Collection__heatmap``: this renders the heatmap of the Products footprints. - ``Collection__TRUE_COLOR``, ``Collection__FALSE_COLOR``, ``Collection__NDVI``: these are the browse visualizations with the definintions from earlier. @@ -401,6 +436,8 @@ The following list shows all of these rendering options with an example product +-----------------------------------+---------------------------------------------------+ | ``Collection__outlined`` | .. figure:: images/product_outlined.png | +-----------------------------------+---------------------------------------------------+ + | ``Collection__heatmap`` | .. figure:: images/product_heatmap.png | + +-----------------------------------+---------------------------------------------------+ | ``Collection__validity`` | .. figure:: images/product_validity.png | +-----------------------------------+---------------------------------------------------+ | ``Collection__masked_validity`` | .. figure:: images/product_masked_validity.png | diff --git a/documentation/users/services/wms.rst b/documentation/users/services/wms.rst index 220cdad16..0e0bff905 100644 --- a/documentation/users/services/wms.rst +++ b/documentation/users/services/wms.rst @@ -122,7 +122,11 @@ parameters that are available with GetMap requests. | | mask of the provided ``mask-name``. | | | | | - ````: renders the product(s) | | | | | according to the browse types instructions (or uses | | | - | | an already existing browse if available. | | | + | | an already existing browse if available) | | | + | | | | | + | | - ``Collection`` | | | + | | - ``heatmap``: renders the contained products in a | | | + | | heatmap. | | | +---------------------------+-----------------------------------------------------------+----------------------------------+--------------------------------+ | styles | The style for each of the rendered layers to be | | M | | | rendered with. This must be either empty or a | | | @@ -133,7 +137,7 @@ parameters that are available with GetMap requests. | | The available styles depend on the layer type. Outline | | | | | and mask layers can be rendered in the basic colors. | | | | | Single band output can be styled using a range of | | | - | | color scales. | | | + | | color scales (Raster styles may apply). | | | | | | | | | | The Capabilities document lists the available styles per | | | | | layer. | | | diff --git a/docker/eoxserver-entrypoint.sh b/entrypoint.sh similarity index 63% rename from docker/eoxserver-entrypoint.sh rename to entrypoint.sh index f36c88128..689f01291 100755 --- a/docker/eoxserver-entrypoint.sh +++ b/entrypoint.sh @@ -3,24 +3,6 @@ # eoxserver-entrypoint.sh # This is the docker ENTRYPOINT script (https://docs.docker.com/engine/reference/builder/#entrypoint) # It ensures a database connection and a running instance before refering execution to the passed command -# -# environment variables: -# - DB: Specify the used database type. either of "spatialite" or "postgis" -# - DB_PW, DB_NAME, DB_HOST, DB_USER: these credentials will be used to establish a -# connection to the postgres database when DB is set to "postgis" in order to wait -# for it to come online -# - INSTANCE_NAME: the name of the instance passed to `eoxserver-instance.py` -# - DJANGO_USER, DJANGO_MAIL, DJANGO_PASSWORD: when set, these credentials will be -# used to create a superuser to be used for the Django Admin. By default, no user is -# created -# - COLLECT_STATIC: if set to "true" (the default), static files will be collected -# upon initialization -# - PREINIT_SCRIPTS: if set, the list of commands that will be executed before -# the instance is initialized -# - INIT_SCRIPTS: if set, the list of commands that will be executed once -# when the instance is initialized -# - STARTUP_SCRIPTS: if set, the list of commands that will be executed before -# the command is run # select python interpreter PYTHON=$(which python3 || which python) @@ -45,8 +27,8 @@ if [ ! -d "${INSTANCE_DIR}" ]; then source $f done fi - - eoxserver-instance.py "${INSTANCE_NAME}" "${INSTANCE_DIR}" + + /opt/eoxserver/eoxserver/scripts/eoxserver-instance.py "${INSTANCE_NAME}" "${INSTANCE_DIR}" cd "${INSTANCE_DIR}" # create the database schema @@ -78,7 +60,5 @@ if [ ! -z "${STARTUP_SCRIPTS}" ] ; then done fi -cd "${INSTANCE_DIR}" - # run the initial command -exec $@ +exec "$@" diff --git a/eoxserver/COMMITTERS b/eoxserver/COMMITTERS deleted file mode 100644 index 051fe6869..000000000 --- a/eoxserver/COMMITTERS +++ /dev/null @@ -1,15 +0,0 @@ -============== ===================== =================================== ================================== - Login Name Email / Contact Area(s) -============== ===================== =================================== ================================== -meissls Stephan Meissl stephan.meissl at eox.at Overall -krauses Stephan Krause stephan.krause at eox.at Overall -schindlerf Fabian Schindler fabian.schindler at eox.at OGC Services -novacek Milan Novacek milan.novacek at siemens.com S2P (WCS) Proxy -martin.paces Martin Paces martin.paces at eox.at WCS-T, WPS -abonitz Arndt Bonitz arndt.bonitz at ait.ac.at IDM -MiroslavHoudek Miroslav Houdek miroslav.houdek at iguassu.cz n/a -ungarj Joachim Ungar joachim.ungar at eox.at Usability & Testing -schillerc Christian Schiller christian.schiller at eox.at Documentation, Usability & Testing -locherm Marko Locher marko.locher at eox.at OSGeo Live Package -santilland Daniel Santillan daniel.santillan at eox.at Client & 3D extensions -============== ===================== =================================== ================================== diff --git a/eoxserver/COPYING b/eoxserver/COPYING deleted file mode 100644 index 8b3e82536..000000000 --- a/eoxserver/COPYING +++ /dev/null @@ -1,22 +0,0 @@ - EOxServer Open License - Version 1, 8 June 2011 - -Copyright (C) 2011 EOX IT Services GmbH - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies of this Software or works derived from this Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/eoxserver/INSTALL b/eoxserver/INSTALL deleted file mode 100644 index 36cf2dd13..000000000 --- a/eoxserver/INSTALL +++ /dev/null @@ -1,110 +0,0 @@ --------------------------------------------------------------------------------- - - Project: EOxServer - Purpose: - Authors: Stephan Krause - Stephan Meissl - --------------------------------------------------------------------------------- -Copyright (C) 2011 EOX IT Services GmbH - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies of this Software or works derived from this Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. --------------------------------------------------------------------------------- - - -############################################## -# Quick installation guide for the impatient # -############################################## - -sudo pip install eoxserver -eoxserver-admin.py create_instance YOUR_INSTANCE_ID --init_spatialite -cd YOUR_INSTANCE_ID -python manage.py syncdb - - -+--------------------------------------------+ -| Running from the command-line -+--------------------------------------------+ - -python manage.py runserver - -# Point your browser to: "http://localhost:8000/" - - -+--------------------------------------------+ -| Running via WSGI interface -+--------------------------------------------+ - -mkdir static -python manage.py collectstatic --noinput - -# Add the following to your Apache web server configuration -# (e.g. /etc/apache2/sites-enabled/eoxserver): ------------------------------------------------------------------ -Alias /static "/static" -Alias /eoxserver "/wsgi.py" -"> - AllowOverride None - Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch - AddHandler wsgi-script .py - Order allow,deny - allow from all - ------------------------------------------------------------------ -# Restart Apache web server and point your browser to: -# "http://. - - --------------------------------------------------------------------------------- -License (see also file named COPYING) --------------------------------------------------------------------------------- - -Copyright (C) 2011 EOX IT Services GmbH - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies of this Software or works derived from this Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - --------------------------------------------------------------------------------- -Credits --------------------------------------------------------------------------------- - -Work on EOxServer has been partly funded by the European Space Agency (ESA) -in the frame of the HMA-FO and O3S projects. -Link: http://rssportal.esa.int/tiki-index.php?page=Open%20Software diff --git a/eoxserver/__init__.py b/eoxserver/__init__.py index 3ef2dabf2..267410703 100644 --- a/eoxserver/__init__.py +++ b/eoxserver/__init__.py @@ -28,7 +28,7 @@ # ------------------------------------------------------------------------------ -__version__ = '1.1.4-dev10' +__version__ = '1.3.3' def get_version(): diff --git a/eoxserver/backends/access.py b/eoxserver/backends/access.py index 3c556e658..53bb51c17 100644 --- a/eoxserver/backends/access.py +++ b/eoxserver/backends/access.py @@ -101,7 +101,7 @@ def retrieve(data_item, cache=None): tmp_path = cache.relative_path(item_id) if not cache.contains(item_id): # actually retrieve the item when not in the cache - handler = handler_cls(path or storage.url) + handler = handler_cls(path or storage.url, storage.streaming) use_cache, path = handler.retrieve( path or child_storage.url, tmp_path ) @@ -112,7 +112,7 @@ def retrieve(data_item, cache=None): if storage_handlers: storage, handler_cls = storage_handlers[-1] - handler = handler_cls(path) + handler = handler_cls(path, storage.streaming) return handler.retrieve(data_item.location)[1] @@ -143,14 +143,23 @@ def get_vsi_path(data_item): location = data_item.location storage = data_item.storage - return get_vsi_storage_path(storage, location) + vsi_path = get_vsi_storage_path(storage, location) + subdataset_type = getattr(data_item, 'subdataset_type', None) + subdataset_locator = getattr(data_item, 'subdataset_locator', None) + + if subdataset_type: + vsi_path = '%s:"%s"' % (subdataset_type, vsi_path) + if subdataset_locator: + vsi_path = '%s:%s' % (vsi_path, subdataset_locator) + + return vsi_path def get_vsi_storage_path(storage, location=None): while storage: handler_cls = get_handler_class_for_model(storage) if handler_cls: - handler = handler_cls(storage.url) + handler = handler_cls(storage.url, storage.streaming) location = handler.get_vsi_path(location or '') else: raise AccessError( @@ -169,7 +178,7 @@ def get_vsi_env(storage): while storage: handler_cls = get_handler_class_for_model(storage) if handler_cls: - handler = handler_cls(storage.url) + handler = handler_cls(storage.url, storage.streaming) env.update(handler.get_vsi_env()) else: raise AccessError( diff --git a/eoxserver/backends/management/commands/storage.py b/eoxserver/backends/management/commands/storage.py index bc335289e..5e3e2e80c 100644 --- a/eoxserver/backends/management/commands/storage.py +++ b/eoxserver/backends/management/commands/storage.py @@ -72,6 +72,20 @@ def add_arguments(self, parser): dest='storage_auth_name', default=None, help='The name of the storage auth to use. Optional', ) + create_parser.add_argument( + '--streaming', action="store_true", + default=False, + help="""If used, respective streaming version of /vsi file + accessor will be used.""" + ) + + create_parser.add_argument( + '--replace', action='store_true', + default=False, + help=( + 'Replace storage definition if already exists.' + ) + ) for parser in [list_parser, env_parser]: parser.add_argument( @@ -103,7 +117,7 @@ def handle(self, subcommand, name, *args, **kwargs): self.handle_env(name, *args, **kwargs) def handle_create(self, name, url, type_name, parent_name, - storage_auth_name, **kwargs): + storage_auth_name, streaming, replace, **kwargs): """ Handle the creation of a new storage. """ url = url[0] @@ -138,11 +152,22 @@ def handle_create(self, name, url, type_name, parent_name, ) else: storage_auth = None - - backends.Storage.objects.create( - name=name, url=url, storage_type=type_name, parent=parent, - storage_auth=storage_auth, - ) + if replace: + backends.Storage.objects.update_or_create( + name=name, + defaults={ + 'url':url, + 'storage_type':type_name, + 'parent':parent, + 'storage_auth':storage_auth, + 'streaming':streaming, + }, + ) + else: + backends.Storage.objects.create( + storage_type=type_name, parent=parent, + storage_auth=storage_auth, streaming=streaming, + ) self.print_msg( 'Successfully created storage %s (%s)' % ( diff --git a/eoxserver/backends/management/commands/storageauth.py b/eoxserver/backends/management/commands/storageauth.py index 9f94be195..5a386141b 100644 --- a/eoxserver/backends/management/commands/storageauth.py +++ b/eoxserver/backends/management/commands/storageauth.py @@ -74,6 +74,14 @@ def add_arguments(self, parser): help='Check access to the storage auth.', ) + create_parser.add_argument( + '--replace', action='store_true', + default=False, + help=( + 'Replace storage auth definition if already exists.' + ) + ) + @transaction.atomic def handle(self, subcommand, name, *args, **kwargs): """ Dispatch sub-commands: create, delete, insert, exclude, purge. @@ -84,7 +92,7 @@ def handle(self, subcommand, name, *args, **kwargs): elif subcommand == "delete": self.handle_delete(name, *args, **kwargs) - def handle_create(self, name, url, type_name, parameters, check, **kwargs): + def handle_create(self, name, url, type_name, parameters, check, replace, **kwargs): """ Handle the creation of a new storage. """ url = url[0] @@ -102,15 +110,24 @@ def parse_parameter(key, value=None, *extra): parse_parameter(*param) for param in parameters ) - - storage_auth = backends.StorageAuth( - name=name, - url=url, - storage_auth_type=type_name, - auth_parameters=json.dumps(parameters), - ) - storage_auth.full_clean() - storage_auth.save() + if replace: + backends.StorageAuth.objects.update_or_create( + name=name, + defaults={ + 'url':url, + 'storage_auth_type':type_name, + 'auth_parameters':json.dumps(parameters), + }, + ) + else: + storage_auth = backends.StorageAuth( + name=name, + url=url, + storage_auth_type=type_name, + auth_parameters=json.dumps(parameters), + ) + storage_auth.full_clean() + storage_auth.save() if check: _ = get_handler_for_model(storage_auth) diff --git a/eoxserver/backends/migrations/0004_storage_streaming.py b/eoxserver/backends/migrations/0004_storage_streaming.py new file mode 100644 index 000000000..ba47210c6 --- /dev/null +++ b/eoxserver/backends/migrations/0004_storage_streaming.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.18 on 2023-04-07 07:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('backends', '0003_nameblank'), + ] + + operations = [ + migrations.AddField( + model_name='storage', + name='streaming', + field=models.BooleanField(blank=True, null=True), + ), + ] diff --git a/eoxserver/backends/models.py b/eoxserver/backends/models.py index dea1605ba..7c6885afe 100644 --- a/eoxserver/backends/models.py +++ b/eoxserver/backends/models.py @@ -65,6 +65,7 @@ class Storage(models.Model): storage_type = models.CharField(max_length=32, **mandatory) name = models.CharField(max_length=1024, null=True, blank=True, unique=True) storage_auth = models.ForeignKey(StorageAuth, on_delete=models.CASCADE, **optional) + streaming = models.BooleanField(null=True, blank=True) parent = models.ForeignKey("self", on_delete=models.CASCADE, **optional) diff --git a/eoxserver/backends/storages.py b/eoxserver/backends/storages.py index 042ae18ad..d07d22ac7 100644 --- a/eoxserver/backends/storages.py +++ b/eoxserver/backends/storages.py @@ -110,9 +110,10 @@ class ZIPStorageHandler(BaseStorageHandler): is_local = True - def __init__(self, package_filename): + def __init__(self, package_filename, streaming): self.package_filename = package_filename self.zipfile = None + self.streaming = streaming def __enter__(self): self.zipfile = zipfile.ZipFile(self.package_filename, "r") @@ -153,9 +154,10 @@ class TARStorageHandler(BaseStorageHandler): is_local = True - def __init__(self, package_filename): + def __init__(self, package_filename, streaming): self.package_filename = package_filename self.tarfile = None + self.streaming = streaming def __enter__(self): self.tarfile = tarfile.TarFile(self.package_filename, "r") @@ -197,8 +199,9 @@ class DirectoryStorageHandler(BaseStorageHandler): is_local = True - def __init__(self, dirpath): + def __init__(self, dirpath, streaming): self.dirpath = dirpath + self.streaming = streaming def retrieve(self, location, path): return False, os.path.join(self.dirpath, location) @@ -224,15 +227,20 @@ class HTTPStorageHandler(BaseStorageHandler): allows_child_storages = True allows_parent_storage = False - def __init__(self, url): + def __init__(self, url, streaming): self.url = url + self.streaming = streaming def retrieve(self, location, path): request.urlretrieve(parse.urljoin(self.url, location), path) return True, path def get_vsi_path(self, location): - return '/vsicurl/%s' % parse.urljoin(self.url, location) + if self.streaming: + prefix = '/vsicurl_streaming/' + else: + prefix = '/vsicurl/' + return '%s%s' % (prefix, parse.urljoin(self.url, location)) @classmethod def test(cls, locator): @@ -251,10 +259,11 @@ class FTPStorageHandler(BaseStorageHandler): allows_parent_storage = True allows_parent_storage = False - def __init__(self, url): + def __init__(self, url, streaming): self.url = url self.parsed_url = urlparse(url) self.ftp = None + self.streaming=streaming def __enter__(self): self.ftp = ftplib.FTP() @@ -285,7 +294,11 @@ def list_files(self, location, glob_pattern=None): return filenames def get_vsi_path(self, location): - return '/vsicurl/%s' % parse.urljoin(self.url, location) + if self.streaming: + prefix = '/vsicurl_streaming/' + else: + prefix = '/vsicurl/' + return '%s%s' % (prefix, parse.urljoin(self.url, location)) @classmethod def test(cls, locator): @@ -301,8 +314,9 @@ class SwiftStorageHandler(BaseStorageHandler): allows_parent_storage = False allows_child_storages = True - def __init__(self, url): + def __init__(self, url, streaming): self.container = url + self.streaming = streaming def retrieve(self, location, path): pass @@ -311,7 +325,12 @@ def list_files(self, location, glob_pattern=None): return [] def get_vsi_path(self, location): - return vsi.join('/vsiswift/%s' % self.container, location) + if self.streaming: + prefix = '/vsiswift_streaming' + else: + prefix = '/vsiswift' + base_path = f'{prefix}/{self.container}' if self.container else prefix + return vsi.join(base_path, location) @classmethod def test(cls, locator): @@ -324,8 +343,9 @@ class S3StorageHandler(BaseStorageHandler): allows_parent_storage = False allows_child_storages = True - def __init__(self, url): + def __init__(self, url, streaming): self.bucket = url + self.streaming = streaming def retrieve(self, location, path): pass @@ -334,13 +354,11 @@ def list_files(self, location, glob_pattern=None): return [] def get_vsi_path(self, location): - import logging - logger = logging.getLogger(__name__) - - # logger.debug() - - - base_path = '/vsis3/%s' % self.bucket if self.bucket else '/vsis3' + if self.streaming: + prefix = '/vsis3_streaming' + else: + prefix = '/vsis3' + base_path = f'{prefix}/{self.bucket}' if self.bucket else prefix return vsi.join(base_path, location) @classmethod diff --git a/eoxserver/contrib/gdal.py b/eoxserver/contrib/gdal.py index 46fe7a0e7..7946ae340 100644 --- a/eoxserver/contrib/gdal.py +++ b/eoxserver/contrib/gdal.py @@ -184,4 +184,10 @@ def config_env(env, fail_on_override=False, reset_old=True): def open_with_env(path, env, shared=True): with config_env(env, False): + # if attempting to load NETCDF file with additional indexing, need to extract only base path + # this loads the variable dataset and allows information to be extracted from first band + if "NETCDF" in path: + if ("https://" in path and path.count(":") == 4) or (not "https://" in path and path.count(":") == 3): + splitpath = path.rsplit(":",1) + path = splitpath[0] return OpenShared(path) if shared else Open(path) diff --git a/eoxserver/contrib/mapserver.py b/eoxserver/contrib/mapserver.py index 6a72d1f6d..94756c6a0 100644 --- a/eoxserver/contrib/mapserver.py +++ b/eoxserver/contrib/mapserver.py @@ -285,7 +285,7 @@ def set_env(map_obj, env, fail_on_override=False, return_old=False): if fail_on_override or return_old: old_value = map_obj.getConfigOption(str(key)) if fail_on_override and old_value is not None \ - and old_value != value: + and old_value != value and old_value != 'None': raise Exception( 'Would override previous value of %s: %s with %s' % (key, old_value, value) diff --git a/eoxserver/contrib/vrt.py b/eoxserver/contrib/vrt.py index 253764b1e..450028a3d 100644 --- a/eoxserver/contrib/vrt.py +++ b/eoxserver/contrib/vrt.py @@ -498,6 +498,9 @@ def stack_bands(filenames, env, save=None): return out_ds +def sign_abs(x): + return 0.0 if abs(x) == 0 else x / abs(x) + def with_extent(filename, extent, save=None): """ Create a VRT and override the underlying files geolocation """ @@ -509,6 +512,9 @@ def with_extent(filename, extent, save=None): x = extent[0] y = extent[3] + source_geotransform = src_ds.GetGeoTransform() + resy_sign_north_up = sign_abs(source_geotransform[5]) + resx = abs(extent[2] - extent[0]) / width resy = abs(extent[3] - extent[1]) / height out_ds.SetGeoTransform([ @@ -517,6 +523,6 @@ def with_extent(filename, extent, save=None): 0, y, 0, - resy, + resy_sign_north_up * resy, ]) return out_ds diff --git a/eoxserver/instance_template/manage.py b/eoxserver/instance_template/manage.py index 6fcfc0d4d..41309844e 100644 --- a/eoxserver/instance_template/manage.py +++ b/eoxserver/instance_template/manage.py @@ -19,4 +19,4 @@ "forget to activate a virtual environment?" ) raise - execute_from_command_line(sys.argv) \ No newline at end of file + execute_from_command_line(sys.argv) diff --git a/eoxserver/instance_template/project_name/settings.py b/eoxserver/instance_template/project_name/settings.py index f606a5a7b..5c0471151 100644 --- a/eoxserver/instance_template/project_name/settings.py +++ b/eoxserver/instance_template/project_name/settings.py @@ -126,11 +126,11 @@ # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. # Example: "/var/www/example.com/static/" -STATIC_ROOT = join(PROJECT_DIR, 'static') +STATIC_ROOT = os.environ.get('STATIC_ROOT', join(PROJECT_DIR, 'static')) # URL prefix for static files. # Example: "http://example.com/static/", "http://static.example.com/" -STATIC_URL = '/{{ project_name }}_static/' +STATIC_URL = os.environ.get('STATIC_URL', '/{{ project_name }}_static/') # Additional locations of static files STATICFILES_DIRS = ( @@ -169,6 +169,7 @@ ] MIDDLEWARE = [ + 'django_prometheus.middleware.PrometheusBeforeMiddleware', 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -179,6 +180,7 @@ 'django.middleware.clickjacking.XFrameOptionsMiddleware', # # For management of the per/request cache system. # 'eoxserver.backends.middleware.BackendsCacheMiddleware', + 'django_prometheus.middleware.PrometheusAfterMiddleware', ] MIDDLEWARE_CLASSES = ( @@ -217,6 +219,7 @@ #'south', # Enable for debugging #'django_extensions', + 'django_prometheus', # Enable EOxServer: 'eoxserver.core', 'eoxserver.services', diff --git a/eoxserver/instance_template/project_name/urls.py b/eoxserver/instance_template/project_name/urls.py index d6a4967c2..87464a0c0 100644 --- a/eoxserver/instance_template/project_name/urls.py +++ b/eoxserver/instance_template/project_name/urls.py @@ -36,6 +36,7 @@ from django.urls import include, re_path from django.contrib import admin +import django_prometheus.exports from eoxserver.views import index @@ -58,4 +59,9 @@ re_path(r'^admin/doc/', include('django.contrib.admindocs.urls')), # Enable the admin: re_path(r'^admin/', admin.site.urls), + re_path( + r'^metrics$', + django_prometheus.exports.ExportToDjangoView, + name="prometheus-django-metrics", + ), ] diff --git a/eoxserver/instance_template/project_name/wsgi.py b/eoxserver/instance_template/project_name/wsgi.py index c4dae84dd..31b18b68c 100644 --- a/eoxserver/instance_template/project_name/wsgi.py +++ b/eoxserver/instance_template/project_name/wsgi.py @@ -48,13 +48,11 @@ if path not in sys.path: sys.path.append(path) -# NOTE: The Apache mod_wsgi, by default, shares the enviroment variables -# between different WSGI apps which leads to conflicts between -# multiple EOxServer instance. Therefore we cannot rely on the -# DJANGO_SETTINGS_MODULE enviromental variable we must always set it -# to the proper value. -#os.environ.setdefault("DJANGO_SETTINGS_MODULE", "{{ project_name }}.settings") -os.environ["DJANGO_SETTINGS_MODULE"] = "{{ project_name }}.settings" +# NOTE: Between 2013 and 2023, this used to override an existing +# DJANGO_SETTINGS_MODULE env var for use in apache with +# multiple eoxserver instances. This is however incompatible +# with the VS use case. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "{{ project_name }}.settings") # Initialize the EOxServer component system. import eoxserver.core diff --git a/eoxserver/render/browse/defaultstyles.py b/eoxserver/render/browse/defaultstyles.py new file mode 100644 index 000000000..cf91bfb7f --- /dev/null +++ b/eoxserver/render/browse/defaultstyles.py @@ -0,0 +1,25 @@ +from eoxserver.render.colors import COLOR_SCALES, BASE_COLORS +from eoxserver.render.browse.objects import ( + GeometryStyle, + RasterStyle, + RasterStyleColorEntry, +) + +DEFAULT_RASTER_STYLES = {} +DEFAULT_GEOMETRY_STYLES = {} + +for name, entries in COLOR_SCALES.items(): + DEFAULT_RASTER_STYLES[name] = RasterStyle( + name, + "ramp", + name, + name, + [ + RasterStyleColorEntry(i, color) + for i, color in entries + ] + ) + + +for name in BASE_COLORS.keys(): + DEFAULT_GEOMETRY_STYLES[name] = GeometryStyle(name, name, name) diff --git a/eoxserver/render/browse/functions.py b/eoxserver/render/browse/functions.py index 15b4874a2..4c80f6882 100644 --- a/eoxserver/render/browse/functions.py +++ b/eoxserver/render/browse/functions.py @@ -28,12 +28,14 @@ import logging from uuid import uuid4 from functools import wraps +from typing import List, SupportsFloat as Numeric import numpy as np from eoxserver.contrib import gdal from eoxserver.contrib import ogr from eoxserver.contrib import gdal_array +from eoxserver.render.browse.util import convert_dtype logger = logging.getLogger(__name__) @@ -53,15 +55,10 @@ def _dem_processing(data, processing, **kwargs): processing, **kwargs ) - - out_ds = gdal.Open(filename) - band = out_ds.GetRasterBand(1) - out_data = band.ReadAsArray() - del out_ds - finally: + except Exception: gdal.Unlink(filename) - - return gdal_array.OpenNumPyArray(out_data, False) + out_ds = gdal.Open(filename) + return out_ds def hillshade(data, zfactor=1, scale=1, azimuth=315, altitude=45, alg='Horn'): @@ -170,14 +167,11 @@ def contours(data, offset=0, interval=100, fill_value=-9999, format='raster'): outputBounds=[xmin, ymin, xmax, ymax], ) - out_ds = gdal.Open(out_filename) - band = out_ds.GetRasterBand(1) - out_data = gdal_array.OpenNumPyArray(band.ReadAsArray(), False) - del out_ds - gdal.Unlink(out_filename) + out_data = gdal.Open(out_filename) elif format == 'vector': out_data = vector_ds - + except Exception: + gdal.Unlink(out_filename) finally: vector_driver.DeleteDataSource(vec_filename) @@ -203,6 +197,12 @@ def pansharpen(pan_ds, *spectral_dss): ) out_ds = gdal_array.OpenNumPyArray(ds.ReadAsArray(), True) + # restore original nodata from pan band to output ds + nodata_value = pan_ds.GetRasterBand(1).GetNoDataValue() + if nodata_value is not None: + for i in range(out_ds.RasterCount): + out_ds.GetRasterBand(i + 1).SetNoDataValue(nodata_value) + return out_ds @@ -212,10 +212,6 @@ def percentile(ds, perc, default=0): if histogram: min_, max_, _, buckets = histogram bucket_diff = (max_ - min_) / len(buckets) - nodata = band.GetNoDataValue() - if nodata is not None: - # Set bucket of nodata value to 0 - buckets[round((nodata - min_) / bucket_diff)] = 0 cumsum = np.cumsum(buckets) bucket_index = np.searchsorted(cumsum, cumsum[-1] * (perc / 100)) return min_ + (bucket_index * bucket_diff) @@ -225,6 +221,7 @@ def percentile(ds, perc, default=0): def _has_stats(band): return 'STATISTICS_MINIMUM' in band.GetMetadata() + def statistics_min(ds, default=0): band = ds.GetRasterBand(1) if _has_stats(band): @@ -240,6 +237,7 @@ def statistics_max(ds, default=0): return max_ return default + def statistics_mean(ds, default=0): band = ds.GetRasterBand(1) if _has_stats(band): @@ -256,52 +254,101 @@ def statistics_stddev(ds, default=0): return default -def interpolate(ds, x1, x2, y1, y2): - """Perform linear interpolation for x between (x1,y1) and (x2,y2) """ +def interpolate( + ds:"gdal.Dataset", x1:Numeric, x2:Numeric, y1:Numeric, y2:Numeric, clip:bool=False, nodata_range:List[Numeric]=None + ): + """Perform linear interpolation for x between (x1,y1) and (x2,y2) with + optional clamp and additional masking out multiple no data value ranges + + Args: + ds (gdal.Dataset): input gdal dataset + x1 (Numeric): linear interpolate from min + x2 (Numeric): linear interpolate from max + y1 (Numeric): linear interpolate to min + y2 (Numeric): linear interpolate to max + clip (bool, optional): if set to True, performs clip (values below y1 set to y1, values above y2 set to y2). Defaults to False. + additional_no_data (List, optional): additionally masks out (sets to band no_data_value) a range of values. Defaults to []. Example [1,5] + + Returns: + gdal.Dataset: Interpolated dataset + """ band = ds.GetRasterBand(1) - x = band.ReadAsArray() - x = ((y2 - y1) * x + x2 * y1 - x1 * y2) / (x2 - x1) - return gdal_array.OpenNumPyArray(x, True) + nodata_value = band.GetNoDataValue() + orig_image = band.ReadAsArray() + # NOTE: the interpolate formula uses large numbers which lead to overflows on uint16 + if orig_image.dtype != convert_dtype(orig_image.dtype): + orig_image = orig_image.astype(convert_dtype(orig_image.dtype)) + interpolated_image = ((y2 - y1) * orig_image + x2 * y1 - x1 * y2) / (x2 - x1) + if clip: + # clamp values below min to min and above max to max + np.clip(interpolated_image, y1, y2, out=interpolated_image) + if nodata_value is not None: + # restore nodata pixels on interpolated array from original array + interpolated_image[orig_image == nodata_value] = nodata_value + if nodata_range: + # apply mask of additional nodata ranges from original array on interpolated array + interpolated_image[(orig_image >= nodata_range[0]) & (orig_image <= nodata_range[1])] = nodata_value + + ds = gdal_array.OpenNumPyArray(interpolated_image, True) + if nodata_value is not None: + ds.GetRasterBand(1).SetNoDataValue(nodata_value) + return ds def wrap_numpy_func(function): @wraps(function) - def inner(ds, *args, **kwargs): - band = ds.GetRasterBand(1) - data = band.ReadAsArray() - function(data, *args, **kwargs) - band.WriteArray(data) - return ds + def inner(*args, **kwargs): + converted_args = [] + for arg in args: + if isinstance(arg, gdal.Dataset): + band = arg.GetRasterBand(1) + data = band.ReadAsArray() + converted_args.append(data) + else: + converted_args.append(arg) + + result = function(*converted_args, **kwargs) + + if np.isscalar(result): + return result + + arg = args[0] + if isinstance(arg, gdal.Dataset): + band = arg.GetRasterBand(1) + band.WriteArray(result) + return arg + else: + return gdal_array.OpenNumPyArray(result, False) return inner function_map = { - 'sin': np.sin, - 'cos': np.cos, - 'tan': np.tan, - 'arcsin': np.arcsin, - 'arccos': np.arccos, - 'arctan': np.arctan, - 'hypot': np.hypot, - 'arctan2': np.arctan2, - 'degrees': np.degrees, - 'radians': np.radians, - 'unwrap': np.unwrap, - 'deg2rad': np.deg2rad, - 'rad2deg': np.rad2deg, - 'sinh': np.sinh, - 'cosh': np.cosh, - 'tanh': np.tanh, - 'arcsinh': np.arcsinh, - 'arccosh': np.arccosh, - 'arctanh': np.arctanh, - 'exp': np.exp, - 'expm1': np.expm1, - 'exp2': np.exp2, - 'log': np.log, - 'log10': np.log10, - 'log2': np.log2, - 'log1p': np.log1p, + 'sin': wrap_numpy_func(np.sin), + 'cos': wrap_numpy_func(np.cos), + 'tan': wrap_numpy_func(np.tan), + 'arcsin': wrap_numpy_func(np.arcsin), + 'arccos': wrap_numpy_func(np.arccos), + 'arctan': wrap_numpy_func(np.arctan), + 'hypot': wrap_numpy_func(np.hypot), + 'arctan2': wrap_numpy_func(np.arctan2), + 'degrees': wrap_numpy_func(np.degrees), + 'radians': wrap_numpy_func(np.radians), + 'unwrap': wrap_numpy_func(np.unwrap), + 'deg2rad': wrap_numpy_func(np.deg2rad), + 'rad2deg': wrap_numpy_func(np.rad2deg), + 'sinh': wrap_numpy_func(np.sinh), + 'cosh': wrap_numpy_func(np.cosh), + 'tanh': wrap_numpy_func(np.tanh), + 'arcsinh': wrap_numpy_func(np.arcsinh), + 'arccosh': wrap_numpy_func(np.arccosh), + 'arctanh': wrap_numpy_func(np.arctanh), + 'exp': wrap_numpy_func(np.exp), + 'expm1': wrap_numpy_func(np.expm1), + 'exp2': wrap_numpy_func(np.exp2), + 'log': wrap_numpy_func(np.log), + 'log10': wrap_numpy_func(np.log10), + 'log2': wrap_numpy_func(np.log2), + 'log1p': wrap_numpy_func(np.log1p), 'hillshade': hillshade, 'slopeshade': slopeshade, 'aspect': aspect, diff --git a/eoxserver/render/browse/generate.py b/eoxserver/render/browse/generate.py index c1af30647..73fb31135 100644 --- a/eoxserver/render/browse/generate.py +++ b/eoxserver/render/browse/generate.py @@ -116,6 +116,7 @@ class BandExpressionError(ValueError): _ast.Add, _ast.Sub, _ast.Num if hasattr(_ast, 'Num') else _ast.Constant, + _ast.List, _ast.BitAnd, _ast.BitOr, @@ -483,6 +484,10 @@ def _evaluate_expression(expr, fields_and_datasets, variables, cache): # Get a copy of the selected band data = value.GetRasterBand(slice_ + 1).ReadAsArray() result = gdal_array.OpenNumPyArray(data, True) + # restore nodata on output + nodata_value = value.GetRasterBand(slice_ + 1).GetNoDataValue() + if nodata_value is not None: + result.GetRasterBand(1).SetNoDataValue(nodata_value) elif hasattr(_ast, 'Num') and isinstance(expr, _ast.Num): result = expr.n @@ -490,6 +495,12 @@ def _evaluate_expression(expr, fields_and_datasets, variables, cache): elif hasattr(_ast, 'Constant') and isinstance(expr, _ast.Constant): result = expr.value + elif hasattr(_ast, 'List') and isinstance(expr, _ast.List): + result = [ + _evaluate_expression( + item, fields_and_datasets, variables, cache, + ) for item in expr.elts + ] else: raise BandExpressionError('Invalid expression node %s' % expr) diff --git a/eoxserver/render/browse/objects.py b/eoxserver/render/browse/objects.py index aa4fe3ef4..89afd2b3f 100644 --- a/eoxserver/render/browse/objects.py +++ b/eoxserver/render/browse/objects.py @@ -25,8 +25,11 @@ # THE SOFTWARE. # ------------------------------------------------------------------------------ +from typing import List, Tuple, Optional, Union + from django.contrib.gis.geos import Polygon from django.contrib.gis.gdal import SpatialReference, CoordTransform, DataSource +from django.conf import settings from eoxserver.contrib import gdal from eoxserver.backends.access import get_vsi_path, get_vsi_env, gdal_open @@ -36,10 +39,15 @@ BROWSE_MODE_RGB = "rgb" BROWSE_MODE_RGBA = "rgba" BROWSE_MODE_GRAYSCALE = "grayscale" +DEFAULT_EOXS_LAYER_SUFFIX_SEPARATOR = '__' + + +OptionalNumeric = Optional[Union[float, int]] class Browse(object): - def __init__(self, name, filename, env, size, extent, crs, mode, footprint): + def __init__(self, name, filename, env, size, extent, crs, mode, footprint, + raster_styles): self._name = name self._filename = filename self._env = env @@ -48,6 +56,7 @@ def __init__(self, name, filename, env, size, extent, crs, mode, footprint): self._crs = crs self._mode = mode self._footprint = footprint + self._raster_styles = raster_styles @property def name(self): @@ -94,7 +103,7 @@ def footprint(self): return polygon @classmethod - def from_model(cls, product_model, browse_model): + def from_model(cls, product_model, browse_model, raster_styles=None): filename = get_vsi_path(browse_model) env = get_vsi_env(browse_model.storage) size = (browse_model.width, browse_model.height) @@ -106,10 +115,13 @@ def from_model(cls, product_model, browse_model): ds = gdal_open(browse_model) mode = _get_ds_mode(ds) ds = None - - if browse_model.browse_type: - name = '%s__%s' % ( - product_model.identifier, browse_model.browse_type.name + suffix_separator = getattr( + settings, 'EOXS_LAYER_SUFFIX_SEPARATOR', + DEFAULT_EOXS_LAYER_SUFFIX_SEPARATOR + ) + if browse_model.browse_type and browse_model.browse_type.name: + name = '%s%s%s' % ( + product_model.identifier, suffix_separator, browse_model.browse_type.name ) else: name = product_model.identifier @@ -117,11 +129,12 @@ def from_model(cls, product_model, browse_model): return cls( name, filename, env, size, extent, browse_model.coordinate_reference_system, mode, - product_model.footprint + product_model.footprint, + raster_styles if raster_styles is not None else {} ) @classmethod - def from_file(cls, filename, env=None): + def from_file(cls, filename, env=None, raster_styles=None): env = env or {} ds = gdal.Open(filename) size = (ds.RasterXSize, ds.RasterYSize) @@ -130,13 +143,16 @@ def from_file(cls, filename, env=None): return cls( filename, env, filename, size, extent, - ds.GetProjection(), mode, None + ds.GetProjection(), mode, None, + raster_styles if raster_styles is not None else {}, ) class GeneratedBrowse(Browse): def __init__(self, name, band_expressions, ranges, nodata_values, - fields_and_coverages, field_list, footprint, variables): + fields_and_coverages, field_list, footprint, raster_styles, + variables, show_out_of_bounds_data=False, + ): self._name = name self._band_expressions = band_expressions self._ranges = ranges @@ -144,7 +160,9 @@ def __init__(self, name, band_expressions, ranges, nodata_values, self._fields_and_coverages = fields_and_coverages self._field_list = field_list self._footprint = footprint + self._raster_styles = raster_styles self._variables = variables + self._show_out_of_bounds_data = show_out_of_bounds_data @property def name(self): @@ -185,11 +203,11 @@ def band_expressions(self): return self._band_expressions @property - def ranges(self): + def ranges(self) -> List[Tuple[OptionalNumeric, OptionalNumeric]]: return self._ranges @property - def nodata_values(self): + def nodata_values(self) -> List[OptionalNumeric]: return self._nodata_values @property @@ -204,10 +222,19 @@ def field_list(self): def variables(self): return self._variables + @property + def raster_styles(self): + return self._raster_styles + + @property + def show_out_of_bounds_data(self) -> bool: + return self._show_out_of_bounds_data + @classmethod def from_coverage_models(cls, band_expressions, ranges, nodata_values, fields_and_coverage_models, - product_model, variables): + product_model, variables, raster_styles, + show_out_of_bounds_data): fields_and_coverages = { field_name: [ @@ -229,7 +256,9 @@ def from_coverage_models(cls, band_expressions, ranges, nodata_values, for field_name in fields_and_coverages.keys() ], product_model.footprint, + raster_styles, variables, + show_out_of_bounds_data, ) @@ -302,6 +331,95 @@ def from_models(cls, product_model, browse_model, mask_model, ) +class BaseStyle(object): + def __init__(self, name, title, abstract): + self._name = name + self._title = title or '' + self._abstract = abstract or '' + + @property + def name(self): + return self._name + + @property + def title(self): + return self._title + + @property + def abstract(self): + return self._abstract + + +class GeometryStyle(BaseStyle): + pass + + +class RasterStyle(BaseStyle): + def __init__(self, name, type, title, abstract, entries): + super().__init__(name, title, abstract) + self._type = type + self._entries = entries + + @property + def type(self): + return self._type + + @property + def entries(self): + return self._entries + + @classmethod + def from_model(cls, raster_style_model, name=None): + return cls( + name or raster_style_model.name, + raster_style_model.type, + raster_style_model.title, + raster_style_model.abstract, + [ + RasterStyleColorEntry.from_model(entry_model) + for entry_model in raster_style_model.color_entries.all() + ] + ) + + +def hex_to_rgb(hexa): + hexa = hexa.lstrip("#") + return tuple(int(hexa[i:i + 2], 16) for i in (0, 2, 4)) + + +class RasterStyleColorEntry(object): + def __init__(self, value, color, opacity=1.0, label=None): + self._value = value + self._color = color + self._opacity = opacity + self._label = label + + @property + def value(self): + return self._value + + @property + def color(self): + return self._color + + @property + def opacity(self): + return self._opacity + + @property + def label(self): + return self._label + + @classmethod + def from_model(cls, raster_style_color_entry_model): + return cls( + raster_style_color_entry_model.value, + hex_to_rgb(raster_style_color_entry_model.color), + raster_style_color_entry_model.opacity, + raster_style_color_entry_model.label, + ) + + def _get_ds_mode(ds): first = ds.GetRasterBand(1) diff --git a/eoxserver/render/browse/util.py b/eoxserver/render/browse/util.py index d89b112e2..fa4461310 100644 --- a/eoxserver/render/browse/util.py +++ b/eoxserver/render/browse/util.py @@ -1,8 +1,12 @@ from uuid import uuid4 +import numpy as np +import logging from eoxserver.contrib import gdal, osr from eoxserver.resources.coverages import crss +logger = logging.getLogger(__name__) + def create_mem_ds(width, height, data_type): driver = gdal.GetDriverByName('MEM') @@ -78,3 +82,27 @@ def warp_fields(coverages, field_name, bbox, crs, width, height): gdal.Unlink(vrt_filename) return out_ds + + +def convert_dtype(dtype:np.dtype): + """Maps numpy dtype to a larger itemsize + to avoid value overflow during mathematical operations + + Args: + dtype (np.dtype): input dtype + + Returns: + dtype (np.dtype): either one size larger dtype or original dtype + """ + mapping = { + np.dtype(np.int8): np.dtype(np.int16), + np.dtype(np.int16): np.dtype(np.int32), + np.dtype(np.int32): np.dtype(np.int64), + np.dtype(np.uint8): np.dtype(np.int16), + np.dtype(np.uint16): np.dtype(np.int32), + np.dtype(np.uint32): np.dtype(np.int64), + np.dtype(np.float16): np.dtype(np.float32), + np.dtype(np.float32): np.dtype(np.float64), + } + output_dtype = mapping.get(dtype, dtype) + return output_dtype diff --git a/eoxserver/render/colors.py b/eoxserver/render/colors.py index e49ab2421..c876cb38b 100644 --- a/eoxserver/render/colors.py +++ b/eoxserver/render/colors.py @@ -440,9 +440,9 @@ def linear(colors): ]), "brylgn" : linear([ - (130,67,0), - (255,200,110), - (255,255,179), + (130, 67, 0), + (255, 200, 110), + (255, 255, 179), (116, 234, 118), (0, 109, 0), ]), diff --git a/eoxserver/render/coverage/objects.py b/eoxserver/render/coverage/objects.py index 2ee700969..c6bf8a18f 100644 --- a/eoxserver/render/coverage/objects.py +++ b/eoxserver/render/coverage/objects.py @@ -31,6 +31,7 @@ from itertools import zip_longest as izip_longest from copy import deepcopy +from typing import List, Optional, Union from django.utils.six import string_types from eoxserver.core.util.timetools import parse_iso8601, parse_duration @@ -63,7 +64,7 @@ def __init__(self, index, identifier, description, definition, self._data_type_range = data_type_range @property - def index(self): + def index(self) -> int: return self._index @property @@ -476,11 +477,11 @@ def __init__(self, path, env, format, start_field, end_field, band_statistics): self._band_statistics = band_statistics @property - def start_field(self): + def start_field(self) -> int: return self._start_field @property - def end_field(self): + def end_field(self) -> int: return self._end_field @property @@ -539,7 +540,7 @@ def origin(self): return self._origin @property - def grid(self): + def grid(self) -> Grid: return self._grid @property @@ -555,7 +556,7 @@ def native_format(self): ) @property - def arraydata_locations(self): + def arraydata_locations(self) -> List[ArraydataLocation]: return self._arraydata_locations @property @@ -596,7 +597,9 @@ def extent(self): elif self.footprint: return self.footprint.extent - def lookup_field(self, field_or_identifier): + def lookup_field( + self, field_or_identifier: Union[Field, str] + ) -> Optional[Field]: if isinstance(field_or_identifier, Field): field = field_or_identifier if field not in self.range_type: @@ -612,7 +615,9 @@ def lookup_field(self, field_or_identifier): except StopIteration: return None - def get_location_for_field(self, field_or_identifier): + def get_location_for_field( + self, field_or_identifier: Union[Field, str], + ) -> Optional[ArraydataLocation]: field = self.lookup_field(field_or_identifier) index = field.index @@ -738,7 +743,7 @@ def end_time(self): return self._eo_metadata.end_time if self._eo_metadata else None @property - def range_type(self): + def range_type(self) -> RangeType: return self._range_type @property diff --git a/eoxserver/render/map/objects.py b/eoxserver/render/map/objects.py index 5150b64fc..17de17121 100644 --- a/eoxserver/render/map/objects.py +++ b/eoxserver/render/map/objects.py @@ -26,9 +26,12 @@ # ------------------------------------------------------------------------------ from weakref import proxy +from typing import List, Optional, Tuple + +from django.contrib.gis.geos import GEOSGeometry from eoxserver.render.coverage.objects import ( - GRID_TYPE_TEMPORAL, GRID_TYPE_ELEVATION + GRID_TYPE_TEMPORAL, GRID_TYPE_ELEVATION, Coverage, Mosaic, ) @@ -71,7 +74,7 @@ def __init__(self, name, style, coverage, bands, wavelengths, time, self._ranges = ranges @property - def coverage(self): + def coverage(self) -> Coverage: return self._coverage @property @@ -109,7 +112,7 @@ def __init__(self, name, style, coverages, bands, wavelengths, time, self._ranges = ranges @property - def coverages(self): + def coverages(self) -> List[Coverage]: return self._coverages @property @@ -184,7 +187,7 @@ def __init__(self, name, style, mosaic, coverages, bands, wavelengths, time, self._ranges = ranges @property - def mosaic(self): + def mosaic(self) -> Mosaic: return self._mosaic @property @@ -292,10 +295,28 @@ def fill(self): return self._fill +class HeatmapLayer(Layer): + """ Representation of a heatmap layer. + """ + def __init__(self, name: str, style: str, footprints: List[GEOSGeometry], + range: Optional[Tuple[float, float]] = None): + super(HeatmapLayer, self).__init__(name, style) + self._footprints = footprints + self._range = range + + @property + def footprints(self) -> List[GEOSGeometry]: + return self._footprints + + @property + def range(self) -> Optional[Tuple[float, float]]: + return self._range + + class Map(object): """ Abstract interpretation of a map to be drawn. """ - def __init__(self, layers, width, height, format, bbox, crs, bgcolor=None, + def __init__(self, layers: List[Layer], width, height, format, bbox, crs, bgcolor=None, transparent=True, time=None, elevation=None): self._layers = layers self._width = int(width) @@ -312,7 +333,7 @@ def __init__(self, layers, width, height, format, bbox, crs, bgcolor=None, layer.map = self @property - def layers(self): + def layers(self) -> List[Layer]: return self._layers @property diff --git a/eoxserver/render/mapserver/config.py b/eoxserver/render/mapserver/config.py index 08e75c806..156a1e37f 100644 --- a/eoxserver/render/mapserver/config.py +++ b/eoxserver/render/mapserver/config.py @@ -34,4 +34,11 @@ 'eoxserver.render.mapserver.factories.MaskLayerFactory', 'eoxserver.render.mapserver.factories.MaskedBrowseLayerFactory', 'eoxserver.render.mapserver.factories.OutlinesLayerFactory', + 'eoxserver.render.mapserver.factories.HeatmapLayerFactory', ] + + +# default for EOXS_MAPSERVER_HEATMAP_RANGE_DEFAULT: the default range for Heatmap +# render requests + +DEFAULT_EOXS_MAPSERVER_HEATMAP_RANGE_DEFAULT = (0, 10) diff --git a/eoxserver/render/mapserver/factories.py b/eoxserver/render/mapserver/factories.py index 3a2d5ecb3..2784f3153 100644 --- a/eoxserver/render/mapserver/factories.py +++ b/eoxserver/render/mapserver/factories.py @@ -26,6 +26,7 @@ # ------------------------------------------------------------------------------ from os.path import join +from typing import List, Type, Iterable, Tuple from uuid import uuid4 try: from itertools import izip_longest @@ -33,23 +34,28 @@ from itertools import zip_longest as izip_longest from django.conf import settings +from django.contrib.gis.geos import GEOSGeometry from django.utils.module_loading import import_string from eoxserver.core.util.iteratortools import pairwise_iterative from eoxserver.contrib import mapserver as ms -from eoxserver.contrib import vsi, vrt, gdal, osr +from eoxserver.contrib import vsi, vrt, gdal, osr, ogr from eoxserver.render.browse.objects import ( - Browse, GeneratedBrowse, BROWSE_MODE_GRAYSCALE + Browse, GeneratedBrowse, BROWSE_MODE_GRAYSCALE, BROWSE_MODE_RGBA ) from eoxserver.render.browse.generate import ( generate_browse, FilenameGenerator ) +from eoxserver.render.browse.defaultstyles import DEFAULT_RASTER_STYLES from eoxserver.render.map.objects import ( - CoverageLayer, CoveragesLayer, MosaicLayer, OutlinedCoveragesLayer, + CoverageLayer, CoveragesLayer, HeatmapLayer, MosaicLayer, OutlinedCoveragesLayer, BrowseLayer, OutlinedBrowseLayer, - MaskLayer, MaskedBrowseLayer, OutlinesLayer + MaskLayer, MaskedBrowseLayer, OutlinesLayer, + Layer, Map, ) +from eoxserver.render.coverage.objects import Coverage, Field from eoxserver.render.mapserver.config import ( + DEFAULT_EOXS_MAPSERVER_HEATMAP_RANGE_DEFAULT, DEFAULT_EOXS_MAPSERVER_LAYER_FACTORIES, ) from eoxserver.render.colors import BASE_COLORS, COLOR_SCALES, OFFSITE_COLORS @@ -64,23 +70,25 @@ class BaseMapServerLayerFactory(object): - handled_layer_types = [] + handled_layer_types: List[Type[Layer]] = [] @classmethod - def supports(self, layer_type): + def supports(self, layer_type: Type[Layer]): return layer_type in self.handled_layer_types - def create(self, map_obj, layer): + def create(self, map_obj: Map, layer: Layer): pass - def destroy(self, map_obj, layer, data): + def destroy(self, map_obj: Map, layer: Layer, data): pass class CoverageLayerFactoryMixIn(object): """ Base class for factories dealing with coverages. """ - def get_fields(self, fields, bands, wavelengths): + def get_fields( + self, fields: Iterable[Field], bands, wavelengths + ) -> List[Field]: """ Get the field subset for the given bands/wavelengths selection """ if bands: @@ -112,7 +120,7 @@ def get_fields(self, fields, bands, wavelengths): return fields - def create_coverage_layer(self, map_obj, coverage, fields, + def create_coverage_layer(self, map_obj: Map, coverage: Coverage, fields: List[Field], style=None, ranges=None): """ Creates a mapserver layer object for the given coverage """ @@ -132,8 +140,8 @@ def create_coverage_layer(self, map_obj, coverage, fields, # TODO: apply subsets in time/elevation dims num_locations = len(set(locations)) if num_locations == 1: + location = field_locations[0][1] if not coverage.grid.is_referenceable: - location = field_locations[0][1] data = location.path ms.set_env(map_obj, location.env, True) else: @@ -145,12 +153,13 @@ def create_coverage_layer(self, map_obj, coverage, fields, wkt = osr.SpatialReference(map_obj.getProjection()).wkt # TODO: env? - reftools.create_rectified_vrt( - field_locations[0][1].path, vrt_path, - order=1, max_error=10, - resolution=(resx, -resy), srid_or_wkt=wkt - ) - data = vrt_path + with gdal.config_env(location.env): + reftools.create_rectified_vrt( + location.path, vrt_path, + order=1, max_error=10, + resolution=(resx, -resy), srid_or_wkt=wkt + ) + data = vrt_path elif num_locations > 1: paths_set = set( @@ -177,7 +186,7 @@ def create_coverage_layer(self, map_obj, coverage, fields, sr = osr.SpatialReference(map_obj.getProjection()) layer_objs = _create_raster_layer_objs( - map_obj, extent, sr, data, filename_generator + map_obj, extent, sr, data, filename_generator, location.env, ) for i, layer_obj in enumerate(layer_objs): @@ -207,7 +216,10 @@ def create_coverage_layer(self, map_obj, coverage, fields, for layer_obj in layer_objs: _create_raster_style( - style or "blackwhite", layer_obj, range_[0], range_[1], [ + DEFAULT_RASTER_STYLES[style or "blackwhite"], + layer_obj, + range_[0], + range_[1], [ nil_value[0] for nil_value in field.nil_values ] ) @@ -270,7 +282,7 @@ class OutlinedCoverageLayerFactory(CoverageLayerFactoryMixIn, BaseMapServerLayerFactory): handled_layer_types = [OutlinedCoveragesLayer] - def create(self, map_obj, layer): + def create(self, map_obj, layer: CoveragesLayer): coverages = layer.coverages style = layer.style @@ -311,7 +323,7 @@ def destroy(self, map_obj, layer, data): class MosaicLayerFactory(CoverageLayerFactoryMixIn, BaseMapServerLayerFactory): handled_layer_types = [MosaicLayer] - def create(self, map_obj, layer): + def create(self, map_obj, layer: MosaicLayer): mosaic = layer.mosaic fields = self.get_fields( mosaic.range_type, layer.bands, layer.wavelengths @@ -379,8 +391,28 @@ def make_browse_layer_generator(self, map_obj, browses, map_, layer_obj.setMetaData("wms_srs", short_epsg) layer_obj.setProjection(sr.proj) - if browse.mode == BROWSE_MODE_GRAYSCALE: - field = browse.field_list[0] + if browse.mode == BROWSE_MODE_GRAYSCALE: + field = browse.field_list[0] + if ranges: + browse_range = ranges[0] + elif browse.ranges[0] != (None, None): + browse_range = browse.ranges[0] + else: + browse_range = _get_range(field) + + for layer_obj in layer_objs: + raster_style = browse.raster_styles.get(style or "blackwhite") or DEFAULT_RASTER_STYLES[style or "blackwhite"] + _create_raster_style( + raster_style, layer_obj, + browse_range[0], browse_range[1], + browse.nodata_values + ) + + else: + browse_iter = enumerate( + zip(browse.field_list, browse.ranges, browse.nodata_values), start=1 + ) + for i, (field, field_range, nodata_value) in browse_iter: if ranges: browse_range = ranges[0] elif browse.ranges[0] != (None, None): @@ -394,6 +426,34 @@ def make_browse_layer_generator(self, map_obj, browses, map_, browse_range[0], browse_range[1], browse.nodata_values ) + # NOTE: Only works if browsetype nodata is lower than browse_type_min by at least 1 + if browse.show_out_of_bounds_data: + # final LUT for min,max 200,700 and nodata=0 should look like: + # 0:0,1:1,200:1,700:256 + lut_inputs = { + range_[0]: 1, + range_[1]: 256, + } + if nodata_value is not None: + # no_data_value_plus +1 to ensure that only no_data_value is + # rendered as black (transparent) + nodata_value_plus = nodata_value + 1 + lut_inputs[nodata_value] = 0 + lut_inputs[nodata_value_plus] = 1 + + # LUT inputs needs to be ascending + sorted_inputs = { + k: v for k, v in sorted(list(lut_inputs.items())) + } + lut = ",".join("%d:%d" % (k,v) for k,v in sorted_inputs.items()) + + layer_obj.setProcessingKey("LUT_%d" % i, lut) + else: + # due to offsite 0,0,0 will make all pixels below or equal to min transparent + layer_obj.setProcessingKey( + "SCALE_%d" % i, + "%s,%s" % tuple(range_) + ) else: browse_iter = enumerate( @@ -419,10 +479,8 @@ def make_browse_layer_generator(self, map_obj, browses, map_, elif isinstance(browse, Browse): layer_objs = _create_raster_layer_objs( map_obj, browse.extent, browse.spatial_reference, - browse.filename, filename_generator + browse.filename, filename_generator, browse.env, browse.mode, ) - for layer_obj in layer_objs: - layer_obj.data = browse.filename ms.set_env(map_obj, browse.env, True) elif browse is None: # TODO: figure out why and deal with it? @@ -629,20 +687,126 @@ def create(self, map_obj, layer): layer_obj.insertClass(class_obj) +class HeatmapLayerFactory(BaseMapServerLayerFactory): + handled_layer_types = [HeatmapLayer] + + def _create_vector_ds(self, footprints: List[GEOSGeometry]) -> gdal.Dataset: + """ Stores the given footprints in a GDAL vector dataset. It uses the in-memory + driver to reduce disk IO. + """ + driver = gdal.GetDriverByName("Memory") + ds = driver.Create("", 0, 0, 0, gdal.GDT_Unknown) + layer = ds.CreateLayer("data") + from osgeo import osr + sr = osr.SpatialReference() + sr.ImportFromEPSG(4326) + for footprint in footprints: + feature = ogr.Feature(ogr.FeatureDefn()) + geom = ogr.CreateGeometryFromWkt(footprint.wkt, sr) + feature.SetGeometryDirectly(geom) + layer.CreateFeature(feature) + + return ds + + def _rasterize_footprints(self, map_obj: ms.mapObj, vector_ds: gdal.Dataset, + filename: str): + """ Rasterizes the footprints from a vector datasets into a raster dataset + of type Uint16 with the same dimension and spatial bounds as the provided map. + It uses additive mode in order to calculate how many geometries overlap with + a specific pixel. + The dataset is stored under a given filename (vsimem possible). + """ + sr = osr.SpatialReference() + sr.ImportFromProj4(map_obj.getProjection()) + extent = map_obj.extent + + gdal.Rasterize( + filename, + vector_ds, + format="GTiff", + width=map_obj.width, + height=map_obj.height, + outputType=gdal.GDT_UInt16, + outputBounds=(extent.minx, extent.miny, extent.maxx, extent.maxy), + outputSRS=sr.ExportToWkt(), + initValues=[0], + burnValues=[1], + add=True + ) + + def create(self, map_obj: ms.mapObj, layer: Layer): + """_summary_ + + Args: + map_obj (ms.mapObj): _description_ + layer (Layer): _description_ + + Returns: + _type_: _description_ + """ + assert isinstance(layer, HeatmapLayer) + + vector_ds = self._create_vector_ds(layer.footprints) + filename_generator = FilenameGenerator('/vsimem/{uuid}.{extension}') + filename = filename_generator.generate("tif") + self._rasterize_footprints(map_obj, vector_ds, filename) + + layer_obj = ms.layerObj(map_obj) + layer_obj.type = ms.MS_LAYER_RASTER + layer_obj.status = ms.MS_ON + layer_obj.data = filename + + layer_obj.setProjection(map_obj.getProjection()) + + default_range = getattr( + settings, 'EOXS_MAPSERVER_HEATMAP_RANGE_DEFAULT', + DEFAULT_EOXS_MAPSERVER_HEATMAP_RANGE_DEFAULT + ) + + range_ = layer.range or default_range + _create_raster_style( + DEFAULT_RASTER_STYLES[layer.style or "plasma"], + layer_obj, + range_[0], + range_[1], + [0], + ) + + return filename_generator + + def destroy(self, map_obj, layer, filename_generator): + # cleanup temporary files + for filename in filename_generator.filenames: + vsi.unlink(filename) + # ------------------------------------------------------------------------------ # utils # ------------------------------------------------------------------------------ -def _create_raster_layer_objs(map_obj, extent, sr, data, filename_generator, - resample=None): +def _create_raster_layer_objs(map_obj, extent, sr, data, filename_generator, env, + browse_mode=None, resample=None) -> List[ms.layerObj]: layer_obj = ms.layerObj(map_obj) layer_obj.type = ms.MS_LAYER_RASTER layer_obj.status = ms.MS_ON - layer_obj.data = data + # if attempting to load NETCDF file, need to use a temporary file to extract only + # the required band data + if "NETCDF" in data: + # determine if attempting to load a specific band index + if ("https://" in data and data.count(":") == 4) or (not "https://" in data and data.count(":") == 3): + splitpath = data.rsplit(":",1) + data = splitpath[0] + index = int(splitpath[1]) + 1 + temp_path = filename_generator.generate() + # extract only desired index to new temporary file + gdal.Translate(temp_path, data, bandList=[index]) + data = temp_path - layer_obj.offsite = ms.colorObj(0, 0, 0) + layer_obj.data = data + # assumption that RGBA already has transparency in alpha band + if browse_mode != BROWSE_MODE_RGBA: + layer_obj.offsite = ms.colorObj(0, 0, 0) if extent: layer_obj.setMetaData("wms_extent", "%f %f %f %f" % extent) @@ -666,10 +830,12 @@ def _create_raster_layer_objs(map_obj, extent, sr, data, filename_generator, wrapped_layer_obj.status = ms.MS_ON wrapped_data = filename_generator.generate() - vrt.with_extent(data, wrapped_extent, wrapped_data) + with gdal.config_env(env): + vrt.with_extent(data, wrapped_extent, wrapped_data) wrapped_layer_obj.data = wrapped_data - - wrapped_layer_obj.offsite = ms.colorObj(0, 0, 0) + # assumption that RGBA already has transparency in alpha band + if browse_mode != BROWSE_MODE_RGBA: + wrapped_layer_obj.offsite = ms.colorObj(0, 0, 0) wrapped_layer_obj.setMetaData("ows_srs", short_epsg) wrapped_layer_obj.setMetaData("wms_srs", short_epsg) @@ -759,10 +925,49 @@ def _build_vrt(size, field_locations): return path -def _create_raster_style(name, layer, minvalue=0, maxvalue=255, +def _create_raster_style(raster_style, layer, minvalue=0, maxvalue=255, nil_values=None): - colors = COLOR_SCALES[name] + if raster_style.type == "ramp": + return _create_raster_style_ramp( + raster_style, layer, minvalue, maxvalue, nil_values + ) + elif raster_style.type == "values": + for entry in raster_style.entries: + value = entry.value + if int(value) == value: + value = int(value) + cls = ms.classObj() + cls.setExpression("([pixel] = %s)" % value) + cls.group = entry.label + + style = ms.styleObj() + style.color = ms.colorObj(*entry.color) + style.opacity = int(entry.opacity * 100) + cls.insertStyle(style) + layer.insertClass(cls) + cls = ms.classObj() + style = ms.styleObj() + style.color = ms.colorObj(0, 0, 0, 0) + style.opacity = 0 + cls.insertStyle(style) + layer.insertClass(cls) + return + + elif raster_style.type == "intervals": + # TODO + return + raise ValueError("Invalid raster style type %r" % raster_style.type) + + +def _create_raster_style_ramp(raster_style, layer, minvalue=0, maxvalue=255, + nil_values=None): + name = raster_style.name + + colors = [ + (entry.value, entry.color) + for entry in raster_style.entries + ] if nil_values and all(v is not None for v in nil_values): nil_values = [float(nil_value) for nil_value in nil_values] else: @@ -820,7 +1025,7 @@ def _create_raster_style(name, layer, minvalue=0, maxvalue=255, next_perc, next_color = next_item cls = ms.classObj() - cls.setExpression("([pixel] > %s AND [pixel] < %s)" % ( + cls.setExpression("([pixel] > %s AND [pixel] <= %s)" % ( (minvalue + prev_perc * interval), (minvalue + next_perc * interval) )) @@ -844,7 +1049,7 @@ def _create_raster_style(name, layer, minvalue=0, maxvalue=255, high_nil = min(high_nil_values) cls = ms.classObj() cls.setExpression( - "([pixel] > %s AND [pixel] < %s)" % (maxvalue, high_nil) + "([pixel] > %s AND [pixel] <= %s)" % (maxvalue, high_nil) ) cls.group = name style = ms.styleObj() @@ -861,7 +1066,7 @@ def _create_raster_style(name, layer, minvalue=0, maxvalue=255, layer.insertClass(cls) -def _get_range(field, range_=None): +def _get_range(field: Field, range_=None) -> Tuple[int, int]: """ Gets the numeric range of a field """ if range_: @@ -893,7 +1098,7 @@ def _setup_factories(): ] -def get_layer_factories(): +def get_layer_factories() -> List[BaseMapServerLayerFactory]: if LAYER_FACTORIES is None: _setup_factories() return LAYER_FACTORIES diff --git a/eoxserver/render/mapserver/map_renderer.py b/eoxserver/render/mapserver/map_renderer.py index 4f96fd6d9..8f650e2cd 100644 --- a/eoxserver/render/mapserver/map_renderer.py +++ b/eoxserver/render/mapserver/map_renderer.py @@ -27,13 +27,19 @@ import logging import tempfile +from typing import List, Tuple, Type from uuid import uuid4 from contextlib import contextmanager from eoxserver.contrib import mapserver as ms from eoxserver.contrib import vsi -from eoxserver.render.colors import BASE_COLORS, COLOR_SCALES -from eoxserver.render.mapserver.factories import get_layer_factories +from eoxserver.render.browse.defaultstyles import ( + DEFAULT_RASTER_STYLES, DEFAULT_GEOMETRY_STYLES +) +from eoxserver.render.mapserver.factories import ( + BaseMapServerLayerFactory, get_layer_factories +) +from eoxserver.render.map.objects import Map, Layer from eoxserver.resources.coverages.formats import getFormatRegistry @@ -55,10 +61,10 @@ class MapserverMapRenderer(object): ] def get_geometry_styles(self): - return BASE_COLORS.keys() + return list(DEFAULT_GEOMETRY_STYLES.values()) def get_raster_styles(self): - return COLOR_SCALES.keys() + return list(DEFAULT_RASTER_STYLES.values()) def get_supported_layer_types(self): layer_types = [] @@ -221,10 +227,13 @@ def _prepare_map( for layer, factory, data in layers_plus_factories_plus_data: factory.destroy(map_obj, layer, data) - def _get_layers_plus_factories(self, layers): + def _get_layers_plus_factories( + self, + render_map: Map, + ) -> List[Tuple[Layer, BaseMapServerLayerFactory]]: layers_plus_factories = [] type_to_layer_factory = {} - for layer in layers: + for layer in render_map.layers: layer_type = type(layer) if layer_type in type_to_layer_factory: factory = type_to_layer_factory[layer_type] @@ -236,7 +245,7 @@ def _get_layers_plus_factories(self, layers): return layers_plus_factories - def _get_layer_factory(self, layer_type): + def _get_layer_factory(self, layer_type: Type[Layer]): for factory in get_layer_factories(): if factory.supports(layer_type): return factory diff --git a/eoxserver/resources/coverages/admin.py b/eoxserver/resources/coverages/admin.py index 47f6cc530..008d0a1ba 100644 --- a/eoxserver/resources/coverages/admin.py +++ b/eoxserver/resources/coverages/admin.py @@ -1,11 +1,11 @@ -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ # # Project: EOxServer # Authors: Fabian Schindler # Stephan Meissl # Stephan Krause # -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ # Copyright (C) 2011 EOX IT Services GmbH # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -25,7 +25,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ from django.contrib.gis import admin try: @@ -34,6 +34,7 @@ from django.urls import reverse, NoReverseMatch from django.utils.safestring import mark_safe from django.conf import settings +from django.forms import ModelForm, TextInput from eoxserver.resources.coverages import models @@ -93,7 +94,11 @@ class BrowseTypeInline(admin.StackedInline): 'alpha_expression', 'alpha_nodata_value', ('alpha_range_min', 'alpha_range_max'), ) - }) + }), + ("Show out of bounds data", { + 'classes': ('collapse', 'collapsed'), + 'fields': ('show_out_of_bounds_data',), + }), ) @@ -123,7 +128,9 @@ class MetaDataItemInline(admin.StackedInline): def download_link(self, obj): try: return mark_safe('Download'.format( - reverse('metadata', kwargs=dict( + reverse( + 'metadata', + kwargs=dict( identifier=obj.eo_object.identifier, semantic=dict( models.MetaDataItem.SEMANTIC_CHOICES @@ -197,6 +204,7 @@ def default_zoom(self): class CoverageTypeAdmin(admin.ModelAdmin): inlines = [FieldTypeInline] + admin.site.register(models.CoverageType, CoverageTypeAdmin) @@ -204,24 +212,28 @@ class ProductTypeAdmin(admin.ModelAdmin): inlines = [BrowseTypeInline, MaskTypeInline] filter_horizontal = ['allowed_coverage_types'] + admin.site.register(models.ProductType, ProductTypeAdmin) class CollectionTypeAdmin(admin.ModelAdmin): filter_horizontal = ['allowed_product_types', 'allowed_coverage_types'] + admin.site.register(models.CollectionType, CollectionTypeAdmin) class MaskTypeAdmin(admin.ModelAdmin): pass + admin.site.register(models.MaskType, MaskTypeAdmin) class GridAdmin(admin.ModelAdmin): pass + admin.site.register(models.Grid, GridAdmin) # ============================================================================== @@ -232,6 +244,7 @@ class GridAdmin(admin.ModelAdmin): class CoverageAdmin(EOObjectAdmin): inlines = [CoverageMetadataInline, MetaDataItemInline, ArrayDataItemInline] + admin.site.register(models.Coverage, CoverageAdmin) @@ -240,12 +253,14 @@ class ProductAdmin(EOObjectAdmin): MaskInline, BrowseInline, ProductDataItemInline, MetaDataItemInline, ProductMetadataInline ] + admin.site.register(models.Product, ProductAdmin) class MosaicAdmin(EOObjectAdmin): inlines = [] + admin.site.register(models.Mosaic, MosaicAdmin) @@ -275,6 +290,7 @@ class IndexHiddenAdmin(admin.ModelAdmin): def get_model_perms(self, request): return {} + admin.site.register(models.OrbitNumber, IndexHiddenAdmin) admin.site.register(models.Track, IndexHiddenAdmin) admin.site.register(models.Frame, IndexHiddenAdmin) @@ -288,3 +304,35 @@ def get_model_perms(self, request): admin.site.register(models.ProcessingMode, IndexHiddenAdmin) admin.site.register(models.AcquisitionStation, IndexHiddenAdmin) admin.site.register(models.AcquisitionSubType, IndexHiddenAdmin) + + +# ============================================================================== +# Raster Style models +# ============================================================================== + + +class RasterStyleColorEntryForm(ModelForm): + class Meta: + model = models.RasterStyleColorEntry + fields = '__all__' + widgets = { + 'color': TextInput(attrs={'type': 'color'}), + } + + +class RasterStyleColorEntryInline(admin.TabularInline): + model = models.RasterStyleColorEntry + form = RasterStyleColorEntryForm + extra = 0 + + +class RasterStyleToBrowseTypeThroughInline(admin.TabularInline): + model = models.RasterStyleToBrowseTypeThrough + extra = 0 + + +class RasterStyleAdmin(admin.ModelAdmin): + inlines = [RasterStyleToBrowseTypeThroughInline, RasterStyleColorEntryInline] + + +admin.site.register(models.RasterStyle, RasterStyleAdmin) diff --git a/eoxserver/resources/coverages/management/commands/browsetype.py b/eoxserver/resources/coverages/management/commands/browsetype.py index 391408bc6..a38920a46 100644 --- a/eoxserver/resources/coverages/management/commands/browsetype.py +++ b/eoxserver/resources/coverages/management/commands/browsetype.py @@ -104,6 +104,19 @@ def add_arguments(self, parser): '--alpha-nodata', type=float, dest='alpha_nodata', default=None, ) + create_parser.add_argument( + '--show-out-of-bounds-data', + action="store_true", + default=False, + ) + create_parser.add_argument( + '--replace', action='store_true', + default=False, + help=( + '''Change browse type if browse type already exists.''' + ) + ) + list_parser.add_argument( 'product_type_name', nargs=1, @@ -134,6 +147,8 @@ def handle_create(self, product_type_name, browse_type_name, blue_range=(None, None), alpha_range=(None, None), red_or_grey_nodata=None, green_nodata=None, blue_nodata=None, alpha_nodata=None, + show_out_of_bounds_data=False, + replace=False, *args, **kwargs): """ Handle the creation of a new browse type. """ @@ -151,27 +166,52 @@ def handle_create(self, product_type_name, browse_type_name, green_min, green_max = green_range blue_min, blue_max = blue_range alpha_min, alpha_max = alpha_range - - models.BrowseType.objects.create( - product_type=product_type, - name=browse_type_name, - red_or_grey_expression=red_or_grey_expression, - green_expression=green_expression, - blue_expression=blue_expression, - alpha_expression=alpha_expression, - red_or_grey_range_min=red_min, - red_or_grey_range_max=red_max, - green_range_min=green_min, - green_range_max=green_max, - blue_range_min=blue_min, - blue_range_max=blue_max, - alpha_range_min=alpha_min, - alpha_range_max=alpha_max, - red_or_grey_nodata_value=red_or_grey_nodata, - green_nodata_value=green_nodata, - blue_nodata_value=blue_nodata, - alpha_nodata_value=alpha_nodata, - ) + if replace: + models.BrowseType.objects.update_or_create( + product_type=product_type, + name=browse_type_name, + defaults={ + 'red_or_grey_expression':red_or_grey_expression, + 'green_expression':green_expression, + 'blue_expression':blue_expression, + 'alpha_expression':alpha_expression, + 'red_or_grey_range_min':red_min, + 'red_or_grey_range_max':red_max, + 'green_range_min':green_min, + 'green_range_max':green_max, + 'blue_range_min':blue_min, + 'blue_range_max':blue_max, + 'alpha_range_min':alpha_min, + 'alpha_range_max':alpha_max, + 'red_or_grey_nodata_value':red_or_grey_nodata, + 'green_nodata_value':green_nodata, + 'blue_nodata_value':blue_nodata, + 'alpha_nodata_value':alpha_nodata, + 'show_out_of_bounds_data':show_out_of_bounds_data, + }, + ) + else: + models.BrowseType.objects.create( + product_type=product_type, + name=browse_type_name, + red_or_grey_expression=red_or_grey_expression, + green_expression=green_expression, + blue_expression=blue_expression, + alpha_expression=alpha_expression, + red_or_grey_range_min=red_min, + red_or_grey_range_max=red_max, + green_range_min=green_min, + green_range_max=green_max, + blue_range_min=blue_min, + blue_range_max=blue_max, + alpha_range_min=alpha_min, + alpha_range_max=alpha_max, + red_or_grey_nodata_value=red_or_grey_nodata, + green_nodata_value=green_nodata, + blue_nodata_value=blue_nodata, + alpha_nodata_value=alpha_nodata, + show_out_of_bounds_data=show_out_of_bounds_data, + ) if not browse_type_name: print( diff --git a/eoxserver/resources/coverages/management/commands/collection.py b/eoxserver/resources/coverages/management/commands/collection.py index e5a4cfb0f..f4cfab4e3 100644 --- a/eoxserver/resources/coverages/management/commands/collection.py +++ b/eoxserver/resources/coverages/management/commands/collection.py @@ -73,6 +73,14 @@ def add_arguments(self, parser): '"platform".' ) ) + create_parser.add_argument( + '--replace', action='store_true', + default=False, + help=( + '''Change collection type references according to parameters + if collection type already exists.''' + ) + ) delete_parser.add_argument( '--all', '-a', action="store_true", default=False, dest='all_collections', @@ -155,7 +163,7 @@ def handle(self, subcommand, identifier, *args, **kwargs): elif subcommand == "summary": self.handle_summary(identifier[0], *args, **kwargs) - def handle_create(self, identifier, type_name, grid_name, **kwargs): + def handle_create(self, identifier, type_name, grid_name, replace, **kwargs): """ Handle the creation of a new collection. """ if grid_name: @@ -176,11 +184,18 @@ def handle_create(self, identifier, type_name, grid_name, **kwargs): raise CommandError( "Collection type %r does not exist." % type_name ) - - models.Collection.objects.create( - identifier=identifier, - collection_type=collection_type, grid=grid - ) + if replace: + models.Collection.objects.update_or_create( + identifier=identifier, + defaults={ + 'collection_type':collection_type, + 'grid':grid, + } + ) + else: + models.Collection.objects.create( + identifier=identifier, collection_type=collection_type, grid=grid + ) print('Successfully created collection %r' % identifier) diff --git a/eoxserver/resources/coverages/management/commands/collectiontype.py b/eoxserver/resources/coverages/management/commands/collectiontype.py index ad1941d36..fb93a5c17 100644 --- a/eoxserver/resources/coverages/management/commands/collectiontype.py +++ b/eoxserver/resources/coverages/management/commands/collectiontype.py @@ -66,6 +66,15 @@ def add_arguments(self, parser): ) ) + create_parser.add_argument( + '--replace', action='store_true', + default=False, + help=( + '''Change collection type references according to parameters + if collection type already exists.''' + ) + ) + delete_parser.add_argument( '--force', '-f', action='store_true', default=False, help='Also remove all collections associated with that type.' @@ -88,37 +97,39 @@ def handle(self, subcommand, *args, **kwargs): self.handle_list(*args, **kwargs) def handle_create(self, name, allowed_coverage_type_names, - allowed_product_type_names, **kwargs): + allowed_product_type_names, replace, **kwargs): """ Handle the creation of a new collection type. """ - - collection_type = models.CollectionType.objects.create(name=name) + if replace: + collection_type = models.CollectionType.objects.get_or_create(name=name)[0] + else: + collection_type = models.CollectionType.objects.create(name=name) for allowed_coverage_type_name in allowed_coverage_type_names: - try: - collection_type.allowed_coverage_types.add( - models.CoverageType.objects.get( - name=allowed_coverage_type_name - ) - ) - except models.CoverageType.DoesNotExist: - raise CommandError( - 'Coverage type %r does not exist.' % - allowed_coverage_type_name - ) + if replace: + if not collection_type.allowed_coverage_types.filter(name=allowed_coverage_type_name).exists(): + self.add_allowed_coverage_type_name(collection_type, allowed_coverage_type_name) + else: + self.add_allowed_coverage_type_name(collection_type, allowed_coverage_type_name) + if replace: + # remove allowed coverage types not part of definition of collection type + referenced_coverage_types = collection_type.allowed_coverage_types.all() + for ct in referenced_coverage_types: + if ct.name not in allowed_coverage_type_names: + collection_type.allowed_coverage_types.remove(ct) for allowed_product_type_name in allowed_product_type_names: - try: - collection_type.allowed_product_types.add( - models.ProductType.objects.get( - name=allowed_product_type_name - ) - ) - except models.ProductType.DoesNotExist: - raise CommandError( - 'Product type %r does not exist.' % - allowed_product_type_name - ) + if replace: + if not collection_type.allowed_product_types.filter(name=allowed_product_type_name).exists(): + self.add_allowed_product_type_name(collection_type, allowed_product_type_name) + else: + self.add_allowed_product_type_name(collection_type, allowed_product_type_name) + if replace: + # remove allowed product types not part of definition of collection type + referenced_product_types = collection_type.allowed_product_types.all() + for pt in referenced_product_types: + if pt.name not in allowed_product_type_names: + collection_type.allowed_product_types.remove(pt) print('Successfully created collection type %r' % name) @@ -139,3 +150,30 @@ def handle_list(self, detail, *args, **kwargs): # if detail: # for coverage_type in collection_type.allowed_coverage_types.all(): # print("\t%s" % coverage_type.name) + + def add_allowed_coverage_type_name(self, collection_type, allowed_coverage_type_name): + try: + collection_type.allowed_coverage_types.add( + models.CoverageType.objects.get( + name=allowed_coverage_type_name + ) + ) + except models.CoverageType.DoesNotExist: + raise CommandError( + 'Coverage type %r does not exist.' % + allowed_coverage_type_name + ) + + + def add_allowed_product_type_name(self, collection_type, allowed_product_type_name): + try: + collection_type.allowed_product_types.add( + models.ProductType.objects.get( + name=allowed_product_type_name + ) + ) + except models.CoverageType.DoesNotExist: + raise CommandError( + 'Product type %r does not exist.' % + allowed_product_type_name + ) diff --git a/eoxserver/resources/coverages/management/commands/coveragetype.py b/eoxserver/resources/coverages/management/commands/coveragetype.py index 54f2d52ae..4dfc500a7 100644 --- a/eoxserver/resources/coverages/management/commands/coveragetype.py +++ b/eoxserver/resources/coverages/management/commands/coveragetype.py @@ -78,6 +78,15 @@ def add_arguments(self, parser): help='Read the definition from stdin instead from a file.' ) + for parser in [create_parser, import_parser]: + parser.add_argument( + '--replace', action='store_true', + default=False, + help=( + 'Replace field types if coverage type already exists.' + ) + ) + delete_parser.add_argument( '--force', '-f', action='store_true', default=False, help='Also remove all collections associated with that type.' @@ -103,42 +112,42 @@ def handle(self, subcommand, *args, **kwargs): elif subcommand == "list": self.handle_list(*args, **kwargs) - def handle_create(self, name, field_types, **kwargs): + def handle_create(self, name, field_types, replace, **kwargs): """ Handle the creation of a new coverage type. """ coverage_type = self._create_coverage_type(name) - - self._create_field_types(coverage_type, {}, [ - dict( - identifier=field_type_definition[0], - description=field_type_definition[1], - definition=field_type_definition[2], - unit_of_measure=field_type_definition[3], - wavelength=field_type_definition[4] - ) - for field_type_definition in field_types - ]) + if replace or not models.FieldType.objects.filter(coverage_type=coverage_type).exists(): + self._create_field_types(coverage_type, {}, [ + dict( + identifier=field_type_definition[0], + description=field_type_definition[1], + definition=field_type_definition[2], + unit_of_measure=field_type_definition[3], + wavelength=field_type_definition[4] + ) + for field_type_definition in field_types + ], replace) print('Successfully created coverage type %r' % name) def handle_import(self, locations, *args, **kwargs): - def _import(definitions): + def _import(definitions, replace): if isinstance(definitions, dict): definitions = [definitions] for definition in definitions: - self._import_definition(definition) + self._import_definition(definition, replace) - if kwargs['stdin']: + if kwargs.get('stdin'): try: - _import(json.load(sys.stdin)) + _import(json.load(sys.stdin), kwargs.get('replace')) except ValueError: raise CommandError('Could not parse JSON from stdin') else: for location in locations: with open(location) as f: try: - _import(json.load(f)) + _import(json.load(f), kwargs.get('replace')) except ValueError: raise CommandError( 'Could not parse JSON from %r' % location @@ -183,32 +192,37 @@ def handle_list(self, detail, *args, **kwargs): for coverage_type in coverage_type.field_types.all(): print("\t%s" % coverage_type.identifier) - def _import_definition(self, definition): + def _import_definition(self, definition, replace): name = str(definition['name']) coverage_type = self._create_coverage_type(name) field_type_definitions = ( definition.get('field_type') or definition.get('bands') ) - self._create_field_types( - coverage_type, definition, field_type_definitions - ) - self.print_msg('Successfully imported coverage type %r' % name) + if replace or not models.FieldType.objects.filter(coverage_type=coverage_type).exists(): + self._create_field_types( + coverage_type, definition, field_type_definitions, replace + ) + self.print_msg('Successfully imported coverage type %r' % name) def _create_coverage_type(self, name): - try: - return models.CoverageType.objects.create(name=name) - except IntegrityError: - raise CommandError("Coverage type %r already exists." % name) + return models.CoverageType.objects.get_or_create(name=name)[0] def _create_field_types(self, coverage_type, coverage_type_definition, - field_type_definitions): + field_type_definitions, replace): for i, field_type_definition in enumerate(field_type_definitions): + if i == 0: + # only in first iteration consider replace + if replace: + # delete FieldTypes attached to CoverageType if any exist + field_types = models.FieldType.objects.filter( + coverage_type=coverage_type, + ) + field_types.delete() uom = ( field_type_definition.get('unit_of_measure') or field_type_definition.get('uom') ) - - field_type = models.FieldType( + field_type = models.FieldType.objects.create( coverage_type=coverage_type, index=i, identifier=field_type_definition.get('identifier'), diff --git a/eoxserver/resources/coverages/management/commands/product.py b/eoxserver/resources/coverages/management/commands/product.py index a775a7751..e371762ef 100644 --- a/eoxserver/resources/coverages/management/commands/product.py +++ b/eoxserver/resources/coverages/management/commands/product.py @@ -216,7 +216,7 @@ def add_arguments(self, parser): discover_parser.add_argument( 'identifier', default=None, - help='The identifier of the product to descover.' + help='The identifier of the product to discover.' ) discover_parser.add_argument( diff --git a/eoxserver/resources/coverages/management/commands/producttype.py b/eoxserver/resources/coverages/management/commands/producttype.py index 9ce13614b..f1492b54c 100644 --- a/eoxserver/resources/coverages/management/commands/producttype.py +++ b/eoxserver/resources/coverages/management/commands/producttype.py @@ -73,6 +73,15 @@ def add_arguments(self, parser): ) ) + create_parser.add_argument( + '--replace', action='store_true', + default=False, + help=( + '''Change product type references according to parameters + if product type already exists.''' + ) + ) + delete_parser.add_argument( '--force', '-f', action='store_true', default=False, help='Also remove all products associated with that type.' @@ -83,6 +92,7 @@ def add_arguments(self, parser): help="Disable the printing of details of the product type." ) + @transaction.atomic def handle(self, subcommand, *args, **kwargs): """ Dispatch sub-commands: create, delete, list. @@ -95,13 +105,15 @@ def handle(self, subcommand, *args, **kwargs): self.handle_list(*args, **kwargs) def handle_create(self, name, coverage_type_names, mask_type_names, - validity_mask_type_names, browse_type_names, + validity_mask_type_names, browse_type_names, replace, *args, **kwargs): """ Handle the creation of a new product type. """ - - product_type = models.ProductType.objects.create(name=name) - + product_type = None + if replace: + product_type = models.ProductType.objects.get_or_create(name=name)[0] + else: + product_type = models.ProductType.objects.create(name=name) for coverage_type_name in coverage_type_names: try: coverage_type = models.CoverageType.objects.get( @@ -112,22 +124,46 @@ def handle_create(self, name, coverage_type_names, mask_type_names, raise CommandError( 'Coverage type %r does not exist' % coverage_type_name ) + if replace: + # remove allowed coverage types not part of definition of product type + referenced_coverage_types = product_type.allowed_coverage_types.all() + for ct in referenced_coverage_types: + if ct.name not in coverage_type_names: + product_type.allowed_coverage_types.remove(ct) for mask_type_name in mask_type_names: - models.MaskType.objects.create( - name=mask_type_name, product_type=product_type - ) + if replace: + mt = models.MaskType.objects.get_or_create( + name=mask_type_name, product_type=product_type + )[0] + mt.validity = False + else: + models.MaskType.objects.create( + name=mask_type_name, product_type=product_type, + validity=False + ) for mask_type_name in validity_mask_type_names: - models.MaskType.objects.create( - name=mask_type_name, product_type=product_type, - validity=True - ) + if replace: + mt = models.MaskType.objects.get_or_create( + name=mask_type_name, product_type=product_type + )[0] + mt.validity = True + else: + models.MaskType.objects.create( + name=mask_type_name, product_type=product_type, + validity=True + ) for browse_type_name in browse_type_names: - models.BrowseType.objects.create( - name=browse_type_name, product_type=product_type - ) + if replace: + models.BrowseType.objects.get_or_create( + name=browse_type_name, product_type=product_type + ) + else: + models.BrowseType.objects.create( + name=browse_type_name, product_type=product_type + ) print('Successfully created product type %r' % name) diff --git a/eoxserver/resources/coverages/management/commands/rasterstyle.py b/eoxserver/resources/coverages/management/commands/rasterstyle.py new file mode 100644 index 000000000..5a48d170e --- /dev/null +++ b/eoxserver/resources/coverages/management/commands/rasterstyle.py @@ -0,0 +1,271 @@ +# ------------------------------------------------------------------------------ +# +# Project: EOxServer +# Authors: Fabian Schindler +# +# ------------------------------------------------------------------------------ +# Copyright (C) 2017 EOX IT Services GmbH +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies of this Software or works derived from this Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# ------------------------------------------------------------------------------ + +from django.core.management.base import CommandError, BaseCommand +from django.db import transaction +from lxml import etree + +from eoxserver.resources.coverages import models +from eoxserver.resources.coverages.management.commands import ( + CommandOutputMixIn, SubParserMixIn +) + + +class Command(CommandOutputMixIn, SubParserMixIn, BaseCommand): + """ Command to manage product types. This command uses sub-commands for the + specific tasks: create, delete + """ + def add_arguments(self, parser): + create_parser = self.add_subparser(parser, 'create') + import_parser = self.add_subparser(parser, 'import') + delete_parser = self.add_subparser(parser, 'delete') + list_parser = self.add_subparser(parser, 'list') + link_parser = self.add_subparser(parser, 'link') + + for parser in [create_parser, delete_parser, link_parser]: + parser.add_argument( + 'name', nargs=1, help='The raster style name. Mandatory.' + ) + + create_parser.add_argument( + '--type', '-t', action="store", default="ramp", + choices=["ramp", "values", "intervals"], + help="Specify this raster style type" + ) + create_parser.add_argument( + '--title', action="store", + help="Specify this raster style title" + ) + create_parser.add_argument( + '--abstract', action="store", + help="Specify this raster style abstract" + ) + create_parser.add_argument( + '--color-entry', '-c', + action='append', dest='color_entries', default=[], nargs=4, + help=( + "A color style entry. Must consist of , , " + ",