Merge pull request #79 from duttonw/develop

.docker/Dockerfile-template.ckan

@@ -1,8 +1,9 @@
-FROM openknowledge/ckan-dev:{CKAN_VERSION}
+FROM ckan/ckan-dev:{CKAN_VERSION}
 
 ARG SITE_URL=http://ckan:5000/
 ENV PYTHON_VERSION={PYTHON_VERSION}
 ENV CKAN_VERSION={CKAN_VERSION}
+ENV SOLR_VERSION={SOLR_VERSION}
 ENV CKAN_SITE_URL="${SITE_URL}"
 ENV PYTHON={PYTHON}
 
@@ -21,6 +22,11 @@ COPY bin/ckan_cli /usr/bin/
 
 RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli
 
+# Install setuptools conditionally
+RUN if [ "$CKAN_VERSION" = "2.9" ]; then \
+        pip install "setuptools>=44.1.0,<71"; \
+    fi
+
 # Init current extension.
 RUN ${APP_DIR}/bin/init-ext.sh
 

.github/workflows/test-direct.yml

@@ -0,0 +1,113 @@
+name: Tests (direct)
+on:
+  push:
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --extend-exclude ckan
+
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        include: #ckan-image see https://github.com/ckan/ckan-docker-base, ckan-version controls other image tags
+          - ckan-version: "2.11"
+            ckan-image: "2.11-py3.10"
+            ckan-solr: "2.11-solr9"
+            experimental: true
+          - ckan-version: "2.10"
+            ckan-image: "2.10-py3.10"
+            ckan-solr: "2.10-solr9"
+            experimental: false
+          - ckan-version: "2.9"
+            ckan-image: "2.9-py3.9"
+            ckan-solr: "2.9-solr8"
+            experimental: false
+#          - ckan-version: "master"
+#            ckan-image: "master"
+#            ckan-solr: "master-solr9"
+#            experimental: true  # master is unstable, good to know if we are compatible or not
+      fail-fast: false
+
+    name: ${{ matrix.experimental && '**Fail_Ignored** ' || '' }} CKAN ${{ matrix.ckan-version }}
+    runs-on: ubuntu-latest
+    container:
+      image: ckan/ckan-dev:${{ matrix.ckan-image }}
+      options: --user root
+    services:
+      solr:
+        image: ckan/ckan-solr:${{ matrix.ckan-solr }}
+      postgres:
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+#        ports:
+#          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+          image: redis:3
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+
+    steps:
+    - uses: actions/checkout@v4
+      continue-on-error: ${{ matrix.experimental }}
+
+    - name: Pin setuptools for ckan 2.9 only
+      if: ${{ matrix.ckan-version == 2.9 }}
+      run: pip install "setuptools>=44.1.0,<71"
+      continue-on-error: ${{ matrix.experimental }}
+
+    - name: Install requirements
+      continue-on-error: ${{ matrix.experimental }}
+      run: |
+        pip install -r dev-requirements.txt
+        pip install -r requirements.txt
+        pip install -e .
+        # Replace default path to CKAN core config file with the one on the container
+        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+
+    - name: Pin jinja2 for ckan 2.9 only (last jinja version that had escape class)
+      if: ${{ matrix.ckan-version == 2.9 }}
+      run: pip install "jinja2<3.1"
+      continue-on-error: ${{ matrix.experimental }}
+
+    - name: Setup extension
+      continue-on-error: ${{ matrix.experimental }}
+      run: |
+        ckan -c test.ini db init
+
+    - name: Run tests
+      continue-on-error: ${{ matrix.experimental }}
+      run: pytest --ckan-ini=test.ini --cov=ckanext.validation --cov-report=xml --cov-append --disable-warnings ckanext/validation/tests -vv  --junit-xml=/tmp/artifacts/junit/results.xml
+
+    - name: Test Summary
+      uses: test-summary/action@v2
+      continue-on-error: ${{ matrix.experimental }}
+      with:
+        paths: "/tmp/artifacts/junit/*.xml"
+      if: always()
+
+    - name: "Upload coverage report to codecov flag: ${{ matrix.ckan-version }}"
+      uses: codecov/codecov-action@v5
+      continue-on-error: ${{ matrix.experimental }}
+      with:
+        file: ./coverage.xml
+        token: ${{ secrets.CODECOV_TOKEN }}
+        flags: "${{ matrix.ckan-version }}"

.github/workflows/test.yml

@@ -30,13 +30,23 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ckan-version: ["2.10", 2.9]
+        include: #ckan-image see https://github.com/ckan/ckan-docker-base, ckan-version controls other image tags
+          - ckan-version: "2.11"
+            solr-version: "9"
+            experimental: true
+          - ckan-version: "2.10"
+            solr-version: "9"
+            experimental: false
+          - ckan-version: "2.9"
+            solr-version: "8"
+            experimental: false
 
-    name: Continuous Integration build on CKAN ${{ matrix.ckan-version }}
+    name: ${{ matrix.experimental && '**Fail_Ignored** ' || '' }} CKAN ${{ matrix.ckan-version }} - Continuous Integration build
     runs-on: ubuntu-latest
     container: drevops/ci-runner:23.12.0
     env:
       CKAN_VERSION: ${{ matrix.ckan-version }}
+      SOLR_VERSION: ${{ matrix.solr-version }}
 
     steps:
       # Patch https://github.com/actions/runner/issues/863
@@ -49,20 +59,24 @@ jobs:
       - name: Build
         run: bin/build.sh
         timeout-minutes: 15
+        continue-on-error: ${{ matrix.experimental }}
 
       - name: Test
         run: bin/test.sh
         timeout-minutes: 20
+        continue-on-error: ${{ matrix.experimental }}
 
       - name: Retrieve logs
         if: always()
         run: ahoy logs
         timeout-minutes: 1
+        continue-on-error: ${{ matrix.experimental }}
 
       - name: Retrieve screenshots
         if: always()
         run: bin/process-artifacts.sh
         timeout-minutes: 1
+        continue-on-error: ${{ matrix.experimental }}
 
       - name: Upload screenshots
         if: failure()
@@ -71,3 +85,4 @@ jobs:
           name: CKAN ${{ matrix.ckan-version }} screenshots
           path: /tmp/artifacts/behave/screenshots
         timeout-minutes: 1
+        continue-on-error: ${{ matrix.experimental }}

README.md

@@ -696,6 +696,10 @@ To run the tests, do:
 
     pytest --ckan-ini=test.ini ckanext/validation/tests/
 
+## CICD Github actions locally
+
+act -W '.github/workflows/test.yml' --matrix ckan-version:2.9 --container-architecture linux/amd64
+
 
 ## Copying and License
 

bin/build.sh

@@ -23,6 +23,7 @@ else
 fi
 
 sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \
+    | sed "s|{SOLR_VERSION}|$SOLR_VERSION|g" \
     | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" \
     | sed "s|{PYTHON}|$PYTHON|g" \
     > .docker/Dockerfile.ckan

bin/docker-compose.sh

@@ -1,14 +1,4 @@
 #!/bin/sh
-
-# Pass commands to Docker Compose v1 or v2 depending on what is present
-
-if (which docker-compose >/dev/null); then
-    # Docker Compose v1
-    docker-compose $*
-elif (docker compose ls >/dev/null); then
-    # Docker Compose v2
-    docker compose $*
-else
-    # Docker Compose not found
-    exit 1
-fi
+export DOCKER_DEFAULT_PLATFORM=linux/amd64
+# Pass commands to Docker Compose v2 depending on what is present
+docker compose $*

ckanext/validation/jobs.py

@@ -115,9 +115,8 @@ def _validate_table(source, _format=u'csv', schema=None, **options):
 
     http_session = options.pop('http_session', None) or requests.Session()
 
-    use_proxy = 'ckan.download_proxy' in t.config
-    if use_proxy:
-        proxy = t.config.get('ckan.download_proxy')
+    proxy = t.config.get('ckan.download_proxy', None)
+    if proxy is not None:
         log.debug(u'Download resource for validation via proxy: %s', proxy)
         http_session.proxies.update({'http': proxy, 'https': proxy})
     report = validate(source, format=_format, schema=schema, http_session=http_session, **options)

ckanext/validation/tests/test_interfaces.py

@@ -45,6 +45,11 @@ def receive_validation_report(self, validation_report):
         self.calls += 1
 
 
+def _reset_plugin_counter():
+    for plugin in p.PluginImplementations(IDataValidation):
+        plugin.reset_counter()
+
+
 def _get_data_plugin_calls():
     for plugin in p.PluginImplementations(IDataValidation):
         return plugin.calls
@@ -71,6 +76,7 @@ class TestInterfaceSync(BaseTestInterfaces):
 
     def test_can_validate_called_on_create_sync(self, mock_validation,
                                                 resource_factory):
+        _reset_plugin_counter()
         """Plugin must be called once for SYNC mode on create
         1. resource before_create
         """
@@ -82,6 +88,7 @@ def test_can_validate_called_on_create_sync(self, mock_validation,
 
     def test_can_validate_called_on_create_sync_no_validation(
             self, mock_validation, resource_factory):
+        _reset_plugin_counter()
         """Plugin must be called once for SYNC mode on create
         1. resource before_create
         """
@@ -93,6 +100,7 @@ def test_can_validate_called_on_create_sync_no_validation(
 
     def test_can_validate_called_on_update_sync(self, mock_validation,
                                                 resource_factory):
+        _reset_plugin_counter()
         """Plugin must be called 2 times for SYNC mode.
         1. resource before_create on resource create
         2. resource before_update on resource update
@@ -113,6 +121,7 @@ def test_can_validate_called_on_update_sync(self, mock_validation,
 
     def test_can_validate_called_on_update_sync_no_validation(
             self, mock_validation, resource_factory):
+        _reset_plugin_counter()
         """Plugin must be called 2 times for SYNC mode.
         1. resource before_create on resource create
         2. resource before_update on resource update
@@ -137,6 +146,7 @@ class TestInterfaceAsync(BaseTestInterfaces):
 
     def test_can_validate_called_on_create_async(self, mock_validation,
                                                  resource_factory):
+        _reset_plugin_counter()
         """Plugin must be called once for ASYNC mode on create
         1. resource after_create
         """
@@ -149,6 +159,7 @@ def test_can_validate_called_on_create_async(self, mock_validation,
 
     def test_can_validate_called_on_create_async_no_validation(
             self, mock_validation, resource_factory):
+        _reset_plugin_counter()
         """Plugin must be called once for ASYNC mode on create
         1. resource after_create
         """
@@ -161,6 +172,7 @@ def test_can_validate_called_on_create_async_no_validation(
 
     def test_can_validate_called_on_update_async(self, mock_validation,
                                                  resource_factory):
+        _reset_plugin_counter()
         """Plugin must be called 3 times for ASYNC mode.
         1. resource after_create on resource create
         2. resource before_update on resource update
@@ -181,6 +193,7 @@ def test_can_validate_called_on_update_async(self, mock_validation,
 
     def test_can_validate_called_on_update_async_no_validation(
             self, mock_validation, resource_factory):
+        _reset_plugin_counter()
         """Plugin must be called 2 times for ASYNC mode.
         1. resource after_create on resource create
         2. resource before_update on resource update

docker-compose.yml

@@ -61,7 +61,7 @@ services:
       - default
 
   solr:
-    image: ckan/ckan-solr:${CKAN_VERSION}
+    image: ckan/ckan-solr:${CKAN_VERSION}-solr${SOLR_VERSION}
     ports:
       - "8983"
     environment:

test.ini

@@ -1,3 +1,16 @@
+#
+# CKAN - Pylons configuration
+#
+# These are some of the configuration options available for your CKAN
+# instance. Check the documentation in 'doc/configuration.rst' or at the
+# following URL for a description of what they do and the full list of
+# available options:
+#
+# http://docs.ckan.org/en/latest/maintaining/configuration.html
+#
+# The %(here)s variable will be replaced with the parent directory of this file
+#
+
 [DEFAULT]
 debug = false
 smtp_server = localhost
@@ -11,13 +24,17 @@ port = 5000
 [app:main]
 use = config:../ckan/test-core.ini
 
+# Disable CSRF on local testing TODO: make it work with this enabled
+WTF_CSRF_ENABLED = False
+
 # Insert any custom config settings to be used when running your extension's
 # tests here.
 ckan.plugins =
     validation
     scheming_datasets
     test_validation_plugin
 
+# ckanext-validation
 ckanext.validation.run_on_update_async = False
 ckanext.validation.run_on_create_async = False
 
@@ -29,7 +46,7 @@ ckan.requests.timeout = 5
 
 # Logging configuration
 [loggers]
-keys = root, ckan, sqlalchemy
+keys = root, ckan, ckanext, sqlalchemy
 
 [handlers]
 keys = console
@@ -42,9 +59,16 @@ level = WARN
 handlers = console
 
 [logger_ckan]
-qualname = ckan
-handlers =
 level = INFO
+handlers = console
+qualname = ckan
+propagate = 0
+
+[logger_ckanext]
+level = DEBUG
+handlers = console
+qualname = ckanext
+propagate = 0
 
 [logger_sqlalchemy]
 handlers =