Refactored Config management

.github/workflows/e2e.yml

@@ -11,18 +11,17 @@ jobs:
     strategy:
       matrix:
         py_version: ['3.7', '3.8', '3.9', '3.10']
-        pg_version: ['10-2.5', '13-3.3', '15-3.3']
 
     # create the postgis service - the version is hardcoded atm
     services:
       postgis:
-        image: postgis/postgis:${{ matrix.pg_version }}
+        image: postgis/postgis:15-3.3
+        ports:
+          - 5432:5432
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: postgres
-        ports:
-          - 5432/tcp
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
     steps:
@@ -38,19 +37,9 @@ jobs:
     # e2e-tests rely on pytest and pytest-depends so far 
     - name: Install dependencies
       run: | 
-        pip install -r requirements.txt
-        python setup.py develop
+        pip install -e .
         pip install pytest pytest-depends pytest-cov
-
-    # activate the IO extension
-    - name: Activate builtin IOExtension
-      run: python -c "from metacatalog import ext; ext.activate_extension('io', 'metacatalog.ext.io', 'IOExtension')"
-
-    - name: Activate builtin StandardsExportExtension, install additional dependencies
-      run: |
-        pip install jinja2 lxml
-        python -c "from metacatalog import ext; ext.activate_extension('standards_export', 'metacatalog.ext.standards_export', 'StandardsExportExtension')"
-
+
     # run the actual tests
     # they need the arbitrary port number which is mapped to 5432
     # in the VM host 
@@ -59,10 +48,10 @@ jobs:
     - name: e2e Tests
       run: pytest --cov-config=.coveragerc --cov=./ --cov-report=xml
       env:
-        POSTGRES_PORT: ${{ job.services.postgis.ports[5432] }}  
+        METACATALOG_URI: postgresql://postgres:postgres@localhost:5432/postgres
 
     - name: Upload coverage to Codecov
-      if: ${{ matrix.py_version }} == '3.9' && ${{ matrix.pg_version == '10.2-5' }}
+      if: ${{ matrix.py_version }} == '3.9'
       uses: codecov/[email protected]
       with:
         file: ./coverage.xml

.github/workflows/pages.yml

@@ -27,10 +27,6 @@ jobs:
         run: |
           pip install metacatalog-corr metacatalog-search
           pip install jinja2 lxml
-          python -c "from metacatalog import ext; ext.activate_extension('io', 'metacatalog.ext.io', 'IOExtension')"
-          python -c "from metacatalog import ext; ext.activate_extension('standards_export', 'metacatalog.ext.standards_export', 'StandardsExportExtension')"
-          python -c "from metacatalog import ext; ext.activate_extension('search', 'metacatalog_search.extension', 'SearchExtension')"
-          python -c "from metacatalog import ext; ext.activate_extension('corr', 'metacatalog_corr.extension', 'CorrExtension')"
 
       - name: Build Sphinx
         run: |

.gitignore

@@ -109,6 +109,7 @@ DBNAME
 tutorials
 Playground.ipynb
 Test.ipynb
+pg_data
 
 # macOS .DS_Store
 .DS_Store

Dockerfile

@@ -0,0 +1,24 @@
+# Use the python image with a build argument for the version
+ARG PYTHON_VERSION=3.11
+FROM python:${PYTHON_VERSION}
+
+# update pip
+RUN pip install --upgrade pip
+
+# install some stuff to make interacting easier
+RUN pip install ipython jupyterlab jupyter
+
+# install dependencies to run tests in the container
+RUN pip install pytest pytest-depends
+
+# Copy the metacatalog package
+RUN mkdir -p /app/metacatalog
+COPY metacatalog /app/metacatalog
+COPY MANIFEST.in /app/MANIFEST.in
+COPY setup.py /app/setup.py
+COPY README.md /app/README.md
+COPY requirements.txt /app/requirements.txt
+
+# Install the metacatalog package
+WORKDIR /app
+RUN pip install -e .

Playground.ipynb

@@ -21,41 +21,42 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 4,
    "id": "493e045e",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "Engine(postgresql://postgres:***@localhost:5432/test_bjnewnuf)"
+       "Engine(postgresql://postgres:***@localhost:5432/metacatalog)"
       ]
      },
-     "execution_count": 2,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "session = api.connect_database(f\"postgresql://postgres:lordblaupause6@localhost:5432/test_bjnewnuf\")\n",
+    "session = api.connect_database(f\"postgresql://postgres:lordblaupause6@localhost:5432/metacatalog\")\n",
     "session.bind"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "id": "f4d1f029",
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "text/plain": [
-       "'b4b6af0bb85b75bbb276db0e4b059d04'"
-      ]
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
+     "ename": "IndexError",
+     "evalue": "list index out of range",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
+      "\u001b[0;32m/tmp/ipykernel_5588/682838852.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mapi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_entry\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtitle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"Dummy*\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mas_result\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mres\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchecksum\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
+     ]
     }
    ],
    "source": [
@@ -4365,7 +4366,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3.9.13 ('base')",
+   "display_name": "base",
    "language": "python",
    "name": "python3"
   },
@@ -4383,7 +4384,7 @@
   },
   "vscode": {
    "interpreter": {
-    "hash": "2c57ebfed52ffd848a0d2f36f1ea9c0a9060c9b67397fbb725d6aa92a9494b08"
+    "hash": "f54d8176e82297fa872ac8c77277e50c0e193f921954c1c4a0b1ae2e8be99b71"
    }
   }
  },

docker-compose.yml

@@ -0,0 +1,24 @@
+version: '3.8'
+
+services:
+  db:
+    image: postgis/postgis:15-3.4
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: metacatalog
+    volumes:
+      - ./pg_data/data:/var/lib/postgresql/data
+    expose:
+      - 5432
+    ports:
+      - 5432:5432
+
+  metacatalog:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    links:
+      - db
+    environment:
+      METACATALOG_URI: postgresql://postgres:postgres@db:5432/metacatalog