feat: improve the discovery of projects at sites (#278)

* feat: improve the discovery of projects at sites

Moved everything into a python module and include testing for it,
hopefully will allow us to detect issues before deploymentt

* Linting

* Black

* More linting fixes

.github/workflows/python.yml

@@ -0,0 +1,19 @@
+name: Python testing
+
+on: pull_request
+
+jobs:
+  test:
+    name: test python code
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 
+      uses: actions/setup-python@v2
+    - name: Test cloud-info-generator 
+      run: |
+        cd cloud-info
+        pip install -r requirements.txt
+        pip install -e .
+        python3 -m cloud_info_catchall.test 

.gitignore

@@ -1,3 +1,4 @@
 # keep secrets out
 secrets.yaml
 .venv
+__pycache__

cloud-info/Dockerfile

@@ -2,18 +2,12 @@ FROM python:3
 
 LABEL org.opencontainers.image.source=https://github.com/EGI-Federation/fedcloud-catchall-operations
 
-ARG CLOUD_INFO_VERSION=f6f6a2e265cc9608d791f31a8ef2903302ca33f6
-
-# hadolint ignore=DL3013
-RUN pip install --no-cache-dir \
-        "git+https://github.com/EGI-Federation/cloud-info-provider.git@$CLOUD_INFO_VERSION" \
-        "git+https://github.com/ARGOeu/argo-ams-library@devel" \
-        python-glanceclient python-novaclient python-keystoneclient keystoneauth1 yq \
-	fedcloudclient
-
-
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
+RUN mkdir /cloud-info
+COPY requirements.txt /cloud-info/requirements.txt
+RUN pip install --no-cache-dir -r /cloud-info/requirements.txt
+
 # CA certificates: install and add to python
 # hadolint ignore=DL3015, DL3008
 RUN curl -Ls \
@@ -27,10 +21,13 @@ RUN curl -Ls \
     && rm -rf /var/lib/apt/lists/* \
     && cat /etc/grid-security/certificates/*.pem >> "$(python -m requests.certs)"
 
+
+COPY . /cloud-info/
+RUN pip install --no-cache-dir /cloud-info
+
 COPY ams-wrapper.sh /usr/local/bin/ams-wrapper.sh
 COPY publisher.sh /usr/local/bin/publisher.sh
 COPY openstack.rc /etc/cloud-info-provider/openstack.rc
 COPY openstack.yaml /etc/cloud-info-provider/openstack.yaml
-COPY config_generator.py /usr/local/bin/config_generator.py
 
 CMD ["publisher.sh"]

cloud-info/ams-wrapper.sh

@@ -35,7 +35,7 @@ curl -f "https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC?key=$AMS_T
 # Attempt to generate the site configuration
 AUTO_CONFIG_PATH="$(mktemp -d)"
 export CHECKIN_SECRETS_FILE="$CHECKIN_SECRETS_PATH/secrets.yaml"
-if VO_SECRETS_PATH="$AUTO_CONFIG_PATH/vos" config_generator.py > "$AUTO_CONFIG_PATH/site.yaml"; then
+if VO_SECRETS_PATH="$AUTO_CONFIG_PATH/vos" config-generator > "$AUTO_CONFIG_PATH/site.yaml"; then
     # this worked, let's update the env
     export CHECKIN_SECRETS_PATH="$AUTO_CONFIG_PATH/vos"
     export CLOUD_INFO_CONFIG="$AUTO_CONFIG_PATH/site.yaml"

cloud-info/cloud_info_catchall/config_generator.py

@@ -0,0 +1,116 @@
+"""Discover projects for cloud-info-povider and generate configuration
+
+Takes its own configuration from env variables:
+CHECKIN_SECRETS_FILE: yaml file with the check-in secrets to get access tokens
+CHECKIN_OIDC_TOKEN: URL for token refreshal
+OS_AUTH_URL, OS_IDENTITY_PROVIDER, OS_PROTOCOL: OpenStack endpoint config
+SITE_NAME: site name
+"""
+
+import logging
+import os
+
+import fedcloudclient.endpoint as fedcli
+import yaml
+from cloud_info_provider.auth_refreshers.oidc_refresh import OidcRefreshToken
+
+
+class ShareDiscovery:
+    def __init__(self, auth_url, identity_provider, protocol, token_url, vo_dir):
+        self.auth_url = auth_url
+        self.identity_provider = identity_provider
+        self.protocol = protocol
+        self.token_url = token_url
+        self.vo_dir = vo_dir
+
+    def refresh_token(self, secret):
+        # fake the options for refreshing
+        # avoids code duplication but not very clean
+        class Opt:
+            timeout = 10
+
+        refresher = OidcRefreshToken(Opt)
+        return refresher._refresh_token(
+            self.token_url,
+            secret.get("client_id", None),
+            secret.get("client_secret", None),
+            secret.get("refresh_token", None),
+            "openid email profile voperson_id eduperson_entitlement",
+        )
+
+    def get_token_shares(self, access_token):
+        # rely on fedcloudclient for getting token
+        # exchange access_token for Keystone token
+        shares = {}
+        try:
+            token = fedcli.retrieve_unscoped_token(
+                self.auth_url, access_token, self.protocol
+            )
+        except fedcli.TokenException:
+            # this check-in account does not have access to the site, ignore
+            return shares
+        projects = fedcli.get_projects_from_single_site(self.auth_url, token)
+        for p in projects:
+            vo = p.get("VO", None)
+            if not vo:
+                logging.warning(
+                    "Discarding project %s as it does not have VO property", p["name"]
+                )
+                continue
+            if not p.get("enabled", False):
+                logging.warning("Discarding project %s as it is not enabled", p["name"])
+                continue
+            shares[vo] = {"auth": {"project_id": p["id"]}}
+        return shares
+
+    def generate_shares(self, secrets):
+        shares = {}
+        for s in secrets:
+            # not our thing
+            if not isinstance(secrets[s], dict):
+                continue
+            access_token = self.refresh_token(secrets[s])
+            token_shares = self.get_token_shares(access_token)
+            shares.update(token_shares)
+            # create the directory structure for the cloud-info-provider
+            for d in token_shares:
+                dir_path = os.path.join(self.vo_dir, d)
+                os.makedirs(dir_path, exist_ok=True)
+                for field in "client_id", "client_secret", "refresh_token":
+                    with open(os.path.join(dir_path, field), "w+") as f:
+                        f.write(secrets[s].get(field, None) or "")
+        if not shares:
+            logging.error("No shares generated!")
+            raise Exception("No shares found!")
+        return shares
+
+    def generate_config(self, site_name, secrets):
+        shares = self.generate_shares(secrets)
+        return {"site": {"name": site_name}, "compute": {"shares": shares}}
+
+
+def read_secrets(secrets_file):
+    with open(secrets_file, "r") as f:
+        return yaml.load(f.read(), Loader=yaml.SafeLoader)
+
+
+def main():
+    logging.basicConfig()
+    # get config from env
+    checkin_secrets_file = os.environ["CHECKIN_SECRETS_FILE"]
+    checkin_token_url = os.environ["CHECKIN_OIDC_TOKEN"]
+    os_auth_url = os.environ["OS_AUTH_URL"]
+    os_identity_provider = os.environ["OS_IDENTITY_PROVIDER"]
+    os_protocol = os.environ["OS_PROTOCOL"]
+    site_name = os.environ["SITE_NAME"]
+    vo_dir = os.environ["VO_SECRETS_PATH"]
+    secrets = read_secrets(checkin_secrets_file)
+    disc = ShareDiscovery(
+        os_auth_url, os_identity_provider, os_protocol, checkin_token_url, vo_dir
+    )
+    config = disc.generate_config(site_name, secrets)
+    print(yaml.dump(config))
+
+
+if __name__ == "__main__":
+    main()

cloud-info/cloud_info_catchall/test.py

@@ -0,0 +1,124 @@
+""" Tests for the config generator """
+
+import unittest
+from unittest.mock import call, mock_open, patch
+
+from cloud_info_catchall.config_generator import ShareDiscovery
+from fedcloudclient.endpoint import TokenException
+
+
+class TestConfig(unittest.TestCase):
+    @patch(
+        "cloud_info_provider.auth_refreshers.oidc_refresh.OidcRefreshToken._refresh_token"
+    )
+    def test_token_refresh(self, m):
+        d = ShareDiscovery(
+            "https://openstack.org", "egi.eu", "oidc", "https://aai.egi.eu", "vo"
+        )
+        t = d.refresh_token(
+            {"client_id": "id", "client_secret": "secret", "refresh_token": "token"}
+        )
+        m.assert_called_with(
+            "https://aai.egi.eu",
+            "id",
+            "secret",
+            "token",
+            "openid email profile voperson_id eduperson_entitlement",
+        )
+        self.assertEqual(t, m.return_value)
+
+    @patch("fedcloudclient.endpoint.retrieve_unscoped_token")
+    def test_failed_token_shares(self, m):
+        d = ShareDiscovery(
+            "https://openstack.org", "egi.eu", "oidc", "https://aai.egi.eu", "vo"
+        )
+        m.side_effect = TokenException()
+        s = d.get_token_shares("foobar")
+        m.assert_called_with("https://openstack.org", "foobar", "oidc")
+        self.assertEqual(s, {})
+
+    @patch("fedcloudclient.endpoint.get_projects_from_single_site")
+    @patch("fedcloudclient.endpoint.retrieve_unscoped_token")
+    def test_token_shares(self, m_token, m_proj):
+        d = ShareDiscovery(
+            "https://openstack.org", "egi.eu", "oidc", "https://aai.egi.eu", "vo"
+        )
+        m_proj.return_value = [
+            {
+                "VO": "foobar.eu",
+                "id": "id1",
+                "name": "enabled foobar VO",
+                "enabled": True,
+            },
+            {"VO": "disabled.eu", "id": "id2", "name": "disabled VO", "enabled": False},
+            {"id": "id3", "name": "not VO project", "enabled": True},
+        ]
+        s = d.get_token_shares("foobar")
+        m_token.assert_called_with("https://openstack.org", "foobar", "oidc")
+        m_proj.assert_called_with("https://openstack.org", m_token.return_value)
+        # return only the enabled with VO
+        self.assertEqual(s, {"foobar.eu": {"auth": {"project_id": "id1"}}})
+
+    @patch.object(ShareDiscovery, "refresh_token")
+    @patch.object(ShareDiscovery, "get_token_shares")
+    @patch("os.makedirs")
+    def test_generate_shares(self, m_makedirs, m_shares, m_refresh):
+        d = ShareDiscovery(
+            "https://openstack.org", "egi.eu", "oidc", "https://aai.egi.eu", "vo"
+        )
+        vos = {
+            "foobar.eu": {
+                "client_id": "bar",
+                "client_secret": "foo",
+                "refresh_token": "foobar",
+            },
+            "baz.eu": {
+                "client_id": "barz",
+                "refresh_token": "foobarz",
+            },
+        }
+        m_shares.side_effect = [
+            {"foobar.eu": {"auth": {"project_id": "id1"}}},
+            {"baz.eu": {"auth": {"project_id": "id2"}}},
+        ]
+        with patch("builtins.open", mock_open()) as m_file:
+            s = d.generate_shares({"s1": vos["foobar.eu"], "s2": vos["baz.eu"]})
+        handle = m_file()
+        for vo in vos:
+            for field in vos[vo]:
+                m_file.assert_any_call(f"vo/{vo}/{field}", "w+"),
+                handle.write.assert_any_call(vos[vo][field])
+        m_refresh.assert_has_calls([call(vos["foobar.eu"]), call(vos["baz.eu"])])
+        m_shares.assert_called_with(m_refresh.return_value)
+        m_makedirs.assert_has_calls(
+            [call("vo/foobar.eu", exist_ok=True), call("vo/baz.eu", exist_ok=True)]
+        )
+        self.assertEqual(
+            s,
+            {
+                "foobar.eu": {"auth": {"project_id": "id1"}},
+                "baz.eu": {"auth": {"project_id": "id2"}},
+            },
+        )
+
+    def test_generate_empty_shares(self):
+        d = ShareDiscovery(
+            "https://openstack.org", "egi.eu", "oidc", "https://aai.egi.eu", "vo"
+        )
+        with self.assertRaises(Exception):
+            d.generate_shares({})
+
+    @patch.object(ShareDiscovery, "generate_shares")
+    def test_generate_config(self, m):
+        d = ShareDiscovery(
+            "https://openstack.org", "egi.eu", "oidc", "https://aai.egi.eu", "vo"
+        )
+        s = d.generate_config("site", {})
+        m.assert_called_with({})
+        self.assertEqual(
+            s, {"site": {"name": "site"}, "compute": {"shares": m.return_value}}
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()