From 274dce59811647025b7b392981219927f9e98801 Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Fri, 13 Dec 2024 09:08:39 +0100 Subject: [PATCH] Try to fix API and integration workflows --- .../workflows/api-and-integration-tests.yml | 8 ++--- .github/workflows/api-documentation.yml | 13 ++------- setup.cfg | 2 +- tests/conftest.py | 4 +-- tools/api/generate-openapi.py | 29 +++++++++++-------- util/user.py | 2 +- 6 files changed, 27 insertions(+), 31 deletions(-) diff --git a/.github/workflows/api-and-integration-tests.yml b/.github/workflows/api-and-integration-tests.yml index 37044bf0d..462583f92 100644 --- a/.github/workflows/api-and-integration-tests.yml +++ b/.github/workflows/api-and-integration-tests.yml @@ -20,7 +20,6 @@ jobs: ) }} steps: - - uses: actions/checkout@v4 - name: Set up Python @@ -76,8 +75,8 @@ jobs: run: | cd yoda/docker/compose docker exec provider.yoda sh -c 'set -x ; cd /etc/irods/yoda-ruleset && sudo chown irods:irods -R /etc/irods/yoda-ruleset && sudo -u irods git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && sudo -u irods git pull && sudo -u irods git status' - docker exec provider.yoda sh -c "set -x ; cd /etc/irods/yoda-ruleset && sudo -u irods git checkout ${{ steps.extract_branch.outputs.branch }} && sudo -u irods python -m pip --no-cache-dir install --user -r /etc/irods/yoda-ruleset/requirements.txt && sudo -u irods make && sudo -u irods make install" - docker exec provider.yoda sh -c "set -x ; sudo -u irods /var/lib/irods/irodsctl restart" + docker exec provider.yoda sh -c "set -x ; cd /etc/irods/yoda-ruleset && sudo -u irods git checkout ${{ steps.extract_branch.outputs.branch }} && sudo -u irods python3 -m pip --no-cache-dir install --user --break-system-packages -r /etc/irods/yoda-ruleset/requirements.txt && sudo -u irods make && sudo -u irods make install" + docker exec provider.yoda sh -c 'set -x ; sudo -u irods /var/lib/irods/irodsctl restart' - name: Pull and install latest version of portal shell: bash @@ -122,8 +121,7 @@ jobs: - name: Output rodsLogs if: failure() run: | - docker exec provider.yoda sh -c 'set -x ; cat /var/lib/irods/log/rodsLog*' - + docker exec provider.yoda sh -c 'set -x ; tail -f /var/log/syslog' - name: Output web server logs if: failure() diff --git a/.github/workflows/api-documentation.yml b/.github/workflows/api-documentation.yml index 80a3c1e3a..8b7c62fed 100644 --- a/.github/workflows/api-documentation.yml +++ b/.github/workflows/api-documentation.yml @@ -14,21 +14,14 @@ on: jobs: build: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - python-version: [2.7] + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Set up Python - # setup-python stopped supporting Python 2.7, use https://github.com/MatteoH2O1999/setup-python - uses: MatteoH2O1999/setup-python@v3.2.1 + uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - allow-build: info - cache-build: true + python-version: 3.12 - name: Install dependencies run: | diff --git a/setup.cfg b/setup.cfg index 6c349c1fd..880b648c2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ import-order-style=smarkets strictness=short docstring_style=sphinx max-line-length=127 -exclude=__init__.py,tools,tests +exclude=__init__.py,tools,tests/env/ application-import-names=avu,conftest,util,api,config,constants,data_access_token,datacite,datarequest,data_object,epic,error,folder,groups,groups_import,json_datacite,json_landing_page,jsonutil,log,mail,meta,meta_form,msi,notifications,schema,schema_transformation,schema_transformations,settings,pathutil,provenance,policies_intake,policies_datamanager,policies_datapackage_status,policies_folder_status,policies_datarequest_status,publication,query,replication,revisions,revision_strategies,revision_utils,rule,user,vault,sram,arb_data_manager,cached_data_manager,resource,yoda_names,policies_utils [mypy] diff --git a/tests/conftest.py b/tests/conftest.py index f79a870f6..364153e10 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -197,7 +197,7 @@ def login(user, password): p = re.compile("tokenValue: '([a-zA-Z0-9._-]*)'") found_csrf_tokens = p.findall(content) if len(found_csrf_tokens) == 0: - print(f"Error: could not find login CSRF token in response from server for login of user {user}. Response was:") + print(f"Error: could not find login CSRF token in response from server for login of user {user}. Response was: ") print(content) sys.exit(1) csrf = found_csrf_tokens[0] @@ -215,7 +215,7 @@ def login(user, password): p = re.compile("tokenValue: '([a-zA-Z0-9._-]*)'") found_csrf_tokens = p.findall(content) if len(found_csrf_tokens) == 0: - print(f"Error: could not find authenticated CSRF token in response from server for login of user {user}. Response was:") + print(f"Error: could not find authenticated CSRF token in response from server for login of user {user}. Response was: ") print(content) sys.exit(1) csrf = found_csrf_tokens[0] diff --git a/tools/api/generate-openapi.py b/tools/api/generate-openapi.py index b922b86e1..fe194ac85 100755 --- a/tools/api/generate-openapi.py +++ b/tools/api/generate-openapi.py @@ -12,24 +12,21 @@ This module imports (and therefore executes) ruleset code. Do not run it on untrusted codebases. """ -from __future__ import print_function __copyright__ = 'Copyright (c) 2020-2024, Utrecht University' __license__ = 'GPLv3, see LICENSE' -__author__ = ('Chris Smeele') -__author__ = ('Lazlo Westerhof') +__author__ = ('Chris Smeele') +__author__ = ('Lazlo Westerhof') # (in alphabetical order) -import sys -import re +import argparse import inspect import json - -from importlib import import_module +import re +import sys from collections import OrderedDict - -import argparse +from importlib import import_module parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('ruleset', metavar='RULESET', type=str, @@ -44,6 +41,7 @@ core = args.core module = args.module + # Strategy: Import the requested ruleset with an instrumented environment, and # apply introspection to extract API function information. @@ -55,9 +53,15 @@ class Sandbag(object): Used as a stub for various internal irods modules so that we can import rulesets without errors. """ - def __init__(self, *_, **kw): self._data = kw - def __call__(self, *_, **__): return Sandbag() - def __getattr__(self, k): return self._data.get(k, Sandbag()) + def __init__(self, *_, **kw): + self._data = kw + + def __call__(self, *_, **__): + return Sandbag() + + def __getattr__(self, k): + return self._data.get(k, Sandbag()) + def __setattr__(self, k, v): if k == '_data': return super(Sandbag, self).__setattr__(k, v) @@ -80,6 +84,7 @@ def f(g): return g return f + # Inject iRODS modules. sys.modules['irods_types'] = Sandbag() sys.modules['genquery'] = Sandbag() diff --git a/util/user.py b/util/user.py index 6ca228c3f..a52d0838b 100644 --- a/util/user.py +++ b/util/user.py @@ -123,7 +123,7 @@ def number_of_connections(ctx: 'rule.Context') -> int: try: # We don't use the -a option with the ips command, because this takes # significantly more time, which would significantly reduce performance. - ips = subprocess.check_output(["ips"]) + ips = (subprocess.check_output(["ips"])).decode("utf-8") username = session_vars.get_map(ctx.rei)['client_user']['user_name'] connections = ips.count(username) except Exception as e: