Refactor: Archive Functionality (#13)

* refactor(config-definition): remove upload from archive Signed-off-by: Tarik Zegmott <[email protected]> * feat(workspace): define examples of the config archive yaml entries Signed-off-by: Tarik Zegmott <[email protected]> * refactor(lifecycle-archive): respect archive methods set as available in workspace refactor methods into separate posix and s3 modules Signed-off-by: Tarik Zegmott <[email protected]> * refactor(lifecycle-archive): add placeholder for http methods Signed-off-by: Tarik Zegmott <[email protected]> * feat(results-transfer-daemon): now respects configuration set in workspace Signed-off-by: Tarik Zegmott <[email protected]> * feat(lifecycle-archive): s3 move and copy functions Signed-off-by: Tarik Zegmott <[email protected]> * test(archive): added WIP * build(pre-commit): autoupdate deps Signed-off-by: Tarik Zegmott <[email protected]> * style(lifecycle-archive): linting Signed-off-by: Tarik Zegmott <[email protected]> * ci(gh-action): trigger ci on pr ready for review Signed-off-by: Tarik Zegmott <[email protected]> * test(archive): additional tests Signed-off-by: Tarik Zegmott <[email protected]> * ci(gh-action): ci on pr sync Signed-off-by: Tarik Zegmott <[email protected]> * feat(utils): new function to add loki handler to logger linting Signed-off-by: Tarik Zegmott <[email protected]> * ci(gh-action): ci python version bumped to 3 10 reason: workflow/definitions/work.py:137: error: X Y syntax for unions requires Python 3.10 Signed-off-by: Tarik Zegmott <[email protected]> * build(pyproject): add logging-loki dep Signed-off-by: Tarik Zegmott <[email protected]> * test(pytest): set up requisit initial config * test(archive): additional tests and code refactor * feat(archive): read permission command user and group from workspace * docs(workspaces): added info on archive configuration Signed-off-by: Tarik Zegmott <[email protected]> --------- Signed-off-by: Tarik Zegmott <[email protected]> Co-authored-by: Shiny [ˈʃaɪni] <[email protected]>
CHIMEFRB · Jan 28, 2024 · 319e644 · 319e644
1 parent e5a2dd0
commit 319e644
Show file tree

Hide file tree

Showing 20 changed files with 2,149 additions and 1,040 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -5,18 +5,20 @@ on:
     branches:
       - main
   pull_request:
-     branches:
-       - "**"
-     paths-ignore:
-       - "docs/**"
-     types: [ready_for_review]
+    branches:
+      - "**"
+    paths-ignore:
+      - "docs/**"
+    types:
+      - ready_for_review
+      - synchronize
 
 jobs:
   pre-commit:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.9']
+        python-version: ["3.10"]
     runs-on: ${{ matrix.os }}
     steps:
     -
@@ -34,7 +36,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.9']
+        python-version: ["3.10"]
     runs-on: ${{ matrix.os }}
     needs: pre-commit
     steps:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,98 +1,99 @@
 repos:
-- hooks:
-  - args:
-    - --multi-line=3
-    - --trailing-comma
-    - --force-grid-wrap=0
-    - --use-parentheses
-    - --line-width=88
-    - --ensure-newline-before-comments
-    id: isort
-  repo: https://github.com/pycqa/isort
-  rev: 5.12.0
-- hooks:
-  - args:
-    - --max-line-length=89
-    - --max-complexity=15
-    id: flake8
-  repo: https://github.com/pycqa/flake8
-  rev: 6.1.0
-- hooks:
-  - args:
-    - --py36-plus
-    id: pyupgrade
-  repo: https://github.com/asottile/pyupgrade
-  rev: v3.13.0
-- hooks:
-  - id: black
-  repo: https://github.com/psf/black
-  rev: 23.9.1
-- hooks:
-  - additional_dependencies:
-    - types-attrs
-    - types-requests
-    - types-setuptools
-    - types-PyYAML
-    - types-toml
-    args:
-    - --ignore-missing-imports
-    - --no-implicit-optional
-    id: mypy
-  repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.5.1
-- hooks:
-  - args:
-    - --convention=google
-    - --add-ignore=D104
-    id: pydocstyle
-  repo: https://github.com/pycqa/pydocstyle
-  rev: 6.3.0
-- hooks:
-  - id: blacken-docs
-  repo: https://github.com/asottile/blacken-docs
-  rev: 1.16.0
-- hooks:
-  - id: trailing-whitespace
-  - id: end-of-file-fixer
-  - id: debug-statements
-  - id: check-case-conflict
-  - id: check-json
-  - id: check-added-large-files
-  - id: check-yaml
-  - id: mixed-line-ending
-  - id: check-toml
-  - id: pretty-format-json
-  - id: check-docstring-first
-  - id: check-symlinks
-  - id: detect-private-key
-  - id: pretty-format-json
-    args:
-      - --autofix
-  repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
-- hooks:
-  - args:
-    - -iii
-    - -lll
-    id: bandit
-  repo: https://github.com/PyCQA/bandit
-  rev: 1.7.5
-- hooks:
-  - additional_dependencies:
-    - radon
-    args:
-    - cc
-    - --min=C
-    - --total-average
-    entry: radon
-    id: radon
-    language: python
-    name: radon
-    verbose: true
-  repo: local
-- hooks:
-  - id: commitizen
-    stages:
-    - commit-msg
-  repo: https://github.com/commitizen-tools/commitizen
-  rev: 3.10.0
+  - hooks:
+      - args:
+          - --multi-line=3
+          - --trailing-comma
+          - --force-grid-wrap=0
+          - --use-parentheses
+          - --line-width=88
+          - --ensure-newline-before-comments
+        id: isort
+    repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+  - hooks:
+      - args:
+          - --max-line-length=89
+          - --max-complexity=15
+          - --extend-ignore=E203
+        id: flake8
+    repo: https://github.com/pycqa/flake8
+    rev: 7.0.0
+  - hooks:
+      - args:
+          - --py36-plus
+        id: pyupgrade
+    repo: https://github.com/asottile/pyupgrade
+    rev: v3.15.0
+  - hooks:
+      - id: black
+    repo: https://github.com/psf/black
+    rev: 23.9.1
+  - hooks:
+      - additional_dependencies:
+          - types-attrs
+          - types-requests
+          - types-setuptools
+          - types-PyYAML
+          - types-toml
+        args:
+          - --ignore-missing-imports
+          - --no-implicit-optional
+        id: mypy
+    repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.8.0
+  - hooks:
+      - args:
+          - --convention=google
+          - --add-ignore=D104
+        id: pydocstyle
+    repo: https://github.com/pycqa/pydocstyle
+    rev: 6.3.0
+  - hooks:
+      - id: blacken-docs
+    repo: https://github.com/asottile/blacken-docs
+    rev: 1.16.0
+  - hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: debug-statements
+      - id: check-case-conflict
+      - id: check-json
+      - id: check-added-large-files
+      - id: check-yaml
+      - id: mixed-line-ending
+      - id: check-toml
+      - id: pretty-format-json
+      - id: check-docstring-first
+      - id: check-symlinks
+      - id: detect-private-key
+      - id: pretty-format-json
+        args:
+          - --autofix
+    repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+  - hooks:
+      - args:
+          - -iii
+          - -lll
+        id: bandit
+    repo: https://github.com/PyCQA/bandit
+    rev: 1.7.6
+  - hooks:
+      - additional_dependencies:
+          - radon
+        args:
+          - cc
+          - --min=C
+          - --total-average
+        entry: radon
+        id: radon
+        language: python
+        name: radon
+        verbose: true
+    repo: local
+  - hooks:
+      - id: commitizen
+        stages:
+          - commit-msg
+    repo: https://github.com/commitizen-tools/commitizen
+    rev: v3.13.0