From e8301f9c81ff838b11936ead15629b77f8e2c24d Mon Sep 17 00:00:00 2001 From: "Meinel, Michael" Date: Tue, 2 Apr 2024 14:50:12 +0200 Subject: [PATCH 01/33] Clean up and fix logging --- src/hermes/commands/cli.py | 3 +++ src/hermes/logger.py | 16 +++++----------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/hermes/commands/cli.py b/src/hermes/commands/cli.py index e13cfcf0..5ca827de 100644 --- a/src/hermes/commands/cli.py +++ b/src/hermes/commands/cli.py @@ -10,6 +10,7 @@ """ import argparse +from hermes import logger from hermes.commands import HermesHelpCommand, HermesCleanCommand, HermesHarvestCommand, HermesProcessCommand, \ HermesCurateCommand, HermesDepositCommand, HermesPostprocessCommand from hermes.commands.base import HermesCommand @@ -56,6 +57,8 @@ def main() -> None: # Actually parse the command line, configure it and execute the selected sub-command. args = parser.parse_args() + logger.init_logging() + args.command.load_settings(args) args.command.patch_settings(args) args.command(args) diff --git a/src/hermes/logger.py b/src/hermes/logger.py index cd8b28bc..7b6dd981 100644 --- a/src/hermes/logger.py +++ b/src/hermes/logger.py @@ -31,7 +31,7 @@ 'class': "logging.FileHandler", 'formatter': "logfile", 'level': "DEBUG", - 'filename': "./.hermes/hermes.log", + 'filename': "./hermes.log", }, 'auditfile': { @@ -50,12 +50,6 @@ }, } -# This dict caches all the different configuration sections already loaded -_config = { - # We need some basic logging configuration to get logging up and running at all - 'logging': _logging_config, -} - _loggers = {} @@ -64,14 +58,14 @@ def init_logging(): return # Make sure the directories to hold the log files exists (or else create) - pathlib.Path(_config['logging']['handlers']['logfile']['filename']).parent.mkdir(exist_ok=True, parents=True) - pathlib.Path(_config['logging']['handlers']['auditfile']['filename']).parent.mkdir(exist_ok=True, parents=True) + pathlib.Path(_logging_config['handlers']['logfile']['filename']).parent.mkdir(exist_ok=True, parents=True) + pathlib.Path(_logging_config['handlers']['auditfile']['filename']).parent.mkdir(exist_ok=True, parents=True) # Inintialize logging system import logging.config - logging.config.dictConfig(_config['logging']) - for log_name in _config['logging']['loggers']: + logging.config.dictConfig(_logging_config) + for log_name in _logging_config['loggers']: _loggers[log_name] = logging.getLogger(log_name) From fb918f83a57107ce282735dff8f59a8f857a471d Mon Sep 17 00:00:00 2001 From: "Meinel, Michael" Date: Tue, 2 Apr 2024 14:53:17 +0200 Subject: [PATCH 02/33] Remove debug print statements --- src/hermes/commands/deposit/base.py | 1 - src/hermes/commands/harvest/base.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/hermes/commands/deposit/base.py b/src/hermes/commands/deposit/base.py index e92249ad..5f0290c2 100644 --- a/src/hermes/commands/deposit/base.py +++ b/src/hermes/commands/deposit/base.py @@ -120,7 +120,6 @@ def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None: def __call__(self, args: argparse.Namespace) -> None: self.args = args plugin_name = self.settings.target - print(self.args) ctx = CodeMetaContext() codemeta_file = ctx.get_cache("curate", ctx.hermes_name) diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index dee745c5..629342e7 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -48,10 +48,8 @@ def __call__(self, args: argparse.Namespace) -> None: try: plugin_func = self.plugins[plugin_name]() harvested_data, tags = plugin_func(self) - print(harvested_data) - with HermesHarvestContext( - ctx, plugin_name - ) as harvest_ctx: + + with HermesHarvestContext(ctx, plugin_name) as harvest_ctx: harvest_ctx.update_from(harvested_data, plugin=plugin_name, timestamp=datetime.now().isoformat(), **tags) From 7a4aa40de40c62d644753b28a82636dac2ddba08 Mon Sep 17 00:00:00 2001 From: "Meinel, Michael" Date: Thu, 4 Apr 2024 10:34:15 +0200 Subject: [PATCH 03/33] Start re-activating and adding some tests --- test/hermes_test/conftest.py | 2 ++ test/hermes_test/test_cli.py | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/test/hermes_test/conftest.py b/test/hermes_test/conftest.py index 16f3b7a0..2d3e52b2 100644 --- a/test/hermes_test/conftest.py +++ b/test/hermes_test/conftest.py @@ -23,8 +23,10 @@ def __setitem__(self, path, data): def __enter__(self): self.test_path.mkdir(parents=True, exist_ok=True) + for file_name, data in self.test_files.items(): file_path = self.test_path / file_name + file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_text(data) os.chdir(self.test_path) diff --git a/test/hermes_test/test_cli.py b/test/hermes_test/test_cli.py index 9261c391..85b40e5e 100644 --- a/test/hermes_test/test_cli.py +++ b/test/hermes_test/test_cli.py @@ -9,23 +9,27 @@ from hermes.commands import cli -def test_hermes_full(capsys): +def test_hermes_full(): with pytest.raises(SystemExit) as se: cli.main() assert "choose from" in se -@pytest.mark.skip(reason="Needs update") def test_hermes_harvest(hermes_env): + hermes_env['hermes.toml'] = "" + with hermes_env: result = hermes_env.run("harvest") assert result.returncode == 0 -@pytest.mark.skip(reason="Needs update") def test_hermes_process(hermes_env): + hermes_env['hermes.toml'] = "" + hermes_env['.hermes/harvest/test.json'] = "" + with hermes_env: result = hermes_env.run("process") + print(result.stdout.read()) assert result.returncode == 0 From 991c16c073594e00edcd7b014eaa8554b8e3ea8e Mon Sep 17 00:00:00 2001 From: SKernchen Date: Mon, 6 May 2024 08:54:18 +0200 Subject: [PATCH 04/33] Start of tutorial for writing a plugin --- .../tutorials/writing-a-plugin-for-hermes.md | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 docs/source/tutorials/writing-a-plugin-for-hermes.md diff --git a/docs/source/tutorials/writing-a-plugin-for-hermes.md b/docs/source/tutorials/writing-a-plugin-for-hermes.md new file mode 100644 index 00000000..8c20d7dd --- /dev/null +++ b/docs/source/tutorials/writing-a-plugin-for-hermes.md @@ -0,0 +1,105 @@ + + + + +# Write a plugin for HERMES + + +This tutorial will present the basic steps for writing an additional harvester. +At the moment only the harvest architecture is stable. +The full code and structure is available at [harvest-git](https://github.com/hermes-hmc/hermes-git). +This plugin extracts information from the local git history. +The harvest-git plugin will help to gather contributing and branch metadata. +```{note} +For this tutorial you should be familiar with HERMES. +If you never used HERMES before, you might want to check the tutorial: [Automated Publication with HERMES](https://docs.software-metadata.pub/en/latest/tutorials/automated-publication-with-ci.html). +``` + +## Plugin Architecture + +HERMES uses a plugin architecture. Therefore, users are invited to contribute own features. +The structure for every plugin follows the same schema. +There is a base class for every plugin. In this HermesPlugin class there is one abstract method __ call __ which needs to be overwritten. +Furthermore, the HermesCommand class provides all needs for writing a plugin used in a HERMES command. +So the HermesPlugins call method uses an Instance of the HermesCommand that triggered this plugin to run. +In our case this will be the HermesHarvestCommand which calls all harvest plugins. +The Plugin class also uses a derivative of HermesSettings to add parameters. +HermesSettings are the base class for command specific settings. +It uses pydantic settings to specify and validate the parameters. +The user can either set the parameters in the hermes.toml or overwrite them in the command line. +To overwrite the configuration, you use the -O operator with the dotted parameter name and the value. + +## Set Up Plugin +To write a new plugin, it is important to follow the given structure. +This means your plugins source code has a pydantic class with Settings and the plugin class which inherits from one base class. +For our specific case, we want to write a git harvest plugin. +Our class Structure should look like this: + + +```{code-block} python +from hermes.commands.harvest.base import HermesHarvestPlugin +from pydantic import BaseModel + + +class GitHarvestSettings(BaseModel): + from_branch: str = 'main' + + +class GitHarvestPlugin(HermesHarvestPlugin): + settings_class = GitHarvestSettings + + def __call__(self, command): + print("Hello World!") + + return {}, {} +``` + +The Code uses the HermesHarvestPlugin as base class and pydantics Basemodel for the Settings. In the GitHarvestSettings you +can see that one setting is made. The Parameter from_branch is specific for this plugin and can be reached through self.settings.harvest.git.git_branch as long as our plugin will be named git. +In the hermes.toml this would be achieved by [harvest.{plugin_name}]. +The GitHarvestSettings are assigned to the GitHarvestPlugin. In the plugin you need to overwrite the __ call __ method. +For now a simple Hello World will do. The method return two dictionaries. These will later depict the harvested data in codemeta (json-ld) and information for generating hermes metadata. +That is the basic structure for the plugins source code. + +To integrate this code, you have to register it as a plugin in the pyproject.toml. To learn more about the pyproject.toml check https://python-poetry.org/docs/pyproject/. +We will just look at the important places for this plugin. There are two ways to integrate this plugin. First we will show how to use the plugin environment as the running base with HERMES as a dependency. +Then we say how to integrate this plugin in HERMES itself. + +### Include HERMES as Dependency +```{code-block} toml +... +[tool.poetry.dependencies] +python = "^3.10" +hermes = "^0.8.0" +... +... +[tool.poetry.plugins."hermes.harvest"] +git = "hermes_git.harvest:GitHarvestPlugin" +... +``` +### Write Plugin to be included in HERMES +```{code-block} toml +... +[tool.poetry.dependencies] +... +pydantic-settings = "^2.1.0" +hermes-git = { git = "https://github.com/hermes-hmc/hermes-git.git", branch = "main" } +... +... +[tool.poetry.plugins."hermes.harvest"] +cff = "hermes.commands.harvest.cff:CffHarvestPlugin" +codemeta = "hermes.commands.harvest.codemeta:CodeMetaHarvestPlugin" +git = "hermes_git.harvest:GitHarvestPlugin" +... +``` + +```{admonition} Congratulations! +You can now write plugins for HERMES. +``` From 9e4baec5803dda82c3573fdd4b28854d27f4de66 Mon Sep 17 00:00:00 2001 From: SKernchen Date: Fri, 10 May 2024 05:16:48 +0200 Subject: [PATCH 05/33] First version of tutorial, needs correction --- .../tutorials/writing-a-plugin-for-hermes.md | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/source/tutorials/writing-a-plugin-for-hermes.md b/docs/source/tutorials/writing-a-plugin-for-hermes.md index 8c20d7dd..045e5d83 100644 --- a/docs/source/tutorials/writing-a-plugin-for-hermes.md +++ b/docs/source/tutorials/writing-a-plugin-for-hermes.md @@ -73,6 +73,11 @@ We will just look at the important places for this plugin. There are two ways to Then we say how to integrate this plugin in HERMES itself. ### Include HERMES as Dependency +This is probably the more common way, where you can see HERMES as a framework. +The idea is that your project is the main part. You create the pyproject.toml as usual. +In the dependencies block you need to include hermes. Then you just have to declare your plugin. +The HERMES software will look for plugins and install these. +In the code below you can see the parts of the pyproject.toml that are important. ```{code-block} toml ... [tool.poetry.dependencies] @@ -84,7 +89,18 @@ hermes = "^0.8.0" git = "hermes_git.harvest:GitHarvestPlugin" ... ``` +As you can see the plugin class from hermes_git is declared as git for hermes.harvest. +To use the plugin you have to adapt the harvest Settings in the hermes.toml. +We will discuss the exact step after showing the other pyproject.toml configuration. +```{note} +You have to run poetry install to add and install all entrypoints declared in the pyproject.toml. +``` + ### Write Plugin to be included in HERMES +This variant is used to contribute to the HERMES community or adapt the HERMES workflow for own purposes. +If you want to contribute, see the [Contribution Guidelines](https://docs.software-metadata.pub/en/latest/dev/contribute.html). +After cloning the HERMES workflow repository you can adapt the pyproject.toml. +In the code below you see the parts with the important lines. ```{code-block} toml ... [tool.poetry.dependencies] @@ -99,7 +115,42 @@ codemeta = "hermes.commands.harvest.codemeta:CodeMetaHarvestPlugin" git = "hermes_git.harvest:GitHarvestPlugin" ... ``` +In the dependencies you have to install your plugin. If your Plugin is pip installable than you can just give the name and the version. +If your plugin is in a buildable git repository, you can install it with the given expression. +Note that this differs with the accessibility and your wishes, check [Explicit Package Sources](https://python-poetry.org/docs/repositories/#explicit-package-sources). + +The second thing to adapt is to declare the access point for the plugin. +You can do that with `git = "hermes_git.harvest:GitHarvestPlugin"`. +This expression makes the GitHarvestPlugin from the hermes_git package, a hermes.harvest plugin named git. +So you need to configure this line with your plugin properties. + +Now you just need to add the plugin to the hermes.toml and reinstall the adapted poetry package. + +### Configure hermes.toml +To use the plugin, you have to set it in the hermes.toml. +The settings for the plugins are also set there. +For the harvest plugin the hermes.toml could look like this: +```{code-block} toml +[harvest] +sources = [ "cff", "git" ] # ordered priority (first one is most important) + +[harvest.cff] +enable_validation = false + +[harvest.git] +from_branch = "develop" +... +``` +In [harvest] you define that this plugin is used with less priority than the built-in cff plugin. +in [harvest.git] you set the configuration for the plugin. +In the Beginning of this tutorial we set the parameter `from_branch` in the git settings. Now we change the default from_branch to develop. +With this Configuration the plugin will be used. If you run hermes harvest, you should see the "Hello World" message. +Of course the hermes.toml is always changeable as you desire. ```{admonition} Congratulations! You can now write plugins for HERMES. ``` +To fill the plugin with code, you can check our [harvest_git](https://github.com/hermes-hmc/hermes-git) repository. +There is the code to check the local git history and extract contributors of the given branch. + +If you have any questions, wishes or requests, feel free to contact us. From 13171de18aa4454e72bfe8a8a2c311289f2c646f Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 11:22:09 +0200 Subject: [PATCH 06/33] Use template to publish --- .github/workflows/hermes-zenodo-sandbox.yml | 146 ++++++++++++++++++-- 1 file changed, 134 insertions(+), 12 deletions(-) diff --git a/.github/workflows/hermes-zenodo-sandbox.yml b/.github/workflows/hermes-zenodo-sandbox.yml index 5a036b26..4a6c6aae 100644 --- a/.github/workflows/hermes-zenodo-sandbox.yml +++ b/.github/workflows/hermes-zenodo-sandbox.yml @@ -8,24 +8,146 @@ name: Software publication on Zenodo Sandbox on: push: - tags: - - "*" + branches: + - feature/261-put-hermes-on-zenodo + #tags: + # - "*" + + # NOTE: Do not delete the trigger on closed pull requests, the HERMES workflow needs this. + pull_request: + types: + - closed jobs: - hermes: - name: HERMES + hermes-prepare: + name: Prepare Metadata for Curation + runs-on: ubuntu-latest + # This condition becomes much easier when we only react to push events on the release branch. + # We still need to exclude the merge commit push of the post processing PR + + # ADAPT + # Depending on the event you react to in the 'on:' section above, you will need to adapt this + # to react on the specific events. + # NOTE: You will probably still need to keep the exclusion check for commit messages provided by the workflow ('hermes/'/'hermes/post'). + if: > + github.event_name == 'push' && ! ( + startsWith(github.ref_name, 'hermes/') || + contains(github.event.head_commit.message, 'hermes/post') + ) + + permissions: + contents: write # Allow creation of new branches + pull-requests: write # Postprocessing should be able to create a pull request with changes + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - run: pip install hermes + - run: hermes harvest + - run: hermes process + - run: hermes curate + + - run: | + # Cache current branch for PR close job + git branch --show-current > .hermes/curate/target_branch + + # Shorten the SHA for the PR title + SHORT_SHA=$(echo "$GITHUB_SHA" | cut -c -8) + echo "SHORT_SHA=$SHORT_SHA" >> "$GITHUB_ENV" + + # Create a curation branch + git branch -c "hermes/curate-$SHORT_SHA" + git push origin "hermes/curate-$SHORT_SHA" + + # Explicitly add to-be-curated metadata (which is ignored via .gitignore!) + git add -f .hermes/curate + - uses: peter-evans/create-pull-request@v5 + with: + base: hermes/curate-${{ env.SHORT_SHA }} + branch: hermes/curate-result-${{ env.SHORT_SHA }} + title: Metadata Curation for Commit ${{ env.SHORT_SHA }} + body: | + Please carefully review the attached metadata. + If you are satisfied with the result, you may merge this PR, which will trigger publication. + (Any temporary branches will be cleaned up.) + delete-branch: true + + hermes-curate: + name: Publish Software with Curated Metadata + if: github.event.pull_request.merged == true && startsWith( github.base_ref , 'hermes/curate-') + runs-on: ubuntu-latest permissions: - contents: read # We will only read content from the repo - # pull-requests: write # Postprocessing should be able to create a pull request with changes + contents: write # Allow creation of new branches + pull-requests: write # Postprocessing should be able to create a pull request with changes + steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: python-version: '3.10' - - run: pip install git+https://github.com/sdruskat/workflow.git@deRSE23 + - run: pip install hermes + + # ADAPT + # If you want to publish artifacts (e.g., a zipped snapshot of your repository), + # you can prepare this here. - run: git archive --format zip HEAD src > hermes.zip - - run: hermes - - run: hermes deposit --auth-token ${{ secrets.ZENODO }} --file hermes.zip --file README.md - # - run: hermes postprocess # (Not implemented yet) - # - uses: peter-evans/create-pull-request@v4 (TODO once postprocessing has it) + + # Run the HERMES deposition and postprocessing steps. + # ADAPT + # 1. You need to have an authentication token for your target publication platform + # as a GitHub secret in your repository (in this example, this is called ZENODO_SANDBOX). + # 2. Adapt the files you want to deposit. In the example, showcase.zip and README.md are deposited alongside the metadata. + # 3. Check if you want to run with '--initial', as this may potentially create a completely new record (collection), + # rather than a new version of the same collection! + - run: hermes deposit --initial -O invenio_rdm.auth_token ${{ secrets.ZENODO_SANDBOX }} --file hermes.zip --file README.md + + # ADAPT + # Remove this command if you don't want to do any postprocessing + - run: hermes postprocess + + # ADAPT + # If you don't want to run postprocessing, remove this complete section (next '-run' and 'uses: peter-evans/...' bullets). + # + # Note 1: We change the base branch here for the PR. This flow runs so far within the "curated-metadata-*" branch, + # but now we want to add the changes done by deposit and post processing to the branch that was initially + # meant to be published using HERMES. + # Note 2: The create-pull-request action will NOT inherit the commits we did in the previous job. It will only look at the + # changes within this local workspace we did *now*. + - run: echo "TARGET_BRANCH=$(cat .hermes/curate/target_branch)" >> "$GITHUB_ENV" + - uses: peter-evans/create-pull-request@v5 + with: + branch: hermes/post-${{ github.run_id }} + base: ${{ env.TARGET_BRANCH }} + title: Review hermes post-processing results + body: | + This is an automated pull request created by HERMES post-processing. + + Please carefully review the changes and finally merge them into your + + # Delete all the curation branches + - run: | + for BRANCH in $(git ls-remote origin 'refs/heads/hermes/curate-*' | cut -f2 | cut -d'/' -f'3-'); do + git push origin --delete "$BRANCH" + done + # TODO: if: failure() --- delete the curation branches when the deposition failed + + + hermes-cleanup: + name: Cleanup aborted curation branches + if: github.event.pull_request.merged == false && startsWith( github.base_ref , 'hermes/curate-') + + runs-on: ubuntu-latest + permissions: + contents: write # Allow creation of new branches + pull-requests: write # Postprocessing should be able to create a pull request with changes + + steps: + - uses: actions/checkout@v3 + # Delete all the curation branches + - run: | + for BRANCH in $(git ls-remote origin 'refs/heads/hermes/curate-*' | cut -f2 | cut -d'/' -f'3-'); do + git push origin --delete "$BRANCH" + done From d8a9476aa7b018c220a845b8d31042819b97314a Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 12:57:11 +0200 Subject: [PATCH 07/33] Update to newer version --- CITATION.cff | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 54830b3a..6eed50bc 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -14,9 +14,9 @@ title: hermes message: >- If you use this software, please cite it using the metadata from this file. -version: "proof-of-concept" +version: 0.8.0b1 license: "Apache-2.0" -abstract: "Proof-of-concept implementation of the HERMES workflow." +abstract: "Tool to automate software publication. Not stable yet." type: software authors: - given-names: Michael @@ -49,3 +49,8 @@ authors: email: d.pape@hzdr.de affiliation: Helmholtz-Zentrum Dresden-Rossendorf (HZDR) orcid: 'https://orcid.org/0000-0002-3145-9880' + - given-names: Kernchen + family-names: Sophie + email: sophie.kernchen@dlr.de + affiliation: German Aerospace Center (DLR) + orcid: 'https://orcid.org/0009-0005-4430-6743' From 4a33aec5300f18293fc6d0b05297dbbe65952cc0 Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 13:04:45 +0200 Subject: [PATCH 08/33] Correct variable name --- .github/workflows/hermes-zenodo-sandbox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hermes-zenodo-sandbox.yml b/.github/workflows/hermes-zenodo-sandbox.yml index 4a6c6aae..a815df40 100644 --- a/.github/workflows/hermes-zenodo-sandbox.yml +++ b/.github/workflows/hermes-zenodo-sandbox.yml @@ -102,7 +102,7 @@ jobs: # 2. Adapt the files you want to deposit. In the example, showcase.zip and README.md are deposited alongside the metadata. # 3. Check if you want to run with '--initial', as this may potentially create a completely new record (collection), # rather than a new version of the same collection! - - run: hermes deposit --initial -O invenio_rdm.auth_token ${{ secrets.ZENODO_SANDBOX }} --file hermes.zip --file README.md + - run: hermes deposit --initial -O invenio_rdm.auth_token ${{ secrets.ZENODO }} --file hermes.zip --file README.md # ADAPT # Remove this command if you don't want to do any postprocessing From f96789573ce14727517a07cb4b63cc78821002ad Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 13:13:31 +0200 Subject: [PATCH 09/33] Correct version --- CITATION.cff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index 6eed50bc..eea43ed5 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -14,7 +14,7 @@ title: hermes message: >- If you use this software, please cite it using the metadata from this file. -version: 0.8.0b1 +version: 0.8.1b1 license: "Apache-2.0" abstract: "Tool to automate software publication. Not stable yet." type: software From 78b2727a601c287a56bfc828aab467097fb2b28d Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 13:22:55 +0200 Subject: [PATCH 10/33] Push on Zenodo not sandbox --- .../{hermes-zenodo-sandbox.yml => hermes-zenodo.yml} | 6 ++---- hermes.toml | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) rename .github/workflows/{hermes-zenodo-sandbox.yml => hermes-zenodo.yml} (98%) diff --git a/.github/workflows/hermes-zenodo-sandbox.yml b/.github/workflows/hermes-zenodo.yml similarity index 98% rename from .github/workflows/hermes-zenodo-sandbox.yml rename to .github/workflows/hermes-zenodo.yml index a815df40..2db5ae99 100644 --- a/.github/workflows/hermes-zenodo-sandbox.yml +++ b/.github/workflows/hermes-zenodo.yml @@ -8,10 +8,8 @@ name: Software publication on Zenodo Sandbox on: push: - branches: - - feature/261-put-hermes-on-zenodo - #tags: - # - "*" + tags: + - "*" # NOTE: Do not delete the trigger on closed pull requests, the HERMES workflow needs this. pull_request: diff --git a/hermes.toml b/hermes.toml index 8884aea4..e2ea3690 100644 --- a/hermes.toml +++ b/hermes.toml @@ -12,7 +12,7 @@ enable_validation = false target = "invenio_rdm" [deposit.invenio_rdm] -site_url = "https://sandbox.zenodo.org" +site_url = "https://zenodo.org" communities = [] access_right = "open" From ea92fde20712c5760d7684c06382f09a4540f5c4 Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 13:24:51 +0200 Subject: [PATCH 11/33] Change name --- .github/workflows/hermes-zenodo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hermes-zenodo.yml b/.github/workflows/hermes-zenodo.yml index 2db5ae99..46cf0390 100644 --- a/.github/workflows/hermes-zenodo.yml +++ b/.github/workflows/hermes-zenodo.yml @@ -4,7 +4,7 @@ # SPDX-FileContributor: Stephan Druskat -name: Software publication on Zenodo Sandbox +name: Software publication on Zenodo on: push: From d42aa8f41e507718e9b23d11825911aa4096c9aa Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 14:02:43 +0200 Subject: [PATCH 12/33] Adapt for release tag not branch --- .github/workflows/hermes-zenodo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hermes-zenodo.yml b/.github/workflows/hermes-zenodo.yml index 46cf0390..402c5944 100644 --- a/.github/workflows/hermes-zenodo.yml +++ b/.github/workflows/hermes-zenodo.yml @@ -56,7 +56,7 @@ jobs: echo "SHORT_SHA=$SHORT_SHA" >> "$GITHUB_ENV" # Create a curation branch - git branch -c "hermes/curate-$SHORT_SHA" + git switch -c "hermes/curate-$SHORT_SHA" git push origin "hermes/curate-$SHORT_SHA" # Explicitly add to-be-curated metadata (which is ignored via .gitignore!) From 0f9dcd83a7a0518de2a05d9821586d8675a212c8 Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 14:17:38 +0200 Subject: [PATCH 13/33] Add record id and DOI --- CITATION.cff | 4 ++++ hermes.toml | 1 + 2 files changed, 5 insertions(+) diff --git a/CITATION.cff b/CITATION.cff index eea43ed5..36db497f 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -54,3 +54,7 @@ authors: email: sophie.kernchen@dlr.de affiliation: German Aerospace Center (DLR) orcid: 'https://orcid.org/0009-0005-4430-6743' +identifiers: + - type: doi + value: 10.5281/zenodo.13221384 + description: Version 0.8.1b1 diff --git a/hermes.toml b/hermes.toml index e2ea3690..f455746a 100644 --- a/hermes.toml +++ b/hermes.toml @@ -15,6 +15,7 @@ target = "invenio_rdm" site_url = "https://zenodo.org" communities = [] access_right = "open" +record_id = 13221384 [deposit.invenio_rdm.api_paths] depositions = "api/deposit/depositions" From 7f64f102e916c76dc44404b77ab2a80f5a4e59b1 Mon Sep 17 00:00:00 2001 From: "Meinel, Michael" Date: Tue, 6 Aug 2024 14:41:28 +0200 Subject: [PATCH 14/33] Add basic error handling to console entry point --- src/hermes/commands/cli.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/hermes/commands/cli.py b/src/hermes/commands/cli.py index 5ca827de..dae31858 100644 --- a/src/hermes/commands/cli.py +++ b/src/hermes/commands/cli.py @@ -9,6 +9,7 @@ This module provides the main entry point for the HERMES command line application. """ import argparse +import sys from hermes import logger from hermes.commands import HermesHelpCommand, HermesCleanCommand, HermesHarvestCommand, HermesProcessCommand, \ @@ -58,7 +59,22 @@ def main() -> None: args = parser.parse_args() logger.init_logging() + log = logger.getLogger("hermes.cli") + log.debug("Running hermes with the following command line arguments: %s", args) - args.command.load_settings(args) - args.command.patch_settings(args) - args.command(args) + try: + log.info("Loading settings...") + args.command.load_settings(args) + + log.debug("Update settings from command line...") + args.command.patch_settings(args) + + log.info("Run subcommand %s", args.command.command_name) + args.command(args) + except BaseException as e: + log.error("An error occurred during execution of %s", args.command.command_name) + log.debug("Original exception was: %s", e) + + sys.exit(1) + + sys.exit(0) From c27192c055853a321f1d64d20748ba267f80bd3c Mon Sep 17 00:00:00 2001 From: "Meinel, Michael" Date: Tue, 6 Aug 2024 14:59:10 +0200 Subject: [PATCH 15/33] Adapt commands and plugins to propagate errors There is no consistent use yet, some exceptions are re-raised, some are logged to an internal error register of the command... --- src/hermes/commands/base.py | 1 + src/hermes/commands/cli.py | 7 ++++++- src/hermes/commands/deposit/base.py | 4 +++- src/hermes/commands/deposit/invenio.py | 2 +- src/hermes/commands/harvest/base.py | 4 +++- src/hermes/commands/postprocess/invenio.py | 4 ++-- src/hermes/commands/postprocess/invenio_rdm.py | 4 ++-- src/hermes/commands/process/base.py | 3 ++- 8 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/hermes/commands/base.py b/src/hermes/commands/base.py index dea22412..82692975 100644 --- a/src/hermes/commands/base.py +++ b/src/hermes/commands/base.py @@ -43,6 +43,7 @@ def __init__(self, parser: argparse.ArgumentParser): self.settings = None self.log = logging.getLogger(f"hermes.{self.command_name}") + self.errors = [] def init_plugins(self): """Collect and initialize the plugins available for the HERMES command.""" diff --git a/src/hermes/commands/cli.py b/src/hermes/commands/cli.py index dae31858..167d1beb 100644 --- a/src/hermes/commands/cli.py +++ b/src/hermes/commands/cli.py @@ -71,10 +71,15 @@ def main() -> None: log.info("Run subcommand %s", args.command.command_name) args.command(args) - except BaseException as e: + except Exception as e: log.error("An error occurred during execution of %s", args.command.command_name) log.debug("Original exception was: %s", e) + sys.exit(2) + + if args.command.errors: + for e in args.command.errors: + log.error(e) sys.exit(1) sys.exit(0) diff --git a/src/hermes/commands/deposit/base.py b/src/hermes/commands/deposit/base.py index 5f0290c2..0a8e10f5 100644 --- a/src/hermes/commands/deposit/base.py +++ b/src/hermes/commands/deposit/base.py @@ -134,11 +134,13 @@ def __call__(self, args: argparse.Namespace) -> None: try: plugin_func = self.plugins[plugin_name](self, ctx) - except KeyError: + except KeyError as e: self.log.error("Plugin '%s' not found.", plugin_name) + self.errors.append(e) try: plugin_func(self) except HermesValidationError as e: self.log.error("Error while executing %s: %s", plugin_name, e) + self.errors.append(e) diff --git a/src/hermes/commands/deposit/invenio.py b/src/hermes/commands/deposit/invenio.py index 6c8ff139..ddee7e68 100644 --- a/src/hermes/commands/deposit/invenio.py +++ b/src/hermes/commands/deposit/invenio.py @@ -105,7 +105,7 @@ class InvenioResolver: invenio_client_class = InvenioClient def __init__(self, client=None): - self.client = client or self.invenio_client_class() + self.client = client or self.invenio_client_class(InvenioDepositSettings()) def resolve_latest_id( self, record_id=None, doi=None, codemeta_identifier=None diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index 629342e7..4d2a1731 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -57,8 +57,10 @@ def __call__(self, args: argparse.Namespace) -> None: if any(v != _value and t == _tag for v, t in _trace): raise MergeError(_key, None, _value) - except KeyError: + except KeyError as e: self.log.error("Plugin '%s' not found.", plugin_name) + self.errors.append(e) except HermesValidationError as e: self.log.error("Error while executing %s: %s", plugin_name, e) + self.errors.append(e) diff --git a/src/hermes/commands/postprocess/invenio.py b/src/hermes/commands/postprocess/invenio.py index 7d039ae1..bf3a7df3 100644 --- a/src/hermes/commands/postprocess/invenio.py +++ b/src/hermes/commands/postprocess/invenio.py @@ -23,8 +23,8 @@ def config_record_id(ctx): try: conf.deposit.invenio.record_id = deposition['record_id'] toml.dump(conf, open('hermes.toml', 'w')) - except KeyError: - raise RuntimeError("No deposit.invenio configuration available to store record id in") from None + except KeyError as e: + raise RuntimeError("No deposit.invenio configuration available to store record id in") from e def cff_doi(ctx): diff --git a/src/hermes/commands/postprocess/invenio_rdm.py b/src/hermes/commands/postprocess/invenio_rdm.py index 7b2bf37c..ddbc8cb4 100644 --- a/src/hermes/commands/postprocess/invenio_rdm.py +++ b/src/hermes/commands/postprocess/invenio_rdm.py @@ -22,5 +22,5 @@ def config_record_id(ctx): try: conf['deposit']['invenio_rdm']['record_id'] = deposition['record_id'] toml.dump(conf, open('hermes.toml', 'w')) - except KeyError: - raise RuntimeError("No deposit.invenio configuration available to store record id in") from None + except KeyError as e: + raise RuntimeError("No deposit.invenio configuration available to store record id in") from e diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 8abeacc5..9e29d1e6 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -58,7 +58,8 @@ def __call__(self, args: argparse.Namespace) -> None: ctx.merge_contexts_from(harvest_context) if ctx._errors: - self.log.error('!!! warning "Errors during merge"') + self.log.error('Errors during merge') + self.errors.extend(ctx._errors) for ep, error in ctx._errors: self.log.info(" - %s: %s", ep.name, error) From 6289b945a22403b44ebc4a953a7bff702c3955aa Mon Sep 17 00:00:00 2001 From: Sophie <133236526+SKernchen@users.noreply.github.com> Date: Fri, 9 Aug 2024 09:54:18 +0200 Subject: [PATCH 16/33] Correct form and improve text Co-authored-by: Michael Meinel --- .../tutorials/writing-a-plugin-for-hermes.md | 66 ++++++++++--------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/docs/source/tutorials/writing-a-plugin-for-hermes.md b/docs/source/tutorials/writing-a-plugin-for-hermes.md index 045e5d83..e6781977 100644 --- a/docs/source/tutorials/writing-a-plugin-for-hermes.md +++ b/docs/source/tutorials/writing-a-plugin-for-hermes.md @@ -13,10 +13,10 @@ SPDX-FileContributor: Sophie Kernchen This tutorial will present the basic steps for writing an additional harvester. -At the moment only the harvest architecture is stable. -The full code and structure is available at [harvest-git](https://github.com/hermes-hmc/hermes-git). +At the moment only the architecture for harvester plugins is stable. +The full code and structure is available at [hermes-plugin-git](https://github.com/softwarepub/hermes-plugin-git). This plugin extracts information from the local git history. -The harvest-git plugin will help to gather contributing and branch metadata. +The hermes-plugin-git will help to gather contributing and branch metadata. ```{note} For this tutorial you should be familiar with HERMES. If you never used HERMES before, you might want to check the tutorial: [Automated Publication with HERMES](https://docs.software-metadata.pub/en/latest/tutorials/automated-publication-with-ci.html). @@ -26,15 +26,18 @@ If you never used HERMES before, you might want to check the tutorial: [Automate HERMES uses a plugin architecture. Therefore, users are invited to contribute own features. The structure for every plugin follows the same schema. -There is a base class for every plugin. In this HermesPlugin class there is one abstract method __ call __ which needs to be overwritten. -Furthermore, the HermesCommand class provides all needs for writing a plugin used in a HERMES command. -So the HermesPlugins call method uses an Instance of the HermesCommand that triggered this plugin to run. -In our case this will be the HermesHarvestCommand which calls all harvest plugins. -The Plugin class also uses a derivative of HermesSettings to add parameters. -HermesSettings are the base class for command specific settings. -It uses pydantic settings to specify and validate the parameters. -The user can either set the parameters in the hermes.toml or overwrite them in the command line. -To overwrite the configuration, you use the -O operator with the dotted parameter name and the value. +There is a top-level base class for every plugin. In this `HermesPlugin` class there is one abstract method `__ call __` which needs to be overwritten. +Furthermore, the `HermesCommand` class provides all needs for writing a plugin used in a HERMES command. +So the `HermesPlugin`s call method gets an instance of the `HermesCommand` that triggered this plugin to run. +In our case this will be the `HermesHarvestCommand` which calls all harvest plugins. +The plugin class also uses a derivative of `HermesSettings` to add parameters that can be adapted by the configuration file. +`HermesSettings` are the base class for command specific settings. +It uses [pydantic](https://docs.pydantic.dev/latest/) [settings](https://docs.pydantic.dev/latest/api/pydantic_settings/) to specify and validate the parameters. +The user can either set the parameters in the `hermes.toml` or overwrite them in the command line. +To overwrite a parameter from command line, use the `-O` command line option followed by the dotted parameter name and the value. +E.g., you can set your authentication token for InvenioRDM by adding the following options to your call to `hermes deposit`: +```shell +hermes deposit -O invenio_rdm.auth_token YourSecretAuthToken ## Set Up Plugin To write a new plugin, it is important to follow the given structure. @@ -61,23 +64,23 @@ class GitHarvestPlugin(HermesHarvestPlugin): return {}, {} ``` -The Code uses the HermesHarvestPlugin as base class and pydantics Basemodel for the Settings. In the GitHarvestSettings you -can see that one setting is made. The Parameter from_branch is specific for this plugin and can be reached through self.settings.harvest.git.git_branch as long as our plugin will be named git. -In the hermes.toml this would be achieved by [harvest.{plugin_name}]. -The GitHarvestSettings are assigned to the GitHarvestPlugin. In the plugin you need to overwrite the __ call __ method. +The code uses the `HermesHarvestPlugin` as base class and pydantics Basemodel for the settings. In the `GitHarvestSettings` you +can see that an additional parameter is defined. The Parameter `from_branch` is specific for this plugin and can be accessed inside the plugin using `self.settings.harvest.git.git_branch` as long as our plugin will be named git. +In the `hermes.toml` this would be achieved by [harvest.{plugin_name}]. +The `GitHarvestSettings` are associated with the `GitHarvestPlugin`. In the plugin you need to overwrite the `__ call __` method. For now a simple Hello World will do. The method return two dictionaries. These will later depict the harvested data in codemeta (json-ld) and information for generating hermes metadata. That is the basic structure for the plugins source code. -To integrate this code, you have to register it as a plugin in the pyproject.toml. To learn more about the pyproject.toml check https://python-poetry.org/docs/pyproject/. +To integrate this code, you have to register it as a plugin in the `pyproject.toml`. To learn more about the `pyproject.toml` check https://python-poetry.org/docs/pyproject/ or refer to [PEP621](https://peps.python.org/pep-0621/). We will just look at the important places for this plugin. There are two ways to integrate this plugin. First we will show how to use the plugin environment as the running base with HERMES as a dependency. Then we say how to integrate this plugin in HERMES itself. ### Include HERMES as Dependency This is probably the more common way, where you can see HERMES as a framework. -The idea is that your project is the main part. You create the pyproject.toml as usual. -In the dependencies block you need to include hermes. Then you just have to declare your plugin. -The HERMES software will look for plugins and install these. -In the code below you can see the parts of the pyproject.toml that are important. +The idea is that your project is the main part. You create the `pyproject.toml` as usual. +In the dependencies block you need to include `hermes`. Then you just have to declare your plugin. +The HERMES software will look for installed plugins and use them. +In the code below you can see the parts of the `pyproject.toml` that are important. ```{code-block} toml ... [tool.poetry.dependencies] @@ -89,9 +92,9 @@ hermes = "^0.8.0" git = "hermes_git.harvest:GitHarvestPlugin" ... ``` -As you can see the plugin class from hermes_git is declared as git for hermes.harvest. -To use the plugin you have to adapt the harvest Settings in the hermes.toml. -We will discuss the exact step after showing the other pyproject.toml configuration. +As you can see the plugin class from `hermes_git` is declared as `git` for the `hermes.harvest` entrypoint. +To use the plugin you have to adapt the harvest settings in the `hermes.toml`. +We will discuss the exact step after showing the other `pyproject.toml` configuration. ```{note} You have to run poetry install to add and install all entrypoints declared in the pyproject.toml. ``` @@ -127,9 +130,9 @@ So you need to configure this line with your plugin properties. Now you just need to add the plugin to the hermes.toml and reinstall the adapted poetry package. ### Configure hermes.toml -To use the plugin, you have to set it in the hermes.toml. +To use the plugin, you have to activate it in the `hermes.toml`. The settings for the plugins are also set there. -For the harvest plugin the hermes.toml could look like this: +For the harvest plugin the `hermes.toml` could look like this: ```{code-block} toml [harvest] sources = [ "cff", "git" ] # ordered priority (first one is most important) @@ -141,16 +144,15 @@ enable_validation = false from_branch = "develop" ... ``` -In [harvest] you define that this plugin is used with less priority than the built-in cff plugin. -in [harvest.git] you set the configuration for the plugin. -In the Beginning of this tutorial we set the parameter `from_branch` in the git settings. Now we change the default from_branch to develop. -With this Configuration the plugin will be used. If you run hermes harvest, you should see the "Hello World" message. -Of course the hermes.toml is always changeable as you desire. +In the `[harvest]` section you define that this plugin is used with less priority than the built-in `cff` plugin. +in the `[harvest.git]` section you set the configuration for the plugin. +In the beginning of this tutorial we set the parameter `from_branch` in the git settings. Now we change the default `from_branch` to `develop`. +With this configuration the plugin will be used. If you run `hermes harvest`, you should see the "Hello World" message. ```{admonition} Congratulations! You can now write plugins for HERMES. ``` -To fill the plugin with code, you can check our [harvest_git](https://github.com/hermes-hmc/hermes-git) repository. +To fill the plugin with code, you can check our [hermes-plugin-git](https://github.com/softwarepub/hermes-plugin-git) repository. There is the code to check the local git history and extract contributors of the given branch. If you have any questions, wishes or requests, feel free to contact us. From 62b9799dc8191b37d1043c962c02a8bf5ce09f17 Mon Sep 17 00:00:00 2001 From: Sophie <133236526+SKernchen@users.noreply.github.com> Date: Fri, 9 Aug 2024 09:56:18 +0200 Subject: [PATCH 17/33] Correct duplicated word --- docs/source/tutorials/writing-a-plugin-for-hermes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/tutorials/writing-a-plugin-for-hermes.md b/docs/source/tutorials/writing-a-plugin-for-hermes.md index e6781977..6eadb5f3 100644 --- a/docs/source/tutorials/writing-a-plugin-for-hermes.md +++ b/docs/source/tutorials/writing-a-plugin-for-hermes.md @@ -68,7 +68,7 @@ The code uses the `HermesHarvestPlugin` as base class and pydantics Basemodel fo can see that an additional parameter is defined. The Parameter `from_branch` is specific for this plugin and can be accessed inside the plugin using `self.settings.harvest.git.git_branch` as long as our plugin will be named git. In the `hermes.toml` this would be achieved by [harvest.{plugin_name}]. The `GitHarvestSettings` are associated with the `GitHarvestPlugin`. In the plugin you need to overwrite the `__ call __` method. -For now a simple Hello World will do. The method return two dictionaries. These will later depict the harvested data in codemeta (json-ld) and information for generating hermes metadata. +For now a simple Hello World will do. The method returns two dictionaries. These will contain the harvested data in CodeMeta (JSON-LD) and additional information, e.g., to provide provenance information. That is the basic structure for the plugins source code. To integrate this code, you have to register it as a plugin in the `pyproject.toml`. To learn more about the `pyproject.toml` check https://python-poetry.org/docs/pyproject/ or refer to [PEP621](https://peps.python.org/pep-0621/). From 46a7b9f2475865616cb07f1e88724bc0fe36ed28 Mon Sep 17 00:00:00 2001 From: Sophie <133236526+SKernchen@users.noreply.github.com> Date: Fri, 9 Aug 2024 10:42:09 +0200 Subject: [PATCH 18/33] Add name convention to hermes_plugin_git everywhere --- docs/source/tutorials/writing-a-plugin-for-hermes.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/tutorials/writing-a-plugin-for-hermes.md b/docs/source/tutorials/writing-a-plugin-for-hermes.md index 6eadb5f3..91f0c4bd 100644 --- a/docs/source/tutorials/writing-a-plugin-for-hermes.md +++ b/docs/source/tutorials/writing-a-plugin-for-hermes.md @@ -89,10 +89,10 @@ hermes = "^0.8.0" ... ... [tool.poetry.plugins."hermes.harvest"] -git = "hermes_git.harvest:GitHarvestPlugin" +git = "hermes_plugin_git.harvest:GitHarvestPlugin" ... ``` -As you can see the plugin class from `hermes_git` is declared as `git` for the `hermes.harvest` entrypoint. +As you can see the plugin class from `hermes_plugin_git` is declared as `git` for the `hermes.harvest` entrypoint. To use the plugin you have to adapt the harvest settings in the `hermes.toml`. We will discuss the exact step after showing the other `pyproject.toml` configuration. ```{note} @@ -109,13 +109,13 @@ In the code below you see the parts with the important lines. [tool.poetry.dependencies] ... pydantic-settings = "^2.1.0" -hermes-git = { git = "https://github.com/hermes-hmc/hermes-git.git", branch = "main" } +hermes-plugin-git = { git = "https://github.com/softwarepub/hermes-plugin-git.git", branch = "main" } ... ... [tool.poetry.plugins."hermes.harvest"] cff = "hermes.commands.harvest.cff:CffHarvestPlugin" codemeta = "hermes.commands.harvest.codemeta:CodeMetaHarvestPlugin" -git = "hermes_git.harvest:GitHarvestPlugin" +git = "hermes_plugin_git.harvest:GitHarvestPlugin" ... ``` In the dependencies you have to install your plugin. If your Plugin is pip installable than you can just give the name and the version. @@ -123,8 +123,8 @@ If your plugin is in a buildable git repository, you can install it with the giv Note that this differs with the accessibility and your wishes, check [Explicit Package Sources](https://python-poetry.org/docs/repositories/#explicit-package-sources). The second thing to adapt is to declare the access point for the plugin. -You can do that with `git = "hermes_git.harvest:GitHarvestPlugin"`. -This expression makes the GitHarvestPlugin from the hermes_git package, a hermes.harvest plugin named git. +You can do that with `git = "hermes_plugin_git.harvest:GitHarvestPlugin"`. +This expression makes the GitHarvestPlugin from the hermes_plugin_git package, a hermes.harvest plugin named git. So you need to configure this line with your plugin properties. Now you just need to add the plugin to the hermes.toml and reinstall the adapted poetry package. From 241f91b1df7605d05f9629188fe6f77cb115df56 Mon Sep 17 00:00:00 2001 From: Michael Meinel Date: Tue, 13 Aug 2024 13:51:32 +0200 Subject: [PATCH 19/33] Apply suggestions from code review --- src/hermes/commands/cli.py | 2 +- src/hermes/commands/postprocess/invenio.py | 4 ++-- src/hermes/commands/postprocess/invenio_rdm.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/hermes/commands/cli.py b/src/hermes/commands/cli.py index 167d1beb..542ff891 100644 --- a/src/hermes/commands/cli.py +++ b/src/hermes/commands/cli.py @@ -63,7 +63,7 @@ def main() -> None: log.debug("Running hermes with the following command line arguments: %s", args) try: - log.info("Loading settings...") + log.debug("Loading settings...") args.command.load_settings(args) log.debug("Update settings from command line...") diff --git a/src/hermes/commands/postprocess/invenio.py b/src/hermes/commands/postprocess/invenio.py index bf3a7df3..a7ba6b53 100644 --- a/src/hermes/commands/postprocess/invenio.py +++ b/src/hermes/commands/postprocess/invenio.py @@ -23,8 +23,8 @@ def config_record_id(ctx): try: conf.deposit.invenio.record_id = deposition['record_id'] toml.dump(conf, open('hermes.toml', 'w')) - except KeyError as e: - raise RuntimeError("No deposit.invenio configuration available to store record id in") from e + except KeyError: + raise RuntimeError("No deposit.invenio configuration available to store record id in") def cff_doi(ctx): diff --git a/src/hermes/commands/postprocess/invenio_rdm.py b/src/hermes/commands/postprocess/invenio_rdm.py index ddbc8cb4..9553f47b 100644 --- a/src/hermes/commands/postprocess/invenio_rdm.py +++ b/src/hermes/commands/postprocess/invenio_rdm.py @@ -22,5 +22,5 @@ def config_record_id(ctx): try: conf['deposit']['invenio_rdm']['record_id'] = deposition['record_id'] toml.dump(conf, open('hermes.toml', 'w')) - except KeyError as e: - raise RuntimeError("No deposit.invenio configuration available to store record id in") from e + except KeyError: + raise RuntimeError("No deposit.invenio_rdm configuration available to store record id in") From 7602e3d2647c4f877a525bc36cdb1cd583fd01b4 Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 11:22:09 +0200 Subject: [PATCH 20/33] Use template to publish --- .github/workflows/hermes-zenodo-sandbox.yml | 146 ++++++++++++++++++-- 1 file changed, 134 insertions(+), 12 deletions(-) diff --git a/.github/workflows/hermes-zenodo-sandbox.yml b/.github/workflows/hermes-zenodo-sandbox.yml index 5a036b26..4a6c6aae 100644 --- a/.github/workflows/hermes-zenodo-sandbox.yml +++ b/.github/workflows/hermes-zenodo-sandbox.yml @@ -8,24 +8,146 @@ name: Software publication on Zenodo Sandbox on: push: - tags: - - "*" + branches: + - feature/261-put-hermes-on-zenodo + #tags: + # - "*" + + # NOTE: Do not delete the trigger on closed pull requests, the HERMES workflow needs this. + pull_request: + types: + - closed jobs: - hermes: - name: HERMES + hermes-prepare: + name: Prepare Metadata for Curation + runs-on: ubuntu-latest + # This condition becomes much easier when we only react to push events on the release branch. + # We still need to exclude the merge commit push of the post processing PR + + # ADAPT + # Depending on the event you react to in the 'on:' section above, you will need to adapt this + # to react on the specific events. + # NOTE: You will probably still need to keep the exclusion check for commit messages provided by the workflow ('hermes/'/'hermes/post'). + if: > + github.event_name == 'push' && ! ( + startsWith(github.ref_name, 'hermes/') || + contains(github.event.head_commit.message, 'hermes/post') + ) + + permissions: + contents: write # Allow creation of new branches + pull-requests: write # Postprocessing should be able to create a pull request with changes + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - run: pip install hermes + - run: hermes harvest + - run: hermes process + - run: hermes curate + + - run: | + # Cache current branch for PR close job + git branch --show-current > .hermes/curate/target_branch + + # Shorten the SHA for the PR title + SHORT_SHA=$(echo "$GITHUB_SHA" | cut -c -8) + echo "SHORT_SHA=$SHORT_SHA" >> "$GITHUB_ENV" + + # Create a curation branch + git branch -c "hermes/curate-$SHORT_SHA" + git push origin "hermes/curate-$SHORT_SHA" + + # Explicitly add to-be-curated metadata (which is ignored via .gitignore!) + git add -f .hermes/curate + - uses: peter-evans/create-pull-request@v5 + with: + base: hermes/curate-${{ env.SHORT_SHA }} + branch: hermes/curate-result-${{ env.SHORT_SHA }} + title: Metadata Curation for Commit ${{ env.SHORT_SHA }} + body: | + Please carefully review the attached metadata. + If you are satisfied with the result, you may merge this PR, which will trigger publication. + (Any temporary branches will be cleaned up.) + delete-branch: true + + hermes-curate: + name: Publish Software with Curated Metadata + if: github.event.pull_request.merged == true && startsWith( github.base_ref , 'hermes/curate-') + runs-on: ubuntu-latest permissions: - contents: read # We will only read content from the repo - # pull-requests: write # Postprocessing should be able to create a pull request with changes + contents: write # Allow creation of new branches + pull-requests: write # Postprocessing should be able to create a pull request with changes + steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: python-version: '3.10' - - run: pip install git+https://github.com/sdruskat/workflow.git@deRSE23 + - run: pip install hermes + + # ADAPT + # If you want to publish artifacts (e.g., a zipped snapshot of your repository), + # you can prepare this here. - run: git archive --format zip HEAD src > hermes.zip - - run: hermes - - run: hermes deposit --auth-token ${{ secrets.ZENODO }} --file hermes.zip --file README.md - # - run: hermes postprocess # (Not implemented yet) - # - uses: peter-evans/create-pull-request@v4 (TODO once postprocessing has it) + + # Run the HERMES deposition and postprocessing steps. + # ADAPT + # 1. You need to have an authentication token for your target publication platform + # as a GitHub secret in your repository (in this example, this is called ZENODO_SANDBOX). + # 2. Adapt the files you want to deposit. In the example, showcase.zip and README.md are deposited alongside the metadata. + # 3. Check if you want to run with '--initial', as this may potentially create a completely new record (collection), + # rather than a new version of the same collection! + - run: hermes deposit --initial -O invenio_rdm.auth_token ${{ secrets.ZENODO_SANDBOX }} --file hermes.zip --file README.md + + # ADAPT + # Remove this command if you don't want to do any postprocessing + - run: hermes postprocess + + # ADAPT + # If you don't want to run postprocessing, remove this complete section (next '-run' and 'uses: peter-evans/...' bullets). + # + # Note 1: We change the base branch here for the PR. This flow runs so far within the "curated-metadata-*" branch, + # but now we want to add the changes done by deposit and post processing to the branch that was initially + # meant to be published using HERMES. + # Note 2: The create-pull-request action will NOT inherit the commits we did in the previous job. It will only look at the + # changes within this local workspace we did *now*. + - run: echo "TARGET_BRANCH=$(cat .hermes/curate/target_branch)" >> "$GITHUB_ENV" + - uses: peter-evans/create-pull-request@v5 + with: + branch: hermes/post-${{ github.run_id }} + base: ${{ env.TARGET_BRANCH }} + title: Review hermes post-processing results + body: | + This is an automated pull request created by HERMES post-processing. + + Please carefully review the changes and finally merge them into your + + # Delete all the curation branches + - run: | + for BRANCH in $(git ls-remote origin 'refs/heads/hermes/curate-*' | cut -f2 | cut -d'/' -f'3-'); do + git push origin --delete "$BRANCH" + done + # TODO: if: failure() --- delete the curation branches when the deposition failed + + + hermes-cleanup: + name: Cleanup aborted curation branches + if: github.event.pull_request.merged == false && startsWith( github.base_ref , 'hermes/curate-') + + runs-on: ubuntu-latest + permissions: + contents: write # Allow creation of new branches + pull-requests: write # Postprocessing should be able to create a pull request with changes + + steps: + - uses: actions/checkout@v3 + # Delete all the curation branches + - run: | + for BRANCH in $(git ls-remote origin 'refs/heads/hermes/curate-*' | cut -f2 | cut -d'/' -f'3-'); do + git push origin --delete "$BRANCH" + done From de92c6cf3609c819142f7db6288873e3f5e995d1 Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 12:57:11 +0200 Subject: [PATCH 21/33] Update to newer version --- CITATION.cff | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 54830b3a..6eed50bc 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -14,9 +14,9 @@ title: hermes message: >- If you use this software, please cite it using the metadata from this file. -version: "proof-of-concept" +version: 0.8.0b1 license: "Apache-2.0" -abstract: "Proof-of-concept implementation of the HERMES workflow." +abstract: "Tool to automate software publication. Not stable yet." type: software authors: - given-names: Michael @@ -49,3 +49,8 @@ authors: email: d.pape@hzdr.de affiliation: Helmholtz-Zentrum Dresden-Rossendorf (HZDR) orcid: 'https://orcid.org/0000-0002-3145-9880' + - given-names: Kernchen + family-names: Sophie + email: sophie.kernchen@dlr.de + affiliation: German Aerospace Center (DLR) + orcid: 'https://orcid.org/0009-0005-4430-6743' From 44899564733ecaac592102055d10386ee8c35791 Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 13:04:45 +0200 Subject: [PATCH 22/33] Correct variable name --- .github/workflows/hermes-zenodo-sandbox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hermes-zenodo-sandbox.yml b/.github/workflows/hermes-zenodo-sandbox.yml index 4a6c6aae..a815df40 100644 --- a/.github/workflows/hermes-zenodo-sandbox.yml +++ b/.github/workflows/hermes-zenodo-sandbox.yml @@ -102,7 +102,7 @@ jobs: # 2. Adapt the files you want to deposit. In the example, showcase.zip and README.md are deposited alongside the metadata. # 3. Check if you want to run with '--initial', as this may potentially create a completely new record (collection), # rather than a new version of the same collection! - - run: hermes deposit --initial -O invenio_rdm.auth_token ${{ secrets.ZENODO_SANDBOX }} --file hermes.zip --file README.md + - run: hermes deposit --initial -O invenio_rdm.auth_token ${{ secrets.ZENODO }} --file hermes.zip --file README.md # ADAPT # Remove this command if you don't want to do any postprocessing From 133f8b78f6317b71edd3db888103fb6e168283ca Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 13:13:31 +0200 Subject: [PATCH 23/33] Correct version --- CITATION.cff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index 6eed50bc..eea43ed5 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -14,7 +14,7 @@ title: hermes message: >- If you use this software, please cite it using the metadata from this file. -version: 0.8.0b1 +version: 0.8.1b1 license: "Apache-2.0" abstract: "Tool to automate software publication. Not stable yet." type: software From 1b3b140d99d8278f15bb3f4dfb2ce9676373ded6 Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 13:22:55 +0200 Subject: [PATCH 24/33] Push on Zenodo not sandbox --- .../{hermes-zenodo-sandbox.yml => hermes-zenodo.yml} | 6 ++---- hermes.toml | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) rename .github/workflows/{hermes-zenodo-sandbox.yml => hermes-zenodo.yml} (98%) diff --git a/.github/workflows/hermes-zenodo-sandbox.yml b/.github/workflows/hermes-zenodo.yml similarity index 98% rename from .github/workflows/hermes-zenodo-sandbox.yml rename to .github/workflows/hermes-zenodo.yml index a815df40..2db5ae99 100644 --- a/.github/workflows/hermes-zenodo-sandbox.yml +++ b/.github/workflows/hermes-zenodo.yml @@ -8,10 +8,8 @@ name: Software publication on Zenodo Sandbox on: push: - branches: - - feature/261-put-hermes-on-zenodo - #tags: - # - "*" + tags: + - "*" # NOTE: Do not delete the trigger on closed pull requests, the HERMES workflow needs this. pull_request: diff --git a/hermes.toml b/hermes.toml index 8884aea4..e2ea3690 100644 --- a/hermes.toml +++ b/hermes.toml @@ -12,7 +12,7 @@ enable_validation = false target = "invenio_rdm" [deposit.invenio_rdm] -site_url = "https://sandbox.zenodo.org" +site_url = "https://zenodo.org" communities = [] access_right = "open" From 9f8ef98e76773ec3647509e78d8e96c0b1bb7010 Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 13:24:51 +0200 Subject: [PATCH 25/33] Change name --- .github/workflows/hermes-zenodo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hermes-zenodo.yml b/.github/workflows/hermes-zenodo.yml index 2db5ae99..46cf0390 100644 --- a/.github/workflows/hermes-zenodo.yml +++ b/.github/workflows/hermes-zenodo.yml @@ -4,7 +4,7 @@ # SPDX-FileContributor: Stephan Druskat -name: Software publication on Zenodo Sandbox +name: Software publication on Zenodo on: push: From 4f27bdd3f6b0829c2dc58ec5e92265fc750d47b5 Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 14:02:43 +0200 Subject: [PATCH 26/33] Adapt for release tag not branch --- .github/workflows/hermes-zenodo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hermes-zenodo.yml b/.github/workflows/hermes-zenodo.yml index 46cf0390..402c5944 100644 --- a/.github/workflows/hermes-zenodo.yml +++ b/.github/workflows/hermes-zenodo.yml @@ -56,7 +56,7 @@ jobs: echo "SHORT_SHA=$SHORT_SHA" >> "$GITHUB_ENV" # Create a curation branch - git branch -c "hermes/curate-$SHORT_SHA" + git switch -c "hermes/curate-$SHORT_SHA" git push origin "hermes/curate-$SHORT_SHA" # Explicitly add to-be-curated metadata (which is ignored via .gitignore!) From ad75da370a5d222b452b236b24ad9314367b5010 Mon Sep 17 00:00:00 2001 From: "Kernchen, Sophie" Date: Mon, 5 Aug 2024 14:17:38 +0200 Subject: [PATCH 27/33] Add record id and DOI --- CITATION.cff | 4 ++++ hermes.toml | 1 + 2 files changed, 5 insertions(+) diff --git a/CITATION.cff b/CITATION.cff index eea43ed5..36db497f 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -54,3 +54,7 @@ authors: email: sophie.kernchen@dlr.de affiliation: German Aerospace Center (DLR) orcid: 'https://orcid.org/0009-0005-4430-6743' +identifiers: + - type: doi + value: 10.5281/zenodo.13221384 + description: Version 0.8.1b1 diff --git a/hermes.toml b/hermes.toml index e2ea3690..f455746a 100644 --- a/hermes.toml +++ b/hermes.toml @@ -15,6 +15,7 @@ target = "invenio_rdm" site_url = "https://zenodo.org" communities = [] access_right = "open" +record_id = 13221384 [deposit.invenio_rdm.api_paths] depositions = "api/deposit/depositions" From 1c85dec4e681c63f242d6f3d0bd3243e4a0ec64b Mon Sep 17 00:00:00 2001 From: "Meinel, Michael" Date: Tue, 6 Aug 2024 15:09:31 +0200 Subject: [PATCH 28/33] Bump version of hermes --- hermes.toml | 3 --- pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/hermes.toml b/hermes.toml index f455746a..fd743ce4 100644 --- a/hermes.toml +++ b/hermes.toml @@ -5,9 +5,6 @@ [harvest] sources = [ "cff" ] # ordered priority (first one is most important) -[harvest.cff] -enable_validation = false - [deposit] target = "invenio_rdm" diff --git a/pyproject.toml b/pyproject.toml index 41bad489..8dd98b12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ [tool.poetry] # Reference at https://python-poetry.org/docs/pyproject/ name = "hermes" -version = "0.1.0" +version = "0.8.1" description = "Workflow to publish research software with rich metadata" homepage = "https://software-metadata.pub" license = "Apache-2.0" From a3164e9565b643571ff5c6122defe640ec8ee6ce Mon Sep 17 00:00:00 2001 From: "Meinel, Michael" Date: Tue, 6 Aug 2024 15:11:38 +0200 Subject: [PATCH 29/33] Remove old settings handler --- src/hermes/settings.py | 44 ------------------------------------------ 1 file changed, 44 deletions(-) delete mode 100644 src/hermes/settings.py diff --git a/src/hermes/settings.py b/src/hermes/settings.py deleted file mode 100644 index fbae7b92..00000000 --- a/src/hermes/settings.py +++ /dev/null @@ -1,44 +0,0 @@ -# SPDX-FileCopyrightText: 2024 German Aerospace Center (DLR) -# -# SPDX-License-Identifier: Apache-2.0 - -# SPDX-FileContributor: Sophie Kernchen -# SPDX-FileContributor: Michael Meinel - -import pathlib -from typing import Any, Dict, Tuple - -import toml -from pydantic.fields import FieldInfo -from pydantic_settings import PydanticBaseSettingsSource - - -class TomlConfigSettingsSource(PydanticBaseSettingsSource): - """ - A simple settings source class that loads variables from a TOML file - at the project's root. - - Here we happen to choose to use the `env_file_encoding` from Config - when reading `config.json` - """ - def __init__(self, settings_cls, config_path): - super().__init__(settings_cls) - self.__config_path = config_path - - def get_field_value(self, field: FieldInfo, field_name: str) -> Tuple[Any, str, bool]: - file_content_toml = toml.load(pathlib.Path(self.__config_path)) - field_value = file_content_toml.get(field_name) - return field_value, field_name, False - - def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any: - return value - - def __call__(self) -> Dict[str, Any]: - d: Dict[str, Any] = {} - for field_name, field in self.settings_cls.model_fields.items(): - field_value, field_key, value_is_complex = self.get_field_value(field, field_name) - field_value = self.prepare_field_value(field_name, field, field_value, value_is_complex) - if field_value is not None: - d[field_key] = field_value - - return d From 9e40d7e46d23ed7b21eeb13809677a645d66f14d Mon Sep 17 00:00:00 2001 From: "Meinel, Michael" Date: Tue, 6 Aug 2024 15:12:04 +0200 Subject: [PATCH 30/33] Add missing author to Python package info --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8dd98b12..42a4747b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ authors = [ "Jeffrey Kelling ", "Oliver Knodel ", "David Pape ", + "Sophie Kernchen ", ] readme = "README.md" From a418fec5d97cbf19c363c04dc4e960bc74161aae Mon Sep 17 00:00:00 2001 From: "Meinel, Michael" Date: Tue, 13 Aug 2024 13:54:40 +0200 Subject: [PATCH 31/33] Add log output to ignored files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d7c36ebe..5856425f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ dist/ # HERMES workflow specifics .hermes/ +hermes.log From 797b0ca3678aa28c810a32600e35d3021094d8ca Mon Sep 17 00:00:00 2001 From: "Meinel, Michael" Date: Tue, 13 Aug 2024 13:57:06 +0200 Subject: [PATCH 32/33] Enable harvesting from pyproject.toml --- .github/workflows/hermes-zenodo.yml | 2 +- hermes.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/hermes-zenodo.yml b/.github/workflows/hermes-zenodo.yml index 402c5944..302fdeb0 100644 --- a/.github/workflows/hermes-zenodo.yml +++ b/.github/workflows/hermes-zenodo.yml @@ -42,7 +42,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.10' - - run: pip install hermes + - run: pip install hermes hermes-plugin-python - run: hermes harvest - run: hermes process - run: hermes curate diff --git a/hermes.toml b/hermes.toml index fd743ce4..3aa44a8f 100644 --- a/hermes.toml +++ b/hermes.toml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: CC0-1.0 [harvest] -sources = [ "cff" ] # ordered priority (first one is most important) +sources = [ "cff", "toml" ] # ordered priority (first one is most important) [deposit] target = "invenio_rdm" From 92a1da57cba3935d45650176378e5680463d5b3d Mon Sep 17 00:00:00 2001 From: "Meinel, Michael" Date: Tue, 13 Aug 2024 13:57:38 +0200 Subject: [PATCH 33/33] Bump version number --- CITATION.cff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index 36db497f..dfb7efe6 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -14,7 +14,7 @@ title: hermes message: >- If you use this software, please cite it using the metadata from this file. -version: 0.8.1b1 +version: 0.8.1 license: "Apache-2.0" abstract: "Tool to automate software publication. Not stable yet." type: software