Merge branch 'main' of https://github.com/HakaiInstitute/hakai-metada…

…ta-conversion

.github/workflows/run-tests.yaml

@@ -17,7 +17,13 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-
+        with: 
+          path: "."
+      - name: Checkout hakai metadata filesxz
+        uses: actions/checkout@v4
+        with:
+          repository: HakaiInstitute/hakai-metadata-entry-form-files
+          path: ./tests/records/hakai-metadata-entry-form-files
       - name: Set up Python 3.11
         uses: actions/setup-python@v5
         with:
@@ -34,8 +40,9 @@ jobs:
       - name: Run pytest
         run: poetry run pytest
 
-      - name: Convert test record to CITATION.cff
+      - name: Convert test record to CITATION.cff if main branch
         uses: hakaiinstitute/hakai-metadata-conversion@main
+        if: github.ref == 'refs/heads/main'
         with:
           input: tests/records/test_record1.json
           output-file: CITATION.cff

.gitignore

@@ -1,3 +1,4 @@
 **/__pycache__
 .DS_Store
-tests/records/hakai-metadata-entry-form-files
+tests/records/hakai-metadata-entry-form-files
+tests/results

README.md

@@ -58,7 +58,7 @@ job:
         step:
             - use: action/checkout@v4
             - name: Sync metadata
-              use: hakaiinsitute/hakai-metadata-conversion
+              use: hakaiinstitute/hakai-metadata-conversion
               with:
                 - input: url or file path within repo
                 - output-file: CITATION.cff

action.yaml

@@ -67,7 +67,7 @@ runs:
     - name: Run conversion
       shell: bash
       run: |
-        hakai_metadata_conversion \
+        hakai_metadata_conversion convert \
         --input ${{ inputs.input }} \
         --output-dir ${{ inputs.output-dir }} \
         --output-file ${{ inputs.output-file }} \

hakai_metadata_conversion/__main__.py

@@ -39,7 +39,7 @@ def load(input, format, encoding="utf-8") -> dict:
         return input_formats[format](f)
 
 
-def convert(record, format) -> str:
+def converter(record, format) -> str:
     """Run the conversion to the desired format."""
     if format == "json":
         return json.dumps(record, indent=2)
@@ -53,7 +53,18 @@ def convert(record, format) -> str:
         raise ValueError(f"Unknown output format: {format}")
 
 
-@click.command()
+@click.group(name="hakai-metadata-conversion")
+def cli():
+    """Hakai Metadata Conversion CLI.
+    Convert metadata records to different metadata formats or standards.
+    """
+    pass
+
+
+cli.add_command(erddap.update, name="erddap-update")
+
+
+@cli.command(name="convert")
 @click.option("--input", "-i", required=True, help="Input file.")
 @click.option(
     "--recursive", "-r", is_flag=True, help="Process files recursively.", default=False
@@ -100,20 +111,21 @@ def convert(record, format) -> str:
     show_default=True,
 )
 @logger.catch(reraise=True)
-def cli_main(**kwargs):
+def cli_convert(**kwargs):
     """Convert metadata records to different metadata formats or standards."""
-    main(**kwargs)
+    convert(**kwargs)
 
 
-def main(
+@logger.catch(reraise=True)
+def convert(
     input,
-    recursive,
-    input_file_format,
-    encoding,
-    output_dir,
-    output_file,
-    output_format,
-    output_encoding,
+    output_format: str,
+    recursive: bool = False,
+    input_file_format: str = "yaml",
+    encoding: str = "utf-8",
+    output_dir: str = ".",
+    output_file: str = None,
+    output_encoding: str = "utf-8",
 ):
     """Convert metadata records to different metadata formats or standards."""
 
@@ -142,7 +154,7 @@ def main(
             continue
 
         logger.debug(f"Converting to {output_format}")
-        converted_record = convert(record, output_format)
+        converted_record = converter(record, output_format)
 
         # Generate output file path
         if output_dir and not output_file:
@@ -163,4 +175,4 @@ def main(
 
 
 if __name__ == "__main__":
-    cli_main()
+    cli()

hakai_metadata_conversion/citation_cff.py

@@ -10,6 +10,15 @@
 from hakai_metadata_conversion.utils import drop_empty_values
 
 
+def _get_placeholder(language):
+    if language == "en":
+        return "Not available"
+    elif language == "fr":
+        return "Non disponible"
+    else:
+        return "Not available"
+
+
 def _get_country_code(country_name):
     if not country_name:
         return None
@@ -71,6 +80,58 @@ def get_cff_contact(contact):
     )
 
 
+def _get_doi(record):
+    if not record["identification"].get("identifier", ""):
+        return []
+    return [
+        {
+            "description": "DOI",
+            "type": "doi",
+            "value": (
+                record["identification"]["identifier"].replace("https://doi.org/", "")
+                if "doi.org" in record["identification"].get("identifier", "")
+                else None
+            ),
+        }
+    ]
+
+
+def _get_ressources(record, language):
+    ressources = []
+    for distribution in record["distribution"]:
+        if not distribution["url"].startswith("http"):
+            logger.warning(f"Invalid URL: {distribution['url']}")
+            continue
+        ressources.append(
+            {
+                "description": ": ".join(
+                    [
+                        item
+                        for item in [
+                            distribution.get("name", {}).get(language, ""),
+                            distribution.get("description", {}).get(
+                                language, _get_placeholder(language)
+                            ),
+                        ]
+                        if item
+                    ]
+                ),
+                "type": "url",
+                "value": distribution["url"],
+            }
+        )
+    return ressources
+
+
+def _get_unique_authors(record):
+    authors = []
+    for author in record["contact"]:
+        contact = get_cff_contact(author)
+        if contact not in authors:
+            authors.append(contact)
+    return authors
+
+
 def citation_cff(
     record,
     output_format="yaml",
@@ -90,14 +151,11 @@ def citation_cff(
         + "_"
         + record["metadata"]["identifier"]
     )
+
     record = {
         "cff-version": "1.2.0",
         "message": message,
-        "authors": [
-            get_cff_contact(contact)
-            for contact in record["contact"]
-            if contact["inCitation"]
-        ],
+        "authors": _get_unique_authors(record),
         "title": record["identification"]["title"].get(language),
         "abstract": record["identification"]["abstract"].get(language),
         "date-released": record["metadata"]["dates"]["revision"].split("T")[0],
@@ -113,52 +171,29 @@ def citation_cff(
                 "value": record["metadata"]["identifier"],
             },
             {
-                "description": "Hakai Metadata record URL",
+                "description": "Metadata record URL",
                 "type": "url",
                 "value": resource_url,
             },
+            *_get_doi(record),
             {
-                "description": "Hakai Metadata record DOI",
-                "type": "doi",
-                "value": (
-                    record["identification"]["identifier"].replace(
-                        "https://doi.org/", ""
-                    )
-                    if "doi.org" in record["identification"].get("identifier", "")
-                    else None
-                ),
-            },
-            {
-                "description": "Hakai Metadata Form used to generate this record",
+                "description": "Metadata Form used to generate this record",
                 "type": "url",
                 "value": record["metadata"]["maintenance_note"].replace(
                     "Generated from ", ""
                 ),
             },
-            # Generate ressources links
-            *[
-                {
-                    "description": ": ".join(
-                        [
-                            item
-                            for item in [
-                                distribution.get("name", {}).get(language, ""),
-                                distribution.get("description", {}).get(language),
-                            ]
-                            if item
-                        ]
-                    ),
-                    "type": "url",
-                    "value": distribution["url"],
-                }
-                for distribution in record["distribution"]
-            ],
-        ],
-        "keywords": [
-            keyword
-            for _, group in record["identification"]["keywords"].items()
-            for keyword in group.get(language, [])
+            *_get_ressources(record, language=language),
         ],
+        "keywords": list(
+            set(
+                [
+                    keyword
+                    for _, group in record["identification"]["keywords"].items()
+                    for keyword in group.get(language, [])
+                ]
+            )
+        ),
         "license": record["metadata"]["use_constraints"].get("licence", {}).get("code"),
         "license-url": record["metadata"]["use_constraints"]
         .get("licence", {})