Merge branch 'develop' into trivy-scan-sapbert

.github/workflows/build-push-dev-image.yml

@@ -48,20 +48,20 @@ jobs:
     # https://github.com/marketplace/actions/build-and-push-docker-images
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
       with:
         driver-opts: |
           network=host
 
     - name: Login to DockerHub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
         logout: true
 
     - name: Login to Container Registry
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: containers.renci.org
         username: ${{ secrets.CONTAINERHUB_USERNAME }}
@@ -72,7 +72,7 @@ jobs:
     # Notes on Cache: 
     # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
     - name: Build Push Container
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: .
         push: true

.github/workflows/build-push-release.yml

@@ -63,20 +63,20 @@ jobs:
     # step
     # https://github.com/marketplace/actions/build-and-push-docker-images
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
       with:
         driver-opts: |
           network=host
 
     - name: Login to DockerHub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
         logout: true
 
     - name: Login to Container Registry
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: containers.renci.org
         username: ${{ secrets.CONTAINERHUB_USERNAME }}
@@ -86,7 +86,7 @@ jobs:
     # Notes on Cache: 
     # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
     - name: Build Push Container
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         push: true
         # Push to renci-registry and dockerhub here.

.github/workflows/code-checks.yml

@@ -1,23 +1,27 @@
 # Workflow responsible for core acceptance testing.
 # Tests Currently Run:
 #     - flake8-linter
-#     - image-build-test
-#    
-# This workflow only validates images can build
-# but does not push images to any repository.
-#
+#     - PYTest
+#     - Bandit
+# For PR Vulnerability Scanning a separate workflow will run.
 # The build-push-dev-image and build-push-release workflows 
 # handle the develop and release image storage respectively.
 #
 #
 
 name: Code-Checks
 on:
-  push:
-    branches-ignore:
-      - master
-      - main
+  # push:
+  #   branches-ignore:
+  #     - master
+  #     - main
+  #     - develop
+  pull_request:
+    branches:
       - develop
+      - master
+      - main 
+    types: [ opened, synchronize ]
     paths-ignore:
       - README.md
       - .old_cicd/*
@@ -27,13 +31,6 @@ on:
       - .gitignore
       - .dockerignore
       - .githooks
-  pull_request:
-    branches:
-      - develop
-      - master
-      - main 
-    types: [ opened, synchronize ]
-
 
 jobs:
 ############################## flake8-linter ##############################
@@ -69,35 +66,45 @@ jobs:
       # flake8 --ignore=E,W --exit-zero . 
       continue-on-error: true
 
-############################## test-image-build ##############################
-  test-image-build:
-    # needs: flake8-linter
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
+# ############################## build-vuln-test ##############################
+  # build-vuln-test:
+  #   # needs: flake8-linter
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #   - uses: actions/checkout@v3
 
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
-      with:
-        driver-opts: |
-          network=host
+  #   - name: Set up Docker Buildx
+  #     uses: docker/setup-buildx-action@v3
+  #     with:
+  #       driver-opts: |
+  #         network=host
 
-    - name: Login to DockerHub
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-        logout: true
+  #   - name: Login to DockerHub
+  #     uses: docker/login-action@v3
+  #     with:
+  #       username: ${{ secrets.DOCKERHUB_USERNAME }}
+  #       password: ${{ secrets.DOCKERHUB_TOKEN }}
+  #       logout: true
+
+  #   # Notes on Cache: 
+  #   # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
+  #   - name: Build Container
+  #     uses: docker/build-push-action@v5
+  #     with:
+  #       context: .
+  #       push: false
+  #       load: true
+  #       tag: ${{ github.repository }}:vuln-test
+  #       cache-from: type=registry,ref=${{ github.repository }}:buildcache
+  #       cache-to: type=registry,ref=${{ github.repository }}:buildcache,mode=max
+  #   ####### Run for Fidelity ######
+  #   - name: Run Trivy vulnerability scanner
+  #     uses: aquasecurity/trivy-action@master
+  #     with:
+  #       image-ref: '${{ github.repository }}:vuln-test'
+  #       severity: 'CRITICAL,HIGH'
+  #       exit-code: '1'
 
-    # Notes on Cache: 
-    # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
-    - name: Build Container
-      uses: docker/build-push-action@v4
-      with:
-        context: .
-        push: false
-        cache-from: type=registry,ref=${{ github.repository }}:buildcache
-        cache-to: type=registry,ref=${{ github.repository }}:buildcache,mode=max
 ################################### PYTEST ###################################
   pytest:
     runs-on: ubuntu-latest

.github/workflows/trivy-pr-scan.yml

@@ -0,0 +1,67 @@
+
+name: trivy-pr-scan 
+on:
+  pull_request:
+    branches:
+      - develop
+      - master
+      - main 
+    types: [ opened, synchronize ]
+    paths-ignore:
+    - README.md
+    - .old_cicd/*
+    - .github/*
+    - .github/workflows/*
+    - LICENSE
+    - .gitignore
+    - .dockerignore
+    - .githooks
+
+jobs:
+ trivy-pr-scan:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      with:
+        driver-opts: |
+          network=host
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+        logout: true
+
+    # Notes on Cache: 
+    # https://docs.docker.com/build/ci/github-actions/examples/#inline-cache
+    - name: Build Container
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        push: false
+        load: true
+        tags: ${{ github.repository }}:vuln-test
+        cache-from: type=registry,ref=${{ github.repository }}:buildcache
+        cache-to: type=registry,ref=${{ github.repository }}:buildcache,mode=max
+
+    # We will not be concerned with Medium and Low vulnerabilities
+    - name: Run Trivy vulnerability scanner
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: '${{ github.repository }}:vuln-test'
+        format: 'sarif'
+        severity: 'CRITICAL,HIGH'
+        output: 'trivy-results.sarif'
+        exit-code: '1'
+    # Scan results should be viewable in GitHub Security Dashboard
+    # We still fail the job if results are found, so below will always run
+    # unless manually canceled.
+    - name: Upload Trivy scan results to GitHub Security tab
+      uses: github/codeql-action/upload-sarif@v2
+      if: '!cancelled()'
+      with:
+        sarif_file: 'trivy-results.sarif'

README.md

@@ -290,3 +290,8 @@ TOPMed phenotypic concept data is [here](https://github.com/helxplatform/dug/tre
 ## Release
 
 To release, commit the change and select feature.
+
+#### Fail on Vulnerability Detection
+
+During PR's several vulnerability scanners are run. If there are vulnerabilities detected, the pr checks will fail and a report will be sent to Github Security Dashboard for viewing. Please ensure the vulnerability is mitigated prior to continuing the merge to protected branches.
+

bin/vlmd_to_dbgap_xml.py

@@ -161,10 +161,12 @@ def vlmd_to_dbgap_xml(input_file, output, file_format, study_id, appl_id, study_
                 # description later if that is useful.
                 if row.get('constraints.pattern'):
                     counters['constraints.pattern'] += 1
-                    logging.warning(f"`constraints.pattern` of {row['constraints.pattern']} found in row {row_index}, skipped.")
+                    logging.warning(f"`constraints.pattern` of {row['constraints.pattern']} found in row {row_index}, "
+                                    f"but pattern constraints are not currently being written.")
                 if row.get('format'):
                     counters['format'] += 1
-                    logging.warning(f"Found `format` of {row['format']} found in row {row_index}, skipped.")
+                    logging.warning(f"Found `format` of {row['format']} found in row {row_index}, but format is not "
+                                    f"currently being written.")
 
                 # Process enumerated and encoded values.
                 encs = {}

requirements.txt

@@ -24,5 +24,6 @@ six
 # We use Click 7.0 because that's what one of the pinned packages above use.
 click
 httpx
+linkml-runtime==1.6.0
 bmt==1.1.0
 urllib3

src/dug/config.py

@@ -83,7 +83,7 @@ class Config:
                 "desc": "summary",
                 "collection_name": "cde_category",
                 "collection_id":  "cde_category",
-                "collection_action": "files"
+                "action": "files"
             }
         }
     })

src/dug/core/async_search.py

@@ -651,6 +651,7 @@ async def search_vars_unscored(self, concept="", query="",
                 new_results = new_results[data_type]
             else:
                 new_results = {}
+        new_results.update({'total_items': total_items['count']})
         return new_results
 
     async def search_kg(self, unique_id, query, offset=0, size=None,

src/dug/core/crawler.py

@@ -264,7 +264,7 @@ def expand_to_dug_element(self,
                                 for key in attribute_mapping:
                                     mapped_value = node.get(attribute_mapping[key], "")
                                     # treat all attributes as strings 
-                                    if key in array_to_string and isinstance(mapped_value, list) and len(mapped_value) > 0:
+                                    if attribute_mapping[key] in array_to_string and isinstance(mapped_value, list) and len(mapped_value) > 0:
                                         mapped_value = mapped_value[0]
                                     element_attribute_args.update({key: mapped_value})
                                 element = DugElement(

src/dug/core/tranql.py

@@ -113,11 +113,14 @@ def get_node_names(self, include_curie=True):
         return node_names
 
     def get_node_synonyms(self, include_curie=True):
+        # @TODO call name-resolver 
         node_synonyms = []
         curie_ids = self.get_curie_ids()
         for node in self.get_nodes():
             if include_curie or node['id'] not in curie_ids:
-                node_synonyms += node.get('synonyms') or []
+                syn = node.get('synonyms') 
+                if isinstance(syn,list):
+                    node_synonyms +=  syn 
         return node_synonyms
 
     def get_curie_ids(self):

src/dug/server.py

@@ -3,6 +3,7 @@
 import uvicorn
 
 from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
 from dug.config import Config
 from dug.core.async_search import Search
 from pydantic import BaseModel
@@ -15,6 +16,13 @@
     root_path=os.environ.get("ROOT_PATH", "/"),
 )
 
+APP.add_middleware(
+    CORSMiddleware,
+    allow_origins=['*'],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
 
 class GetFromIndex(BaseModel):
     index: str = "concepts_index"