generate directive (#3)

* Use generate directive

* Readme update

* Temp commit

.github/workflows/pythonpublish.yml

@@ -4,13 +4,12 @@ on:
   push:
     branches:
       - main
-      - feature/*
     tags:
       - 'v*'
   workflow_dispatch:
 
 env:
-  REGISTRY: ghcr.iopy
+  REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
@@ -32,6 +31,7 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/')
         run: |
           poetry install --no-cache --without dev
+          poetry run pytest
           poetry publish --build --username $PYPI_USERNAME --password $PYPI_PASSWORD
         env:
           PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}

README.md

@@ -70,3 +70,7 @@ curl -F 'file=@/tmp/bom.json' http://0.0.0.0:9000/import -H "Authorization: Bear
 ## Cloud Setup
 
 Refer to the instructions under [contrib](contrib/microk8s/INSTALL.md) to setup a microk8s cluster with threat-db and dgraph.
+
+## Discord support
+
+The developers could be reached via the [discord](https://discord.gg/DCNxzaeUpd) channel.

contrib/microk8s/INSTALL.md

@@ -186,7 +186,7 @@ dev-db-dgraph-zero-2    1/1     Running   0          29m
 dev-db-dgraph-alpha-2   1/1     Running   0          29m
 ```
 
-````
+```
 $ microk8s kubectl get service
 NAME                           TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)             AGE
 kubernetes                     ClusterIP   10.152.183.1     <none>        443/TCP             52m
@@ -397,4 +397,4 @@ Check pod limits
 microk8s kubectl get pods dev-db-dgraph-zero-0 -o jsonpath='{range .spec.containers[*]}{"Container Name: "}{.name}{"\n"}{"Requests:"}{.resources.requests}{"\n"}{"Limits:"}{.resources.limits}{"\n"}{end}'
 ```
 
-````
+

pyproject.toml

@@ -15,7 +15,6 @@ classifiers = [
     "Intended Audience :: System Administrators",
     "Topic :: Utilities",
     "Topic :: Security",
-    "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",

queries-graphql.md

@@ -1,3 +1,32 @@
+
+## Aggregate queries
+
+Count of BoM, Component and Vulnerability
+
+```
+query {
+  aggregateBom {
+    count
+  }
+}
+```
+
+```
+query {
+  aggregateComponent {
+    count
+  }
+}
+```
+
+```
+query {
+  aggregateVulnerability {
+    count
+  }
+}
+```
+
 # Get component by purl
 
 ```

tests/test_loader.py

@@ -0,0 +1,31 @@
+import os
+
+import threat_db.loader as loader
+
+
+def test_get_pkg_vulns():
+    parsed_obj = loader.get_pkg_vulns_json(
+        os.path.join(
+            os.path.dirname(__file__), "data", "NodeGoat", "sbom-nodejs.vex.json"
+        )
+    )
+    assert (
+        parsed_obj["serial_number"] == "urn:uuid:25223366-1379-41f4-8abf-3a4e11d6548f"
+    )
+    assert parsed_obj["metadata"]
+    assert len(parsed_obj["components"]) == 1080
+    assert len(parsed_obj["services"]) == 0
+    assert len(parsed_obj["vulnerabilities"]) == 99
+
+    parsed_obj = loader.get_pkg_vulns_json(
+        os.path.join(
+            os.path.dirname(__file__), "data", "NodeGoat", "sbom-yaml-manifest.vex.json"
+        )
+    )
+    assert (
+        parsed_obj["serial_number"] == "urn:uuid:1a1ffcb3-2c26-43af-8032-d312627ab9f8"
+    )
+    assert parsed_obj["metadata"]
+    assert len(parsed_obj["components"]) == 207
+    assert len(parsed_obj["services"]) == 2
+    assert len(parsed_obj["vulnerabilities"]) == 11

threat_db/base-schema.graphql

@@ -31,6 +31,10 @@ List of roles
 """
     roles: [UserRole]
 """
+Is this a bot user for automation?
+"""
+    isBot: Boolean
+"""
 Is this user account disabled?
 """
     disabled: Boolean
@@ -115,7 +119,13 @@ Is this role disabled?
 """
 Results from a scan
 """
-type ScanResult @withSubscription {
+type ScanResult @generate(
+    mutation: {
+        add: true,
+        delete: false
+    },
+    subscription: true
+) {
 """
 Unique ID such as UUID or ULID
 """
@@ -189,7 +199,13 @@ Direct pdf report url for this scan result
 """
 Scan of an application
 """
-type Scan {
+type Scan @generate(
+    mutation: {
+        add: true,
+        delete: false
+    },
+    subscription: false
+) {
 """
 Unique ID such as UUID or ULID
 """
@@ -287,7 +303,13 @@ VCS Commit Hash
 """
 Represents an application
 """
-type Application @withSubscription {
+type Application @generate(
+    mutation: {
+        add: true,
+        delete: false
+    },
+    subscription: true
+) {
 """
 Unique ID such as UUID or ULID
 """
@@ -353,6 +375,10 @@ Modified date
 """
     modified: DateTime! @search
 """
+For application groups, there could be dependent sub-applications
+"""
+    subApplications: [Application]
+"""
 List of teams
 """
     teams: [Team]
@@ -572,7 +598,13 @@ List of properties
 """
 Bom type
 """
-type Bom {
+type Bom @generate(
+    mutation: {
+        add: true,
+        delete: false
+    },
+    subscription: false
+) {
 """
 Serial number
 """
@@ -805,7 +837,9 @@ Details for the state
 """
 Vulnerability type
 """
-type Vulnerability @withSubscription {
+type Vulnerability @generate(
+    subscription: true
+) {
 """
 Unique reference within a BoM
 """

threat_db/loader.py

@@ -32,18 +32,18 @@ def cleanup_license_string(license_str):
 def get_pkg_vulns_json(jsonfile):
     """Method to extract packages from a bom json file"""
     if not os.path.exists(jsonfile):
-        return None, None, None, None
+        return {}
     with open(jsonfile) as fp:
         try:
             bom_data = orjson.loads(fp.read())
             if bom_data:
                 return get_pkg_vulns_from_bom(bom_data)
         except Exception:
-            return None, None, None, None
+            return {}
 
 
 def get_pkg_vulns_from_bom(bom_data):
-    pkgs = []
+    components = []
     serial_number = None
     metadata = {}
     metadata = bom_data.get("metadata")
@@ -94,7 +94,7 @@ def get_pkg_vulns_from_bom(bom_data):
                 "appearsIn": [{"serialNumber": serial_number}],
             }
             del fcomp["bom-ref"]
-            pkgs.append(fcomp)
+            components.append(fcomp)
         for avuln in bom_data.get("vulnerabilities", []):
             bomRef = avuln.get("bom-ref")
             if added_vkeys.get(bomRef):
@@ -128,7 +128,14 @@ def get_pkg_vulns_from_bom(bom_data):
             del fvuln["bom-ref"]
             vulnerabilities.append(fvuln)
             added_vkeys[bomRef] = True
-    return (serial_number, metadata, pkgs, vulnerabilities)
+    return {
+        "serial_number": serial_number,
+        "metadata": metadata,
+        "components": components,
+        "services": bom_data.get("services", []),
+        "vulnerabilities": vulnerabilities,
+        "bom_data": bom_data,
+    }
 
 
 def process_vex(client, data_dir, remove_on_success=False):
@@ -140,23 +147,20 @@ def process_vex(client, data_dir, remove_on_success=False):
 
 
 def process_vex_file(client, jsonf):
+    parsed_obj = {}
     if isinstance(jsonf, SpooledTemporaryFile):
         try:
-            (
-                serial_number,
-                metadata,
-                components,
-                vulnerabilities,
-            ) = get_pkg_vulns_from_bom(orjson.loads(jsonf.read()))
+            parsed_obj = get_pkg_vulns_from_bom(orjson.loads(jsonf.read()))
         except Exception as ex:
             LOG.warn("Exception while converting to json from tempfile")
             LOG.exception(ex)
             return False
     else:
         LOG.debug(f"Processing {jsonf}")
-        (serial_number, metadata, components, vulnerabilities) = get_pkg_vulns_json(
-            jsonf
-        )
+        parsed_obj = get_pkg_vulns_json(jsonf)
+    serial_number = parsed_obj.get("serial_number")
+    components = parsed_obj.get("components")
+    metadata = parsed_obj.get("metadata")
     if serial_number and components:
         LOG.info(f"Creating Bom with {len(components)} components from {jsonf}")
         root_component = metadata.get("component", None)
@@ -175,7 +179,8 @@ def process_vex_file(client, jsonf):
                         "component": root_component,
                     },
                     "components": components,
-                    "vulnerabilities": vulnerabilities,
+                    "services": parsed_obj.get("services"),
+                    "vulnerabilities": parsed_obj.get("vulnerabilities"),
                 }
             ],
         )