Merge pull request #155 from swimlane/2_7_0_release

* Added comments List field to `Actor` model
* Added `last_updated` timestamp to support new data generation process
* Updated depdencies and CI
* Added Splunks Security Content repo data for detections into dataset

.github/workflows/generatepyattckdata.yml

@@ -33,31 +33,30 @@ jobs:
       uses: abatilo/[email protected]
       with:
         poetry-version: ${{ matrix.poetry-version }}
-    - name: Install dependencies
-      run: |
-        pip install -r requirements.txt
     - name: Generate new data_collector data
       run: |
         cd data_collector
         poetry install
+        poetry install pip install -r ../requirements.txt
         poetry run python run.py
-    - name: Generate ATTCK Data
-      env:
-        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        cd ..
-        poetry run pip install --upgrade pip
-        poetry run pip install PyGithub>=1.55
-        poetry run pip install deep-translator
-        poetry install
-        poetry run python run.py
+    # - name: Generate ATTCK Data
+    #   env:
+    #     GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    #   run: |
+    #     cd ..
+    #     poetry run pip install --upgrade pip
+    #     poetry run pip install PyGithub>=1.55
+    #     poetry run pip install deep-translator
+    #     poetry install
+    #     poetry run python run.py
     - name: Configure AWS Credentials
       uses: aws-actions/configure-aws-credentials@v1
       with:
         role-to-assume: ${{ secrets.SWIMLANE_PYATTCK_ROLE }}
         aws-region: us-west-2
     - name: Upload files to S3 with AWS CLI
       run: |
+        cd ..
         aws s3 cp generated_attck_data.json s3://${{ secrets.SWIMLANE_PYATTCK_AWS_BUCKET_NAME }} --acl public-read
         aws s3 cp generated_attck_data_v2.json s3://${{ secrets.SWIMLANE_PYATTCK_AWS_BUCKET_NAME }} --acl public-read
         aws s3 cp attck_to_nist_controls.json s3://${{ secrets.SWIMLANE_PYATTCK_AWS_BUCKET_NAME }} --acl public-read

.github/workflows/tests.yml

@@ -22,16 +22,16 @@ jobs:
 
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v4.5.0
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
 
       - name: Upgrade pip
         run: |
-          pip install --constraint=.github/workflows/constraints.txt pip
+          pip install --upgrade pip
           pip --version
 
       - name: Upgrade pip in virtual environments
@@ -45,13 +45,13 @@ jobs:
 
       - name: Install Poetry
         run: |
-          pipx install --pip-args=--constraint=.github/workflows/constraints.txt poetry
+          pipx install poetry
           poetry --version
 
       - name: Install Nox
         run: |
-          pipx install --pip-args=--constraint=.github/workflows/constraints.txt nox
-          pipx inject --pip-args=--constraint=.github/workflows/constraints.txt nox nox-poetry
+          pipx install nox
+          pipx inject nox nox-poetry
           nox --version
 
       - name: Compute pre-commit cache key
@@ -70,7 +70,7 @@ jobs:
           print("::set-output name=result::{}".format(result))
 
       - name: Restore pre-commit cache
-        uses: actions/cache@v3.3.1
+        uses: actions/cache@v3
         if: matrix.session == 'pre-commit'
         with:
           path: ~/.cache/pre-commit
@@ -101,27 +101,27 @@ jobs:
     needs: tests
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3
 
       - name: Set up Python
-        uses: actions/setup-python@v4.5.0
+        uses: actions/setup-python@v4
         with:
           python-version: "3.10"
 
       - name: Upgrade pip
         run: |
-          pip install --constraint=.github/workflows/constraints.txt pip
+          pip install --upgrade pip
           pip --version
 
       - name: Install Poetry
         run: |
-          pipx install --pip-args=--constraint=.github/workflows/constraints.txt poetry
+          pipx install poetry
           poetry --version
 
       - name: Install Nox
         run: |
-          pipx install --pip-args=--constraint=.github/workflows/constraints.txt nox
-          pipx inject --pip-args=--constraint=.github/workflows/constraints.txt nox nox-poetry
+          pipx install nox
+          pipx inject nox nox-poetry
           nox --version
 
       - name: Download coverage data
@@ -138,4 +138,4 @@ jobs:
           nox --force-color --session=coverage -- xml
 
       - name: Upload coverage report
-        uses: codecov/[email protected].1
+        uses: codecov/[email protected].4

data_collector/pyproject.toml

@@ -23,7 +23,7 @@ pygithub = "^1.59.0"
 pydantic = "^1.10.9"
 deep-translator = "^1.11.1"
 orjson = "^3.9.1"
-
+pyattck-data = { path = ".." }
 
 [build-system]
 requires = ["poetry-core"]

data_collector/src/data_collector/collect.py

@@ -24,6 +24,7 @@
 from .services.osqueryattack import OsqueryAttack
 from .services.stockpile import MitreStockpile
 from .services.sysmonhunter import SysmonHunter
+from .services.splunkcontent import SplunkSecurityContent
 
 
 class Collector(Base):
@@ -53,6 +54,7 @@ def collect(self) -> None:
             OsqueryAttack,
             MitreStockpile,
             SysmonHunter,
+            SplunkSecurityContent,
         ]:
             self.__logger.info(f"Collecting data from {service.__name__}")
             service().parse()

data_collector/src/data_collector/services/splunkcontent.py

@@ -0,0 +1,34 @@
+from .base import Base
+
+
+class SplunkSecurityContent(Base):
+    """
+    Data Source: https://raw.githubusercontent.com/splunk/security_content/develop/dist/api/detections.json
+    Authors:
+        - Splunk
+
+    This class is a wrapper for the above data set
+    """
+    URL = 'https://raw.githubusercontent.com/splunk/security_content/develop/dist/api/detections.json'
+
+    def parse(self):
+        self.count = 0
+        self.actor_count = 0
+        data = self.session.get(self.URL).json()
+        for item in data.get("detections"):
+            if item.get("tags"):
+                if item["tags"].get("mitre_attack_enrichments"):
+                    for enrichment in item["tags"]["mitre_attack_enrichments"]:
+                        if enrichment.get("mitre_attack_id"):
+                            tech = self.helper.get_object_by_external_id(enrichment["mitre_attack_id"], "attack-pattern")
+                            tech.possible_detections.append({"name": item["name"], "description": item["description"], "search": item["search"], "tags": item["tags"]})
+                            tech.external_references.extend(item.get("external_references",[]))
+                            self.count += 1
+                            self.helper.replace_object(tech)
+                        if enrichment.get("mitre_attack_groups"):
+                            for group in enrichment["mitre_attack_groups"]:
+                                actor = self.helper.get_object_by_name_or_aliases(group, "intrusion-set")
+                                actor.links.extend(item.get("external_references", []))
+                                self.actor_count += 1
+                                self.helper.replace_object(actor)
+        self.__logger.info(f"Updated {self.count} techniques and {self.actor_count} actors")