[TAXII 2 Server] Revoke objects, Support SCO structure (demisto#24939)

Change the default order of retrieved objects to ascending
Add relationships objects

Packs/Base/.pack-ignore

@@ -64,3 +64,4 @@ SSLAdapter
 SSL
 appendContext
 refang
+IndicatorsSearcher

Packs/Base/ReleaseNotes/1_31_72.md

@@ -0,0 +1,4 @@
+
+#### Scripts
+##### CommonServerPython
+- Added support for the *sort* parameter in the **IndicatorsSearcher** class, which enables sorting of retrieved indicators based on specific criteria.

Packs/Base/Scripts/CommonServerPython/CommonServerPython.py

@@ -9428,6 +9428,9 @@ class IndicatorsSearcher:
     :type limit: ``Optional[int]``
     :param limit: the current upper limit of the search (can be updated after init)
 
+    :type sort: ``List[Dict]``
+    :param sort: An array of sort params ordered by importance. Item structure: {"field": string, "asc": boolean}
+
     :return: No data returned
     :rtype: ``None``
     """
@@ -9441,7 +9444,9 @@ def __init__(self,
                  size=100,
                  to_date=None,
                  value='',
-                 limit=None):
+                 limit=None,
+                 sort=None,
+                 ):
         # searchAfter is available in searchIndicators from version 6.1.0
         self._can_use_search_after = is_demisto_version_ge('6.1.0')
         # populateFields merged in https://github.com/demisto/server/pull/18398
@@ -9457,6 +9462,7 @@ def __init__(self,
         self._value = value
         self._limit = limit
         self._total_iocs_fetched = 0
+        self._sort = sort
 
     def __iter__(self):
         return self
@@ -9550,8 +9556,10 @@ def search_indicators_by_version(self, from_date=None, query='', size=100, to_da
             searchAfter=self._search_after_param if self._can_use_search_after else None,
             populateFields=self._filter_fields if self._can_use_filter_fields else None,
             # use paging as fallback when cannot use search_after
-            page=self.page if not self._can_use_search_after else None
+            page=self.page if not self._can_use_search_after else None,
         )
+        if is_demisto_version_ge('6.6.0'):
+            search_args['sort'] = self._sort
         res = demisto.searchIndicators(**search_args)
         if isinstance(self._page, int):
             self._page += 1  # advance pages

Packs/Base/Scripts/CommonServerPython/CommonServerPython_test.py

@@ -6736,6 +6736,30 @@ def test_iterator__research_flow(self, mocker):
             results.append(res)
         assert len(results) == 1
 
+    def test_search_indicators_with_sort(self, mocker):
+        """
+        Given:
+          - Searching indicators with a custom sort parameter.
+          - Mocking the searchIndicators function.
+        When:
+          - Calling the searchIndicators function with the custom sort parameter.
+        Then:
+          - Ensure that the sort parameter is set correctly.
+          - Ensure that the searchIndicators function is called with the expected arguments.
+        """
+        from CommonServerPython import IndicatorsSearcher
+        get_demisto_version._version = None  # clear cache between runs of the test
+        mocker.patch.object(demisto, 'demistoVersion', return_value={'version': '6.6.0'})
+
+        mocker.patch.object(demisto, 'searchIndicators')
+        sort_param = [{"field": "created", "asc": False}]
+        search_indicators_obj_search_after = IndicatorsSearcher(sort=sort_param)
+        search_indicators_obj_search_after.search_indicators_by_version()
+        expected_args = {'size': 100, 'sort': [{'asc': False, 'field': 'created'}]}
+        assert search_indicators_obj_search_after._sort == sort_param
+        demisto.searchIndicators.assert_called_once_with(**expected_args)
+
+
 
 class TestAutoFocusKeyRetriever:
     def test_instantiate_class_with_param_key(self, mocker, clear_version_cache):

Packs/Base/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Base",
     "description": "The base pack for Cortex XSOAR.",
     "support": "xsoar",
-    "currentVersion": "1.31.71",
+    "currentVersion": "1.31.72",
     "author": "Cortex XSOAR",
     "serverMinVersion": "6.0.0",
     "url": "https://www.paloaltonetworks.com/cortex",

Packs/TAXIIServer/.secrets-ignore

@@ -5,4 +5,7 @@ http://www.rsyslog.com
 18.163.0.0
 [email protected]
 http://host
-test_taxii_wrong_auth
+test_taxii_wrong_auth
+6.6.6.6
+@APP.route('/taxii2/'
+uuid.uuid5(SCO_DET_ID_NAMESPACE

Packs/TAXIIServer/Integrations/TAXII2Server/README.md

@@ -54,14 +54,14 @@ For more information, visit [TAXII2 Documentation](http://docs.oasis-open.org/ct
 
 ## TAXII v2.1 API Enpoints
 
-| **URL** | **Method** | **Response** | **TAXII2 Documentation** |
-| --- | --- | --- | --- |
-| /taxii2/ | GET | Server Discovery Information. | [Server Discovery](https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107526) |
-| /{api_root}/ | GET | XSOAR API root is *threatintel*. | [API Root Information](https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107528) |
-| /{api_root}/collections/ | GET | All Cortex XSOAR collections that configure in Collection JSON parameter. | [Collections Resource](https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107533) |
-| /{api_root}/collections/{collection_id}/ | GET | Cortex XSOAR Collection with given *collection_id*. | [Collection Response](https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107535) |
-| /{api_root}/collections/{collection_id}/manifest/ | GET | Object manifests from the given collection. | [Objects Manifest Resource](https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107537) |
-| /{api_root}/collections/{collection_id}/objects/ | GET | Objects (Cortex XSOAR Indicators) from the given collection. | [Object Resource](https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107539) |
+| **URL**                                           | **Method** | **Response**                                                                   | **TAXII2 Documentation**                                                                                   |
+|---------------------------------------------------|------------|--------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------|
+| /taxii2/                                          | GET        | Server Discovery Information.                                                  | [Server Discovery](https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107526)          |
+| /{api_root}/                                      | GET        | XSOAR API root is *threatintel*.                                               | [API Root Information](https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107528)      |
+| /{api_root}/collections/                          | GET        | All Cortex XSOAR collections that configure in Collection JSON parameter.      | [Collections Resource](https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107533)      |
+| /{api_root}/collections/{collection_id}/          | GET        | Cortex XSOAR Collection with given *collection_id*.                            | [Collection Response](https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107535)       |
+| /{api_root}/collections/{collection_id}/manifest/ | GET        | Object manifests from the given collection.                                    | [Objects Manifest Resource](https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107537) |
+| /{api_root}/collections/{collection_id}/objects/  | GET        | Objects (Cortex XSOAR Indicators and Relationships) from the given collection. | [Object Resource](https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107539)           |
 
 For more information, visit [TAXII2 Documentation](https://docs.oasis-open.org/cti/taxii/v2.1/taxii-v2.1.html).
 

Packs/TAXIIServer/Integrations/TAXII2Server/TAXII2Server.py

@@ -567,6 +567,7 @@ def find_indicators(query: str, types: list, added_after, limit: int, offset: in
     new_limit = offset + limit
     iocs = []
     extensions = []
+
     if is_manifest:
         field_filters: Optional[str] = ','.join(TAXII_REQUIRED_FILTER_FIELDS)
     elif SERVER.fields_to_present:
@@ -582,7 +583,8 @@ def find_indicators(query: str, types: list, added_after, limit: int, offset: in
         query=new_query,
         limit=new_limit,
         size=PAGE_SIZE,
-        from_date=added_after
+        from_date=added_after,
+        sort=[{"field": "modified", "asc": True}],
     )
 
     total = 0
@@ -606,7 +608,11 @@ def find_indicators(query: str, types: list, added_after, limit: int, offset: in
                         extensions.append(extension_definition)
                 elif stix_ioc:
                     iocs.append(stix_ioc)
-
+    if not is_manifest and iocs and is_demisto_version_ge('6.6.0'):
+        if relationships := create_relationships_objects(iocs, extensions):
+            total += len(relationships)
+            iocs.extend(relationships)
+            iocs = sorted(iocs, key=lambda k: k['modified'])
     return iocs, extensions, total
 
 
@@ -1200,6 +1206,91 @@ def get_server_collections_command(integration_context):
     return result
 
 
+def create_relationships_objects(stix_iocs: list[dict[str, Any]], extensions: list) -> list[dict[str, Any]]:
+    """
+    Create entries for the relationships returned by the searchRelationships command.
+    :param stix_iocs: Entries for the Stix objects associated with given indicators
+    :param extensions: A list of dictionaries representing extension properties to include in the generated STIX objects.
+    :return: A list of dictionaries representing the relationships objects, including entityBs objects
+    """
+    relationships_list: list[dict[str, Any]] = []
+    iocs_value_to_id = {(stix_ioc.get('value') or stix_ioc.get('name')): stix_ioc.get('id') for stix_ioc in stix_iocs}
+    search_relationships = demisto.searchRelationships({'entities': list(iocs_value_to_id.keys())}).get('data') or []
+    demisto.debug(f"Found {len(search_relationships)} relationships for {len(iocs_value_to_id)} Stix IOC values.")
+
+    relationships_list.extend(create_entity_b_stix_objects(search_relationships, iocs_value_to_id, extensions))
+
+    for relationship in search_relationships:
+
+        if not iocs_value_to_id.get(relationship.get('entityB')):
+            demisto.debug(f"WARNING: Invalid entity B - Relationships will not be created to entity A:"
+                          f" {relationship.get('entityA')} with relationship name {relationship.get('name')}")
+            continue
+        try:
+            created_parsed = parse(relationship.get('createdInSystem')).strftime(STIX_DATE_FORMAT)
+            modified_parsed = parse(relationship.get('modified')).strftime(STIX_DATE_FORMAT)
+        except Exception as e:
+            created_parsed, modified_parsed = '', ''
+            demisto.debug(f"Error parsing dates for relationship {relationship.get('id')}: {e}")
+
+        relationship_unique_id = uuid.uuid5(SERVER.namespace_uuid, f'relationship:{relationship.get("id")}')
+        relationship_stix_id = f'relationship--{relationship_unique_id}'
+
+        relationship_object: dict[str, Any] = {
+            'type': "relationship",
+            'spec_version': SERVER.version,
+            'id': relationship_stix_id,
+            'created': created_parsed,
+            'modified': modified_parsed,
+            "relationship_type": relationship.get('name'),
+            'source_ref': iocs_value_to_id.get(relationship.get('entityA')),
+            'target_ref': iocs_value_to_id.get(relationship.get('entityB')),
+        }
+        if description := demisto.get(relationship, 'CustomFields.description'):
+            relationship_object['Description'] = description
+
+        relationships_list.append(relationship_object)
+
+    return relationships_list
+
+
+def create_entity_b_stix_objects(relationships: list[dict[str, Any]], iocs_value_to_id: dict, extensions: list) -> list:
+    """
+    Generates a list of STIX objects for the 'entityB' values in the provided 'relationships' list.
+    :param relationships: A list of dictionaries representing relationships between entities
+    :param iocs_value_to_id: A dictionary mapping IOC values to their corresponding ID values.
+    :param extensions: A list of dictionaries representing extension properties to include in the generated STIX objects.
+    :return: A list of dictionaries representing STIX objects for the 'entityB' values
+    """
+    entity_b_objects: list[dict[str, Any]] = []
+    entity_b_values = ""
+    for relationship in relationships:
+        if (entity_b_value := relationship.get('entityB')) and entity_b_value not in iocs_value_to_id:
+            iocs_value_to_id[entity_b_value] = ""
+            entity_b_values += f'\"{entity_b_value}\" '
+    if not entity_b_values:
+        return entity_b_objects
+
+    found_indicators = demisto.searchIndicators(query=f'value:({entity_b_values})').get('iocs') or []
+
+    extensions_dict: dict = {}
+    for xsoar_indicator in found_indicators:
+        xsoar_type = xsoar_indicator.get('indicator_type')
+        stix_ioc, extension_definition, extensions_dict = create_stix_object(xsoar_indicator, xsoar_type, extensions_dict)
+        if XSOAR_TYPES_TO_STIX_SCO.get(xsoar_type) in SERVER.types_for_indicator_sdo:
+            stix_ioc = convert_sco_to_indicator_sdo(stix_ioc, xsoar_indicator)
+        if SERVER.has_extension and stix_ioc:
+            entity_b_objects.append(stix_ioc)
+            if extension_definition:
+                extensions.append(extension_definition)
+        elif stix_ioc:
+            entity_b_objects.append(stix_ioc)
+        iocs_value_to_id[(stix_ioc.get('value') or stix_ioc.get('name'))] = stix_ioc.get('id')
+
+    demisto.debug(f"Generated {len(entity_b_objects)} STIX objects for 'entityB' values.")
+    return entity_b_objects
+
+
 def main():  # pragma: no cover
     """
     Main

Packs/TAXIIServer/Integrations/TAXII2Server/TAXII2Server.yml

@@ -131,7 +131,7 @@ script:
     - contextPath: TAXIIServer.ServerInfo.description
       description: The server description
       type: String
-  dockerimage: demisto/flask-nginx:1.0.0.48906
+  dockerimage: demisto/flask-nginx:1.0.0.49636
   feed: false
   isfetch: false
   longRunning: true

Packs/TAXIIServer/Integrations/TAXII2Server/TAXII2Server_test.py

@@ -591,3 +591,40 @@ def test_convert_sco_to_indicator_sdo_with_type_file(mocker):
     assert 'file:hashes.' in output.get('pattern', '')
     assert 'MD5' in output.get('pattern', '')
     assert 'pattern_type' in output.keys()
+
+
+def test_taxii21_objects_with_relationships(mocker, taxii2_server_v21):
+    """
+        Given
+            TAXII Server v2.1, collection_id, no_extension
+        When
+            Calling get objects api request for given collection
+        Then
+            Validate that right objects are returned.
+            Ensure that searchRelationships is called with the expected arguments.
+
+    """
+    from CommonServerPython import get_demisto_version
+
+    get_demisto_version._version = None  # clear cache between runs of the test
+    mocker.patch.object(demisto, 'demistoVersion', return_value={'version': '6.6.0'})
+
+    mocker.patch('TAXII2Server.SERVER', taxii2_server_v21)
+    mocker.patch('TAXII2Server.SERVER.has_extension', False)
+    mock_search_relationships_response = util_load_json('test_data/searchRelationships-response.json')
+    mocker.patch.object(demisto, 'searchRelationships', return_value=mock_search_relationships_response)
+
+    objects = util_load_json('test_data/objects21_ip_with_relationships.json')
+    mock_iocs = util_load_json('test_data/sort_ip_iocs.json')
+    mock_entity_b_iocs = util_load_json('test_data/entity_b_iocs.json')
+    mocker.patch.object(demisto, 'searchIndicators', side_effect=[mock_iocs,
+                                                                  mock_entity_b_iocs])
+
+    mocker.patch.object(demisto, 'params', return_value={'res_size': '4'})
+    with APP.test_client() as test_client:
+        response = test_client.get('/threatintel/collections/4c649e16-2bb7-50f5-8826-2a2d0a0b9631/objects/',
+                                   headers=HEADERS)
+        assert response.status_code == 200
+        assert response.content_type == 'application/taxii+json;version=2.1'
+        demisto.searchRelationships.assert_called_once_with({'entities': ["1.1.1.1", "8.8.8.8", "3.3.3.3", "1.2.3.4"]})
+        assert response.json == objects