RSS Feed - create indicators without published field (#37437)

* set default valur to published * revert server_url value * bump RN * empty commit * add test_parsed_indicators_from_response_no_publish_field * pre commit * Update 1_0_14.md
demisto · Dec 1, 2024 · e601a3a · e601a3a
1 parent 1cb4bfe
commit e601a3a
Show file tree

Hide file tree

Showing 5 changed files with 68 additions and 31 deletions.
diff --git a/Packs/FeedRSS/Integrations/FeedRSS/FeedRSS.py b/Packs/FeedRSS/Integrations/FeedRSS/FeedRSS.py
@@ -63,10 +63,11 @@ def create_indicators_from_response(self):
 
             publications = []
             if indicator:
-                published = dateparser.parse(indicator.published)
-                if not published:
-                    continue
-                published_iso = published.strftime('%Y-%m-%dT%H:%M:%S')
+                published = None
+                if hasattr(indicator, 'published'):
+                    published = dateparser.parse(indicator.published)
+                published_iso = published.strftime('%Y-%m-%dT%H:%M:%S') if published else ''
+
                 publications.append({
                     'timestamp': published_iso,
                     'link': link,

diff --git a/Packs/FeedRSS/Integrations/FeedRSS/FeedRSS_test.py b/Packs/FeedRSS/Integrations/FeedRSS/FeedRSS_test.py
@@ -2,7 +2,7 @@
 import pytest
 from FeedRSS import *
 from requests.models import Response
-from test_data.test_variables import HTML_CONTENT, FEED_DATA, TEST_DATA_MAX_SIZE
+from test_data.test_variables import HTML_CONTENT, FEED_DATA, TEST_DATA_MAX_SIZE, FEED_DATA_NO_PUBLISH_FIELD
 
 
 def side_effect_feed_url(mocker, client):
@@ -51,31 +51,6 @@ def test_parsed_indicators_from_response(mocker, parse_response, expected_output
     assert indicators == expected_output
 
 
-@pytest.mark.parametrize('parse_response,expected_output', FEED_DATA)
-def test_parsed_indicators_enrichment_excluded(mocker, parse_response, expected_output):
-    """
-    Given:
-    - RSS feed url
-    - Enrichment excluded is set
-
-    When:
-    - Calling fetch_indicators
-
-    Then:
-    - Ensure all indicator fields extracted properly, with enrichmentExcluded set to True
-    """
-
-    client = mock_client(mocker, parse_response, enrichment_excluded=True)
-
-    mocker.patch.object(Client, 'get_url_content', return_value='test description')
-    indicators = fetch_indicators(client)
-
-    for ind in expected_output:
-        ind['enrichmentExcluded'] = True
-
-    assert indicators == expected_output
-
-
 def test_get_url_content(mocker):
     """
     Given:
@@ -124,3 +99,23 @@ def test_content_max_size(mocker, article_content, expected_output):
     type(article_content_res).content = article_content
     mocker.patch.object(Client, '_http_request', return_value=article_content_res)
     assert client.get_url_content('test-link.com') == expected_output
+
+
+@pytest.mark.parametrize('parse_response,expected_output', FEED_DATA_NO_PUBLISH_FIELD)
+def test_parsed_indicators_from_response_no_publish_field(mocker, parse_response, expected_output):
+    """
+    Given:
+    - RSS feed url with indicators without publish filed
+
+    When:
+    - After parsing the feed content, we hold a list of items and create a Report indicator from each one of them
+
+    Then:
+    - Ensure all indicator fields extracted properly
+    """
+
+    client = mock_client(mocker, parse_response)
+
+    mocker.patch.object(Client, 'get_url_content', return_value='test description')
+    indicators = fetch_indicators(client)
+    assert indicators == expected_output
diff --git a/Packs/FeedRSS/Integrations/FeedRSS/test_data/test_variables.py b/Packs/FeedRSS/Integrations/FeedRSS/test_data/test_variables.py
@@ -78,3 +78,38 @@
                        'tags': [],
                        'description': 'this is summary'
                    }}])]
+
+
+FEED_DATA_NO_PUBLISH_FIELD = [({'bozo': False,
+                                'entries': [feedparser.util.FeedParserDict({'title': 'Test Article, with comma',
+                                                                            'link': 'https://test-article.com/',
+                                                                            'authors': [{'name': 'Example'}],
+                                                                            'tags': [{'term': 'Malware', 'scheme': None, 'label': None}],
+                                                                            'id': 'xxxx',
+                                                                            'guidislink': False,
+                                                                            'summary': "this is summary"})]
+                                }, [{
+                                    "type": 'Report',
+                                    "value": "Test Article with comma",
+                                    "rawJSON": {'value': {'authors': [{'name': 'Example'}],
+                                                          'guidislink': False,
+                                                          'id': 'xxxx',
+                                                          'link': 'https://test-article.com/',
+                                                          'summary': 'this is summary',
+                                                          'tags': [{'label': None,
+                                                                    'scheme': None,
+                                                                    'term': 'Malware'}],
+                                                          'title': 'Test Article, with comma'},
+                                                'type': 'Report', "firstseenbysource": ''},
+                                    "reliability": "F - Reliability cannot be judged",
+                                    "fields": {
+                                        'publications': [{
+                                            'timestamp': '',
+                                            'link': 'https://test-article.com/',
+                                            'source': 'test.com',
+                                            'title': 'Test Article, with comma'
+                                        }],
+                                        'rssfeedrawcontent': 'test description',
+                                        'tags': [],
+                                        'description': 'this is summary'
+                                    }}])]
diff --git a/Packs/FeedRSS/ReleaseNotes/1_0_14.md b/Packs/FeedRSS/ReleaseNotes/1_0_14.md
@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### RSS Feed
+
+Fixed an issue where indicators without a published value were not fetched by the ***fetch-indicators*** command.
diff --git a/Packs/FeedRSS/pack_metadata.json b/Packs/FeedRSS/pack_metadata.json
@@ -2,7 +2,7 @@
     "name": "RSS Feed",
     "description": "RSS Feed reader, imports new articles as Report indicator.  All rss fields (e.g. author, published, tags) are available.",
     "support": "xsoar",
-    "currentVersion": "1.0.13",
+    "currentVersion": "1.0.14",
     "author": "Cortex XSOAR",
     "url": "",
     "email": "",