Release 3.1.0.1 - th2 5191 cherry pick (#94)

* Release 3.1.0.1 - [TH2-5193] Bug fixes

* TH2-5191 fix with new orjson

* Update CI python test versions

* Fix readme and related script

* Fix CI build

* Improve release notes

(cherry picked from commit 07dec2a)

* Ruff improvements

.github/workflows/ci-tests.yml

@@ -9,7 +9,7 @@ jobs:
 
     strategy:
       matrix:
-        py_version: [ '3.7', '3.8', '3.9' ]
+        py_version: [ '3.8', '3.9', '3.10' ]
         os: [ ubuntu-20.04, windows-latest ]
 
     runs-on: ${{ matrix.os }}

.github/workflows/dev-build-to-pypi.yml

@@ -8,16 +8,17 @@ on:
 
 jobs:
   build-n-publish:
-    name: Dev build and Python distributions to PyPI
+    name: Dev build and publish Python distributions to PyPI
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
 
       - name: Wait for tests to succeed
-        uses: lewagon/[email protected].1
+        uses: lewagon/[email protected].2
         with:
           ref: ${{ github.ref }}
-          running-workflow-name: 'Dev build and Python distributions to PyPI'
+#          check-name: 'Run tests'
+          running-workflow-name: 'Dev build and publish Python distributions to PyPI'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           wait-interval: 10
           allowed-conclusions: success
@@ -45,10 +46,10 @@ jobs:
           value: ${{ steps.release_ver.outputs.value }}
 
       # Build and publish
-      - name: Set up Python 3.7
+      - name: Set up Python 3.8
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7
+          python-version: 3.8
 
       - name: Install dependencies
         run: |

README.md

@@ -1,10 +1,11 @@
-# Lightweight Data Provider Data Source (major version 3).
-# Introduction
+# Lightweight Data Provider Data Source (major version 3)
+
+## Introduction
 This repository is an implementation of data-services data source for Lightweight Data Provider V3.
 
-Most commands that this library provides can be seen in example below.
+Most commands that this library provides can be seen in the example below.
 
-# Example
+## Example
 <!-- start get_started_example.py -->
 ```python
 from typing import List
@@ -64,7 +65,7 @@ START_TIME = datetime(year=2023, month=1, day=5, hour=13, minute=57, second=5, m
 END_TIME = datetime(year=2023, month=1, day=5, hour=13, minute=57, second=6, microsecond=0)
 
 # [1] Create data source object to connect to lightweight data provider.
-provider_url_link = f"http://10.100.66.105:32681"
+provider_url_link = "http://10.100.66.105:32681"
 data_source = DataSource(provider_url_link)
 
 # [2] Getting books, pages, scopes, groups and aliases.
@@ -198,18 +199,17 @@ etc = EventTreeCollection(etc_driver)
 etc.build(events)
 etc.recover_unknown_events()
 # See more info about how to use ETC in th2-data-services lib documentation.
-
 ```
 <!-- end get_started_example.py -->
 
-# Changes in LwDP 3.0.0
+## Changes in LwDP 3.* against LwDP 2.*
 
 Changes mostly affect how messages are represented in LwDP V3.
 
-In V3 message id will also get a group section and new format will look like:
-book:group:session_alias:direction:timestamp:sequence
+In V3 message id will also get a group section, and the new format will look like:
+`book:group:session_alias:direction:timestamp:sequence`
 
-Main changes are in body field of message:
+The Main changes are in the body field of a message:
 * METADATA
   * the metadata does not contain duplicated information from the top message (direction, sequence, timestamp, sessionId). Only subsequence, messageType and protocol are left (inside metadata).
   * metadata block doesn’t have fixed structure. So if there is only 1 parsed was produced from raw message, metadata will not have this key.
@@ -276,3 +276,4 @@ mfr = MessageFieldResolver()
 message_data = message_data.map(mfr.expand_message)
 print(message_data) # we should now have 2 messages built from the body list of original message.
 ```
+

attach_file_to.sh

@@ -1,51 +1,35 @@
 #!/bin/bash
 
-set -euo pipefail
+function get_line_number () {
+    local text=$1;
+    local file=$2;
+    grep "$text" $file -n -m1 | cut -d':' -f1
+}
 
 attachable_file=$1
 attachable_file_path=$(find "$PWD" -name "$attachable_file")
-output_file_path=$PWD"/"$2
-
-echo $attachable_file_path
+attach_to=$2
 
 function main() {
-  status=$(git status)
-
-#  if [[ "$status" =~ .*"$attachable_file".* ]]; then
-  count=0
-  start_line=0
-  end_line=0
-
-  exec 0<"$output_file_path"
-  while read -r line
-  do
-    count=$((count+1))
-
-    if [[ $line =~ "<!--".*"start".*"$attachable_file" ]]; then
-      start_line=$count
-    fi
-
-    if [[ $line =~ "<!--".*"end".*"$attachable_file" ]]; then
-      end_line=$count
-    fi
-  done
-
-  echo $start_line
-  echo $end_line
-
-  head -n $start_line "$output_file_path" > buffer
-  {
-    printf '```python\n'
-    cat "$attachable_file_path"
-    printf '\n```\n'
-    tail -n $((count - end_line + 1)) "$output_file_path"
-  }>> buffer
-  rm "$output_file_path"
-  mv -f buffer "$output_file_path"
-  git add "$output_file_path"
-#  fi
-
-  exit 0
+    local status=$(git status)
+
+    # Check If Update Is Needed
+    if ! [[ "$status" =~ .*"$attachable_file".* ]]; then return; fi;
+
+    # Get Start & End Indexes Of Code Block
+    local start=$(get_line_number "<!-- start $attachable_file" $attach_to);
+    local end=$(get_line_number "<!-- end $attachable_file" $attach_to);
+
+    ((start+=2)); # Increment By 2, Start After "```"
+    ((end-=2)); # Decrement By 2, End Before "```"
+
+    # Delete Code Block Lines
+    sed -i -e "${start},${end}d;" $attach_to;
+
+    ((start-=1)); # Increment By 2, Start After "```"
+
+    # Add Code Block Lines
+    sed -i "${start}r $attachable_file_path" $attach_to;
 }
 
 main

examples/get_started_example.py

@@ -55,7 +55,7 @@
 END_TIME = datetime(year=2023, month=1, day=5, hour=13, minute=57, second=6, microsecond=0)
 
 # [1] Create data source object to connect to lightweight data provider.
-provider_url_link = f"http://10.100.66.105:32681"
+provider_url_link = "http://10.100.66.105:32681"
 data_source = DataSource(provider_url_link)
 
 # [2] Getting books, pages, scopes, groups and aliases.

package_info.json

@@ -1,5 +1,5 @@
 {
   "package_name": "th2-data-services-lwdp",
-  "package_version": "3.1.0.0"
+  "package_version": "3.1.0.1"
 }
 

release_notes.md

@@ -176,7 +176,14 @@ BugFixes without ticket
 2. [TH2-4945] - Refactoring: Remove GRPC.
 3. [TH2-4959] - Added deprecation warning.
 4. [TH2-5048] - Added typing hints for resolver methods.
-5. [TH2-4974] - Added resolver for getting group in message.
+5. [TH2-4974] - Added resolver for getting a group in the message.
+
+
+# v3.0.1.0
+
+## Features
+
+1. [Th2-4975] Added streams parameter to download and get-messages-by-groups/pages sse commands.
 
 
 # v3.1.0.0
@@ -223,3 +230,21 @@ BugFixes without ticket
     - 2023-12-27 23:42:15.123456
     - 2023-12-27 23:42:15.123
     - 2023-12-27 23:42:15
+
+
+# v3.1.0.1
+
+## User impact and migration instructions
+
+1. [I] new version of `orjson` & `aiohttp` lib require python 3.8+.
+   [M] Change your python version if you use 3.7 to 3.8+.
+
+## BugFixes
+1. [TH2-5191] - The following commands `GetEventScopes`, `GetMessageAliases`, 
+   `GetMessageGroups` return list of lists instead of a single list.
+2. [TH2-5193] - orjson versions 3.7.0 through 3.9.14 library has vulnerability
+   https://devhub.checkmarx.com/cve-details/CVE-2024-27454/. 
+   Updated to >=3.10,<4.
+3. [TH2-5193] - aiohttp library versions < 3.9.4 have vulnerability
+   https://security.snyk.io/package/pip/aiohttp. 
+   Updated to >=3.9.5,<4.

requirements.txt

@@ -1,8 +1,8 @@
-# Uncomment it when ds libe v2 will be released
+# TODO - Uncomment it when ds libe v2 will be released
 #th2-data-services>=2,<3  # ds core 2.*
 
-th2-data-services~=2.0.0.dev8154123929
+th2-data-services~=2.0.0.dev8779326749
 # We don't need it now, because we don't suport grpc now
 # th2_grpc_lw_data_provider==2.0.0.dev3656626108
-orjson>=3,<4
-aiohttp>=3,<4
+orjson>=3.10,<4
+aiohttp>=3.9.5,<4

setup.py

@@ -40,7 +40,7 @@
     author_email="[email protected]",
     url="https://github.com/th2-net/th2-ds-source-lwdp",
     license="Apache License 2.0",
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     install_requires=requirements,
     packages=find_namespace_packages(include=["th2_data_services", "th2_data_services.*"]),
     include_package_data=True,

tests/tests_integration/tests_common/test_data_source.py

@@ -17,7 +17,7 @@
     root_event_body,
     plain_event_1_body,
     filter_event_3_body,
-) 
+)
 from tests.tests_integration.test_bodies.http.message_bodies import message_1_body, message_2_body
 
 
@@ -28,8 +28,11 @@ def test_issue_events(all_events):
 def test_issue_messages(all_messages):
     assert list(all_messages.data) == all_messages.expected_data_values
 
-def test_find_messages_by_book_by_groups(get_messages_by_book_by_groups,all_messages):
-    assert sorted(list(get_messages_by_book_by_groups),key=lambda a: a['messageId']) == sorted(all_messages.expected_data_values,key=lambda a: a['messageId']) 
+
+def test_find_messages_by_book_by_groups(get_messages_by_book_by_groups, all_messages):
+    assert sorted(list(get_messages_by_book_by_groups), key=lambda a: a["messageId"]) == sorted(
+        all_messages.expected_data_values, key=lambda a: a["messageId"]
+    )
 
 
 def test_find_messages_by_pages_by_groups(get_messages_by_page_by_groups: Data):
@@ -133,7 +136,7 @@ def test_get_x_with_filters(
     get_events_with_filters: Data,
 ):
     event_case = [filter_event_3_body]
-    assert list(get_events_with_one_filter) == event_case and len(event_case) is 1
+    assert list(get_events_with_one_filter) == event_case and len(event_case) == 1
     assert list(get_events_with_filters) == event_case
 
 
@@ -152,7 +155,7 @@ def test_get_events_from_data_provider_with_error(http_data_source: DataSource):
             )
         )
         list(events)
-    #assert "replace() takes no keyword arguments" in str(exc_info)
+    # assert "replace() takes no keyword arguments" in str(exc_info)
     assert "Provided timestamp should be `datetime` object" in str(exc_info)
 
 
@@ -167,7 +170,7 @@ def test_get_messages_from_data_provider_with_error(http_data_source: DataSource
             )
         )
         list(messages)
-    #assert "replace() takes no keyword arguments" in str(exc_info)
+    # assert "replace() takes no keyword arguments" in str(exc_info)
     assert "Provided timestamp should be `datetime` object" in str(exc_info)
 
 
@@ -238,7 +241,7 @@ def test_messages_for_data_loss(all_messages):
     It might happen if source inside of Data is object of generator instead of
     generator function.
     """
-    
+
     messages = all_messages.data
     for _ in range(3):
         for _ in messages:

tests/tests_unit/tests_common/test_data_source.py

@@ -21,7 +21,7 @@ def test_command_without_end_timestamp():
                 year=2022, month=6, day=30, hour=14, minute=0, second=0, microsecond=0
             ),
         )
-    except TypeError as te:
+    except TypeError:
         assert False