From 5a6da9db3bd21f1c99d5e27abfe96f7ae90810f7 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 15 Aug 2023 17:22:50 +0300 Subject: [PATCH 001/268] Async RPClient: WIP --- reportportal_client/client_async.py | 238 ++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 239 insertions(+) create mode 100644 reportportal_client/client_async.py diff --git a/reportportal_client/client_async.py b/reportportal_client/client_async.py new file mode 100644 index 00000000..ba69366b --- /dev/null +++ b/reportportal_client/client_async.py @@ -0,0 +1,238 @@ +"""This module contains asynchronous implementation of Report Portal Client.""" + +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import asyncio +import logging +from queue import LifoQueue +from typing import Union, Tuple, List, Dict, Any, Optional, TextIO + +import aiohttp + +from .core.rp_issues import Issue +from .logs.log_manager import LogManager, MAX_LOG_BATCH_PAYLOAD_SIZE +from .static.defines import NOT_FOUND +from .steps import StepReporter + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + + +class _LifoQueue(LifoQueue): + def last(self): + with self.mutex: + if self._qsize(): + return self.queue[-1] + + +class _RPClient: + _log_manager: LogManager = ... + api_v1: str = ... + api_v2: str = ... + base_url_v1: str = ... + base_url_v2: str = ... + endpoint: str = ... + is_skipped_an_issue: bool = ... + launch_id: str = ... + log_batch_size: int = ... + log_batch_payload_size: int = ... + project: str = ... + api_key: str = ... + verify_ssl: Union[bool, str] = ... + retries: int = ... + max_pool_size: int = ... + http_timeout: Union[float, Tuple[float, float]] = ... + session: aiohttp.ClientSession = ... + step_reporter: StepReporter = ... + mode: str = ... + launch_uuid_print: Optional[bool] = ... + print_output: Optional[TextIO] = ... + _skip_analytics: str = ... + _item_stack: _LifoQueue = ... + + def __init__( + self, + endpoint: str, + project: str, + api_key: str = None, + log_batch_size: int = 20, + is_skipped_an_issue: bool = True, + verify_ssl: bool = True, + retries: int = None, + max_pool_size: int = 50, + launch_id: str = None, + http_timeout: Union[float, Tuple[float, float]] = (10, 10), + log_batch_payload_size: int = MAX_LOG_BATCH_PAYLOAD_SIZE, + mode: str = 'DEFAULT', + launch_uuid_print: bool = False, + print_output: Optional[TextIO] = None, + **kwargs: Any + ) -> None: + self._item_stack = _LifoQueue() + + async def finish_launch(self, + end_time: str, + status: str = None, + attributes: Optional[Union[List, Dict]] = None, + **kwargs: Any) -> Optional[str]: + pass + + async def finish_test_item(self, + item_id: Union[asyncio.Future, str], + end_time: str, + *, + status: str = None, + issue: Optional[Issue] = None, + attributes: Optional[Union[List, Dict]] = None, + description: str = None, + retry: bool = False, + **kwargs: Any) -> Optional[str]: + pass + + async def get_item_id_by_uuid(self, uuid: Union[asyncio.Future, str]) -> Optional[str]: + pass + + async def get_launch_info(self) -> Optional[Dict]: + pass + + async def get_launch_ui_id(self) -> Optional[Dict]: + pass + + async def get_launch_ui_url(self) -> Optional[str]: + pass + + async def get_project_settings(self) -> Optional[Dict]: + pass + + async def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, + attachment: Optional[Dict] = None, + item_id: Optional[Union[asyncio.Future, str]] = None) -> None: + pass + + async def start_launch(self, + name: str, + start_time: str, + description: Optional[str] = None, + attributes: Optional[Union[List, Dict]] = None, + rerun: bool = False, + rerun_of: Optional[str] = None, + **kwargs) -> Optional[str]: + pass + + async def start_test_item(self, + name: str, + start_time: str, + item_type: str, + *, + description: Optional[str] = None, + attributes: Optional[List[Dict]] = None, + parameters: Optional[Dict] = None, + parent_item_id: Optional[Union[asyncio.Future, str]] = None, + has_stats: bool = True, + code_ref: Optional[str] = None, + retry: bool = False, + test_case_id: Optional[str] = None, + **_: Any) -> Optional[str]: + pass + + async def update_test_item(self, item_uuid: Union[asyncio.Future, str], + attributes: Optional[Union[List, Dict]] = None, + description: Optional[str] = None) -> Optional[str]: + pass + + def _add_current_item(self, item: Union[asyncio.Future, str]) -> None: + """Add the last item from the self._items queue.""" + self._item_stack.put(item) + + def _remove_current_item(self) -> None: + """Remove the last item from the self._items queue.""" + return self._item_stack.get() + + def current_item(self) -> Union[asyncio.Future, str]: + """Retrieve the last item reported by the client.""" + return self._item_stack.last() + + def clone(self) -> '_RPClient': + """Clone the client object, set current Item ID as cloned item ID. + + :returns: Cloned client object + :rtype: _RPClient + """ + cloned = _RPClient( + endpoint=self.endpoint, + project=self.project, + api_key=self.api_key, + log_batch_size=self.log_batch_size, + is_skipped_an_issue=self.is_skipped_an_issue, + verify_ssl=self.verify_ssl, + retries=self.retries, + max_pool_size=self.max_pool_size, + launch_id=self.launch_id, + http_timeout=self.http_timeout, + log_batch_payload_size=self.log_batch_payload_size, + mode=self.mode + ) + current_item = self.current_item() + if current_item: + cloned._add_current_item(current_item) + return cloned + + +class RPClientAsync(_RPClient): + + async def start_test_item(self, + name: str, + start_time: str, + item_type: str, + *, + description: Optional[str] = None, + attributes: Optional[List[Dict]] = None, + parameters: Optional[Dict] = None, + parent_item_id: Optional[Union[asyncio.Future, str]] = None, + has_stats: bool = True, + code_ref: Optional[str] = None, + retry: bool = False, + test_case_id: Optional[str] = None, + **_: Any) -> Optional[str]: + item_id = await super().start_test_item(name, start_time, item_type, description=description, + attributes=attributes, parameters=parameters, + parent_item_id=parent_item_id, has_stats=has_stats, + code_ref=code_ref, retry=retry, test_case_id=test_case_id) + if item_id and item_id is not NOT_FOUND: + super()._add_current_item(item_id) + return item_id + + async def finish_test_item(self, + item_id: Union[asyncio.Future, str], + end_time: str, + *, + status: str = None, + issue: Optional[Issue] = None, + attributes: Optional[Union[List, Dict]] = None, + description: str = None, + retry: bool = False, + **kwargs: Any) -> Optional[str]: + result = await super().finish_test_item(item_id, end_time, status=status, issue=issue, + attributes=attributes, description=description, retry=retry) + super()._remove_current_item() + return result + + +class RPClientSync: + + client: _RPClient + + def __init__(self, client: _RPClient): + self.client = client + diff --git a/requirements.txt b/requirements.txt index 549e8374..5c507f12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ aenum requests>=2.27.1 six>=1.16.0 +aiohttp==3.8.5 From 6ef4137d93050d059334a7456fb6153b04027d4c Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 15 Aug 2023 18:33:14 +0300 Subject: [PATCH 002/268] Async RPClient: WIP --- reportportal_client/client_async.py | 77 ++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 12 deletions(-) diff --git a/reportportal_client/client_async.py b/reportportal_client/client_async.py index ba69366b..71206f9a 100644 --- a/reportportal_client/client_async.py +++ b/reportportal_client/client_async.py @@ -15,6 +15,7 @@ import asyncio import logging +import threading from queue import LifoQueue from typing import Union, Tuple, List, Dict, Any, Optional, TextIO @@ -65,6 +66,7 @@ def __init__( self, endpoint: str, project: str, + *, api_key: str = None, log_batch_size: int = 20, is_skipped_an_issue: bool = True, @@ -127,7 +129,7 @@ async def start_launch(self, attributes: Optional[Union[List, Dict]] = None, rerun: bool = False, rerun_of: Optional[str] = None, - **kwargs) -> Optional[str]: + **kwargs) -> Optional[Union[asyncio.Future, str]]: pass async def start_test_item(self, @@ -143,8 +145,10 @@ async def start_test_item(self, code_ref: Optional[str] = None, retry: bool = False, test_case_id: Optional[str] = None, - **_: Any) -> Optional[str]: - pass + **_: Any) -> Optional[Union[asyncio.Future, str]]: + parent = parent_item_id + if parent_item_id and asyncio.isfuture(parent_item_id): + parent = await parent_item_id async def update_test_item(self, item_uuid: Union[asyncio.Future, str], attributes: Optional[Union[List, Dict]] = None, @@ -204,11 +208,12 @@ async def start_test_item(self, code_ref: Optional[str] = None, retry: bool = False, test_case_id: Optional[str] = None, - **_: Any) -> Optional[str]: + **kwargs: Any) -> Optional[Union[asyncio.Future, str]]: item_id = await super().start_test_item(name, start_time, item_type, description=description, attributes=attributes, parameters=parameters, parent_item_id=parent_item_id, has_stats=has_stats, - code_ref=code_ref, retry=retry, test_case_id=test_case_id) + code_ref=code_ref, retry=retry, test_case_id=test_case_id, + **kwargs) if item_id and item_id is not NOT_FOUND: super()._add_current_item(item_id) return item_id @@ -224,15 +229,63 @@ async def finish_test_item(self, retry: bool = False, **kwargs: Any) -> Optional[str]: result = await super().finish_test_item(item_id, end_time, status=status, issue=issue, - attributes=attributes, description=description, retry=retry) + attributes=attributes, description=description, retry=retry, + **kwargs) super()._remove_current_item() return result -class RPClientSync: - - client: _RPClient - - def __init__(self, client: _RPClient): - self.client = client +class RPClientSync(_RPClient): + loop: asyncio.AbstractEventLoop + thread: threading.Thread + def __init__( + self, + endpoint: str, + project: str, + *, + api_key: str = None, + log_batch_size: int = 20, + is_skipped_an_issue: bool = True, + verify_ssl: bool = True, + retries: int = None, + max_pool_size: int = 50, + launch_id: str = None, + http_timeout: Union[float, Tuple[float, float]] = (10, 10), + log_batch_payload_size: int = MAX_LOG_BATCH_PAYLOAD_SIZE, + mode: str = 'DEFAULT', + launch_uuid_print: bool = False, + print_output: Optional[TextIO] = None, + **kwargs: Any + ) -> None: + super().__init__(endpoint, project, api_key=api_key, log_batch_size=log_batch_size, + is_skipped_an_issue=is_skipped_an_issue, verify_ssl=verify_ssl, retries=retries, + max_pool_size=max_pool_size, launch_id=launch_id, http_timeout=http_timeout, + log_batch_payload_size=log_batch_payload_size, mode=mode, + launch_uuid_print=launch_uuid_print, print_output=print_output, **kwargs) + self.loop = asyncio.new_event_loop() + self.thread = threading.Thread(target=self.loop.run_forever(), name='RP-Async-Client', daemon=True) + self.thread.start() + + def start_test_item(self, + name: str, + start_time: str, + item_type: str, + *, + description: Optional[str] = None, + attributes: Optional[List[Dict]] = None, + parameters: Optional[Dict] = None, + parent_item_id: Optional[Union[asyncio.Future, str]] = None, + has_stats: bool = True, + code_ref: Optional[str] = None, + retry: bool = False, + test_case_id: Optional[str] = None, + **kwargs: Any) -> Optional[Union[asyncio.Future, str]]: + item_id_coro = super().start_test_item(name, start_time, item_type, description=description, + attributes=attributes, parameters=parameters, + parent_item_id=parent_item_id, has_stats=has_stats, + code_ref=code_ref, retry=retry, test_case_id=test_case_id, + **kwargs) + item_id_task = self.loop.create_task(item_id_coro) + super()._add_current_item(item_id_task) + return item_id_task From 7c4b35cfaa321fbf9b81c31d262b8f6398259d01 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 30 Aug 2023 12:51:38 +0300 Subject: [PATCH 003/268] Async RPClient: WIP --- reportportal_client/__init__.py | 9 +++--- reportportal_client/async/__init__.py | 12 ++++++++ .../{client_async.py => async/client.py} | 28 +++++++++---------- reportportal_client/steps/__init__.py | 1 + 4 files changed, 32 insertions(+), 18 deletions(-) create mode 100644 reportportal_client/async/__init__.py rename reportportal_client/{client_async.py => async/client.py} (94%) diff --git a/reportportal_client/__init__.py b/reportportal_client/__init__.py index bde50a59..7b12f8c4 100644 --- a/reportportal_client/__init__.py +++ b/reportportal_client/__init__.py @@ -14,10 +14,11 @@ limitations under the License. """ -from ._local import current -from .logs import RPLogger, RPLogHandler -from .client import RPClient -from .steps import step +# noinspection PyProtectedMember +from reportportal_client._local import current +from reportportal_client.logs import RPLogger, RPLogHandler +from reportportal_client.client import RPClient +from reportportal_client.steps import step __all__ = [ 'current', diff --git a/reportportal_client/async/__init__.py b/reportportal_client/async/__init__.py new file mode 100644 index 00000000..1c768643 --- /dev/null +++ b/reportportal_client/async/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License diff --git a/reportportal_client/client_async.py b/reportportal_client/async/client.py similarity index 94% rename from reportportal_client/client_async.py rename to reportportal_client/async/client.py index 71206f9a..0e7c2a0e 100644 --- a/reportportal_client/client_async.py +++ b/reportportal_client/async/client.py @@ -21,10 +21,10 @@ import aiohttp -from .core.rp_issues import Issue -from .logs.log_manager import LogManager, MAX_LOG_BATCH_PAYLOAD_SIZE -from .static.defines import NOT_FOUND -from .steps import StepReporter +from reportportal_client.core.rp_issues import Issue +from reportportal_client.logs.log_manager import LogManager, MAX_LOG_BATCH_PAYLOAD_SIZE +from reportportal_client.static.defines import NOT_FOUND +from reportportal_client.steps import StepReporter logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -37,7 +37,7 @@ def last(self): return self.queue[-1] -class _RPClient: +class _RPClientAsync: _log_manager: LogManager = ... api_v1: str = ... api_v2: str = ... @@ -129,7 +129,7 @@ async def start_launch(self, attributes: Optional[Union[List, Dict]] = None, rerun: bool = False, rerun_of: Optional[str] = None, - **kwargs) -> Optional[Union[asyncio.Future, str]]: + **kwargs) -> Optional[str]: pass async def start_test_item(self, @@ -145,7 +145,7 @@ async def start_test_item(self, code_ref: Optional[str] = None, retry: bool = False, test_case_id: Optional[str] = None, - **_: Any) -> Optional[Union[asyncio.Future, str]]: + **_: Any) -> Optional[str]: parent = parent_item_id if parent_item_id and asyncio.isfuture(parent_item_id): parent = await parent_item_id @@ -167,13 +167,13 @@ def current_item(self) -> Union[asyncio.Future, str]: """Retrieve the last item reported by the client.""" return self._item_stack.last() - def clone(self) -> '_RPClient': + def clone(self) -> '_RPClientAsync': """Clone the client object, set current Item ID as cloned item ID. :returns: Cloned client object - :rtype: _RPClient + :rtype: _RPClientAsync """ - cloned = _RPClient( + cloned = _RPClientAsync( endpoint=self.endpoint, project=self.project, api_key=self.api_key, @@ -193,7 +193,7 @@ def clone(self) -> '_RPClient': return cloned -class RPClientAsync(_RPClient): +class RPClientAsync(_RPClientAsync): async def start_test_item(self, name: str, @@ -208,7 +208,7 @@ async def start_test_item(self, code_ref: Optional[str] = None, retry: bool = False, test_case_id: Optional[str] = None, - **kwargs: Any) -> Optional[Union[asyncio.Future, str]]: + **kwargs: Any) -> Optional[str]: item_id = await super().start_test_item(name, start_time, item_type, description=description, attributes=attributes, parameters=parameters, parent_item_id=parent_item_id, has_stats=has_stats, @@ -235,7 +235,7 @@ async def finish_test_item(self, return result -class RPClientSync(_RPClient): +class RPClientSync(_RPClientAsync): loop: asyncio.AbstractEventLoop thread: threading.Thread @@ -280,7 +280,7 @@ def start_test_item(self, code_ref: Optional[str] = None, retry: bool = False, test_case_id: Optional[str] = None, - **kwargs: Any) -> Optional[Union[asyncio.Future, str]]: + **kwargs: Any) -> Optional[asyncio.Future]: item_id_coro = super().start_test_item(name, start_time, item_type, description=description, attributes=attributes, parameters=parameters, parent_item_id=parent_item_id, has_stats=has_stats, diff --git a/reportportal_client/steps/__init__.py b/reportportal_client/steps/__init__.py index 23fc9669..6052d091 100644 --- a/reportportal_client/steps/__init__.py +++ b/reportportal_client/steps/__init__.py @@ -10,6 +10,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License + """Report portal Nested Step handling module. The module for handling and reporting Report Portal Nested Steps inside python From 41abcedc73039daf058d86ee9d52c3b66c9c7815 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 30 Aug 2023 17:06:24 +0300 Subject: [PATCH 004/268] Async RPClient: WIP --- reportportal_client/core/rp_requests.py | 229 +++++++++++++++-------- reportportal_client/core/rp_requests.pyi | 214 --------------------- reportportal_client/static/abstract.py | 30 ++- 3 files changed, 165 insertions(+), 308 deletions(-) delete mode 100644 reportportal_client/core/rp_requests.pyi diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 9f2b6b18..4cb6e18d 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -20,6 +20,7 @@ import json import logging +from typing import Callable, Text, Optional, Union, Dict, List, ByteString, IO, Tuple from reportportal_client import helpers from reportportal_client.core.rp_file import RPFile @@ -32,9 +33,9 @@ from reportportal_client.static.defines import ( DEFAULT_PRIORITY, LOW_PRIORITY, - RP_LOG_LEVELS + RP_LOG_LEVELS, Priority ) -from .rp_responses import RPResponse +from reportportal_client.core.rp_responses import RPResponse logger = logging.getLogger(__name__) @@ -42,16 +43,31 @@ class HttpRequest: """This model stores attributes related to RP HTTP requests.""" - def __init__(self, session_method, url, data=None, json=None, - files=None, verify_ssl=True, http_timeout=(10, 10), - name=None): + session_method: Callable + url: Text + files: Optional[Dict] + data: Optional[Union[Dict, List[Union[tuple, ByteString]], IO]] + json: Optional[Dict] + verify_ssl: Optional[bool] + http_timeout: Union[float, Tuple[float, float]] + name: Optional[Text] + + def __init__(self, + session_method: Callable, + url: Text, + data: Optional[Union[Dict, List[Union[tuple, ByteString, IO]]]] = None, + json_data: Optional[Dict] = None, + files_data: Optional[Dict] = None, + verify_ssl: Optional[bool] = None, + http_timeout: Union[float, Tuple[float, float]] = (10, 10), + name: Optional[Text] = None) -> None: """Initialize instance attributes. :param session_method: Method of the requests.Session instance :param url: Request URL :param data: Dictionary, list of tuples, bytes, or file-like object to send in the body of the request - :param json: JSON to be sent in the body of the request + :param json_data: JSON to be sent in the body of the request :param verify_ssl: Is SSL certificate verification required :param http_timeout: a float in seconds for connect and read timeout. Use a Tuple to specific connect and @@ -59,8 +75,8 @@ def __init__(self, session_method, url, data=None, json=None, :param name: request name """ self.data = data - self.files = files - self.json = json + self.files = files_data + self.json = json_data self.session_method = session_method self.url = url self.verify_ssl = verify_ssl @@ -84,53 +100,81 @@ def make(self): ) -class RPRequestBase(object): +class AsyncHttpRequest(HttpRequest): + """This model stores attributes related to RP HTTP requests.""" + + def __init__(self, session_method: Callable, url: str, data=None, json=None, + files=None, verify_ssl=True, http_timeout=(10, 10), + name=None) -> None: + super().__init__(session_method, url, data, json, files, verify_ssl, http_timeout, name) + + async def make(self): + """Make HTTP request to the Report Portal API.""" + try: + return RPResponse(self.session_method( + self.url, data=self.data, json=self.json, + files=self.files, verify=self.verify_ssl, + timeout=self.http_timeout) + ) + # https://github.com/reportportal/client-Python/issues/39 + except (KeyError, IOError, ValueError, TypeError) as exc: + logger.warning( + "Report Portal %s request failed", + self.name, + exc_info=exc + ) + + +class RPRequestBase(metaclass=AbstractBaseClass): """Base class for the rest of the RP request models.""" __metaclass__ = AbstractBaseClass + _http_request: Optional[HttpRequest] = ... + _priority: Priority = ... + _response: Optional[RPResponse] = ... - def __init__(self): + def __init__(self) -> None: """Initialize instance attributes.""" self._http_request = None self._priority = DEFAULT_PRIORITY self._response = None - def __lt__(self, other): + def __lt__(self, other) -> bool: """Priority protocol for the PriorityQueue.""" return self.priority < other.priority @property - def http_request(self): + def http_request(self) -> HttpRequest: """Get the HttpRequest object of the request.""" return self._http_request @http_request.setter - def http_request(self, value): + def http_request(self, value: HttpRequest) -> None: """Set the HttpRequest object of the request.""" self._http_request = value @property - def priority(self): + def priority(self) -> Priority: """Get the priority of the request.""" return self._priority @priority.setter - def priority(self, value): + def priority(self, value: Priority) -> None: """Set the priority of the request.""" self._priority = value @property - def response(self): + def response(self) -> Optional[RPResponse]: """Get the response object for the request.""" return self._response @response.setter - def response(self, value): + def response(self, value: RPResponse) -> None: """Set the response object for the request.""" self._response = value @abstractmethod - def payload(self): + def payload(self) -> Dict: """Abstract interface for getting HTTP request payload.""" raise NotImplementedError('Payload interface is not implemented!') @@ -140,15 +184,23 @@ class LaunchStartRequest(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#start-launch """ + attributes: Optional[Union[List, Dict]] + description: Text + mode: Text + name: Text + rerun: bool + rerun_of: Text + start_time: Text + uuid: Text def __init__(self, - name, - start_time, - attributes=None, - description=None, - mode='default', - rerun=False, - rerun_of=None): + name: Text, + start_time: Text, + attributes: Optional[Union[List, Dict]] = None, + description: Optional[Text] = None, + mode: Text = 'default', + rerun: bool = False, + rerun_of: Optional[Text] = None) -> None: """Initialize instance attributes. :param name: Name of the launch @@ -160,7 +212,7 @@ def __init__(self, :param rerun_of: Rerun mode. Specifies launch to be re-runned. Uses with the 'rerun' attribute. """ - super(LaunchStartRequest, self).__init__() + super().__init__() self.attributes = attributes self.description = description self.mode = mode @@ -170,7 +222,7 @@ def __init__(self, self.start_time = start_time @property - def payload(self): + def payload(self) -> Dict: """Get HTTP payload for the request.""" if self.attributes and isinstance(self.attributes, dict): self.attributes = dict_to_payload(self.attributes) @@ -192,10 +244,10 @@ class LaunchFinishRequest(RPRequestBase): """ def __init__(self, - end_time, - status=None, - attributes=None, - description=None): + end_time: Text, + status: Optional[Text] = None, + attributes: Optional[Union[List, Dict]] = None, + description: Optional[Text] = None) -> None: """Initialize instance attributes. :param end_time: Launch end time @@ -206,14 +258,14 @@ def __init__(self, Overrides attributes on start :param description: Launch description. Overrides description on start """ - super(LaunchFinishRequest, self).__init__() + super().__init__() self.attributes = attributes self.description = description self.end_time = end_time self.status = status @property - def payload(self): + def payload(self) -> Dict: """Get HTTP payload for the request.""" if self.attributes and isinstance(self.attributes, dict): self.attributes = dict_to_payload(self.attributes) @@ -230,21 +282,30 @@ class ItemStartRequest(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#start-rootsuite-item """ + attributes: Optional[Union[List, Dict]] + code_ref: Optional[Text] + description: Optional[Text] + has_stats: bool + launch_uuid: Text + name: Text + parameters: Optional[Union[List, Dict]] + retry: bool + start_time: Text + test_case_id: Optional[Text] + type_: Text def __init__(self, - name, - start_time, - type_, - launch_uuid, - attributes=None, - code_ref=None, - description=None, - has_stats=True, - parameters=None, - retry=False, - test_case_id=None, - uuid=None, - unique_id=None): + name: Text, + start_time: Text, + type_: Text, + launch_uuid: Text, + attributes: Optional[Union[List, Dict]] = None, + code_ref: Optional[Text] = None, + description: Optional[Text] = None, + has_stats: bool = True, + parameters: Optional[Union[List, Dict]] = None, + retry: bool = False, + test_case_id: Optional[Text] = None) -> None: """Initialize instance attributes. :param name: Name of the test item @@ -264,10 +325,8 @@ def __init__(self, :param retry: Used to report retry of the test. Allowable values: "True" or "False" :param test_case_id:Test case ID from integrated TMS - :param uuid: Test item UUID (auto generated) - :param unique_id: Test item ID (auto generated) """ - super(ItemStartRequest, self).__init__() + super().__init__() self.attributes = attributes self.code_ref = code_ref self.description = description @@ -279,11 +338,9 @@ def __init__(self, self.start_time = start_time self.test_case_id = test_case_id self.type_ = type_ - self.uuid = uuid - self.unique_id = unique_id @property - def payload(self): + def payload(self) -> Dict: """Get HTTP payload for the request.""" if self.attributes and isinstance(self.attributes, dict): self.attributes = dict_to_payload(self.attributes) @@ -309,16 +366,24 @@ class ItemFinishRequest(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#finish-child-item """ + attributes: Optional[Union[List, Dict]] + description: Text + end_time: Text + is_skipped_an_issue: bool + issue: Issue + launch_uuid: Text + status: Text + retry: bool def __init__(self, - end_time, - launch_uuid, - status, - attributes=None, - description=None, - is_skipped_an_issue=True, - issue=None, - retry=False): + end_time: Text, + launch_uuid: Text, + status: Text, + attributes: Optional[Union[List, Dict]] = None, + description: Optional[str] = None, + is_skipped_an_issue: bool = True, + issue: Optional[Issue] = None, + retry: bool = False) -> None: """Initialize instance attributes. :param end_time: Test item end time @@ -336,7 +401,7 @@ def __init__(self, :param retry: Used to report retry of the test. Allowable values: "True" or "False" """ - super(ItemFinishRequest, self).__init__() + super().__init__() self.attributes = attributes self.description = description self.end_time = end_time @@ -347,7 +412,7 @@ def __init__(self, self.retry = retry @property - def payload(self): + def payload(self) -> Dict: """Get HTTP payload for the request.""" if self.attributes and isinstance(self.attributes, dict): self.attributes = dict_to_payload(self.attributes) @@ -373,14 +438,20 @@ class RPRequestLog(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#save-single-log-without-attachment """ + file: Optional[RPFile] + launch_uuid: Text + level: Text + message: Optional[Text] + time: Text + item_uuid: Optional[Text] def __init__(self, - launch_uuid, - time, - file=None, - item_uuid=None, - level=RP_LOG_LEVELS[40000], - message=None): + launch_uuid: Text, + time: Text, + file: Optional[RPFile] = None, + item_uuid: Optional[Text] = None, + level: Text = RP_LOG_LEVELS[40000], + message: Optional[Text] = None) -> None: """Initialize instance attributes. :param launch_uuid: Launch UUID @@ -392,7 +463,7 @@ def __init__(self, trace(5000), fatal(50000), unknown(60000) :param message: Log message """ - super(RPRequestLog, self).__init__() + super().__init__() self.file = file # type: RPFile self.launch_uuid = launch_uuid self.level = level @@ -401,14 +472,14 @@ def __init__(self, self.item_uuid = item_uuid self.priority = LOW_PRIORITY - def __file(self): + def __file(self) -> Dict: """Form file payload part of the payload.""" if not self.file: return {} return {'file': {'name': self.file.name}} @property - def payload(self): + def payload(self) -> Dict: """Get HTTP payload for the request.""" payload = { 'launchUuid': self.launch_uuid, @@ -421,7 +492,7 @@ def payload(self): return payload @property - def multipart_size(self): + def multipart_size(self) -> int: """Calculate request size how it would transfer in Multipart HTTP.""" size = helpers.calculate_json_part_size(self.payload) size += helpers.calculate_file_part_size(self.file) @@ -433,24 +504,26 @@ class RPLogBatch(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#batch-save-logs """ + default_content: Text = ... + log_reqs: List[RPRequestLog] = ... - def __init__(self, log_reqs): + def __init__(self, log_reqs: List[RPRequestLog]) -> None: """Initialize instance attributes. :param log_reqs: """ - super(RPLogBatch, self).__init__() + super().__init__() self.default_content = 'application/octet-stream' self.log_reqs = log_reqs self.priority = LOW_PRIORITY - def __get_file(self, rp_file): + def __get_file(self, rp_file) -> Tuple[str, tuple]: """Form a tuple for the single file.""" return ('file', (rp_file.name, rp_file.content, rp_file.content_type or self.default_content)) - def __get_files(self): + def __get_files(self) -> List[Tuple[str, tuple]]: """Get list of files for the JSON body.""" files = [] for req in self.log_reqs: @@ -458,7 +531,7 @@ def __get_files(self): files.append(self.__get_file(req.file)) return files - def __get_request_part(self): + def __get_request_part(self) -> List[Tuple[str, tuple]]: r"""Form JSON body for the request. Example: @@ -487,6 +560,6 @@ def __get_request_part(self): return body @property - def payload(self): + def payload(self) -> List[Tuple[str, tuple]]: """Get HTTP payload for the request.""" return self.__get_request_part() diff --git a/reportportal_client/core/rp_requests.pyi b/reportportal_client/core/rp_requests.pyi deleted file mode 100644 index 332cccf3..00000000 --- a/reportportal_client/core/rp_requests.pyi +++ /dev/null @@ -1,214 +0,0 @@ -# Copyright (c) 2022 EPAM Systems -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License - -from typing import Any, Callable, ByteString, Dict, IO, List, Optional, Text, \ - Union, Tuple - -from reportportal_client.core.rp_file import RPFile as RPFile -from reportportal_client.core.rp_issues import Issue as Issue -from reportportal_client.core.rp_responses import RPResponse as RPResponse -from reportportal_client.static.abstract import AbstractBaseClass -from reportportal_client.static.defines import Priority as Priority - - -class HttpRequest: - session_method: Callable = ... - url: Text = ... - files: Optional[Dict] = ... - data: Optional[Union[Dict, List[Union[tuple, ByteString]], IO]] = ... - json: Optional[Dict] = ... - verify_ssl: Optional[bool] = ... - http_timeout: Union[float, Tuple[float, float]] = ... - name: Optional[Text] = ... - - def __init__(self, - session_method: Callable, - url: Text, - data=Optional[ - Union[Dict, List[Union[tuple, ByteString, IO]]]], - json=Optional[Dict], - files=Optional[Dict], - verify_ssl=Optional[bool], - name=Optional[Text]) -> None: ... - - def make(self) -> Optional[RPResponse]: ... - - -class RPRequestBase(metaclass=AbstractBaseClass): - __metaclass__: AbstractBaseClass = ... - _http_request: Optional[HttpRequest] = ... - _priority: Priority = ... - _response: Optional[RPResponse] = ... - - def __init__(self) -> None: ... - - def __lt__(self, other: RPRequestBase) -> bool: ... - - @property - def http_request(self) -> HttpRequest: ... - - @http_request.setter - def http_request(self, value: HttpRequest) -> None: ... - - @property - def priority(self) -> Priority: ... - - @priority.setter - def priority(self, value: Priority) -> None: ... - - @property - def response(self) -> Optional[RPResponse]: ... - - @response.setter - def response(self, value: RPResponse) -> None: ... - - def payload(self) -> Dict: ... - - -class LaunchStartRequest(RPRequestBase): - attributes: Optional[Union[List, Dict]] = ... - description: Text = ... - mode: Text = ... - name: Text = ... - rerun: bool = ... - rerun_of: Text = ... - start_time: Text = ... - uuid: Text = ... - - def __init__(self, - name: Text, - start_time: Text, - attributes: Optional[Union[List, Dict]] = ..., - description: Optional[Text] = ..., - mode: Text = ..., - rerun: bool = ..., - rerun_of: Optional[Text] = ..., - uuid: Optional[Text] = ...) -> None: ... - - @property - def payload(self) -> Dict: ... - - -class LaunchFinishRequest(RPRequestBase): - attributes: Optional[Union[List, Dict]] = ... - description: Text = ... - end_time: Text = ... - status: Text = ... - - def __init__(self, - end_time: Text, - status: Optional[Text] = ..., - attributes: Optional[Union[List, Dict]] = ..., - description: Optional[Text] = ...) -> None: ... - - @property - def payload(self) -> Dict: ... - - -class ItemStartRequest(RPRequestBase): - attributes: Optional[Union[List, Dict]] = ... - code_ref: Text = ... - description: Text = ... - has_stats: bool = ... - launch_uuid: Text = ... - name: Text = ... - parameters: Optional[Union[List, Dict]] = ... - retry: bool = ... - start_time: Text = ... - test_case_id: Optional[Text] = ... - type_: Text = ... - uuid: Text = ... - unique_id: Text = ... - - def __init__(self, - name: Text, - start_time: Text, - type_: Text, - launch_uuid: Text, - attributes: Optional[Union[List, Dict]] = ..., - code_ref: Optional[Text] = ..., - description: Optional[Text] = ..., - has_stats: bool = ..., - parameters: Optional[Union[List, Dict]] = ..., - retry: bool = ..., - test_case_id: Optional[Text] = ..., - uuid: Optional[Any] = ..., - unique_id: Optional[Any] = ...) -> None: ... - - @property - def payload(self) -> Dict: ... - - -class ItemFinishRequest(RPRequestBase): - attributes: Optional[Union[List, Dict]] = ... - description: Text = ... - end_time: Text = ... - is_skipped_an_issue: bool = ... - issue: Issue = ... - launch_uuid: Text = ... - status: Text = ... - retry: bool = ... - - def __init__(self, - end_time: Text, - launch_uuid: Text, - status: Text, - attributes: Optional[Union[List, Dict]] = ..., - description: Optional[Any] = ..., - is_skipped_an_issue: bool = ..., - issue: Optional[Issue] = ..., - retry: bool = ...) -> None: ... - - @property - def payload(self) -> Dict: ... - - -class RPRequestLog(RPRequestBase): - file: RPFile = ... - launch_uuid: Text = ... - level: Text = ... - message: Text = ... - time: Text = ... - item_uuid: Text = ... - - def __init__(self, - launch_uuid: Text, - time: Text, - file: Optional[RPFile] = ..., - item_uuid: Optional[Text] = ..., - level: Text = ..., - message: Optional[Text] = ...) -> None: ... - - def __file(self) -> Dict: ... - - @property - def payload(self) -> Dict: ... - - @property - def multipart_size(self) -> int: ... - - -class RPLogBatch(RPRequestBase): - default_content: Text = ... - log_reqs: List[RPRequestLog] = ... - - def __init__(self, log_reqs: List[RPRequestLog]) -> None: ... - - def __get_file(self, rp_file: RPFile) -> tuple: ... - - def __get_files(self) -> List: ... - - def __get_request_part(self) -> Dict: ... - - @property - def payload(self) -> Dict: ... diff --git a/reportportal_client/static/abstract.py b/reportportal_client/static/abstract.py index 8bc59d9b..85e1e9ae 100644 --- a/reportportal_client/static/abstract.py +++ b/reportportal_client/static/abstract.py @@ -1,19 +1,17 @@ -"""This module provides base abstract class for RP request objects. - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" +"""This module provides base abstract class for RP request objects.""" + +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License from abc import ABCMeta as _ABCMeta, abstractmethod From 5c4b6218b2526bf4a96add4fb06720c8fb0360dc Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 31 Aug 2023 10:53:43 +0300 Subject: [PATCH 005/268] Bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 81884edd..cb28102c 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, find_packages -__version__ = '5.4.1' +__version__ = '5.5.0' TYPE_STUBS = ['*.pyi'] From f9a9c2072c2e0f6ac8478c8609befcf3919e4a6e Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 31 Aug 2023 10:54:22 +0300 Subject: [PATCH 006/268] Supported Python versions update --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index cb28102c..f5986161 100644 --- a/setup.py +++ b/setup.py @@ -38,12 +38,11 @@ def read_file(fname): license='Apache 2.0.', keywords=['testing', 'reporting', 'reportportal', 'client'], classifiers=[ - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', ], install_requires=read_file('requirements.txt').splitlines(), ) From 5c6c0336050010564cb8fc9770722995b09328de Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 31 Aug 2023 10:56:26 +0300 Subject: [PATCH 007/268] Add more packages to publish stubs --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f5986161..a1d82a3e 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,10 @@ def read_file(fname): packages=find_packages(exclude=('tests', 'tests.*')), package_data={ 'reportportal_client': TYPE_STUBS, - 'reportportal_client.steps': TYPE_STUBS + 'reportportal_client.steps': TYPE_STUBS, + 'reportportal_client.core': TYPE_STUBS, + 'reportportal_client.logs': TYPE_STUBS, + 'reportportal_client.services': TYPE_STUBS, }, version=__version__, description='Python client for Report Portal v5.', From 1185d54206f3b0b9f0a36f0f71f4857790c9245e Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 31 Aug 2023 14:03:41 +0300 Subject: [PATCH 008/268] Remove deprecated code --- reportportal_client/client.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 412d5680..ee532aac 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -384,31 +384,14 @@ def start_launch(self, re-run. Should be used with the 'rerun' option. """ url = uri_join(self.base_url_v2, 'launch') - - # We are moving 'mode' param to the constructor, next code for the - # transition period only. - my_kwargs = dict(kwargs) - mode = my_kwargs.get('mode') - if 'mode' in my_kwargs: - warnings.warn( - message='Argument `mode` is deprecated since 5.2.5 and will be subject for removing in the ' - 'next major version. Use `mode` argument in the class constructor instead.', - category=DeprecationWarning, - stacklevel=2 - ) - del my_kwargs['mode'] - if not mode: - mode = self.mode - request_payload = LaunchStartRequest( name=name, start_time=start_time, attributes=attributes, description=description, - mode=mode, + mode=self.mode, rerun=rerun, - rerun_of=rerun_of or kwargs.get('rerunOf'), - **my_kwargs + rerun_of=rerun_of or kwargs.get('rerunOf') ).payload response = HttpRequest(self.session.post, url=url, From d765a55038af31d18111728b9fb8415e209deae2 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 31 Aug 2023 14:04:02 +0300 Subject: [PATCH 009/268] Async RPClient: WIP --- reportportal_client/core/rp_requests.py | 126 ++++++++++++------------ 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 4cb6e18d..01dfa8fc 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -18,13 +18,14 @@ # See the License for the specific language governing permissions and # limitations under the License -import json +import json as json_converter import logging -from typing import Callable, Text, Optional, Union, Dict, List, ByteString, IO, Tuple +from typing import Callable, Text, Optional, Union, List, ByteString, IO, Tuple from reportportal_client import helpers from reportportal_client.core.rp_file import RPFile from reportportal_client.core.rp_issues import Issue +from reportportal_client.core.rp_responses import RPResponse from reportportal_client.helpers import dict_to_payload from reportportal_client.static.abstract import ( AbstractBaseClass, @@ -35,7 +36,6 @@ LOW_PRIORITY, RP_LOG_LEVELS, Priority ) -from reportportal_client.core.rp_responses import RPResponse logger = logging.getLogger(__name__) @@ -44,20 +44,20 @@ class HttpRequest: """This model stores attributes related to RP HTTP requests.""" session_method: Callable - url: Text - files: Optional[Dict] - data: Optional[Union[Dict, List[Union[tuple, ByteString]], IO]] - json: Optional[Dict] + url: str + files: Optional[dict] + data: Optional[Union[dict, List[Union[tuple, ByteString]], IO]] + json: Optional[dict] verify_ssl: Optional[bool] http_timeout: Union[float, Tuple[float, float]] name: Optional[Text] def __init__(self, session_method: Callable, - url: Text, - data: Optional[Union[Dict, List[Union[tuple, ByteString, IO]]]] = None, - json_data: Optional[Dict] = None, - files_data: Optional[Dict] = None, + url: str, + data: Optional[Union[dict, List[Union[tuple, ByteString, IO]]]] = None, + json: Optional[dict] = None, + files: Optional[dict] = None, verify_ssl: Optional[bool] = None, http_timeout: Union[float, Tuple[float, float]] = (10, 10), name: Optional[Text] = None) -> None: @@ -67,7 +67,7 @@ def __init__(self, :param url: Request URL :param data: Dictionary, list of tuples, bytes, or file-like object to send in the body of the request - :param json_data: JSON to be sent in the body of the request + :param json: JSON to be sent in the body of the request :param verify_ssl: Is SSL certificate verification required :param http_timeout: a float in seconds for connect and read timeout. Use a Tuple to specific connect and @@ -75,8 +75,8 @@ def __init__(self, :param name: request name """ self.data = data - self.files = files_data - self.json = json_data + self.files = files + self.json = json self.session_method = session_method self.url = url self.verify_ssl = verify_ssl @@ -174,7 +174,7 @@ def response(self, value: RPResponse) -> None: self._response = value @abstractmethod - def payload(self) -> Dict: + def payload(self) -> dict: """Abstract interface for getting HTTP request payload.""" raise NotImplementedError('Payload interface is not implemented!') @@ -184,21 +184,21 @@ class LaunchStartRequest(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#start-launch """ - attributes: Optional[Union[List, Dict]] - description: Text - mode: Text - name: Text + attributes: Optional[Union[list, dict]] + description: str + mode: str + name: str rerun: bool - rerun_of: Text - start_time: Text - uuid: Text + rerun_of: str + start_time: str + uuid: str def __init__(self, - name: Text, - start_time: Text, - attributes: Optional[Union[List, Dict]] = None, + name: str, + start_time: str, + attributes: Optional[Union[list, dict]] = None, description: Optional[Text] = None, - mode: Text = 'default', + mode: str = 'default', rerun: bool = False, rerun_of: Optional[Text] = None) -> None: """Initialize instance attributes. @@ -222,7 +222,7 @@ def __init__(self, self.start_time = start_time @property - def payload(self) -> Dict: + def payload(self) -> dict: """Get HTTP payload for the request.""" if self.attributes and isinstance(self.attributes, dict): self.attributes = dict_to_payload(self.attributes) @@ -244,9 +244,9 @@ class LaunchFinishRequest(RPRequestBase): """ def __init__(self, - end_time: Text, + end_time: str, status: Optional[Text] = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, description: Optional[Text] = None) -> None: """Initialize instance attributes. @@ -265,7 +265,7 @@ def __init__(self, self.status = status @property - def payload(self) -> Dict: + def payload(self) -> dict: """Get HTTP payload for the request.""" if self.attributes and isinstance(self.attributes, dict): self.attributes = dict_to_payload(self.attributes) @@ -282,28 +282,28 @@ class ItemStartRequest(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#start-rootsuite-item """ - attributes: Optional[Union[List, Dict]] + attributes: Optional[Union[list, dict]] code_ref: Optional[Text] description: Optional[Text] has_stats: bool - launch_uuid: Text - name: Text - parameters: Optional[Union[List, Dict]] + launch_uuid: str + name: str + parameters: Optional[Union[list, dict]] retry: bool - start_time: Text + start_time: str test_case_id: Optional[Text] - type_: Text + type_: str def __init__(self, - name: Text, - start_time: Text, - type_: Text, - launch_uuid: Text, - attributes: Optional[Union[List, Dict]] = None, + name: str, + start_time: str, + type_: str, + launch_uuid: str, + attributes: Optional[Union[list, dict]] = None, code_ref: Optional[Text] = None, description: Optional[Text] = None, has_stats: bool = True, - parameters: Optional[Union[List, Dict]] = None, + parameters: Optional[Union[list, dict]] = None, retry: bool = False, test_case_id: Optional[Text] = None) -> None: """Initialize instance attributes. @@ -340,7 +340,7 @@ def __init__(self, self.type_ = type_ @property - def payload(self) -> Dict: + def payload(self) -> dict: """Get HTTP payload for the request.""" if self.attributes and isinstance(self.attributes, dict): self.attributes = dict_to_payload(self.attributes) @@ -366,20 +366,20 @@ class ItemFinishRequest(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#finish-child-item """ - attributes: Optional[Union[List, Dict]] - description: Text - end_time: Text + attributes: Optional[Union[list, dict]] + description: str + end_time: str is_skipped_an_issue: bool issue: Issue - launch_uuid: Text - status: Text + launch_uuid: str + status: str retry: bool def __init__(self, - end_time: Text, - launch_uuid: Text, - status: Text, - attributes: Optional[Union[List, Dict]] = None, + end_time: str, + launch_uuid: str, + status: str, + attributes: Optional[Union[list, dict]] = None, description: Optional[str] = None, is_skipped_an_issue: bool = True, issue: Optional[Issue] = None, @@ -412,7 +412,7 @@ def __init__(self, self.retry = retry @property - def payload(self) -> Dict: + def payload(self) -> dict: """Get HTTP payload for the request.""" if self.attributes and isinstance(self.attributes, dict): self.attributes = dict_to_payload(self.attributes) @@ -439,18 +439,18 @@ class RPRequestLog(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#save-single-log-without-attachment """ file: Optional[RPFile] - launch_uuid: Text - level: Text + launch_uuid: str + level: str message: Optional[Text] - time: Text + time: str item_uuid: Optional[Text] def __init__(self, - launch_uuid: Text, - time: Text, + launch_uuid: str, + time: str, file: Optional[RPFile] = None, item_uuid: Optional[Text] = None, - level: Text = RP_LOG_LEVELS[40000], + level: str = RP_LOG_LEVELS[40000], message: Optional[Text] = None) -> None: """Initialize instance attributes. @@ -472,14 +472,14 @@ def __init__(self, self.item_uuid = item_uuid self.priority = LOW_PRIORITY - def __file(self) -> Dict: + def __file(self) -> dict: """Form file payload part of the payload.""" if not self.file: return {} return {'file': {'name': self.file.name}} @property - def payload(self) -> Dict: + def payload(self) -> dict: """Get HTTP payload for the request.""" payload = { 'launchUuid': self.launch_uuid, @@ -504,7 +504,7 @@ class RPLogBatch(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#batch-save-logs """ - default_content: Text = ... + default_content: str = ... log_reqs: List[RPRequestLog] = ... def __init__(self, log_reqs: List[RPRequestLog]) -> None: @@ -552,7 +552,7 @@ def __get_request_part(self) -> List[Tuple[str, tuple]]: body = [( 'json_request_part', ( None, - json.dumps([log.payload for log in self.log_reqs]), + json_converter.dumps([log.payload for log in self.log_reqs]), 'application/json' ) )] From e02b08bb2a86735e65f5614a51d20c77fd251e3a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 31 Aug 2023 17:48:49 +0300 Subject: [PATCH 010/268] Async RPClient: WIP --- reportportal_client/core/rp_requests.py | 69 ++++++++++++++++------- reportportal_client/core/rp_responses.py | 20 ++----- reportportal_client/core/rp_responses.pyi | 1 - 3 files changed, 53 insertions(+), 37 deletions(-) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 01dfa8fc..f7bebf7d 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -18,9 +18,13 @@ # See the License for the specific language governing permissions and # limitations under the License +import asyncio import json as json_converter import logging -from typing import Callable, Text, Optional, Union, List, ByteString, IO, Tuple +import ssl +from typing import Callable, Text, Optional, Union, List, Tuple, Any, TypeVar + +import aiohttp from reportportal_client import helpers from reportportal_client.core.rp_file import RPFile @@ -38,26 +42,36 @@ ) logger = logging.getLogger(__name__) +T = TypeVar("T") + + +async def await_if_necessary(obj: Optional[Any]) -> Any: + if obj: + if asyncio.isfuture(obj) or asyncio.iscoroutine(obj): + return await obj + elif asyncio.iscoroutinefunction(obj): + return await obj() + return obj class HttpRequest: """This model stores attributes related to RP HTTP requests.""" session_method: Callable - url: str - files: Optional[dict] - data: Optional[Union[dict, List[Union[tuple, ByteString]], IO]] - json: Optional[dict] - verify_ssl: Optional[bool] + url: Any + files: Optional[Any] + data: Optional[Any] + json: Optional[Any] + verify_ssl: Optional[Union[bool, str]] http_timeout: Union[float, Tuple[float, float]] - name: Optional[Text] + name: Optional[str] def __init__(self, session_method: Callable, - url: str, - data: Optional[Union[dict, List[Union[tuple, ByteString, IO]]]] = None, - json: Optional[dict] = None, - files: Optional[dict] = None, + url: Any, + data: Optional[Any] = None, + json: Optional[Any] = None, + files: Optional[Any] = None, verify_ssl: Optional[bool] = None, http_timeout: Union[float, Tuple[float, float]] = (10, 10), name: Optional[Text] = None) -> None: @@ -86,11 +100,8 @@ def __init__(self, def make(self): """Make HTTP request to the Report Portal API.""" try: - return RPResponse(self.session_method( - self.url, data=self.data, json=self.json, - files=self.files, verify=self.verify_ssl, - timeout=self.http_timeout) - ) + return RPResponse(self.session_method(self.url, data=self.data, json=self.json, files=self.files, + verify=self.verify_ssl, timeout=self.http_timeout)) # https://github.com/reportportal/client-Python/issues/39 except (KeyError, IOError, ValueError, TypeError) as exc: logger.warning( @@ -110,11 +121,29 @@ def __init__(self, session_method: Callable, url: str, data=None, json=None, async def make(self): """Make HTTP request to the Report Portal API.""" + ssl_config = self.verify_ssl + if ssl_config and type(ssl_config) == str: + ssl_context = ssl.create_default_context() + ssl_context.load_cert_chain(ssl_config) + ssl_config = ssl_context + + timeout_config = self.http_timeout + if not timeout_config or not type(timeout_config) == tuple: + timeout_config = (timeout_config, timeout_config) + + data = self.data + if self.files: + data = self.files + try: - return RPResponse(self.session_method( - self.url, data=self.data, json=self.json, - files=self.files, verify=self.verify_ssl, - timeout=self.http_timeout) + return RPResponse( + await self.session_method( + await await_if_necessary(self.url), + data=data, + json=self.json, + ssl=ssl_config, + timeout=aiohttp.ClientTimeout(connect=timeout_config[0], sock_read=timeout_config[1]) + ) ) # https://github.com/reportportal/client-Python/issues/39 except (KeyError, IOError, ValueError, TypeError) as exc: diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index 910e3b11..6e70c6cf 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -19,6 +19,7 @@ # limitations under the License import logging +from functools import lru_cache from reportportal_client.static.defines import NOT_FOUND @@ -51,7 +52,7 @@ def is_empty(self): return self.message is NOT_FOUND -class RPResponse(object): +class RPResponse: """Class representing RP API response.""" __slots__ = ['_data', '_resp'] @@ -61,7 +62,6 @@ def __init__(self, data): :param data: requests.Response object """ - self._data = self._get_json(data) self._resp = data @staticmethod @@ -83,25 +83,13 @@ def is_success(self): """Check if response to API has been successful.""" return self._resp.ok - def _iter_messages(self): - """Generate RPMessage for each response.""" - data = self.json.get('responses', [self.json]) - for chunk in data: - message = RPMessage(chunk) - if not message.is_empty: - yield message - @property + @lru_cache() def json(self): """Get the response in dictionary.""" - return self._data + return self._get_json(self._resp) @property def message(self): """Get value of the 'message' key.""" return self.json.get('message', NOT_FOUND) - - @property - def messages(self): - """Get list of messages received.""" - return tuple(self._iter_messages()) diff --git a/reportportal_client/core/rp_responses.pyi b/reportportal_client/core/rp_responses.pyi index 92d95c05..f25774c2 100644 --- a/reportportal_client/core/rp_responses.pyi +++ b/reportportal_client/core/rp_responses.pyi @@ -22,7 +22,6 @@ class RPMessage: def is_empty(self) -> bool: ... class RPResponse: - _data: Dict = ... _resp: Response = ... def __init__(self, data: Response) -> None: ... def _get_json(self, data: Response) -> Dict: ... From a1beb2b34e1ef0bf56ba47af6574c8465f5b4ba1 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 31 Aug 2023 17:49:23 +0300 Subject: [PATCH 011/268] Async RPClient: WIP --- reportportal_client/core/rp_responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index 6e70c6cf..024d2ac9 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -55,7 +55,7 @@ def is_empty(self): class RPResponse: """Class representing RP API response.""" - __slots__ = ['_data', '_resp'] + __slots__ = ['_resp'] def __init__(self, data): """Initialize instance attributes. From 8412f4e4bc6379b8e7314c00bba5ef6ea2bd4857 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 31 Aug 2023 17:49:35 +0300 Subject: [PATCH 012/268] Async RPClient: WIP --- reportportal_client/core/rp_responses.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index 024d2ac9..133e948c 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -55,8 +55,6 @@ def is_empty(self): class RPResponse: """Class representing RP API response.""" - __slots__ = ['_resp'] - def __init__(self, data): """Initialize instance attributes. From 1c0269aaae876e6840fef7e0990c4d84291da247 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 31 Aug 2023 17:49:48 +0300 Subject: [PATCH 013/268] Async RPClient: WIP --- reportportal_client/core/rp_responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index 133e948c..dacc0246 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) -class RPMessage(object): +class RPMessage: """Model for the message returned by RP API.""" __slots__ = ['message', 'error_code'] From f548e14924d38309afa5d1d84af0a66aa106fa74 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 1 Sep 2023 12:34:34 +0300 Subject: [PATCH 014/268] Remove test manager as not used code, refactor core model, move priority and response methods into HttpRequest class --- reportportal_client/core/rp_requests.py | 83 +++---- reportportal_client/core/test_manager.py | 230 ------------------ reportportal_client/core/worker.py | 9 +- reportportal_client/core/worker.pyi | 6 +- reportportal_client/items/__init__.py | 16 -- reportportal_client/items/item_weight.py | 23 -- reportportal_client/items/rp_base_item.py | 64 ----- .../items/rp_log_items/__init__.py | 16 -- .../items/rp_log_items/rp_log_item.py | 70 ------ .../items/rp_test_items/__init__.py | 16 -- .../items/rp_test_items/rp_base_test_item.py | 89 ------- .../items/rp_test_items/rp_child_test_item.py | 76 ------ .../items/rp_test_items/rp_root_test_item.py | 70 ------ reportportal_client/logs/log_manager.py | 3 +- reportportal_client/static/__init__.py | 30 ++- reportportal_client/static/defines.py | 32 ++- reportportal_client/static/errors.py | 31 ++- reportportal_client/steps/__init__.py | 27 +- 18 files changed, 100 insertions(+), 791 deletions(-) delete mode 100644 reportportal_client/core/test_manager.py delete mode 100644 reportportal_client/items/__init__.py delete mode 100644 reportportal_client/items/item_weight.py delete mode 100644 reportportal_client/items/rp_base_item.py delete mode 100644 reportportal_client/items/rp_log_items/__init__.py delete mode 100644 reportportal_client/items/rp_log_items/rp_log_item.py delete mode 100644 reportportal_client/items/rp_test_items/__init__.py delete mode 100644 reportportal_client/items/rp_test_items/rp_base_test_item.py delete mode 100644 reportportal_client/items/rp_test_items/rp_child_test_item.py delete mode 100644 reportportal_client/items/rp_test_items/rp_root_test_item.py diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index f7bebf7d..94a2edfc 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -65,6 +65,8 @@ class HttpRequest: verify_ssl: Optional[Union[bool, str]] http_timeout: Union[float, Tuple[float, float]] name: Optional[str] + _priority: Priority + _response: Optional[RPResponse] def __init__(self, session_method: Callable, @@ -96,12 +98,37 @@ def __init__(self, self.verify_ssl = verify_ssl self.http_timeout = http_timeout self.name = name + self._priority = DEFAULT_PRIORITY + self._response = None + + def __lt__(self, other) -> bool: + """Priority protocol for the PriorityQueue.""" + return self.priority < other.priority + + @property + def priority(self) -> Priority: + """Get the priority of the request.""" + return self._priority + + @priority.setter + def priority(self, value: Priority) -> None: + """Set the priority of the request.""" + self._priority = value + + @property + def response(self) -> Optional[RPResponse]: + """Get the response object for the request.""" + if not self._response: + return self.make() + return self._response def make(self): """Make HTTP request to the Report Portal API.""" try: - return RPResponse(self.session_method(self.url, data=self.data, json=self.json, files=self.files, - verify=self.verify_ssl, timeout=self.http_timeout)) + self._response = RPResponse( + self.session_method(self.url, data=self.data, json=self.json, files=self.files, + verify=self.verify_ssl, timeout=self.http_timeout)) + return self._response # https://github.com/reportportal/client-Python/issues/39 except (KeyError, IOError, ValueError, TypeError) as exc: logger.warning( @@ -158,49 +185,6 @@ class RPRequestBase(metaclass=AbstractBaseClass): """Base class for the rest of the RP request models.""" __metaclass__ = AbstractBaseClass - _http_request: Optional[HttpRequest] = ... - _priority: Priority = ... - _response: Optional[RPResponse] = ... - - def __init__(self) -> None: - """Initialize instance attributes.""" - self._http_request = None - self._priority = DEFAULT_PRIORITY - self._response = None - - def __lt__(self, other) -> bool: - """Priority protocol for the PriorityQueue.""" - return self.priority < other.priority - - @property - def http_request(self) -> HttpRequest: - """Get the HttpRequest object of the request.""" - return self._http_request - - @http_request.setter - def http_request(self, value: HttpRequest) -> None: - """Set the HttpRequest object of the request.""" - self._http_request = value - - @property - def priority(self) -> Priority: - """Get the priority of the request.""" - return self._priority - - @priority.setter - def priority(self, value: Priority) -> None: - """Set the priority of the request.""" - self._priority = value - - @property - def response(self) -> Optional[RPResponse]: - """Get the response object for the request.""" - return self._response - - @response.setter - def response(self, value: RPResponse) -> None: - """Set the response object for the request.""" - self._response = value @abstractmethod def payload(self) -> dict: @@ -229,7 +213,8 @@ def __init__(self, description: Optional[Text] = None, mode: str = 'default', rerun: bool = False, - rerun_of: Optional[Text] = None) -> None: + rerun_of: Optional[Text] = None, + uuid: str = None) -> None: """Initialize instance attributes. :param name: Name of the launch @@ -249,13 +234,14 @@ def __init__(self, self.rerun = rerun self.rerun_of = rerun_of self.start_time = start_time + self.uuid = uuid @property def payload(self) -> dict: """Get HTTP payload for the request.""" if self.attributes and isinstance(self.attributes, dict): self.attributes = dict_to_payload(self.attributes) - return { + result = { 'attributes': self.attributes, 'description': self.description, 'mode': self.mode, @@ -264,6 +250,9 @@ def payload(self) -> dict: 'rerunOf': self.rerun_of, 'startTime': self.start_time } + if self.uuid: + result['uuid'] = self.uuid + return result class LaunchFinishRequest(RPRequestBase): diff --git a/reportportal_client/core/test_manager.py b/reportportal_client/core/test_manager.py deleted file mode 100644 index 9fdc2ddc..00000000 --- a/reportportal_client/core/test_manager.py +++ /dev/null @@ -1,230 +0,0 @@ -"""This module contains functional for test items management.""" - -# Copyright (c) 2022 EPAM Systems -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License - -from reportportal_client.helpers import generate_uuid, dict_to_payload -from reportportal_client.items.rp_log_items.rp_log_item import RPLogItem -from reportportal_client.items.rp_test_items.rp_child_test_item import \ - RPChildTestItem -from reportportal_client.items.rp_test_items.rp_root_test_item import \ - RPRootTestItem - - -class TestManager(object): - """Manage test items during single launch. - - Test item types (item_type) can be: - (SUITE, STORY, TEST, SCENARIO, STEP, BEFORE_CLASS, - BEFORE_GROUPS, BEFORE_METHOD, BEFORE_SUITE, BEFORE_TEST, AFTER_CLASS, - AFTER_GROUPS, AFTER_METHOD, AFTER_SUITE, AFTER_TEST). - - 'attributes' and 'parameters' should be a dictionary - with the following format: - { - "": "", - "": "", - ... - } - """ - - def __init__(self, - session, - endpoint, - project_name, - launch_id=None - ): - """Initialize instance attributes. - - :param session: Session object - :param endpoint: Endpoint url - :param launch_id: Report portal launch UUID - :param project_name: RP project name - """ - self.session = session - self.endpoint = endpoint - self.project_name = project_name - self.launch_id = launch_id - self.__storage = [] - - def start_test_item(self, - api_version, - name, - start_time, - item_type, - description=None, - attributes=None, - parameters=None, - parent_uuid=None, - has_stats=True, - **kwargs): - """Start new test item. - - :param api_version: RP API version - :param name: test item name - :param start_time: test item execution start time - :param item_type: test item type (see class doc string) - :param description: test item description - :param attributes: test item attributes(tags) - Pairs of key and value (see class doc string) - :param parameters: test item set of parameters - (for parametrized tests) (see class doc string) - :param parent_uuid: UUID of parent test item - :param has_stats: True - regular test item, False - test item - without statistics (nested step) - :param kwargs: other parameters - :return: test item UUID - """ - if attributes and isinstance(attributes, dict): - attributes = dict_to_payload(attributes) - if parameters: - parameters = dict_to_payload(parameters) - - item_data = { - "description": description, - "attributes": attributes, - "parameters": parameters, - "has_stats": has_stats - } - kwargs and item_data.update(kwargs) - uuid = generate_uuid() - if not parent_uuid: - test_item = RPRootTestItem(self.endpoint, - self.session, - self.project_name, - name, - item_type, - self.launch_id, - uuid, - **item_data) - self.__storage.append(test_item) - else: - parent_item = self.get_test_item(parent_uuid) - test_item = RPChildTestItem(self.endpoint, - self.session, - self.project_name, - parent_item, - name, - item_type, - self.launch_id, - uuid, - **item_data) - test_item.start(api_version, start_time) - return uuid - - def update_test_item(self, api_version, item_uuid, attributes=None, - description=None, **kwargs): - """Update existing test item at the Report Portal. - - :param api_version: RP API version - :param str item_uuid: test item UUID returned on the item start - :param str description: test item description - :param dict attributes: test item attributes(tags) - Pairs of key and value (see class doc string) - """ - self.get_test_item(item_uuid) - raise NotImplementedError() - - def finish_test_item(self, - api_version, - item_uuid, - end_time, - status, - issue=None, - attributes=None, - **kwargs): - """Finish test item. - - :param api_version:RP API version - :param item_uuid: id of the test item - :param end_time: time in UTC format - :param status: status of the test - :param issue: description of an issue - :param attributes: dict with attributes - :param kwargs: other parameters - """ - # check if the test is skipped, if not - do not mark as TO INVESTIGATE - if issue is None and status == "SKIPPED": - issue = {"issue_type": "NOT_ISSUE"} - if attributes and isinstance(attributes, dict): - attributes = dict_to_payload(attributes) - self.get_test_item(item_uuid).finish(api_version, - end_time, - status, - issue=issue, - attributes=attributes, - **kwargs) - - def remove_test_item(self, api_version, item_uuid): - """Remove test item by uuid. - - :param api_version: RP API version - :param item_uuid: test item uuid - """ - self.get_test_item(item_uuid) - raise NotImplementedError() - - def log(self, api_version, time, message=None, level=None, attachment=None, - item_id=None): - """Log message. Can be added to test item in any state. - - :param api_version: RP API version - :param time: log time - :param message: log message - :param level: log level - :param attachment: attachments to log (images,files,etc.) - :param item_id: parent item UUID - :return: log item UUID - """ - uuid = generate_uuid() - log_item = RPLogItem(self.endpoint, - self.session, - self.project_name, - self.launch_id, - uuid) - log_item.create(api_version, time, attachment, item_id, level, message) - return uuid - - def get_test_item(self, item_uuid): - """Get test item by its uuid in the storage. - - :param item_uuid: test item uuid - :return: test item object if found else None - """ - # Todo: add 'force' parameter to get item from report portal server - # instead of cache and update cache data according to this request - return self._find_item(item_uuid, self.__storage) - - def _find_item(self, item_uuid, storage): - """Find test item by its uuid in given storage. - - :param item_uuid: test item uuid - :param storage: list with test item objects - :return: test item object if found else None - """ - for test_item in reversed(storage): - if item_uuid == test_item.generated_id: - return test_item - else: - if hasattr(test_item, "child_items") and test_item.child_items: - found_item = self._find_item(item_uuid, - test_item.child_items) - if found_item: - return found_item - - def get_storage(self): - """Get storage. - - :return: storage with test items - """ - return self.__storage diff --git a/reportportal_client/core/worker.py b/reportportal_client/core/worker.py index 92acdf04..b787c7bd 100644 --- a/reportportal_client/core/worker.py +++ b/reportportal_client/core/worker.py @@ -86,12 +86,13 @@ def _command_process(self, cmd): self._stop_immediately() else: self._stop() + self._queue.task_done() def _request_process(self, request): """Send request to RP and update response attribute of the request.""" logger.debug('[%s] Processing {%s} request', self.name, request) try: - request.response = request.http_request.make() + request.make() except Exception as err: logger.exception('[%s] Unknown exception has occurred. ' 'Skipping it.', err) @@ -127,11 +128,7 @@ def _stop(self): This method process everything in worker's queue first, ignoring commands and terminates thread only after. """ - request = self._command_get() - while request is not None: - if not isinstance(request, ControlCommand): - self._request_process(request) - request = self._command_get() + self._queue.join() self._stop_immediately() def _stop_immediately(self): diff --git a/reportportal_client/core/worker.pyi b/reportportal_client/core/worker.pyi index ad0c9a3e..036c59bd 100644 --- a/reportportal_client/core/worker.pyi +++ b/reportportal_client/core/worker.pyi @@ -19,7 +19,7 @@ from typing import Any, Optional, Text, Union from aenum import Enum -from reportportal_client.core.rp_requests import RPRequestBase as RPRequest +from reportportal_client.core.rp_requests import RPRequestBase as RPRequest, HttpRequest from static.defines import Priority logger: Logger @@ -53,7 +53,7 @@ class APIWorker: def _command_process(self, cmd: Optional[ControlCommand]) -> None: ... - def _request_process(self, request: Optional[RPRequest]) -> None: ... + def _request_process(self, request: Optional[HttpRequest]) -> None: ... def _monitor(self) -> None: ... @@ -63,7 +63,7 @@ class APIWorker: def is_alive(self) -> bool: ... - def send(self, cmd: Union[ControlCommand, RPRequest]) -> Any: ... + def send(self, cmd: Union[ControlCommand, HttpRequest]) -> Any: ... def start(self) -> None: ... diff --git a/reportportal_client/items/__init__.py b/reportportal_client/items/__init__.py deleted file mode 100644 index a4f66a81..00000000 --- a/reportportal_client/items/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -"""This package provides common static objects and variables. - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" diff --git a/reportportal_client/items/item_weight.py b/reportportal_client/items/item_weight.py deleted file mode 100644 index f1b40b90..00000000 --- a/reportportal_client/items/item_weight.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -This module contains common classes-helpers for the items. - -Copyright (c) 2018 http://reportportal.io . -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at -http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -from aenum import IntEnum - - -class ItemWeight(IntEnum): - """Enum with weights for different item types.""" - - ROOT_ITEM_WEIGHT = 1 - LOG_ITEM_WEIGHT = 10 diff --git a/reportportal_client/items/rp_base_item.py b/reportportal_client/items/rp_base_item.py deleted file mode 100644 index 6979e2a4..00000000 --- a/reportportal_client/items/rp_base_item.py +++ /dev/null @@ -1,64 +0,0 @@ -"""This module contains functional for Base RP items management.""" - -# Copyright (c) 2023 EPAM Systems -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License - -from reportportal_client.core.rp_requests import HttpRequest - - -class BaseRPItem(object): - """This model stores attributes related to RP item.""" - - def __init__(self, rp_url, session, project_name, - launch_uuid, generated_id): - """Initialize instance attributes. - - :param rp_url: report portal url - :param session: Session object - :param project_name: RP project name - :param launch_uuid: Parent launch UUID - :param generated_id: Id generated to speed up client - """ - self.uuid = None - self.weight = None - self.generated_id = generated_id - self.http_requests = [] - self.responses = [] - self.rp_url = rp_url - self.session = session - self.project_name = project_name - self.launch_uuid = launch_uuid - - @property - def http_request(self): - """Get last http request. - - :return: request object - """ - return self.http_requests[-1] if self.http_requests else None - - def add_request(self, endpoint, method, request_class, *args, **kwargs): - """Add new request object. - - :param endpoint: request endpoint - :param method: Session object method. Allowable values: get, - post, put, delete - :param request_class: request class object - :param args: request object attributes - :param kwargs: request object named attributes - :return: None - """ - rp_request = request_class(*args, **kwargs) - rp_request.http_request = HttpRequest(method, endpoint) - rp_request.priority = self.weight - self.http_requests.append(rp_request) diff --git a/reportportal_client/items/rp_log_items/__init__.py b/reportportal_client/items/rp_log_items/__init__.py deleted file mode 100644 index a4f66a81..00000000 --- a/reportportal_client/items/rp_log_items/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -"""This package provides common static objects and variables. - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" diff --git a/reportportal_client/items/rp_log_items/rp_log_item.py b/reportportal_client/items/rp_log_items/rp_log_item.py deleted file mode 100644 index fedd473c..00000000 --- a/reportportal_client/items/rp_log_items/rp_log_item.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -This module contains functional for RP log items management. - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -from reportportal_client.core.rp_requests import RPRequestLog -from reportportal_client.items.item_weight import ItemWeight -from reportportal_client.items.rp_base_item import BaseRPItem -from reportportal_client.static.defines import RP_LOG_LEVELS - - -class RPLogItem(BaseRPItem): - """This model stores attributes for RP log items.""" - - def __init__(self, rp_url, session, project_name, - launch_uuid, generated_id): - """Initialize instance attributes. - - :param rp_url: report portal URL - :param session: Session object - :param project_name: RP project name - :param launch_uuid: Parent launch UUID - :param generated_id: Id generated to speed up client - """ - super(RPLogItem, self).__init__(rp_url, session, - project_name, launch_uuid, - generated_id) - self.weight = ItemWeight.LOG_ITEM_WEIGHT - - @property - def response(self): - """Get the response object for RP log item.""" - return self.responses[0] - - @response.setter - def response(self, value): - """Set the response object for RP log item.""" - raise NotImplementedError - - def create(self, api_version, time, file_obj=None, item_uuid=None, - level=RP_LOG_LEVELS[40000], message=None): - """Add request for log item creation. - - :param api_version: RP API version - :param time: Log item time - :param file_obj: Object of the RPFile - :param item_uuid: Parent test item UUID - :param level: Log level. Allowable values: error(40000), - warn(30000), info(20000), debug(10000), - trace(5000), fatal(50000), unknown(60000) - :param message: Log message - """ - endpoint = "/api/{version}/{projectName}/log".format( - version=api_version, projectName=self.project_name) - self.add_request(endpoint, self.session.post, RPRequestLog, - self.launch_uuid, time, file=file_obj, - item_uuid=item_uuid, level=level, message=message) diff --git a/reportportal_client/items/rp_test_items/__init__.py b/reportportal_client/items/rp_test_items/__init__.py deleted file mode 100644 index a4f66a81..00000000 --- a/reportportal_client/items/rp_test_items/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -"""This package provides common static objects and variables. - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" diff --git a/reportportal_client/items/rp_test_items/rp_base_test_item.py b/reportportal_client/items/rp_test_items/rp_base_test_item.py deleted file mode 100644 index 5b4d318a..00000000 --- a/reportportal_client/items/rp_test_items/rp_base_test_item.py +++ /dev/null @@ -1,89 +0,0 @@ -""" -This module contains functional for Base RP test items management. - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -from reportportal_client.items.rp_base_item import BaseRPItem -from reportportal_client.core.rp_requests import ItemFinishRequest - - -class RPBaseTestItem(BaseRPItem): - """This model stores common attributes for RP test items.""" - - def __init__(self, rp_url, session, project_name, item_name, - item_type, launch_uuid, generated_id, **kwargs): - """Initialize instance attributes. - - :param rp_url: report portal url - :param session: Session object - :param project_name: RP project name - :param item_name: RP item name - :param item_type: Type of the test item. Allowable values: "suite", - "story", "test", "scenario", "step", - "before_class", "before_groups", "before_method", - "before_suite", "before_test", "after_class", - "after_groups", "after_method", "after_suite", - "after_test" - :param launch_uuid: Parent launch UUID - :param generated_id: Id generated to speed up client - :param has_stats: If item has stats - :param kwargs: Dict of additional named parameters - """ - super(RPBaseTestItem, self).__init__(rp_url, session, - project_name, launch_uuid, - generated_id) - self.item_name = item_name - self.item_type = item_type - self.description = kwargs.get("description") - self.attributes = kwargs.get("attributes") - self.uuid = kwargs.get("uuid") - self.code_ref = kwargs.get("code_ref") - self.parameters = kwargs.get("parameters") - self.unique_id = kwargs.get("unique_id") - self.retry = kwargs.get("retry", False) - self.has_stats = kwargs.get("has_stats", True) - self.child_items = [] - - def add_child_item(self, item): - """Add new child item to the list. - - :param item: test item object - :return: None - """ - self.child_items.append(item) - - def finish(self, api_version, end_time, status=None, description=None, - attributes=None, issue=None): - """Form finish request for RP test item. - - :param api_version: RP API version - :param end_time: Test item end time - :param status: Test status. Allowable values: "passed", - "failed", "stopped", "skipped", "interrupted", - "cancelled" - :param description: Test item description. - :param attributes: List with attributes - :param issue: Issue of the current test item - """ - attributes = attributes or self.attributes - endpoint = "{url}/api/{version}/{projectName}/item/{itemUuid}". \ - format(url=self.rp_url, version=api_version, - projectName=self.project_name, itemUuid=self.uuid) - - self.add_request(endpoint, self.session.post, ItemFinishRequest, - end_time, self.launch_uuid, status, - attributes=attributes, description=description, - issue=issue, retry=self.retry) diff --git a/reportportal_client/items/rp_test_items/rp_child_test_item.py b/reportportal_client/items/rp_test_items/rp_child_test_item.py deleted file mode 100644 index 3a09c01d..00000000 --- a/reportportal_client/items/rp_test_items/rp_child_test_item.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -This module contains functional for Child RP test items management. - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -from weakref import proxy - -from reportportal_client.core.rp_requests import ItemStartRequest -from reportportal_client.items.rp_test_items.rp_base_test_item import \ - RPBaseTestItem - - -class RPChildTestItem(RPBaseTestItem): - """This model stores attributes for RP child test items.""" - - def __init__(self, rp_url, session, project_name, parent_item, - item_name, item_type, launch_uuid, generated_id, - **kwargs): - """Initialize instance attributes. - - :param rp_url: report portal url - :param session: Session object - :param project_name: RP project name - :param item_name: RP item name - :param item_type: Type of the test item. Allowable values: "suite", - "story", "test", "scenario", "step", - "before_class", "before_groups", "before_method", - "before_suite", "before_test", "after_class", - "after_groups", "after_method", "after_suite", - "after_test" - :param launch_uuid: Parent launch UUID - :param generated_id: Id generated to speed up client - :param kwargs: Dict of additional named parameters - """ - super(RPChildTestItem, self).__init__(rp_url, session, - project_name, item_name, - item_type, launch_uuid, - generated_id, **kwargs) - self.parent_item = proxy(parent_item) - self.parent_item.add_child_item(self) - self.weight = self.parent_item.weight + 1 - - def start(self, api_version, start_time): - """Create request object to start child test item. - - :param api_version: RP API version - :param start_time: Test item start time - """ - endpoint = "{url}/{api_version}/{project_name}/item/" \ - "{parentItemUuid}". \ - format(url=self.rp_url, api_version=api_version, - project_name=self.project_name, - parentItemUuid=self.parent_item.uuid) - - self.add_request(endpoint, self.session.post, ItemStartRequest, - self.item_name, start_time, - self.item_type, self.launch_uuid, - attributes=self.attributes, code_ref=self.code_ref, - description=self.description, - has_stats=self.has_stats, - parameters=self.parameters, - retry=self.retry, uuid=self.uuid, - unique_id=self.unique_id) diff --git a/reportportal_client/items/rp_test_items/rp_root_test_item.py b/reportportal_client/items/rp_test_items/rp_root_test_item.py deleted file mode 100644 index 40d887f7..00000000 --- a/reportportal_client/items/rp_test_items/rp_root_test_item.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -This module contains functional for Root RP test items management. - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -from reportportal_client.core.rp_requests import ItemStartRequest -from reportportal_client.items.item_weight import ItemWeight -from reportportal_client.items.rp_test_items.rp_base_test_item import \ - RPBaseTestItem - - -class RPRootTestItem(RPBaseTestItem): - """This model stores attributes for RP Root test items.""" - - def __init__(self, rp_url, session, project_name, item_name, - item_type, launch_uuid, generated_id, **kwargs): - """Initialize instance attributes. - - :param rp_url: report portal url - :param session: Session object - :param project_name: RP project name - :param item_name: RP item name - :param item_type: Type of the test item. Allowable values: - "suite", "story", "test", "scenario", "step", - "before_class", "before_groups", "before_method", - "before_suite", "before_test", "after_class", - "after_groups", "after_method", "after_suite", - "after_test" - :param launch_uuid: Parent launch UUID - :param generated_id: Id generated to speed up client - :param kwargs: Dict of additional named parameters - """ - super(RPRootTestItem, self).__init__(rp_url, session, - project_name, item_name, - item_type, launch_uuid, - generated_id, **kwargs) - self.weight = ItemWeight.ROOT_ITEM_WEIGHT - - def start(self, api_version, start_time): - """Create request object to start root test item. - - :param api_version: RP API version - :param start_time: Test item start time - """ - endpoint = "{rp_url}/api/{api_version}/{project_name}/item".format( - rp_url=self.rp_url, - api_version=api_version, - project_name=self.project_name) - - self.add_request(endpoint, self.session.post, ItemStartRequest, - self.item_name, start_time, - self.item_type, self.launch_uuid, - attributes=self.attributes, code_ref=self.code_ref, - description=self.description, - has_stats=self.has_stats, parameters=self.parameters, - retry=self.retry, uuid=self.uuid, - unique_id=self.unique_id) diff --git a/reportportal_client/logs/log_manager.py b/reportportal_client/logs/log_manager.py index c93642fe..c53a6f63 100644 --- a/reportportal_client/logs/log_manager.py +++ b/reportportal_client/logs/log_manager.py @@ -79,8 +79,7 @@ def _send_batch(self): http_request = HttpRequest( self.session.post, self._log_endpoint, files=batch.payload, verify_ssl=self.verify_ssl) - batch.http_request = http_request - self._worker.send(batch) + self._worker.send(http_request) self._batch = [] self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH diff --git a/reportportal_client/static/__init__.py b/reportportal_client/static/__init__.py index a4f66a81..77137a71 100644 --- a/reportportal_client/static/__init__.py +++ b/reportportal_client/static/__init__.py @@ -1,16 +1,14 @@ -"""This package provides common static objects and variables. - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" +"""This package provides common static objects and variables.""" + +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License diff --git a/reportportal_client/static/defines.py b/reportportal_client/static/defines.py index 25ca1c52..fa9ac3f1 100644 --- a/reportportal_client/static/defines.py +++ b/reportportal_client/static/defines.py @@ -1,19 +1,17 @@ -"""This module provides RP client static objects and variables. - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" +"""This module provides RP client static objects and variables.""" + +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License import aenum as enum @@ -29,7 +27,7 @@ } -class _PresenceSentinel(object): +class _PresenceSentinel: """Sentinel object for the None type.""" def __nonzero__(self): diff --git a/reportportal_client/static/errors.py b/reportportal_client/static/errors.py index dbbdee9f..50d04645 100644 --- a/reportportal_client/static/errors.py +++ b/reportportal_client/static/errors.py @@ -1,20 +1,17 @@ -"""This modules includes exceptions used by the client. - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - +"""This module includes exceptions used by the client.""" + +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License class RPExceptionBase(Exception): """General exception for the package.""" diff --git a/reportportal_client/steps/__init__.py b/reportportal_client/steps/__init__.py index 6052d091..0689a31b 100644 --- a/reportportal_client/steps/__init__.py +++ b/reportportal_client/steps/__init__.py @@ -1,16 +1,3 @@ -# Copyright (c) 2022 https://reportportal.io . -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License - """Report portal Nested Step handling module. The module for handling and reporting Report Portal Nested Steps inside python @@ -42,6 +29,20 @@ def test_my_nested_step(): pass """ + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + from functools import wraps from reportportal_client._local import current From 75b0073f0b9fdb1d2248f135cec779cc06dc8f19 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 1 Sep 2023 12:37:10 +0300 Subject: [PATCH 015/268] Add test timeout --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1fe16d42..46036112 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -50,6 +50,7 @@ jobs: pip install tox tox-gh-actions - name: Test with tox + timeout-minutes: 10 run: tox - name: Upload coverage to Codecov From 6e2502c72231b2041699f8bbbac4a162f36cee97 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 1 Sep 2023 17:47:39 +0300 Subject: [PATCH 016/268] A cosmetic import fix --- reportportal_client/logs/log_manager.pyi | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/reportportal_client/logs/log_manager.pyi b/reportportal_client/logs/log_manager.pyi index cd59fc29..dbed1c36 100644 --- a/reportportal_client/logs/log_manager.pyi +++ b/reportportal_client/logs/log_manager.pyi @@ -18,9 +18,7 @@ from typing import Dict, List, Optional, Text from requests import Session from six.moves import queue -from reportportal_client.core.rp_requests import ( - RPRequestLog as RPRequestLog -) +from reportportal_client.core.rp_requests import RPRequestLog from reportportal_client.core.worker import APIWorker as APIWorker logger: Logger From 1d208e46393c2b52097f0f3016bc901ea073e799 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 1 Sep 2023 17:47:52 +0300 Subject: [PATCH 017/268] Async RPClient: WIP --- reportportal_client/core/rp_requests.py | 347 ++++++++---------------- 1 file changed, 111 insertions(+), 236 deletions(-) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 94a2edfc..89442a51 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -22,6 +22,7 @@ import json as json_converter import logging import ssl +from dataclasses import dataclass from typing import Callable, Text, Optional, Union, List, Tuple, Any, TypeVar import aiohttp @@ -66,7 +67,6 @@ class HttpRequest: http_timeout: Union[float, Tuple[float, float]] name: Optional[str] _priority: Priority - _response: Optional[RPResponse] def __init__(self, session_method: Callable, @@ -99,7 +99,6 @@ def __init__(self, self.http_timeout = http_timeout self.name = name self._priority = DEFAULT_PRIORITY - self._response = None def __lt__(self, other) -> bool: """Priority protocol for the PriorityQueue.""" @@ -115,21 +114,11 @@ def priority(self, value: Priority) -> None: """Set the priority of the request.""" self._priority = value - @property - def response(self) -> Optional[RPResponse]: - """Get the response object for the request.""" - if not self._response: - return self.make() - return self._response - def make(self): """Make HTTP request to the Report Portal API.""" try: - self._response = RPResponse( - self.session_method(self.url, data=self.data, json=self.json, files=self.files, - verify=self.verify_ssl, timeout=self.http_timeout)) - return self._response - # https://github.com/reportportal/client-Python/issues/39 + return RPResponse(self.session_method(self.url, data=self.data, json=self.json, files=self.files, + verify=self.verify_ssl, timeout=self.http_timeout)) except (KeyError, IOError, ValueError, TypeError) as exc: logger.warning( "Report Portal %s request failed", @@ -141,10 +130,8 @@ def make(self): class AsyncHttpRequest(HttpRequest): """This model stores attributes related to RP HTTP requests.""" - def __init__(self, session_method: Callable, url: str, data=None, json=None, - files=None, verify_ssl=True, http_timeout=(10, 10), - name=None) -> None: - super().__init__(session_method, url, data, json, files, verify_ssl, http_timeout, name) + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) async def make(self): """Make HTTP request to the Report Portal API.""" @@ -154,25 +141,21 @@ async def make(self): ssl_context.load_cert_chain(ssl_config) ssl_config = ssl_context - timeout_config = self.http_timeout - if not timeout_config or not type(timeout_config) == tuple: - timeout_config = (timeout_config, timeout_config) + timeout = None + if self.http_timeout: + if type(self.http_timeout) == tuple: + connect_timeout, read_timeout = self.http_timeout + else: + connect_timeout, read_timeout = self.http_timeout, self.http_timeout + timeout = aiohttp.ClientTimeout(connect=connect_timeout, sock_read=read_timeout) data = self.data if self.files: data = self.files try: - return RPResponse( - await self.session_method( - await await_if_necessary(self.url), - data=data, - json=self.json, - ssl=ssl_config, - timeout=aiohttp.ClientTimeout(connect=timeout_config[0], sock_read=timeout_config[1]) - ) - ) - # https://github.com/reportportal/client-Python/issues/39 + return RPResponse(await self.session_method(await await_if_necessary(self.url), data=data, + json=self.json, ssl=ssl_config, timeout=timeout)) except (KeyError, IOError, ValueError, TypeError) as exc: logger.warning( "Report Portal %s request failed", @@ -192,57 +175,29 @@ def payload(self) -> dict: raise NotImplementedError('Payload interface is not implemented!') +@dataclass(frozen=True) class LaunchStartRequest(RPRequestBase): """RP start launch request model. https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#start-launch """ - attributes: Optional[Union[list, dict]] - description: str - mode: str name: str - rerun: bool - rerun_of: str start_time: str - uuid: str - - def __init__(self, - name: str, - start_time: str, - attributes: Optional[Union[list, dict]] = None, - description: Optional[Text] = None, - mode: str = 'default', - rerun: bool = False, - rerun_of: Optional[Text] = None, - uuid: str = None) -> None: - """Initialize instance attributes. - - :param name: Name of the launch - :param start_time: Launch start time - :param attributes: Launch attributes - :param description: Description of the launch - :param mode: Launch mode. Allowable values 'default' or 'debug' - :param rerun: Rerun mode. Allowable values 'True' of 'False' - :param rerun_of: Rerun mode. Specifies launch to be re-runned. Uses - with the 'rerun' attribute. - """ - super().__init__() - self.attributes = attributes - self.description = description - self.mode = mode - self.name = name - self.rerun = rerun - self.rerun_of = rerun_of - self.start_time = start_time - self.uuid = uuid + attributes: Optional[Union[list, dict]] = None + description: Optional[str] = None + mode: str = 'default' + rerun: bool = False + rerun_of: str = None + uuid: str = None @property def payload(self) -> dict: """Get HTTP payload for the request.""" + my_attributes = None if self.attributes and isinstance(self.attributes, dict): - self.attributes = dict_to_payload(self.attributes) + my_attributes = dict_to_payload(self.attributes) result = { - 'attributes': self.attributes, + 'attributes': my_attributes, 'description': self.description, 'mode': self.mode, 'name': self.name, @@ -255,46 +210,33 @@ def payload(self) -> dict: return result +@dataclass(frozen=True) class LaunchFinishRequest(RPRequestBase): """RP finish launch request model. https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#finish-launch """ - def __init__(self, - end_time: str, - status: Optional[Text] = None, - attributes: Optional[Union[list, dict]] = None, - description: Optional[Text] = None) -> None: - """Initialize instance attributes. - - :param end_time: Launch end time - :param status: Launch status. Allowable values: "passed", - "failed", "stopped", "skipped", "interrupted", - "cancelled" - :param attributes: Launch attributes(tags). Pairs of key and value. - Overrides attributes on start - :param description: Launch description. Overrides description on start - """ - super().__init__() - self.attributes = attributes - self.description = description - self.end_time = end_time - self.status = status + end_time: str + status: Optional[Text] = None + attributes: Optional[Union[list, dict]] = None + description: Optional[str] = None @property def payload(self) -> dict: """Get HTTP payload for the request.""" + my_attributes = None if self.attributes and isinstance(self.attributes, dict): - self.attributes = dict_to_payload(self.attributes) + my_attributes = dict_to_payload(self.attributes) return { - 'attributes': self.attributes, + 'attributes': my_attributes, 'description': self.description, 'endTime': self.end_time, 'status': self.status } +@dataclass(frozen=True) class ItemStartRequest(RPRequestBase): """RP start test item request model. @@ -304,7 +246,7 @@ class ItemStartRequest(RPRequestBase): code_ref: Optional[Text] description: Optional[Text] has_stats: bool - launch_uuid: str + launch_uuid: Any name: str parameters: Optional[Union[list, dict]] retry: bool @@ -312,73 +254,48 @@ class ItemStartRequest(RPRequestBase): test_case_id: Optional[Text] type_: str - def __init__(self, - name: str, - start_time: str, - type_: str, - launch_uuid: str, - attributes: Optional[Union[list, dict]] = None, - code_ref: Optional[Text] = None, - description: Optional[Text] = None, - has_stats: bool = True, - parameters: Optional[Union[list, dict]] = None, - retry: bool = False, - test_case_id: Optional[Text] = None) -> None: - """Initialize instance attributes. - - :param name: Name of the test item - :param start_time: Test item start time - :param type_: Type of the test item. Allowable values: "suite", - "story", "test", "scenario", "step", - "before_class", "before_groups", "before_method", - "before_suite", "before_test", "after_class", - "after_groups", "after_method", "after_suite", - "after_test" - :param launch_uuid: Parent launch UUID - :param attributes: Test item attributes - :param code_ref: Physical location of the test item - :param description: Test item description - :param has_stats: Set to False if test item is nested step - :param parameters: Set of parameters (for parametrized test items) - :param retry: Used to report retry of the test. Allowable values: - "True" or "False" - :param test_case_id:Test case ID from integrated TMS - """ - super().__init__() - self.attributes = attributes - self.code_ref = code_ref - self.description = description - self.has_stats = has_stats - self.launch_uuid = launch_uuid - self.name = name - self.parameters = parameters - self.retry = retry - self.start_time = start_time - self.test_case_id = test_case_id - self.type_ = type_ + @staticmethod + def create_request(**kwargs) -> dict: + request = { + 'codeRef': kwargs.get('code_ref'), + 'description': kwargs.get('description'), + 'hasStats': kwargs.get('has_stats'), + 'name': kwargs['name'], + 'retry': kwargs.get('retry'), + 'startTime': kwargs['start_time'], + 'testCaseId': kwargs.get('test_case_id'), + 'type': kwargs['type'], + 'launchUuid': kwargs['launch_uuid'] + } + if 'attributes' in kwargs: + request['attributes'] = dict_to_payload(kwargs['attributes']) + if 'parameters' in kwargs: + request['parameters'] = dict_to_payload(kwargs['parameters']) + return request @property def payload(self) -> dict: """Get HTTP payload for the request.""" - if self.attributes and isinstance(self.attributes, dict): - self.attributes = dict_to_payload(self.attributes) - if self.parameters: - self.parameters = dict_to_payload(self.parameters) - return { - 'attributes': self.attributes, - 'codeRef': self.code_ref, - 'description': self.description, - 'hasStats': self.has_stats, - 'launchUuid': self.launch_uuid, - 'name': self.name, - 'parameters': self.parameters, - 'retry': self.retry, - 'startTime': self.start_time, - 'testCaseId': self.test_case_id, - 'type': self.type_ - } + data = self.__dict__.copy() + data['type'] = data.pop('type_') + return ItemStartRequest.create_request(**data) + +class ItemStartRequestAsync(ItemStartRequest): + def __int__(self, *args, **kwargs) -> None: + super.__init__(*args, **kwargs) + + @property + async def payload(self) -> dict: + """Get HTTP payload for the request.""" + data = self.__dict__.copy() + data['type'] = data.pop('type_') + data['launch_uuid'] = await_if_necessary(data.pop('launch_uuid')) + return ItemStartRequest.create_request(**data) + + +@dataclass(frozen=True) class ItemFinishRequest(RPRequestBase): """RP finish test item request model. @@ -389,106 +306,64 @@ class ItemFinishRequest(RPRequestBase): end_time: str is_skipped_an_issue: bool issue: Issue - launch_uuid: str + launch_uuid: Any status: str retry: bool - def __init__(self, - end_time: str, - launch_uuid: str, - status: str, - attributes: Optional[Union[list, dict]] = None, - description: Optional[str] = None, - is_skipped_an_issue: bool = True, - issue: Optional[Issue] = None, - retry: bool = False) -> None: - """Initialize instance attributes. + @staticmethod + def create_request(**kwargs) -> dict: + request = { + 'description': kwargs.get('description'), + 'endTime': kwargs['end_time'], + 'launchUuid': kwargs['launch_uuid'], + 'status': kwargs.get('status'), + 'retry': kwargs.get('retry') + } + if 'attributes' in kwargs: + request['attributes'] = dict_to_payload(kwargs['attributes']) - :param end_time: Test item end time - :param launch_uuid: Parent launch UUID - :param status: Test status. Allowable values: "passed", - "failed", "stopped", "skipped", - "interrupted", "cancelled". - :param attributes: Test item attributes(tags). Pairs of key - and value. Overrides attributes on start - :param description: Test item description. Overrides - description from start request. - :param is_skipped_an_issue: Option to mark skipped tests as not - 'To Investigate' items in UI - :param issue: Issue of the current test item - :param retry: Used to report retry of the test. - Allowable values: "True" or "False" - """ - super().__init__() - self.attributes = attributes - self.description = description - self.end_time = end_time - self.is_skipped_an_issue = is_skipped_an_issue - self.issue = issue # type: Issue - self.launch_uuid = launch_uuid - self.status = status - self.retry = retry + if kwargs.get('issue') is None and ( + kwargs.get('status') is not None and kwargs.get('status').lower() == 'skipped' + ) and not kwargs.get('is_skipped_an_issue'): + issue_payload = {'issue_type': 'NOT_ISSUE'} + elif kwargs.get('issue') is not None: + issue_payload = kwargs.get('issue').payload + else: + issue_payload = None + request['issue'] = issue_payload + return request @property def payload(self) -> dict: """Get HTTP payload for the request.""" - if self.attributes and isinstance(self.attributes, dict): - self.attributes = dict_to_payload(self.attributes) - if self.issue is None and ( - self.status is not None and self.status.lower() == 'skipped' - ) and not self.is_skipped_an_issue: - issue_payload = {'issue_type': 'NOT_ISSUE'} - else: - issue_payload = None - return { - 'attributes': self.attributes, - 'description': self.description, - 'endTime': self.end_time, - 'issue': getattr(self.issue, 'payload', issue_payload), - 'launchUuid': self.launch_uuid, - 'status': self.status, - 'retry': self.retry - } + return ItemFinishRequest.create_request(**self.__dict__) + +class ItemFinishRequestAsync(ItemFinishRequest): + + def __int__(self, *args, **kwargs) -> None: + super.__init__(*args, **kwargs) + + @property + async def payload(self) -> dict: + """Get HTTP payload for the request.""" + data = self.__dict__.copy() + data['launch_uuid'] = await_if_necessary(data.pop('launch_uuid')) + return ItemFinishRequest.create_request(**data) + +@dataclass(frozen=True) class RPRequestLog(RPRequestBase): """RP log save request model. https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#save-single-log-without-attachment """ - file: Optional[RPFile] launch_uuid: str - level: str - message: Optional[Text] time: str - item_uuid: Optional[Text] - - def __init__(self, - launch_uuid: str, - time: str, - file: Optional[RPFile] = None, - item_uuid: Optional[Text] = None, - level: str = RP_LOG_LEVELS[40000], - message: Optional[Text] = None) -> None: - """Initialize instance attributes. - - :param launch_uuid: Launch UUID - :param time: Log time - :param file: Object of the RPFile - :param item_uuid: Test item UUID - :param level: Log level. Allowable values: error(40000), - warn(30000), info(20000), debug(10000), - trace(5000), fatal(50000), unknown(60000) - :param message: Log message - """ - super().__init__() - self.file = file # type: RPFile - self.launch_uuid = launch_uuid - self.level = level - self.message = message - self.time = time - self.item_uuid = item_uuid - self.priority = LOW_PRIORITY + file: Optional[RPFile] = None + item_uuid: Optional[Text] = None + level: str = RP_LOG_LEVELS[40000] + message: Optional[Text] = None def __file(self) -> dict: """Form file payload part of the payload.""" From cbb710fc488bffb35143b838fb4114e5ae676803 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 4 Sep 2023 16:48:27 +0300 Subject: [PATCH 018/268] Async RPClient: WIP --- reportportal_client/core/rp_requests.py | 113 ++++++++++++++++++------ 1 file changed, 85 insertions(+), 28 deletions(-) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 89442a51..e7338760 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -291,7 +291,7 @@ async def payload(self) -> dict: """Get HTTP payload for the request.""" data = self.__dict__.copy() data['type'] = data.pop('type_') - data['launch_uuid'] = await_if_necessary(data.pop('launch_uuid')) + data['launch_uuid'] = await await_if_necessary(data.pop('launch_uuid')) return ItemStartRequest.create_request(**data) @@ -348,7 +348,7 @@ def __int__(self, *args, **kwargs) -> None: async def payload(self) -> dict: """Get HTTP payload for the request.""" data = self.__dict__.copy() - data['launch_uuid'] = await_if_necessary(data.pop('launch_uuid')) + data['launch_uuid'] = await await_if_necessary(data.pop('launch_uuid')) return ItemFinishRequest.create_request(**data) @@ -358,31 +358,31 @@ class RPRequestLog(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#save-single-log-without-attachment """ - launch_uuid: str + launch_uuid: Any time: str file: Optional[RPFile] = None - item_uuid: Optional[Text] = None + item_uuid: Optional[Any] = None level: str = RP_LOG_LEVELS[40000] - message: Optional[Text] = None + message: Optional[str] = None - def __file(self) -> dict: - """Form file payload part of the payload.""" - if not self.file: - return {} - return {'file': {'name': self.file.name}} + @staticmethod + def create_request(**kwargs) -> dict: + request = { + 'launchUuid': kwargs['launch_uuid'], + 'level': kwargs['level'], + 'message': kwargs.get('message'), + 'time': kwargs['time'], + 'itemUuid': kwargs.get('item_uuid'), + 'file': kwargs.get('file') + } + if 'file' in kwargs and kwargs['file']: + request['file'] = {'name': kwargs['file'].name} + return request @property def payload(self) -> dict: """Get HTTP payload for the request.""" - payload = { - 'launchUuid': self.launch_uuid, - 'level': self.level, - 'message': self.message, - 'time': self.time, - 'itemUuid': self.item_uuid - } - payload.update(self.__file()) - return payload + return RPRequestLog.create_request(**self.__dict__) @property def multipart_size(self) -> int: @@ -392,15 +392,32 @@ def multipart_size(self) -> int: return size +class RPRequestLogAsync(RPRequestLog): + + def __int__(self, *args, **kwargs) -> None: + super.__init__(*args, **kwargs) + + @property + async def payload(self) -> dict: + """Get HTTP payload for the request.""" + data = self.__dict__.copy() + uuids = await asyncio.gather(await_if_necessary(data.pop('launch_uuid')), + await_if_necessary(data.pop('item_uuid'))) + data['launch_uuid'] = uuids[0] + data['item_uuid'] = uuids[1] + return RPRequestLog.create_request(**data) + + class RPLogBatch(RPRequestBase): """RP log save batches with attachments request model. https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#batch-save-logs """ - default_content: str = ... - log_reqs: List[RPRequestLog] = ... + default_content: str + log_reqs: List[Union[RPRequestLog, RPRequestLogAsync]] + priority: Priority - def __init__(self, log_reqs: List[RPRequestLog]) -> None: + def __init__(self, log_reqs: List[Union[RPRequestLog, RPRequestLogAsync]]) -> None: """Initialize instance attributes. :param log_reqs: @@ -425,7 +442,18 @@ def __get_files(self) -> List[Tuple[str, tuple]]: return files def __get_request_part(self) -> List[Tuple[str, tuple]]: - r"""Form JSON body for the request. + body = [( + 'json_request_part', ( + None, + json_converter.dumps([log.payload for log in self.log_reqs]), + 'application/json' + ) + )] + return body + + @property + def payload(self) -> List[Tuple[str, tuple]]: + r"""Get HTTP payload for the request. Example: [('json_request_part', @@ -442,17 +470,46 @@ def __get_request_part(self) -> List[Tuple[str, tuple]]: '\n

Paragraph

', 'text/html'))] """ + body = self.__get_request_part() + body.extend(self.__get_files()) + return body + + +class RPLogBatchAsync(RPLogBatch): + + def __int__(self, *args, **kwargs) -> None: + super.__init__(*args, **kwargs) + + async def __get_request_part(self) -> List[Tuple[str, tuple]]: + coroutines = [log.payload for log in self.log_reqs] body = [( 'json_request_part', ( None, - json_converter.dumps([log.payload for log in self.log_reqs]), + json_converter.dumps(await asyncio.gather(*coroutines)), 'application/json' ) )] - body.extend(self.__get_files()) return body @property - def payload(self) -> List[Tuple[str, tuple]]: - """Get HTTP payload for the request.""" - return self.__get_request_part() + async def payload(self) -> List[Tuple[str, tuple]]: + r"""Get HTTP payload for the request. + + Example: + [('json_request_part', + (None, + '[{"launchUuid": "bf6edb74-b092-4b32-993a-29967904a5b4", + "time": "1588936537081", + "message": "Html report", + "level": "INFO", + "itemUuid": "d9dc2514-2c78-4c4f-9369-ee4bca4c78f8", + "file": {"name": "Detailed report"}}]', + 'application/json')), + ('file', + ('Detailed report', + '\n

Paragraph

', + 'text/html'))] + """ + body = await self.__get_request_part() + body.extend(self.__get_files()) + return body From 100db9fdffb642536fded7ee3e2d38412840a6b3 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 4 Sep 2023 18:50:22 +0300 Subject: [PATCH 019/268] Async RPClient: WIP --- reportportal_client/_local/__init__.pyi | 6 +- .../{async => aio}/__init__.py | 0 reportportal_client/{async => aio}/client.py | 66 +++++++++++++++++-- reportportal_client/client.py | 1 - reportportal_client/core/rp_requests.py | 25 ++++--- reportportal_client/logs/__init__.py | 3 + reportportal_client/logs/log_manager.py | 4 +- reportportal_client/steps/__init__.pyi | 4 +- 8 files changed, 85 insertions(+), 24 deletions(-) rename reportportal_client/{async => aio}/__init__.py (100%) rename reportportal_client/{async => aio}/client.py (83%) diff --git a/reportportal_client/_local/__init__.pyi b/reportportal_client/_local/__init__.pyi index b5f2ab4e..4711a6a5 100644 --- a/reportportal_client/_local/__init__.pyi +++ b/reportportal_client/_local/__init__.pyi @@ -11,12 +11,14 @@ # See the License for the specific language governing permissions and # limitations under the License -from typing import Optional +from typing import Optional, Union +# noinspection PyProtectedMember +from reportportal_client.aio.client import _RPClientAsync from reportportal_client.client import RPClient def current() -> Optional[RPClient]: ... -def set_current(client: Optional[RPClient]) -> None: ... +def set_current(client: Optional[Union[RPClient, _RPClientAsync]]) -> None: ... diff --git a/reportportal_client/async/__init__.py b/reportportal_client/aio/__init__.py similarity index 100% rename from reportportal_client/async/__init__.py rename to reportportal_client/aio/__init__.py diff --git a/reportportal_client/async/client.py b/reportportal_client/aio/client.py similarity index 83% rename from reportportal_client/async/client.py rename to reportportal_client/aio/client.py index 0e7c2a0e..4b2fb31e 100644 --- a/reportportal_client/async/client.py +++ b/reportportal_client/aio/client.py @@ -15,14 +15,20 @@ import asyncio import logging +import sys import threading +import warnings +from os import getenv from queue import LifoQueue from typing import Union, Tuple, List, Dict, Any, Optional, TextIO import aiohttp +from helpers import uri_join +# noinspection PyProtectedMember +from reportportal_client._local import set_current from reportportal_client.core.rp_issues import Issue -from reportportal_client.logs.log_manager import LogManager, MAX_LOG_BATCH_PAYLOAD_SIZE +from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.static.defines import NOT_FOUND from reportportal_client.steps import StepReporter @@ -38,9 +44,8 @@ def last(self): class _RPClientAsync: - _log_manager: LogManager = ... - api_v1: str = ... - api_v2: str = ... + api_v1: str + api_v2: str base_url_v1: str = ... base_url_v2: str = ... endpoint: str = ... @@ -82,6 +87,59 @@ def __init__( **kwargs: Any ) -> None: self._item_stack = _LifoQueue() + set_current(self) + self.api_v1, self.api_v2 = 'v1', 'v2' + self.endpoint = endpoint + self.project = project + self.base_url_v1 = uri_join( + self.endpoint, 'api/{}'.format(self.api_v1), self.project) + self.base_url_v2 = uri_join( + self.endpoint, 'api/{}'.format(self.api_v2), self.project) + self.is_skipped_an_issue = is_skipped_an_issue + self.launch_id = launch_id + self.log_batch_size = log_batch_size + self.log_batch_payload_size = log_batch_payload_size + self.verify_ssl = verify_ssl + self.retries = retries + self.max_pool_size = max_pool_size + self.http_timeout = http_timeout + self.step_reporter = StepReporter(self) + self._item_stack = _LifoQueue() + self.mode = mode + self._skip_analytics = getenv('AGENT_NO_ANALYTICS') + self.launch_uuid_print = launch_uuid_print + self.print_output = print_output or sys.stdout + + self.api_key = api_key + if not self.api_key: + if 'token' in kwargs: + warnings.warn( + message='Argument `token` is deprecated since 5.3.5 and ' + 'will be subject for removing in the next major ' + 'version. Use `api_key` argument instead.', + category=DeprecationWarning, + stacklevel=2 + ) + self.api_key = kwargs['token'] + + if not self.api_key: + warnings.warn( + message='Argument `api_key` is `None` or empty string, ' + 'that is not supposed to happen because Report ' + 'Portal is usually requires an authorization key. ' + 'Please check your code.', + category=RuntimeWarning, + stacklevel=2 + ) + + self.__init_session() + + def __init_session(self) -> None: + headers = {} + if self.api_key: + headers['Authorization'] = f'Bearer {self.api_key}' + session = aiohttp.ClientSession(headers=headers) + self.session = session async def finish_launch(self, end_time: str, diff --git a/reportportal_client/client.py b/reportportal_client/client.py index ee532aac..a7988db6 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -124,7 +124,6 @@ def __init__( be processed in one batch """ set_current(self) - self._batch_logs = [] self.api_v1, self.api_v2 = 'v1', 'v2' self.endpoint = endpoint self.project = project diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index e7338760..39da0e7f 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -26,6 +26,7 @@ from typing import Callable, Text, Optional, Union, List, Tuple, Any, TypeVar import aiohttp +from aiohttp import MultipartWriter, Payload from reportportal_client import helpers from reportportal_client.core.rp_file import RPFile @@ -480,19 +481,12 @@ class RPLogBatchAsync(RPLogBatch): def __int__(self, *args, **kwargs) -> None: super.__init__(*args, **kwargs) - async def __get_request_part(self) -> List[Tuple[str, tuple]]: + async def __get_request_part(self) -> str: coroutines = [log.payload for log in self.log_reqs] - body = [( - 'json_request_part', ( - None, - json_converter.dumps(await asyncio.gather(*coroutines)), - 'application/json' - ) - )] - return body + return json_converter.dumps(await asyncio.gather(*coroutines)) @property - async def payload(self) -> List[Tuple[str, tuple]]: + async def payload(self) -> MultipartWriter: r"""Get HTTP payload for the request. Example: @@ -510,6 +504,11 @@ async def payload(self) -> List[Tuple[str, tuple]]: '\n

Paragraph

', 'text/html'))] """ - body = await self.__get_request_part() - body.extend(self.__get_files()) - return body + json_payload = Payload(await self.__get_request_part(), content_type='application/json') + json_payload.set_content_disposition('form-data', name='json_request_part') + mpwriter = MultipartWriter('form-data') + mpwriter.append_payload(json_payload) + for _, file in self.__get_files(): + file_payload = Payload(file[1], content_type=file[2], filename=file[0]) + mpwriter.append_payload(file_payload) + return mpwriter diff --git a/reportportal_client/logs/__init__.py b/reportportal_client/logs/__init__.py index 269018ed..64ba74d2 100644 --- a/reportportal_client/logs/__init__.py +++ b/reportportal_client/logs/__init__.py @@ -23,6 +23,9 @@ from reportportal_client._local import current, set_current from reportportal_client.helpers import timestamp +MAX_LOG_BATCH_SIZE = 20 +MAX_LOG_BATCH_PAYLOAD_SIZE = 65000000 + class RPLogger(logging.getLoggerClass()): """RPLogger class for low-level logging in tests.""" diff --git a/reportportal_client/logs/log_manager.py b/reportportal_client/logs/log_manager.py index c53a6f63..f80a2d4e 100644 --- a/reportportal_client/logs/log_manager.py +++ b/reportportal_client/logs/log_manager.py @@ -26,13 +26,11 @@ RPRequestLog ) from reportportal_client.core.worker import APIWorker +from reportportal_client.logs import MAX_LOG_BATCH_SIZE, MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.static.defines import NOT_FOUND logger = logging.getLogger(__name__) -MAX_LOG_BATCH_SIZE = 20 -MAX_LOG_BATCH_PAYLOAD_SIZE = 65000000 - class LogManager(object): """Manager of the log items.""" diff --git a/reportportal_client/steps/__init__.pyi b/reportportal_client/steps/__init__.pyi index 748d8c79..636802ec 100644 --- a/reportportal_client/steps/__init__.pyi +++ b/reportportal_client/steps/__init__.pyi @@ -13,13 +13,15 @@ from typing import Text, Optional, Dict, Any, Callable, Union +# noinspection PyProtectedMember +from reportportal_client.aio.client import _RPClientAsync from reportportal_client.client import RPClient class StepReporter: client: RPClient = ... - def __init__(self, rp_client: RPClient) -> None: ... + def __init__(self, rp_client: Union[RPClient, _RPClientAsync]) -> None: ... def start_nested_step(self, name: Text, From eb6e8e2305ecfe3a5ba07b63d8fc1481e9c90d7e Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 5 Sep 2023 18:22:03 +0300 Subject: [PATCH 020/268] Async RPClient: WIP --- reportportal_client/aio/client.py | 419 ++++++++++++++------ reportportal_client/core/rp_requests.py | 82 ++-- reportportal_client/services/statistics.py | 60 ++- reportportal_client/services/statistics.pyi | 29 -- 4 files changed, 386 insertions(+), 204 deletions(-) delete mode 100644 reportportal_client/services/statistics.pyi diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 4b2fb31e..5f5c298b 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -15,6 +15,7 @@ import asyncio import logging +import ssl import sys import threading import warnings @@ -24,11 +25,14 @@ import aiohttp -from helpers import uri_join +from core.rp_requests import LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, \ + AsyncItemFinishRequest, LaunchFinishRequest +from helpers import uri_join, verify_value_length # noinspection PyProtectedMember from reportportal_client._local import set_current from reportportal_client.core.rp_issues import Issue from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE +from reportportal_client.services.statistics import async_send_event from reportportal_client.static.defines import NOT_FOUND from reportportal_client.steps import StepReporter @@ -43,29 +47,31 @@ def last(self): return self.queue[-1] +# TODO: Correct url and launch id handling for sync and async classes class _RPClientAsync: api_v1: str api_v2: str - base_url_v1: str = ... - base_url_v2: str = ... - endpoint: str = ... - is_skipped_an_issue: bool = ... - launch_id: str = ... - log_batch_size: int = ... - log_batch_payload_size: int = ... - project: str = ... - api_key: str = ... - verify_ssl: Union[bool, str] = ... - retries: int = ... - max_pool_size: int = ... - http_timeout: Union[float, Tuple[float, float]] = ... - session: aiohttp.ClientSession = ... - step_reporter: StepReporter = ... - mode: str = ... - launch_uuid_print: Optional[bool] = ... - print_output: Optional[TextIO] = ... - _skip_analytics: str = ... - _item_stack: _LifoQueue = ... + base_url_v1: str + base_url_v2: str + endpoint: str + is_skipped_an_issue: bool + launch_id: asyncio.Future + use_own_launch: bool + log_batch_size: int + log_batch_payload_size: int + project: str + api_key: str + verify_ssl: Union[bool, str] + retries: int + max_pool_size: int + http_timeout: Union[float, Tuple[float, float]] + step_reporter: StepReporter + mode: str + launch_uuid_print: Optional[bool] + print_output: Optional[TextIO] + _skip_analytics: str + _item_stack: _LifoQueue + __session: aiohttp.ClientSession def __init__( self, @@ -75,10 +81,10 @@ def __init__( api_key: str = None, log_batch_size: int = 20, is_skipped_an_issue: bool = True, - verify_ssl: bool = True, + verify_ssl: Union[bool, str] = True, retries: int = None, max_pool_size: int = 50, - launch_id: str = None, + launch_id: Optional[asyncio.Future] = None, http_timeout: Union[float, Tuple[float, float]] = (10, 10), log_batch_payload_size: int = MAX_LOG_BATCH_PAYLOAD_SIZE, mode: str = 'DEFAULT', @@ -96,7 +102,11 @@ def __init__( self.base_url_v2 = uri_join( self.endpoint, 'api/{}'.format(self.api_v2), self.project) self.is_skipped_an_issue = is_skipped_an_issue - self.launch_id = launch_id + if launch_id: + self.launch_id = launch_id + self.use_own_launch = False + else: + self.use_own_launch = True self.log_batch_size = log_batch_size self.log_batch_payload_size = log_batch_payload_size self.verify_ssl = verify_ssl @@ -132,24 +142,151 @@ def __init__( stacklevel=2 ) - self.__init_session() + @property + def session(self) -> aiohttp.ClientSession: + # TODO: add retry handler + if self.__session: + return self.__session + + ssl_config = self.verify_ssl + if ssl_config and type(ssl_config) == str: + ssl_context = ssl.create_default_context() + ssl_context.load_cert_chain(ssl_config) + ssl_config = ssl_context + connector = aiohttp.TCPConnector(ssl=ssl_config, limit=self.max_pool_size) + + timeout = None + if self.http_timeout: + if type(self.http_timeout) == tuple: + connect_timeout, read_timeout = self.http_timeout + else: + connect_timeout, read_timeout = self.http_timeout, self.http_timeout + timeout = aiohttp.ClientTimeout(connect=connect_timeout, sock_read=read_timeout) - def __init_session(self) -> None: headers = {} if self.api_key: headers['Authorization'] = f'Bearer {self.api_key}' - session = aiohttp.ClientSession(headers=headers) - self.session = session + self.__session = aiohttp.ClientSession(self.endpoint, connector=connector, headers=headers, + timeout=timeout) + return self.__session + + async def __get_item_url(self, id_future: asyncio.Future) -> Optional[str]: + item_id = await id_future + if item_id is NOT_FOUND: + logger.warning('Attempt to make request for non-existent id.') + return + return uri_join(self.base_url_v2, 'item', item_id) + + async def __get_launch_url(self) -> Optional[str]: + launch_id = await self.launch_id + if launch_id is NOT_FOUND: + logger.warning('Attempt to make request for non-existent launch.') + return + return uri_join(self.base_url_v2, 'launch', launch_id, 'finish') - async def finish_launch(self, - end_time: str, - status: str = None, - attributes: Optional[Union[List, Dict]] = None, - **kwargs: Any) -> Optional[str]: - pass + async def start_launch(self, + name: str, + start_time: str, + description: Optional[str] = None, + attributes: Optional[Union[List, Dict]] = None, + rerun: bool = False, + rerun_of: Optional[str] = None, + **kwargs) -> Optional[str]: + """Start a new launch with the given parameters. + + :param name: Launch name + :param start_time: Launch start time + :param description: Launch description + :param attributes: Launch attributes + :param rerun: Start launch in rerun mode + :param rerun_of: For rerun mode specifies which launch will be + re-run. Should be used with the 'rerun' option. + """ + if not self.use_own_launch: + return self.launch_id + url = uri_join(self.base_url_v2, 'launch') + request_payload = LaunchStartRequest( + name=name, + start_time=start_time, + attributes=attributes, + description=description, + mode=self.mode, + rerun=rerun, + rerun_of=rerun_of or kwargs.get('rerunOf') + ).payload + + launch_coro = AsyncHttpRequest(self.session.post, + url=url, + json=request_payload).make() + + stat_coro = None + if not self._skip_analytics: + agent_name, agent_version = None, None + + agent_attribute = [a for a in attributes if + a.get('key') == 'agent'] if attributes else [] + if len(agent_attribute) > 0 and agent_attribute[0].get('value'): + agent_name, agent_version = agent_attribute[0]['value'].split( + '|') + stat_coro = async_send_event('start_launch', agent_name, agent_version) + + if not stat_coro: + response = await launch_coro + else: + response = (await asyncio.gather(launch_coro, stat_coro))[0] + + if not response: + return + + launch_id = response.id + logger.debug(f'start_launch - ID: %s', launch_id) + if self.launch_uuid_print and self.print_output: + print(f'Report Portal Launch UUID: {self.launch_id}', file=self.print_output) + return launch_id + + async def start_test_item(self, + name: str, + start_time: str, + item_type: str, + *, + description: Optional[str] = None, + attributes: Optional[List[Dict]] = None, + parameters: Optional[Dict] = None, + parent_item_id: Optional[asyncio.Future] = None, + has_stats: bool = True, + code_ref: Optional[str] = None, + retry: bool = False, + test_case_id: Optional[str] = None, + **_: Any) -> Optional[str]: + if parent_item_id: + url = self.__get_item_url(parent_item_id) + else: + url = uri_join(self.base_url_v2, 'item') + request_payload = AsyncItemStartRequest( + name, + start_time, + item_type, + self.launch_id, + attributes=verify_value_length(attributes), + code_ref=code_ref, + description=description, + has_stats=has_stats, + parameters=parameters, + retry=retry, + test_case_id=test_case_id + ).payload + + response = await AsyncHttpRequest(self.session.post, url=url, json=request_payload).make() + if not response: + return + item_id = response.id + if item_id is NOT_FOUND: + logger.warning('start_test_item - invalid response: %s', + str(response.json)) + return item_id async def finish_test_item(self, - item_id: Union[asyncio.Future, str], + item_id: asyncio.Future, end_time: str, *, status: str = None, @@ -158,7 +295,45 @@ async def finish_test_item(self, description: str = None, retry: bool = False, **kwargs: Any) -> Optional[str]: - pass + url = self.__get_item_url(item_id) + request_payload = AsyncItemFinishRequest( + end_time, + self.launch_id, + status, + attributes=attributes, + description=description, + is_skipped_an_issue=self.is_skipped_an_issue, + issue=issue, + retry=retry + ).payload + response = await AsyncHttpRequest(self.session.put, url=url, json=request_payload).make() + if not response: + return + logger.debug('finish_test_item - ID: %s', item_id) + logger.debug('response message: %s', response.message) + return response.message + + async def finish_launch(self, + end_time: str, + status: str = None, + attributes: Optional[Union[List, Dict]] = None, + **kwargs: Any) -> Optional[str]: + if not self.use_own_launch: + return "" + url = self.__get_launch_url() + request_payload = LaunchFinishRequest( + end_time, + status=status, + attributes=attributes, + description=kwargs.get('description') + ).payload + response = await AsyncHttpRequest(self.session.put, url=url, json=request_payload, + name='Finish Launch').make() + if not response: + return + logger.debug('finish_launch - ID: %s', self.launch_id) + logger.debug('response message: %s', response.message) + return response.message async def get_item_id_by_uuid(self, uuid: Union[asyncio.Future, str]) -> Optional[str]: pass @@ -180,34 +355,6 @@ async def log(self, time: str, message: str, level: Optional[Union[int, str]] = item_id: Optional[Union[asyncio.Future, str]] = None) -> None: pass - async def start_launch(self, - name: str, - start_time: str, - description: Optional[str] = None, - attributes: Optional[Union[List, Dict]] = None, - rerun: bool = False, - rerun_of: Optional[str] = None, - **kwargs) -> Optional[str]: - pass - - async def start_test_item(self, - name: str, - start_time: str, - item_type: str, - *, - description: Optional[str] = None, - attributes: Optional[List[Dict]] = None, - parameters: Optional[Dict] = None, - parent_item_id: Optional[Union[asyncio.Future, str]] = None, - has_stats: bool = True, - code_ref: Optional[str] = None, - retry: bool = False, - test_case_id: Optional[str] = None, - **_: Any) -> Optional[str]: - parent = parent_item_id - if parent_item_id and asyncio.isfuture(parent_item_id): - parent = await parent_item_id - async def update_test_item(self, item_uuid: Union[asyncio.Future, str], attributes: Optional[Union[List, Dict]] = None, description: Optional[str] = None) -> Optional[str]: @@ -225,33 +372,25 @@ def current_item(self) -> Union[asyncio.Future, str]: """Retrieve the last item reported by the client.""" return self._item_stack.last() - def clone(self) -> '_RPClientAsync': - """Clone the client object, set current Item ID as cloned item ID. - :returns: Cloned client object - :rtype: _RPClientAsync - """ - cloned = _RPClientAsync( - endpoint=self.endpoint, - project=self.project, - api_key=self.api_key, - log_batch_size=self.log_batch_size, - is_skipped_an_issue=self.is_skipped_an_issue, - verify_ssl=self.verify_ssl, - retries=self.retries, - max_pool_size=self.max_pool_size, - launch_id=self.launch_id, - http_timeout=self.http_timeout, - log_batch_payload_size=self.log_batch_payload_size, - mode=self.mode - ) - current_item = self.current_item() - if current_item: - cloned._add_current_item(current_item) - return cloned +class RPClientAsync(_RPClientAsync): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) -class RPClientAsync(_RPClientAsync): + async def start_launch(self, + name: str, + start_time: str, + description: Optional[str] = None, + attributes: Optional[Union[List, Dict]] = None, + rerun: bool = False, + rerun_of: Optional[str] = None, + **kwargs) -> Optional[str]: + launch_id = await super().start_launch(name=name, start_time=start_time, description=description, + attributes=attributes, rerun=rerun, rerun_of=rerun_of, + **kwargs) + self.launch_id = launch_id + return launch_id async def start_test_item(self, name: str, @@ -261,7 +400,7 @@ async def start_test_item(self, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, parameters: Optional[Dict] = None, - parent_item_id: Optional[Union[asyncio.Future, str]] = None, + parent_item_id: Optional[str] = None, has_stats: bool = True, code_ref: Optional[str] = None, retry: bool = False, @@ -273,6 +412,7 @@ async def start_test_item(self, code_ref=code_ref, retry=retry, test_case_id=test_case_id, **kwargs) if item_id and item_id is not NOT_FOUND: + logger.debug('start_test_item - ID: %s', item_id) super()._add_current_item(item_id) return item_id @@ -292,38 +432,54 @@ async def finish_test_item(self, super()._remove_current_item() return result + def clone(self) -> 'RPClientAsync': + """Clone the client object, set current Item ID as cloned item ID. + + :returns: Cloned client object + :rtype: RPClientAsync + """ + cloned = RPClientAsync( + endpoint=self.endpoint, + project=self.project, + api_key=self.api_key, + log_batch_size=self.log_batch_size, + is_skipped_an_issue=self.is_skipped_an_issue, + verify_ssl=self.verify_ssl, + retries=self.retries, + max_pool_size=self.max_pool_size, + launch_id=self.launch_id, + http_timeout=self.http_timeout, + log_batch_payload_size=self.log_batch_payload_size, + mode=self.mode + ) + current_item = self.current_item() + if current_item: + cloned._add_current_item(current_item) + return cloned + class RPClientSync(_RPClientAsync): loop: asyncio.AbstractEventLoop thread: threading.Thread - - def __init__( - self, - endpoint: str, - project: str, - *, - api_key: str = None, - log_batch_size: int = 20, - is_skipped_an_issue: bool = True, - verify_ssl: bool = True, - retries: int = None, - max_pool_size: int = 50, - launch_id: str = None, - http_timeout: Union[float, Tuple[float, float]] = (10, 10), - log_batch_payload_size: int = MAX_LOG_BATCH_PAYLOAD_SIZE, - mode: str = 'DEFAULT', - launch_uuid_print: bool = False, - print_output: Optional[TextIO] = None, - **kwargs: Any - ) -> None: - super().__init__(endpoint, project, api_key=api_key, log_batch_size=log_batch_size, - is_skipped_an_issue=is_skipped_an_issue, verify_ssl=verify_ssl, retries=retries, - max_pool_size=max_pool_size, launch_id=launch_id, http_timeout=http_timeout, - log_batch_payload_size=log_batch_payload_size, mode=mode, - launch_uuid_print=launch_uuid_print, print_output=print_output, **kwargs) - self.loop = asyncio.new_event_loop() - self.thread = threading.Thread(target=self.loop.run_forever(), name='RP-Async-Client', daemon=True) - self.thread.start() + self_loop: bool + self_thread: bool + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + if 'loop' in kwargs and kwargs['loop']: + self.loop = kwargs['loop'] + self.self_loop = False + else: + self.loop = asyncio.new_event_loop() + self.self_loop = True + if 'thread' in kwargs and kwargs['thread']: + self.thread = kwargs['thread'] + self.self_thread = False + else: + self.thread = threading.Thread(target=self.loop.run_forever(), name='RP-Async-Client', + daemon=True) + self.thread.start() + self.self_thread = True def start_test_item(self, name: str, @@ -333,7 +489,7 @@ def start_test_item(self, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, parameters: Optional[Dict] = None, - parent_item_id: Optional[Union[asyncio.Future, str]] = None, + parent_item_id: Optional[asyncio.Future] = None, has_stats: bool = True, code_ref: Optional[str] = None, retry: bool = False, @@ -347,3 +503,30 @@ def start_test_item(self, item_id_task = self.loop.create_task(item_id_coro) super()._add_current_item(item_id_task) return item_id_task + + def clone(self) -> 'RPClientSync': + """Clone the client object, set current Item ID as cloned item ID. + + :returns: Cloned client object + :rtype: RPClientSync + """ + cloned = RPClientSync( + endpoint=self.endpoint, + project=self.project, + api_key=self.api_key, + log_batch_size=self.log_batch_size, + is_skipped_an_issue=self.is_skipped_an_issue, + verify_ssl=self.verify_ssl, + retries=self.retries, + max_pool_size=self.max_pool_size, + launch_id=self.launch_id, + http_timeout=self.http_timeout, + log_batch_payload_size=self.log_batch_payload_size, + mode=self.mode, + loop=self.loop, + thread=self.thread + ) + current_item = self.current_item() + if current_item: + cloned._add_current_item(current_item) + return cloned diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 39da0e7f..d5ad74f0 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -21,12 +21,10 @@ import asyncio import json as json_converter import logging -import ssl from dataclasses import dataclass from typing import Callable, Text, Optional, Union, List, Tuple, Any, TypeVar import aiohttp -from aiohttp import MultipartWriter, Payload from reportportal_client import helpers from reportportal_client.core.rp_file import RPFile @@ -75,7 +73,7 @@ def __init__(self, data: Optional[Any] = None, json: Optional[Any] = None, files: Optional[Any] = None, - verify_ssl: Optional[bool] = None, + verify_ssl: Optional[Union[bool, str]] = None, http_timeout: Union[float, Tuple[float, float]] = (10, 10), name: Optional[Text] = None) -> None: """Initialize instance attributes. @@ -85,6 +83,7 @@ def __init__(self, :param data: Dictionary, list of tuples, bytes, or file-like object to send in the body of the request :param json: JSON to be sent in the body of the request + :param files Dictionary for multipart encoding upload. :param verify_ssl: Is SSL certificate verification required :param http_timeout: a float in seconds for connect and read timeout. Use a Tuple to specific connect and @@ -131,32 +130,33 @@ def make(self): class AsyncHttpRequest(HttpRequest): """This model stores attributes related to RP HTTP requests.""" - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) + def __init__(self, + session_method: Callable, + url: Any, + data: Optional[Any] = None, + json: Optional[Any] = None, + http_timeout: Union[float, Tuple[float, float]] = (10, 10), + name: Optional[Text] = None) -> None: + """Initialize instance attributes. + + :param session_method: Method of the requests.Session instance + :param url: Request URL + :param data: Dictionary, list of tuples, bytes, or file-like object to send in the body of + the request + :param json: JSON to be sent in the body of the request + :param name: request name + """ + super().__init__(session_method=session_method, url=url, data=data, json=json, name=name) async def make(self): """Make HTTP request to the Report Portal API.""" - ssl_config = self.verify_ssl - if ssl_config and type(ssl_config) == str: - ssl_context = ssl.create_default_context() - ssl_context.load_cert_chain(ssl_config) - ssl_config = ssl_context - - timeout = None - if self.http_timeout: - if type(self.http_timeout) == tuple: - connect_timeout, read_timeout = self.http_timeout - else: - connect_timeout, read_timeout = self.http_timeout, self.http_timeout - timeout = aiohttp.ClientTimeout(connect=connect_timeout, sock_read=read_timeout) - - data = self.data - if self.files: - data = self.files + url = await_if_necessary(self.url) + if not url: + return try: - return RPResponse(await self.session_method(await await_if_necessary(self.url), data=data, - json=self.json, ssl=ssl_config, timeout=timeout)) + return RPResponse(await self.session_method(await await_if_necessary(self.url), data=self.data, + json=self.json)) except (KeyError, IOError, ValueError, TypeError) as exc: logger.warning( "Report Portal %s request failed", @@ -243,17 +243,17 @@ class ItemStartRequest(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#start-rootsuite-item """ + name: str + start_time: str + type_: str + launch_uuid: Any attributes: Optional[Union[list, dict]] code_ref: Optional[Text] description: Optional[Text] has_stats: bool - launch_uuid: Any - name: str parameters: Optional[Union[list, dict]] retry: bool - start_time: str test_case_id: Optional[Text] - type_: str @staticmethod def create_request(**kwargs) -> dict: @@ -282,7 +282,7 @@ def payload(self) -> dict: return ItemStartRequest.create_request(**data) -class ItemStartRequestAsync(ItemStartRequest): +class AsyncItemStartRequest(ItemStartRequest): def __int__(self, *args, **kwargs) -> None: super.__init__(*args, **kwargs) @@ -302,13 +302,13 @@ class ItemFinishRequest(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#finish-child-item """ + end_time: str + launch_uuid: Any + status: str attributes: Optional[Union[list, dict]] description: str - end_time: str is_skipped_an_issue: bool issue: Issue - launch_uuid: Any - status: str retry: bool @staticmethod @@ -340,7 +340,7 @@ def payload(self) -> dict: return ItemFinishRequest.create_request(**self.__dict__) -class ItemFinishRequestAsync(ItemFinishRequest): +class AsyncItemFinishRequest(ItemFinishRequest): def __int__(self, *args, **kwargs) -> None: super.__init__(*args, **kwargs) @@ -393,7 +393,7 @@ def multipart_size(self) -> int: return size -class RPRequestLogAsync(RPRequestLog): +class AsyncRPRequestLog(RPRequestLog): def __int__(self, *args, **kwargs) -> None: super.__init__(*args, **kwargs) @@ -415,10 +415,10 @@ class RPLogBatch(RPRequestBase): https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#batch-save-logs """ default_content: str - log_reqs: List[Union[RPRequestLog, RPRequestLogAsync]] + log_reqs: List[Union[RPRequestLog, AsyncRPRequestLog]] priority: Priority - def __init__(self, log_reqs: List[Union[RPRequestLog, RPRequestLogAsync]]) -> None: + def __init__(self, log_reqs: List[Union[RPRequestLog, AsyncRPRequestLog]]) -> None: """Initialize instance attributes. :param log_reqs: @@ -476,7 +476,7 @@ def payload(self) -> List[Tuple[str, tuple]]: return body -class RPLogBatchAsync(RPLogBatch): +class AsyncRPLogBatch(RPLogBatch): def __int__(self, *args, **kwargs) -> None: super.__init__(*args, **kwargs) @@ -486,7 +486,7 @@ async def __get_request_part(self) -> str: return json_converter.dumps(await asyncio.gather(*coroutines)) @property - async def payload(self) -> MultipartWriter: + async def payload(self) -> aiohttp.MultipartWriter: r"""Get HTTP payload for the request. Example: @@ -504,11 +504,11 @@ async def payload(self) -> MultipartWriter: '\n

Paragraph

', 'text/html'))] """ - json_payload = Payload(await self.__get_request_part(), content_type='application/json') + json_payload = aiohttp.Payload(await self.__get_request_part(), content_type='application/json') json_payload.set_content_disposition('form-data', name='json_request_part') - mpwriter = MultipartWriter('form-data') + mpwriter = aiohttp.MultipartWriter('form-data') mpwriter.append_payload(json_payload) for _, file in self.__get_files(): - file_payload = Payload(file[1], content_type=file[2], filename=file[0]) + file_payload = aiohttp.Payload(file[1], content_type=file[2], filename=file[0]) mpwriter.append_payload(file_payload) return mpwriter diff --git a/reportportal_client/services/statistics.py b/reportportal_client/services/statistics.py index 94e003eb..0c2bfa3e 100644 --- a/reportportal_client/services/statistics.py +++ b/reportportal_client/services/statistics.py @@ -15,19 +15,21 @@ import logging from platform import python_version +from typing import Optional +import aiohttp import requests from pkg_resources import get_distribution -from .client_id import get_client_id -from .constants import CLIENT_INFO, ENDPOINT +from reportportal_client.services.client_id import get_client_id +from reportportal_client.services.constants import CLIENT_INFO, ENDPOINT logger = logging.getLogger(__name__) ID, KEY = CLIENT_INFO.split(':') -def _get_client_info(): +def _get_client_info() -> tuple[str, str]: """Get name of the client and its version. :return: ('reportportal-client', '5.0.4') @@ -36,7 +38,7 @@ def _get_client_info(): return client.project_name, client.version -def _get_platform_info(): +def _get_platform_info() -> str: """Get current platform basic info, e.g.: 'Python 3.6.1'. :return: str represents the current platform, e.g.: 'Python 3.6.1' @@ -44,15 +46,7 @@ def _get_platform_info(): return 'Python ' + python_version() -def send_event(event_name, agent_name, agent_version): - """Send an event to statistics service. - - Use client and agent versions with their names. - - :param event_name: Event name to be used - :param agent_name: Name of the agent that uses the client - :param agent_version: Version of the agent - """ +def get_payload(event_name: str, agent_name: Optional[str], agent_version: Optional[str]) -> dict: client_name, client_version = _get_client_info() request_params = { 'client_name': client_name, @@ -67,20 +61,54 @@ def send_event(event_name, agent_name, agent_version): if agent_version: request_params['agent_version'] = agent_version - payload = { + return { 'client_id': get_client_id(), 'events': [{ 'name': event_name, 'params': request_params }] } + + +def send_event(event_name: str, agent_name: Optional[str], agent_version: Optional[str]) -> requests.Response: + """Send an event to statistics service. + + Use client and agent versions with their names. + + :param event_name: Event name to be used + :param agent_name: Name of the agent that uses the client + :param agent_version: Version of the agent + """ headers = {'User-Agent': 'python-requests'} query_params = { 'measurement_id': ID, 'api_secret': KEY } try: - return requests.post(url=ENDPOINT, json=payload, headers=headers, - params=query_params) + return requests.post(url=ENDPOINT, json=get_payload(event_name, agent_name, agent_version), + headers=headers, params=query_params) except requests.exceptions.RequestException as err: logger.debug('Failed to send data to Statistics service: %s', str(err)) + + +async def async_send_event(event_name: str, agent_name: Optional[str], + agent_version: Optional[str]) -> aiohttp.ClientResponse: + """Send an event to statistics service. + + Use client and agent versions with their names. + + :param event_name: Event name to be used + :param agent_name: Name of the agent that uses the client + :param agent_version: Version of the agent + """ + headers = {'User-Agent': 'python-aiohttp'} + query_params = { + 'measurement_id': ID, + 'api_secret': KEY + } + async with aiohttp.ClientSession() as session: + result = await session.post(url=ENDPOINT, json=get_payload(event_name, agent_name, agent_version), + headers=headers, params=query_params) + if not result.ok: + logger.debug('Failed to send data to Statistics service: %s', result.reason) + return result diff --git a/reportportal_client/services/statistics.pyi b/reportportal_client/services/statistics.pyi deleted file mode 100644 index f061bf70..00000000 --- a/reportportal_client/services/statistics.pyi +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2023 EPAM Systems -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License - -from logging import Logger -from typing import Text, Optional - -import requests - -logger: Logger - - -def _get_client_info() -> tuple[Text, Text]: ... - - -def _get_platform_info() -> Text: ... - - -def send_event(event_name: Text, agent_name: Optional[Text], - agent_version: Optional[Text]) -> requests.Response: ... From 3fe68df679faecbadab1ca9b9cbb28131c1f39fc Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 5 Sep 2023 22:25:28 +0300 Subject: [PATCH 021/268] Async RPClient: WIP --- reportportal_client/aio/client.py | 143 ++++++++++++++++++------ reportportal_client/core/rp_requests.py | 12 +- reportportal_client/helpers.py | 81 +++++++------- reportportal_client/helpers.pyi | 32 ------ 4 files changed, 147 insertions(+), 121 deletions(-) delete mode 100644 reportportal_client/helpers.pyi diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 5f5c298b..97cc0b20 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -25,12 +25,12 @@ import aiohttp -from core.rp_requests import LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, \ - AsyncItemFinishRequest, LaunchFinishRequest -from helpers import uri_join, verify_value_length # noinspection PyProtectedMember from reportportal_client._local import set_current from reportportal_client.core.rp_issues import Issue +from reportportal_client.core.rp_requests import LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, \ + AsyncItemFinishRequest, LaunchFinishRequest +from reportportal_client.helpers import uri_join, verify_value_length, await_if_necessary from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.services.statistics import async_send_event from reportportal_client.static.defines import NOT_FOUND @@ -47,7 +47,6 @@ def last(self): return self.queue[-1] -# TODO: Correct url and launch id handling for sync and async classes class _RPClientAsync: api_v1: str api_v2: str @@ -55,8 +54,6 @@ class _RPClientAsync: base_url_v2: str endpoint: str is_skipped_an_issue: bool - launch_id: asyncio.Future - use_own_launch: bool log_batch_size: int log_batch_payload_size: int project: str @@ -84,7 +81,7 @@ def __init__( verify_ssl: Union[bool, str] = True, retries: int = None, max_pool_size: int = 50, - launch_id: Optional[asyncio.Future] = None, + http_timeout: Union[float, Tuple[float, float]] = (10, 10), log_batch_payload_size: int = MAX_LOG_BATCH_PAYLOAD_SIZE, mode: str = 'DEFAULT', @@ -102,11 +99,6 @@ def __init__( self.base_url_v2 = uri_join( self.endpoint, 'api/{}'.format(self.api_v2), self.project) self.is_skipped_an_issue = is_skipped_an_issue - if launch_id: - self.launch_id = launch_id - self.use_own_launch = False - else: - self.use_own_launch = True self.log_batch_size = log_batch_size self.log_batch_payload_size = log_batch_payload_size self.verify_ssl = verify_ssl @@ -170,15 +162,15 @@ def session(self) -> aiohttp.ClientSession: timeout=timeout) return self.__session - async def __get_item_url(self, id_future: asyncio.Future) -> Optional[str]: - item_id = await id_future + async def __get_item_url(self, item_id_future: Union[str, asyncio.Future]) -> Optional[str]: + item_id = await await_if_necessary(item_id_future) if item_id is NOT_FOUND: logger.warning('Attempt to make request for non-existent id.') return return uri_join(self.base_url_v2, 'item', item_id) - async def __get_launch_url(self) -> Optional[str]: - launch_id = await self.launch_id + async def __get_launch_url(self, launch_id_future: Union[str, asyncio.Future]) -> Optional[str]: + launch_id = await await_if_necessary(launch_id_future) if launch_id is NOT_FOUND: logger.warning('Attempt to make request for non-existent launch.') return @@ -187,6 +179,7 @@ async def __get_launch_url(self) -> Optional[str]: async def start_launch(self, name: str, start_time: str, + *, description: Optional[str] = None, attributes: Optional[Union[List, Dict]] = None, rerun: bool = False, @@ -202,8 +195,6 @@ async def start_launch(self, :param rerun_of: For rerun mode specifies which launch will be re-run. Should be used with the 'rerun' option. """ - if not self.use_own_launch: - return self.launch_id url = uri_join(self.base_url_v2, 'launch') request_payload = LaunchStartRequest( name=name, @@ -241,18 +232,19 @@ async def start_launch(self, launch_id = response.id logger.debug(f'start_launch - ID: %s', launch_id) if self.launch_uuid_print and self.print_output: - print(f'Report Portal Launch UUID: {self.launch_id}', file=self.print_output) + print(f'Report Portal Launch UUID: {launch_id}', file=self.print_output) return launch_id async def start_test_item(self, name: str, start_time: str, item_type: str, + launch_id: Union[str, asyncio.Future], *, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, parameters: Optional[Dict] = None, - parent_item_id: Optional[asyncio.Future] = None, + parent_item_id: Optional[Union[str, asyncio.Future]] = None, has_stats: bool = True, code_ref: Optional[str] = None, retry: bool = False, @@ -266,7 +258,7 @@ async def start_test_item(self, name, start_time, item_type, - self.launch_id, + launch_id, attributes=verify_value_length(attributes), code_ref=code_ref, description=description, @@ -286,8 +278,9 @@ async def start_test_item(self, return item_id async def finish_test_item(self, - item_id: asyncio.Future, + item_id: Union[str, asyncio.Future], end_time: str, + launch_id: Union[str, asyncio.Future], *, status: str = None, issue: Optional[Issue] = None, @@ -298,7 +291,7 @@ async def finish_test_item(self, url = self.__get_item_url(item_id) request_payload = AsyncItemFinishRequest( end_time, - self.launch_id, + launch_id, status, attributes=attributes, description=description, @@ -314,13 +307,13 @@ async def finish_test_item(self, return response.message async def finish_launch(self, + launch_id: Union[str, asyncio.Future], end_time: str, + *, status: str = None, attributes: Optional[Union[List, Dict]] = None, **kwargs: Any) -> Optional[str]: - if not self.use_own_launch: - return "" - url = self.__get_launch_url() + url = self.__get_launch_url(launch_id) request_payload = LaunchFinishRequest( end_time, status=status, @@ -331,7 +324,7 @@ async def finish_launch(self, name='Finish Launch').make() if not response: return - logger.debug('finish_launch - ID: %s', self.launch_id) + logger.debug('finish_launch - ID: %s', launch_id) logger.debug('response message: %s', response.message) return response.message @@ -374,9 +367,17 @@ def current_item(self) -> Union[asyncio.Future, str]: class RPClientAsync(_RPClientAsync): + launch_id: Optional[str] + use_own_launch: bool - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) + def __init__(self, endpoint: str, project: str, *, launch_id: Optional[str] = None, + **kwargs: Any) -> None: + super().__init__(endpoint, project, **kwargs) + if launch_id: + self.launch_id = launch_id + self.use_own_launch = False + else: + self.use_own_launch = True async def start_launch(self, name: str, @@ -386,7 +387,9 @@ async def start_launch(self, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Optional[str]: - launch_id = await super().start_launch(name=name, start_time=start_time, description=description, + if not self.use_own_launch: + return self.launch_id + launch_id = await super().start_launch(name, start_time, description=description, attributes=attributes, rerun=rerun, rerun_of=rerun_of, **kwargs) self.launch_id = launch_id @@ -426,12 +429,22 @@ async def finish_test_item(self, description: str = None, retry: bool = False, **kwargs: Any) -> Optional[str]: - result = await super().finish_test_item(item_id, end_time, status=status, issue=issue, + result = await super().finish_test_item(item_id, end_time, self.launch_id, status=status, issue=issue, attributes=attributes, description=description, retry=retry, **kwargs) super()._remove_current_item() return result + async def finish_launch(self, + end_time: str, + status: str = None, + attributes: Optional[Union[List, Dict]] = None, + **kwargs: Any) -> Optional[str]: + if not self.use_own_launch: + return "" + return await super().finish_launch(self.launch_id, end_time, status=status, attributes=attributes, + **kwargs) + def clone(self) -> 'RPClientAsync': """Clone the client object, set current Item ID as cloned item ID. @@ -463,9 +476,18 @@ class RPClientSync(_RPClientAsync): thread: threading.Thread self_loop: bool self_thread: bool + launch_id: Optional[asyncio.Future] + use_own_launch: bool + + def __init__(self, endpoint: str, project: str, *, launch_id: Optional[asyncio.Future] = None, + **kwargs: Any) -> None: + super().__init__(endpoint, project, **kwargs) + if launch_id: + self.launch_id = launch_id + self.use_own_launch = False + else: + self.use_own_launch = True - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) if 'loop' in kwargs and kwargs['loop']: self.loop = kwargs['loop'] self.self_loop = False @@ -481,6 +503,26 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.thread.start() self.self_thread = True + async def __empty_line(self): + return "" + + def start_launch(self, + name: str, + start_time: str, + description: Optional[str] = None, + attributes: Optional[Union[List, Dict]] = None, + rerun: bool = False, + rerun_of: Optional[str] = None, + **kwargs) -> asyncio.Future: + if not self.use_own_launch: + return self.launch_id + launch_id_coro = super().start_launch(name, start_time, description=description, + attributes=attributes, rerun=rerun, rerun_of=rerun_of, + **kwargs) + launch_id_task = self.loop.create_task(launch_id_coro) + self.launch_id = launch_id_task + return launch_id_task + def start_test_item(self, name: str, start_time: str, @@ -494,8 +536,10 @@ def start_test_item(self, code_ref: Optional[str] = None, retry: bool = False, test_case_id: Optional[str] = None, - **kwargs: Any) -> Optional[asyncio.Future]: - item_id_coro = super().start_test_item(name, start_time, item_type, description=description, + **kwargs: Any) -> asyncio.Future: + + item_id_coro = super().start_test_item(name, start_time, item_type, launch_id=self.launch_id, + description=description, attributes=attributes, parameters=parameters, parent_item_id=parent_item_id, has_stats=has_stats, code_ref=code_ref, retry=retry, test_case_id=test_case_id, @@ -504,6 +548,35 @@ def start_test_item(self, super()._add_current_item(item_id_task) return item_id_task + def finish_test_item(self, + item_id: asyncio.Future, + end_time: str, + *, + status: str = None, + issue: Optional[Issue] = None, + attributes: Optional[Union[List, Dict]] = None, + description: str = None, + retry: bool = False, + **kwargs: Any) -> asyncio.Future: + result_coro = super().finish_test_item(item_id, end_time, self.launch_id, status=status, issue=issue, + attributes=attributes, description=description, retry=retry, + **kwargs) + result_task = self.loop.create_task(result_coro) + super()._remove_current_item() + return result_task + + def finish_launch(self, + end_time: str, + status: str = None, + attributes: Optional[Union[List, Dict]] = None, + **kwargs: Any) -> asyncio.Future: + if not self.use_own_launch: + return self.loop.create_task(self.__empty_line()) + result_coro = super().finish_launch(self.launch_id, end_time, status=status, attributes=attributes, + **kwargs) + result_task = self.loop.create_task(result_coro) + return result_task + def clone(self) -> 'RPClientSync': """Clone the client object, set current Item ID as cloned item ID. diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index d5ad74f0..6a1eb89a 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -30,7 +30,7 @@ from reportportal_client.core.rp_file import RPFile from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_responses import RPResponse -from reportportal_client.helpers import dict_to_payload +from reportportal_client.helpers import dict_to_payload, await_if_necessary from reportportal_client.static.abstract import ( AbstractBaseClass, abstractmethod @@ -45,15 +45,6 @@ T = TypeVar("T") -async def await_if_necessary(obj: Optional[Any]) -> Any: - if obj: - if asyncio.isfuture(obj) or asyncio.iscoroutine(obj): - return await obj - elif asyncio.iscoroutinefunction(obj): - return await obj() - return obj - - class HttpRequest: """This model stores attributes related to RP HTTP requests.""" @@ -135,7 +126,6 @@ def __init__(self, url: Any, data: Optional[Any] = None, json: Optional[Any] = None, - http_timeout: Union[float, Tuple[float, float]] = (10, 10), name: Optional[Text] = None) -> None: """Initialize instance attributes. diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index 51a0f8b8..f432718b 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -11,39 +11,29 @@ See the License for the specific language governing permissions and limitations under the License. """ +import asyncio import inspect import json import logging import time import uuid from platform import machine, processor, system +from typing import Optional, Any, List, Dict, Callable -import six from pkg_resources import DistributionNotFound, get_distribution -from .static.defines import ATTRIBUTE_LENGTH_LIMIT +from reportportal_client.core.rp_file import RPFile +from reportportal_client.static.defines import ATTRIBUTE_LENGTH_LIMIT -logger = logging.getLogger(__name__) +logger: logging.Logger = logging.getLogger(__name__) -def generate_uuid(): +def generate_uuid() -> str: """Generate UUID.""" return str(uuid.uuid4()) -def convert_string(value): - """Support and convert strings in py2 and py3. - - :param value: input string - :return value: converted string - """ - if isinstance(value, six.text_type): - # Don't try to encode 'unicode' in Python 2. - return value - return str(value) - - -def dict_to_payload(dictionary): +def dict_to_payload(dictionary: dict) -> List[dict]: """Convert incoming dictionary to the list of dictionaries. This function transforms the given dictionary of tags/attributes into @@ -55,12 +45,12 @@ def dict_to_payload(dictionary): """ hidden = dictionary.pop('system', False) return [ - {'key': key, 'value': convert_string(value), 'system': hidden} + {'key': key, 'value': str(value), 'system': hidden} for key, value in sorted(dictionary.items()) ] -def gen_attributes(rp_attributes): +def gen_attributes(rp_attributes: List[str]) -> List[Dict[str, str]]: """Generate list of attributes for the API request. Example of input list: @@ -89,7 +79,7 @@ def gen_attributes(rp_attributes): return attrs -def get_launch_sys_attrs(): +def get_launch_sys_attrs() -> Dict[str]: """Generate attributes for the launch containing system information. :return: dict {'os': 'Windows', @@ -104,7 +94,7 @@ def get_launch_sys_attrs(): } -def get_package_version(package_name): +def get_package_version(package_name) -> str: """Get version of the given package. :param package_name: Name of the package @@ -117,11 +107,11 @@ def get_package_version(package_name): return package_version -def verify_value_length(attributes): +def verify_value_length(attributes: List[dict]) -> List[dict]: """Verify length of the attribute value. The length of the attribute value should have size from '1' to '128'. - Otherwise HTTP response will return an error. + Otherwise, HTTP response will return an error. Example of the input list: [{'key': 'tag_name', 'value': 'tag_value1'}, {'value': 'tag_value2'}] :param attributes: List of attributes(tags) @@ -141,12 +131,12 @@ def verify_value_length(attributes): return attributes -def timestamp(): +def timestamp() -> str: """Return string representation of the current time in milliseconds.""" return str(int(time.time() * 1000)) -def uri_join(*uri_parts): +def uri_join(*uri_parts: str) -> str: """Join uri parts. Avoiding usage of urlparse.urljoin and os.path.join @@ -160,7 +150,7 @@ def uri_join(*uri_parts): return '/'.join(str(s).strip('/').strip('\\') for s in uri_parts) -def get_function_params(func, args, kwargs): +def get_function_params(func: Callable, args: tuple, kwargs: Dict[str, Any]) -> Dict[str, Any]: """Extract argument names from the function and combine them with values. :param func: the function to get arg names @@ -168,11 +158,7 @@ def get_function_params(func, args, kwargs): :param kwargs: function's kwargs :return: a dictionary of values """ - if six.PY2: - # Use deprecated method for python 2.7 compatibility - arg_spec = inspect.getargspec(func) - else: - arg_spec = inspect.getfullargspec(func) + arg_spec = inspect.getfullargspec(func) result = dict() for i, arg_name in enumerate(arg_spec.args): if i >= len(args): @@ -183,36 +169,36 @@ def get_function_params(func, args, kwargs): return result if len(result.items()) > 0 else None -TYPICAL_MULTIPART_BOUNDARY = '--972dbca3abacfd01fb4aea0571532b52' +TYPICAL_MULTIPART_BOUNDARY: str = '--972dbca3abacfd01fb4aea0571532b52' -TYPICAL_JSON_PART_HEADER = TYPICAL_MULTIPART_BOUNDARY + '''\r +TYPICAL_JSON_PART_HEADER: str = TYPICAL_MULTIPART_BOUNDARY + '''\r Content-Disposition: form-data; name="json_request_part"\r Content-Type: application/json\r \r ''' -TYPICAL_FILE_PART_HEADER = TYPICAL_MULTIPART_BOUNDARY + '''\r +TYPICAL_FILE_PART_HEADER: str = TYPICAL_MULTIPART_BOUNDARY + '''\r Content-Disposition: form-data; name="file"; filename="{0}"\r Content-Type: {1}\r \r ''' -TYPICAL_JSON_PART_HEADER_LENGTH = len(TYPICAL_JSON_PART_HEADER) +TYPICAL_JSON_PART_HEADER_LENGTH: int = len(TYPICAL_JSON_PART_HEADER) -TYPICAL_MULTIPART_FOOTER = '\r\n' + TYPICAL_MULTIPART_BOUNDARY + '--' +TYPICAL_MULTIPART_FOOTER: str = '\r\n' + TYPICAL_MULTIPART_BOUNDARY + '--' -TYPICAL_MULTIPART_FOOTER_LENGTH = len(TYPICAL_MULTIPART_FOOTER) +TYPICAL_MULTIPART_FOOTER_LENGTH: int = len(TYPICAL_MULTIPART_FOOTER) -TYPICAL_JSON_ARRAY = '[]' +TYPICAL_JSON_ARRAY: str = '[]' -TYPICAL_JSON_ARRAY_LENGTH = len(TYPICAL_JSON_ARRAY) +TYPICAL_JSON_ARRAY_LENGTH: int = len(TYPICAL_JSON_ARRAY) -TYPICAL_JSON_ARRAY_ELEMENT = ',' +TYPICAL_JSON_ARRAY_ELEMENT: str = ',' -TYPICAL_JSON_ARRAY_ELEMENT_LENGTH = len(TYPICAL_JSON_ARRAY_ELEMENT) +TYPICAL_JSON_ARRAY_ELEMENT_LENGTH: int = len(TYPICAL_JSON_ARRAY_ELEMENT) -def calculate_json_part_size(json_dict): +def calculate_json_part_size(json_dict: dict) -> int: """Predict a JSON part size of Multipart request. :param json_dict: a dictionary representing the JSON @@ -225,7 +211,7 @@ def calculate_json_part_size(json_dict): return size -def calculate_file_part_size(file): +def calculate_file_part_size(file: RPFile) -> int: """Predict a file part size of Multipart request. :param file: RPFile class instance @@ -236,3 +222,12 @@ def calculate_file_part_size(file): size = len(TYPICAL_FILE_PART_HEADER.format(file.name, file.content_type)) size += len(file.content) return size + + +async def await_if_necessary(obj: Optional[Any]) -> Any: + if obj: + if asyncio.isfuture(obj) or asyncio.iscoroutine(obj): + return await obj + elif asyncio.iscoroutinefunction(obj): + return await obj() + return obj diff --git a/reportportal_client/helpers.pyi b/reportportal_client/helpers.pyi deleted file mode 100644 index 8d52d805..00000000 --- a/reportportal_client/helpers.pyi +++ /dev/null @@ -1,32 +0,0 @@ -from core.rp_file import RPFile -from .errors import EntryCreatedError as EntryCreatedError, \ - OperationCompletionError as OperationCompletionError, \ - ResponseError as ResponseError -from logging import Logger - -from typing import Text, Tuple, List, Callable, Any, Dict, Optional -from requests import Response - -logger: Logger - -TYPICAL_FILE_PART_HEADER: Text -TYPICAL_MULTIPART_FOOTER_LENGTH: int - -def generate_uuid() -> Text: ... -def convert_string(value: Text) -> Text: ... -def dict_to_payload(dictionary: dict) -> list[dict]: ... -def gen_attributes(rp_attributes: List[Text]) -> List[Dict[Text, Text]]: ... -def get_launch_sys_attrs()-> dict[Text]: ... -def get_package_version(package_name:Text) -> Text: ... -def uri_join(*uri_parts: Text) -> Text: ... -def get_id(response: Response) -> Text: ... -def get_msg(response: Response) -> dict: ... -def get_data(response: Response) -> dict: ... -def get_json(response: Response) -> dict: ... -def get_error_messages(data: dict) -> list: ... -def verify_value_length(attributes: list[dict]) -> list[dict]: ... -def timestamp() -> Text: ... -def get_function_params(func: Callable, args: Tuple[Any, ...], - kwargs: Dict[Text, Any]) -> Dict[Text, Any]: ... -def calculate_json_part_size(json_dict: dict) -> int: ... -def calculate_file_part_size(file: RPFile) -> int: ... From 06976fac2e08cc5b089fd36f9f1ee8ecfa3095d5 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 5 Sep 2023 22:27:06 +0300 Subject: [PATCH 022/268] Async RPClient: WIP --- reportportal_client/aio/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 97cc0b20..58e60e09 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -565,6 +565,7 @@ def finish_test_item(self, super()._remove_current_item() return result_task + # TODO: implement loop task finish wait def finish_launch(self, end_time: str, status: str = None, From fb1f403b736dfca18eec7150f5a83aaf06780b11 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 5 Sep 2023 22:28:51 +0300 Subject: [PATCH 023/268] Async RPClient: WIP --- reportportal_client/aio/client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 58e60e09..13d35727 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -366,7 +366,7 @@ def current_item(self) -> Union[asyncio.Future, str]: return self._item_stack.last() -class RPClientAsync(_RPClientAsync): +class AsyncRPClient(_RPClientAsync): launch_id: Optional[str] use_own_launch: bool @@ -445,13 +445,13 @@ async def finish_launch(self, return await super().finish_launch(self.launch_id, end_time, status=status, attributes=attributes, **kwargs) - def clone(self) -> 'RPClientAsync': + def clone(self) -> 'AsyncRPClient': """Clone the client object, set current Item ID as cloned item ID. :returns: Cloned client object - :rtype: RPClientAsync + :rtype: AsyncRPClient """ - cloned = RPClientAsync( + cloned = AsyncRPClient( endpoint=self.endpoint, project=self.project, api_key=self.api_key, @@ -471,7 +471,7 @@ def clone(self) -> 'RPClientAsync': return cloned -class RPClientSync(_RPClientAsync): +class SyncRPClient(_RPClientAsync): loop: asyncio.AbstractEventLoop thread: threading.Thread self_loop: bool @@ -578,13 +578,13 @@ def finish_launch(self, result_task = self.loop.create_task(result_coro) return result_task - def clone(self) -> 'RPClientSync': + def clone(self) -> 'SyncRPClient': """Clone the client object, set current Item ID as cloned item ID. :returns: Cloned client object - :rtype: RPClientSync + :rtype: SyncRPClient """ - cloned = RPClientSync( + cloned = SyncRPClient( endpoint=self.endpoint, project=self.project, api_key=self.api_key, From 02c933b3c9dde3eb5525ac1eb2d7fa7bb1e80598 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 5 Sep 2023 22:43:38 +0300 Subject: [PATCH 024/268] Async RPClient: WIP --- reportportal_client/_local/__init__.pyi | 4 +- reportportal_client/aio/client.py | 59 +++++++++++++++---------- reportportal_client/core/rp_requests.py | 5 ++- reportportal_client/static/abstract.py | 2 +- reportportal_client/steps/__init__.pyi | 4 +- 5 files changed, 43 insertions(+), 31 deletions(-) diff --git a/reportportal_client/_local/__init__.pyi b/reportportal_client/_local/__init__.pyi index 4711a6a5..7ba4b639 100644 --- a/reportportal_client/_local/__init__.pyi +++ b/reportportal_client/_local/__init__.pyi @@ -14,11 +14,11 @@ from typing import Optional, Union # noinspection PyProtectedMember -from reportportal_client.aio.client import _RPClientAsync +from reportportal_client.aio.client import _AsyncRPClient from reportportal_client.client import RPClient def current() -> Optional[RPClient]: ... -def set_current(client: Optional[Union[RPClient, _RPClientAsync]]) -> None: ... +def set_current(client: Optional[Union[RPClient, _AsyncRPClient]]) -> None: ... diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 13d35727..75285eda 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -25,6 +25,10 @@ import aiohttp +from reportportal_client.static.abstract import ( + AbstractBaseClass, + abstractmethod +) # noinspection PyProtectedMember from reportportal_client._local import set_current from reportportal_client.core.rp_issues import Issue @@ -47,7 +51,9 @@ def last(self): return self.queue[-1] -class _RPClientAsync: +class _AsyncRPClient(metaclass=AbstractBaseClass): + __metaclass__ = AbstractBaseClass + api_v1: str api_v2: str base_url_v1: str @@ -162,14 +168,14 @@ def session(self) -> aiohttp.ClientSession: timeout=timeout) return self.__session - async def __get_item_url(self, item_id_future: Union[str, asyncio.Future]) -> Optional[str]: + async def __get_item_url(self, item_id_future: Union[str, asyncio.Task]) -> Optional[str]: item_id = await await_if_necessary(item_id_future) if item_id is NOT_FOUND: logger.warning('Attempt to make request for non-existent id.') return return uri_join(self.base_url_v2, 'item', item_id) - async def __get_launch_url(self, launch_id_future: Union[str, asyncio.Future]) -> Optional[str]: + async def __get_launch_url(self, launch_id_future: Union[str, asyncio.Task]) -> Optional[str]: launch_id = await await_if_necessary(launch_id_future) if launch_id is NOT_FOUND: logger.warning('Attempt to make request for non-existent launch.') @@ -239,12 +245,12 @@ async def start_test_item(self, name: str, start_time: str, item_type: str, - launch_id: Union[str, asyncio.Future], + launch_id: Union[str, asyncio.Task], *, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, parameters: Optional[Dict] = None, - parent_item_id: Optional[Union[str, asyncio.Future]] = None, + parent_item_id: Optional[Union[str, asyncio.Task]] = None, has_stats: bool = True, code_ref: Optional[str] = None, retry: bool = False, @@ -278,9 +284,9 @@ async def start_test_item(self, return item_id async def finish_test_item(self, - item_id: Union[str, asyncio.Future], + item_id: Union[str, asyncio.Task], end_time: str, - launch_id: Union[str, asyncio.Future], + launch_id: Union[str, asyncio.Task], *, status: str = None, issue: Optional[Issue] = None, @@ -307,7 +313,7 @@ async def finish_test_item(self, return response.message async def finish_launch(self, - launch_id: Union[str, asyncio.Future], + launch_id: Union[str, asyncio.Task], end_time: str, *, status: str = None, @@ -328,7 +334,7 @@ async def finish_launch(self, logger.debug('response message: %s', response.message) return response.message - async def get_item_id_by_uuid(self, uuid: Union[asyncio.Future, str]) -> Optional[str]: + async def get_item_id_by_uuid(self, uuid: Union[asyncio.Task, str]) -> Optional[str]: pass async def get_launch_info(self) -> Optional[Dict]: @@ -345,15 +351,15 @@ async def get_project_settings(self) -> Optional[Dict]: async def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, - item_id: Optional[Union[asyncio.Future, str]] = None) -> None: + item_id: Optional[Union[asyncio.Task, str]] = None) -> None: pass - async def update_test_item(self, item_uuid: Union[asyncio.Future, str], + async def update_test_item(self, item_uuid: Union[asyncio.Task, str], attributes: Optional[Union[List, Dict]] = None, description: Optional[str] = None) -> Optional[str]: pass - def _add_current_item(self, item: Union[asyncio.Future, str]) -> None: + def _add_current_item(self, item: Union[asyncio.Task, str]) -> None: """Add the last item from the self._items queue.""" self._item_stack.put(item) @@ -361,12 +367,17 @@ def _remove_current_item(self) -> None: """Remove the last item from the self._items queue.""" return self._item_stack.get() - def current_item(self) -> Union[asyncio.Future, str]: + def current_item(self) -> Union[asyncio.Task, str]: """Retrieve the last item reported by the client.""" return self._item_stack.last() + @abstractmethod + def clone(self) -> '_AsyncRPClient': + """Abstract interface for cloning the client.""" + raise NotImplementedError('Clone interface is not implemented!') + -class AsyncRPClient(_RPClientAsync): +class AsyncRPClient(_AsyncRPClient): launch_id: Optional[str] use_own_launch: bool @@ -420,7 +431,7 @@ async def start_test_item(self, return item_id async def finish_test_item(self, - item_id: Union[asyncio.Future, str], + item_id: Union[asyncio.Task, str], end_time: str, *, status: str = None, @@ -471,15 +482,15 @@ def clone(self) -> 'AsyncRPClient': return cloned -class SyncRPClient(_RPClientAsync): +class SyncRPClient(_AsyncRPClient): loop: asyncio.AbstractEventLoop thread: threading.Thread self_loop: bool self_thread: bool - launch_id: Optional[asyncio.Future] + launch_id: Optional[asyncio.Task] use_own_launch: bool - def __init__(self, endpoint: str, project: str, *, launch_id: Optional[asyncio.Future] = None, + def __init__(self, endpoint: str, project: str, *, launch_id: Optional[asyncio.Task] = None, **kwargs: Any) -> None: super().__init__(endpoint, project, **kwargs) if launch_id: @@ -513,7 +524,7 @@ def start_launch(self, attributes: Optional[Union[List, Dict]] = None, rerun: bool = False, rerun_of: Optional[str] = None, - **kwargs) -> asyncio.Future: + **kwargs) -> asyncio.Task: if not self.use_own_launch: return self.launch_id launch_id_coro = super().start_launch(name, start_time, description=description, @@ -531,12 +542,12 @@ def start_test_item(self, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, parameters: Optional[Dict] = None, - parent_item_id: Optional[asyncio.Future] = None, + parent_item_id: Optional[asyncio.Task] = None, has_stats: bool = True, code_ref: Optional[str] = None, retry: bool = False, test_case_id: Optional[str] = None, - **kwargs: Any) -> asyncio.Future: + **kwargs: Any) -> asyncio.Task: item_id_coro = super().start_test_item(name, start_time, item_type, launch_id=self.launch_id, description=description, @@ -549,7 +560,7 @@ def start_test_item(self, return item_id_task def finish_test_item(self, - item_id: asyncio.Future, + item_id: asyncio.Task, end_time: str, *, status: str = None, @@ -557,7 +568,7 @@ def finish_test_item(self, attributes: Optional[Union[List, Dict]] = None, description: str = None, retry: bool = False, - **kwargs: Any) -> asyncio.Future: + **kwargs: Any) -> asyncio.Task: result_coro = super().finish_test_item(item_id, end_time, self.launch_id, status=status, issue=issue, attributes=attributes, description=description, retry=retry, **kwargs) @@ -570,7 +581,7 @@ def finish_launch(self, end_time: str, status: str = None, attributes: Optional[Union[List, Dict]] = None, - **kwargs: Any) -> asyncio.Future: + **kwargs: Any) -> asyncio.Task: if not self.use_own_launch: return self.loop.create_task(self.__empty_line()) result_coro = super().finish_launch(self.launch_id, end_time, status=status, attributes=attributes, diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 6a1eb89a..47570b2a 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -108,8 +108,9 @@ def priority(self, value: Priority) -> None: def make(self): """Make HTTP request to the Report Portal API.""" try: - return RPResponse(self.session_method(self.url, data=self.data, json=self.json, files=self.files, - verify=self.verify_ssl, timeout=self.http_timeout)) + return RPResponse(self.session_method(self.url, data=self.data, json=self.json, + files=self.files, verify=self.verify_ssl, + timeout=self.http_timeout)) except (KeyError, IOError, ValueError, TypeError) as exc: logger.warning( "Report Portal %s request failed", diff --git a/reportportal_client/static/abstract.py b/reportportal_client/static/abstract.py index 85e1e9ae..82a4c87c 100644 --- a/reportportal_client/static/abstract.py +++ b/reportportal_client/static/abstract.py @@ -19,7 +19,7 @@ class AbstractBaseClass(_ABCMeta): - """Metaclass fot pure Interfacing. + """Metaclass for pure Interfacing. Being set as __metaclass__, forbids direct object creation from this class, allowing only inheritance. I.e. diff --git a/reportportal_client/steps/__init__.pyi b/reportportal_client/steps/__init__.pyi index 636802ec..4e62fb02 100644 --- a/reportportal_client/steps/__init__.pyi +++ b/reportportal_client/steps/__init__.pyi @@ -14,14 +14,14 @@ from typing import Text, Optional, Dict, Any, Callable, Union # noinspection PyProtectedMember -from reportportal_client.aio.client import _RPClientAsync +from reportportal_client.aio.client import _AsyncRPClient from reportportal_client.client import RPClient class StepReporter: client: RPClient = ... - def __init__(self, rp_client: Union[RPClient, _RPClientAsync]) -> None: ... + def __init__(self, rp_client: Union[RPClient, _AsyncRPClient]) -> None: ... def start_nested_step(self, name: Text, From be2a0a312ab5fe1f6d0a991f92ec75abaf22d969 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 5 Sep 2023 22:55:22 +0300 Subject: [PATCH 025/268] Async RPClient: WIP --- reportportal_client/aio/client.py | 29 +++++++++++------------------ reportportal_client/client.py | 26 ++++++++++---------------- reportportal_client/helpers.py | 10 +++++++++- 3 files changed, 30 insertions(+), 35 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 75285eda..d7c1b166 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -25,18 +25,18 @@ import aiohttp -from reportportal_client.static.abstract import ( - AbstractBaseClass, - abstractmethod -) # noinspection PyProtectedMember from reportportal_client._local import set_current from reportportal_client.core.rp_issues import Issue -from reportportal_client.core.rp_requests import LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, \ - AsyncItemFinishRequest, LaunchFinishRequest -from reportportal_client.helpers import uri_join, verify_value_length, await_if_necessary +from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, + AsyncItemFinishRequest, LaunchFinishRequest) +from reportportal_client.helpers import uri_join, verify_value_length, await_if_necessary, agent_name_version from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.services.statistics import async_send_event +from reportportal_client.static.abstract import ( + AbstractBaseClass, + abstractmethod +) from reportportal_client.static.defines import NOT_FOUND from reportportal_client.steps import StepReporter @@ -218,19 +218,12 @@ async def start_launch(self, stat_coro = None if not self._skip_analytics: - agent_name, agent_version = None, None - - agent_attribute = [a for a in attributes if - a.get('key') == 'agent'] if attributes else [] - if len(agent_attribute) > 0 and agent_attribute[0].get('value'): - agent_name, agent_version = agent_attribute[0]['value'].split( - '|') - stat_coro = async_send_event('start_launch', agent_name, agent_version) + stat_coro = async_send_event('start_launch', *agent_name_version(attributes)) - if not stat_coro: - response = await launch_coro - else: + if stat_coro: response = (await asyncio.gather(launch_coro, stat_coro))[0] + else: + response = await launch_coro if not response: return diff --git a/reportportal_client/client.py b/reportportal_client/client.py index a7988db6..25903010 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -23,20 +23,21 @@ import requests from requests.adapters import HTTPAdapter, Retry, DEFAULT_RETRIES -from ._local import set_current -from .core.rp_issues import Issue -from .core.rp_requests import ( +# noinspection PyProtectedMember +from reportportal_client._local import set_current +from reportportal_client.core.rp_issues import Issue +from reportportal_client.core.rp_requests import ( HttpRequest, ItemStartRequest, ItemFinishRequest, LaunchStartRequest, LaunchFinishRequest ) -from .helpers import uri_join, verify_value_length -from .logs.log_manager import LogManager, MAX_LOG_BATCH_PAYLOAD_SIZE -from .services.statistics import send_event -from .static.defines import NOT_FOUND -from .steps import StepReporter +from reportportal_client.helpers import uri_join, verify_value_length, agent_name_version +from reportportal_client.logs.log_manager import LogManager, MAX_LOG_BATCH_PAYLOAD_SIZE +from reportportal_client.services.statistics import send_event +from reportportal_client.static.defines import NOT_FOUND +from reportportal_client.steps import StepReporter logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -400,14 +401,7 @@ def start_launch(self, return if not self._skip_analytics: - agent_name, agent_version = None, None - - agent_attribute = [a for a in attributes if - a.get('key') == 'agent'] if attributes else [] - if len(agent_attribute) > 0 and agent_attribute[0].get('value'): - agent_name, agent_version = agent_attribute[0]['value'].split( - '|') - send_event('start_launch', agent_name, agent_version) + send_event('start_launch', *agent_name_version(attributes)) self._log_manager.launch_id = self.launch_id = response.id logger.debug('start_launch - ID: %s', self.launch_id) diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index f432718b..2d65b412 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -18,7 +18,7 @@ import time import uuid from platform import machine, processor, system -from typing import Optional, Any, List, Dict, Callable +from typing import Optional, Any, List, Dict, Callable, Tuple, Union from pkg_resources import DistributionNotFound, get_distribution @@ -224,6 +224,14 @@ def calculate_file_part_size(file: RPFile) -> int: return size +def agent_name_version(attributes: Optional[Union[List, Dict]] = None) -> Tuple[Optional[str], Optional[str]]: + agent_name, agent_version = None, None + agent_attribute = [a for a in attributes if a.get('key') == 'agent'] if attributes else [] + if len(agent_attribute) > 0 and agent_attribute[0].get('value'): + agent_name, agent_version = agent_attribute[0]['value'].split('|') + return agent_name, agent_version + + async def await_if_necessary(obj: Optional[Any]) -> Any: if obj: if asyncio.isfuture(obj) or asyncio.iscoroutine(obj): From 089702c4cfe5d0e35f89eb37d7fe9dd963ace9b7 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 5 Sep 2023 23:09:50 +0300 Subject: [PATCH 026/268] Async RPClient: WIP --- reportportal_client/aio/client.py | 127 ++++++++++++++++-------------- 1 file changed, 67 insertions(+), 60 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index d7c1b166..ab2a8599 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -175,12 +175,12 @@ async def __get_item_url(self, item_id_future: Union[str, asyncio.Task]) -> Opti return return uri_join(self.base_url_v2, 'item', item_id) - async def __get_launch_url(self, launch_id_future: Union[str, asyncio.Task]) -> Optional[str]: - launch_id = await await_if_necessary(launch_id_future) - if launch_id is NOT_FOUND: + async def __get_launch_url(self, launch_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: + launch_uuid = await await_if_necessary(launch_uuid_future) + if launch_uuid is NOT_FOUND: logger.warning('Attempt to make request for non-existent launch.') return - return uri_join(self.base_url_v2, 'launch', launch_id, 'finish') + return uri_join(self.base_url_v2, 'launch', launch_uuid, 'finish') async def start_launch(self, name: str, @@ -228,17 +228,17 @@ async def start_launch(self, if not response: return - launch_id = response.id - logger.debug(f'start_launch - ID: %s', launch_id) + launch_uuid = response.id + logger.debug(f'start_launch - ID: %s', launch_uuid) if self.launch_uuid_print and self.print_output: - print(f'Report Portal Launch UUID: {launch_id}', file=self.print_output) - return launch_id + print(f'Report Portal Launch UUID: {launch_uuid}', file=self.print_output) + return launch_uuid async def start_test_item(self, + launch_uuid: Union[str, asyncio.Task], name: str, start_time: str, item_type: str, - launch_id: Union[str, asyncio.Task], *, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, @@ -257,7 +257,7 @@ async def start_test_item(self, name, start_time, item_type, - launch_id, + launch_uuid, attributes=verify_value_length(attributes), code_ref=code_ref, description=description, @@ -277,9 +277,9 @@ async def start_test_item(self, return item_id async def finish_test_item(self, + launch_uuid: Union[str, asyncio.Task], item_id: Union[str, asyncio.Task], end_time: str, - launch_id: Union[str, asyncio.Task], *, status: str = None, issue: Optional[Issue] = None, @@ -290,7 +290,7 @@ async def finish_test_item(self, url = self.__get_item_url(item_id) request_payload = AsyncItemFinishRequest( end_time, - launch_id, + launch_uuid, status, attributes=attributes, description=description, @@ -306,13 +306,13 @@ async def finish_test_item(self, return response.message async def finish_launch(self, - launch_id: Union[str, asyncio.Task], + launch_uuid: Union[str, asyncio.Task], end_time: str, *, status: str = None, attributes: Optional[Union[List, Dict]] = None, **kwargs: Any) -> Optional[str]: - url = self.__get_launch_url(launch_id) + url = self.__get_launch_url(launch_uuid) request_payload = LaunchFinishRequest( end_time, status=status, @@ -323,7 +323,7 @@ async def finish_launch(self, name='Finish Launch').make() if not response: return - logger.debug('finish_launch - ID: %s', launch_id) + logger.debug('finish_launch - ID: %s', launch_uuid) logger.debug('response message: %s', response.message) return response.message @@ -342,12 +342,20 @@ async def get_launch_ui_url(self) -> Optional[str]: async def get_project_settings(self) -> Optional[Dict]: pass - async def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, + async def log(self, + launch_uuid: Union[str, asyncio.Task], + time: str, + message: str, + *, + level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, - item_id: Optional[Union[asyncio.Task, str]] = None) -> None: + item_id: Optional[Union[str, asyncio.Task]] = None) -> None: pass - async def update_test_item(self, item_uuid: Union[asyncio.Task, str], + async def update_test_item(self, + launch_uuid: Union[str, asyncio.Task], + item_uuid: Union[asyncio.Task, str], + *, attributes: Optional[Union[List, Dict]] = None, description: Optional[str] = None) -> Optional[str]: pass @@ -371,14 +379,14 @@ def clone(self) -> '_AsyncRPClient': class AsyncRPClient(_AsyncRPClient): - launch_id: Optional[str] + launch_uuid: Optional[str] use_own_launch: bool - def __init__(self, endpoint: str, project: str, *, launch_id: Optional[str] = None, + def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[str] = None, **kwargs: Any) -> None: super().__init__(endpoint, project, **kwargs) - if launch_id: - self.launch_id = launch_id + if launch_uuid: + self.launch_uuid = launch_uuid self.use_own_launch = False else: self.use_own_launch = True @@ -392,12 +400,12 @@ async def start_launch(self, rerun_of: Optional[str] = None, **kwargs) -> Optional[str]: if not self.use_own_launch: - return self.launch_id - launch_id = await super().start_launch(name, start_time, description=description, - attributes=attributes, rerun=rerun, rerun_of=rerun_of, - **kwargs) - self.launch_id = launch_id - return launch_id + return self.launch_uuid + launch_uuid = await super().start_launch(name, start_time, description=description, + attributes=attributes, rerun=rerun, rerun_of=rerun_of, + **kwargs) + self.launch_uuid = launch_uuid + return launch_uuid async def start_test_item(self, name: str, @@ -413,11 +421,11 @@ async def start_test_item(self, retry: bool = False, test_case_id: Optional[str] = None, **kwargs: Any) -> Optional[str]: - item_id = await super().start_test_item(name, start_time, item_type, description=description, - attributes=attributes, parameters=parameters, - parent_item_id=parent_item_id, has_stats=has_stats, - code_ref=code_ref, retry=retry, test_case_id=test_case_id, - **kwargs) + item_id = await super().start_test_item(self.launch_uuid, name, start_time, item_type, + description=description, attributes=attributes, + parameters=parameters, parent_item_id=parent_item_id, + has_stats=has_stats, code_ref=code_ref, retry=retry, + test_case_id=test_case_id, **kwargs) if item_id and item_id is not NOT_FOUND: logger.debug('start_test_item - ID: %s', item_id) super()._add_current_item(item_id) @@ -433,9 +441,9 @@ async def finish_test_item(self, description: str = None, retry: bool = False, **kwargs: Any) -> Optional[str]: - result = await super().finish_test_item(item_id, end_time, self.launch_id, status=status, issue=issue, - attributes=attributes, description=description, retry=retry, - **kwargs) + result = await super().finish_test_item(self.launch_uuid, item_id, end_time, status=status, + issue=issue, attributes=attributes, description=description, + retry=retry, **kwargs) super()._remove_current_item() return result @@ -446,7 +454,7 @@ async def finish_launch(self, **kwargs: Any) -> Optional[str]: if not self.use_own_launch: return "" - return await super().finish_launch(self.launch_id, end_time, status=status, attributes=attributes, + return await super().finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, **kwargs) def clone(self) -> 'AsyncRPClient': @@ -464,7 +472,7 @@ def clone(self) -> 'AsyncRPClient': verify_ssl=self.verify_ssl, retries=self.retries, max_pool_size=self.max_pool_size, - launch_id=self.launch_id, + launch_uuid=self.launch_uuid, http_timeout=self.http_timeout, log_batch_payload_size=self.log_batch_payload_size, mode=self.mode @@ -480,14 +488,14 @@ class SyncRPClient(_AsyncRPClient): thread: threading.Thread self_loop: bool self_thread: bool - launch_id: Optional[asyncio.Task] + launch_uuid: Optional[asyncio.Task] use_own_launch: bool - def __init__(self, endpoint: str, project: str, *, launch_id: Optional[asyncio.Task] = None, + def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio.Task] = None, **kwargs: Any) -> None: super().__init__(endpoint, project, **kwargs) - if launch_id: - self.launch_id = launch_id + if launch_uuid: + self.launch_uuid = launch_uuid self.use_own_launch = False else: self.use_own_launch = True @@ -519,13 +527,13 @@ def start_launch(self, rerun_of: Optional[str] = None, **kwargs) -> asyncio.Task: if not self.use_own_launch: - return self.launch_id - launch_id_coro = super().start_launch(name, start_time, description=description, - attributes=attributes, rerun=rerun, rerun_of=rerun_of, - **kwargs) - launch_id_task = self.loop.create_task(launch_id_coro) - self.launch_id = launch_id_task - return launch_id_task + return self.launch_uuid + launch_uuid_coro = super().start_launch(name, start_time, description=description, + attributes=attributes, rerun=rerun, rerun_of=rerun_of, + **kwargs) + launch_uuid_task = self.loop.create_task(launch_uuid_coro) + self.launch_uuid = launch_uuid_task + return launch_uuid_task def start_test_item(self, name: str, @@ -542,12 +550,11 @@ def start_test_item(self, test_case_id: Optional[str] = None, **kwargs: Any) -> asyncio.Task: - item_id_coro = super().start_test_item(name, start_time, item_type, launch_id=self.launch_id, - description=description, - attributes=attributes, parameters=parameters, - parent_item_id=parent_item_id, has_stats=has_stats, - code_ref=code_ref, retry=retry, test_case_id=test_case_id, - **kwargs) + item_id_coro = super().start_test_item(self.launch_uuid, name, start_time, item_type, + description=description, attributes=attributes, + parameters=parameters, parent_item_id=parent_item_id, + has_stats=has_stats, code_ref=code_ref, retry=retry, + test_case_id=test_case_id, **kwargs) item_id_task = self.loop.create_task(item_id_coro) super()._add_current_item(item_id_task) return item_id_task @@ -562,9 +569,9 @@ def finish_test_item(self, description: str = None, retry: bool = False, **kwargs: Any) -> asyncio.Task: - result_coro = super().finish_test_item(item_id, end_time, self.launch_id, status=status, issue=issue, - attributes=attributes, description=description, retry=retry, - **kwargs) + result_coro = super().finish_test_item(self.launch_uuid, item_id, end_time, status=status, + issue=issue, attributes=attributes, description=description, + retry=retry, **kwargs) result_task = self.loop.create_task(result_coro) super()._remove_current_item() return result_task @@ -577,7 +584,7 @@ def finish_launch(self, **kwargs: Any) -> asyncio.Task: if not self.use_own_launch: return self.loop.create_task(self.__empty_line()) - result_coro = super().finish_launch(self.launch_id, end_time, status=status, attributes=attributes, + result_coro = super().finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, **kwargs) result_task = self.loop.create_task(result_coro) return result_task @@ -597,7 +604,7 @@ def clone(self) -> 'SyncRPClient': verify_ssl=self.verify_ssl, retries=self.retries, max_pool_size=self.max_pool_size, - launch_id=self.launch_id, + launch_uuid=self.launch_uuid, http_timeout=self.http_timeout, log_batch_payload_size=self.log_batch_payload_size, mode=self.mode, From 9c05fdd2f6d3660896c95a12b067746ed339dc45 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 6 Sep 2023 12:33:54 +0300 Subject: [PATCH 027/268] Async RPClient: WIP --- reportportal_client/aio/client.py | 254 +++++++++++++++--------- reportportal_client/core/rp_requests.py | 7 +- 2 files changed, 166 insertions(+), 95 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index ab2a8599..f1a26015 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -33,10 +33,6 @@ from reportportal_client.helpers import uri_join, verify_value_length, await_if_necessary, agent_name_version from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.services.statistics import async_send_event -from reportportal_client.static.abstract import ( - AbstractBaseClass, - abstractmethod -) from reportportal_client.static.defines import NOT_FOUND from reportportal_client.steps import StepReporter @@ -51,9 +47,7 @@ def last(self): return self.queue[-1] -class _AsyncRPClient(metaclass=AbstractBaseClass): - __metaclass__ = AbstractBaseClass - +class _AsyncRPClient: api_v1: str api_v2: str base_url_v1: str @@ -73,7 +67,6 @@ class _AsyncRPClient(metaclass=AbstractBaseClass): launch_uuid_print: Optional[bool] print_output: Optional[TextIO] _skip_analytics: str - _item_stack: _LifoQueue __session: aiohttp.ClientSession def __init__( @@ -95,7 +88,6 @@ def __init__( print_output: Optional[TextIO] = None, **kwargs: Any ) -> None: - self._item_stack = _LifoQueue() set_current(self) self.api_v1, self.api_v2 = 'v1', 'v2' self.endpoint = endpoint @@ -327,11 +319,48 @@ async def finish_launch(self, logger.debug('response message: %s', response.message) return response.message - async def get_item_id_by_uuid(self, uuid: Union[asyncio.Task, str]) -> Optional[str]: - pass + async def __get_item_uuid_url(self, item_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: + item_uuid = await await_if_necessary(item_uuid_future) + if item_uuid is NOT_FOUND: + logger.warning('Attempt to make request for non-existent UUID.') + return + return uri_join(self.base_url_v1, 'item', 'uuid', item_uuid) - async def get_launch_info(self) -> Optional[Dict]: - pass + async def get_item_id_by_uuid(self, item_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: + """Get test Item ID by the given Item UUID. + + :param item_uuid_future: Str or asyncio.Task UUID returned on the Item start + :return: Test item ID + """ + url = self.__get_item_uuid_url(item_uuid_future) + response = await AsyncHttpRequest(self.session.get, url=url).make() + return response.id if response else None + + async def __get_launch_uuid_url(self, launch_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: + launch_uuid = await await_if_necessary(launch_uuid_future) + if launch_uuid is NOT_FOUND: + logger.warning('Attempt to make request for non-existent Launch UUID.') + return + logger.debug('get_launch_info - ID: %s', launch_uuid) + return uri_join(self.base_url_v1, 'launch', 'uuid', launch_uuid) + + async def get_launch_info(self, launch_uuid_future: Union[str, asyncio.Task]) -> Optional[Dict]: + """Get the launch information by Launch UUID. + + :param launch_uuid_future: Str or asyncio.Task UUID returned on the Launch start + :return dict: Launch information in dictionary + """ + url = self.__get_launch_uuid_url(launch_uuid_future) + response = await AsyncHttpRequest(self.session.get, url=url).make() + if not response: + return + if response.is_success: + launch_info = response.json + logger.debug('get_launch_info - Launch info: %s', response.json) + else: + logger.warning('get_launch_info - Launch info: Failed to fetch launch ID from the API.') + launch_info = {} + return launch_info async def get_launch_ui_id(self) -> Optional[Dict]: pass @@ -360,31 +389,41 @@ async def update_test_item(self, description: Optional[str] = None) -> Optional[str]: pass - def _add_current_item(self, item: Union[asyncio.Task, str]) -> None: - """Add the last item from the self._items queue.""" - self._item_stack.put(item) - - def _remove_current_item(self) -> None: - """Remove the last item from the self._items queue.""" - return self._item_stack.get() - - def current_item(self) -> Union[asyncio.Task, str]: - """Retrieve the last item reported by the client.""" - return self._item_stack.last() - - @abstractmethod def clone(self) -> '_AsyncRPClient': - """Abstract interface for cloning the client.""" - raise NotImplementedError('Clone interface is not implemented!') + """Clone the client object, set current Item ID as cloned item ID. + + :returns: Cloned client object + :rtype: AsyncRPClient + """ + cloned = _AsyncRPClient( + endpoint=self.endpoint, + project=self.project, + api_key=self.api_key, + log_batch_size=self.log_batch_size, + is_skipped_an_issue=self.is_skipped_an_issue, + verify_ssl=self.verify_ssl, + retries=self.retries, + max_pool_size=self.max_pool_size, + http_timeout=self.http_timeout, + log_batch_payload_size=self.log_batch_payload_size, + mode=self.mode + ) + return cloned -class AsyncRPClient(_AsyncRPClient): +class AsyncRPClient: + __client: _AsyncRPClient + _item_stack: _LifoQueue launch_uuid: Optional[str] use_own_launch: bool def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[str] = None, - **kwargs: Any) -> None: - super().__init__(endpoint, project, **kwargs) + client: Optional[_AsyncRPClient] = None, **kwargs: Any) -> None: + self._item_stack = _LifoQueue() + if client: + self.__client = client + else: + self.__client = _AsyncRPClient(endpoint, project, **kwargs) if launch_uuid: self.launch_uuid = launch_uuid self.use_own_launch = False @@ -401,9 +440,9 @@ async def start_launch(self, **kwargs) -> Optional[str]: if not self.use_own_launch: return self.launch_uuid - launch_uuid = await super().start_launch(name, start_time, description=description, - attributes=attributes, rerun=rerun, rerun_of=rerun_of, - **kwargs) + launch_uuid = await self.__client.start_launch(name, start_time, description=description, + attributes=attributes, rerun=rerun, rerun_of=rerun_of, + **kwargs) self.launch_uuid = launch_uuid return launch_uuid @@ -421,14 +460,14 @@ async def start_test_item(self, retry: bool = False, test_case_id: Optional[str] = None, **kwargs: Any) -> Optional[str]: - item_id = await super().start_test_item(self.launch_uuid, name, start_time, item_type, - description=description, attributes=attributes, - parameters=parameters, parent_item_id=parent_item_id, - has_stats=has_stats, code_ref=code_ref, retry=retry, - test_case_id=test_case_id, **kwargs) + item_id = await self.__client.start_test_item(self.launch_uuid, name, start_time, item_type, + description=description, attributes=attributes, + parameters=parameters, parent_item_id=parent_item_id, + has_stats=has_stats, code_ref=code_ref, retry=retry, + test_case_id=test_case_id, **kwargs) if item_id and item_id is not NOT_FOUND: logger.debug('start_test_item - ID: %s', item_id) - super()._add_current_item(item_id) + self._add_current_item(item_id) return item_id async def finish_test_item(self, @@ -441,10 +480,11 @@ async def finish_test_item(self, description: str = None, retry: bool = False, **kwargs: Any) -> Optional[str]: - result = await super().finish_test_item(self.launch_uuid, item_id, end_time, status=status, - issue=issue, attributes=attributes, description=description, - retry=retry, **kwargs) - super()._remove_current_item() + result = await self.__client.finish_test_item(self.launch_uuid, item_id, end_time, status=status, + issue=issue, attributes=attributes, + description=description, + retry=retry, **kwargs) + self._remove_current_item() return result async def finish_launch(self, @@ -454,8 +494,26 @@ async def finish_launch(self, **kwargs: Any) -> Optional[str]: if not self.use_own_launch: return "" - return await super().finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, - **kwargs) + return await self.__client.finish_launch(self.launch_uuid, end_time, status=status, + attributes=attributes, + **kwargs) + + async def get_launch_info(self) -> Optional[Dict]: + if not self.launch_uuid: + return {} + return await self.__client.get_launch_info(self.launch_uuid) + + def _add_current_item(self, item: str) -> None: + """Add the last item from the self._items queue.""" + self._item_stack.put(item) + + def _remove_current_item(self) -> str: + """Remove the last item from the self._items queue.""" + return self._item_stack.get() + + def current_item(self) -> str: + """Retrieve the last item reported by the client.""" + return self._item_stack.last() def clone(self) -> 'AsyncRPClient': """Clone the client object, set current Item ID as cloned item ID. @@ -463,19 +521,13 @@ def clone(self) -> 'AsyncRPClient': :returns: Cloned client object :rtype: AsyncRPClient """ + cloned_client = self.__client.clone() + # noinspection PyTypeChecker cloned = AsyncRPClient( - endpoint=self.endpoint, - project=self.project, - api_key=self.api_key, - log_batch_size=self.log_batch_size, - is_skipped_an_issue=self.is_skipped_an_issue, - verify_ssl=self.verify_ssl, - retries=self.retries, - max_pool_size=self.max_pool_size, - launch_uuid=self.launch_uuid, - http_timeout=self.http_timeout, - log_batch_payload_size=self.log_batch_payload_size, - mode=self.mode + endpoint=None, + project=None, + client=cloned_client, + launch_uuid=self.launch_uuid ) current_item = self.current_item() if current_item: @@ -483,7 +535,9 @@ def clone(self) -> 'AsyncRPClient': return cloned -class SyncRPClient(_AsyncRPClient): +class SyncRPClient: + __client: _AsyncRPClient + _item_stack: _LifoQueue loop: asyncio.AbstractEventLoop thread: threading.Thread self_loop: bool @@ -492,22 +546,27 @@ class SyncRPClient(_AsyncRPClient): use_own_launch: bool def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio.Task] = None, - **kwargs: Any) -> None: - super().__init__(endpoint, project, **kwargs) + client: Optional[_AsyncRPClient] = None, loop: Optional[asyncio.AbstractEventLoop] = None, + thread: Optional[threading.Thread] = None, **kwargs: Any) -> None: + self._item_stack = _LifoQueue() + if client: + self.__client = client + else: + self.__client = _AsyncRPClient(endpoint, project, **kwargs) if launch_uuid: self.launch_uuid = launch_uuid self.use_own_launch = False else: self.use_own_launch = True - if 'loop' in kwargs and kwargs['loop']: - self.loop = kwargs['loop'] + if loop: + self.loop = loop self.self_loop = False else: self.loop = asyncio.new_event_loop() self.self_loop = True - if 'thread' in kwargs and kwargs['thread']: - self.thread = kwargs['thread'] + if thread: + self.thread = thread self.self_thread = False else: self.thread = threading.Thread(target=self.loop.run_forever(), name='RP-Async-Client', @@ -528,9 +587,9 @@ def start_launch(self, **kwargs) -> asyncio.Task: if not self.use_own_launch: return self.launch_uuid - launch_uuid_coro = super().start_launch(name, start_time, description=description, - attributes=attributes, rerun=rerun, rerun_of=rerun_of, - **kwargs) + launch_uuid_coro = self.__client.start_launch(name, start_time, description=description, + attributes=attributes, rerun=rerun, rerun_of=rerun_of, + **kwargs) launch_uuid_task = self.loop.create_task(launch_uuid_coro) self.launch_uuid = launch_uuid_task return launch_uuid_task @@ -550,13 +609,13 @@ def start_test_item(self, test_case_id: Optional[str] = None, **kwargs: Any) -> asyncio.Task: - item_id_coro = super().start_test_item(self.launch_uuid, name, start_time, item_type, - description=description, attributes=attributes, - parameters=parameters, parent_item_id=parent_item_id, - has_stats=has_stats, code_ref=code_ref, retry=retry, - test_case_id=test_case_id, **kwargs) + item_id_coro = self.__client.start_test_item(self.launch_uuid, name, start_time, item_type, + description=description, attributes=attributes, + parameters=parameters, parent_item_id=parent_item_id, + has_stats=has_stats, code_ref=code_ref, retry=retry, + test_case_id=test_case_id, **kwargs) item_id_task = self.loop.create_task(item_id_coro) - super()._add_current_item(item_id_task) + self._add_current_item(item_id_task) return item_id_task def finish_test_item(self, @@ -569,11 +628,12 @@ def finish_test_item(self, description: str = None, retry: bool = False, **kwargs: Any) -> asyncio.Task: - result_coro = super().finish_test_item(self.launch_uuid, item_id, end_time, status=status, - issue=issue, attributes=attributes, description=description, - retry=retry, **kwargs) + result_coro = self.__client.finish_test_item(self.launch_uuid, item_id, end_time, status=status, + issue=issue, attributes=attributes, + description=description, + retry=retry, **kwargs) result_task = self.loop.create_task(result_coro) - super()._remove_current_item() + self._remove_current_item() return result_task # TODO: implement loop task finish wait @@ -584,30 +644,42 @@ def finish_launch(self, **kwargs: Any) -> asyncio.Task: if not self.use_own_launch: return self.loop.create_task(self.__empty_line()) - result_coro = super().finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, - **kwargs) + result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, + attributes=attributes, + **kwargs) result_task = self.loop.create_task(result_coro) return result_task + def get_item_id_by_uuid(self, item_uuid_future: asyncio.Task) -> asyncio.Task: + result_coro = self.__client.get_item_id_by_uuid(item_uuid_future) + result_task = self.loop.create_task(result_coro) + return result_task + + def _add_current_item(self, item: asyncio.Task) -> None: + """Add the last item from the self._items queue.""" + self._item_stack.put(item) + + def _remove_current_item(self) -> asyncio.Task: + """Remove the last item from the self._items queue.""" + return self._item_stack.get() + + def current_item(self) -> asyncio.Task: + """Retrieve the last item reported by the client.""" + return self._item_stack.last() + def clone(self) -> 'SyncRPClient': """Clone the client object, set current Item ID as cloned item ID. :returns: Cloned client object :rtype: SyncRPClient """ + cloned_client = self.__client.clone() + # noinspection PyTypeChecker cloned = SyncRPClient( - endpoint=self.endpoint, - project=self.project, - api_key=self.api_key, - log_batch_size=self.log_batch_size, - is_skipped_an_issue=self.is_skipped_an_issue, - verify_ssl=self.verify_ssl, - retries=self.retries, - max_pool_size=self.max_pool_size, + endpoint=None, + project=None, launch_uuid=self.launch_uuid, - http_timeout=self.http_timeout, - log_batch_payload_size=self.log_batch_payload_size, - mode=self.mode, + client=cloned_client, loop=self.loop, thread=self.thread ) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 47570b2a..66ac8b8d 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -105,7 +105,7 @@ def priority(self, value: Priority) -> None: """Set the priority of the request.""" self._priority = value - def make(self): + def make(self) -> Optional[RPResponse]: """Make HTTP request to the Report Portal API.""" try: return RPResponse(self.session_method(self.url, data=self.data, json=self.json, @@ -139,15 +139,14 @@ def __init__(self, """ super().__init__(session_method=session_method, url=url, data=data, json=json, name=name) - async def make(self): + async def make(self) -> Optional[RPResponse]: """Make HTTP request to the Report Portal API.""" url = await_if_necessary(self.url) if not url: return try: - return RPResponse(await self.session_method(await await_if_necessary(self.url), data=self.data, - json=self.json)) + return RPResponse(await self.session_method(url, data=self.data, json=self.json)) except (KeyError, IOError, ValueError, TypeError) as exc: logger.warning( "Report Portal %s request failed", From c5ebaffb809e8998a675a687de6c5bb819d928bd Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 6 Sep 2023 15:31:40 +0300 Subject: [PATCH 028/268] Async RPClient: WIP --- reportportal_client/aio/client.py | 77 +++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index f1a26015..6e855d22 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -33,6 +33,10 @@ from reportportal_client.helpers import uri_join, verify_value_length, await_if_necessary, agent_name_version from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.services.statistics import async_send_event +from reportportal_client.static.abstract import ( + AbstractBaseClass, + abstractmethod +) from reportportal_client.static.defines import NOT_FOUND from reportportal_client.steps import StepReporter @@ -411,7 +415,64 @@ def clone(self) -> '_AsyncRPClient': return cloned -class AsyncRPClient: +class RPClient(metaclass=AbstractBaseClass): + __metaclass__ = AbstractBaseClass + + @abstractmethod + def start_launch(self, + name: str, + start_time: str, + description: Optional[str] = None, + attributes: Optional[Union[List, Dict]] = None, + rerun: bool = False, + rerun_of: Optional[str] = None, + **kwargs) -> Union[Optional[str], asyncio.Task]: + raise NotImplementedError('"start_launch" method is not implemented!') + + @abstractmethod + def start_test_item(self, + name: str, + start_time: str, + item_type: str, + *, + description: Optional[str] = None, + attributes: Optional[List[Dict]] = None, + parameters: Optional[Dict] = None, + parent_item_id: Union[Optional[str], asyncio.Task] = None, + has_stats: bool = True, + code_ref: Optional[str] = None, + retry: bool = False, + test_case_id: Optional[str] = None, + **kwargs: Any) -> Union[Optional[str], asyncio.Task]: + raise NotImplementedError('"start_test_item" method is not implemented!') + + @abstractmethod + def finish_test_item(self, + item_id: Union[str, asyncio.Task], + end_time: str, + *, + status: str = None, + issue: Optional[Issue] = None, + attributes: Optional[Union[List, Dict]] = None, + description: str = None, + retry: bool = False, + **kwargs: Any) -> Union[Optional[str], asyncio.Task]: + raise NotImplementedError('"finish_test_item" method is not implemented!') + + @abstractmethod + def finish_launch(self, + end_time: str, + status: str = None, + attributes: Optional[Union[List, Dict]] = None, + **kwargs: Any) -> Union[Optional[str], asyncio.Task]: + raise NotImplementedError('"finish_launch" method is not implemented!') + + @abstractmethod + def get_launch_info(self) -> Union[Optional[dict], asyncio.Task]: + raise NotImplementedError('"get_launch_info" method is not implemented!') + + +class AsyncRPClient(RPClient): __client: _AsyncRPClient _item_stack: _LifoQueue launch_uuid: Optional[str] @@ -498,7 +559,7 @@ async def finish_launch(self, attributes=attributes, **kwargs) - async def get_launch_info(self) -> Optional[Dict]: + async def get_launch_info(self) -> Optional[dict]: if not self.launch_uuid: return {} return await self.__client.get_launch_info(self.launch_uuid) @@ -535,7 +596,7 @@ def clone(self) -> 'AsyncRPClient': return cloned -class SyncRPClient: +class SyncRPClient(RPClient): __client: _AsyncRPClient _item_stack: _LifoQueue loop: asyncio.AbstractEventLoop @@ -650,6 +711,16 @@ def finish_launch(self, result_task = self.loop.create_task(result_coro) return result_task + async def __empty_dict(self): + return {} + + def get_launch_info(self) -> asyncio.Task: + if not self.launch_uuid: + return self.loop.create_task(self.__empty_dict()) + result_coro = self.__client.get_launch_info(self.launch_uuid) + result_task = self.loop.create_task(result_coro) + return result_task + def get_item_id_by_uuid(self, item_uuid_future: asyncio.Task) -> asyncio.Task: result_coro = self.__client.get_item_id_by_uuid(item_uuid_future) result_task = self.loop.create_task(result_coro) From 2da024a9516ab61617b262c0dde3467095375ce4 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 7 Sep 2023 12:06:51 +0300 Subject: [PATCH 029/268] Async RPClient: WIP --- reportportal_client/_local/__init__.pyi | 5 +- reportportal_client/aio/client.py | 182 +++++++++++++++++++----- reportportal_client/client.py | 2 +- reportportal_client/steps/__init__.pyi | 5 +- 4 files changed, 154 insertions(+), 40 deletions(-) diff --git a/reportportal_client/_local/__init__.pyi b/reportportal_client/_local/__init__.pyi index 7ba4b639..a2bdd9fd 100644 --- a/reportportal_client/_local/__init__.pyi +++ b/reportportal_client/_local/__init__.pyi @@ -13,12 +13,11 @@ from typing import Optional, Union -# noinspection PyProtectedMember -from reportportal_client.aio.client import _AsyncRPClient +from reportportal_client.aio.client import RPClient as AsyncRPClient from reportportal_client.client import RPClient def current() -> Optional[RPClient]: ... -def set_current(client: Optional[Union[RPClient, _AsyncRPClient]]) -> None: ... +def set_current(client: Optional[Union[RPClient, AsyncRPClient]]) -> None: ... diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 6e855d22..ae7ad41f 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -66,7 +66,6 @@ class _AsyncRPClient: retries: int max_pool_size: int http_timeout: Union[float, Tuple[float, float]] - step_reporter: StepReporter mode: str launch_uuid_print: Optional[bool] print_output: Optional[TextIO] @@ -92,7 +91,6 @@ def __init__( print_output: Optional[TextIO] = None, **kwargs: Any ) -> None: - set_current(self) self.api_v1, self.api_v2 = 'v1', 'v2' self.endpoint = endpoint self.project = project @@ -107,7 +105,6 @@ def __init__( self.retries = retries self.max_pool_size = max_pool_size self.http_timeout = http_timeout - self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() self.mode = mode self._skip_analytics = getenv('AGENT_NO_ANALYTICS') @@ -323,6 +320,23 @@ async def finish_launch(self, logger.debug('response message: %s', response.message) return response.message + async def update_test_item(self, + item_uuid: Union[str, asyncio.Task], + *, + attributes: Optional[Union[List, Dict]] = None, + description: Optional[str] = None) -> Optional[str]: + data = { + 'description': description, + 'attributes': verify_value_length(attributes), + } + item_id = await self.get_item_id_by_uuid(item_uuid) + url = uri_join(self.base_url_v1, 'item', item_id, 'update') + response = await AsyncHttpRequest(self.session.put, url=url, json=data).make() + if not response: + return + logger.debug('update_test_item - Item: %s', item_id) + return response.message + async def __get_item_uuid_url(self, item_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: item_uuid = await await_if_necessary(item_uuid_future) if item_uuid is NOT_FOUND: @@ -366,14 +380,33 @@ async def get_launch_info(self, launch_uuid_future: Union[str, asyncio.Task]) -> launch_info = {} return launch_info - async def get_launch_ui_id(self) -> Optional[Dict]: - pass + async def get_launch_ui_id(self, launch_uuid_future: Union[str, asyncio.Task]) -> Optional[int]: + launch_info = await self.get_launch_info(launch_uuid_future) + return launch_info.get('id') if launch_info else None - async def get_launch_ui_url(self) -> Optional[str]: - pass + async def get_launch_ui_url(self, launch_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: + launch_uuid = await await_if_necessary(launch_uuid_future) + launch_info = await self.get_launch_info(launch_uuid) + ui_id = launch_info.get('id') if launch_info else None + if not ui_id: + return + mode = launch_info.get('mode') if launch_info else None + if not mode: + mode = self.mode + + launch_type = 'launches' if mode.upper() == 'DEFAULT' else 'userdebug' + + path = 'ui/#{project_name}/{launch_type}/all/{launch_id}'.format( + project_name=self.project.lower(), launch_type=launch_type, + launch_id=ui_id) + url = uri_join(self.endpoint, path) + logger.debug('get_launch_ui_url - ID: %s', launch_uuid) + return url async def get_project_settings(self) -> Optional[Dict]: - pass + url = uri_join(self.base_url_v1, 'settings') + response = await AsyncHttpRequest(self.session.get, url=url).make() + return response.json if response else None async def log(self, launch_uuid: Union[str, asyncio.Task], @@ -385,14 +418,6 @@ async def log(self, item_id: Optional[Union[str, asyncio.Task]] = None) -> None: pass - async def update_test_item(self, - launch_uuid: Union[str, asyncio.Task], - item_uuid: Union[asyncio.Task, str], - *, - attributes: Optional[Union[List, Dict]] = None, - description: Optional[str] = None) -> Optional[str]: - pass - def clone(self) -> '_AsyncRPClient': """Clone the client object, set current Item ID as cloned item ID. @@ -467,19 +492,48 @@ def finish_launch(self, **kwargs: Any) -> Union[Optional[str], asyncio.Task]: raise NotImplementedError('"finish_launch" method is not implemented!') + @abstractmethod + def update_test_item(self, item_uuid: str, attributes: Optional[Union[List, Dict]] = None, + description: Optional[str] = None) -> Optional[str]: + raise NotImplementedError('"update_test_item" method is not implemented!') + @abstractmethod def get_launch_info(self) -> Union[Optional[dict], asyncio.Task]: raise NotImplementedError('"get_launch_info" method is not implemented!') + @abstractmethod + def get_item_id_by_uuid(self, item_uuid: str) -> Optional[str]: + raise NotImplementedError('"get_item_id_by_uuid" method is not implemented!') + + @abstractmethod + def get_launch_ui_id(self) -> Optional[int]: + raise NotImplementedError('"get_launch_ui_id" method is not implemented!') + + @abstractmethod + def get_launch_ui_url(self) -> Optional[str]: + raise NotImplementedError('"get_launch_ui_id" method is not implemented!') + + @abstractmethod + def get_project_settings(self) -> Optional[Dict]: + raise NotImplementedError('"get_project_settings" method is not implemented!') + + @abstractmethod + def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, + attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: + raise NotImplementedError('"log" method is not implemented!') + class AsyncRPClient(RPClient): __client: _AsyncRPClient _item_stack: _LifoQueue launch_uuid: Optional[str] use_own_launch: bool + step_reporter: StepReporter def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[str] = None, client: Optional[_AsyncRPClient] = None, **kwargs: Any) -> None: + set_current(self) + self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() if client: self.__client = client @@ -559,10 +613,9 @@ async def finish_launch(self, attributes=attributes, **kwargs) - async def get_launch_info(self) -> Optional[dict]: - if not self.launch_uuid: - return {} - return await self.__client.get_launch_info(self.launch_uuid) + async def update_test_item(self, item_uuid: str, attributes: Optional[Union[List, Dict]] = None, + description: Optional[str] = None) -> Optional[str]: + return await self.__client.update_test_item(item_uuid, attributes=attributes, description=description) def _add_current_item(self, item: str) -> None: """Add the last item from the self._items queue.""" @@ -576,6 +629,31 @@ def current_item(self) -> str: """Retrieve the last item reported by the client.""" return self._item_stack.last() + async def get_launch_info(self) -> Optional[dict]: + if not self.launch_uuid: + return {} + return await self.__client.get_launch_info(self.launch_uuid) + + async def get_item_id_by_uuid(self, item_uuid: str) -> Optional[str]: + return await self.__client.get_item_id_by_uuid(item_uuid) + + async def get_launch_ui_id(self) -> Optional[int]: + if not self.launch_uuid: + return + return await self.__client.get_launch_ui_id(self.launch_uuid) + + async def get_launch_ui_url(self) -> Optional[str]: + if not self.launch_uuid: + return + return await self.__client.get_launch_ui_url(self.launch_uuid) + + async def get_project_settings(self) -> Optional[Dict]: + return await self.__client.get_project_settings() + + async def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, + attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: + return + def clone(self) -> 'AsyncRPClient': """Clone the client object, set current Item ID as cloned item ID. @@ -596,7 +674,7 @@ def clone(self) -> 'AsyncRPClient': return cloned -class SyncRPClient(RPClient): +class ScheduledRPClient(RPClient): __client: _AsyncRPClient _item_stack: _LifoQueue loop: asyncio.AbstractEventLoop @@ -605,10 +683,13 @@ class SyncRPClient(RPClient): self_thread: bool launch_uuid: Optional[asyncio.Task] use_own_launch: bool + step_reporter: StepReporter def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio.Task] = None, client: Optional[_AsyncRPClient] = None, loop: Optional[asyncio.AbstractEventLoop] = None, thread: Optional[threading.Thread] = None, **kwargs: Any) -> None: + set_current(self) + self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() if client: self.__client = client @@ -711,9 +792,33 @@ def finish_launch(self, result_task = self.loop.create_task(result_coro) return result_task + def update_test_item(self, + item_uuid: asyncio.Task, + attributes: Optional[Union[List, Dict]] = None, + description: Optional[str] = None) -> asyncio.Task: + result_coro = self.__client.update_test_item(item_uuid, attributes=attributes, + description=description) + result_task = self.loop.create_task(result_coro) + return result_task + + def _add_current_item(self, item: asyncio.Task) -> None: + """Add the last item from the self._items queue.""" + self._item_stack.put(item) + + def _remove_current_item(self) -> asyncio.Task: + """Remove the last item from the self._items queue.""" + return self._item_stack.get() + async def __empty_dict(self): return {} + async def __none_value(self): + return + + def current_item(self) -> asyncio.Task: + """Retrieve the last item reported by the client.""" + return self._item_stack.last() + def get_launch_info(self) -> asyncio.Task: if not self.launch_uuid: return self.loop.create_task(self.__empty_dict()) @@ -726,27 +831,38 @@ def get_item_id_by_uuid(self, item_uuid_future: asyncio.Task) -> asyncio.Task: result_task = self.loop.create_task(result_coro) return result_task - def _add_current_item(self, item: asyncio.Task) -> None: - """Add the last item from the self._items queue.""" - self._item_stack.put(item) + def get_launch_ui_id(self) -> asyncio.Task: + if not self.launch_uuid: + return self.loop.create_task(self.__none_value()) + result_coro = self.__client.get_launch_ui_id(self.launch_uuid) + result_task = self.loop.create_task(result_coro) + return result_task - def _remove_current_item(self) -> asyncio.Task: - """Remove the last item from the self._items queue.""" - return self._item_stack.get() + def get_launch_ui_url(self) -> asyncio.Task: + if not self.launch_uuid: + return self.loop.create_task(self.__none_value()) + result_coro = self.__client.get_launch_ui_url(self.launch_uuid) + result_task = self.loop.create_task(result_coro) + return result_task - def current_item(self) -> asyncio.Task: - """Retrieve the last item reported by the client.""" - return self._item_stack.last() + def get_project_settings(self) -> asyncio.Task: + result_coro = self.__client.get_project_settings() + result_task = self.loop.create_task(result_coro) + return result_task + + def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, + attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: + return None - def clone(self) -> 'SyncRPClient': + def clone(self) -> 'ScheduledRPClient': """Clone the client object, set current Item ID as cloned item ID. :returns: Cloned client object - :rtype: SyncRPClient + :rtype: ScheduledRPClient """ cloned_client = self.__client.clone() # noinspection PyTypeChecker - cloned = SyncRPClient( + cloned = ScheduledRPClient( endpoint=None, project=None, launch_uuid=self.launch_uuid, diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 25903010..47260ad8 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -309,7 +309,7 @@ def get_launch_info(self) -> Optional[Dict]: launch_info = {} return launch_info - def get_launch_ui_id(self) -> Optional[Dict]: + def get_launch_ui_id(self) -> Optional[int]: """Get UI ID of the current launch. :return: UI ID of the given launch. None if UI ID has not been found. diff --git a/reportportal_client/steps/__init__.pyi b/reportportal_client/steps/__init__.pyi index 4e62fb02..2a583751 100644 --- a/reportportal_client/steps/__init__.pyi +++ b/reportportal_client/steps/__init__.pyi @@ -13,15 +13,14 @@ from typing import Text, Optional, Dict, Any, Callable, Union -# noinspection PyProtectedMember -from reportportal_client.aio.client import _AsyncRPClient +from reportportal_client.aio.client import RPClient as AsyncRPClient from reportportal_client.client import RPClient class StepReporter: client: RPClient = ... - def __init__(self, rp_client: Union[RPClient, _AsyncRPClient]) -> None: ... + def __init__(self, rp_client: Union[RPClient, AsyncRPClient]) -> None: ... def start_nested_step(self, name: Text, From f117a2fbb3ebc3ecf34f56a2e29bcf109e03f081 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 7 Sep 2023 13:47:42 +0300 Subject: [PATCH 030/268] Fix typing --- reportportal_client/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index 2d65b412..3b60fca4 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -79,7 +79,7 @@ def gen_attributes(rp_attributes: List[str]) -> List[Dict[str, str]]: return attrs -def get_launch_sys_attrs() -> Dict[str]: +def get_launch_sys_attrs() -> Dict[str, str]: """Generate attributes for the launch containing system information. :return: dict {'os': 'Windows', From 3d41c9c13e373c10c5c3f431104da78608028e6f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 7 Sep 2023 15:32:19 +0300 Subject: [PATCH 031/268] Fix loop run method reference --- reportportal_client/aio/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index ae7ad41f..ea327cf4 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -711,7 +711,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio self.thread = thread self.self_thread = False else: - self.thread = threading.Thread(target=self.loop.run_forever(), name='RP-Async-Client', + self.thread = threading.Thread(target=self.loop.run_forever, name='RP-Async-Client', daemon=True) self.thread.start() self.self_thread = True From a611524a7346b60852c641847274654346ea9876 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 7 Sep 2023 16:45:22 +0300 Subject: [PATCH 032/268] Fix bugs --- reportportal_client/aio/client.py | 30 ++++++++++++------------- reportportal_client/core/rp_requests.py | 2 +- reportportal_client/helpers.py | 14 ++++++++++++ 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index ea327cf4..b8f8c6e5 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -30,7 +30,8 @@ from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, AsyncItemFinishRequest, LaunchFinishRequest) -from reportportal_client.helpers import uri_join, verify_value_length, await_if_necessary, agent_name_version +from reportportal_client.helpers import (root_uri_join, verify_value_length, await_if_necessary, + agent_name_version) from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.services.statistics import async_send_event from reportportal_client.static.abstract import ( @@ -70,7 +71,7 @@ class _AsyncRPClient: launch_uuid_print: Optional[bool] print_output: Optional[TextIO] _skip_analytics: str - __session: aiohttp.ClientSession + __session: Optional[aiohttp.ClientSession] def __init__( self, @@ -94,10 +95,8 @@ def __init__( self.api_v1, self.api_v2 = 'v1', 'v2' self.endpoint = endpoint self.project = project - self.base_url_v1 = uri_join( - self.endpoint, 'api/{}'.format(self.api_v1), self.project) - self.base_url_v2 = uri_join( - self.endpoint, 'api/{}'.format(self.api_v2), self.project) + self.base_url_v1 = root_uri_join(f'api/{self.api_v1}', self.project) + self.base_url_v2 = root_uri_join(f'api/{self.api_v2}', self.project) self.is_skipped_an_issue = is_skipped_an_issue self.log_batch_size = log_batch_size self.log_batch_payload_size = log_batch_payload_size @@ -110,6 +109,7 @@ def __init__( self._skip_analytics = getenv('AGENT_NO_ANALYTICS') self.launch_uuid_print = launch_uuid_print self.print_output = print_output or sys.stdout + self.__session = None self.api_key = api_key if not self.api_key: @@ -166,14 +166,14 @@ async def __get_item_url(self, item_id_future: Union[str, asyncio.Task]) -> Opti if item_id is NOT_FOUND: logger.warning('Attempt to make request for non-existent id.') return - return uri_join(self.base_url_v2, 'item', item_id) + return root_uri_join(self.base_url_v2, 'item', item_id) async def __get_launch_url(self, launch_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: launch_uuid = await await_if_necessary(launch_uuid_future) if launch_uuid is NOT_FOUND: logger.warning('Attempt to make request for non-existent launch.') return - return uri_join(self.base_url_v2, 'launch', launch_uuid, 'finish') + return root_uri_join(self.base_url_v2, 'launch', launch_uuid, 'finish') async def start_launch(self, name: str, @@ -194,7 +194,7 @@ async def start_launch(self, :param rerun_of: For rerun mode specifies which launch will be re-run. Should be used with the 'rerun' option. """ - url = uri_join(self.base_url_v2, 'launch') + url = root_uri_join(self.base_url_v2, 'launch') request_payload = LaunchStartRequest( name=name, start_time=start_time, @@ -245,7 +245,7 @@ async def start_test_item(self, if parent_item_id: url = self.__get_item_url(parent_item_id) else: - url = uri_join(self.base_url_v2, 'item') + url = root_uri_join(self.base_url_v2, 'item') request_payload = AsyncItemStartRequest( name, start_time, @@ -330,7 +330,7 @@ async def update_test_item(self, 'attributes': verify_value_length(attributes), } item_id = await self.get_item_id_by_uuid(item_uuid) - url = uri_join(self.base_url_v1, 'item', item_id, 'update') + url = root_uri_join(self.base_url_v1, 'item', item_id, 'update') response = await AsyncHttpRequest(self.session.put, url=url, json=data).make() if not response: return @@ -342,7 +342,7 @@ async def __get_item_uuid_url(self, item_uuid_future: Union[str, asyncio.Task]) if item_uuid is NOT_FOUND: logger.warning('Attempt to make request for non-existent UUID.') return - return uri_join(self.base_url_v1, 'item', 'uuid', item_uuid) + return root_uri_join(self.base_url_v1, 'item', 'uuid', item_uuid) async def get_item_id_by_uuid(self, item_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: """Get test Item ID by the given Item UUID. @@ -360,7 +360,7 @@ async def __get_launch_uuid_url(self, launch_uuid_future: Union[str, asyncio.Tas logger.warning('Attempt to make request for non-existent Launch UUID.') return logger.debug('get_launch_info - ID: %s', launch_uuid) - return uri_join(self.base_url_v1, 'launch', 'uuid', launch_uuid) + return root_uri_join(self.base_url_v1, 'launch', 'uuid', launch_uuid) async def get_launch_info(self, launch_uuid_future: Union[str, asyncio.Task]) -> Optional[Dict]: """Get the launch information by Launch UUID. @@ -399,12 +399,12 @@ async def get_launch_ui_url(self, launch_uuid_future: Union[str, asyncio.Task]) path = 'ui/#{project_name}/{launch_type}/all/{launch_id}'.format( project_name=self.project.lower(), launch_type=launch_type, launch_id=ui_id) - url = uri_join(self.endpoint, path) + url = root_uri_join(self.endpoint, path) logger.debug('get_launch_ui_url - ID: %s', launch_uuid) return url async def get_project_settings(self) -> Optional[Dict]: - url = uri_join(self.base_url_v1, 'settings') + url = root_uri_join(self.base_url_v1, 'settings') response = await AsyncHttpRequest(self.session.get, url=url).make() return response.json if response else None diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 66ac8b8d..6605c656 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -141,7 +141,7 @@ def __init__(self, async def make(self) -> Optional[RPResponse]: """Make HTTP request to the Report Portal API.""" - url = await_if_necessary(self.url) + url = await await_if_necessary(self.url) if not url: return diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index 3b60fca4..c1dcbefc 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -150,6 +150,20 @@ def uri_join(*uri_parts: str) -> str: return '/'.join(str(s).strip('/').strip('\\') for s in uri_parts) +def root_uri_join(*uri_parts: str) -> str: + """Join uri parts. Format it as path from server root. + + Avoiding usage of urlparse.urljoin and os.path.join + as it does not clearly join parts. + Args: + *uri_parts: tuple of values for join, can contain back and forward + slashes (will be stripped up). + Returns: + An uri string. + """ + return '/' + uri_join(*uri_parts) + + def get_function_params(func: Callable, args: tuple, kwargs: Dict[str, Any]) -> Dict[str, Any]: """Extract argument names from the function and combine them with values. From a528086036edf770418f9b97103890b78eedfcc2 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 7 Sep 2023 17:21:20 +0300 Subject: [PATCH 033/268] Async RPClient: WIP --- reportportal_client/aio/client.py | 25 ++++--- reportportal_client/core/rp_requests.py | 6 +- reportportal_client/core/rp_responses.py | 82 +++++++++++++---------- reportportal_client/core/rp_responses.pyi | 38 ----------- 4 files changed, 62 insertions(+), 89 deletions(-) delete mode 100644 reportportal_client/core/rp_responses.pyi diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index b8f8c6e5..16b2b516 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -221,7 +221,7 @@ async def start_launch(self, if not response: return - launch_uuid = response.id + launch_uuid = await response.id logger.debug(f'start_launch - ID: %s', launch_uuid) if self.launch_uuid_print and self.print_output: print(f'Report Portal Launch UUID: {launch_uuid}', file=self.print_output) @@ -263,10 +263,11 @@ async def start_test_item(self, response = await AsyncHttpRequest(self.session.post, url=url, json=request_payload).make() if not response: return - item_id = response.id + item_id = await response.id if item_id is NOT_FOUND: - logger.warning('start_test_item - invalid response: %s', - str(response.json)) + logger.warning('start_test_item - invalid response: %s', str(await response.json)) + else: + logger.debug('start_test_item - ID: %s', item_id) return item_id async def finish_test_item(self, @@ -294,9 +295,10 @@ async def finish_test_item(self, response = await AsyncHttpRequest(self.session.put, url=url, json=request_payload).make() if not response: return - logger.debug('finish_test_item - ID: %s', item_id) - logger.debug('response message: %s', response.message) - return response.message + message = await response.message + logger.debug('finish_test_item - ID: %s', await await_if_necessary(item_id)) + logger.debug('response message: %s', message) + return message async def finish_launch(self, launch_uuid: Union[str, asyncio.Task], @@ -316,9 +318,10 @@ async def finish_launch(self, name='Finish Launch').make() if not response: return - logger.debug('finish_launch - ID: %s', launch_uuid) - logger.debug('response message: %s', response.message) - return response.message + message = await response.message + logger.debug('finish_launch - ID: %s', await await_if_necessary(launch_uuid)) + logger.debug('response message: %s', message) + return message async def update_test_item(self, item_uuid: Union[str, asyncio.Task], @@ -335,7 +338,7 @@ async def update_test_item(self, if not response: return logger.debug('update_test_item - Item: %s', item_id) - return response.message + return await response.message async def __get_item_uuid_url(self, item_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: item_uuid = await await_if_necessary(item_uuid_future) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 6605c656..20d00522 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -29,7 +29,7 @@ from reportportal_client import helpers from reportportal_client.core.rp_file import RPFile from reportportal_client.core.rp_issues import Issue -from reportportal_client.core.rp_responses import RPResponse +from reportportal_client.core.rp_responses import RPResponse, AsyncRPResponse from reportportal_client.helpers import dict_to_payload, await_if_necessary from reportportal_client.static.abstract import ( AbstractBaseClass, @@ -139,14 +139,14 @@ def __init__(self, """ super().__init__(session_method=session_method, url=url, data=data, json=json, name=name) - async def make(self) -> Optional[RPResponse]: + async def make(self) -> Optional[AsyncRPResponse]: """Make HTTP request to the Report Portal API.""" url = await await_if_necessary(self.url) if not url: return try: - return RPResponse(await self.session_method(url, data=self.data, json=self.json)) + return AsyncRPResponse(await self.session_method(url, data=self.data, json=self.json)) except (KeyError, IOError, ValueError, TypeError) as exc: logger.warning( "Report Portal %s request failed", diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index dacc0246..9318300e 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -19,75 +19,83 @@ # limitations under the License import logging -from functools import lru_cache +from typing import Any, Optional + +from aiohttp import ClientResponse +from requests import Response from reportportal_client.static.defines import NOT_FOUND logger = logging.getLogger(__name__) -class RPMessage: - """Model for the message returned by RP API.""" - - __slots__ = ['message', 'error_code'] +class RPResponse: + """Class representing RP API response.""" + _resp: Response + __json: Any - def __init__(self, data): + def __init__(self, data: Response) -> None: """Initialize instance attributes. - :param data: Dictionary representation of the API response + :param data: requests.Response object """ - self.error_code = data.get('error_code', NOT_FOUND) - self.message = data.get('message', NOT_FOUND) + self._resp = data + self.__json = None - def __str__(self): - """Change string representation of the class.""" - if self.error_code is NOT_FOUND: - return self.message - return '{error_code}: {message}'.format(error_code=self.error_code, - message=self.message) + @property + def id(self) -> Optional[str]: + """Get value of the 'id' key.""" + return self.json.get('id', NOT_FOUND) @property - def is_empty(self): - """Check if returned message is empty.""" - return self.message is NOT_FOUND + def is_success(self) -> bool: + """Check if response to API has been successful.""" + return self._resp.ok + @property + def json(self) -> Any: + """Get the response in dictionary.""" + if not self.__json: + self.__json = self._resp.json() + return self.__json -class RPResponse: + @property + def message(self) -> Optional[str]: + """Get value of the 'message' key.""" + return self.json.get('message') + + +class AsyncRPResponse: """Class representing RP API response.""" + _resp: Response + __json: Any - def __init__(self, data): + def __init__(self, data: Response) -> None: """Initialize instance attributes. :param data: requests.Response object """ self._resp = data - - @staticmethod - def _get_json(data): - """Get response in dictionary. - - :param data: requests.Response object - :return: dict - """ - return data.json() + self.__json = None @property - def id(self): + async def id(self) -> Optional[str]: """Get value of the 'id' key.""" - return self.json.get('id', NOT_FOUND) + return (await self.json).get('id', NOT_FOUND) @property - def is_success(self): + def is_success(self) -> bool: """Check if response to API has been successful.""" return self._resp.ok @property - @lru_cache() - def json(self): + async def json(self) -> Any: """Get the response in dictionary.""" - return self._get_json(self._resp) + if not self.__json: + self.__json = await self._resp.json() + return self.__json @property - def message(self): + async def message(self) -> Optional[str]: """Get value of the 'message' key.""" - return self.json.get('message', NOT_FOUND) + return (await self.json).get('message') diff --git a/reportportal_client/core/rp_responses.pyi b/reportportal_client/core/rp_responses.pyi deleted file mode 100644 index f25774c2..00000000 --- a/reportportal_client/core/rp_responses.pyi +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2022 EPAM Systems -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License - -from requests import Response -from typing import Dict, Iterable, Optional, Text, Tuple - -class RPMessage: - error_code: Text = ... - message: Text = ... - def __init__(self, data: Dict) -> None: ... - @property - def is_empty(self) -> bool: ... - -class RPResponse: - _resp: Response = ... - def __init__(self, data: Response) -> None: ... - def _get_json(self, data: Response) -> Dict: ... - def _iter_messages(self) -> Iterable[RPMessage]: ... - @property - def id(self) -> Optional[Text]: ... - @property - def is_success(self) -> bool: ... - @property - def json(self) -> Dict: ... - @property - def message(self) -> Text: ... - @property - def messages(self) -> Tuple[RPMessage]: ... From 7ea23a305a8ff749919c02eed195c10bbd644c4c Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 7 Sep 2023 17:24:54 +0300 Subject: [PATCH 034/268] Async RPClient: WIP --- reportportal_client/aio/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 16b2b516..fc1db4ac 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -376,8 +376,8 @@ async def get_launch_info(self, launch_uuid_future: Union[str, asyncio.Task]) -> if not response: return if response.is_success: - launch_info = response.json - logger.debug('get_launch_info - Launch info: %s', response.json) + launch_info = await response.json + logger.debug('get_launch_info - Launch info: %s', launch_info) else: logger.warning('get_launch_info - Launch info: Failed to fetch launch ID from the API.') launch_info = {} @@ -409,7 +409,7 @@ async def get_launch_ui_url(self, launch_uuid_future: Union[str, asyncio.Task]) async def get_project_settings(self) -> Optional[Dict]: url = root_uri_join(self.base_url_v1, 'settings') response = await AsyncHttpRequest(self.session.get, url=url).make() - return response.json if response else None + return await response.json if response else None async def log(self, launch_uuid: Union[str, asyncio.Task], From 10f434f64e5cef9d9128f2089981c52e50ab5b38 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 8 Sep 2023 11:43:27 +0300 Subject: [PATCH 035/268] Fix tests --- reportportal_client/core/rp_requests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 20d00522..ddf065c4 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -258,9 +258,9 @@ def create_request(**kwargs) -> dict: 'type': kwargs['type'], 'launchUuid': kwargs['launch_uuid'] } - if 'attributes' in kwargs: + if kwargs.get('attributes'): request['attributes'] = dict_to_payload(kwargs['attributes']) - if 'parameters' in kwargs: + if kwargs.get('parameters'): request['parameters'] = dict_to_payload(kwargs['parameters']) return request @@ -310,7 +310,7 @@ def create_request(**kwargs) -> dict: 'status': kwargs.get('status'), 'retry': kwargs.get('retry') } - if 'attributes' in kwargs: + if kwargs.get('attributes'): request['attributes'] = dict_to_payload(kwargs['attributes']) if kwargs.get('issue') is None and ( From cc397d53f08a79696e50a85f312a7ce0b2ef7d14 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 8 Sep 2023 15:02:21 +0300 Subject: [PATCH 036/268] Async RPClient: WIP --- reportportal_client/aio/client.py | 78 ++++++++++++------------- reportportal_client/core/rp_requests.py | 5 +- 2 files changed, 39 insertions(+), 44 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index fc1db4ac..a070393d 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -72,6 +72,7 @@ class _AsyncRPClient: print_output: Optional[TextIO] _skip_analytics: str __session: Optional[aiohttp.ClientSession] + __stat_task: Optional[asyncio.Task] def __init__( self, @@ -104,12 +105,12 @@ def __init__( self.retries = retries self.max_pool_size = max_pool_size self.http_timeout = http_timeout - self._item_stack = _LifoQueue() self.mode = mode self._skip_analytics = getenv('AGENT_NO_ANALYTICS') self.launch_uuid_print = launch_uuid_print self.print_output = print_output or sys.stdout self.__session = None + self.__stat_task = None self.api_key = api_key if not self.api_key: @@ -205,21 +206,13 @@ async def start_launch(self, rerun_of=rerun_of or kwargs.get('rerunOf') ).payload - launch_coro = AsyncHttpRequest(self.session.post, - url=url, - json=request_payload).make() + response = await AsyncHttpRequest(self.session.post, url=url, json=request_payload).make() + if not response: + return - stat_coro = None if not self._skip_analytics: stat_coro = async_send_event('start_launch', *agent_name_version(attributes)) - - if stat_coro: - response = (await asyncio.gather(launch_coro, stat_coro))[0] - else: - response = await launch_coro - - if not response: - return + self.__stat_task = asyncio.create_task(stat_coro, name='Statistics update') launch_uuid = await response.id logger.debug(f'start_launch - ID: %s', launch_uuid) @@ -680,8 +673,8 @@ def clone(self) -> 'AsyncRPClient': class ScheduledRPClient(RPClient): __client: _AsyncRPClient _item_stack: _LifoQueue - loop: asyncio.AbstractEventLoop - thread: threading.Thread + __loop: Optional[asyncio.AbstractEventLoop] + __thread: Optional[threading.Thread] self_loop: bool self_thread: bool launch_uuid: Optional[asyncio.Task] @@ -690,7 +683,7 @@ class ScheduledRPClient(RPClient): def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio.Task] = None, client: Optional[_AsyncRPClient] = None, loop: Optional[asyncio.AbstractEventLoop] = None, - thread: Optional[threading.Thread] = None, **kwargs: Any) -> None: + **kwargs: Any) -> None: set_current(self) self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() @@ -704,20 +697,22 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio else: self.use_own_launch = True + self.__thread = None if loop: - self.loop = loop + self.__loop = loop self.self_loop = False else: - self.loop = asyncio.new_event_loop() + self.__loop = asyncio.new_event_loop() self.self_loop = True - if thread: - self.thread = thread - self.self_thread = False - else: - self.thread = threading.Thread(target=self.loop.run_forever, name='RP-Async-Client', - daemon=True) - self.thread.start() - self.self_thread = True + + def create_task(self, coro: Any) -> asyncio.Task: + loop = self.__loop + result = loop.create_task(coro) + if not self.__thread and self.self_loop: + self.__thread = threading.Thread(target=loop.run_forever, name='RP-Async-Client', + daemon=True) + self.__thread.start() + return result async def __empty_line(self): return "" @@ -735,7 +730,7 @@ def start_launch(self, launch_uuid_coro = self.__client.start_launch(name, start_time, description=description, attributes=attributes, rerun=rerun, rerun_of=rerun_of, **kwargs) - launch_uuid_task = self.loop.create_task(launch_uuid_coro) + launch_uuid_task = self.create_task(launch_uuid_coro) self.launch_uuid = launch_uuid_task return launch_uuid_task @@ -759,7 +754,7 @@ def start_test_item(self, parameters=parameters, parent_item_id=parent_item_id, has_stats=has_stats, code_ref=code_ref, retry=retry, test_case_id=test_case_id, **kwargs) - item_id_task = self.loop.create_task(item_id_coro) + item_id_task = self.create_task(item_id_coro) self._add_current_item(item_id_task) return item_id_task @@ -777,7 +772,7 @@ def finish_test_item(self, issue=issue, attributes=attributes, description=description, retry=retry, **kwargs) - result_task = self.loop.create_task(result_coro) + result_task = self.create_task(result_coro) self._remove_current_item() return result_task @@ -788,11 +783,11 @@ def finish_launch(self, attributes: Optional[Union[List, Dict]] = None, **kwargs: Any) -> asyncio.Task: if not self.use_own_launch: - return self.loop.create_task(self.__empty_line()) + return self.create_task(self.__empty_line()) result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, **kwargs) - result_task = self.loop.create_task(result_coro) + result_task = self.create_task(result_coro) return result_task def update_test_item(self, @@ -801,7 +796,7 @@ def update_test_item(self, description: Optional[str] = None) -> asyncio.Task: result_coro = self.__client.update_test_item(item_uuid, attributes=attributes, description=description) - result_task = self.loop.create_task(result_coro) + result_task = self.create_task(result_coro) return result_task def _add_current_item(self, item: asyncio.Task) -> None: @@ -824,33 +819,33 @@ def current_item(self) -> asyncio.Task: def get_launch_info(self) -> asyncio.Task: if not self.launch_uuid: - return self.loop.create_task(self.__empty_dict()) + return self.create_task(self.__empty_dict()) result_coro = self.__client.get_launch_info(self.launch_uuid) - result_task = self.loop.create_task(result_coro) + result_task = self.create_task(result_coro) return result_task def get_item_id_by_uuid(self, item_uuid_future: asyncio.Task) -> asyncio.Task: result_coro = self.__client.get_item_id_by_uuid(item_uuid_future) - result_task = self.loop.create_task(result_coro) + result_task = self.create_task(result_coro) return result_task def get_launch_ui_id(self) -> asyncio.Task: if not self.launch_uuid: - return self.loop.create_task(self.__none_value()) + return self.create_task(self.__none_value()) result_coro = self.__client.get_launch_ui_id(self.launch_uuid) - result_task = self.loop.create_task(result_coro) + result_task = self.create_task(result_coro) return result_task def get_launch_ui_url(self) -> asyncio.Task: if not self.launch_uuid: - return self.loop.create_task(self.__none_value()) + return self.create_task(self.__none_value()) result_coro = self.__client.get_launch_ui_url(self.launch_uuid) - result_task = self.loop.create_task(result_coro) + result_task = self.create_task(result_coro) return result_task def get_project_settings(self) -> asyncio.Task: result_coro = self.__client.get_project_settings() - result_task = self.loop.create_task(result_coro) + result_task = self.create_task(result_coro) return result_task def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, @@ -870,8 +865,7 @@ def clone(self) -> 'ScheduledRPClient': project=None, launch_uuid=self.launch_uuid, client=cloned_client, - loop=self.loop, - thread=self.thread + loop=self.__loop ) current_item = self.current_item() if current_item: diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index ddf065c4..05c38f67 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -144,9 +144,10 @@ async def make(self) -> Optional[AsyncRPResponse]: url = await await_if_necessary(self.url) if not url: return - + data = await_if_necessary(self.data) + json = await_if_necessary(self.json) try: - return AsyncRPResponse(await self.session_method(url, data=self.data, json=self.json)) + return AsyncRPResponse(await self.session_method(url, data=data, json=json)) except (KeyError, IOError, ValueError, TypeError) as exc: logger.warning( "Report Portal %s request failed", From 8dde2f668d0b8b4eb92a405190ec4383e53d8cff Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 8 Sep 2023 15:10:12 +0300 Subject: [PATCH 037/268] Async RPClient: WIP --- reportportal_client/core/rp_requests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 05c38f67..1d652ba8 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -144,8 +144,8 @@ async def make(self) -> Optional[AsyncRPResponse]: url = await await_if_necessary(self.url) if not url: return - data = await_if_necessary(self.data) - json = await_if_necessary(self.json) + data = await await_if_necessary(self.data) + json = await await_if_necessary(self.json) try: return AsyncRPResponse(await self.session_method(url, data=data, json=json)) except (KeyError, IOError, ValueError, TypeError) as exc: From 6fb8344fd86cddc459ed0e87e509bac28e4a61ce Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 8 Sep 2023 16:07:45 +0300 Subject: [PATCH 038/268] Fix SSL Context for async_send_event function --- reportportal_client/services/statistics.py | 5 ++++- requirements.txt | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/reportportal_client/services/statistics.py b/reportportal_client/services/statistics.py index 0c2bfa3e..f090cc71 100644 --- a/reportportal_client/services/statistics.py +++ b/reportportal_client/services/statistics.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License +import ssl +import certifi import logging from platform import python_version from typing import Optional @@ -106,9 +108,10 @@ async def async_send_event(event_name: str, agent_name: Optional[str], 'measurement_id': ID, 'api_secret': KEY } + sslcontext = ssl.create_default_context(cafile=certifi.where()) async with aiohttp.ClientSession() as session: result = await session.post(url=ENDPOINT, json=get_payload(event_name, agent_name, agent_version), - headers=headers, params=query_params) + headers=headers, params=query_params, ssl=sslcontext) if not result.ok: logger.debug('Failed to send data to Statistics service: %s', result.reason) return result diff --git a/requirements.txt b/requirements.txt index 5c507f12..3375cf78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ aenum requests>=2.27.1 six>=1.16.0 -aiohttp==3.8.5 +aiohttp>=3.8.5 +certifi>=2023.7.22 From aca919c239104995104e2bbd0d2d5ac47b887c14 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 8 Sep 2023 16:19:26 +0300 Subject: [PATCH 039/268] Minor style changes --- reportportal_client/aio/client.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index a070393d..7385e712 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -730,9 +730,8 @@ def start_launch(self, launch_uuid_coro = self.__client.start_launch(name, start_time, description=description, attributes=attributes, rerun=rerun, rerun_of=rerun_of, **kwargs) - launch_uuid_task = self.create_task(launch_uuid_coro) - self.launch_uuid = launch_uuid_task - return launch_uuid_task + self.launch_uuid = self.create_task(launch_uuid_coro) + return self.launch_uuid def start_test_item(self, name: str, @@ -785,8 +784,7 @@ def finish_launch(self, if not self.use_own_launch: return self.create_task(self.__empty_line()) result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, - attributes=attributes, - **kwargs) + attributes=attributes, **kwargs) result_task = self.create_task(result_coro) return result_task From a9d9f5764914a64836bc6da35b0da5444223a98d Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 8 Sep 2023 16:24:28 +0300 Subject: [PATCH 040/268] Add certify on connection configuration --- reportportal_client/aio/client.py | 12 ++++++++---- reportportal_client/services/statistics.py | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 7385e712..56cc0cc3 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -14,6 +14,7 @@ # limitations under the License import asyncio +import certifi import logging import ssl import sys @@ -141,10 +142,13 @@ def session(self) -> aiohttp.ClientSession: return self.__session ssl_config = self.verify_ssl - if ssl_config and type(ssl_config) == str: - ssl_context = ssl.create_default_context() - ssl_context.load_cert_chain(ssl_config) - ssl_config = ssl_context + if ssl_config: + if type(ssl_config) == str: + sl_config = ssl.create_default_context() + sl_config.load_cert_chain(ssl_config) + else: + ssl_config = ssl.create_default_context(cafile=certifi.where()) + connector = aiohttp.TCPConnector(ssl=ssl_config, limit=self.max_pool_size) timeout = None diff --git a/reportportal_client/services/statistics.py b/reportportal_client/services/statistics.py index f090cc71..fa300daa 100644 --- a/reportportal_client/services/statistics.py +++ b/reportportal_client/services/statistics.py @@ -108,10 +108,10 @@ async def async_send_event(event_name: str, agent_name: Optional[str], 'measurement_id': ID, 'api_secret': KEY } - sslcontext = ssl.create_default_context(cafile=certifi.where()) + ssl_context = ssl.create_default_context(cafile=certifi.where()) async with aiohttp.ClientSession() as session: result = await session.post(url=ENDPOINT, json=get_payload(event_name, agent_name, agent_version), - headers=headers, params=query_params, ssl=sslcontext) + headers=headers, params=query_params, ssl=ssl_context) if not result.ok: logger.debug('Failed to send data to Statistics service: %s', result.reason) return result From 68915523cee5ae005e09f3a5052182eb8df1b952 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 8 Sep 2023 17:00:00 +0300 Subject: [PATCH 041/268] Fix payload conversion --- reportportal_client/core/rp_requests.py | 43 ++++++++++++------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 1d652ba8..721ce983 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -108,15 +108,10 @@ def priority(self, value: Priority) -> None: def make(self) -> Optional[RPResponse]: """Make HTTP request to the Report Portal API.""" try: - return RPResponse(self.session_method(self.url, data=self.data, json=self.json, - files=self.files, verify=self.verify_ssl, - timeout=self.http_timeout)) + return RPResponse(self.session_method(self.url, data=self.data, json=self.json, files=self.files, + verify=self.verify_ssl, timeout=self.http_timeout)) except (KeyError, IOError, ValueError, TypeError) as exc: - logger.warning( - "Report Portal %s request failed", - self.name, - exc_info=exc - ) + logger.warning("Report Portal %s request failed", self.name, exc_info=exc) class AsyncHttpRequest(HttpRequest): @@ -149,11 +144,7 @@ async def make(self) -> Optional[AsyncRPResponse]: try: return AsyncRPResponse(await self.session_method(url, data=data, json=json)) except (KeyError, IOError, ValueError, TypeError) as exc: - logger.warning( - "Report Portal %s request failed", - self.name, - exc_info=exc - ) + logger.warning("Report Portal %s request failed", self.name, exc_info=exc) class RPRequestBase(metaclass=AbstractBaseClass): @@ -185,8 +176,8 @@ class LaunchStartRequest(RPRequestBase): @property def payload(self) -> dict: """Get HTTP payload for the request.""" - my_attributes = None - if self.attributes and isinstance(self.attributes, dict): + my_attributes = self.attributes + if my_attributes and isinstance(self.attributes, dict): my_attributes = dict_to_payload(self.attributes) result = { 'attributes': my_attributes, @@ -217,8 +208,8 @@ class LaunchFinishRequest(RPRequestBase): @property def payload(self) -> dict: """Get HTTP payload for the request.""" - my_attributes = None - if self.attributes and isinstance(self.attributes, dict): + my_attributes = self.attributes + if my_attributes and isinstance(self.attributes, dict): my_attributes = dict_to_payload(self.attributes) return { 'attributes': my_attributes, @@ -259,10 +250,14 @@ def create_request(**kwargs) -> dict: 'type': kwargs['type'], 'launchUuid': kwargs['launch_uuid'] } - if kwargs.get('attributes'): - request['attributes'] = dict_to_payload(kwargs['attributes']) - if kwargs.get('parameters'): - request['parameters'] = dict_to_payload(kwargs['parameters']) + attributes = kwargs.get('attributes') + if attributes and isinstance(attributes, dict): + attributes = dict_to_payload(kwargs['attributes']) + request['attributes'] = attributes + parameters = kwargs.get('parameters') + if parameters and isinstance(parameters, dict): + parameters = dict_to_payload(kwargs['parameters']) + request['parameters'] = parameters return request @property @@ -311,8 +306,10 @@ def create_request(**kwargs) -> dict: 'status': kwargs.get('status'), 'retry': kwargs.get('retry') } - if kwargs.get('attributes'): - request['attributes'] = dict_to_payload(kwargs['attributes']) + attributes = kwargs.get('attributes') + if attributes and isinstance(attributes, dict): + attributes = dict_to_payload(kwargs['attributes']) + request['attributes'] = attributes if kwargs.get('issue') is None and ( kwargs.get('status') is not None and kwargs.get('status').lower() == 'skipped' From 4d7d9fae90cb2ded7bb61d2defb39087b8dc7a36 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 8 Sep 2023 17:42:47 +0300 Subject: [PATCH 042/268] Revert worker update since it degrades performance --- reportportal_client/core/worker.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/reportportal_client/core/worker.py b/reportportal_client/core/worker.py index b787c7bd..d145b085 100644 --- a/reportportal_client/core/worker.py +++ b/reportportal_client/core/worker.py @@ -86,7 +86,6 @@ def _command_process(self, cmd): self._stop_immediately() else: self._stop() - self._queue.task_done() def _request_process(self, request): """Send request to RP and update response attribute of the request.""" @@ -96,7 +95,6 @@ def _request_process(self, request): except Exception as err: logger.exception('[%s] Unknown exception has occurred. ' 'Skipping it.', err) - self._queue.task_done() def _monitor(self): """Monitor worker queues and process them. @@ -128,7 +126,11 @@ def _stop(self): This method process everything in worker's queue first, ignoring commands and terminates thread only after. """ - self._queue.join() + request = self._command_get() + while request is not None: + if not isinstance(request, ControlCommand): + self._request_process(request) + request = self._command_get() self._stop_immediately() def _stop_immediately(self): From af2306bcef6f184ba246a551c8fef5a217189584 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 8 Sep 2023 17:47:20 +0300 Subject: [PATCH 043/268] Add backward compatible methods --- reportportal_client/aio/client.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 56cc0cc3..522b7429 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -522,6 +522,12 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: raise NotImplementedError('"log" method is not implemented!') + def start(self) -> None: + pass # For backward compatibility + + def terminate(self, *_: Any, **__: Any) -> None: + pass # For backward compatibility + class AsyncRPClient(RPClient): __client: _AsyncRPClient From 4b3ffe62c10458dd9e5d260e79c829aeacb33d10 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 8 Sep 2023 19:00:49 +0300 Subject: [PATCH 044/268] Add task finishing wait --- reportportal_client/aio/client.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 522b7429..aa159726 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -19,6 +19,7 @@ import ssl import sys import threading +import time import warnings from os import getenv from queue import LifoQueue @@ -45,6 +46,9 @@ logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) +TASK_TIMEOUT: int = 60 +SHUTDOWN_TIMEOUT: int = 120 + class _LifoQueue(LifoQueue): def last(self): @@ -685,6 +689,7 @@ class ScheduledRPClient(RPClient): _item_stack: _LifoQueue __loop: Optional[asyncio.AbstractEventLoop] __thread: Optional[threading.Thread] + __task_list: List[asyncio.Task] self_loop: bool self_thread: bool launch_uuid: Optional[asyncio.Task] @@ -707,6 +712,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio else: self.use_own_launch = True + self.__task_list = [] self.__thread = None if loop: self.__loop = loop @@ -718,12 +724,29 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio def create_task(self, coro: Any) -> asyncio.Task: loop = self.__loop result = loop.create_task(coro) + self.__task_list.append(result) if not self.__thread and self.self_loop: self.__thread = threading.Thread(target=loop.run_forever, name='RP-Async-Client', daemon=True) self.__thread.start() + i = 0 + for i, task in enumerate(self.__task_list): + if not task.done(): + break + self.__task_list = self.__task_list[i:] return result + def finish_tasks(self): + shutdown_start_time = time.time() + for task in self.__task_list: + task_start_time = time.time() + while not task.done() and (time.time() - task_start_time < TASK_TIMEOUT) and ( + time.time() - shutdown_start_time < SHUTDOWN_TIMEOUT): + time.sleep(0.001) + if time.time() - shutdown_start_time >= SHUTDOWN_TIMEOUT: + break + self.__task_list = [] + async def __empty_line(self): return "" @@ -796,6 +819,7 @@ def finish_launch(self, result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, **kwargs) result_task = self.create_task(result_coro) + self.finish_tasks() return result_task def update_test_item(self, From 69c5b1dcd24f481fee3d505dc99fa6866fbbf9d1 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Sat, 9 Sep 2023 00:21:38 +0300 Subject: [PATCH 045/268] Correct launch finish logic --- reportportal_client/aio/client.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index aa159726..d16d224f 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -14,7 +14,6 @@ # limitations under the License import asyncio -import certifi import logging import ssl import sys @@ -26,6 +25,7 @@ from typing import Union, Tuple, List, Dict, Any, Optional, TextIO import aiohttp +import certifi # noinspection PyProtectedMember from reportportal_client._local import set_current @@ -814,10 +814,12 @@ def finish_launch(self, status: str = None, attributes: Optional[Union[List, Dict]] = None, **kwargs: Any) -> asyncio.Task: - if not self.use_own_launch: - return self.create_task(self.__empty_line()) - result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, - attributes=attributes, **kwargs) + if self.use_own_launch: + result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, + attributes=attributes, **kwargs) + else: + result_coro = self.create_task(self.__empty_line()) + result_task = self.create_task(result_coro) self.finish_tasks() return result_task From 73b37f5a037914de9360d1636aa013bd36ac7533 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Sat, 9 Sep 2023 01:02:17 +0300 Subject: [PATCH 046/268] Remove one TODO, add another one --- reportportal_client/aio/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index d16d224f..d33be4c3 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -808,7 +808,6 @@ def finish_test_item(self, self._remove_current_item() return result_task - # TODO: implement loop task finish wait def finish_launch(self, end_time: str, status: str = None, @@ -884,6 +883,7 @@ def get_project_settings(self) -> asyncio.Task: def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: + # TODO: implement logging return None def clone(self) -> 'ScheduledRPClient': From c89b566be2846cc16ebe6d38abd6538c73ea5ca5 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Sat, 9 Sep 2023 01:20:41 +0300 Subject: [PATCH 047/268] Correct task wait sleep time --- reportportal_client/aio/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index d33be4c3..61d3d8d7 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -737,12 +737,13 @@ def create_task(self, coro: Any) -> asyncio.Task: return result def finish_tasks(self): + sleep_time = sys.getswitchinterval() shutdown_start_time = time.time() for task in self.__task_list: task_start_time = time.time() while not task.done() and (time.time() - task_start_time < TASK_TIMEOUT) and ( time.time() - shutdown_start_time < SHUTDOWN_TIMEOUT): - time.sleep(0.001) + time.sleep(sleep_time) if time.time() - shutdown_start_time >= SHUTDOWN_TIMEOUT: break self.__task_list = [] From 04f957cd3184a4ca263a8cbaa32b7cdabe40e4a7 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Sat, 9 Sep 2023 13:31:13 +0300 Subject: [PATCH 048/268] Add batched client implementation --- reportportal_client/aio/client.py | 206 ++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 61d3d8d7..506c1e73 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -906,3 +906,209 @@ def clone(self) -> 'ScheduledRPClient': if current_item: cloned._add_current_item(current_item) return cloned + + +class BatchedRPClient(RPClient): + __client: _AsyncRPClient + _item_stack: _LifoQueue + __loop: Optional[asyncio.AbstractEventLoop] + __thread: Optional[threading.Thread] + __task_list: List[asyncio.Task] + launch_uuid: Optional[asyncio.Task] + use_own_launch: bool + step_reporter: StepReporter + + def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio.Task] = None, + client: Optional[_AsyncRPClient] = None, loop: Optional[asyncio.AbstractEventLoop] = None, + **kwargs: Any) -> None: + set_current(self) + self.step_reporter = StepReporter(self) + self._item_stack = _LifoQueue() + if client: + self.__client = client + else: + self.__client = _AsyncRPClient(endpoint, project, **kwargs) + if launch_uuid: + self.launch_uuid = launch_uuid + self.use_own_launch = False + else: + self.use_own_launch = True + + self.__task_list = [] + self.__loop = asyncio.new_event_loop() + + def run_tasks(self) -> None: + tasks = self.__task_list + if len(tasks) <= 0: + return + self.__task_list = [] + self.__loop.run_until_complete(asyncio.gather(*tasks)) + + def create_task(self, coro: Any) -> asyncio.Task: + result = self.__loop.create_task(coro) + self.__task_list.append(result) + if len(self.__task_list) >= 10: + self.run_tasks() + return result + + def finish_tasks(self): + self.run_tasks() + + async def __empty_line(self): + return "" + + def start_launch(self, + name: str, + start_time: str, + description: Optional[str] = None, + attributes: Optional[Union[List, Dict]] = None, + rerun: bool = False, + rerun_of: Optional[str] = None, + **kwargs) -> asyncio.Task: + if not self.use_own_launch: + return self.launch_uuid + launch_uuid_coro = self.__client.start_launch(name, start_time, description=description, + attributes=attributes, rerun=rerun, rerun_of=rerun_of, + **kwargs) + self.launch_uuid = self.create_task(launch_uuid_coro) + return self.launch_uuid + + def start_test_item(self, + name: str, + start_time: str, + item_type: str, + *, + description: Optional[str] = None, + attributes: Optional[List[Dict]] = None, + parameters: Optional[Dict] = None, + parent_item_id: Optional[asyncio.Task] = None, + has_stats: bool = True, + code_ref: Optional[str] = None, + retry: bool = False, + test_case_id: Optional[str] = None, + **kwargs: Any) -> asyncio.Task: + + item_id_coro = self.__client.start_test_item(self.launch_uuid, name, start_time, item_type, + description=description, attributes=attributes, + parameters=parameters, parent_item_id=parent_item_id, + has_stats=has_stats, code_ref=code_ref, retry=retry, + test_case_id=test_case_id, **kwargs) + item_id_task = self.create_task(item_id_coro) + self._add_current_item(item_id_task) + return item_id_task + + def finish_test_item(self, + item_id: asyncio.Task, + end_time: str, + *, + status: str = None, + issue: Optional[Issue] = None, + attributes: Optional[Union[List, Dict]] = None, + description: str = None, + retry: bool = False, + **kwargs: Any) -> asyncio.Task: + result_coro = self.__client.finish_test_item(self.launch_uuid, item_id, end_time, status=status, + issue=issue, attributes=attributes, + description=description, + retry=retry, **kwargs) + result_task = self.create_task(result_coro) + self._remove_current_item() + return result_task + + def finish_launch(self, + end_time: str, + status: str = None, + attributes: Optional[Union[List, Dict]] = None, + **kwargs: Any) -> asyncio.Task: + if self.use_own_launch: + result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, + attributes=attributes, **kwargs) + else: + result_coro = self.create_task(self.__empty_line()) + + result_task = self.create_task(result_coro) + self.finish_tasks() + return result_task + + def update_test_item(self, + item_uuid: asyncio.Task, + attributes: Optional[Union[List, Dict]] = None, + description: Optional[str] = None) -> asyncio.Task: + result_coro = self.__client.update_test_item(item_uuid, attributes=attributes, + description=description) + result_task = self.create_task(result_coro) + return result_task + + def _add_current_item(self, item: asyncio.Task) -> None: + """Add the last item from the self._items queue.""" + self._item_stack.put(item) + + def _remove_current_item(self) -> asyncio.Task: + """Remove the last item from the self._items queue.""" + return self._item_stack.get() + + async def __empty_dict(self): + return {} + + async def __none_value(self): + return + + def current_item(self) -> asyncio.Task: + """Retrieve the last item reported by the client.""" + return self._item_stack.last() + + def get_launch_info(self) -> asyncio.Task: + if not self.launch_uuid: + return self.create_task(self.__empty_dict()) + result_coro = self.__client.get_launch_info(self.launch_uuid) + result_task = self.create_task(result_coro) + return result_task + + def get_item_id_by_uuid(self, item_uuid_future: asyncio.Task) -> asyncio.Task: + result_coro = self.__client.get_item_id_by_uuid(item_uuid_future) + result_task = self.create_task(result_coro) + return result_task + + def get_launch_ui_id(self) -> asyncio.Task: + if not self.launch_uuid: + return self.create_task(self.__none_value()) + result_coro = self.__client.get_launch_ui_id(self.launch_uuid) + result_task = self.create_task(result_coro) + return result_task + + def get_launch_ui_url(self) -> asyncio.Task: + if not self.launch_uuid: + return self.create_task(self.__none_value()) + result_coro = self.__client.get_launch_ui_url(self.launch_uuid) + result_task = self.create_task(result_coro) + return result_task + + def get_project_settings(self) -> asyncio.Task: + result_coro = self.__client.get_project_settings() + result_task = self.create_task(result_coro) + return result_task + + def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, + attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: + # TODO: implement logging + return None + + def clone(self) -> 'ScheduledRPClient': + """Clone the client object, set current Item ID as cloned item ID. + + :returns: Cloned client object + :rtype: ScheduledRPClient + """ + cloned_client = self.__client.clone() + # noinspection PyTypeChecker + cloned = ScheduledRPClient( + endpoint=None, + project=None, + launch_uuid=self.launch_uuid, + client=cloned_client, + loop=self.__loop + ) + current_item = self.current_item() + if current_item: + cloned._add_current_item(current_item) + return cloned \ No newline at end of file From f54cf5c109f803a4258439b10a74db448d352ae0 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 11 Sep 2023 23:13:27 +0300 Subject: [PATCH 049/268] Batched client update --- reportportal_client/aio/client.py | 65 +++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 506c1e73..f499f0eb 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -46,6 +46,8 @@ logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) +TASK_RUN_NUM_THRESHOLD: int = 10 +TASK_RUN_INTERVAL: float = 1.0 TASK_TIMEOUT: int = 60 SHUTDOWN_TIMEOUT: int = 120 @@ -684,7 +686,7 @@ def clone(self) -> 'AsyncRPClient': return cloned -class ScheduledRPClient(RPClient): +class ThreadedRPClient(RPClient): __client: _AsyncRPClient _item_stack: _LifoQueue __loop: Optional[asyncio.AbstractEventLoop] @@ -887,15 +889,15 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, # TODO: implement logging return None - def clone(self) -> 'ScheduledRPClient': + def clone(self) -> 'ThreadedRPClient': """Clone the client object, set current Item ID as cloned item ID. :returns: Cloned client object - :rtype: ScheduledRPClient + :rtype: ThreadedRPClient """ cloned_client = self.__client.clone() # noinspection PyTypeChecker - cloned = ScheduledRPClient( + cloned = ThreadedRPClient( endpoint=None, project=None, launch_uuid=self.launch_uuid, @@ -911,9 +913,10 @@ def clone(self) -> 'ScheduledRPClient': class BatchedRPClient(RPClient): __client: _AsyncRPClient _item_stack: _LifoQueue - __loop: Optional[asyncio.AbstractEventLoop] - __thread: Optional[threading.Thread] + __loop: asyncio.AbstractEventLoop __task_list: List[asyncio.Task] + __task_mutex: threading.Lock + __last_run_time: float launch_uuid: Optional[asyncio.Task] use_own_launch: bool step_reporter: StepReporter @@ -935,24 +938,44 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio self.use_own_launch = True self.__task_list = [] + self.__task_mutex = threading.Lock() + self.__last_run_time = time.time() self.__loop = asyncio.new_event_loop() - def run_tasks(self) -> None: - tasks = self.__task_list - if len(tasks) <= 0: - return - self.__task_list = [] - self.__loop.run_until_complete(asyncio.gather(*tasks)) + @property + def loop(self): + return self.__loop + + def __ready_to_run(self) -> bool: + current_time = time.time() + last_time = self.__last_run_time + if len(self.__task_list) <= 0: + return False + if len(self.__task_list) > TASK_RUN_NUM_THRESHOLD or current_time - last_time >= TASK_RUN_INTERVAL: + self.__last_run_time = current_time + return True + return False def create_task(self, coro: Any) -> asyncio.Task: result = self.__loop.create_task(coro) - self.__task_list.append(result) - if len(self.__task_list) >= 10: - self.run_tasks() + tasks = None + with self.__task_mutex: + self.__task_list.append(result) + if self.__ready_to_run(): + tasks = self.__task_list + self.__task_list = [] + if tasks: + self.__loop.run_until_complete(asyncio.gather(*tasks)) return result - def finish_tasks(self): - self.run_tasks() + def finish_tasks(self) -> None: + tasks = None + with self.__task_mutex: + if len(self.__task_list) > 0: + tasks = self.__task_list + self.__task_list = [] + if tasks: + self.__loop.run_until_complete(asyncio.gather(*tasks)) async def __empty_line(self): return "" @@ -1093,15 +1116,15 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, # TODO: implement logging return None - def clone(self) -> 'ScheduledRPClient': + def clone(self) -> 'BatchedRPClient': """Clone the client object, set current Item ID as cloned item ID. :returns: Cloned client object - :rtype: ScheduledRPClient + :rtype: BatchedRPClient """ cloned_client = self.__client.clone() # noinspection PyTypeChecker - cloned = ScheduledRPClient( + cloned = BatchedRPClient( endpoint=None, project=None, launch_uuid=self.launch_uuid, @@ -1111,4 +1134,4 @@ def clone(self) -> 'ScheduledRPClient': current_item = self.current_item() if current_item: cloned._add_current_item(current_item) - return cloned \ No newline at end of file + return cloned From df5a9d0272c8a6fea6176b8b15a781255251fdf9 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 12 Sep 2023 16:39:19 +0300 Subject: [PATCH 050/268] Self Task type add --- reportportal_client/aio/client.py | 230 +++++++++++++++++++----------- 1 file changed, 146 insertions(+), 84 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index f499f0eb..8a2017a6 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -20,9 +20,11 @@ import threading import time import warnings +from asyncio import Future from os import getenv from queue import LifoQueue -from typing import Union, Tuple, List, Dict, Any, Optional, TextIO +from typing import Union, Tuple, List, Dict, Any, Optional, TextIO, Coroutine, TypeVar, Generic, Generator, \ + Awaitable import aiohttp import certifi @@ -52,6 +54,68 @@ SHUTDOWN_TIMEOUT: int = 120 +T = TypeVar('T') + + +class Task(Generic[T], asyncio.Task, metaclass=AbstractBaseClass): + __metaclass__ = AbstractBaseClass + + def __init__( + self, + coro: Union[Generator[Future[object], None, T], Awaitable[T]], + *, + loop: asyncio.AbstractEventLoop, + name: Optional[str] = None + ) -> None: + super().__init__(coro, loop=loop, name=name) + + @abstractmethod + def blocking_result(self) -> T: + raise NotImplementedError('"blocking_result" method is not implemented!') + + +class BatchedTask(Generic[T], Task[T]): + __loop: asyncio.AbstractEventLoop + __thread: threading.Thread + + def __init__( + self, + coro: Union[Generator[Future[object], None, T], Awaitable[T]], + *, + loop: asyncio.AbstractEventLoop, + name: Optional[str] = None, + thread: threading.Thread + ) -> None: + super().__init__(coro, loop=loop, name=name) + self.__loop = loop + self.__thread = thread + + def blocking_result(self) -> T: + if self.done(): + return self.result() + if self.__thread is not threading.current_thread(): + warnings.warn("The method was called from different thread which was used to create the" + "task, unexpected behavior is possible during the execution.", RuntimeWarning, + stacklevel=3) + return self.__loop.run_until_complete(self) + + +class _BatchedTaskFactory: + __loop: asyncio.AbstractEventLoop + __thread: threading.Thread + + def __init__(self, loop: asyncio.AbstractEventLoop, thread: threading.Thread): + self.__loop = loop + self.__thread = thread + + def __call__( + self, + loop: asyncio.AbstractEventLoop, + factory: Union[Coroutine[Any, Any, T], Generator[Any, None, T]] + ) -> Task[T]: + return BatchedTask(factory, loop=self.__loop, thread=self.__thread) + + class _LifoQueue(LifoQueue): def last(self): with self.mutex: @@ -79,7 +143,7 @@ class _AsyncRPClient: print_output: Optional[TextIO] _skip_analytics: str __session: Optional[aiohttp.ClientSession] - __stat_task: Optional[asyncio.Task] + __stat_task: Optional[Task[aiohttp.ClientResponse]] def __init__( self, @@ -172,14 +236,14 @@ def session(self) -> aiohttp.ClientSession: timeout=timeout) return self.__session - async def __get_item_url(self, item_id_future: Union[str, asyncio.Task]) -> Optional[str]: + async def __get_item_url(self, item_id_future: Union[str, Task[str]]) -> Optional[str]: item_id = await await_if_necessary(item_id_future) if item_id is NOT_FOUND: logger.warning('Attempt to make request for non-existent id.') return return root_uri_join(self.base_url_v2, 'item', item_id) - async def __get_launch_url(self, launch_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: + async def __get_launch_url(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[str]: launch_uuid = await await_if_necessary(launch_uuid_future) if launch_uuid is NOT_FOUND: logger.warning('Attempt to make request for non-existent launch.') @@ -231,7 +295,7 @@ async def start_launch(self, return launch_uuid async def start_test_item(self, - launch_uuid: Union[str, asyncio.Task], + launch_uuid: Union[str, Task[str]], name: str, start_time: str, item_type: str, @@ -239,7 +303,7 @@ async def start_test_item(self, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, parameters: Optional[Dict] = None, - parent_item_id: Optional[Union[str, asyncio.Task]] = None, + parent_item_id: Optional[Union[str, Task[str]]] = None, has_stats: bool = True, code_ref: Optional[str] = None, retry: bool = False, @@ -274,8 +338,8 @@ async def start_test_item(self, return item_id async def finish_test_item(self, - launch_uuid: Union[str, asyncio.Task], - item_id: Union[str, asyncio.Task], + launch_uuid: Union[str, Task[str]], + item_id: Union[str, Task[str]], end_time: str, *, status: str = None, @@ -304,7 +368,7 @@ async def finish_test_item(self, return message async def finish_launch(self, - launch_uuid: Union[str, asyncio.Task], + launch_uuid: Union[str, Task[str]], end_time: str, *, status: str = None, @@ -327,7 +391,7 @@ async def finish_launch(self, return message async def update_test_item(self, - item_uuid: Union[str, asyncio.Task], + item_uuid: Union[str, Task[str]], *, attributes: Optional[Union[List, Dict]] = None, description: Optional[str] = None) -> Optional[str]: @@ -343,24 +407,24 @@ async def update_test_item(self, logger.debug('update_test_item - Item: %s', item_id) return await response.message - async def __get_item_uuid_url(self, item_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: + async def __get_item_uuid_url(self, item_uuid_future: Union[str, Task[str]]) -> Optional[str]: item_uuid = await await_if_necessary(item_uuid_future) if item_uuid is NOT_FOUND: logger.warning('Attempt to make request for non-existent UUID.') return return root_uri_join(self.base_url_v1, 'item', 'uuid', item_uuid) - async def get_item_id_by_uuid(self, item_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: + async def get_item_id_by_uuid(self, item_uuid_future: Union[str, Task[str]]) -> Optional[str]: """Get test Item ID by the given Item UUID. - :param item_uuid_future: Str or asyncio.Task UUID returned on the Item start + :param item_uuid_future: Str or Task UUID returned on the Item start :return: Test item ID """ url = self.__get_item_uuid_url(item_uuid_future) response = await AsyncHttpRequest(self.session.get, url=url).make() return response.id if response else None - async def __get_launch_uuid_url(self, launch_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: + async def __get_launch_uuid_url(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[str]: launch_uuid = await await_if_necessary(launch_uuid_future) if launch_uuid is NOT_FOUND: logger.warning('Attempt to make request for non-existent Launch UUID.') @@ -368,10 +432,10 @@ async def __get_launch_uuid_url(self, launch_uuid_future: Union[str, asyncio.Tas logger.debug('get_launch_info - ID: %s', launch_uuid) return root_uri_join(self.base_url_v1, 'launch', 'uuid', launch_uuid) - async def get_launch_info(self, launch_uuid_future: Union[str, asyncio.Task]) -> Optional[Dict]: + async def get_launch_info(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[Dict]: """Get the launch information by Launch UUID. - :param launch_uuid_future: Str or asyncio.Task UUID returned on the Launch start + :param launch_uuid_future: Str or Task UUID returned on the Launch start :return dict: Launch information in dictionary """ url = self.__get_launch_uuid_url(launch_uuid_future) @@ -386,11 +450,11 @@ async def get_launch_info(self, launch_uuid_future: Union[str, asyncio.Task]) -> launch_info = {} return launch_info - async def get_launch_ui_id(self, launch_uuid_future: Union[str, asyncio.Task]) -> Optional[int]: + async def get_launch_ui_id(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[int]: launch_info = await self.get_launch_info(launch_uuid_future) return launch_info.get('id') if launch_info else None - async def get_launch_ui_url(self, launch_uuid_future: Union[str, asyncio.Task]) -> Optional[str]: + async def get_launch_ui_url(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[str]: launch_uuid = await await_if_necessary(launch_uuid_future) launch_info = await self.get_launch_info(launch_uuid) ui_id = launch_info.get('id') if launch_info else None @@ -415,13 +479,13 @@ async def get_project_settings(self) -> Optional[Dict]: return await response.json if response else None async def log(self, - launch_uuid: Union[str, asyncio.Task], + launch_uuid: Union[str, Task[str]], time: str, message: str, *, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, - item_id: Optional[Union[str, asyncio.Task]] = None) -> None: + item_id: Optional[Union[str, Task[str]]] = None) -> None: pass def clone(self) -> '_AsyncRPClient': @@ -457,7 +521,7 @@ def start_launch(self, attributes: Optional[Union[List, Dict]] = None, rerun: bool = False, rerun_of: Optional[str] = None, - **kwargs) -> Union[Optional[str], asyncio.Task]: + **kwargs) -> Union[Optional[str], Task[str]]: raise NotImplementedError('"start_launch" method is not implemented!') @abstractmethod @@ -469,17 +533,17 @@ def start_test_item(self, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, parameters: Optional[Dict] = None, - parent_item_id: Union[Optional[str], asyncio.Task] = None, + parent_item_id: Union[Optional[str], Task[str]] = None, has_stats: bool = True, code_ref: Optional[str] = None, retry: bool = False, test_case_id: Optional[str] = None, - **kwargs: Any) -> Union[Optional[str], asyncio.Task]: + **kwargs: Any) -> Union[Optional[str], Task[str]]: raise NotImplementedError('"start_test_item" method is not implemented!') @abstractmethod def finish_test_item(self, - item_id: Union[str, asyncio.Task], + item_id: Union[str, Task[str]], end_time: str, *, status: str = None, @@ -487,7 +551,7 @@ def finish_test_item(self, attributes: Optional[Union[List, Dict]] = None, description: str = None, retry: bool = False, - **kwargs: Any) -> Union[Optional[str], asyncio.Task]: + **kwargs: Any) -> Union[Optional[str], Task[str]]: raise NotImplementedError('"finish_test_item" method is not implemented!') @abstractmethod @@ -495,7 +559,7 @@ def finish_launch(self, end_time: str, status: str = None, attributes: Optional[Union[List, Dict]] = None, - **kwargs: Any) -> Union[Optional[str], asyncio.Task]: + **kwargs: Any) -> Union[Optional[str], Task[str]]: raise NotImplementedError('"finish_launch" method is not implemented!') @abstractmethod @@ -504,11 +568,11 @@ def update_test_item(self, item_uuid: str, attributes: Optional[Union[List, Dict raise NotImplementedError('"update_test_item" method is not implemented!') @abstractmethod - def get_launch_info(self) -> Union[Optional[dict], asyncio.Task]: + def get_launch_info(self) -> Union[Optional[dict], Task[str]]: raise NotImplementedError('"get_launch_info" method is not implemented!') @abstractmethod - def get_item_id_by_uuid(self, item_uuid: str) -> Optional[str]: + def get_item_id_by_uuid(self, item_uuid: Union[str, Task[str]]) -> Optional[str]: raise NotImplementedError('"get_item_id_by_uuid" method is not implemented!') @abstractmethod @@ -525,7 +589,7 @@ def get_project_settings(self) -> Optional[Dict]: @abstractmethod def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: + attachment: Optional[Dict] = None, item_id: Union[Optional[str], Task[str]] = None) -> None: raise NotImplementedError('"log" method is not implemented!') def start(self) -> None: @@ -598,7 +662,7 @@ async def start_test_item(self, return item_id async def finish_test_item(self, - item_id: Union[asyncio.Task, str], + item_id: str, end_time: str, *, status: str = None, @@ -691,14 +755,14 @@ class ThreadedRPClient(RPClient): _item_stack: _LifoQueue __loop: Optional[asyncio.AbstractEventLoop] __thread: Optional[threading.Thread] - __task_list: List[asyncio.Task] + __task_list: List[Task[T]] self_loop: bool self_thread: bool - launch_uuid: Optional[asyncio.Task] + launch_uuid: Optional[Task[str]] use_own_launch: bool step_reporter: StepReporter - def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio.Task] = None, + def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, client: Optional[_AsyncRPClient] = None, loop: Optional[asyncio.AbstractEventLoop] = None, **kwargs: Any) -> None: set_current(self) @@ -723,7 +787,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio self.__loop = asyncio.new_event_loop() self.self_loop = True - def create_task(self, coro: Any) -> asyncio.Task: + def create_task(self, coro: Coroutine[Any, Any, T]) -> Task[T]: loop = self.__loop result = loop.create_task(coro) self.__task_list.append(result) @@ -760,7 +824,7 @@ def start_launch(self, attributes: Optional[Union[List, Dict]] = None, rerun: bool = False, rerun_of: Optional[str] = None, - **kwargs) -> asyncio.Task: + **kwargs) -> Task[str]: if not self.use_own_launch: return self.launch_uuid launch_uuid_coro = self.__client.start_launch(name, start_time, description=description, @@ -777,12 +841,12 @@ def start_test_item(self, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, parameters: Optional[Dict] = None, - parent_item_id: Optional[asyncio.Task] = None, + parent_item_id: Optional[Task[str]] = None, has_stats: bool = True, code_ref: Optional[str] = None, retry: bool = False, test_case_id: Optional[str] = None, - **kwargs: Any) -> asyncio.Task: + **kwargs: Any) -> Task[str]: item_id_coro = self.__client.start_test_item(self.launch_uuid, name, start_time, item_type, description=description, attributes=attributes, @@ -794,7 +858,7 @@ def start_test_item(self, return item_id_task def finish_test_item(self, - item_id: asyncio.Task, + item_id: Task[str], end_time: str, *, status: str = None, @@ -802,7 +866,7 @@ def finish_test_item(self, attributes: Optional[Union[List, Dict]] = None, description: str = None, retry: bool = False, - **kwargs: Any) -> asyncio.Task: + **kwargs: Any) -> Task[str]: result_coro = self.__client.finish_test_item(self.launch_uuid, item_id, end_time, status=status, issue=issue, attributes=attributes, description=description, @@ -815,7 +879,7 @@ def finish_launch(self, end_time: str, status: str = None, attributes: Optional[Union[List, Dict]] = None, - **kwargs: Any) -> asyncio.Task: + **kwargs: Any) -> Task[str]: if self.use_own_launch: result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, **kwargs) @@ -827,65 +891,65 @@ def finish_launch(self, return result_task def update_test_item(self, - item_uuid: asyncio.Task, + item_uuid: Task[str], attributes: Optional[Union[List, Dict]] = None, - description: Optional[str] = None) -> asyncio.Task: + description: Optional[str] = None) -> Task[str]: result_coro = self.__client.update_test_item(item_uuid, attributes=attributes, description=description) result_task = self.create_task(result_coro) return result_task - def _add_current_item(self, item: asyncio.Task) -> None: + def _add_current_item(self, item: Task[T]) -> None: """Add the last item from the self._items queue.""" self._item_stack.put(item) - def _remove_current_item(self) -> asyncio.Task: + def _remove_current_item(self) -> Task[T]: """Remove the last item from the self._items queue.""" return self._item_stack.get() + def current_item(self) -> Task[T]: + """Retrieve the last item reported by the client.""" + return self._item_stack.last() + async def __empty_dict(self): return {} async def __none_value(self): return - def current_item(self) -> asyncio.Task: - """Retrieve the last item reported by the client.""" - return self._item_stack.last() - - def get_launch_info(self) -> asyncio.Task: + def get_launch_info(self) -> Task[dict]: if not self.launch_uuid: return self.create_task(self.__empty_dict()) result_coro = self.__client.get_launch_info(self.launch_uuid) result_task = self.create_task(result_coro) return result_task - def get_item_id_by_uuid(self, item_uuid_future: asyncio.Task) -> asyncio.Task: + def get_item_id_by_uuid(self, item_uuid_future: Task[str]) -> Task[str]: result_coro = self.__client.get_item_id_by_uuid(item_uuid_future) result_task = self.create_task(result_coro) return result_task - def get_launch_ui_id(self) -> asyncio.Task: + def get_launch_ui_id(self) -> Task[str]: if not self.launch_uuid: return self.create_task(self.__none_value()) result_coro = self.__client.get_launch_ui_id(self.launch_uuid) result_task = self.create_task(result_coro) return result_task - def get_launch_ui_url(self) -> asyncio.Task: + def get_launch_ui_url(self) -> Task[str]: if not self.launch_uuid: return self.create_task(self.__none_value()) result_coro = self.__client.get_launch_ui_url(self.launch_uuid) result_task = self.create_task(result_coro) return result_task - def get_project_settings(self) -> asyncio.Task: + def get_project_settings(self) -> Task[dict]: result_coro = self.__client.get_project_settings() result_task = self.create_task(result_coro) return result_task def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: + attachment: Optional[Dict] = None, item_id: Optional[Task[str]] = None) -> None: # TODO: implement logging return None @@ -914,16 +978,16 @@ class BatchedRPClient(RPClient): __client: _AsyncRPClient _item_stack: _LifoQueue __loop: asyncio.AbstractEventLoop - __task_list: List[asyncio.Task] + __task_list: List[Task[T]] __task_mutex: threading.Lock __last_run_time: float - launch_uuid: Optional[asyncio.Task] + __thread: threading.Thread + launch_uuid: Optional[Task[str]] use_own_launch: bool step_reporter: StepReporter - def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio.Task] = None, - client: Optional[_AsyncRPClient] = None, loop: Optional[asyncio.AbstractEventLoop] = None, - **kwargs: Any) -> None: + def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, + client: Optional[_AsyncRPClient] = None, **kwargs: Any) -> None: set_current(self) self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() @@ -941,10 +1005,8 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[asyncio self.__task_mutex = threading.Lock() self.__last_run_time = time.time() self.__loop = asyncio.new_event_loop() - - @property - def loop(self): - return self.__loop + self.__thread = threading.current_thread() + self.__loop.set_task_factory(_BatchedTaskFactory(self.__loop, self.__thread)) def __ready_to_run(self) -> bool: current_time = time.time() @@ -956,7 +1018,7 @@ def __ready_to_run(self) -> bool: return True return False - def create_task(self, coro: Any) -> asyncio.Task: + def create_task(self, coro: Coroutine[Any, Any, T]) -> Task[T]: result = self.__loop.create_task(coro) tasks = None with self.__task_mutex: @@ -987,7 +1049,7 @@ def start_launch(self, attributes: Optional[Union[List, Dict]] = None, rerun: bool = False, rerun_of: Optional[str] = None, - **kwargs) -> asyncio.Task: + **kwargs) -> Task[str]: if not self.use_own_launch: return self.launch_uuid launch_uuid_coro = self.__client.start_launch(name, start_time, description=description, @@ -1004,12 +1066,12 @@ def start_test_item(self, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, parameters: Optional[Dict] = None, - parent_item_id: Optional[asyncio.Task] = None, + parent_item_id: Optional[Task[str]] = None, has_stats: bool = True, code_ref: Optional[str] = None, retry: bool = False, test_case_id: Optional[str] = None, - **kwargs: Any) -> asyncio.Task: + **kwargs: Any) -> Task[str]: item_id_coro = self.__client.start_test_item(self.launch_uuid, name, start_time, item_type, description=description, attributes=attributes, @@ -1021,7 +1083,7 @@ def start_test_item(self, return item_id_task def finish_test_item(self, - item_id: asyncio.Task, + item_id: Task[str], end_time: str, *, status: str = None, @@ -1029,7 +1091,7 @@ def finish_test_item(self, attributes: Optional[Union[List, Dict]] = None, description: str = None, retry: bool = False, - **kwargs: Any) -> asyncio.Task: + **kwargs: Any) -> Task[str]: result_coro = self.__client.finish_test_item(self.launch_uuid, item_id, end_time, status=status, issue=issue, attributes=attributes, description=description, @@ -1042,7 +1104,7 @@ def finish_launch(self, end_time: str, status: str = None, attributes: Optional[Union[List, Dict]] = None, - **kwargs: Any) -> asyncio.Task: + **kwargs: Any) -> Task[str]: if self.use_own_launch: result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, **kwargs) @@ -1054,65 +1116,65 @@ def finish_launch(self, return result_task def update_test_item(self, - item_uuid: asyncio.Task, + item_uuid: Task[str], attributes: Optional[Union[List, Dict]] = None, - description: Optional[str] = None) -> asyncio.Task: + description: Optional[str] = None) -> Task: result_coro = self.__client.update_test_item(item_uuid, attributes=attributes, description=description) result_task = self.create_task(result_coro) return result_task - def _add_current_item(self, item: asyncio.Task) -> None: + def _add_current_item(self, item: Task[T]) -> None: """Add the last item from the self._items queue.""" self._item_stack.put(item) - def _remove_current_item(self) -> asyncio.Task: + def _remove_current_item(self) -> Task[T]: """Remove the last item from the self._items queue.""" return self._item_stack.get() + def current_item(self) -> Task[T]: + """Retrieve the last item reported by the client.""" + return self._item_stack.last() + async def __empty_dict(self): return {} async def __none_value(self): return - def current_item(self) -> asyncio.Task: - """Retrieve the last item reported by the client.""" - return self._item_stack.last() - - def get_launch_info(self) -> asyncio.Task: + def get_launch_info(self) -> Task[dict]: if not self.launch_uuid: return self.create_task(self.__empty_dict()) result_coro = self.__client.get_launch_info(self.launch_uuid) result_task = self.create_task(result_coro) return result_task - def get_item_id_by_uuid(self, item_uuid_future: asyncio.Task) -> asyncio.Task: + def get_item_id_by_uuid(self, item_uuid_future: Task[str]) -> Task[str]: result_coro = self.__client.get_item_id_by_uuid(item_uuid_future) result_task = self.create_task(result_coro) return result_task - def get_launch_ui_id(self) -> asyncio.Task: + def get_launch_ui_id(self) -> Task[str]: if not self.launch_uuid: return self.create_task(self.__none_value()) result_coro = self.__client.get_launch_ui_id(self.launch_uuid) result_task = self.create_task(result_coro) return result_task - def get_launch_ui_url(self) -> asyncio.Task: + def get_launch_ui_url(self) -> Task[str]: if not self.launch_uuid: return self.create_task(self.__none_value()) result_coro = self.__client.get_launch_ui_url(self.launch_uuid) result_task = self.create_task(result_coro) return result_task - def get_project_settings(self) -> asyncio.Task: + def get_project_settings(self) -> Task[dict]: result_coro = self.__client.get_project_settings() result_task = self.create_task(result_coro) return result_task def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: + attachment: Optional[Dict] = None, item_id: Union[Optional[str], Task[str]] = None) -> None: # TODO: implement logging return None From 868da8195c29d4f5d1c1d4937b79a8a57bab6f2c Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 12 Sep 2023 16:43:06 +0300 Subject: [PATCH 051/268] Fix typing --- reportportal_client/aio/client.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 8a2017a6..40c26719 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -814,7 +814,7 @@ def finish_tasks(self): break self.__task_list = [] - async def __empty_line(self): + async def __empty_str(self): return "" def start_launch(self, @@ -884,7 +884,7 @@ def finish_launch(self, result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, **kwargs) else: - result_coro = self.create_task(self.__empty_line()) + result_coro = self.create_task(self.__empty_str()) result_task = self.create_task(result_coro) self.finish_tasks() @@ -914,8 +914,8 @@ def current_item(self) -> Task[T]: async def __empty_dict(self): return {} - async def __none_value(self): - return + async def __int_value(self): + return -1 def get_launch_info(self) -> Task[dict]: if not self.launch_uuid: @@ -929,16 +929,16 @@ def get_item_id_by_uuid(self, item_uuid_future: Task[str]) -> Task[str]: result_task = self.create_task(result_coro) return result_task - def get_launch_ui_id(self) -> Task[str]: + def get_launch_ui_id(self) -> Task[int]: if not self.launch_uuid: - return self.create_task(self.__none_value()) + return self.create_task(self.__int_value()) result_coro = self.__client.get_launch_ui_id(self.launch_uuid) result_task = self.create_task(result_coro) return result_task def get_launch_ui_url(self) -> Task[str]: if not self.launch_uuid: - return self.create_task(self.__none_value()) + return self.create_task(self.__empty_str()) result_coro = self.__client.get_launch_ui_url(self.launch_uuid) result_task = self.create_task(result_coro) return result_task @@ -1039,7 +1039,7 @@ def finish_tasks(self) -> None: if tasks: self.__loop.run_until_complete(asyncio.gather(*tasks)) - async def __empty_line(self): + async def __empty_str(self): return "" def start_launch(self, @@ -1109,7 +1109,7 @@ def finish_launch(self, result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, **kwargs) else: - result_coro = self.create_task(self.__empty_line()) + result_coro = self.create_task(self.__empty_str()) result_task = self.create_task(result_coro) self.finish_tasks() @@ -1139,8 +1139,8 @@ def current_item(self) -> Task[T]: async def __empty_dict(self): return {} - async def __none_value(self): - return + async def __int_value(self): + return -1 def get_launch_info(self) -> Task[dict]: if not self.launch_uuid: @@ -1154,16 +1154,16 @@ def get_item_id_by_uuid(self, item_uuid_future: Task[str]) -> Task[str]: result_task = self.create_task(result_coro) return result_task - def get_launch_ui_id(self) -> Task[str]: + def get_launch_ui_id(self) -> Task[int]: if not self.launch_uuid: - return self.create_task(self.__none_value()) + return self.create_task(self.__int_value()) result_coro = self.__client.get_launch_ui_id(self.launch_uuid) result_task = self.create_task(result_coro) return result_task def get_launch_ui_url(self) -> Task[str]: if not self.launch_uuid: - return self.create_task(self.__none_value()) + return self.create_task(self.__empty_str()) result_coro = self.__client.get_launch_ui_url(self.launch_uuid) result_task = self.create_task(result_coro) return result_task From d6a487021b225a46cab4d1c6cdd2408edaa1e91a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 12 Sep 2023 16:51:32 +0300 Subject: [PATCH 052/268] Refactoring --- reportportal_client/aio/__init__.py | 8 +++ reportportal_client/aio/client.py | 89 +++++------------------------ reportportal_client/aio/tasks.py | 82 ++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 75 deletions(-) create mode 100644 reportportal_client/aio/tasks.py diff --git a/reportportal_client/aio/__init__.py b/reportportal_client/aio/__init__.py index 1c768643..6c63bc5d 100644 --- a/reportportal_client/aio/__init__.py +++ b/reportportal_client/aio/__init__.py @@ -10,3 +10,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License + +from reportportal_client.aio.tasks import Task, BatchedTask, BatchedTaskFactory + +__all__ = [ + 'Task', + 'BatchedTask', + 'BatchedTaskFactory' +] \ No newline at end of file diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 40c26719..2642edfa 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -20,17 +20,16 @@ import threading import time import warnings -from asyncio import Future from os import getenv from queue import LifoQueue -from typing import Union, Tuple, List, Dict, Any, Optional, TextIO, Coroutine, TypeVar, Generic, Generator, \ - Awaitable +from typing import Union, Tuple, List, Dict, Any, Optional, TextIO, Coroutine, TypeVar import aiohttp import certifi # noinspection PyProtectedMember from reportportal_client._local import set_current +from reportportal_client.aio import Task, BatchedTaskFactory from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, AsyncItemFinishRequest, LaunchFinishRequest) @@ -53,67 +52,7 @@ TASK_TIMEOUT: int = 60 SHUTDOWN_TIMEOUT: int = 120 - -T = TypeVar('T') - - -class Task(Generic[T], asyncio.Task, metaclass=AbstractBaseClass): - __metaclass__ = AbstractBaseClass - - def __init__( - self, - coro: Union[Generator[Future[object], None, T], Awaitable[T]], - *, - loop: asyncio.AbstractEventLoop, - name: Optional[str] = None - ) -> None: - super().__init__(coro, loop=loop, name=name) - - @abstractmethod - def blocking_result(self) -> T: - raise NotImplementedError('"blocking_result" method is not implemented!') - - -class BatchedTask(Generic[T], Task[T]): - __loop: asyncio.AbstractEventLoop - __thread: threading.Thread - - def __init__( - self, - coro: Union[Generator[Future[object], None, T], Awaitable[T]], - *, - loop: asyncio.AbstractEventLoop, - name: Optional[str] = None, - thread: threading.Thread - ) -> None: - super().__init__(coro, loop=loop, name=name) - self.__loop = loop - self.__thread = thread - - def blocking_result(self) -> T: - if self.done(): - return self.result() - if self.__thread is not threading.current_thread(): - warnings.warn("The method was called from different thread which was used to create the" - "task, unexpected behavior is possible during the execution.", RuntimeWarning, - stacklevel=3) - return self.__loop.run_until_complete(self) - - -class _BatchedTaskFactory: - __loop: asyncio.AbstractEventLoop - __thread: threading.Thread - - def __init__(self, loop: asyncio.AbstractEventLoop, thread: threading.Thread): - self.__loop = loop - self.__thread = thread - - def __call__( - self, - loop: asyncio.AbstractEventLoop, - factory: Union[Coroutine[Any, Any, T], Generator[Any, None, T]] - ) -> Task[T]: - return BatchedTask(factory, loop=self.__loop, thread=self.__thread) +_T = TypeVar('_T') class _LifoQueue(LifoQueue): @@ -755,7 +694,7 @@ class ThreadedRPClient(RPClient): _item_stack: _LifoQueue __loop: Optional[asyncio.AbstractEventLoop] __thread: Optional[threading.Thread] - __task_list: List[Task[T]] + __task_list: List[Task[_T]] self_loop: bool self_thread: bool launch_uuid: Optional[Task[str]] @@ -787,7 +726,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self.__loop = asyncio.new_event_loop() self.self_loop = True - def create_task(self, coro: Coroutine[Any, Any, T]) -> Task[T]: + def create_task(self, coro: Coroutine[Any, Any, _T]) -> Task[_T]: loop = self.__loop result = loop.create_task(coro) self.__task_list.append(result) @@ -899,15 +838,15 @@ def update_test_item(self, result_task = self.create_task(result_coro) return result_task - def _add_current_item(self, item: Task[T]) -> None: + def _add_current_item(self, item: Task[_T]) -> None: """Add the last item from the self._items queue.""" self._item_stack.put(item) - def _remove_current_item(self) -> Task[T]: + def _remove_current_item(self) -> Task[_T]: """Remove the last item from the self._items queue.""" return self._item_stack.get() - def current_item(self) -> Task[T]: + def current_item(self) -> Task[_T]: """Retrieve the last item reported by the client.""" return self._item_stack.last() @@ -978,7 +917,7 @@ class BatchedRPClient(RPClient): __client: _AsyncRPClient _item_stack: _LifoQueue __loop: asyncio.AbstractEventLoop - __task_list: List[Task[T]] + __task_list: List[Task[_T]] __task_mutex: threading.Lock __last_run_time: float __thread: threading.Thread @@ -1006,7 +945,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self.__last_run_time = time.time() self.__loop = asyncio.new_event_loop() self.__thread = threading.current_thread() - self.__loop.set_task_factory(_BatchedTaskFactory(self.__loop, self.__thread)) + self.__loop.set_task_factory(BatchedTaskFactory(self.__loop, self.__thread)) def __ready_to_run(self) -> bool: current_time = time.time() @@ -1018,7 +957,7 @@ def __ready_to_run(self) -> bool: return True return False - def create_task(self, coro: Coroutine[Any, Any, T]) -> Task[T]: + def create_task(self, coro: Coroutine[Any, Any, _T]) -> Task[_T]: result = self.__loop.create_task(coro) tasks = None with self.__task_mutex: @@ -1124,15 +1063,15 @@ def update_test_item(self, result_task = self.create_task(result_coro) return result_task - def _add_current_item(self, item: Task[T]) -> None: + def _add_current_item(self, item: Task[_T]) -> None: """Add the last item from the self._items queue.""" self._item_stack.put(item) - def _remove_current_item(self) -> Task[T]: + def _remove_current_item(self) -> Task[_T]: """Remove the last item from the self._items queue.""" return self._item_stack.get() - def current_item(self) -> Task[T]: + def current_item(self) -> Task[_T]: """Retrieve the last item reported by the client.""" return self._item_stack.last() diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py new file mode 100644 index 00000000..67d1f3f4 --- /dev/null +++ b/reportportal_client/aio/tasks.py @@ -0,0 +1,82 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import asyncio +import threading +import warnings +from abc import abstractmethod +from asyncio import Future +from typing import TypeVar, Generic, Union, Generator, Awaitable, Optional, Coroutine, Any + +from reportportal_client.static.abstract import AbstractBaseClass + +_T = TypeVar('_T') + + +class Task(Generic[_T], asyncio.Task, metaclass=AbstractBaseClass): + __metaclass__ = AbstractBaseClass + + def __init__( + self, + coro: Union[Generator[Future[object], None, _T], Awaitable[_T]], + *, + loop: asyncio.AbstractEventLoop, + name: Optional[str] = None + ) -> None: + super().__init__(coro, loop=loop, name=name) + + @abstractmethod + def blocking_result(self) -> _T: + raise NotImplementedError('"blocking_result" method is not implemented!') + + +class BatchedTask(Generic[_T], Task[_T]): + __loop: asyncio.AbstractEventLoop + __thread: threading.Thread + + def __init__( + self, + coro: Union[Generator[Future[object], None, _T], Awaitable[_T]], + *, + loop: asyncio.AbstractEventLoop, + name: Optional[str] = None, + thread: threading.Thread + ) -> None: + super().__init__(coro, loop=loop, name=name) + self.__loop = loop + self.__thread = thread + + def blocking_result(self) -> _T: + if self.done(): + return self.result() + if self.__thread is not threading.current_thread(): + warnings.warn("The method was called from different thread which was used to create the" + "task, unexpected behavior is possible during the execution.", RuntimeWarning, + stacklevel=3) + return self.__loop.run_until_complete(self) + + +class BatchedTaskFactory: + __loop: asyncio.AbstractEventLoop + __thread: threading.Thread + + def __init__(self, loop: asyncio.AbstractEventLoop, thread: threading.Thread): + self.__loop = loop + self.__thread = thread + + def __call__( + self, + loop: asyncio.AbstractEventLoop, + factory: Union[Coroutine[Any, Any, _T], Generator[Any, None, _T]] + ) -> Task[_T]: + return BatchedTask(factory, loop=self.__loop, thread=self.__thread) From a2111b796eb8ec87fc75df4fbc6e67fceb120bed Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 13 Sep 2023 16:26:07 +0300 Subject: [PATCH 053/268] Add batch logging async method --- reportportal_client/aio/client.py | 41 +++++++++++------------- reportportal_client/core/rp_requests.py | 33 ++++++++----------- reportportal_client/core/rp_responses.py | 28 +++++++++++++--- reportportal_client/helpers.py | 2 +- 4 files changed, 57 insertions(+), 47 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 2642edfa..eb55f932 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -32,7 +32,8 @@ from reportportal_client.aio import Task, BatchedTaskFactory from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, - AsyncItemFinishRequest, LaunchFinishRequest) + AsyncItemFinishRequest, LaunchFinishRequest, + AsyncRPRequestLog, AsyncRPLogBatch) from reportportal_client.helpers import (root_uri_join, verify_value_length, await_if_necessary, agent_name_version) from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE @@ -417,15 +418,10 @@ async def get_project_settings(self) -> Optional[Dict]: response = await AsyncHttpRequest(self.session.get, url=url).make() return await response.json if response else None - async def log(self, - launch_uuid: Union[str, Task[str]], - time: str, - message: str, - *, - level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, - item_id: Optional[Union[str, Task[str]]] = None) -> None: - pass + async def log_batch(self, log_batch: List[AsyncRPRequestLog]) -> Tuple[str, ...]: + url = root_uri_join(self.base_url_v2, 'log') + response = await AsyncHttpRequest(self.session.post, url=url, data=AsyncRPLogBatch(log_batch)).make() + return await response.messages def clone(self) -> '_AsyncRPClient': """Clone the client object, set current Item ID as cloned item ID. @@ -527,7 +523,7 @@ def get_project_settings(self) -> Optional[Dict]: raise NotImplementedError('"get_project_settings" method is not implemented!') @abstractmethod - def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, + def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, item_id: Union[Optional[str], Task[str]] = None) -> None: raise NotImplementedError('"log" method is not implemented!') @@ -665,7 +661,7 @@ async def get_launch_ui_url(self) -> Optional[str]: async def get_project_settings(self) -> Optional[Dict]: return await self.__client.get_project_settings() - async def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, + async def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: return @@ -689,7 +685,14 @@ def clone(self) -> 'AsyncRPClient': return cloned -class ThreadedRPClient(RPClient): +class _SyncRPClient(RPClient, metaclass=AbstractBaseClass): + __metaclass__ = AbstractBaseClass + + async def __empty_str(self): + return "" + + +class ThreadedRPClient(_SyncRPClient): __client: _AsyncRPClient _item_stack: _LifoQueue __loop: Optional[asyncio.AbstractEventLoop] @@ -753,9 +756,6 @@ def finish_tasks(self): break self.__task_list = [] - async def __empty_str(self): - return "" - def start_launch(self, name: str, start_time: str, @@ -887,7 +887,7 @@ def get_project_settings(self) -> Task[dict]: result_task = self.create_task(result_coro) return result_task - def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, + def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, item_id: Optional[Task[str]] = None) -> None: # TODO: implement logging return None @@ -913,7 +913,7 @@ def clone(self) -> 'ThreadedRPClient': return cloned -class BatchedRPClient(RPClient): +class BatchedRPClient(_SyncRPClient): __client: _AsyncRPClient _item_stack: _LifoQueue __loop: asyncio.AbstractEventLoop @@ -978,9 +978,6 @@ def finish_tasks(self) -> None: if tasks: self.__loop.run_until_complete(asyncio.gather(*tasks)) - async def __empty_str(self): - return "" - def start_launch(self, name: str, start_time: str, @@ -1112,7 +1109,7 @@ def get_project_settings(self) -> Task[dict]: result_task = self.create_task(result_coro) return result_task - def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, + def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, item_id: Union[Optional[str], Task[str]] = None) -> None: # TODO: implement logging return None diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 721ce983..8077423f 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -373,12 +373,16 @@ def payload(self) -> dict: """Get HTTP payload for the request.""" return RPRequestLog.create_request(**self.__dict__) + @staticmethod + def _multipart_size(payload: dict, file: Optional[RPFile]): + size = helpers.calculate_json_part_size(payload) + size += helpers.calculate_file_part_size(file) + return size + @property def multipart_size(self) -> int: """Calculate request size how it would transfer in Multipart HTTP.""" - size = helpers.calculate_json_part_size(self.payload) - size += helpers.calculate_file_part_size(self.file) - return size + return RPRequestLog._multipart_size(self.payload, self.file) class AsyncRPRequestLog(RPRequestLog): @@ -396,6 +400,11 @@ async def payload(self) -> dict: data['item_uuid'] = uuids[1] return RPRequestLog.create_request(**data) + @property + async def multipart_size(self) -> int: + """Calculate request size how it would transfer in Multipart HTTP.""" + return RPRequestLog._multipart_size(await self.payload, self.file) + class RPLogBatch(RPRequestBase): """RP log save batches with attachments request model. @@ -475,23 +484,7 @@ async def __get_request_part(self) -> str: @property async def payload(self) -> aiohttp.MultipartWriter: - r"""Get HTTP payload for the request. - - Example: - [('json_request_part', - (None, - '[{"launchUuid": "bf6edb74-b092-4b32-993a-29967904a5b4", - "time": "1588936537081", - "message": "Html report", - "level": "INFO", - "itemUuid": "d9dc2514-2c78-4c4f-9369-ee4bca4c78f8", - "file": {"name": "Detailed report"}}]', - 'application/json')), - ('file', - ('Detailed report', - '\n

Paragraph

', - 'text/html'))] - """ + """Get HTTP payload for the request.""" json_payload = aiohttp.Payload(await self.__get_request_part(), content_type='application/json') json_payload.set_content_disposition('form-data', name='json_request_part') mpwriter = aiohttp.MultipartWriter('form-data') diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index 9318300e..b71d9b9c 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -19,7 +19,7 @@ # limitations under the License import logging -from typing import Any, Optional +from typing import Any, Optional, Generator, Mapping, Tuple, Protocol from aiohttp import ClientResponse from requests import Response @@ -29,7 +29,17 @@ logger = logging.getLogger(__name__) -class RPResponse: +def _iter_json_messages(json: Any) -> Generator[str]: + if not isinstance(json, Mapping): + return + data = json.get('responses', [json]) + for chunk in data: + message = chunk.get('message', chunk.get('error_code')) + if message: + yield message + + +class RPResponse(Protocol): """Class representing RP API response.""" _resp: Response __json: Any @@ -64,13 +74,18 @@ def message(self) -> Optional[str]: """Get value of the 'message' key.""" return self.json.get('message') + @property + def messages(self) -> Tuple[str, ...]: + """Get list of messages received.""" + return tuple(_iter_json_messages(self.json)) + class AsyncRPResponse: """Class representing RP API response.""" - _resp: Response + _resp: ClientResponse __json: Any - def __init__(self, data: Response) -> None: + def __init__(self, data: ClientResponse) -> None: """Initialize instance attributes. :param data: requests.Response object @@ -99,3 +114,8 @@ async def json(self) -> Any: async def message(self) -> Optional[str]: """Get value of the 'message' key.""" return (await self.json).get('message') + + @property + async def messages(self) -> Tuple[str, ...]: + """Get list of messages received.""" + return tuple(_iter_json_messages(await self.json)) diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index c1dcbefc..d0c2b68e 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -225,7 +225,7 @@ def calculate_json_part_size(json_dict: dict) -> int: return size -def calculate_file_part_size(file: RPFile) -> int: +def calculate_file_part_size(file: Optional[RPFile]) -> int: """Predict a file part size of Multipart request. :param file: RPFile class instance From 4463ba219cd5f3a4b77d541464653aa3a2fde9be Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 13 Sep 2023 16:41:19 +0300 Subject: [PATCH 054/268] Add launch_id getter for backward compatibility --- reportportal_client/aio/client.py | 45 ++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index eb55f932..37cd835b 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -533,11 +533,26 @@ def start(self) -> None: def terminate(self, *_: Any, **__: Any) -> None: pass # For backward compatibility + @abstractmethod + @property + def launch_uuid(self) -> Optional[Union[str, Task[str]]]: + raise NotImplementedError('"launch_uuid" property is not implemented!') + + @property + def launch_id(self) -> Optional[Union[str, Task[str]]]: + warnings.warn( + message='`launch_id` property is deprecated since 5.5.0 and will be subject for removing in the' + ' next major version. Use `launch_uuid` property instead.', + category=DeprecationWarning, + stacklevel=2 + ) + return self.launch_uuid + class AsyncRPClient(RPClient): __client: _AsyncRPClient _item_stack: _LifoQueue - launch_uuid: Optional[str] + __launch_uuid: Optional[str] use_own_launch: bool step_reporter: StepReporter @@ -665,6 +680,14 @@ async def log(self, datetime: str, message: str, level: Optional[Union[int, str] attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: return + @property + def launch_uuid(self) -> Optional[str]: + return self.__launch_uuid + + @launch_uuid.setter + def launch_uuid(self, value: Optional[str]) -> None: + self.__launch_uuid = value + def clone(self) -> 'AsyncRPClient': """Clone the client object, set current Item ID as cloned item ID. @@ -700,7 +723,7 @@ class ThreadedRPClient(_SyncRPClient): __task_list: List[Task[_T]] self_loop: bool self_thread: bool - launch_uuid: Optional[Task[str]] + __launch_uuid: Optional[Task[str]] use_own_launch: bool step_reporter: StepReporter @@ -892,6 +915,14 @@ def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = No # TODO: implement logging return None + @property + def launch_uuid(self) -> Optional[Task[str]]: + return self.__launch_uuid + + @launch_uuid.setter + def launch_uuid(self, value: Optional[Task[str]]) -> None: + self.__launch_uuid = value + def clone(self) -> 'ThreadedRPClient': """Clone the client object, set current Item ID as cloned item ID. @@ -921,7 +952,7 @@ class BatchedRPClient(_SyncRPClient): __task_mutex: threading.Lock __last_run_time: float __thread: threading.Thread - launch_uuid: Optional[Task[str]] + __launch_uuid: Optional[Task[str]] use_own_launch: bool step_reporter: StepReporter @@ -1114,6 +1145,14 @@ def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = No # TODO: implement logging return None + @property + def launch_uuid(self) -> Optional[Task[str]]: + return self.__launch_uuid + + @launch_uuid.setter + def launch_uuid(self, value: Optional[Task[str]]) -> None: + self.__launch_uuid = value + def clone(self) -> 'BatchedRPClient': """Clone the client object, set current Item ID as cloned item ID. From 44460dd7ea9c0164b7cecdf376c76f067831c6a4 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 13 Sep 2023 17:50:31 +0300 Subject: [PATCH 055/268] Refactoring: move common logic for Threaded and Batched clients to abstract class --- reportportal_client/aio/client.py | 323 +++++++++--------------------- 1 file changed, 91 insertions(+), 232 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 37cd835b..3dd1de67 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -548,6 +548,10 @@ def launch_id(self) -> Optional[Union[str, Task[str]]]: ) return self.launch_uuid + @abstractmethod + def clone(self) -> 'RPClient': + raise NotImplementedError('"clone" method is not implemented!') + class AsyncRPClient(RPClient): __client: _AsyncRPClient @@ -711,25 +715,21 @@ def clone(self) -> 'AsyncRPClient': class _SyncRPClient(RPClient, metaclass=AbstractBaseClass): __metaclass__ = AbstractBaseClass - async def __empty_str(self): - return "" - - -class ThreadedRPClient(_SyncRPClient): - __client: _AsyncRPClient _item_stack: _LifoQueue - __loop: Optional[asyncio.AbstractEventLoop] - __thread: Optional[threading.Thread] - __task_list: List[Task[_T]] - self_loop: bool - self_thread: bool __launch_uuid: Optional[Task[str]] use_own_launch: bool step_reporter: StepReporter + @property + def launch_uuid(self) -> Optional[Task[str]]: + return self.__launch_uuid + + @launch_uuid.setter + def launch_uuid(self, value: Optional[Task[str]]) -> None: + self.__launch_uuid = value + def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, - client: Optional[_AsyncRPClient] = None, loop: Optional[asyncio.AbstractEventLoop] = None, - **kwargs: Any) -> None: + client: Optional[_AsyncRPClient] = None, **kwargs: Any) -> None: set_current(self) self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() @@ -743,41 +743,34 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st else: self.use_own_launch = True - self.__task_list = [] - self.__thread = None - if loop: - self.__loop = loop - self.self_loop = False - else: - self.__loop = asyncio.new_event_loop() - self.self_loop = True - + @abstractmethod def create_task(self, coro: Coroutine[Any, Any, _T]) -> Task[_T]: - loop = self.__loop - result = loop.create_task(coro) - self.__task_list.append(result) - if not self.__thread and self.self_loop: - self.__thread = threading.Thread(target=loop.run_forever, name='RP-Async-Client', - daemon=True) - self.__thread.start() - i = 0 - for i, task in enumerate(self.__task_list): - if not task.done(): - break - self.__task_list = self.__task_list[i:] - return result + raise NotImplementedError('"create_task" method is not implemented!') - def finish_tasks(self): - sleep_time = sys.getswitchinterval() - shutdown_start_time = time.time() - for task in self.__task_list: - task_start_time = time.time() - while not task.done() and (time.time() - task_start_time < TASK_TIMEOUT) and ( - time.time() - shutdown_start_time < SHUTDOWN_TIMEOUT): - time.sleep(sleep_time) - if time.time() - shutdown_start_time >= SHUTDOWN_TIMEOUT: - break - self.__task_list = [] + @abstractmethod + def finish_tasks(self) -> None: + raise NotImplementedError('"create_task" method is not implemented!') + + def _add_current_item(self, item: Task[_T]) -> None: + """Add the last item from the self._items queue.""" + self._item_stack.put(item) + + def _remove_current_item(self) -> Task[_T]: + """Remove the last item from the self._items queue.""" + return self._item_stack.get() + + def current_item(self) -> Task[_T]: + """Retrieve the last item reported by the client.""" + return self._item_stack.last() + + async def __empty_str(self): + return "" + + async def __empty_dict(self): + return {} + + async def __int_value(self): + return -1 def start_launch(self, name: str, @@ -855,30 +848,12 @@ def finish_launch(self, def update_test_item(self, item_uuid: Task[str], attributes: Optional[Union[List, Dict]] = None, - description: Optional[str] = None) -> Task[str]: + description: Optional[str] = None) -> Task: result_coro = self.__client.update_test_item(item_uuid, attributes=attributes, description=description) result_task = self.create_task(result_coro) return result_task - def _add_current_item(self, item: Task[_T]) -> None: - """Add the last item from the self._items queue.""" - self._item_stack.put(item) - - def _remove_current_item(self) -> Task[_T]: - """Remove the last item from the self._items queue.""" - return self._item_stack.get() - - def current_item(self) -> Task[_T]: - """Retrieve the last item reported by the client.""" - return self._item_stack.last() - - async def __empty_dict(self): - return {} - - async def __int_value(self): - return -1 - def get_launch_info(self) -> Task[dict]: if not self.launch_uuid: return self.create_task(self.__empty_dict()) @@ -915,13 +890,57 @@ def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = No # TODO: implement logging return None - @property - def launch_uuid(self) -> Optional[Task[str]]: - return self.__launch_uuid - @launch_uuid.setter - def launch_uuid(self, value: Optional[Task[str]]) -> None: - self.__launch_uuid = value +class ThreadedRPClient(_SyncRPClient): + __task_list: List[Task[_T]] + __task_mutex: threading.Lock + __loop: Optional[asyncio.AbstractEventLoop] + __thread: Optional[threading.Thread] + self_loop: bool + self_thread: bool + + def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, + client: Optional[_AsyncRPClient] = None, loop: Optional[asyncio.AbstractEventLoop] = None, + **kwargs: Any) -> None: + super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, **kwargs) + self.__task_list = [] + self.__task_mutex = threading.Lock() + self.__thread = None + if loop: + self.__loop = loop + self.self_loop = False + else: + self.__loop = asyncio.new_event_loop() + self.self_loop = True + + def create_task(self, coro: Coroutine[Any, Any, _T]) -> Task[_T]: + loop = self.__loop + result = loop.create_task(coro) + with self.__task_mutex: + self.__task_list.append(result) + if not self.__thread and self.self_loop: + self.__thread = threading.Thread(target=loop.run_forever, name='RP-Async-Client', + daemon=True) + self.__thread.start() + i = 0 + for i, task in enumerate(self.__task_list): + if not task.done(): + break + self.__task_list = self.__task_list[i:] + return result + + def finish_tasks(self): + sleep_time = sys.getswitchinterval() + shutdown_start_time = time.time() + with self.__task_mutex: + for task in self.__task_list: + task_start_time = time.time() + while not task.done() and (time.time() - task_start_time < TASK_TIMEOUT) and ( + time.time() - shutdown_start_time < SHUTDOWN_TIMEOUT): + time.sleep(sleep_time) + if time.time() - shutdown_start_time >= SHUTDOWN_TIMEOUT: + break + self.__task_list = [] def clone(self) -> 'ThreadedRPClient': """Clone the client object, set current Item ID as cloned item ID. @@ -945,31 +964,15 @@ def clone(self) -> 'ThreadedRPClient': class BatchedRPClient(_SyncRPClient): - __client: _AsyncRPClient - _item_stack: _LifoQueue __loop: asyncio.AbstractEventLoop __task_list: List[Task[_T]] __task_mutex: threading.Lock __last_run_time: float __thread: threading.Thread - __launch_uuid: Optional[Task[str]] - use_own_launch: bool - step_reporter: StepReporter def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, client: Optional[_AsyncRPClient] = None, **kwargs: Any) -> None: - set_current(self) - self.step_reporter = StepReporter(self) - self._item_stack = _LifoQueue() - if client: - self.__client = client - else: - self.__client = _AsyncRPClient(endpoint, project, **kwargs) - if launch_uuid: - self.launch_uuid = launch_uuid - self.use_own_launch = False - else: - self.use_own_launch = True + super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, **kwargs) self.__task_list = [] self.__task_mutex = threading.Lock() @@ -1009,150 +1012,6 @@ def finish_tasks(self) -> None: if tasks: self.__loop.run_until_complete(asyncio.gather(*tasks)) - def start_launch(self, - name: str, - start_time: str, - description: Optional[str] = None, - attributes: Optional[Union[List, Dict]] = None, - rerun: bool = False, - rerun_of: Optional[str] = None, - **kwargs) -> Task[str]: - if not self.use_own_launch: - return self.launch_uuid - launch_uuid_coro = self.__client.start_launch(name, start_time, description=description, - attributes=attributes, rerun=rerun, rerun_of=rerun_of, - **kwargs) - self.launch_uuid = self.create_task(launch_uuid_coro) - return self.launch_uuid - - def start_test_item(self, - name: str, - start_time: str, - item_type: str, - *, - description: Optional[str] = None, - attributes: Optional[List[Dict]] = None, - parameters: Optional[Dict] = None, - parent_item_id: Optional[Task[str]] = None, - has_stats: bool = True, - code_ref: Optional[str] = None, - retry: bool = False, - test_case_id: Optional[str] = None, - **kwargs: Any) -> Task[str]: - - item_id_coro = self.__client.start_test_item(self.launch_uuid, name, start_time, item_type, - description=description, attributes=attributes, - parameters=parameters, parent_item_id=parent_item_id, - has_stats=has_stats, code_ref=code_ref, retry=retry, - test_case_id=test_case_id, **kwargs) - item_id_task = self.create_task(item_id_coro) - self._add_current_item(item_id_task) - return item_id_task - - def finish_test_item(self, - item_id: Task[str], - end_time: str, - *, - status: str = None, - issue: Optional[Issue] = None, - attributes: Optional[Union[List, Dict]] = None, - description: str = None, - retry: bool = False, - **kwargs: Any) -> Task[str]: - result_coro = self.__client.finish_test_item(self.launch_uuid, item_id, end_time, status=status, - issue=issue, attributes=attributes, - description=description, - retry=retry, **kwargs) - result_task = self.create_task(result_coro) - self._remove_current_item() - return result_task - - def finish_launch(self, - end_time: str, - status: str = None, - attributes: Optional[Union[List, Dict]] = None, - **kwargs: Any) -> Task[str]: - if self.use_own_launch: - result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, - attributes=attributes, **kwargs) - else: - result_coro = self.create_task(self.__empty_str()) - - result_task = self.create_task(result_coro) - self.finish_tasks() - return result_task - - def update_test_item(self, - item_uuid: Task[str], - attributes: Optional[Union[List, Dict]] = None, - description: Optional[str] = None) -> Task: - result_coro = self.__client.update_test_item(item_uuid, attributes=attributes, - description=description) - result_task = self.create_task(result_coro) - return result_task - - def _add_current_item(self, item: Task[_T]) -> None: - """Add the last item from the self._items queue.""" - self._item_stack.put(item) - - def _remove_current_item(self) -> Task[_T]: - """Remove the last item from the self._items queue.""" - return self._item_stack.get() - - def current_item(self) -> Task[_T]: - """Retrieve the last item reported by the client.""" - return self._item_stack.last() - - async def __empty_dict(self): - return {} - - async def __int_value(self): - return -1 - - def get_launch_info(self) -> Task[dict]: - if not self.launch_uuid: - return self.create_task(self.__empty_dict()) - result_coro = self.__client.get_launch_info(self.launch_uuid) - result_task = self.create_task(result_coro) - return result_task - - def get_item_id_by_uuid(self, item_uuid_future: Task[str]) -> Task[str]: - result_coro = self.__client.get_item_id_by_uuid(item_uuid_future) - result_task = self.create_task(result_coro) - return result_task - - def get_launch_ui_id(self) -> Task[int]: - if not self.launch_uuid: - return self.create_task(self.__int_value()) - result_coro = self.__client.get_launch_ui_id(self.launch_uuid) - result_task = self.create_task(result_coro) - return result_task - - def get_launch_ui_url(self) -> Task[str]: - if not self.launch_uuid: - return self.create_task(self.__empty_str()) - result_coro = self.__client.get_launch_ui_url(self.launch_uuid) - result_task = self.create_task(result_coro) - return result_task - - def get_project_settings(self) -> Task[dict]: - result_coro = self.__client.get_project_settings() - result_task = self.create_task(result_coro) - return result_task - - def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, item_id: Union[Optional[str], Task[str]] = None) -> None: - # TODO: implement logging - return None - - @property - def launch_uuid(self) -> Optional[Task[str]]: - return self.__launch_uuid - - @launch_uuid.setter - def launch_uuid(self, value: Optional[Task[str]]) -> None: - self.__launch_uuid = value - def clone(self) -> 'BatchedRPClient': """Clone the client object, set current Item ID as cloned item ID. From c8ff14ff8b71fc2391018a0d6882c9d91871a26b Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 13 Sep 2023 17:51:48 +0300 Subject: [PATCH 056/268] Add TODO --- reportportal_client/aio/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 3dd1de67..e6b28d1d 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -682,6 +682,7 @@ async def get_project_settings(self) -> Optional[Dict]: async def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: + # TODO: implement logging return @property From 37f1c07f2cd7a46a4020e7136e470f27a49dcf08 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 13 Sep 2023 17:57:38 +0300 Subject: [PATCH 057/268] Fix some typing --- reportportal_client/aio/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index e6b28d1d..883ba873 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -83,7 +83,7 @@ class _AsyncRPClient: print_output: Optional[TextIO] _skip_analytics: str __session: Optional[aiohttp.ClientSession] - __stat_task: Optional[Task[aiohttp.ClientResponse]] + __stat_task: Optional[asyncio.Task[aiohttp.ClientResponse]] def __init__( self, @@ -96,7 +96,6 @@ def __init__( verify_ssl: Union[bool, str] = True, retries: int = None, max_pool_size: int = 50, - http_timeout: Union[float, Tuple[float, float]] = (10, 10), log_batch_payload_size: int = MAX_LOG_BATCH_PAYLOAD_SIZE, mode: str = 'DEFAULT', From 11dbee10d0a16ed6f1d3c18a7610af04bbc106b1 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 12:06:07 +0300 Subject: [PATCH 058/268] Refactoring: move common interface to root --- reportportal_client/__init__.py | 3 +- reportportal_client/aio/client.py | 117 +--------------- reportportal_client/client.py | 223 ++++++++++++++++++++++++------ 3 files changed, 187 insertions(+), 156 deletions(-) diff --git a/reportportal_client/__init__.py b/reportportal_client/__init__.py index 7b12f8c4..d1b6a311 100644 --- a/reportportal_client/__init__.py +++ b/reportportal_client/__init__.py @@ -17,11 +17,12 @@ # noinspection PyProtectedMember from reportportal_client._local import current from reportportal_client.logs import RPLogger, RPLogHandler -from reportportal_client.client import RPClient +from reportportal_client.client import RP, RPClient from reportportal_client.steps import step __all__ = [ 'current', + 'RP', 'RPLogger', 'RPLogHandler', 'RPClient', diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 883ba873..4c3fb1ef 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -27,6 +27,7 @@ import aiohttp import certifi +from reportportal_client import RP # noinspection PyProtectedMember from reportportal_client._local import set_current from reportportal_client.aio import Task, BatchedTaskFactory @@ -444,115 +445,7 @@ def clone(self) -> '_AsyncRPClient': return cloned -class RPClient(metaclass=AbstractBaseClass): - __metaclass__ = AbstractBaseClass - - @abstractmethod - def start_launch(self, - name: str, - start_time: str, - description: Optional[str] = None, - attributes: Optional[Union[List, Dict]] = None, - rerun: bool = False, - rerun_of: Optional[str] = None, - **kwargs) -> Union[Optional[str], Task[str]]: - raise NotImplementedError('"start_launch" method is not implemented!') - - @abstractmethod - def start_test_item(self, - name: str, - start_time: str, - item_type: str, - *, - description: Optional[str] = None, - attributes: Optional[List[Dict]] = None, - parameters: Optional[Dict] = None, - parent_item_id: Union[Optional[str], Task[str]] = None, - has_stats: bool = True, - code_ref: Optional[str] = None, - retry: bool = False, - test_case_id: Optional[str] = None, - **kwargs: Any) -> Union[Optional[str], Task[str]]: - raise NotImplementedError('"start_test_item" method is not implemented!') - - @abstractmethod - def finish_test_item(self, - item_id: Union[str, Task[str]], - end_time: str, - *, - status: str = None, - issue: Optional[Issue] = None, - attributes: Optional[Union[List, Dict]] = None, - description: str = None, - retry: bool = False, - **kwargs: Any) -> Union[Optional[str], Task[str]]: - raise NotImplementedError('"finish_test_item" method is not implemented!') - - @abstractmethod - def finish_launch(self, - end_time: str, - status: str = None, - attributes: Optional[Union[List, Dict]] = None, - **kwargs: Any) -> Union[Optional[str], Task[str]]: - raise NotImplementedError('"finish_launch" method is not implemented!') - - @abstractmethod - def update_test_item(self, item_uuid: str, attributes: Optional[Union[List, Dict]] = None, - description: Optional[str] = None) -> Optional[str]: - raise NotImplementedError('"update_test_item" method is not implemented!') - - @abstractmethod - def get_launch_info(self) -> Union[Optional[dict], Task[str]]: - raise NotImplementedError('"get_launch_info" method is not implemented!') - - @abstractmethod - def get_item_id_by_uuid(self, item_uuid: Union[str, Task[str]]) -> Optional[str]: - raise NotImplementedError('"get_item_id_by_uuid" method is not implemented!') - - @abstractmethod - def get_launch_ui_id(self) -> Optional[int]: - raise NotImplementedError('"get_launch_ui_id" method is not implemented!') - - @abstractmethod - def get_launch_ui_url(self) -> Optional[str]: - raise NotImplementedError('"get_launch_ui_id" method is not implemented!') - - @abstractmethod - def get_project_settings(self) -> Optional[Dict]: - raise NotImplementedError('"get_project_settings" method is not implemented!') - - @abstractmethod - def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, item_id: Union[Optional[str], Task[str]] = None) -> None: - raise NotImplementedError('"log" method is not implemented!') - - def start(self) -> None: - pass # For backward compatibility - - def terminate(self, *_: Any, **__: Any) -> None: - pass # For backward compatibility - - @abstractmethod - @property - def launch_uuid(self) -> Optional[Union[str, Task[str]]]: - raise NotImplementedError('"launch_uuid" property is not implemented!') - - @property - def launch_id(self) -> Optional[Union[str, Task[str]]]: - warnings.warn( - message='`launch_id` property is deprecated since 5.5.0 and will be subject for removing in the' - ' next major version. Use `launch_uuid` property instead.', - category=DeprecationWarning, - stacklevel=2 - ) - return self.launch_uuid - - @abstractmethod - def clone(self) -> 'RPClient': - raise NotImplementedError('"clone" method is not implemented!') - - -class AsyncRPClient(RPClient): +class AsyncRPClient(RP): __client: _AsyncRPClient _item_stack: _LifoQueue __launch_uuid: Optional[str] @@ -650,11 +543,11 @@ def _add_current_item(self, item: str) -> None: """Add the last item from the self._items queue.""" self._item_stack.put(item) - def _remove_current_item(self) -> str: + def _remove_current_item(self) -> Optional[str]: """Remove the last item from the self._items queue.""" return self._item_stack.get() - def current_item(self) -> str: + def current_item(self) -> Optional[str]: """Retrieve the last item reported by the client.""" return self._item_stack.last() @@ -712,7 +605,7 @@ def clone(self) -> 'AsyncRPClient': return cloned -class _SyncRPClient(RPClient, metaclass=AbstractBaseClass): +class _SyncRPClient(RP, metaclass=AbstractBaseClass): __metaclass__ = AbstractBaseClass _item_stack: _LifoQueue diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 47260ad8..86664e8b 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -16,6 +16,7 @@ import logging import sys import warnings +from abc import abstractmethod from os import getenv from queue import LifoQueue from typing import Union, Tuple, List, Dict, Any, Optional, TextIO @@ -23,6 +24,7 @@ import requests from requests.adapters import HTTPAdapter, Retry, DEFAULT_RETRIES +from aio import Task # noinspection PyProtectedMember from reportportal_client._local import set_current from reportportal_client.core.rp_issues import Issue @@ -38,11 +40,126 @@ from reportportal_client.services.statistics import send_event from reportportal_client.static.defines import NOT_FOUND from reportportal_client.steps import StepReporter +from static.abstract import AbstractBaseClass logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) +class RP(metaclass=AbstractBaseClass): + __metaclass__ = AbstractBaseClass + + @abstractmethod + def start_launch(self, + name: str, + start_time: str, + description: Optional[str] = None, + attributes: Optional[Union[List, Dict]] = None, + rerun: bool = False, + rerun_of: Optional[str] = None, + **kwargs) -> Union[Optional[str], Task[str]]: + raise NotImplementedError('"start_launch" method is not implemented!') + + @abstractmethod + def start_test_item(self, + name: str, + start_time: str, + item_type: str, + *, + description: Optional[str] = None, + attributes: Optional[List[Dict]] = None, + parameters: Optional[Dict] = None, + parent_item_id: Union[Optional[str], Task[str]] = None, + has_stats: bool = True, + code_ref: Optional[str] = None, + retry: bool = False, + test_case_id: Optional[str] = None, + **kwargs: Any) -> Union[Optional[str], Task[str]]: + raise NotImplementedError('"start_test_item" method is not implemented!') + + @abstractmethod + def finish_test_item(self, + item_id: Union[str, Task[str]], + end_time: str, + *, + status: str = None, + issue: Optional[Issue] = None, + attributes: Optional[Union[List, Dict]] = None, + description: str = None, + retry: bool = False, + **kwargs: Any) -> Union[Optional[str], Task[str]]: + raise NotImplementedError('"finish_test_item" method is not implemented!') + + @abstractmethod + def finish_launch(self, + end_time: str, + status: str = None, + attributes: Optional[Union[List, Dict]] = None, + **kwargs: Any) -> Union[Optional[str], Task[str]]: + raise NotImplementedError('"finish_launch" method is not implemented!') + + @abstractmethod + def update_test_item(self, item_uuid: str, attributes: Optional[Union[List, Dict]] = None, + description: Optional[str] = None) -> Optional[str]: + raise NotImplementedError('"update_test_item" method is not implemented!') + + @abstractmethod + def get_launch_info(self) -> Union[Optional[dict], Task[str]]: + raise NotImplementedError('"get_launch_info" method is not implemented!') + + @abstractmethod + def get_item_id_by_uuid(self, item_uuid: Union[str, Task[str]]) -> Optional[str]: + raise NotImplementedError('"get_item_id_by_uuid" method is not implemented!') + + @abstractmethod + def get_launch_ui_id(self) -> Optional[int]: + raise NotImplementedError('"get_launch_ui_id" method is not implemented!') + + @abstractmethod + def get_launch_ui_url(self) -> Optional[str]: + raise NotImplementedError('"get_launch_ui_id" method is not implemented!') + + @abstractmethod + def get_project_settings(self) -> Optional[Dict]: + raise NotImplementedError('"get_project_settings" method is not implemented!') + + @abstractmethod + def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, + attachment: Optional[Dict] = None, item_id: Union[Optional[str], Task[str]] = None) -> None: + raise NotImplementedError('"log" method is not implemented!') + + def start(self) -> None: + pass # For backward compatibility + + def terminate(self, *_: Any, **__: Any) -> None: + pass # For backward compatibility + + @abstractmethod + @property + def launch_uuid(self) -> Optional[Union[str, Task[str]]]: + raise NotImplementedError('"launch_uuid" property is not implemented!') + + @property + def launch_id(self) -> Optional[Union[str, Task[str]]]: + warnings.warn( + message='`launch_id` property is deprecated since 5.5.0 and will be subject for removing in the' + ' next major version. Use `launch_uuid` property instead.', + category=DeprecationWarning, + stacklevel=2 + ) + return self.launch_uuid + + @abstractmethod + def current_item(self) -> Optional[Union[str, Task[str]]]: + """Retrieve the last item reported by the client.""" + raise NotImplementedError('"current_item" method is not implemented!') + + @abstractmethod + def clone(self) -> 'RPClient': + raise NotImplementedError('"clone" method is not implemented!') + + + class _LifoQueue(LifoQueue): def last(self): with self.mutex: @@ -50,7 +167,7 @@ def last(self): return self.queue[-1] -class RPClient: +class RPClient(RP): """Report portal client. The class is supposed to use by Report Portal agents: both custom and @@ -61,29 +178,30 @@ class RPClient: thread to avoid request/response messing and other issues. """ - _log_manager: LogManager = ... - api_v1: str = ... - api_v2: str = ... - base_url_v1: str = ... - base_url_v2: str = ... - endpoint: str = ... - is_skipped_an_issue: bool = ... - launch_id: str = ... - log_batch_size: int = ... - log_batch_payload_size: int = ... - project: str = ... - api_key: str = ... - verify_ssl: Union[bool, str] = ... - retries: int = ... - max_pool_size: int = ... - http_timeout: Union[float, Tuple[float, float]] = ... - session: requests.Session = ... - step_reporter: StepReporter = ... - mode: str = ... - launch_uuid_print: Optional[bool] = ... - print_output: Optional[TextIO] = ... - _skip_analytics: str = ... - _item_stack: _LifoQueue = ... + _log_manager: LogManager + api_v1: str + api_v2: str + base_url_v1: str + base_url_v2: str + endpoint: str + is_skipped_an_issue: bool + __launch_uuid: str + use_own_launch: bool + log_batch_size: int + log_batch_payload_size: int + project: str + api_key: str + verify_ssl: Union[bool, str] + retries: int + max_pool_size: int + http_timeout: Union[float, Tuple[float, float]] + session: requests.Session + step_reporter: StepReporter + mode: str + launch_uuid_print: Optional[bool] + print_output: Optional[TextIO] + _skip_analytics: str + _item_stack: _LifoQueue def __init__( self, @@ -95,7 +213,7 @@ def __init__( verify_ssl: bool = True, retries: int = None, max_pool_size: int = 50, - launch_id: str = None, + launch_uuid: str = None, http_timeout: Union[float, Tuple[float, float]] = (10, 10), log_batch_payload_size: int = MAX_LOG_BATCH_PAYLOAD_SIZE, mode: str = 'DEFAULT', @@ -116,8 +234,7 @@ def __init__( :param verify_ssl: Option to skip ssl verification :param max_pool_size: Option to set the maximum number of connections to save the pool. - :param launch_id: a launch id to use instead of starting - own one + :param launch_uuid: a launch UUID to use instead of starting own one :param http_timeout: a float in seconds for connect and read timeout. Use a Tuple to specific connect and read separately. @@ -133,7 +250,18 @@ def __init__( self.base_url_v2 = uri_join( self.endpoint, 'api/{}'.format(self.api_v2), self.project) self.is_skipped_an_issue = is_skipped_an_issue - self.launch_id = launch_id + self.__launch_uuid = launch_uuid + if not self.__launch_uuid: + launch_id = kwargs.get('launch_id') + if launch_id: + warnings.warn( + message='`launch_id` property is deprecated since 5.5.0 and will be subject for removing' + ' in the next major version. Use `launch_uuid` property instead.', + category=DeprecationWarning, + stacklevel=2 + ) + self.__launch_uuid = launch_id + self.use_own_launch = bool(self.__launch_uuid) self.log_batch_size = log_batch_size self.log_batch_payload_size = log_batch_payload_size self.verify_ssl = verify_ssl @@ -172,6 +300,14 @@ def __init__( self.__init_session() self.__init_log_manager() + @property + def launch_uuid(self) -> Optional[str]: + return self.__launch_uuid + + @launch_uuid.setter + def launch_uuid(self, value: Optional[str]) -> None: + self.__launch_uuid = value + def __init_session(self) -> None: retry_strategy = Retry( total=self.retries, @@ -191,11 +327,12 @@ def __init_session(self) -> None: def __init_log_manager(self) -> None: self._log_manager = LogManager( - self.endpoint, self.session, self.api_v2, self.launch_id, + self.endpoint, self.session, self.api_v2, self.launch_uuid, self.project, max_entry_number=self.log_batch_size, max_payload_size=self.log_batch_payload_size, verify_ssl=self.verify_ssl) + def finish_launch(self, end_time: str, status: str = None, @@ -209,10 +346,10 @@ def finish_launch(self, CANCELLED :param attributes: Launch attributes """ - if self.launch_id is NOT_FOUND or not self.launch_id: + if self.launch_uuid is NOT_FOUND or not self.launch_uuid: logger.warning('Attempt to finish non-existent launch') return - url = uri_join(self.base_url_v2, 'launch', self.launch_id, 'finish') + url = uri_join(self.base_url_v2, 'launch', self.launch_uuid, 'finish') request_payload = LaunchFinishRequest( end_time, status=status, @@ -224,7 +361,7 @@ def finish_launch(self, name='Finish Launch').make() if not response: return - logger.debug('finish_launch - ID: %s', self.launch_id) + logger.debug('finish_launch - ID: %s', self.launch_uuid) logger.debug('response message: %s', response.message) return response.message @@ -258,7 +395,7 @@ def finish_test_item(self, url = uri_join(self.base_url_v2, 'item', item_id) request_payload = ItemFinishRequest( end_time, - self.launch_id, + self.launch_uuid, status, attributes=attributes, description=description, @@ -291,10 +428,10 @@ def get_launch_info(self) -> Optional[Dict]: :return dict: Launch information in dictionary """ - if self.launch_id is None: + if self.launch_uuid is None: return {} - url = uri_join(self.base_url_v1, 'launch', 'uuid', self.launch_id) - logger.debug('get_launch_info - ID: %s', self.launch_id) + url = uri_join(self.base_url_v1, 'launch', 'uuid', self.launch_uuid) + logger.debug('get_launch_info - ID: %s', self.launch_uuid) response = HttpRequest(self.session.get, url=url, verify_ssl=self.verify_ssl).make() if not response: @@ -336,7 +473,7 @@ def get_launch_ui_url(self) -> Optional[str]: project_name=self.project.lower(), launch_type=launch_type, launch_id=ui_id) url = uri_join(self.endpoint, path) - logger.debug('get_launch_ui_url - ID: %s', self.launch_id) + logger.debug('get_launch_ui_url - UUID: %s', self.launch_uuid) return url def get_project_settings(self) -> Optional[Dict]: @@ -403,11 +540,11 @@ def start_launch(self, if not self._skip_analytics: send_event('start_launch', *agent_name_version(attributes)) - self._log_manager.launch_id = self.launch_id = response.id - logger.debug('start_launch - ID: %s', self.launch_id) + self._log_manager.launch_uuid = self.launch_uuid = response.id + logger.debug('start_launch - ID: %s', self.launch_uuid) if self.launch_uuid_print and self.print_output: - print(f'Report Portal Launch UUID: {self.launch_id}', file=self.print_output) - return self.launch_id + print(f'Report Portal Launch UUID: {self.launch_uuid}', file=self.print_output) + return self.launch_uuid def start_test_item(self, name: str, @@ -453,7 +590,7 @@ def start_test_item(self, name, start_time, item_type, - self.launch_id, + self.launch_uuid, attributes=verify_value_length(attributes), code_ref=code_ref, description=description, @@ -531,7 +668,7 @@ def clone(self) -> 'RPClient': verify_ssl=self.verify_ssl, retries=self.retries, max_pool_size=self.max_pool_size, - launch_id=self.launch_id, + launch_uuid=self.launch_uuid, http_timeout=self.http_timeout, log_batch_payload_size=self.log_batch_payload_size, mode=self.mode From 8672fec0b4f1cb842abb5d8e4dfa4a3e4e3d0b1e Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 14:46:36 +0300 Subject: [PATCH 059/268] Add ThreadedTask class and its factory --- reportportal_client/aio/client.py | 2 ++ reportportal_client/aio/tasks.py | 54 +++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 4c3fb1ef..ed3f3a40 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -27,6 +27,7 @@ import aiohttp import certifi +from aio.tasks import ThreadedTaskFactory from reportportal_client import RP # noinspection PyProtectedMember from reportportal_client._local import set_current @@ -804,6 +805,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self.self_loop = False else: self.__loop = asyncio.new_event_loop() + self.__loop.set_task_factory(ThreadedTaskFactory(self.__loop)) self.self_loop = True def create_task(self, coro: Coroutine[Any, Any, _T]) -> Task[_T]: diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index 67d1f3f4..8ec97fcc 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -12,7 +12,9 @@ # limitations under the License import asyncio +import sys import threading +import time import warnings from abc import abstractmethod from asyncio import Future @@ -20,9 +22,15 @@ from reportportal_client.static.abstract import AbstractBaseClass +DEFAULT_TASK_WAIT_TIMEOUT: float = 60.0 + _T = TypeVar('_T') +class BlockingOperationError(RuntimeError): + """An issue with task blocking execution.""" + + class Task(Generic[_T], asyncio.Task, metaclass=AbstractBaseClass): __metaclass__ = AbstractBaseClass @@ -66,6 +74,36 @@ def blocking_result(self) -> _T: return self.__loop.run_until_complete(self) +class ThreadedTask(Generic[_T], Task[_T]): + __loop: asyncio.AbstractEventLoop + __wait_timeout: float + + def __init__( + self, + coro: Union[Generator[Future[object], None, _T], Awaitable[_T]], + wait_timeout: float, + *, + loop: asyncio.AbstractEventLoop, + name: Optional[str] = None + ) -> None: + super().__init__(coro, loop=loop, name=name) + self.__loop = loop + self.__wait_timeout = wait_timeout + + def blocking_result(self) -> _T: + if self.done(): + return self.result() + if not self.__loop.is_running() or self.__loop.is_closed(): + raise BlockingOperationError('Running loop is not alive') + start_time = time.time() + slee_time = sys.getswitchinterval() + while not self.done() or time.time() - start_time < DEFAULT_TASK_WAIT_TIMEOUT: + time.sleep(slee_time) + if not self.done(): + raise BlockingOperationError('Timed out waiting for the task execution') + return self.result() + + class BatchedTaskFactory: __loop: asyncio.AbstractEventLoop __thread: threading.Thread @@ -80,3 +118,19 @@ def __call__( factory: Union[Coroutine[Any, Any, _T], Generator[Any, None, _T]] ) -> Task[_T]: return BatchedTask(factory, loop=self.__loop, thread=self.__thread) + + +class ThreadedTaskFactory: + __loop: asyncio.AbstractEventLoop + __wait_timeout: float + + def __init__(self, loop: asyncio.AbstractEventLoop, wait_timeout: float = DEFAULT_TASK_WAIT_TIMEOUT): + self.__loop = loop + self.__wait_timeout = wait_timeout + + def __call__( + self, + loop: asyncio.AbstractEventLoop, + factory: Union[Coroutine[Any, Any, _T], Generator[Any, None, _T]] + ) -> Task[_T]: + return ThreadedTask(factory, self.__wait_timeout, loop=self.__loop) From d468aa2763b2e415ded8285a567c7ff6a0905cda Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 14:46:47 +0300 Subject: [PATCH 060/268] Remove unused module --- reportportal_client/errors.py | 37 ----------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 reportportal_client/errors.py diff --git a/reportportal_client/errors.py b/reportportal_client/errors.py deleted file mode 100644 index 5f1d993b..00000000 --- a/reportportal_client/errors.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - - -class Error(Exception): - """General exception for package.""" - - -class ResponseError(Error): - """Error in response returned by RP.""" - - -class EntryCreatedError(ResponseError): - """Represents error in case no entry is created. - - No 'id' in the json response. - """ - - -class OperationCompletionError(ResponseError): - """Represents error in case of operation failure. - - No 'message' in the json response. - """ From 23878d8b6d98a364aa6aa426ff227a2deeb6f6c3 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 14:57:46 +0300 Subject: [PATCH 061/268] Constant refactoring --- reportportal_client/aio/__init__.py | 17 +++++++++-- reportportal_client/aio/client.py | 44 ++++++++++++++++------------- reportportal_client/aio/tasks.py | 6 ++-- 3 files changed, 41 insertions(+), 26 deletions(-) diff --git a/reportportal_client/aio/__init__.py b/reportportal_client/aio/__init__.py index 6c63bc5d..cfc5f3cb 100644 --- a/reportportal_client/aio/__init__.py +++ b/reportportal_client/aio/__init__.py @@ -11,10 +11,21 @@ # See the License for the specific language governing permissions and # limitations under the License -from reportportal_client.aio.tasks import Task, BatchedTask, BatchedTaskFactory +from reportportal_client.aio.tasks import (Task, BatchedTask, BatchedTaskFactory, ThreadedTask, + ThreadedTaskFactory, BlockingOperationError) + +DEFAULT_TASK_TIMEOUT: float = 60.0 +DEFAULT_SHUTDOWN_TIMEOUT: float = 120.0 +DEFAULT_TASK_TRIGGER_NUM: int = 10 +DEFAULT_TASK_TRIGGER_INTERVAL: float = 1.0 __all__ = [ 'Task', 'BatchedTask', - 'BatchedTaskFactory' -] \ No newline at end of file + 'BatchedTaskFactory', + 'ThreadedTask', + 'ThreadedTaskFactory', + 'BlockingOperationError', + 'DEFAULT_TASK_TIMEOUT', + 'DEFAULT_SHUTDOWN_TIMEOUT' +] diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index ed3f3a40..6c9426d0 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -27,11 +27,12 @@ import aiohttp import certifi -from aio.tasks import ThreadedTaskFactory from reportportal_client import RP # noinspection PyProtectedMember from reportportal_client._local import set_current -from reportportal_client.aio import Task, BatchedTaskFactory +from reportportal_client.aio import (Task, BatchedTaskFactory, ThreadedTaskFactory, DEFAULT_TASK_TIMEOUT, + DEFAULT_SHUTDOWN_TIMEOUT, DEFAULT_TASK_TRIGGER_NUM, + DEFAULT_TASK_TRIGGER_INTERVAL) from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, AsyncItemFinishRequest, LaunchFinishRequest, @@ -50,11 +51,6 @@ logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) -TASK_RUN_NUM_THRESHOLD: int = 10 -TASK_RUN_INTERVAL: float = 1.0 -TASK_TIMEOUT: int = 60 -SHUTDOWN_TIMEOUT: int = 120 - _T = TypeVar('_T') @@ -790,30 +786,34 @@ class ThreadedRPClient(_SyncRPClient): __task_mutex: threading.Lock __loop: Optional[asyncio.AbstractEventLoop] __thread: Optional[threading.Thread] - self_loop: bool - self_thread: bool + __self_loop: bool + __task_timeout: float + __shutdown_timeout: float def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, client: Optional[_AsyncRPClient] = None, loop: Optional[asyncio.AbstractEventLoop] = None, - **kwargs: Any) -> None: + task_timeout: float = DEFAULT_TASK_TIMEOUT, + shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, **kwargs: Any) -> None: super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, **kwargs) + self.__task_timeout = task_timeout + self.__shutdown_timeout = shutdown_timeout self.__task_list = [] self.__task_mutex = threading.Lock() self.__thread = None if loop: self.__loop = loop - self.self_loop = False + self.__self_loop = False else: self.__loop = asyncio.new_event_loop() - self.__loop.set_task_factory(ThreadedTaskFactory(self.__loop)) - self.self_loop = True + self.__loop.set_task_factory(ThreadedTaskFactory(self.__loop, self.__task_timeout)) + self.__self_loop = True def create_task(self, coro: Coroutine[Any, Any, _T]) -> Task[_T]: loop = self.__loop result = loop.create_task(coro) with self.__task_mutex: self.__task_list.append(result) - if not self.__thread and self.self_loop: + if not self.__thread and self.__self_loop: self.__thread = threading.Thread(target=loop.run_forever, name='RP-Async-Client', daemon=True) self.__thread.start() @@ -830,10 +830,10 @@ def finish_tasks(self): with self.__task_mutex: for task in self.__task_list: task_start_time = time.time() - while not task.done() and (time.time() - task_start_time < TASK_TIMEOUT) and ( - time.time() - shutdown_start_time < SHUTDOWN_TIMEOUT): + while not task.done() and (time.time() - task_start_time < DEFAULT_TASK_TIMEOUT) and ( + time.time() - shutdown_start_time < DEFAULT_SHUTDOWN_TIMEOUT): time.sleep(sleep_time) - if time.time() - shutdown_start_time >= SHUTDOWN_TIMEOUT: + if time.time() - shutdown_start_time >= DEFAULT_SHUTDOWN_TIMEOUT: break self.__task_list = [] @@ -864,9 +864,12 @@ class BatchedRPClient(_SyncRPClient): __task_mutex: threading.Lock __last_run_time: float __thread: threading.Thread + __trigger_num: int + __trigger_interval: float def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, - client: Optional[_AsyncRPClient] = None, **kwargs: Any) -> None: + client: Optional[_AsyncRPClient] = None, trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, + trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL, **kwargs: Any) -> None: super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, **kwargs) self.__task_list = [] @@ -875,13 +878,16 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self.__loop = asyncio.new_event_loop() self.__thread = threading.current_thread() self.__loop.set_task_factory(BatchedTaskFactory(self.__loop, self.__thread)) + self.__trigger_num = trigger_num + self.__trigger_interval = trigger_interval def __ready_to_run(self) -> bool: current_time = time.time() last_time = self.__last_run_time if len(self.__task_list) <= 0: return False - if len(self.__task_list) > TASK_RUN_NUM_THRESHOLD or current_time - last_time >= TASK_RUN_INTERVAL: + if (len(self.__task_list) >= self.__trigger_num + or current_time - last_time >= self.__trigger_interval): self.__last_run_time = current_time return True return False diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index 8ec97fcc..99cd7bd8 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -22,8 +22,6 @@ from reportportal_client.static.abstract import AbstractBaseClass -DEFAULT_TASK_WAIT_TIMEOUT: float = 60.0 - _T = TypeVar('_T') @@ -97,7 +95,7 @@ def blocking_result(self) -> _T: raise BlockingOperationError('Running loop is not alive') start_time = time.time() slee_time = sys.getswitchinterval() - while not self.done() or time.time() - start_time < DEFAULT_TASK_WAIT_TIMEOUT: + while not self.done() or time.time() - start_time < self.__wait_timeout: time.sleep(slee_time) if not self.done(): raise BlockingOperationError('Timed out waiting for the task execution') @@ -124,7 +122,7 @@ class ThreadedTaskFactory: __loop: asyncio.AbstractEventLoop __wait_timeout: float - def __init__(self, loop: asyncio.AbstractEventLoop, wait_timeout: float = DEFAULT_TASK_WAIT_TIMEOUT): + def __init__(self, loop: asyncio.AbstractEventLoop, wait_timeout: float): self.__loop = loop self.__wait_timeout = wait_timeout From 424b9c6b96c7f924541ee9eb7856804e72ce0aeb Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 15:29:49 +0300 Subject: [PATCH 062/268] Class rename --- reportportal_client/aio/client.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 6c9426d0..b9954657 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -61,7 +61,7 @@ def last(self): return self.queue[-1] -class _AsyncRPClient: +class Client: api_v1: str api_v2: str base_url_v1: str @@ -420,13 +420,13 @@ async def log_batch(self, log_batch: List[AsyncRPRequestLog]) -> Tuple[str, ...] response = await AsyncHttpRequest(self.session.post, url=url, data=AsyncRPLogBatch(log_batch)).make() return await response.messages - def clone(self) -> '_AsyncRPClient': + def clone(self) -> 'Client': """Clone the client object, set current Item ID as cloned item ID. :returns: Cloned client object :rtype: AsyncRPClient """ - cloned = _AsyncRPClient( + cloned = Client( endpoint=self.endpoint, project=self.project, api_key=self.api_key, @@ -443,21 +443,21 @@ def clone(self) -> '_AsyncRPClient': class AsyncRPClient(RP): - __client: _AsyncRPClient + __client: Client _item_stack: _LifoQueue __launch_uuid: Optional[str] use_own_launch: bool step_reporter: StepReporter def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[str] = None, - client: Optional[_AsyncRPClient] = None, **kwargs: Any) -> None: + client: Optional[Client] = None, **kwargs: Any) -> None: set_current(self) self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() if client: self.__client = client else: - self.__client = _AsyncRPClient(endpoint, project, **kwargs) + self.__client = Client(endpoint, project, **kwargs) if launch_uuid: self.launch_uuid = launch_uuid self.use_own_launch = False @@ -619,14 +619,14 @@ def launch_uuid(self, value: Optional[Task[str]]) -> None: self.__launch_uuid = value def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, - client: Optional[_AsyncRPClient] = None, **kwargs: Any) -> None: + client: Optional[Client] = None, **kwargs: Any) -> None: set_current(self) self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() if client: self.__client = client else: - self.__client = _AsyncRPClient(endpoint, project, **kwargs) + self.__client = Client(endpoint, project, **kwargs) if launch_uuid: self.launch_uuid = launch_uuid self.use_own_launch = False @@ -791,7 +791,8 @@ class ThreadedRPClient(_SyncRPClient): __shutdown_timeout: float def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, - client: Optional[_AsyncRPClient] = None, loop: Optional[asyncio.AbstractEventLoop] = None, + client: Optional[Client] = None, + loop: Optional[asyncio.AbstractEventLoop] = None, task_timeout: float = DEFAULT_TASK_TIMEOUT, shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, **kwargs: Any) -> None: super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, **kwargs) @@ -868,7 +869,7 @@ class BatchedRPClient(_SyncRPClient): __trigger_interval: float def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, - client: Optional[_AsyncRPClient] = None, trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, + client: Optional[Client] = None, trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL, **kwargs: Any) -> None: super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, **kwargs) From 5fefe6ddb749f8607d4882b88b871149d7b439bb Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 17:50:26 +0300 Subject: [PATCH 063/268] Add log batching --- reportportal_client/aio/client.py | 53 +++++++++++++++++++--- reportportal_client/client.py | 30 ++++++++----- reportportal_client/logs/batcher.py | 70 +++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 19 deletions(-) create mode 100644 reportportal_client/logs/batcher.py diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index b9954657..6be3e6e4 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -27,6 +27,7 @@ import aiohttp import certifi +from logs.batcher import LogBatcher from reportportal_client import RP # noinspection PyProtectedMember from reportportal_client._local import set_current @@ -35,7 +36,7 @@ DEFAULT_TASK_TRIGGER_INTERVAL) from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, - AsyncItemFinishRequest, LaunchFinishRequest, + AsyncItemFinishRequest, LaunchFinishRequest, RPFile, AsyncRPRequestLog, AsyncRPLogBatch) from reportportal_client.helpers import (root_uri_join, verify_value_length, await_if_necessary, agent_name_version) @@ -443,8 +444,9 @@ def clone(self) -> 'Client': class AsyncRPClient(RP): - __client: Client _item_stack: _LifoQueue + _log_batcher: LogBatcher + __client: Client __launch_uuid: Optional[str] use_own_launch: bool step_reporter: StepReporter @@ -454,6 +456,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[str] = set_current(self) self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() + self._log_batcher = LogBatcher() if client: self.__client = client else: @@ -570,9 +573,25 @@ async def get_project_settings(self) -> Optional[Dict]: return await self.__client.get_project_settings() async def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: - # TODO: implement logging - return + attachment: Optional[Dict] = None, + parent_item: Optional[str] = None) -> Optional[Tuple[str, ...]]: + """Log message. Can be added to test item in any state. + + :param datetime: Log time + :param message: Log message + :param level: Log level + :param attachment: Attachments(images,files,etc.) + :param parent_item: Parent item UUID + """ + if parent_item is NOT_FOUND: + logger.warning("Attempt to log to non-existent item") + return + rp_file = RPFile(**attachment) if attachment else None + rp_log = AsyncRPRequestLog(self.launch_uuid, datetime, rp_file, parent_item, level, message) + batch = await self._log_batcher.append_async(rp_log) + if batch: + return await self.__client.log_batch(batch) + @property def launch_uuid(self) -> Optional[str]: @@ -606,6 +625,7 @@ class _SyncRPClient(RP, metaclass=AbstractBaseClass): __metaclass__ = AbstractBaseClass _item_stack: _LifoQueue + _log_batcher: LogBatcher __launch_uuid: Optional[Task[str]] use_own_launch: bool step_reporter: StepReporter @@ -623,6 +643,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st set_current(self) self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() + self._log_batcher = LogBatcher() if client: self.__client = client else: @@ -775,9 +796,27 @@ def get_project_settings(self) -> Task[dict]: result_task = self.create_task(result_coro) return result_task + async def _log(self, log_rq: AsyncRPRequestLog) -> Optional[Tuple[str, ...]]: + batch = await self._log_batcher.append_async(log_rq) + if batch: + return await self.__client.log_batch(batch) + def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, item_id: Optional[Task[str]] = None) -> None: - # TODO: implement logging + attachment: Optional[Dict] = None, parent_item: Optional[Task[str]] = None) -> None: + """Log message. Can be added to test item in any state. + + :param datetime: Log time + :param message: Log message + :param level: Log level + :param attachment: Attachments(images,files,etc.) + :param parent_item: Parent item UUID + """ + if parent_item is NOT_FOUND: + logger.warning("Attempt to log to non-existent item") + return + rp_file = RPFile(**attachment) if attachment else None + rp_log = AsyncRPRequestLog(self.launch_uuid, datetime, rp_file, parent_item, level, message) + self.create_task(self._log(rp_log)) return None diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 86664e8b..88b5e3ad 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -24,18 +24,15 @@ import requests from requests.adapters import HTTPAdapter, Retry, DEFAULT_RETRIES -from aio import Task # noinspection PyProtectedMember from reportportal_client._local import set_current +from reportportal_client.aio import Task from reportportal_client.core.rp_issues import Issue -from reportportal_client.core.rp_requests import ( - HttpRequest, - ItemStartRequest, - ItemFinishRequest, - LaunchStartRequest, - LaunchFinishRequest -) +from reportportal_client.core.rp_requests import (HttpRequest, ItemStartRequest, ItemFinishRequest, RPFile, + LaunchStartRequest, LaunchFinishRequest, RPRequestLog, + RPLogBatch) from reportportal_client.helpers import uri_join, verify_value_length, agent_name_version +from reportportal_client.logs.batcher import LogBatcher from reportportal_client.logs.log_manager import LogManager, MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.services.statistics import send_event from reportportal_client.static.defines import NOT_FOUND @@ -159,7 +156,6 @@ def clone(self) -> 'RPClient': raise NotImplementedError('"clone" method is not implemented!') - class _LifoQueue(LifoQueue): def last(self): with self.mutex: @@ -202,6 +198,7 @@ class RPClient(RP): print_output: Optional[TextIO] _skip_analytics: str _item_stack: _LifoQueue + _log_batcher: LogBatcher def __init__( self, @@ -332,7 +329,6 @@ def __init_log_manager(self) -> None: max_payload_size=self.log_batch_payload_size, verify_ssl=self.verify_ssl) - def finish_launch(self, end_time: str, status: str = None, @@ -487,7 +483,7 @@ def get_project_settings(self) -> Optional[Dict]: return response.json if response else None def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> None: + attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> Optional[Tuple[str, ...]]: """Send log message to the Report Portal. :param time: Time in UTC @@ -496,7 +492,17 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, :param attachment: Message's attachments :param item_id: ID of the RP item the message belongs to """ - self._log_manager.log(time, message, level, attachment, item_id) + if item_id is NOT_FOUND: + logger.warning("Attempt to log to non-existent item") + return + url = uri_join(self.base_url_v2, 'log') + rp_file = RPFile(**attachment) if attachment else None + rp_log = RPRequestLog(self.launch_uuid, time, rp_file, item_id, level, message) + batch = self._log_batcher.append(rp_log) + if batch: + response = HttpRequest(self.session.post, url, files=RPLogBatch(batch).payload, + verify_ssl=self.verify_ssl).make() + return response.messages def start(self) -> None: """Start the client.""" diff --git a/reportportal_client/logs/batcher.py b/reportportal_client/logs/batcher.py new file mode 100644 index 00000000..8ae73de3 --- /dev/null +++ b/reportportal_client/logs/batcher.py @@ -0,0 +1,70 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +import logging +import threading +from typing import List, Optional, TypeVar + +from reportportal_client import helpers +from reportportal_client.core.rp_requests import RPRequestLog, AsyncRPRequestLog +from reportportal_client.logs import MAX_LOG_BATCH_SIZE, MAX_LOG_BATCH_PAYLOAD_SIZE + +logger = logging.getLogger(__name__) + +T_co = TypeVar('T_co', bound='RPRequestLog', covariant=True) + + +class LogBatcher: + entry_num: int + payload_limit: int + _lock: threading.Lock + _batch: List[T_co] + _payload_size: int + + def __init__(self, entry_num=MAX_LOG_BATCH_SIZE, payload_limit=MAX_LOG_BATCH_PAYLOAD_SIZE): + self.entry_num = entry_num + self.payload_limit = payload_limit + self._lock = threading.Lock() + self._batch = [] + self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + + def _append(self, size: int, log_req: T_co) -> Optional[List[T_co]]: + with self._lock: + if self._payload_size + size >= self.payload_limit: + if len(self._batch) > 0: + batch = self._batch + self._batch = [log_req] + self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + return batch + self._batch.append(log_req) + self._payload_size += size + if len(self._batch) >= self.entry_num: + batch = self._batch + self._batch = [] + self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + return batch + + def append(self, log_req: RPRequestLog) -> Optional[List[RPRequestLog]]: + """Add a log request object to internal batch and return the batch if it's full. + + :param log_req: log request object + :return ready to send batch or None + """ + return self._append(log_req.multipart_size, log_req) + + async def append_async(self, log_req: AsyncRPRequestLog) -> Optional[List[AsyncRPRequestLog]]: + """Add a log request object to internal batch and return the batch if it's full. + + :param log_req: log request object + :return ready to send batch or None + """ + return self._append(await log_req.multipart_size, log_req) From 55f471fb37192a3c9ec585a799aa131c92a3cc72 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 18:03:15 +0300 Subject: [PATCH 064/268] Add log batching --- reportportal_client/client.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 88b5e3ad..4008b417 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -32,8 +32,8 @@ LaunchStartRequest, LaunchFinishRequest, RPRequestLog, RPLogBatch) from reportportal_client.helpers import uri_join, verify_value_length, agent_name_version +from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.logs.batcher import LogBatcher -from reportportal_client.logs.log_manager import LogManager, MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.services.statistics import send_event from reportportal_client.static.defines import NOT_FOUND from reportportal_client.steps import StepReporter @@ -173,8 +173,6 @@ class RPClient(RP): NOTICE: the class is not thread-safe, use new class instance for every new thread to avoid request/response messing and other issues. """ - - _log_manager: LogManager api_v1: str api_v2: str base_url_v1: str @@ -261,6 +259,7 @@ def __init__( self.use_own_launch = bool(self.__launch_uuid) self.log_batch_size = log_batch_size self.log_batch_payload_size = log_batch_payload_size + self._log_batcher = LogBatcher(self.log_batch_size, self.log_batch_payload_size) self.verify_ssl = verify_ssl self.retries = retries self.max_pool_size = max_pool_size @@ -295,7 +294,6 @@ def __init__( ) self.__init_session() - self.__init_log_manager() @property def launch_uuid(self) -> Optional[str]: @@ -322,13 +320,6 @@ def __init_session(self) -> None: self.api_key) self.session = session - def __init_log_manager(self) -> None: - self._log_manager = LogManager( - self.endpoint, self.session, self.api_v2, self.launch_uuid, - self.project, max_entry_number=self.log_batch_size, - max_payload_size=self.log_batch_payload_size, - verify_ssl=self.verify_ssl) - def finish_launch(self, end_time: str, status: str = None, @@ -506,7 +497,7 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, def start(self) -> None: """Start the client.""" - self._log_manager.start() + pass def start_launch(self, name: str, @@ -546,7 +537,7 @@ def start_launch(self, if not self._skip_analytics: send_event('start_launch', *agent_name_version(attributes)) - self._log_manager.launch_uuid = self.launch_uuid = response.id + self.launch_uuid = response.id logger.debug('start_launch - ID: %s', self.launch_uuid) if self.launch_uuid_print and self.print_output: print(f'Report Portal Launch UUID: {self.launch_uuid}', file=self.print_output) @@ -623,7 +614,7 @@ def start_test_item(self, def terminate(self, *_: Any, **__: Any) -> None: """Call this to terminate the client.""" - self._log_manager.stop() + pass def update_test_item(self, item_uuid: str, attributes: Optional[Union[List, Dict]] = None, description: Optional[str] = None) -> Optional[str]: @@ -705,5 +696,3 @@ def __setstate__(self, state: Dict[str, Any]) -> None: self.__dict__.update(state) # Restore 'session' field self.__init_session() - # Restore '_log_manager' field - self.__init_log_manager() From ea0592cb2545333f2383bc11d8455f27de7ccd88 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 18:15:33 +0300 Subject: [PATCH 065/268] Add log batching --- reportportal_client/aio/client.py | 19 +++++++++---------- reportportal_client/client.py | 17 ++++++++++------- reportportal_client/logs/batcher.py | 11 +++++++++-- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 6be3e6e4..7e991634 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -416,10 +416,12 @@ async def get_project_settings(self) -> Optional[Dict]: response = await AsyncHttpRequest(self.session.get, url=url).make() return await response.json if response else None - async def log_batch(self, log_batch: List[AsyncRPRequestLog]) -> Tuple[str, ...]: + async def log_batch(self, log_batch: Optional[List[AsyncRPRequestLog]]) -> Tuple[str, ...]: url = root_uri_join(self.base_url_v2, 'log') - response = await AsyncHttpRequest(self.session.post, url=url, data=AsyncRPLogBatch(log_batch)).make() - return await response.messages + if log_batch: + response = await AsyncHttpRequest(self.session.post, url=url, + data=AsyncRPLogBatch(log_batch)).make() + return await response.messages def clone(self) -> 'Client': """Clone the client object, set current Item ID as cloned item ID. @@ -529,6 +531,7 @@ async def finish_launch(self, status: str = None, attributes: Optional[Union[List, Dict]] = None, **kwargs: Any) -> Optional[str]: + await self.__client.log_batch(self._log_batcher.flush()) if not self.use_own_launch: return "" return await self.__client.finish_launch(self.launch_uuid, end_time, status=status, @@ -588,10 +591,7 @@ async def log(self, datetime: str, message: str, level: Optional[Union[int, str] return rp_file = RPFile(**attachment) if attachment else None rp_log = AsyncRPRequestLog(self.launch_uuid, datetime, rp_file, parent_item, level, message) - batch = await self._log_batcher.append_async(rp_log) - if batch: - return await self.__client.log_batch(batch) - + return await self.__client.log_batch(await self._log_batcher.append_async(rp_log)) @property def launch_uuid(self) -> Optional[str]: @@ -746,6 +746,7 @@ def finish_launch(self, status: str = None, attributes: Optional[Union[List, Dict]] = None, **kwargs: Any) -> Task[str]: + self.create_task(self.__client.log_batch(self._log_batcher.flush())) if self.use_own_launch: result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, **kwargs) @@ -797,9 +798,7 @@ def get_project_settings(self) -> Task[dict]: return result_task async def _log(self, log_rq: AsyncRPRequestLog) -> Optional[Tuple[str, ...]]: - batch = await self._log_batcher.append_async(log_rq) - if batch: - return await self.__client.log_batch(batch) + return await self.__client.log_batch(await self._log_batcher.append_async(log_rq)) def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, parent_item: Optional[Task[str]] = None) -> None: diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 4008b417..8cd16e46 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -196,7 +196,7 @@ class RPClient(RP): print_output: Optional[TextIO] _skip_analytics: str _item_stack: _LifoQueue - _log_batcher: LogBatcher + _log_batcher: LogBatcher[RPRequestLog] def __init__( self, @@ -333,6 +333,7 @@ def finish_launch(self, CANCELLED :param attributes: Launch attributes """ + self._log(self._log_batcher.flush()) if self.launch_uuid is NOT_FOUND or not self.launch_uuid: logger.warning('Attempt to finish non-existent launch') return @@ -473,6 +474,13 @@ def get_project_settings(self) -> Optional[Dict]: verify_ssl=self.verify_ssl).make() return response.json if response else None + def _log(self, batch: Optional[List[RPRequestLog]]) -> Optional[Tuple[str, ...]]: + url = uri_join(self.base_url_v2, 'log') + if batch: + response = HttpRequest(self.session.post, url, files=RPLogBatch(batch).payload, + verify_ssl=self.verify_ssl).make() + return response.messages + def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> Optional[Tuple[str, ...]]: """Send log message to the Report Portal. @@ -486,14 +494,9 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, if item_id is NOT_FOUND: logger.warning("Attempt to log to non-existent item") return - url = uri_join(self.base_url_v2, 'log') rp_file = RPFile(**attachment) if attachment else None rp_log = RPRequestLog(self.launch_uuid, time, rp_file, item_id, level, message) - batch = self._log_batcher.append(rp_log) - if batch: - response = HttpRequest(self.session.post, url, files=RPLogBatch(batch).payload, - verify_ssl=self.verify_ssl).make() - return response.messages + return self._log(self._log_batcher.append(rp_log)) def start(self) -> None: """Start the client.""" diff --git a/reportportal_client/logs/batcher.py b/reportportal_client/logs/batcher.py index 8ae73de3..9ddbd7ec 100644 --- a/reportportal_client/logs/batcher.py +++ b/reportportal_client/logs/batcher.py @@ -12,7 +12,7 @@ # limitations under the License import logging import threading -from typing import List, Optional, TypeVar +from typing import List, Optional, TypeVar, Generic from reportportal_client import helpers from reportportal_client.core.rp_requests import RPRequestLog, AsyncRPRequestLog @@ -23,7 +23,7 @@ T_co = TypeVar('T_co', bound='RPRequestLog', covariant=True) -class LogBatcher: +class LogBatcher(Generic[T_co]): entry_num: int payload_limit: int _lock: threading.Lock @@ -68,3 +68,10 @@ async def append_async(self, log_req: AsyncRPRequestLog) -> Optional[List[AsyncR :return ready to send batch or None """ return self._append(await log_req.multipart_size, log_req) + + def flush(self) -> Optional[List[T_co]]: + with self._lock: + if len(self._batch) > 0: + batch = self._batch + self._batch = [] + return batch From 07a858336315e79d16f03a4b4f0da30a77d54850 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 18:29:03 +0300 Subject: [PATCH 066/268] Fix issues --- reportportal_client/aio/__init__.py | 4 +++- reportportal_client/core/rp_responses.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/reportportal_client/aio/__init__.py b/reportportal_client/aio/__init__.py index cfc5f3cb..432a4ab3 100644 --- a/reportportal_client/aio/__init__.py +++ b/reportportal_client/aio/__init__.py @@ -27,5 +27,7 @@ 'ThreadedTaskFactory', 'BlockingOperationError', 'DEFAULT_TASK_TIMEOUT', - 'DEFAULT_SHUTDOWN_TIMEOUT' + 'DEFAULT_SHUTDOWN_TIMEOUT', + 'DEFAULT_TASK_TRIGGER_NUM', + 'DEFAULT_TASK_TRIGGER_INTERVAL' ] diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index b71d9b9c..5e509953 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -29,7 +29,7 @@ logger = logging.getLogger(__name__) -def _iter_json_messages(json: Any) -> Generator[str]: +def _iter_json_messages(json: Any) -> Generator[str, None, None]: if not isinstance(json, Mapping): return data = json.get('responses', [json]) From ad21f3e5c161fbf57297174358da3286b59e64b6 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 18:30:52 +0300 Subject: [PATCH 067/268] Fix issues --- reportportal_client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 8cd16e46..24f42f1b 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -35,9 +35,9 @@ from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.logs.batcher import LogBatcher from reportportal_client.services.statistics import send_event +from reportportal_client.static.abstract import AbstractBaseClass from reportportal_client.static.defines import NOT_FOUND from reportportal_client.steps import StepReporter -from static.abstract import AbstractBaseClass logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) From 8849ed3f9fe4238670d102479625794fc078ced5 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 18:33:41 +0300 Subject: [PATCH 068/268] Fix issues --- reportportal_client/aio/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 7e991634..2cb3a4df 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -27,7 +27,6 @@ import aiohttp import certifi -from logs.batcher import LogBatcher from reportportal_client import RP # noinspection PyProtectedMember from reportportal_client._local import set_current @@ -41,6 +40,7 @@ from reportportal_client.helpers import (root_uri_join, verify_value_length, await_if_necessary, agent_name_version) from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE +from reportportal_client.logs.batcher import LogBatcher from reportportal_client.services.statistics import async_send_event from reportportal_client.static.abstract import ( AbstractBaseClass, From 7395f5c7c14823f2b9ffb72f891c5dfb4afb8ad7 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 18:36:41 +0300 Subject: [PATCH 069/268] revert argument rename --- reportportal_client/aio/client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 2cb3a4df..950d6fe1 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -577,20 +577,20 @@ async def get_project_settings(self) -> Optional[Dict]: async def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, - parent_item: Optional[str] = None) -> Optional[Tuple[str, ...]]: + item_id: Optional[str] = None) -> Optional[Tuple[str, ...]]: """Log message. Can be added to test item in any state. :param datetime: Log time :param message: Log message :param level: Log level :param attachment: Attachments(images,files,etc.) - :param parent_item: Parent item UUID + :param item_id: Parent item UUID """ - if parent_item is NOT_FOUND: + if item_id is NOT_FOUND: logger.warning("Attempt to log to non-existent item") return rp_file = RPFile(**attachment) if attachment else None - rp_log = AsyncRPRequestLog(self.launch_uuid, datetime, rp_file, parent_item, level, message) + rp_log = AsyncRPRequestLog(self.launch_uuid, datetime, rp_file, item_id, level, message) return await self.__client.log_batch(await self._log_batcher.append_async(rp_log)) @property @@ -801,20 +801,20 @@ async def _log(self, log_rq: AsyncRPRequestLog) -> Optional[Tuple[str, ...]]: return await self.__client.log_batch(await self._log_batcher.append_async(log_rq)) def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, parent_item: Optional[Task[str]] = None) -> None: + attachment: Optional[Dict] = None, item_id: Optional[Task[str]] = None) -> None: """Log message. Can be added to test item in any state. :param datetime: Log time :param message: Log message :param level: Log level :param attachment: Attachments(images,files,etc.) - :param parent_item: Parent item UUID + :param item_id: Parent item UUID """ - if parent_item is NOT_FOUND: + if item_id is NOT_FOUND: logger.warning("Attempt to log to non-existent item") return rp_file = RPFile(**attachment) if attachment else None - rp_log = AsyncRPRequestLog(self.launch_uuid, datetime, rp_file, parent_item, level, message) + rp_log = AsyncRPRequestLog(self.launch_uuid, datetime, rp_file, item_id, level, message) self.create_task(self._log(rp_log)) return None From 5d9051366792263fea91f803c4e1629bd8ffa385 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 18:41:19 +0300 Subject: [PATCH 070/268] Fix object cloning --- reportportal_client/aio/client.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 950d6fe1..16244a7b 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -626,10 +626,15 @@ class _SyncRPClient(RP, metaclass=AbstractBaseClass): _item_stack: _LifoQueue _log_batcher: LogBatcher + __client: Client __launch_uuid: Optional[Task[str]] use_own_launch: bool step_reporter: StepReporter + @property + def client(self) -> Client: + return self.__client + @property def launch_uuid(self) -> Optional[Task[str]]: return self.__launch_uuid @@ -882,7 +887,7 @@ def clone(self) -> 'ThreadedRPClient': :returns: Cloned client object :rtype: ThreadedRPClient """ - cloned_client = self.__client.clone() + cloned_client = self.client.clone() # noinspection PyTypeChecker cloned = ThreadedRPClient( endpoint=None, @@ -958,7 +963,7 @@ def clone(self) -> 'BatchedRPClient': :returns: Cloned client object :rtype: BatchedRPClient """ - cloned_client = self.__client.clone() + cloned_client = self.client.clone() # noinspection PyTypeChecker cloned = BatchedRPClient( endpoint=None, From ab2c919c4c06b13a57006809dc63c054844ac848 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 18:45:22 +0300 Subject: [PATCH 071/268] Fix issues --- reportportal_client/aio/client.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 16244a7b..5a9c0d41 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -18,7 +18,7 @@ import ssl import sys import threading -import time +import time as datetime import warnings from os import getenv from queue import LifoQueue @@ -575,12 +575,12 @@ async def get_launch_ui_url(self) -> Optional[str]: async def get_project_settings(self) -> Optional[Dict]: return await self.__client.get_project_settings() - async def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, + async def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> Optional[Tuple[str, ...]]: """Log message. Can be added to test item in any state. - :param datetime: Log time + :param time: Log time :param message: Log message :param level: Log level :param attachment: Attachments(images,files,etc.) @@ -590,7 +590,7 @@ async def log(self, datetime: str, message: str, level: Optional[Union[int, str] logger.warning("Attempt to log to non-existent item") return rp_file = RPFile(**attachment) if attachment else None - rp_log = AsyncRPRequestLog(self.launch_uuid, datetime, rp_file, item_id, level, message) + rp_log = AsyncRPRequestLog(self.launch_uuid, time, rp_file, item_id, level, message) return await self.__client.log_batch(await self._log_batcher.append_async(rp_log)) @property @@ -805,11 +805,11 @@ def get_project_settings(self) -> Task[dict]: async def _log(self, log_rq: AsyncRPRequestLog) -> Optional[Tuple[str, ...]]: return await self.__client.log_batch(await self._log_batcher.append_async(log_rq)) - def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, + def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, item_id: Optional[Task[str]] = None) -> None: """Log message. Can be added to test item in any state. - :param datetime: Log time + :param time: Log time :param message: Log message :param level: Log level :param attachment: Attachments(images,files,etc.) @@ -819,7 +819,7 @@ def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = No logger.warning("Attempt to log to non-existent item") return rp_file = RPFile(**attachment) if attachment else None - rp_log = AsyncRPRequestLog(self.launch_uuid, datetime, rp_file, item_id, level, message) + rp_log = AsyncRPRequestLog(self.launch_uuid, time, rp_file, item_id, level, message) self.create_task(self._log(rp_log)) return None @@ -870,14 +870,14 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Task[_T]: def finish_tasks(self): sleep_time = sys.getswitchinterval() - shutdown_start_time = time.time() + shutdown_start_time = datetime.time() with self.__task_mutex: for task in self.__task_list: - task_start_time = time.time() - while not task.done() and (time.time() - task_start_time < DEFAULT_TASK_TIMEOUT) and ( - time.time() - shutdown_start_time < DEFAULT_SHUTDOWN_TIMEOUT): - time.sleep(sleep_time) - if time.time() - shutdown_start_time >= DEFAULT_SHUTDOWN_TIMEOUT: + task_start_time = datetime.time() + while not task.done() and (datetime.time() - task_start_time < DEFAULT_TASK_TIMEOUT) and ( + datetime.time() - shutdown_start_time < DEFAULT_SHUTDOWN_TIMEOUT): + datetime.sleep(sleep_time) + if datetime.time() - shutdown_start_time >= DEFAULT_SHUTDOWN_TIMEOUT: break self.__task_list = [] @@ -918,7 +918,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self.__task_list = [] self.__task_mutex = threading.Lock() - self.__last_run_time = time.time() + self.__last_run_time = datetime.time() self.__loop = asyncio.new_event_loop() self.__thread = threading.current_thread() self.__loop.set_task_factory(BatchedTaskFactory(self.__loop, self.__thread)) @@ -926,7 +926,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self.__trigger_interval = trigger_interval def __ready_to_run(self) -> bool: - current_time = time.time() + current_time = datetime.time() last_time = self.__last_run_time if len(self.__task_list) <= 0: return False From 18a0dcfa3d0fedb724eb1a57cbc11e146114b7c0 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 18:48:05 +0300 Subject: [PATCH 072/268] Fix issues --- reportportal_client/aio/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 5a9c0d41..7c28fd8f 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -420,7 +420,7 @@ async def log_batch(self, log_batch: Optional[List[AsyncRPRequestLog]]) -> Tuple url = root_uri_join(self.base_url_v2, 'log') if log_batch: response = await AsyncHttpRequest(self.session.post, url=url, - data=AsyncRPLogBatch(log_batch)).make() + data=AsyncRPLogBatch(log_batch).payload).make() return await response.messages def clone(self) -> 'Client': From 4f5c3f2ebfe1fc9c847212d8bf2c9ab9dc0e1c21 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 18:53:19 +0300 Subject: [PATCH 073/268] Fix issues --- reportportal_client/core/rp_requests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 8077423f..4ecd68d4 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -431,7 +431,7 @@ def __get_file(self, rp_file) -> Tuple[str, tuple]: rp_file.content, rp_file.content_type or self.default_content)) - def __get_files(self) -> List[Tuple[str, tuple]]: + def _get_files(self) -> List[Tuple[str, tuple]]: """Get list of files for the JSON body.""" files = [] for req in self.log_reqs: @@ -469,7 +469,7 @@ def payload(self) -> List[Tuple[str, tuple]]: 'text/html'))] """ body = self.__get_request_part() - body.extend(self.__get_files()) + body.extend(self._get_files()) return body @@ -485,11 +485,11 @@ async def __get_request_part(self) -> str: @property async def payload(self) -> aiohttp.MultipartWriter: """Get HTTP payload for the request.""" - json_payload = aiohttp.Payload(await self.__get_request_part(), content_type='application/json') + json_payload = aiohttp.JsonPayload(await self.__get_request_part()) json_payload.set_content_disposition('form-data', name='json_request_part') mpwriter = aiohttp.MultipartWriter('form-data') mpwriter.append_payload(json_payload) - for _, file in self.__get_files(): + for _, file in self._get_files(): file_payload = aiohttp.Payload(file[1], content_type=file[2], filename=file[0]) mpwriter.append_payload(file_payload) return mpwriter From 507fa095b26c6bd814e25e6495c28e3853b654d7 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 14 Sep 2023 18:54:14 +0300 Subject: [PATCH 074/268] Fix issues --- reportportal_client/aio/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 7c28fd8f..2af3d22d 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -969,8 +969,7 @@ def clone(self) -> 'BatchedRPClient': endpoint=None, project=None, launch_uuid=self.launch_uuid, - client=cloned_client, - loop=self.__loop + client=cloned_client ) current_item = self.current_item() if current_item: From 7e70eb432d05b61b1571d07715f2e711d5c85ecb Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 13:08:28 +0300 Subject: [PATCH 075/268] Fix parallel execution issues --- reportportal_client/aio/client.py | 44 +++++++++++++++++-------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 2af3d22d..b6fe06ec 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -660,7 +660,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self.use_own_launch = True @abstractmethod - def create_task(self, coro: Coroutine[Any, Any, _T]) -> Task[_T]: + def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: raise NotImplementedError('"create_task" method is not implemented!') @abstractmethod @@ -825,11 +825,11 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, class ThreadedRPClient(_SyncRPClient): + _loop: Optional[asyncio.AbstractEventLoop] + __self_loop: bool __task_list: List[Task[_T]] __task_mutex: threading.Lock - __loop: Optional[asyncio.AbstractEventLoop] __thread: Optional[threading.Thread] - __self_loop: bool __task_timeout: float __shutdown_timeout: float @@ -845,20 +845,21 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self.__task_mutex = threading.Lock() self.__thread = None if loop: - self.__loop = loop + self._loop = loop self.__self_loop = False else: - self.__loop = asyncio.new_event_loop() - self.__loop.set_task_factory(ThreadedTaskFactory(self.__loop, self.__task_timeout)) + self._loop = asyncio.new_event_loop() + self._loop.set_task_factory(ThreadedTaskFactory(self._loop, self.__task_timeout)) self.__self_loop = True - def create_task(self, coro: Coroutine[Any, Any, _T]) -> Task[_T]: - loop = self.__loop - result = loop.create_task(coro) + def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: + if not getattr(self, '_loop', None): + return + result = self._loop.create_task(coro) with self.__task_mutex: self.__task_list.append(result) if not self.__thread and self.__self_loop: - self.__thread = threading.Thread(target=loop.run_forever, name='RP-Async-Client', + self.__thread = threading.Thread(target=self._loop.run_forever, name='RP-Async-Client', daemon=True) self.__thread.start() i = 0 @@ -894,7 +895,7 @@ def clone(self) -> 'ThreadedRPClient': project=None, launch_uuid=self.launch_uuid, client=cloned_client, - loop=self.__loop + loop=self._loop ) current_item = self.current_item() if current_item: @@ -903,8 +904,9 @@ def clone(self) -> 'ThreadedRPClient': class BatchedRPClient(_SyncRPClient): - __loop: asyncio.AbstractEventLoop + _loop: asyncio.AbstractEventLoop __task_list: List[Task[_T]] + __queued_task_list: List[Task[_T]] __task_mutex: threading.Lock __last_run_time: float __thread: threading.Thread @@ -919,9 +921,9 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self.__task_list = [] self.__task_mutex = threading.Lock() self.__last_run_time = datetime.time() - self.__loop = asyncio.new_event_loop() + self._loop = asyncio.new_event_loop() self.__thread = threading.current_thread() - self.__loop.set_task_factory(BatchedTaskFactory(self.__loop, self.__thread)) + self._loop.set_task_factory(BatchedTaskFactory(self._loop, self.__thread)) self.__trigger_num = trigger_num self.__trigger_interval = trigger_interval @@ -936,16 +938,18 @@ def __ready_to_run(self) -> bool: return True return False - def create_task(self, coro: Coroutine[Any, Any, _T]) -> Task[_T]: - result = self.__loop.create_task(coro) + def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: + if not getattr(self, '_loop', None): + return + result = self._loop.create_task(coro) tasks = None with self.__task_mutex: self.__task_list.append(result) if self.__ready_to_run(): tasks = self.__task_list self.__task_list = [] - if tasks: - self.__loop.run_until_complete(asyncio.gather(*tasks)) + if tasks: + self._loop.run_until_complete(asyncio.gather(*tasks)) return result def finish_tasks(self) -> None: @@ -954,8 +958,8 @@ def finish_tasks(self) -> None: if len(self.__task_list) > 0: tasks = self.__task_list self.__task_list = [] - if tasks: - self.__loop.run_until_complete(asyncio.gather(*tasks)) + if tasks: + self._loop.run_until_complete(asyncio.gather(*tasks)) def clone(self) -> 'BatchedRPClient': """Clone the client object, set current Item ID as cloned item ID. From 00cd2069ef61c8dc25b885a103c8b876c3b62a20 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 15:45:57 +0300 Subject: [PATCH 076/268] Fix payload generation --- reportportal_client/core/rp_requests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 4ecd68d4..16f16e8c 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -478,9 +478,9 @@ class AsyncRPLogBatch(RPLogBatch): def __int__(self, *args, **kwargs) -> None: super.__init__(*args, **kwargs) - async def __get_request_part(self) -> str: + async def __get_request_part(self) -> List[dict]: coroutines = [log.payload for log in self.log_reqs] - return json_converter.dumps(await asyncio.gather(*coroutines)) + return list(await asyncio.gather(*coroutines)) @property async def payload(self) -> aiohttp.MultipartWriter: From 7095219a9767c3181d7fb41a6028bb13f9bb496f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 15:46:11 +0300 Subject: [PATCH 077/268] Bypass log batcher --- reportportal_client/aio/client.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index b6fe06ec..203f1401 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -644,11 +644,15 @@ def launch_uuid(self, value: Optional[Task[str]]) -> None: self.__launch_uuid = value def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, - client: Optional[Client] = None, **kwargs: Any) -> None: + client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, + **kwargs: Any) -> None: set_current(self) self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() - self._log_batcher = LogBatcher() + if log_batcher: + self._log_batcher = log_batcher + else: + self._log_batcher = LogBatcher() if client: self.__client = client else: @@ -834,11 +838,12 @@ class ThreadedRPClient(_SyncRPClient): __shutdown_timeout: float def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, - client: Optional[Client] = None, + client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, loop: Optional[asyncio.AbstractEventLoop] = None, task_timeout: float = DEFAULT_TASK_TIMEOUT, shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, **kwargs: Any) -> None: - super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, **kwargs) + super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, + **kwargs) self.__task_timeout = task_timeout self.__shutdown_timeout = shutdown_timeout self.__task_list = [] @@ -895,6 +900,7 @@ def clone(self) -> 'ThreadedRPClient': project=None, launch_uuid=self.launch_uuid, client=cloned_client, + log_batcher=self._log_batcher, loop=self._loop ) current_item = self.current_item() @@ -914,9 +920,11 @@ class BatchedRPClient(_SyncRPClient): __trigger_interval: float def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, - client: Optional[Client] = None, trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, + client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, + trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL, **kwargs: Any) -> None: - super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, **kwargs) + super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, + **kwargs) self.__task_list = [] self.__task_mutex = threading.Lock() @@ -973,6 +981,7 @@ def clone(self) -> 'BatchedRPClient': endpoint=None, project=None, launch_uuid=self.launch_uuid, + log_batcher=self._log_batcher, client=cloned_client ) current_item = self.current_item() From 69ecbb064373d10743732a5960eb1a5265ee7598 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 16:01:57 +0300 Subject: [PATCH 078/268] Update tasks API --- reportportal_client/aio/client.py | 3 +-- reportportal_client/aio/tasks.py | 21 +++++++-------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 203f1401..f4900684 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -912,7 +912,6 @@ def clone(self) -> 'ThreadedRPClient': class BatchedRPClient(_SyncRPClient): _loop: asyncio.AbstractEventLoop __task_list: List[Task[_T]] - __queued_task_list: List[Task[_T]] __task_mutex: threading.Lock __last_run_time: float __thread: threading.Thread @@ -931,7 +930,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self.__last_run_time = datetime.time() self._loop = asyncio.new_event_loop() self.__thread = threading.current_thread() - self._loop.set_task_factory(BatchedTaskFactory(self._loop, self.__thread)) + self._loop.set_task_factory(BatchedTaskFactory(self._loop)) self.__trigger_num = trigger_num self.__trigger_interval = trigger_interval diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index 99cd7bd8..f31a661b 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -48,27 +48,20 @@ def blocking_result(self) -> _T: class BatchedTask(Generic[_T], Task[_T]): __loop: asyncio.AbstractEventLoop - __thread: threading.Thread def __init__( self, coro: Union[Generator[Future[object], None, _T], Awaitable[_T]], *, loop: asyncio.AbstractEventLoop, - name: Optional[str] = None, - thread: threading.Thread + name: Optional[str] = None ) -> None: super().__init__(coro, loop=loop, name=name) self.__loop = loop - self.__thread = thread def blocking_result(self) -> _T: if self.done(): return self.result() - if self.__thread is not threading.current_thread(): - warnings.warn("The method was called from different thread which was used to create the" - "task, unexpected behavior is possible during the execution.", RuntimeWarning, - stacklevel=3) return self.__loop.run_until_complete(self) @@ -104,18 +97,17 @@ def blocking_result(self) -> _T: class BatchedTaskFactory: __loop: asyncio.AbstractEventLoop - __thread: threading.Thread - def __init__(self, loop: asyncio.AbstractEventLoop, thread: threading.Thread): + def __init__(self, loop: asyncio.AbstractEventLoop): self.__loop = loop - self.__thread = thread def __call__( self, loop: asyncio.AbstractEventLoop, - factory: Union[Coroutine[Any, Any, _T], Generator[Any, None, _T]] + factory: Union[Coroutine[Any, Any, _T], Generator[Any, None, _T]], + **_ ) -> Task[_T]: - return BatchedTask(factory, loop=self.__loop, thread=self.__thread) + return BatchedTask(factory, loop=self.__loop) class ThreadedTaskFactory: @@ -129,6 +121,7 @@ def __init__(self, loop: asyncio.AbstractEventLoop, wait_timeout: float): def __call__( self, loop: asyncio.AbstractEventLoop, - factory: Union[Coroutine[Any, Any, _T], Generator[Any, None, _T]] + factory: Union[Coroutine[Any, Any, _T], Generator[Any, None, _T]], + **_ ) -> Task[_T]: return ThreadedTask(factory, self.__wait_timeout, loop=self.__loop) From c8763fb5700b82bbdc95f429e288e97939981974 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 16:28:27 +0300 Subject: [PATCH 079/268] Add task list class --- reportportal_client/aio/__init__.py | 3 +- reportportal_client/aio/client.py | 42 +++++++++++---------------- reportportal_client/aio/tasks.py | 44 +++++++++++++++++++++++++++-- 3 files changed, 59 insertions(+), 30 deletions(-) diff --git a/reportportal_client/aio/__init__.py b/reportportal_client/aio/__init__.py index 432a4ab3..e3d6bc48 100644 --- a/reportportal_client/aio/__init__.py +++ b/reportportal_client/aio/__init__.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License -from reportportal_client.aio.tasks import (Task, BatchedTask, BatchedTaskFactory, ThreadedTask, +from reportportal_client.aio.tasks import (Task, TaskList, BatchedTask, BatchedTaskFactory, ThreadedTask, ThreadedTaskFactory, BlockingOperationError) DEFAULT_TASK_TIMEOUT: float = 60.0 @@ -21,6 +21,7 @@ __all__ = [ 'Task', + 'TaskList', 'BatchedTask', 'BatchedTaskFactory', 'ThreadedTask', diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index f4900684..8281b8de 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -31,7 +31,7 @@ # noinspection PyProtectedMember from reportportal_client._local import set_current from reportportal_client.aio import (Task, BatchedTaskFactory, ThreadedTaskFactory, DEFAULT_TASK_TIMEOUT, - DEFAULT_SHUTDOWN_TIMEOUT, DEFAULT_TASK_TRIGGER_NUM, + DEFAULT_SHUTDOWN_TIMEOUT, DEFAULT_TASK_TRIGGER_NUM, TaskList, DEFAULT_TASK_TRIGGER_INTERVAL) from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, @@ -911,7 +911,7 @@ def clone(self) -> 'ThreadedRPClient': class BatchedRPClient(_SyncRPClient): _loop: asyncio.AbstractEventLoop - __task_list: List[Task[_T]] + __task_list: TaskList[Task[_T]] __task_mutex: threading.Lock __last_run_time: float __thread: threading.Thread @@ -920,13 +920,19 @@ class BatchedRPClient(_SyncRPClient): def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, + task_list: Optional[TaskList] = None, task_mutex: Optional[threading.Lock], trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL, **kwargs: Any) -> None: super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, **kwargs) - - self.__task_list = [] - self.__task_mutex = threading.Lock() + if task_list: + self.__task_list = task_list + else: + self.__task_list = TaskList() + if task_mutex: + self.__task_mutex = task_mutex + else: + self.__task_mutex = threading.Lock() self.__last_run_time = datetime.time() self._loop = asyncio.new_event_loop() self.__thread = threading.current_thread() @@ -934,37 +940,19 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self.__trigger_num = trigger_num self.__trigger_interval = trigger_interval - def __ready_to_run(self) -> bool: - current_time = datetime.time() - last_time = self.__last_run_time - if len(self.__task_list) <= 0: - return False - if (len(self.__task_list) >= self.__trigger_num - or current_time - last_time >= self.__trigger_interval): - self.__last_run_time = current_time - return True - return False - def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: if not getattr(self, '_loop', None): return result = self._loop.create_task(coro) - tasks = None with self.__task_mutex: - self.__task_list.append(result) - if self.__ready_to_run(): - tasks = self.__task_list - self.__task_list = [] + tasks = self.__task_list.append(result) if tasks: self._loop.run_until_complete(asyncio.gather(*tasks)) return result def finish_tasks(self) -> None: - tasks = None with self.__task_mutex: - if len(self.__task_list) > 0: - tasks = self.__task_list - self.__task_list = [] + tasks = self.__task_list.flush() if tasks: self._loop.run_until_complete(asyncio.gather(*tasks)) @@ -980,8 +968,10 @@ def clone(self) -> 'BatchedRPClient': endpoint=None, project=None, launch_uuid=self.launch_uuid, + client=cloned_client, log_batcher=self._log_batcher, - client=cloned_client + task_list=self.__task_list, + task_mutex=self.__task_mutex ) current_item = self.current_item() if current_item: diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index f31a661b..1b8104e9 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -13,13 +13,12 @@ import asyncio import sys -import threading import time -import warnings from abc import abstractmethod from asyncio import Future -from typing import TypeVar, Generic, Union, Generator, Awaitable, Optional, Coroutine, Any +from typing import TypeVar, Generic, Union, Generator, Awaitable, Optional, Coroutine, Any, List +from reportportal_client.aio import DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL from reportportal_client.static.abstract import AbstractBaseClass _T = TypeVar('_T') @@ -125,3 +124,42 @@ def __call__( **_ ) -> Task[_T]: return ThreadedTask(factory, self.__wait_timeout, loop=self.__loop) + + +class TaskList(Generic[_T]): + + __task_list: List[_T] + __last_run_time: float + __trigger_num: int + __trigger_interval: float + + def __init__(self, + trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, + trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL): + self.__last_run_time = time.time() + self.__trigger_num = trigger_num + self.__trigger_interval = trigger_interval + + def __ready_to_run(self) -> bool: + current_time = time.time() + last_time = self.__last_run_time + if len(self.__task_list) <= 0: + return False + if (len(self.__task_list) >= self.__trigger_num + or current_time - last_time >= self.__trigger_interval): + self.__last_run_time = current_time + return True + return False + + def append(self, value: _T) -> Optional[List[_T]]: + self.__task_list.append(value) + if self.__ready_to_run(): + tasks = self.__task_list + self.__task_list = [] + return tasks + + def flush(self) -> Optional[List[_T]]: + if len(self.__task_list) > 0: + tasks = self.__task_list + self.__task_list = [] + return tasks From 5d5f46acb90c39e2bd1bfe2c359e8b8b736c4489 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 16:32:21 +0300 Subject: [PATCH 080/268] Fix circular import --- reportportal_client/aio/__init__.py | 5 ++--- reportportal_client/aio/tasks.py | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/reportportal_client/aio/__init__.py b/reportportal_client/aio/__init__.py index e3d6bc48..0113623a 100644 --- a/reportportal_client/aio/__init__.py +++ b/reportportal_client/aio/__init__.py @@ -12,12 +12,11 @@ # limitations under the License from reportportal_client.aio.tasks import (Task, TaskList, BatchedTask, BatchedTaskFactory, ThreadedTask, - ThreadedTaskFactory, BlockingOperationError) + ThreadedTaskFactory, BlockingOperationError, + DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL) DEFAULT_TASK_TIMEOUT: float = 60.0 DEFAULT_SHUTDOWN_TIMEOUT: float = 120.0 -DEFAULT_TASK_TRIGGER_NUM: int = 10 -DEFAULT_TASK_TRIGGER_INTERVAL: float = 1.0 __all__ = [ 'Task', diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index 1b8104e9..41961aa8 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -18,10 +18,11 @@ from asyncio import Future from typing import TypeVar, Generic, Union, Generator, Awaitable, Optional, Coroutine, Any, List -from reportportal_client.aio import DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL from reportportal_client.static.abstract import AbstractBaseClass _T = TypeVar('_T') +DEFAULT_TASK_TRIGGER_NUM: int = 10 +DEFAULT_TASK_TRIGGER_INTERVAL: float = 1.0 class BlockingOperationError(RuntimeError): From d12aad1b1718c624e7115042c9cd0732dea11c3b Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 16:46:28 +0300 Subject: [PATCH 081/268] Fix decorator order --- reportportal_client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 24f42f1b..88f6fa3c 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -131,8 +131,8 @@ def start(self) -> None: def terminate(self, *_: Any, **__: Any) -> None: pass # For backward compatibility - @abstractmethod @property + @abstractmethod def launch_uuid(self) -> Optional[Union[str, Task[str]]]: raise NotImplementedError('"launch_uuid" property is not implemented!') From 9e10d247641dfa8076d18524c072417c36f83e6e Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 17:10:27 +0300 Subject: [PATCH 082/268] Fix constructor argument --- reportportal_client/aio/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 8281b8de..d9150d37 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -920,7 +920,7 @@ class BatchedRPClient(_SyncRPClient): def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, - task_list: Optional[TaskList] = None, task_mutex: Optional[threading.Lock], + task_list: Optional[TaskList] = None, task_mutex: Optional[threading.Lock] = None, trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL, **kwargs: Any) -> None: super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, From d4cd4135bdb4178ab2c3acc743337a644e9d07cf Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 17:17:05 +0300 Subject: [PATCH 083/268] Fix list init --- reportportal_client/aio/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index 41961aa8..8b24a366 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -137,6 +137,7 @@ class TaskList(Generic[_T]): def __init__(self, trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL): + self.__task_list = [] self.__last_run_time = time.time() self.__trigger_num = trigger_num self.__trigger_interval = trigger_interval From 95c7b0af35e556793fb38de11883a4685c9cf754 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 17:53:11 +0300 Subject: [PATCH 084/268] Add loop bypass --- reportportal_client/aio/client.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index d9150d37..8ed50183 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -914,15 +914,23 @@ class BatchedRPClient(_SyncRPClient): __task_list: TaskList[Task[_T]] __task_mutex: threading.Lock __last_run_time: float - __thread: threading.Thread __trigger_num: int __trigger_interval: float - def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, - client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, - task_list: Optional[TaskList] = None, task_mutex: Optional[threading.Lock] = None, - trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, - trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL, **kwargs: Any) -> None: + def __init__( + self, endpoint: str, + project: str, + *, + launch_uuid: Optional[Task[str]] = None, + client: Optional[Client] = None, + log_batcher: Optional[LogBatcher] = None, + task_list: Optional[TaskList] = None, + task_mutex: Optional[threading.Lock] = None, + loop: Optional[asyncio.AbstractEventLoop] = None, + trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, + trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL, + **kwargs: Any + ) -> None: super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, **kwargs) if task_list: @@ -934,9 +942,11 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st else: self.__task_mutex = threading.Lock() self.__last_run_time = datetime.time() - self._loop = asyncio.new_event_loop() - self.__thread = threading.current_thread() - self._loop.set_task_factory(BatchedTaskFactory(self._loop)) + if loop: + self._loop = loop + else: + self._loop = asyncio.new_event_loop() + self._loop.set_task_factory(BatchedTaskFactory(self._loop)) self.__trigger_num = trigger_num self.__trigger_interval = trigger_interval @@ -971,7 +981,8 @@ def clone(self) -> 'BatchedRPClient': client=cloned_client, log_batcher=self._log_batcher, task_list=self.__task_list, - task_mutex=self.__task_mutex + task_mutex=self.__task_mutex, + loop=self._loop ) current_item = self.current_item() if current_item: From cd0b90f5ed14efccf9d3251f9a68970321fb0d1b Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 17:53:49 +0300 Subject: [PATCH 085/268] Simplify inits --- reportportal_client/aio/client.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 8ed50183..e9f8dbce 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -933,14 +933,8 @@ def __init__( ) -> None: super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, **kwargs) - if task_list: - self.__task_list = task_list - else: - self.__task_list = TaskList() - if task_mutex: - self.__task_mutex = task_mutex - else: - self.__task_mutex = threading.Lock() + self.__task_list = task_list or TaskList() + self.__task_mutex = task_mutex or threading.Lock() self.__last_run_time = datetime.time() if loop: self._loop = loop From dab6a576d8f665a51dfd4c0bc14dcf3c002b0b33 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 18:04:00 +0300 Subject: [PATCH 086/268] Add endpoint property --- reportportal_client/aio/client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index e9f8dbce..99c8a040 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -628,6 +628,7 @@ class _SyncRPClient(RP, metaclass=AbstractBaseClass): _log_batcher: LogBatcher __client: Client __launch_uuid: Optional[Task[str]] + __endpoint: str use_own_launch: bool step_reporter: StepReporter @@ -643,10 +644,14 @@ def launch_uuid(self) -> Optional[Task[str]]: def launch_uuid(self, value: Optional[Task[str]]) -> None: self.__launch_uuid = value + @property + def endpoint(self) -> str: + return self.endpoint + def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, **kwargs: Any) -> None: - set_current(self) + self.__endpoint = endpoint self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() if log_batcher: @@ -662,6 +667,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self.use_own_launch = False else: self.use_own_launch = True + set_current(self) @abstractmethod def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: From 358dd5a322b53b4d2192f034673dd67c1a99397f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 18:05:46 +0300 Subject: [PATCH 087/268] Recursion fix --- reportportal_client/aio/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 99c8a040..0c12375e 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -646,7 +646,7 @@ def launch_uuid(self, value: Optional[Task[str]]) -> None: @property def endpoint(self) -> str: - return self.endpoint + return self.__endpoint def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, From a4e13527906d988116739f7b513e5d0df2a83904 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 15 Sep 2023 18:14:35 +0300 Subject: [PATCH 088/268] Add log batcher bypass --- reportportal_client/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 88f6fa3c..143f29b8 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -214,6 +214,7 @@ def __init__( mode: str = 'DEFAULT', launch_uuid_print: bool = False, print_output: Optional[TextIO] = None, + log_batcher: Optional[LogBatcher[RPRequestLog]] = None, **kwargs: Any ) -> None: """Initialize required attributes. @@ -259,7 +260,7 @@ def __init__( self.use_own_launch = bool(self.__launch_uuid) self.log_batch_size = log_batch_size self.log_batch_payload_size = log_batch_payload_size - self._log_batcher = LogBatcher(self.log_batch_size, self.log_batch_payload_size) + self._log_batcher = log_batcher or LogBatcher(self.log_batch_size, self.log_batch_payload_size) self.verify_ssl = verify_ssl self.retries = retries self.max_pool_size = max_pool_size @@ -671,7 +672,8 @@ def clone(self) -> 'RPClient': launch_uuid=self.launch_uuid, http_timeout=self.http_timeout, log_batch_payload_size=self.log_batch_payload_size, - mode=self.mode + mode=self.mode, + log_batcher=self._log_batcher ) current_item = self.current_item() if current_item: From 79e2f8d957935f57393b0917b7da1cc6acdd32f0 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Sat, 16 Sep 2023 10:35:08 +0300 Subject: [PATCH 089/268] Add project property --- reportportal_client/aio/client.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 0c12375e..1b4ab53e 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -629,6 +629,7 @@ class _SyncRPClient(RP, metaclass=AbstractBaseClass): __client: Client __launch_uuid: Optional[Task[str]] __endpoint: str + __project: str use_own_launch: bool step_reporter: StepReporter @@ -648,10 +649,15 @@ def launch_uuid(self, value: Optional[Task[str]]) -> None: def endpoint(self) -> str: return self.__endpoint + @property + def project(self) -> str: + return self.__project + def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, **kwargs: Any) -> None: self.__endpoint = endpoint + self.__project = project self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() if log_batcher: From 9fc94f0d906537cde874c46c6245b39bda8ce55e Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Sat, 16 Sep 2023 10:40:45 +0300 Subject: [PATCH 090/268] Add task repr and str methods --- reportportal_client/aio/tasks.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index 8b24a366..bdfb0847 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -64,6 +64,16 @@ def blocking_result(self) -> _T: return self.result() return self.__loop.run_until_complete(self) + def __repr__(self) -> str: + if self.done(): + return str(self.result()) + return super().__repr__() + + def __str__(self): + if self.done(): + return str(self.result()) + return super().__str__() + class ThreadedTask(Generic[_T], Task[_T]): __loop: asyncio.AbstractEventLoop @@ -94,6 +104,16 @@ def blocking_result(self) -> _T: raise BlockingOperationError('Timed out waiting for the task execution') return self.result() + def __repr__(self) -> str: + if self.done(): + return str(self.result()) + return super().__repr__() + + def __str__(self): + if self.done(): + return str(self.result()) + return super().__str__() + class BatchedTaskFactory: __loop: asyncio.AbstractEventLoop From d29cf33717bfd649b9bbee6d1024f36f9d54ac00 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Sat, 16 Sep 2023 11:26:35 +0300 Subject: [PATCH 091/268] Add task repr and str methods --- reportportal_client/aio/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index bdfb0847..045640fc 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -66,7 +66,7 @@ def blocking_result(self) -> _T: def __repr__(self) -> str: if self.done(): - return str(self.result()) + return repr(self.result()) return super().__repr__() def __str__(self): @@ -106,7 +106,7 @@ def blocking_result(self) -> _T: def __repr__(self) -> str: if self.done(): - return str(self.result()) + return repr(self.result()) return super().__repr__() def __str__(self): From 566830c32af7a8914cf763bed34af94c03970a1d Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Sat, 16 Sep 2023 11:39:12 +0300 Subject: [PATCH 092/268] Add log append on launch finish --- reportportal_client/aio/client.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 1b4ab53e..e4cd2b31 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -818,8 +818,11 @@ def get_project_settings(self) -> Task[dict]: result_task = self.create_task(result_coro) return result_task + async def _log_batch(self, log_rq: Optional[List[AsyncRPRequestLog]]) -> Optional[Tuple[str, ...]]: + return await self.__client.log_batch(log_rq) + async def _log(self, log_rq: AsyncRPRequestLog) -> Optional[Tuple[str, ...]]: - return await self.__client.log_batch(await self._log_batcher.append_async(log_rq)) + return await self._log_batch(await self._log_batcher.append_async(log_rq)) def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, item_id: Optional[Task[str]] = None) -> None: @@ -844,7 +847,7 @@ class ThreadedRPClient(_SyncRPClient): _loop: Optional[asyncio.AbstractEventLoop] __self_loop: bool __task_list: List[Task[_T]] - __task_mutex: threading.Lock + __task_mutex: threading.RLock __thread: Optional[threading.Thread] __task_timeout: float __shutdown_timeout: float @@ -859,7 +862,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self.__task_timeout = task_timeout self.__shutdown_timeout = shutdown_timeout self.__task_list = [] - self.__task_mutex = threading.Lock() + self.__task_mutex = threading.RLock() self.__thread = None if loop: self._loop = loop @@ -924,7 +927,7 @@ def clone(self) -> 'ThreadedRPClient': class BatchedRPClient(_SyncRPClient): _loop: asyncio.AbstractEventLoop __task_list: TaskList[Task[_T]] - __task_mutex: threading.Lock + __task_mutex: threading.RLock __last_run_time: float __trigger_num: int __trigger_interval: float @@ -937,7 +940,7 @@ def __init__( client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, task_list: Optional[TaskList] = None, - task_mutex: Optional[threading.Lock] = None, + task_mutex: Optional[threading.RLock] = None, loop: Optional[asyncio.AbstractEventLoop] = None, trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL, @@ -946,7 +949,7 @@ def __init__( super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, **kwargs) self.__task_list = task_list or TaskList() - self.__task_mutex = task_mutex or threading.Lock() + self.__task_mutex = task_mutex or threading.RLock() self.__last_run_time = datetime.time() if loop: self._loop = loop @@ -971,6 +974,10 @@ def finish_tasks(self) -> None: tasks = self.__task_list.flush() if tasks: self._loop.run_until_complete(asyncio.gather(*tasks)) + logs = self._log_batcher.flush() + if logs: + log_task = self._loop.create_task(self._log_batch(logs)) + self._loop.run_until_complete(log_task) def clone(self) -> 'BatchedRPClient': """Clone the client object, set current Item ID as cloned item ID. From cd2e717b5943cd8e194e354211ee070c197a8a47 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Sat, 16 Sep 2023 13:03:38 +0300 Subject: [PATCH 093/268] Fix task wait --- reportportal_client/aio/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index 045640fc..77191bdd 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -98,7 +98,7 @@ def blocking_result(self) -> _T: raise BlockingOperationError('Running loop is not alive') start_time = time.time() slee_time = sys.getswitchinterval() - while not self.done() or time.time() - start_time < self.__wait_timeout: + while not self.done() and time.time() - start_time < self.__wait_timeout: time.sleep(slee_time) if not self.done(): raise BlockingOperationError('Timed out waiting for the task execution') From 07235103479d90b48cf843d14a79da620942f6f5 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 18 Sep 2023 17:42:53 +0300 Subject: [PATCH 094/268] Fix ThreadedRPClient --- reportportal_client/aio/__init__.py | 10 +++-- reportportal_client/aio/client.py | 57 ++++++++++++++++------------- reportportal_client/aio/tasks.py | 29 ++++++++++++++- 3 files changed, 65 insertions(+), 31 deletions(-) diff --git a/reportportal_client/aio/__init__.py b/reportportal_client/aio/__init__.py index 0113623a..fa047c58 100644 --- a/reportportal_client/aio/__init__.py +++ b/reportportal_client/aio/__init__.py @@ -11,16 +11,18 @@ # See the License for the specific language governing permissions and # limitations under the License -from reportportal_client.aio.tasks import (Task, TaskList, BatchedTask, BatchedTaskFactory, ThreadedTask, - ThreadedTaskFactory, BlockingOperationError, - DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL) +from reportportal_client.aio.tasks import (Task, TriggerTaskList, BatchedTask, BatchedTaskFactory, + ThreadedTask, ThreadedTaskFactory, BlockingOperationError, + BackgroundTaskList, DEFAULT_TASK_TRIGGER_NUM, + DEFAULT_TASK_TRIGGER_INTERVAL) DEFAULT_TASK_TIMEOUT: float = 60.0 DEFAULT_SHUTDOWN_TIMEOUT: float = 120.0 __all__ = [ 'Task', - 'TaskList', + 'TriggerTaskList', + 'BackgroundTaskList', 'BatchedTask', 'BatchedTaskFactory', 'ThreadedTask', diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index e4cd2b31..f2fd5d7e 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -31,8 +31,8 @@ # noinspection PyProtectedMember from reportportal_client._local import set_current from reportportal_client.aio import (Task, BatchedTaskFactory, ThreadedTaskFactory, DEFAULT_TASK_TIMEOUT, - DEFAULT_SHUTDOWN_TIMEOUT, DEFAULT_TASK_TRIGGER_NUM, TaskList, - DEFAULT_TASK_TRIGGER_INTERVAL) + DEFAULT_SHUTDOWN_TIMEOUT, DEFAULT_TASK_TRIGGER_NUM, TriggerTaskList, + DEFAULT_TASK_TRIGGER_INTERVAL, BackgroundTaskList) from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, AsyncItemFinishRequest, LaunchFinishRequest, RPFile, @@ -846,23 +846,33 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, class ThreadedRPClient(_SyncRPClient): _loop: Optional[asyncio.AbstractEventLoop] __self_loop: bool - __task_list: List[Task[_T]] + __task_list: BackgroundTaskList[Task[_T]] __task_mutex: threading.RLock __thread: Optional[threading.Thread] __task_timeout: float __shutdown_timeout: float - def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, - client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, - loop: Optional[asyncio.AbstractEventLoop] = None, - task_timeout: float = DEFAULT_TASK_TIMEOUT, - shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, **kwargs: Any) -> None: + def __init__( + self, + endpoint: str, + project: str, + *, + launch_uuid: Optional[Task[str]] = None, + client: Optional[Client] = None, + log_batcher: Optional[LogBatcher] = None, + task_list: Optional[BackgroundTaskList[Task[_T]]] = None, + task_mutex: Optional[threading.RLock] = None, + loop: Optional[asyncio.AbstractEventLoop] = None, + task_timeout: float = DEFAULT_TASK_TIMEOUT, + shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, + **kwargs: Any + ) -> None: super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, **kwargs) self.__task_timeout = task_timeout self.__shutdown_timeout = shutdown_timeout - self.__task_list = [] - self.__task_mutex = threading.RLock() + self.__task_list = task_list or BackgroundTaskList() + self.__task_mutex = task_mutex or threading.RLock() self.__thread = None if loop: self._loop = loop @@ -871,6 +881,8 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[st self._loop = asyncio.new_event_loop() self._loop.set_task_factory(ThreadedTaskFactory(self._loop, self.__task_timeout)) self.__self_loop = True + self.__thread = threading.Thread(target=self._loop.run_forever, name='RP-Async-Client', + daemon=True) def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: if not getattr(self, '_loop', None): @@ -878,29 +890,22 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: result = self._loop.create_task(coro) with self.__task_mutex: self.__task_list.append(result) - if not self.__thread and self.__self_loop: - self.__thread = threading.Thread(target=self._loop.run_forever, name='RP-Async-Client', - daemon=True) + if self.__self_loop and not self.__thread.is_alive(): self.__thread.start() - i = 0 - for i, task in enumerate(self.__task_list): - if not task.done(): - break - self.__task_list = self.__task_list[i:] return result def finish_tasks(self): sleep_time = sys.getswitchinterval() shutdown_start_time = datetime.time() with self.__task_mutex: - for task in self.__task_list: + tasks = self.__task_list.flush() + for task in tasks: task_start_time = datetime.time() while not task.done() and (datetime.time() - task_start_time < DEFAULT_TASK_TIMEOUT) and ( datetime.time() - shutdown_start_time < DEFAULT_SHUTDOWN_TIMEOUT): datetime.sleep(sleep_time) if datetime.time() - shutdown_start_time >= DEFAULT_SHUTDOWN_TIMEOUT: break - self.__task_list = [] def clone(self) -> 'ThreadedRPClient': """Clone the client object, set current Item ID as cloned item ID. @@ -916,6 +921,8 @@ def clone(self) -> 'ThreadedRPClient': launch_uuid=self.launch_uuid, client=cloned_client, log_batcher=self._log_batcher, + task_mutex=self.__task_mutex, + task_list=self.__task_list, loop=self._loop ) current_item = self.current_item() @@ -926,7 +933,7 @@ def clone(self) -> 'ThreadedRPClient': class BatchedRPClient(_SyncRPClient): _loop: asyncio.AbstractEventLoop - __task_list: TaskList[Task[_T]] + __task_list: TriggerTaskList[Task[_T]] __task_mutex: threading.RLock __last_run_time: float __trigger_num: int @@ -939,7 +946,7 @@ def __init__( launch_uuid: Optional[Task[str]] = None, client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, - task_list: Optional[TaskList] = None, + task_list: Optional[TriggerTaskList] = None, task_mutex: Optional[threading.RLock] = None, loop: Optional[asyncio.AbstractEventLoop] = None, trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, @@ -948,7 +955,7 @@ def __init__( ) -> None: super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, **kwargs) - self.__task_list = task_list or TaskList() + self.__task_list = task_list or TriggerTaskList() self.__task_mutex = task_mutex or threading.RLock() self.__last_run_time = datetime.time() if loop: @@ -966,14 +973,14 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: with self.__task_mutex: tasks = self.__task_list.append(result) if tasks: - self._loop.run_until_complete(asyncio.gather(*tasks)) + self._loop.run_until_complete(asyncio.wait(tasks)) return result def finish_tasks(self) -> None: with self.__task_mutex: tasks = self.__task_list.flush() if tasks: - self._loop.run_until_complete(asyncio.gather(*tasks)) + self._loop.run_until_complete(asyncio.wait(tasks)) logs = self._log_batcher.flush() if logs: log_task = self._loop.create_task(self._log_batch(logs)) diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index 77191bdd..a7aa87ee 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -147,8 +147,7 @@ def __call__( return ThreadedTask(factory, self.__wait_timeout, loop=self.__loop) -class TaskList(Generic[_T]): - +class TriggerTaskList(Generic[_T]): __task_list: List[_T] __last_run_time: float __trigger_num: int @@ -185,3 +184,29 @@ def flush(self) -> Optional[List[_T]]: tasks = self.__task_list self.__task_list = [] return tasks + + +class BackgroundTaskList(Generic[_T]): + __task_list: List[_T] + + def __init__(self): + self.__task_list = [] + + def __remove_finished(self): + i = -1 + for task in self.__task_list: + if not task.done(): + break + i += 1 + self.__task_list = self.__task_list[i + 1:] + + def append(self, value: _T) -> None: + self.__remove_finished() + self.__task_list.append(value) + + def flush(self) -> Optional[List[_T]]: + self.__remove_finished() + if len(self.__task_list) > 0: + tasks = self.__task_list + self.__task_list = [] + return tasks From 62a01c94ee620e3723c6f607dfd5d1379cf3af88 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 18 Sep 2023 23:45:24 +0300 Subject: [PATCH 095/268] Fix ThreadedRPClient --- reportportal_client/aio/client.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index f2fd5d7e..14f4ab9e 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -77,6 +77,7 @@ class Client: retries: int max_pool_size: int http_timeout: Union[float, Tuple[float, float]] + keepalive_timeout: Optional[float] mode: str launch_uuid_print: Optional[bool] print_output: Optional[TextIO] @@ -96,6 +97,7 @@ def __init__( retries: int = None, max_pool_size: int = 50, http_timeout: Union[float, Tuple[float, float]] = (10, 10), + keepalive_timeout: Optional[float] = None, log_batch_payload_size: int = MAX_LOG_BATCH_PAYLOAD_SIZE, mode: str = 'DEFAULT', launch_uuid_print: bool = False, @@ -114,6 +116,7 @@ def __init__( self.retries = retries self.max_pool_size = max_pool_size self.http_timeout = http_timeout + self.keepalive_timeout = keepalive_timeout self.mode = mode self._skip_analytics = getenv('AGENT_NO_ANALYTICS') self.launch_uuid_print = launch_uuid_print @@ -157,7 +160,8 @@ def session(self) -> aiohttp.ClientSession: else: ssl_config = ssl.create_default_context(cafile=certifi.where()) - connector = aiohttp.TCPConnector(ssl=ssl_config, limit=self.max_pool_size) + connector = aiohttp.TCPConnector(ssl=ssl_config, limit=self.max_pool_size, + keepalive_timeout=self.keepalive_timeout) timeout = None if self.http_timeout: @@ -439,8 +443,11 @@ def clone(self) -> 'Client': retries=self.retries, max_pool_size=self.max_pool_size, http_timeout=self.http_timeout, + keepalive_timeout=self.keepalive_timeout, log_batch_payload_size=self.log_batch_payload_size, - mode=self.mode + mode=self.mode, + launch_uuid_print=self.launch_uuid_print, + print_output=self.print_output ) return cloned @@ -845,7 +852,6 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, class ThreadedRPClient(_SyncRPClient): _loop: Optional[asyncio.AbstractEventLoop] - __self_loop: bool __task_list: BackgroundTaskList[Task[_T]] __task_mutex: threading.RLock __thread: Optional[threading.Thread] @@ -876,13 +882,18 @@ def __init__( self.__thread = None if loop: self._loop = loop - self.__self_loop = False else: self._loop = asyncio.new_event_loop() self._loop.set_task_factory(ThreadedTaskFactory(self._loop, self.__task_timeout)) - self.__self_loop = True + self.__heartbeat() self.__thread = threading.Thread(target=self._loop.run_forever, name='RP-Async-Client', daemon=True) + self.__thread.start() + + def __heartbeat(self): + # We operate on our own loop with daemon thread, so we will exit in any way when main thread exit, + # so we can iterate forever + self._loop.call_at(self._loop.time() + 1, self.__heartbeat) def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: if not getattr(self, '_loop', None): @@ -890,8 +901,6 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: result = self._loop.create_task(coro) with self.__task_mutex: self.__task_list.append(result) - if self.__self_loop and not self.__thread.is_alive(): - self.__thread.start() return result def finish_tasks(self): From 3427713ee42e6860b8dad8a864683817bd8ed9a3 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 19 Sep 2023 00:04:36 +0300 Subject: [PATCH 096/268] Fix ThreadedRPClient --- reportportal_client/aio/client.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 14f4ab9e..d9621a21 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -160,8 +160,13 @@ def session(self) -> aiohttp.ClientSession: else: ssl_config = ssl.create_default_context(cafile=certifi.where()) - connector = aiohttp.TCPConnector(ssl=ssl_config, limit=self.max_pool_size, - keepalive_timeout=self.keepalive_timeout) + params = { + 'ssl': ssl_config, + 'limit': self.max_pool_size + } + if self.keepalive_timeout: + params['keepalive_timeout'] = self.keepalive_timeout + connector = aiohttp.TCPConnector(**params) timeout = None if self.http_timeout: From 26ac3adcef3d05d2882fec106a1acb4b526a2794 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 19 Sep 2023 00:10:19 +0300 Subject: [PATCH 097/268] Fix ThreadedRPClient --- reportportal_client/aio/client.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index d9621a21..d1d07e95 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -913,13 +913,13 @@ def finish_tasks(self): shutdown_start_time = datetime.time() with self.__task_mutex: tasks = self.__task_list.flush() - for task in tasks: - task_start_time = datetime.time() - while not task.done() and (datetime.time() - task_start_time < DEFAULT_TASK_TIMEOUT) and ( - datetime.time() - shutdown_start_time < DEFAULT_SHUTDOWN_TIMEOUT): - datetime.sleep(sleep_time) - if datetime.time() - shutdown_start_time >= DEFAULT_SHUTDOWN_TIMEOUT: - break + for task in tasks: + task_start_time = datetime.time() + while not task.done() and (datetime.time() - task_start_time < DEFAULT_TASK_TIMEOUT) and ( + datetime.time() - shutdown_start_time < DEFAULT_SHUTDOWN_TIMEOUT): + datetime.sleep(sleep_time) + if datetime.time() - shutdown_start_time >= DEFAULT_SHUTDOWN_TIMEOUT: + break def clone(self) -> 'ThreadedRPClient': """Clone the client object, set current Item ID as cloned item ID. From e3c9a080d3a033cd3e49be4bfc1dba48a989c2d6 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 19 Sep 2023 09:20:55 +0300 Subject: [PATCH 098/268] Fix logging at the end of reporting --- reportportal_client/aio/client.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index d1d07e95..0ead27ba 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -784,7 +784,7 @@ def finish_launch(self, result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, **kwargs) else: - result_coro = self.create_task(self.__empty_str()) + result_coro = self.__empty_str() result_task = self.create_task(result_coro) self.finish_tasks() @@ -898,7 +898,7 @@ def __init__( def __heartbeat(self): # We operate on our own loop with daemon thread, so we will exit in any way when main thread exit, # so we can iterate forever - self._loop.call_at(self._loop.time() + 1, self.__heartbeat) + self._loop.call_at(self._loop.time() + 0.1, self.__heartbeat) def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: if not getattr(self, '_loop', None): @@ -920,6 +920,13 @@ def finish_tasks(self): datetime.sleep(sleep_time) if datetime.time() - shutdown_start_time >= DEFAULT_SHUTDOWN_TIMEOUT: break + logs = self._log_batcher.flush() + if logs: + task_start_time = datetime.time() + log_task = self._loop.create_task(self._log_batch(logs)) + while not log_task.done() and (datetime.time() - task_start_time < DEFAULT_TASK_TIMEOUT) and ( + datetime.time() - shutdown_start_time < DEFAULT_SHUTDOWN_TIMEOUT): + datetime.sleep(sleep_time) def clone(self) -> 'ThreadedRPClient': """Clone the client object, set current Item ID as cloned item ID. From e90a8f8ae8fafd7ecd6705c5d6032cb6d4a7cbca Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 20 Sep 2023 12:09:06 +0300 Subject: [PATCH 099/268] Add client closing on Launch finish --- reportportal_client/aio/client.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 0ead27ba..b8ed9c66 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -183,6 +183,10 @@ def session(self) -> aiohttp.ClientSession: timeout=timeout) return self.__session + def close(self): + if self.__session: + self.__session.close() + async def __get_item_url(self, item_id_future: Union[str, Task[str]]) -> Optional[str]: item_id = await await_if_necessary(item_id_future) if item_id is NOT_FOUND: @@ -546,9 +550,11 @@ async def finish_launch(self, await self.__client.log_batch(self._log_batcher.flush()) if not self.use_own_launch: return "" - return await self.__client.finish_launch(self.launch_uuid, end_time, status=status, - attributes=attributes, - **kwargs) + result = await self.__client.finish_launch(self.launch_uuid, end_time, status=status, + attributes=attributes, + **kwargs) + self.__client.close() + return result async def update_test_item(self, item_uuid: str, attributes: Optional[Union[List, Dict]] = None, description: Optional[str] = None) -> Optional[str]: @@ -788,6 +794,7 @@ def finish_launch(self, result_task = self.create_task(result_coro) self.finish_tasks() + self.__client.close() return result_task def update_test_item(self, From 3166cf7780a23f198293da283393161d8d0cfecb Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 20 Sep 2023 13:00:04 +0300 Subject: [PATCH 100/268] Add client closing on Launch finish --- reportportal_client/aio/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index b8ed9c66..28aea003 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -186,6 +186,7 @@ def session(self) -> aiohttp.ClientSession: def close(self): if self.__session: self.__session.close() + self.__session = None async def __get_item_url(self, item_id_future: Union[str, Task[str]]) -> Optional[str]: item_id = await await_if_necessary(item_id_future) From 519cb6fbd9c8a393c42155965b47f97c6ec2b44c Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 20 Sep 2023 15:16:09 +0300 Subject: [PATCH 101/268] Refactor task timeouts --- reportportal_client/aio/client.py | 46 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 28aea003..13caf2d4 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -645,6 +645,8 @@ class _SyncRPClient(RP, metaclass=AbstractBaseClass): _item_stack: _LifoQueue _log_batcher: LogBatcher + _shutdown_timeout: float + _task_timeout: float __client: Client __launch_uuid: Optional[Task[str]] __endpoint: str @@ -672,13 +674,24 @@ def endpoint(self) -> str: def project(self) -> str: return self.__project - def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[Task[str]] = None, - client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, - **kwargs: Any) -> None: + def __init__( + self, + endpoint: str, + project: str, + *, + launch_uuid: Optional[Task[str]] = None, + client: Optional[Client] = None, + log_batcher: Optional[LogBatcher] = None, + task_timeout: float = DEFAULT_TASK_TIMEOUT, + shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, + **kwargs: Any + ) -> None: self.__endpoint = endpoint self.__project = project self.step_reporter = StepReporter(self) self._item_stack = _LifoQueue() + self._shutdown_timeout = shutdown_timeout + self._task_timeout = task_timeout if log_batcher: self._log_batcher = log_batcher else: @@ -868,8 +881,6 @@ class ThreadedRPClient(_SyncRPClient): __task_list: BackgroundTaskList[Task[_T]] __task_mutex: threading.RLock __thread: Optional[threading.Thread] - __task_timeout: float - __shutdown_timeout: float def __init__( self, @@ -882,14 +893,10 @@ def __init__( task_list: Optional[BackgroundTaskList[Task[_T]]] = None, task_mutex: Optional[threading.RLock] = None, loop: Optional[asyncio.AbstractEventLoop] = None, - task_timeout: float = DEFAULT_TASK_TIMEOUT, - shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, **kwargs: Any ) -> None: super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, **kwargs) - self.__task_timeout = task_timeout - self.__shutdown_timeout = shutdown_timeout self.__task_list = task_list or BackgroundTaskList() self.__task_mutex = task_mutex or threading.RLock() self.__thread = None @@ -897,7 +904,7 @@ def __init__( self._loop = loop else: self._loop = asyncio.new_event_loop() - self._loop.set_task_factory(ThreadedTaskFactory(self._loop, self.__task_timeout)) + self._loop.set_task_factory(ThreadedTaskFactory(self._loop, self._task_timeout)) self.__heartbeat() self.__thread = threading.Thread(target=self._loop.run_forever, name='RP-Async-Client', daemon=True) @@ -917,24 +924,17 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: return result def finish_tasks(self): - sleep_time = sys.getswitchinterval() shutdown_start_time = datetime.time() with self.__task_mutex: tasks = self.__task_list.flush() for task in tasks: - task_start_time = datetime.time() - while not task.done() and (datetime.time() - task_start_time < DEFAULT_TASK_TIMEOUT) and ( - datetime.time() - shutdown_start_time < DEFAULT_SHUTDOWN_TIMEOUT): - datetime.sleep(sleep_time) - if datetime.time() - shutdown_start_time >= DEFAULT_SHUTDOWN_TIMEOUT: + task.blocking_result() + if datetime.time() - shutdown_start_time >= self._shutdown_timeout: break logs = self._log_batcher.flush() if logs: - task_start_time = datetime.time() - log_task = self._loop.create_task(self._log_batch(logs)) - while not log_task.done() and (datetime.time() - task_start_time < DEFAULT_TASK_TIMEOUT) and ( - datetime.time() - shutdown_start_time < DEFAULT_SHUTDOWN_TIMEOUT): - datetime.sleep(sleep_time) + log_task: Task[Tuple[str, ...]] = self._loop.create_task(self._log_batch(logs)) + log_task.blocking_result() def clone(self) -> 'ThreadedRPClient': """Clone the client object, set current Item ID as cloned item ID. @@ -1002,14 +1002,14 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: with self.__task_mutex: tasks = self.__task_list.append(result) if tasks: - self._loop.run_until_complete(asyncio.wait(tasks)) + self._loop.run_until_complete(asyncio.wait(tasks, timeout=self._task_timeout)) return result def finish_tasks(self) -> None: with self.__task_mutex: tasks = self.__task_list.flush() if tasks: - self._loop.run_until_complete(asyncio.wait(tasks)) + self._loop.run_until_complete(asyncio.wait(tasks, timeout=self._shutdown_timeout)) logs = self._log_batcher.flush() if logs: log_task = self._loop.create_task(self._log_batch(logs)) From c38ccb3edda71c08a66d521e13e4636de201f75a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 20 Sep 2023 15:33:21 +0300 Subject: [PATCH 102/268] Fix HTTP session close --- reportportal_client/aio/client.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 13caf2d4..fa6d55b5 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -183,9 +183,9 @@ def session(self) -> aiohttp.ClientSession: timeout=timeout) return self.__session - def close(self): + async def close(self): if self.__session: - self.__session.close() + await self.__session.close() self.__session = None async def __get_item_url(self, item_id_future: Union[str, Task[str]]) -> Optional[str]: @@ -554,7 +554,7 @@ async def finish_launch(self, result = await self.__client.finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, **kwargs) - self.__client.close() + await self.__client.close() return result async def update_test_item(self, item_uuid: str, attributes: Optional[Union[List, Dict]] = None, @@ -640,7 +640,7 @@ def clone(self) -> 'AsyncRPClient': return cloned -class _SyncRPClient(RP, metaclass=AbstractBaseClass): +class _RPClient(RP, metaclass=AbstractBaseClass): __metaclass__ = AbstractBaseClass _item_stack: _LifoQueue @@ -808,7 +808,6 @@ def finish_launch(self, result_task = self.create_task(result_coro) self.finish_tasks() - self.__client.close() return result_task def update_test_item(self, @@ -875,8 +874,11 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, self.create_task(self._log(rp_log)) return None + async def _close(self): + await self.__client.close() -class ThreadedRPClient(_SyncRPClient): + +class ThreadedRPClient(_RPClient): _loop: Optional[asyncio.AbstractEventLoop] __task_list: BackgroundTaskList[Task[_T]] __task_mutex: threading.RLock @@ -933,8 +935,8 @@ def finish_tasks(self): break logs = self._log_batcher.flush() if logs: - log_task: Task[Tuple[str, ...]] = self._loop.create_task(self._log_batch(logs)) - log_task.blocking_result() + self._loop.create_task(self._log_batch(logs)).blocking_result() + self._loop.create_task(self._close()).blocking_result() def clone(self) -> 'ThreadedRPClient': """Clone the client object, set current Item ID as cloned item ID. @@ -960,7 +962,7 @@ def clone(self) -> 'ThreadedRPClient': return cloned -class BatchedRPClient(_SyncRPClient): +class BatchedRPClient(_RPClient): _loop: asyncio.AbstractEventLoop __task_list: TriggerTaskList[Task[_T]] __task_mutex: threading.RLock @@ -1014,6 +1016,7 @@ def finish_tasks(self) -> None: if logs: log_task = self._loop.create_task(self._log_batch(logs)) self._loop.run_until_complete(log_task) + self._loop.run_until_complete(self._close()) def clone(self) -> 'BatchedRPClient': """Clone the client object, set current Item ID as cloned item ID. From 2ebefa4fe0f0b2bab43bb908c3c6f8a356d5c940 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 20 Sep 2023 15:48:43 +0300 Subject: [PATCH 103/268] Misprint fix --- reportportal_client/aio/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index a7aa87ee..5a6ec151 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -97,9 +97,9 @@ def blocking_result(self) -> _T: if not self.__loop.is_running() or self.__loop.is_closed(): raise BlockingOperationError('Running loop is not alive') start_time = time.time() - slee_time = sys.getswitchinterval() + sleep_time = sys.getswitchinterval() while not self.done() and time.time() - start_time < self.__wait_timeout: - time.sleep(slee_time) + time.sleep(sleep_time) if not self.done(): raise BlockingOperationError('Timed out waiting for the task execution') return self.result() From c4d2403aea835d172e6acf15b477dd3d69153bdd Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 20 Sep 2023 18:08:43 +0300 Subject: [PATCH 104/268] Remove Python 2.7 artifacts --- reportportal_client/core/rp_issues.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reportportal_client/core/rp_issues.py b/reportportal_client/core/rp_issues.py index 1449a2bc..c860a5d7 100644 --- a/reportportal_client/core/rp_issues.py +++ b/reportportal_client/core/rp_issues.py @@ -40,7 +40,7 @@ # limitations under the License -class Issue(object): +class Issue: """This class represents an issue that can be attached to test result.""" def __init__(self, @@ -81,7 +81,7 @@ def payload(self): } -class ExternalIssue(object): +class ExternalIssue: """This class represents external(BTS) system issue.""" def __init__(self, From 6c24c4d6f396734a898b069c708d362ec984c381 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 20 Sep 2023 18:29:15 +0300 Subject: [PATCH 105/268] Add HTTP Retry client --- reportportal_client/aio/http.py | 59 +++++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 60 insertions(+) create mode 100644 reportportal_client/aio/http.py diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py new file mode 100644 index 00000000..70fab657 --- /dev/null +++ b/reportportal_client/aio/http.py @@ -0,0 +1,59 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import aiohttp +from aenum import auto, Enum +from aiohttp import ClientResponse +from sympy import fibonacci + +DEFAULT_RETRY_NUMBER: int = 5 +DEFAULT_RETRY_DELAY: int = 10 + + +class RetryClass(Enum): + CONNECTION_ERROR = auto() + SERVER_ERROR = auto() + BAD_REQUEST = auto() + THROTTLING = auto() + + +RETRY_FACTOR = { + RetryClass.CONNECTION_ERROR: 5, + RetryClass.SERVER_ERROR: 1, + RetryClass.BAD_REQUEST: 0, + RetryClass.THROTTLING: 10 +} + + +class RetryingClientSession(aiohttp.ClientSession): + __retry_number: int + __retry_delay: int + + def __init__( + self, + *args, + retry_number: int = DEFAULT_RETRY_NUMBER, + retry_delay: int = DEFAULT_RETRY_DELAY, + **kwargs + ): + super().__init__(*args, **kwargs) + self.__retry_number = retry_number + self.__retry_delay = retry_delay + + async def _request( + self, + *args, + **kwargs + ) -> ClientResponse: + fibonacci(5) + return await super()._request(*args, **kwargs) diff --git a/requirements.txt b/requirements.txt index 3375cf78..050de6a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ requests>=2.27.1 six>=1.16.0 aiohttp>=3.8.5 certifi>=2023.7.22 +sympy>=1.10.1 From a21a5844cc44e11d8c981cdf76c6a723f8d65625 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 20 Sep 2023 20:33:39 +0300 Subject: [PATCH 106/268] Remove six --- reportportal_client/core/worker.py | 3 ++- reportportal_client/logs/__init__.py | 27 ++++++----------------- reportportal_client/logs/log_manager.py | 3 +-- reportportal_client/logs/log_manager.pyi | 2 +- reportportal_client/services/client_id.py | 4 +--- requirements.txt | 1 - tests/__init__.py | 3 --- tests/conftest.py | 4 +++- tests/core/test_worker.py | 4 ++-- tests/logs/test_log_manager.py | 3 +-- tests/logs/test_rp_log_handler.py | 4 ++-- tests/logs/test_rp_logger.py | 2 +- tests/steps/conftest.py | 2 +- tests/steps/test_steps.py | 2 +- tests/test_client.py | 2 +- tests/test_helpers.py | 2 +- tests/test_statistics.py | 5 +++-- 17 files changed, 28 insertions(+), 45 deletions(-) diff --git a/reportportal_client/core/worker.py b/reportportal_client/core/worker.py index d145b085..2c245008 100644 --- a/reportportal_client/core/worker.py +++ b/reportportal_client/core/worker.py @@ -14,12 +14,13 @@ # limitations under the License import logging +import queue import threading from threading import current_thread, Thread from aenum import auto, Enum, unique + from reportportal_client.static.defines import Priority -from six.moves import queue logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) diff --git a/reportportal_client/logs/__init__.py b/reportportal_client/logs/__init__.py index 64ba74d2..a511b61b 100644 --- a/reportportal_client/logs/__init__.py +++ b/reportportal_client/logs/__init__.py @@ -15,9 +15,7 @@ import logging import sys import threading - -from six import PY2 -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse # noinspection PyProtectedMember from reportportal_client._local import current, set_current @@ -59,16 +57,11 @@ def _log(self, level, msg, args, exc_info=None, extra=None, # exception on some versions of IronPython. We trap it here so that # IronPython can use logging. try: - if PY2: - # In python2.7 findCaller() don't accept any parameters - # and returns 3 elements - fn, lno, func = self.findCaller() + if 'stacklevel' in kwargs: + fn, lno, func, sinfo = \ + self.findCaller(stack_info, kwargs['stacklevel']) else: - if 'stacklevel' in kwargs: - fn, lno, func, sinfo = \ - self.findCaller(stack_info, kwargs['stacklevel']) - else: - fn, lno, func, sinfo = self.findCaller(stack_info) + fn, lno, func, sinfo = self.findCaller(stack_info) except ValueError: # pragma: no cover fn, lno, func = '(unknown file)', 0, '(unknown function)' @@ -78,14 +71,8 @@ def _log(self, level, msg, args, exc_info=None, extra=None, if exc_info and not isinstance(exc_info, tuple): exc_info = sys.exc_info() - if PY2: - # In python2.7 makeRecord() accepts everything but sinfo - record = self.makeRecord(self.name, level, fn, lno, msg, args, - exc_info, func, extra) - else: - record = self.makeRecord(self.name, level, fn, lno, msg, args, - exc_info, func, extra, sinfo) - + record = self.makeRecord(self.name, level, fn, lno, msg, args, + exc_info, func, extra, sinfo) if not getattr(record, 'attachment', None): record.attachment = attachment self.handle(record) diff --git a/reportportal_client/logs/log_manager.py b/reportportal_client/logs/log_manager.py index f80a2d4e..8e32c0dc 100644 --- a/reportportal_client/logs/log_manager.py +++ b/reportportal_client/logs/log_manager.py @@ -14,10 +14,9 @@ # limitations under the License import logging +import queue from threading import Lock -from six.moves import queue - from reportportal_client import helpers from reportportal_client.core.rp_requests import ( HttpRequest, diff --git a/reportportal_client/logs/log_manager.pyi b/reportportal_client/logs/log_manager.pyi index dbed1c36..31af263a 100644 --- a/reportportal_client/logs/log_manager.pyi +++ b/reportportal_client/logs/log_manager.pyi @@ -11,12 +11,12 @@ # See the License for the specific language governing permissions and # limitations under the License +import queue from logging import Logger from threading import Lock from typing import Dict, List, Optional, Text from requests import Session -from six.moves import queue from reportportal_client.core.rp_requests import RPRequestLog from reportportal_client.core.worker import APIWorker as APIWorker diff --git a/reportportal_client/services/client_id.py b/reportportal_client/services/client_id.py index 70974d34..d333c36d 100644 --- a/reportportal_client/services/client_id.py +++ b/reportportal_client/services/client_id.py @@ -19,8 +19,6 @@ import os from uuid import uuid4 -import six - from .constants import CLIENT_ID_PROPERTY, RP_FOLDER_PATH, \ RP_PROPERTIES_FILE_PATH @@ -37,7 +35,7 @@ def __preprocess_file(self, fp): return io.StringIO(content) def read(self, filenames, encoding=None): - if isinstance(filenames, six.string_types): + if isinstance(filenames, str): filenames = [filenames] for filename in filenames: with open(filename, 'r') as fp: diff --git a/requirements.txt b/requirements.txt index 050de6a1..ee96e312 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ aenum requests>=2.27.1 -six>=1.16.0 aiohttp>=3.8.5 certifi>=2023.7.22 sympy>=1.10.1 diff --git a/tests/__init__.py b/tests/__init__.py index 72af15ef..65b0b270 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1 @@ """This package contains unit tests for the project.""" - -from six import add_move, MovedModule -add_move(MovedModule('mock', 'mock', 'unittest.mock')) diff --git a/tests/conftest.py b/tests/conftest.py index fc040a8f..79d1402d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ """This module contains common Pytest fixtures and hooks for unit tests.""" -from six.moves import mock +from unittest import mock # noinspection PyPackageRequirements from pytest import fixture @@ -11,6 +11,7 @@ @fixture() def response(): """Cook up a mock for the Response with specific arguments.""" + def inner(ret_code, ret_value): """Set up response with the given parameters. @@ -22,6 +23,7 @@ def inner(ret_code, ret_value): resp.status_code = ret_code resp.json.return_value = ret_value return resp + return inner diff --git a/tests/core/test_worker.py b/tests/core/test_worker.py index 27109c2c..dbc3d12e 100644 --- a/tests/core/test_worker.py +++ b/tests/core/test_worker.py @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License +import queue import time - -from six.moves import queue, mock +from unittest import mock from reportportal_client.core.rp_requests import ( HttpRequest, diff --git a/tests/logs/test_log_manager.py b/tests/logs/test_log_manager.py index 3d2896df..8506ea67 100644 --- a/tests/logs/test_log_manager.py +++ b/tests/logs/test_log_manager.py @@ -12,8 +12,7 @@ # limitations under the License import os - -from six.moves import mock +from unittest import mock from reportportal_client import helpers from reportportal_client.logs.log_manager import LogManager, \ diff --git a/tests/logs/test_rp_log_handler.py b/tests/logs/test_rp_log_handler.py index 6f6fcf65..d2b3e165 100644 --- a/tests/logs/test_rp_log_handler.py +++ b/tests/logs/test_rp_log_handler.py @@ -11,11 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License import re +# noinspection PyUnresolvedReferences +from unittest import mock # noinspection PyPackageRequirements import pytest -# noinspection PyUnresolvedReferences -from six.moves import mock # noinspection PyProtectedMember from reportportal_client._local import set_current diff --git a/tests/logs/test_rp_logger.py b/tests/logs/test_rp_logger.py index 9475b3ed..685481f5 100644 --- a/tests/logs/test_rp_logger.py +++ b/tests/logs/test_rp_logger.py @@ -14,9 +14,9 @@ import logging import sys from logging import LogRecord +from unittest import mock import pytest -from six.moves import mock from reportportal_client._local import set_current from reportportal_client.logs import RPLogger, RPLogHandler diff --git a/tests/steps/conftest.py b/tests/steps/conftest.py index cd376d97..f700044d 100644 --- a/tests/steps/conftest.py +++ b/tests/steps/conftest.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License -from six.moves import mock +from unittest import mock from pytest import fixture diff --git a/tests/steps/test_steps.py b/tests/steps/test_steps.py index 2a6615ad..ebbf0080 100644 --- a/tests/steps/test_steps.py +++ b/tests/steps/test_steps.py @@ -12,10 +12,10 @@ # limitations under the License import random import time +from unittest import mock from reportportal_client import step from reportportal_client._local import set_current -from six.moves import mock NESTED_STEP_NAME = 'test nested step' PARENT_STEP_ID = '123-123-1234-123' diff --git a/tests/test_client.py b/tests/test_client.py index 25e5c65b..5b70eabd 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -11,11 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License from io import StringIO +from unittest import mock import pytest from requests import Response from requests.exceptions import ReadTimeout -from six.moves import mock from reportportal_client import RPClient from reportportal_client.helpers import timestamp diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 80748428..3c71e144 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License -from six.moves import mock +from unittest import mock from reportportal_client.helpers import ( gen_attributes, diff --git a/tests/test_statistics.py b/tests/test_statistics.py index c38586ed..dcfb74f9 100644 --- a/tests/test_statistics.py +++ b/tests/test_statistics.py @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License -from requests.exceptions import RequestException # noinspection PyUnresolvedReferences -from six.moves import mock +from unittest import mock + +from requests.exceptions import RequestException from reportportal_client.services.constants import ENDPOINT, CLIENT_INFO from reportportal_client.services.statistics import send_event From bf905a2083e518c68a5daced1c076573fff41b14 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 20 Sep 2023 21:00:29 +0300 Subject: [PATCH 107/268] Decrease library requirements --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ee96e312..daa01535 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ aenum -requests>=2.27.1 -aiohttp>=3.8.5 +requests>=2.28.0 +aiohttp>=3.7.0 certifi>=2023.7.22 sympy>=1.10.1 From e925bec1f2d939444897866763c02376d5e77550 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 20 Sep 2023 22:55:04 +0300 Subject: [PATCH 108/268] Finish HTTP Retry client --- reportportal_client/aio/client.py | 4 +- reportportal_client/aio/http.py | 85 +++++++++++++++++++++++-------- 2 files changed, 66 insertions(+), 23 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index fa6d55b5..6fa41cb0 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -33,6 +33,7 @@ from reportportal_client.aio import (Task, BatchedTaskFactory, ThreadedTaskFactory, DEFAULT_TASK_TIMEOUT, DEFAULT_SHUTDOWN_TIMEOUT, DEFAULT_TASK_TRIGGER_NUM, TriggerTaskList, DEFAULT_TASK_TRIGGER_INTERVAL, BackgroundTaskList) +from reportportal_client.aio.http import RetryingClientSession from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, AsyncItemFinishRequest, LaunchFinishRequest, RPFile, @@ -148,7 +149,6 @@ def __init__( @property def session(self) -> aiohttp.ClientSession: - # TODO: add retry handler if self.__session: return self.__session @@ -179,7 +179,7 @@ def session(self) -> aiohttp.ClientSession: headers = {} if self.api_key: headers['Authorization'] = f'Bearer {self.api_key}' - self.__session = aiohttp.ClientSession(self.endpoint, connector=connector, headers=headers, + self.__session = RetryingClientSession(self.endpoint, connector=connector, headers=headers, timeout=timeout) return self.__session diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py index 70fab657..cd169d6f 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/aio/http.py @@ -11,28 +11,26 @@ # See the License for the specific language governing permissions and # limitations under the License +import asyncio +import sys +from typing import Coroutine + import aiohttp -from aenum import auto, Enum -from aiohttp import ClientResponse +from aenum import Enum +from aiohttp import ClientResponse, ServerConnectionError, \ + ClientResponseError from sympy import fibonacci DEFAULT_RETRY_NUMBER: int = 5 DEFAULT_RETRY_DELAY: int = 10 +THROTTLING_STATUSES: set = {425, 429} +RETRY_STATUSES: set = {408, 500, 502, 503, 507}.union(THROTTLING_STATUSES) -class RetryClass(Enum): - CONNECTION_ERROR = auto() - SERVER_ERROR = auto() - BAD_REQUEST = auto() - THROTTLING = auto() - - -RETRY_FACTOR = { - RetryClass.CONNECTION_ERROR: 5, - RetryClass.SERVER_ERROR: 1, - RetryClass.BAD_REQUEST: 0, - RetryClass.THROTTLING: 10 -} +class RetryClass(Enum, int): + SERVER_ERROR = 1 + CONNECTION_ERROR = 5 + THROTTLING = 7 class RetryingClientSession(aiohttp.ClientSession): @@ -42,18 +40,63 @@ class RetryingClientSession(aiohttp.ClientSession): def __init__( self, *args, - retry_number: int = DEFAULT_RETRY_NUMBER, - retry_delay: int = DEFAULT_RETRY_DELAY, + max_retry_number: int = DEFAULT_RETRY_NUMBER, + base_retry_delay: int = DEFAULT_RETRY_DELAY, **kwargs ): super().__init__(*args, **kwargs) - self.__retry_number = retry_number - self.__retry_delay = retry_delay + self.__retry_number = max_retry_number + self.__retry_delay = base_retry_delay + + async def __nothing(self): + pass + + def __sleep(self, retry_num: int, retry_factor: int) -> Coroutine: + if retry_num > 0: # don't wait at the first retry attempt + return asyncio.sleep(fibonacci(retry_factor + retry_num - 1) * self.__retry_delay) + else: + return self.__nothing() async def _request( self, *args, **kwargs ) -> ClientResponse: - fibonacci(5) - return await super()._request(*args, **kwargs) + result = None + exceptions = [] + for i in range(self.__retry_number + 1): # add one for the first attempt, which is not a retry + retry_factor = None + try: + result = await super()._request(*args, **kwargs) + except Exception as exc: + exceptions.append(exc) + if isinstance(exc, ServerConnectionError) or isinstance(exc, ClientResponseError): + retry_factor = RetryClass.CONNECTION_ERROR + + if not retry_factor: + raise exc + + if result: + if result.ok or result.status not in RETRY_STATUSES: + return result + + if result.status in THROTTLING_STATUSES: + retry_factor = RetryClass.THROTTLING + else: + retry_factor = RetryClass.SERVER_ERROR + + if i + 1 < self.__retry_number: + # don't wait at the last attempt + await self.__sleep(i, retry_factor) + + if exceptions: + if len(exceptions) > 1: + if sys.version_info > (3, 10): + # noinspection PyCompatibility + raise ExceptionGroup('During retry attempts the following exceptions happened', + exceptions) + else: + raise exceptions[-1] + else: + raise exceptions[0] + return result From ca263d21bd452632abba3b7784fa38b248b2a5db Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 20 Sep 2023 23:11:29 +0300 Subject: [PATCH 109/268] Fix Enum init --- reportportal_client/aio/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py index cd169d6f..4c4c7026 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/aio/http.py @@ -27,7 +27,7 @@ RETRY_STATUSES: set = {408, 500, 502, 503, 507}.union(THROTTLING_STATUSES) -class RetryClass(Enum, int): +class RetryClass(int, Enum): SERVER_ERROR = 1 CONNECTION_ERROR = 5 THROTTLING = 7 From 71f994c2f192b4a0002f337a42120de57c195f95 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 20 Sep 2023 23:13:33 +0300 Subject: [PATCH 110/268] Update aiohttp requirement --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index daa01535..7ccc2a30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ aenum requests>=2.28.0 -aiohttp>=3.7.0 +aiohttp>=3.8.0 certifi>=2023.7.22 sympy>=1.10.1 From 5397a7e1a26651fcee3dead19c8dea9bb5a013d9 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 20 Sep 2023 23:16:15 +0300 Subject: [PATCH 111/268] Update aiohttp requirement --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7ccc2a30..c23d3c76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ aenum requests>=2.28.0 -aiohttp>=3.8.0 +aiohttp>=3.8.3 certifi>=2023.7.22 sympy>=1.10.1 From 4bbaaf849183d3112a93f7fc3fb083f7c77fafbb Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 21 Sep 2023 11:36:54 +0300 Subject: [PATCH 112/268] Update retry delay logic --- reportportal_client/aio/http.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py index 4c4c7026..37dcc11e 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/aio/http.py @@ -19,18 +19,17 @@ from aenum import Enum from aiohttp import ClientResponse, ServerConnectionError, \ ClientResponseError -from sympy import fibonacci DEFAULT_RETRY_NUMBER: int = 5 -DEFAULT_RETRY_DELAY: int = 10 +DEFAULT_RETRY_DELAY: int = 5 THROTTLING_STATUSES: set = {425, 429} RETRY_STATUSES: set = {408, 500, 502, 503, 507}.union(THROTTLING_STATUSES) class RetryClass(int, Enum): SERVER_ERROR = 1 - CONNECTION_ERROR = 5 - THROTTLING = 7 + CONNECTION_ERROR = 2 + THROTTLING = 3 class RetryingClientSession(aiohttp.ClientSession): @@ -53,7 +52,7 @@ async def __nothing(self): def __sleep(self, retry_num: int, retry_factor: int) -> Coroutine: if retry_num > 0: # don't wait at the first retry attempt - return asyncio.sleep(fibonacci(retry_factor + retry_num - 1) * self.__retry_delay) + return asyncio.sleep((retry_factor * self.__retry_delay) ** retry_num) else: return self.__nothing() From 5470f98af10660a9e32f2aaab2c57454fc39e4af Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 21 Sep 2023 11:37:12 +0300 Subject: [PATCH 113/268] remove sympy --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c23d3c76..6c8a44c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,3 @@ aenum requests>=2.28.0 aiohttp>=3.8.3 certifi>=2023.7.22 -sympy>=1.10.1 From ace25fef436fa5840e2f2ded5f58ae3f520dc1dc Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 21 Sep 2023 16:33:21 +0300 Subject: [PATCH 114/268] Fix delays --- reportportal_client/aio/http.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py index 37dcc11e..5701fe4f 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/aio/http.py @@ -15,13 +15,12 @@ import sys from typing import Coroutine -import aiohttp from aenum import Enum -from aiohttp import ClientResponse, ServerConnectionError, \ +from aiohttp import ClientSession, ClientResponse, ServerConnectionError, \ ClientResponseError DEFAULT_RETRY_NUMBER: int = 5 -DEFAULT_RETRY_DELAY: int = 5 +DEFAULT_RETRY_DELAY: float = 0.005 THROTTLING_STATUSES: set = {425, 429} RETRY_STATUSES: set = {408, 500, 502, 503, 507}.union(THROTTLING_STATUSES) @@ -32,15 +31,15 @@ class RetryClass(int, Enum): THROTTLING = 3 -class RetryingClientSession(aiohttp.ClientSession): +class RetryingClientSession(ClientSession): __retry_number: int - __retry_delay: int + __retry_delay: float def __init__( self, *args, max_retry_number: int = DEFAULT_RETRY_NUMBER, - base_retry_delay: int = DEFAULT_RETRY_DELAY, + base_retry_delay: float = DEFAULT_RETRY_DELAY, **kwargs ): super().__init__(*args, **kwargs) @@ -52,7 +51,8 @@ async def __nothing(self): def __sleep(self, retry_num: int, retry_factor: int) -> Coroutine: if retry_num > 0: # don't wait at the first retry attempt - return asyncio.sleep((retry_factor * self.__retry_delay) ** retry_num) + delay = (((retry_factor * self.__retry_delay) * 1000) ** retry_num) / 1000 + return asyncio.sleep(delay) else: return self.__nothing() From c7d577e1a921ac59da720bb88570ef72019c0347 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 21 Sep 2023 17:27:45 +0300 Subject: [PATCH 115/268] Add the first retry test --- requirements-dev.txt | 1 + tests/aio/test_http.py | 75 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 tests/aio/test_http.py diff --git a/requirements-dev.txt b/requirements-dev.txt index 9955decc..e01a8206 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ pytest pytest-cov +pytest-asyncio diff --git a/tests/aio/test_http.py b/tests/aio/test_http.py new file mode 100644 index 00000000..333bd524 --- /dev/null +++ b/tests/aio/test_http.py @@ -0,0 +1,75 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +import http.server +import socketserver +import threading +import time +import traceback +from unittest import mock + +import aiohttp +import pytest + +from reportportal_client.aio.http import RetryingClientSession, ClientSession + +HTTP_TIMEOUT_TIME = 1.2 + + +class TimeoutHttpHandler(http.server.BaseHTTPRequestHandler): + + def do_GET(self): + time.sleep(HTTP_TIMEOUT_TIME) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write('{}\n\n'.encode("utf-8")) + self.wfile.flush() + + +SERVER_PORT = 8000 +SERVER_ADDRESS = ('', SERVER_PORT) +SERVER_CLASS = socketserver.TCPServer +SERVER_HANDLER_CLASS = http.server.BaseHTTPRequestHandler + + +def get_http_server(server_class=SERVER_CLASS, server_address=SERVER_ADDRESS, + server_handler=SERVER_HANDLER_CLASS): + httpd = server_class(server_address, server_handler) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + return httpd + + +@pytest.mark.asyncio +async def test_retry_on_request_timeout(): + timeout = aiohttp.ClientTimeout(connect=1, sock_read=1) + session = RetryingClientSession('http://localhost:8000',timeout=timeout, max_retry_number=5, + base_retry_delay=0.01) + parent_request = super(type(session), session)._request + async_mock = mock.AsyncMock() + async_mock.side_effect = parent_request + with get_http_server(server_handler=TimeoutHttpHandler): + with mock.patch('reportportal_client.aio.http.ClientSession._request', async_mock): + async with session: + exception = None + start_time = time.time() + try: + await session.get('/') + except Exception as exc: + exception = exc + total_time = time.time() - start_time + retries_and_delays = 6 + 0.02 + 0.4 + 8 + assert exception is not None + assert async_mock.call_count == 6 + assert total_time > retries_and_delays + assert total_time < retries_and_delays * 1.5 From d381163ee77b539c82ddbb86486cd3225b28fa89 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 21 Sep 2023 18:06:37 +0300 Subject: [PATCH 116/268] Fix some tests --- reportportal_client/client.py | 8 ++++++-- reportportal_client/core/rp_responses.py | 6 +++++- tests/test_client.py | 6 +++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 143f29b8..53ab43b4 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -14,6 +14,7 @@ # limitations under the License import logging +import queue import sys import warnings from abc import abstractmethod @@ -646,9 +647,12 @@ def _add_current_item(self, item: str) -> None: """Add the last item from the self._items queue.""" self._item_stack.put(item) - def _remove_current_item(self) -> str: + def _remove_current_item(self) -> Optional[str]: """Remove the last item from the self._items queue.""" - return self._item_stack.get() + try: + return self._item_stack.get(timeout=0) + except queue.Empty: + return def current_item(self) -> Optional[str]: """Retrieve the last item reported by the client.""" diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index 5e509953..97957790 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -19,6 +19,7 @@ # limitations under the License import logging +from json import JSONDecodeError from typing import Any, Optional, Generator, Mapping, Tuple, Protocol from aiohttp import ClientResponse @@ -66,7 +67,10 @@ def is_success(self) -> bool: def json(self) -> Any: """Get the response in dictionary.""" if not self.__json: - self.__json = self._resp.json() + try: + self.__json = self._resp.json() + except JSONDecodeError: + self.__json = {} return self.__json @property diff --git a/tests/test_client.py b/tests/test_client.py index 5b70eabd..cec79f4a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -57,7 +57,7 @@ def invalid_response(*args, **kwargs): ) def test_connection_errors(rp_client, requests_method, client_method, client_params): - rp_client.launch_id = 'test_launch_id' + rp_client.launch_uuid = 'test_launch_id' getattr(rp_client.session, requests_method).side_effect = connection_error result = getattr(rp_client, client_method)(*client_params) assert result is None @@ -84,7 +84,7 @@ def test_connection_errors(rp_client, requests_method, client_method, ) def test_invalid_responses(rp_client, requests_method, client_method, client_params): - rp_client.launch_id = 'test_launch_id' + rp_client.launch_uuid = 'test_launch_id' getattr(rp_client.session, requests_method).side_effect = invalid_response result = getattr(rp_client, client_method)(*client_params) assert result is None @@ -107,7 +107,7 @@ def test_invalid_responses(rp_client, requests_method, client_method, ] ) def test_launch_url_get(rp_client, launch_mode, project_name, expected_url): - rp_client.launch_id = 'test_launch_id' + rp_client.launch_uuid = 'test_launch_id' rp_client.project = project_name response = mock.Mock() From 0374753cd2e21c9616615387c88f2f396566184d Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 21 Sep 2023 22:43:16 +0300 Subject: [PATCH 117/268] Python version compatibility fix --- reportportal_client/aio/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index 5a6ec151..7e93fb36 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -34,7 +34,7 @@ class Task(Generic[_T], asyncio.Task, metaclass=AbstractBaseClass): def __init__( self, - coro: Union[Generator[Future[object], None, _T], Awaitable[_T]], + coro: Union[Generator[Future[Any], None, _T], Awaitable[_T]], *, loop: asyncio.AbstractEventLoop, name: Optional[str] = None @@ -51,7 +51,7 @@ class BatchedTask(Generic[_T], Task[_T]): def __init__( self, - coro: Union[Generator[Future[object], None, _T], Awaitable[_T]], + coro: Union[Generator[Future[Any], None, _T], Awaitable[_T]], *, loop: asyncio.AbstractEventLoop, name: Optional[str] = None @@ -81,7 +81,7 @@ class ThreadedTask(Generic[_T], Task[_T]): def __init__( self, - coro: Union[Generator[Future[object], None, _T], Awaitable[_T]], + coro: Union[Generator[Future[Any], None, _T], Awaitable[_T]], wait_timeout: float, *, loop: asyncio.AbstractEventLoop, From c0208db3f67d787a05793795a7a383bf1ffa797c Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 21 Sep 2023 22:49:54 +0300 Subject: [PATCH 118/268] Python version compatibility fix --- reportportal_client/aio/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index 7e93fb36..fd15d77d 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -34,7 +34,7 @@ class Task(Generic[_T], asyncio.Task, metaclass=AbstractBaseClass): def __init__( self, - coro: Union[Generator[Future[Any], None, _T], Awaitable[_T]], + coro: Union[Generator[Future, None, _T], Awaitable[_T]], *, loop: asyncio.AbstractEventLoop, name: Optional[str] = None @@ -51,7 +51,7 @@ class BatchedTask(Generic[_T], Task[_T]): def __init__( self, - coro: Union[Generator[Future[Any], None, _T], Awaitable[_T]], + coro: Union[Generator[Future, None, _T], Awaitable[_T]], *, loop: asyncio.AbstractEventLoop, name: Optional[str] = None @@ -81,7 +81,7 @@ class ThreadedTask(Generic[_T], Task[_T]): def __init__( self, - coro: Union[Generator[Future[Any], None, _T], Awaitable[_T]], + coro: Union[Generator[Future, None, _T], Awaitable[_T]], wait_timeout: float, *, loop: asyncio.AbstractEventLoop, From 37a7c3357869e444759482426ac2c440154bccc8 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 21 Sep 2023 22:51:44 +0300 Subject: [PATCH 119/268] Python version compatibility fix --- reportportal_client/services/statistics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reportportal_client/services/statistics.py b/reportportal_client/services/statistics.py index fa300daa..856c4745 100644 --- a/reportportal_client/services/statistics.py +++ b/reportportal_client/services/statistics.py @@ -17,7 +17,7 @@ import certifi import logging from platform import python_version -from typing import Optional +from typing import Optional, Tuple import aiohttp import requests @@ -31,7 +31,7 @@ ID, KEY = CLIENT_INFO.split(':') -def _get_client_info() -> tuple[str, str]: +def _get_client_info() -> Tuple[str, str]: """Get name of the client and its version. :return: ('reportportal-client', '5.0.4') From c876c44f1f4182e69fb9d8e769eba7b10fe00693 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 21 Sep 2023 22:55:12 +0300 Subject: [PATCH 120/268] Python version compatibility fix --- reportportal_client/core/rp_responses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index 97957790..9a6d7592 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -20,7 +20,7 @@ import logging from json import JSONDecodeError -from typing import Any, Optional, Generator, Mapping, Tuple, Protocol +from typing import Any, Optional, Generator, Mapping, Tuple from aiohttp import ClientResponse from requests import Response @@ -40,7 +40,7 @@ def _iter_json_messages(json: Any) -> Generator[str, None, None]: yield message -class RPResponse(Protocol): +class RPResponse: """Class representing RP API response.""" _resp: Response __json: Any From 4b6e82da569c4bf894c8451c4d867261540c0893 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 22 Sep 2023 12:04:15 +0300 Subject: [PATCH 121/268] Fixes for tests --- reportportal_client/client.py | 2 +- reportportal_client/core/rp_responses.py | 42 +++++++++++++++++------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 53ab43b4..2b55d697 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -426,6 +426,7 @@ def get_launch_info(self) -> Optional[Dict]: verify_ssl=self.verify_ssl).make() if not response: return + launch_info = None if response.is_success: launch_info = response.json logger.debug( @@ -433,7 +434,6 @@ def get_launch_info(self) -> Optional[Dict]: else: logger.warning('get_launch_info - Launch info: ' 'Failed to fetch launch ID from the API.') - launch_info = {} return launch_info def get_launch_ui_id(self) -> Optional[int]: diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index 9a6d7592..c7f97e5a 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -25,7 +25,7 @@ from aiohttp import ClientResponse from requests import Response -from reportportal_client.static.defines import NOT_FOUND +from reportportal_client.static.defines import NOT_FOUND, NOT_SET logger = logging.getLogger(__name__) @@ -51,11 +51,13 @@ def __init__(self, data: Response) -> None: :param data: requests.Response object """ self._resp = data - self.__json = None + self.__json = NOT_SET @property def id(self) -> Optional[str]: """Get value of the 'id' key.""" + if self.json is None: + return return self.json.get('id', NOT_FOUND) @property @@ -66,21 +68,25 @@ def is_success(self) -> bool: @property def json(self) -> Any: """Get the response in dictionary.""" - if not self.__json: + if self.__json is NOT_SET: try: self.__json = self._resp.json() except JSONDecodeError: - self.__json = {} + self.__json = None return self.__json @property def message(self) -> Optional[str]: """Get value of the 'message' key.""" + if self.json is None: + return return self.json.get('message') @property - def messages(self) -> Tuple[str, ...]: + def messages(self) -> Optional[Tuple[str, ...]]: """Get list of messages received.""" + if self.json is None: + return return tuple(_iter_json_messages(self.json)) @@ -95,12 +101,15 @@ def __init__(self, data: ClientResponse) -> None: :param data: requests.Response object """ self._resp = data - self.__json = None + self.__json = NOT_SET @property async def id(self) -> Optional[str]: """Get value of the 'id' key.""" - return (await self.json).get('id', NOT_FOUND) + json = await self.json + if json is None: + return + return json.get('id', NOT_FOUND) @property def is_success(self) -> bool: @@ -110,16 +119,25 @@ def is_success(self) -> bool: @property async def json(self) -> Any: """Get the response in dictionary.""" - if not self.__json: - self.__json = await self._resp.json() + if self.__json is NOT_SET: + try: + self.__json = await self._resp.json() + except JSONDecodeError: + self.__json = None return self.__json @property async def message(self) -> Optional[str]: """Get value of the 'message' key.""" - return (await self.json).get('message') + json = await self.json + if json is None: + return + return json.get('message') @property - async def messages(self) -> Tuple[str, ...]: + async def messages(self) -> Optional[Tuple[str, ...]]: """Get list of messages received.""" - return tuple(_iter_json_messages(await self.json)) + json = await self.json + if json is None: + return + return tuple(_iter_json_messages(json)) From 892a43ae424adb8e1b57a0f476d5062d18b63f5e Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 22 Sep 2023 12:06:44 +0300 Subject: [PATCH 122/268] Fixes for tests --- reportportal_client/core/rp_responses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index c7f97e5a..6a341dc1 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -71,7 +71,7 @@ def json(self) -> Any: if self.__json is NOT_SET: try: self.__json = self._resp.json() - except JSONDecodeError: + except (JSONDecodeError, TypeError): self.__json = None return self.__json @@ -122,7 +122,7 @@ async def json(self) -> Any: if self.__json is NOT_SET: try: self.__json = await self._resp.json() - except JSONDecodeError: + except (JSONDecodeError, TypeError): self.__json = None return self.__json From cf13d6e57427eb65d3fe852198cf02115fcf0111 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 22 Sep 2023 12:12:03 +0300 Subject: [PATCH 123/268] Fix tests --- tests/steps/test_steps.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/steps/test_steps.py b/tests/steps/test_steps.py index ebbf0080..9cd9afc5 100644 --- a/tests/steps/test_steps.py +++ b/tests/steps/test_steps.py @@ -81,7 +81,7 @@ def test_nested_step_decorator(rp_client): assert rp_client.session.post.call_count == 1 assert rp_client.session.put.call_count == 1 - assert len(rp_client._log_manager._batch) == 0 + assert len(rp_client._log_batcher._batch) == 0 def test_nested_step_failed(rp_client): @@ -136,8 +136,8 @@ def test_verify_parameters_logging_default_value(rp_client): rp_client.session.post.side_effect = item_id_gen rp_client._add_current_item(PARENT_STEP_ID) nested_step_params(1, 'two') - assert len(rp_client._log_manager._batch) == 1 - assert rp_client._log_manager._batch[0].message \ + assert len(rp_client._log_batcher._batch) == 1 + assert rp_client._log_batcher._batch[0].message \ == "Parameters: param1: 1; param2: two" @@ -145,8 +145,8 @@ def test_verify_parameters_logging_no_default_value(rp_client): rp_client.session.post.side_effect = item_id_gen rp_client._add_current_item(PARENT_STEP_ID) nested_step_params(1, 'two', 'three') - assert len(rp_client._log_manager._batch) == 1 - assert rp_client._log_manager._batch[0].message \ + assert len(rp_client._log_batcher._batch) == 1 + assert rp_client._log_batcher._batch[0].message \ == "Parameters: param1: 1; param2: two; param3: three" @@ -154,8 +154,8 @@ def test_verify_parameters_logging_named_value(rp_client): rp_client.session.post.side_effect = item_id_gen rp_client._add_current_item(PARENT_STEP_ID) nested_step_params(1, 'two', param3='three') - assert len(rp_client._log_manager._batch) == 1 - assert rp_client._log_manager._batch[0].message \ + assert len(rp_client._log_batcher._batch) == 1 + assert rp_client._log_batcher._batch[0].message \ == "Parameters: param1: 1; param2: two; param3: three" @@ -164,8 +164,8 @@ def test_verify_parameters_inline_logging(rp_client): rp_client._add_current_item(PARENT_STEP_ID) with step(NESTED_STEP_NAME, params={'param1': 1, 'param2': 'two'}): pass - assert len(rp_client._log_manager._batch) == 1 - assert rp_client._log_manager._batch[0].message \ + assert len(rp_client._log_batcher._batch) == 1 + assert rp_client._log_batcher._batch[0].message \ == "Parameters: param1: 1; param2: two" @@ -181,7 +181,7 @@ def test_two_level_nested_step_decorator(rp_client): assert rp_client.session.post.call_count == 2 assert rp_client.session.put.call_count == 2 - assert len(rp_client._log_manager._batch) == 0 + assert len(rp_client._log_batcher._batch) == 0 request_uri = rp_client.session.post.call_args_list[0][0][0] first_parent_id = request_uri[request_uri.rindex('/') + 1:] From 81f47deb6042a472d8241d90f0fa62c549801529 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 22 Sep 2023 14:40:17 +0300 Subject: [PATCH 124/268] Fix tests --- tests/logs/test_log_manager.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/logs/test_log_manager.py b/tests/logs/test_log_manager.py index 8506ea67..ce617af9 100644 --- a/tests/logs/test_log_manager.py +++ b/tests/logs/test_log_manager.py @@ -12,9 +12,11 @@ # limitations under the License import os +import json from unittest import mock from reportportal_client import helpers +from reportportal_client.core.rp_requests import HttpRequest from reportportal_client.logs.log_manager import LogManager, \ MAX_LOG_BATCH_PAYLOAD_SIZE @@ -44,8 +46,8 @@ def test_log_batch_send_by_length(): assert log_manager._worker.send.call_count == 1 batch = log_manager._worker.send.call_args[0][0] - assert len(batch.log_reqs) == 5 - assert batch.http_request is not None + assert isinstance(batch, HttpRequest) + assert len(json.loads(batch.files[0][1][1])) == 5 assert 'post' in session._mock_children assert len(log_manager._batch) == 0 assert log_manager._payload_size == helpers.TYPICAL_MULTIPART_FOOTER_LENGTH @@ -66,8 +68,8 @@ def test_log_batch_send_url_format(): assert log_manager._worker.send.call_count == 1 batch = log_manager._worker.send.call_args[0][0] - assert batch.http_request is not None - assert batch.http_request.url == \ + assert isinstance(batch, HttpRequest) + assert batch.url == \ RP_URL + '/api/' + API_VERSION + '/' + PROJECT_NAME + '/log' @@ -104,8 +106,8 @@ def test_log_batch_send_by_stop(): assert log_manager._worker.send.call_count == 1 batch = log_manager._worker.send.call_args[0][0] - assert len(batch.log_reqs) == 4 - assert batch.http_request is not None + assert isinstance(batch, HttpRequest) + assert len(json.loads(batch.files[0][1][1])) == 4 assert 'post' in session._mock_children assert len(log_manager._batch) == 0 assert log_manager._payload_size == helpers.TYPICAL_MULTIPART_FOOTER_LENGTH @@ -158,12 +160,11 @@ def test_log_batch_send_by_size(): assert log_manager._worker.send.call_count == 1 batch = log_manager._worker.send.call_args[0][0] - assert len(batch.log_reqs) == 1 - assert batch.http_request is not None + assert isinstance(batch, HttpRequest) + assert len(json.loads(batch.files[0][1][1])) == 1 assert 'post' in session._mock_children assert len(log_manager._batch) == 1 - assert log_manager._payload_size < \ - helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + 1024 + assert log_manager._payload_size < helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + 1024 # noinspection PyUnresolvedReferences @@ -188,8 +189,8 @@ def test_log_batch_triggers_previous_request_to_send(): assert log_manager._worker.send.call_count == 1 batch = log_manager._worker.send.call_args[0][0] - assert len(batch.log_reqs) == 1 - assert batch.http_request is not None + assert isinstance(batch, HttpRequest) + assert len(json.loads(batch.files[0][1][1])) == 1 assert 'post' in session._mock_children assert len(log_manager._batch) == 1 assert log_manager._payload_size > MAX_LOG_BATCH_PAYLOAD_SIZE From 74d6bc573ebaa738bb966183f0f0377d8c9f6936 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 22 Sep 2023 14:48:44 +0300 Subject: [PATCH 125/268] Fix tests --- tests/core/test_worker.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/core/test_worker.py b/tests/core/test_worker.py index dbc3d12e..34d8ea92 100644 --- a/tests/core/test_worker.py +++ b/tests/core/test_worker.py @@ -44,27 +44,23 @@ def test_worker_continue_working_on_request_error(): http_fail = HttpRequest( fail_session, LOG_REQUEST_URL, files=log_batch.payload, verify_ssl=False) - log_batch.http_request = http_fail - worker.send(log_batch) + worker.send(http_fail) start_time = time.time() while fail_session.call_count < 1 and time.time() - start_time < 10: time.sleep(0.1) assert fail_session.call_count == 1 - assert log_batch.response is None pass_session = mock.Mock() http_pass = HttpRequest( pass_session, LOG_REQUEST_URL, files=log_batch.payload, verify_ssl=False) - log_batch.http_request = http_pass - worker.send(log_batch) + worker.send(http_pass) start_time = time.time() while pass_session.call_count < 1 and time.time() - start_time < 10: time.sleep(0.1) assert pass_session.call_count == 1 - assert log_batch.response worker.stop() From 2d90a8fc153fdcd0623f5c25f57180114822084f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 22 Sep 2023 14:52:42 +0300 Subject: [PATCH 126/268] Fix tests --- tests/aio/test_http.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/aio/test_http.py b/tests/aio/test_http.py index 333bd524..2fc46f19 100644 --- a/tests/aio/test_http.py +++ b/tests/aio/test_http.py @@ -12,6 +12,7 @@ # limitations under the License import http.server import socketserver +import sys import threading import time import traceback @@ -50,6 +51,8 @@ def get_http_server(server_class=SERVER_CLASS, server_address=SERVER_ADDRESS, return httpd +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires Python 3.8 or higher") @pytest.mark.asyncio async def test_retry_on_request_timeout(): timeout = aiohttp.ClientTimeout(connect=1, sock_read=1) From 42c292e1b8baa9cc8f5156236a9938ed347eeb09 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 22 Sep 2023 14:54:57 +0300 Subject: [PATCH 127/268] Fix tests --- tests/aio/test_http.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/aio/test_http.py b/tests/aio/test_http.py index 2fc46f19..9b581757 100644 --- a/tests/aio/test_http.py +++ b/tests/aio/test_http.py @@ -61,6 +61,7 @@ async def test_retry_on_request_timeout(): parent_request = super(type(session), session)._request async_mock = mock.AsyncMock() async_mock.side_effect = parent_request + total_time = 0.0 with get_http_server(server_handler=TimeoutHttpHandler): with mock.patch('reportportal_client.aio.http.ClientSession._request', async_mock): async with session: @@ -71,8 +72,8 @@ async def test_retry_on_request_timeout(): except Exception as exc: exception = exc total_time = time.time() - start_time - retries_and_delays = 6 + 0.02 + 0.4 + 8 - assert exception is not None - assert async_mock.call_count == 6 - assert total_time > retries_and_delays - assert total_time < retries_and_delays * 1.5 + retries_and_delays = 6 + 0.02 + 0.4 + 8 + assert exception is not None + assert async_mock.call_count == 6 + assert total_time > retries_and_delays + assert total_time < retries_and_delays * 1.5 From d698d98ebcd4a13bc8faa6f162e580de336e9b61 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 22 Sep 2023 15:03:35 +0300 Subject: [PATCH 128/268] Fix flake8 --- reportportal_client/aio/client.py | 2 +- reportportal_client/aio/http.py | 7 +++++-- reportportal_client/static/errors.py | 1 + tests/aio/test_http.py | 8 +++----- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 6fa41cb0..dc0ec7f7 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -241,7 +241,7 @@ async def start_launch(self, self.__stat_task = asyncio.create_task(stat_coro, name='Statistics update') launch_uuid = await response.id - logger.debug(f'start_launch - ID: %s', launch_uuid) + logger.debug(f'start_launch - ID: {launch_uuid}') if self.launch_uuid_print and self.print_output: print(f'Report Portal Launch UUID: {launch_uuid}', file=self.print_output) return launch_uuid diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py index 5701fe4f..98b09f7a 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/aio/http.py @@ -92,8 +92,11 @@ async def _request( if len(exceptions) > 1: if sys.version_info > (3, 10): # noinspection PyCompatibility - raise ExceptionGroup('During retry attempts the following exceptions happened', - exceptions) + + raise ExceptionGroup( # noqa: F821 + 'During retry attempts the following exceptions happened', + exceptions + ) else: raise exceptions[-1] else: diff --git a/reportportal_client/static/errors.py b/reportportal_client/static/errors.py index 50d04645..7ebcb49e 100644 --- a/reportportal_client/static/errors.py +++ b/reportportal_client/static/errors.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License + class RPExceptionBase(Exception): """General exception for the package.""" diff --git a/tests/aio/test_http.py b/tests/aio/test_http.py index 9b581757..96300c0d 100644 --- a/tests/aio/test_http.py +++ b/tests/aio/test_http.py @@ -15,13 +15,12 @@ import sys import threading import time -import traceback from unittest import mock import aiohttp import pytest -from reportportal_client.aio.http import RetryingClientSession, ClientSession +from reportportal_client.aio.http import RetryingClientSession HTTP_TIMEOUT_TIME = 1.2 @@ -56,16 +55,15 @@ def get_http_server(server_class=SERVER_CLASS, server_address=SERVER_ADDRESS, @pytest.mark.asyncio async def test_retry_on_request_timeout(): timeout = aiohttp.ClientTimeout(connect=1, sock_read=1) - session = RetryingClientSession('http://localhost:8000',timeout=timeout, max_retry_number=5, + session = RetryingClientSession('http://localhost:8000', timeout=timeout, max_retry_number=5, base_retry_delay=0.01) parent_request = super(type(session), session)._request async_mock = mock.AsyncMock() async_mock.side_effect = parent_request - total_time = 0.0 + exception = None with get_http_server(server_handler=TimeoutHttpHandler): with mock.patch('reportportal_client.aio.http.ClientSession._request', async_mock): async with session: - exception = None start_time = time.time() try: await session.get('/') From f517fc57e74dc5a49821594d4fc2e38fb655da7a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 22 Sep 2023 15:03:54 +0300 Subject: [PATCH 129/268] Fix flake8 --- reportportal_client/aio/http.py | 1 - 1 file changed, 1 deletion(-) diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py index 98b09f7a..4e3e0f19 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/aio/http.py @@ -92,7 +92,6 @@ async def _request( if len(exceptions) > 1: if sys.version_info > (3, 10): # noinspection PyCompatibility - raise ExceptionGroup( # noqa: F821 'During retry attempts the following exceptions happened', exceptions From 30341b8d59cd4e681c2447274bc9be1f36222a4e Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 22 Sep 2023 15:05:17 +0300 Subject: [PATCH 130/268] Fix flake8 --- reportportal_client/aio/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py index 4e3e0f19..80ca827c 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/aio/http.py @@ -94,7 +94,7 @@ async def _request( # noinspection PyCompatibility raise ExceptionGroup( # noqa: F821 'During retry attempts the following exceptions happened', - exceptions + exceptions ) else: raise exceptions[-1] From cce6f5754d3d420842bcf4edf13eb080e820f98f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Sat, 23 Sep 2023 14:00:00 +0300 Subject: [PATCH 131/268] Move LifoQueue to helpers and add pydocs --- reportportal_client/aio/client.py | 18 +++++------------- reportportal_client/client.py | 14 +++----------- reportportal_client/helpers.py | 20 +++++++++++++++++++- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index dc0ec7f7..194e7cfe 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -21,7 +21,6 @@ import time as datetime import warnings from os import getenv -from queue import LifoQueue from typing import Union, Tuple, List, Dict, Any, Optional, TextIO, Coroutine, TypeVar import aiohttp @@ -39,7 +38,7 @@ AsyncItemFinishRequest, LaunchFinishRequest, RPFile, AsyncRPRequestLog, AsyncRPLogBatch) from reportportal_client.helpers import (root_uri_join, verify_value_length, await_if_necessary, - agent_name_version) + agent_name_version, LifoQueue) from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.logs.batcher import LogBatcher from reportportal_client.services.statistics import async_send_event @@ -56,13 +55,6 @@ _T = TypeVar('_T') -class _LifoQueue(LifoQueue): - def last(self): - with self.mutex: - if self._qsize(): - return self.queue[-1] - - class Client: api_v1: str api_v2: str @@ -463,7 +455,7 @@ def clone(self) -> 'Client': class AsyncRPClient(RP): - _item_stack: _LifoQueue + _item_stack: LifoQueue _log_batcher: LogBatcher __client: Client __launch_uuid: Optional[str] @@ -474,7 +466,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[str] = client: Optional[Client] = None, **kwargs: Any) -> None: set_current(self) self.step_reporter = StepReporter(self) - self._item_stack = _LifoQueue() + self._item_stack = LifoQueue() self._log_batcher = LogBatcher() if client: self.__client = client @@ -643,7 +635,7 @@ def clone(self) -> 'AsyncRPClient': class _RPClient(RP, metaclass=AbstractBaseClass): __metaclass__ = AbstractBaseClass - _item_stack: _LifoQueue + _item_stack: LifoQueue _log_batcher: LogBatcher _shutdown_timeout: float _task_timeout: float @@ -689,7 +681,7 @@ def __init__( self.__endpoint = endpoint self.__project = project self.step_reporter = StepReporter(self) - self._item_stack = _LifoQueue() + self._item_stack = LifoQueue() self._shutdown_timeout = shutdown_timeout self._task_timeout = task_timeout if log_batcher: diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 2b55d697..962a60f5 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -19,7 +19,6 @@ import warnings from abc import abstractmethod from os import getenv -from queue import LifoQueue from typing import Union, Tuple, List, Dict, Any, Optional, TextIO import requests @@ -32,7 +31,7 @@ from reportportal_client.core.rp_requests import (HttpRequest, ItemStartRequest, ItemFinishRequest, RPFile, LaunchStartRequest, LaunchFinishRequest, RPRequestLog, RPLogBatch) -from reportportal_client.helpers import uri_join, verify_value_length, agent_name_version +from reportportal_client.helpers import uri_join, verify_value_length, agent_name_version, LifoQueue from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.logs.batcher import LogBatcher from reportportal_client.services.statistics import send_event @@ -157,13 +156,6 @@ def clone(self) -> 'RPClient': raise NotImplementedError('"clone" method is not implemented!') -class _LifoQueue(LifoQueue): - def last(self): - with self.mutex: - if self._qsize(): - return self.queue[-1] - - class RPClient(RP): """Report portal client. @@ -196,7 +188,7 @@ class RPClient(RP): launch_uuid_print: Optional[bool] print_output: Optional[TextIO] _skip_analytics: str - _item_stack: _LifoQueue + _item_stack: LifoQueue _log_batcher: LogBatcher[RPRequestLog] def __init__( @@ -267,7 +259,7 @@ def __init__( self.max_pool_size = max_pool_size self.http_timeout = http_timeout self.step_reporter = StepReporter(self) - self._item_stack = _LifoQueue() + self._item_stack = LifoQueue() self.mode = mode self._skip_analytics = getenv('AGENT_NO_ANALYTICS') self.launch_uuid_print = launch_uuid_print diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index d0c2b68e..7728d5db 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -17,8 +17,9 @@ import logging import time import uuid +import queue from platform import machine, processor, system -from typing import Optional, Any, List, Dict, Callable, Tuple, Union +from typing import Optional, Any, List, Dict, Callable, Tuple, Union, TypeVar, Generic from pkg_resources import DistributionNotFound, get_distribution @@ -26,6 +27,23 @@ from reportportal_client.static.defines import ATTRIBUTE_LENGTH_LIMIT logger: logging.Logger = logging.getLogger(__name__) +_T = TypeVar('_T') + + +class LifoQueue(Generic[_T], queue.LifoQueue[_T]): + """This Queue adds 'last' method to original queue.LifoQueue. + + Unlike 'get' method in queue.LifoQueue the 'last' method do not remove an entity from the queue. + """ + + def last(self) -> _T: + """Return the last element from the queue, but does not remove it. + + :return: the last element in the queue + """ + with self.mutex: + if self._qsize(): + return self.queue[-1] def generate_uuid() -> str: From 5f2dc9be6743e99aa887b3876592c150c7ebee90 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Sat, 23 Sep 2023 14:11:36 +0300 Subject: [PATCH 132/268] Add pydocs --- reportportal_client/client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 962a60f5..4b4a926a 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -44,6 +44,11 @@ class RP(metaclass=AbstractBaseClass): + """Common interface for Report Portal clients. + + This abstract class serves as common interface for different Report Portal clients. It's implemented to + ease migration from version to version and to ensure that each particular client has the same methods. + """ __metaclass__ = AbstractBaseClass @abstractmethod From b9c4dd6696809fd334cec29939129f6808c265c5 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 25 Sep 2023 13:37:47 +0300 Subject: [PATCH 133/268] Update interfaces signatures --- reportportal_client/__init__.py | 2 +- reportportal_client/_local/__init__.pyi | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/reportportal_client/__init__.py b/reportportal_client/__init__.py index d1b6a311..a339035f 100644 --- a/reportportal_client/__init__.py +++ b/reportportal_client/__init__.py @@ -23,8 +23,8 @@ __all__ = [ 'current', 'RP', + 'RPClient', 'RPLogger', 'RPLogHandler', - 'RPClient', 'step', ] diff --git a/reportportal_client/_local/__init__.pyi b/reportportal_client/_local/__init__.pyi index a2bdd9fd..a5d72a7e 100644 --- a/reportportal_client/_local/__init__.pyi +++ b/reportportal_client/_local/__init__.pyi @@ -11,13 +11,12 @@ # See the License for the specific language governing permissions and # limitations under the License -from typing import Optional, Union +from typing import Optional -from reportportal_client.aio.client import RPClient as AsyncRPClient -from reportportal_client.client import RPClient +from reportportal_client import RP -def current() -> Optional[RPClient]: ... +def current() -> Optional[RP]: ... -def set_current(client: Optional[Union[RPClient, AsyncRPClient]]) -> None: ... +def set_current(client: Optional[RP]) -> None: ... From 8300ef1cda6a9deddf42edf789bc1cb4aaed3272 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 25 Sep 2023 13:40:11 +0300 Subject: [PATCH 134/268] Add pydocs and fix interfaces --- reportportal_client/client.py | 70 +++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 3 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 4b4a926a..10bdec37 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -1,4 +1,4 @@ -"""This module contains Report Portal Client class.""" +"""This module contains Report Portal Client interface and synchronous implementation class.""" # Copyright (c) 2023 EPAM Systems # Licensed under the Apache License, Version 2.0 (the "License"); @@ -60,6 +60,17 @@ def start_launch(self, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Union[Optional[str], Task[str]]: + """Start a new launch with the given parameters. + + :param name: Launch name + :param start_time: Launch start time + :param description: Launch description + :param attributes: Launch attributes + :param rerun: Start launch in rerun mode + :param rerun_of: For rerun mode specifies which launch will be + re-run. Should be used with the 'rerun' option. + :returns Launch UUID + """ raise NotImplementedError('"start_launch" method is not implemented!') @abstractmethod @@ -77,6 +88,27 @@ def start_test_item(self, retry: bool = False, test_case_id: Optional[str] = None, **kwargs: Any) -> Union[Optional[str], Task[str]]: + """Start case/step/nested step item. + + :param name: Name of the test item + :param start_time: The item start time + :param item_type: Type of the test item. Allowable values: + "suite", "story", "test", "scenario", "step", + "before_class", "before_groups", + "before_method", "before_suite", + "before_test", "after_class", "after_groups", + "after_method", "after_suite", "after_test" + :param description: The item description + :param attributes: Test item attributes + :param parameters: Set of parameters (for parametrized test items) + :param parent_item_id: An ID of a parent SUITE / STEP + :param has_stats: Set to False if test item is nested step + :param code_ref: Physical location of the test item + :param retry: Used to report retry of the test. Allowable + values: "True" or "False" + :param test_case_id: A unique ID of the current step + :returns Test Item UUID + """ raise NotImplementedError('"start_test_item" method is not implemented!') @abstractmethod @@ -90,6 +122,21 @@ def finish_test_item(self, description: str = None, retry: bool = False, **kwargs: Any) -> Union[Optional[str], Task[str]]: + """Finish suite/case/step/nested step item. + + :param item_id: ID of the test item + :param end_time: The item end time + :param status: Test status. Allowable values: + PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED, INFO, WARN or None + :param issue: Issue which will be attached to the current item + :param attributes: Test item attributes(tags). Pairs of key and value. + Override attributes on start + :param description: Test item description. Overrides description + from start request. + :param retry: Used to report retry of the test. Allowable values: + "True" or "False" + :returns Response message + """ raise NotImplementedError('"finish_test_item" method is not implemented!') @abstractmethod @@ -98,11 +145,28 @@ def finish_launch(self, status: str = None, attributes: Optional[Union[List, Dict]] = None, **kwargs: Any) -> Union[Optional[str], Task[str]]: + """Finish current launch. + + :param end_time: Launch end time + :param status: Launch status. Can be one of the followings: + PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED + :param attributes: Launch attributes + :returns Response message + """ raise NotImplementedError('"finish_launch" method is not implemented!') @abstractmethod - def update_test_item(self, item_uuid: str, attributes: Optional[Union[List, Dict]] = None, - description: Optional[str] = None) -> Optional[str]: + def update_test_item(self, + item_uuid: Union[Optional[str], Task[str]], + attributes: Optional[Union[List, Dict]] = None, + description: Optional[str] = None) -> Union[Optional[str], Task[str]]: + """Update existing test item at the Report Portal. + + :param item_uuid: Test item UUID returned on the item start + :param attributes: Test item attributes: [{'key': 'k_name', 'value': 'k_value'}, ...] + :param description: Test item description + :returns Response message + """ raise NotImplementedError('"update_test_item" method is not implemented!') @abstractmethod From 6ec904dada037109aa36e9e2580d4ac74c1bcdc9 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 25 Sep 2023 14:38:07 +0300 Subject: [PATCH 135/268] Update pydocs, interfaces and properties --- reportportal_client/aio/client.py | 40 +- reportportal_client/client.py | 516 ++++++++++++++----------- reportportal_client/steps/__init__.py | 3 +- reportportal_client/steps/__init__.pyi | 42 +- tests/test_client.py | 8 +- 5 files changed, 331 insertions(+), 278 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 194e7cfe..159737ae 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -459,13 +459,13 @@ class AsyncRPClient(RP): _log_batcher: LogBatcher __client: Client __launch_uuid: Optional[str] + __step_reporter: StepReporter use_own_launch: bool - step_reporter: StepReporter def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[str] = None, client: Optional[Client] = None, **kwargs: Any) -> None: set_current(self) - self.step_reporter = StepReporter(self) + self.__step_reporter = StepReporter(self) self._item_stack = LifoQueue() self._log_batcher = LogBatcher() if client: @@ -478,6 +478,18 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[str] = else: self.use_own_launch = True + @property + def launch_uuid(self) -> Optional[str]: + return self.__launch_uuid + + @launch_uuid.setter + def launch_uuid(self, value: Optional[str]) -> None: + self.__launch_uuid = value + + @property + def step_reporter(self) -> StepReporter: + return self.__step_reporter + async def start_launch(self, name: str, start_time: str, @@ -604,14 +616,6 @@ async def log(self, time: str, message: str, level: Optional[Union[int, str]] = rp_log = AsyncRPRequestLog(self.launch_uuid, time, rp_file, item_id, level, message) return await self.__client.log_batch(await self._log_batcher.append_async(rp_log)) - @property - def launch_uuid(self) -> Optional[str]: - return self.__launch_uuid - - @launch_uuid.setter - def launch_uuid(self, value: Optional[str]) -> None: - self.__launch_uuid = value - def clone(self) -> 'AsyncRPClient': """Clone the client object, set current Item ID as cloned item ID. @@ -644,7 +648,7 @@ class _RPClient(RP, metaclass=AbstractBaseClass): __endpoint: str __project: str use_own_launch: bool - step_reporter: StepReporter + __step_reporter: StepReporter @property def client(self) -> Client: @@ -654,10 +658,6 @@ def client(self) -> Client: def launch_uuid(self) -> Optional[Task[str]]: return self.__launch_uuid - @launch_uuid.setter - def launch_uuid(self, value: Optional[Task[str]]) -> None: - self.__launch_uuid = value - @property def endpoint(self) -> str: return self.__endpoint @@ -666,6 +666,10 @@ def endpoint(self) -> str: def project(self) -> str: return self.__project + @property + def step_reporter(self) -> StepReporter: + return self.__step_reporter + def __init__( self, endpoint: str, @@ -680,7 +684,7 @@ def __init__( ) -> None: self.__endpoint = endpoint self.__project = project - self.step_reporter = StepReporter(self) + self.__step_reporter = StepReporter(self) self._item_stack = LifoQueue() self._shutdown_timeout = shutdown_timeout self._task_timeout = task_timeout @@ -693,7 +697,7 @@ def __init__( else: self.__client = Client(endpoint, project, **kwargs) if launch_uuid: - self.launch_uuid = launch_uuid + self.__launch_uuid = launch_uuid self.use_own_launch = False else: self.use_own_launch = True @@ -741,7 +745,7 @@ def start_launch(self, launch_uuid_coro = self.__client.start_launch(name, start_time, description=description, attributes=attributes, rerun=rerun, rerun_of=rerun_of, **kwargs) - self.launch_uuid = self.create_task(launch_uuid_coro) + self.__launch_uuid = self.create_task(launch_uuid_coro) return self.launch_uuid def start_test_item(self, diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 10bdec37..64c4f916 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -51,6 +51,48 @@ class RP(metaclass=AbstractBaseClass): """ __metaclass__ = AbstractBaseClass + @property + @abstractmethod + def launch_uuid(self) -> Optional[Union[str, Task[str]]]: + """Return current launch UUID. + + :return: UUID string + """ + raise NotImplementedError('"launch_uuid" property is not implemented!') + + @property + def launch_id(self) -> Optional[Union[str, Task[str]]]: + """Return current launch UUID. + + :return: UUID string + """ + warnings.warn( + message='`launch_id` property is deprecated since 5.5.0 and will be subject for removing in the' + ' next major version. Use `launch_uuid` property instead.', + category=DeprecationWarning, + stacklevel=2 + ) + return self.launch_uuid + + @property + @abstractmethod + def endpoint(self) -> str: + raise NotImplementedError('"endpoint" property is not implemented!') + + @property + @abstractmethod + def project(self) -> str: + raise NotImplementedError('"project" property is not implemented!') + + @property + @abstractmethod + def step_reporter(self) -> StepReporter: + """Return StepReporter object for the current launch. + + :return: StepReporter to report steps + """ + raise NotImplementedError('"step_reporter" property is not implemented!') + @abstractmethod def start_launch(self, name: str, @@ -195,25 +237,22 @@ def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = No raise NotImplementedError('"log" method is not implemented!') def start(self) -> None: - pass # For backward compatibility + """Start the client.""" + warnings.warn( + message='`start` method is deprecated since 5.5.0 and will be subject for removing in the' + ' next major version. There is no any necessity to call this method anymore.', + category=DeprecationWarning, + stacklevel=2 + ) def terminate(self, *_: Any, **__: Any) -> None: - pass # For backward compatibility - - @property - @abstractmethod - def launch_uuid(self) -> Optional[Union[str, Task[str]]]: - raise NotImplementedError('"launch_uuid" property is not implemented!') - - @property - def launch_id(self) -> Optional[Union[str, Task[str]]]: + """Call this to terminate the client.""" warnings.warn( - message='`launch_id` property is deprecated since 5.5.0 and will be subject for removing in the' - ' next major version. Use `launch_uuid` property instead.', + message='`terminate` method is deprecated since 5.5.0 and will be subject for removing in the' + ' next major version. There is no any necessity to call this method anymore.', category=DeprecationWarning, stacklevel=2 ) - return self.launch_uuid @abstractmethod def current_item(self) -> Optional[Union[str, Task[str]]]: @@ -239,20 +278,20 @@ class RPClient(RP): api_v2: str base_url_v1: str base_url_v2: str - endpoint: str + __endpoint: str is_skipped_an_issue: bool __launch_uuid: str use_own_launch: bool log_batch_size: int log_batch_payload_size: int - project: str + __project: str api_key: str verify_ssl: Union[bool, str] retries: int max_pool_size: int http_timeout: Union[float, Tuple[float, float]] session: requests.Session - step_reporter: StepReporter + __step_reporter: StepReporter mode: str launch_uuid_print: Optional[bool] print_output: Optional[TextIO] @@ -260,6 +299,39 @@ class RPClient(RP): _item_stack: LifoQueue _log_batcher: LogBatcher[RPRequestLog] + @property + def launch_uuid(self) -> Optional[str]: + return self.__launch_uuid + + @property + def endpoint(self) -> str: + return self.__endpoint + + @property + def project(self) -> str: + return self.__project + + @property + def step_reporter(self) -> StepReporter: + return self.__step_reporter + + def __init_session(self) -> None: + retry_strategy = Retry( + total=self.retries, + backoff_factor=0.1, + status_forcelist=[429, 500, 502, 503, 504] + ) if self.retries else DEFAULT_RETRIES + session = requests.Session() + session.mount('https://', HTTPAdapter( + max_retries=retry_strategy, pool_maxsize=self.max_pool_size)) + # noinspection HttpUrlsUsage + session.mount('http://', HTTPAdapter( + max_retries=retry_strategy, pool_maxsize=self.max_pool_size)) + if self.api_key: + session.headers['Authorization'] = 'Bearer {0}'.format( + self.api_key) + self.session = session + def __init__( self, endpoint: str, @@ -301,12 +373,12 @@ def __init__( """ set_current(self) self.api_v1, self.api_v2 = 'v1', 'v2' - self.endpoint = endpoint - self.project = project + self.__endpoint = endpoint + self.__project = project self.base_url_v1 = uri_join( - self.endpoint, 'api/{}'.format(self.api_v1), self.project) + self.__endpoint, 'api/{}'.format(self.api_v1), self.__project) self.base_url_v2 = uri_join( - self.endpoint, 'api/{}'.format(self.api_v2), self.project) + self.__endpoint, 'api/{}'.format(self.api_v2), self.__project) self.is_skipped_an_issue = is_skipped_an_issue self.__launch_uuid = launch_uuid if not self.__launch_uuid: @@ -327,7 +399,7 @@ def __init__( self.retries = retries self.max_pool_size = max_pool_size self.http_timeout = http_timeout - self.step_reporter = StepReporter(self) + self.__step_reporter = StepReporter(self) self._item_stack = LifoQueue() self.mode = mode self._skip_analytics = getenv('AGENT_NO_ANALYTICS') @@ -358,213 +430,6 @@ def __init__( self.__init_session() - @property - def launch_uuid(self) -> Optional[str]: - return self.__launch_uuid - - @launch_uuid.setter - def launch_uuid(self, value: Optional[str]) -> None: - self.__launch_uuid = value - - def __init_session(self) -> None: - retry_strategy = Retry( - total=self.retries, - backoff_factor=0.1, - status_forcelist=[429, 500, 502, 503, 504] - ) if self.retries else DEFAULT_RETRIES - session = requests.Session() - session.mount('https://', HTTPAdapter( - max_retries=retry_strategy, pool_maxsize=self.max_pool_size)) - # noinspection HttpUrlsUsage - session.mount('http://', HTTPAdapter( - max_retries=retry_strategy, pool_maxsize=self.max_pool_size)) - if self.api_key: - session.headers['Authorization'] = 'Bearer {0}'.format( - self.api_key) - self.session = session - - def finish_launch(self, - end_time: str, - status: str = None, - attributes: Optional[Union[List, Dict]] = None, - **kwargs: Any) -> Optional[str]: - """Finish launch. - - :param end_time: Launch end time - :param status: Launch status. Can be one of the followings: - PASSED, FAILED, STOPPED, SKIPPED, RESETED, - CANCELLED - :param attributes: Launch attributes - """ - self._log(self._log_batcher.flush()) - if self.launch_uuid is NOT_FOUND or not self.launch_uuid: - logger.warning('Attempt to finish non-existent launch') - return - url = uri_join(self.base_url_v2, 'launch', self.launch_uuid, 'finish') - request_payload = LaunchFinishRequest( - end_time, - status=status, - attributes=attributes, - description=kwargs.get('description') - ).payload - response = HttpRequest(self.session.put, url=url, json=request_payload, - verify_ssl=self.verify_ssl, - name='Finish Launch').make() - if not response: - return - logger.debug('finish_launch - ID: %s', self.launch_uuid) - logger.debug('response message: %s', response.message) - return response.message - - def finish_test_item(self, - item_id: str, - end_time: str, - status: str = None, - issue: Optional[Issue] = None, - attributes: Optional[Union[List, Dict]] = None, - description: str = None, - retry: bool = False, - **kwargs: Any) -> Optional[str]: - """Finish suite/case/step/nested step item. - - :param item_id: ID of the test item - :param end_time: The item end time - :param status: Test status. Allowable values: "passed", - "failed", "stopped", "skipped", "interrupted", - "cancelled" or None - :param attributes: Test item attributes(tags). Pairs of key and value. - Override attributes on start - :param description: Test item description. Overrides description - from start request. - :param issue: Issue of the current test item - :param retry: Used to report retry of the test. Allowable values: - "True" or "False" - """ - if item_id is NOT_FOUND or not item_id: - logger.warning('Attempt to finish non-existent item') - return - url = uri_join(self.base_url_v2, 'item', item_id) - request_payload = ItemFinishRequest( - end_time, - self.launch_uuid, - status, - attributes=attributes, - description=description, - is_skipped_an_issue=self.is_skipped_an_issue, - issue=issue, - retry=retry - ).payload - response = HttpRequest(self.session.put, url=url, json=request_payload, - verify_ssl=self.verify_ssl).make() - if not response: - return - self._remove_current_item() - logger.debug('finish_test_item - ID: %s', item_id) - logger.debug('response message: %s', response.message) - return response.message - - def get_item_id_by_uuid(self, uuid: str) -> Optional[str]: - """Get test item ID by the given UUID. - - :param uuid: UUID returned on the item start - :return: Test item ID - """ - url = uri_join(self.base_url_v1, 'item', 'uuid', uuid) - response = HttpRequest(self.session.get, url=url, - verify_ssl=self.verify_ssl).make() - return response.id if response else None - - def get_launch_info(self) -> Optional[Dict]: - """Get the current launch information. - - :return dict: Launch information in dictionary - """ - if self.launch_uuid is None: - return {} - url = uri_join(self.base_url_v1, 'launch', 'uuid', self.launch_uuid) - logger.debug('get_launch_info - ID: %s', self.launch_uuid) - response = HttpRequest(self.session.get, url=url, - verify_ssl=self.verify_ssl).make() - if not response: - return - launch_info = None - if response.is_success: - launch_info = response.json - logger.debug( - 'get_launch_info - Launch info: %s', response.json) - else: - logger.warning('get_launch_info - Launch info: ' - 'Failed to fetch launch ID from the API.') - return launch_info - - def get_launch_ui_id(self) -> Optional[int]: - """Get UI ID of the current launch. - - :return: UI ID of the given launch. None if UI ID has not been found. - """ - launch_info = self.get_launch_info() - return launch_info.get('id') if launch_info else None - - def get_launch_ui_url(self) -> Optional[str]: - """Get UI URL of the current launch. - - :return: launch URL or all launches URL. - """ - launch_info = self.get_launch_info() - ui_id = launch_info.get('id') if launch_info else None - if not ui_id: - return - mode = launch_info.get('mode') if launch_info else None - if not mode: - mode = self.mode - - launch_type = 'launches' if mode.upper() == 'DEFAULT' else 'userdebug' - - path = 'ui/#{project_name}/{launch_type}/all/{launch_id}'.format( - project_name=self.project.lower(), launch_type=launch_type, - launch_id=ui_id) - url = uri_join(self.endpoint, path) - logger.debug('get_launch_ui_url - UUID: %s', self.launch_uuid) - return url - - def get_project_settings(self) -> Optional[Dict]: - """Get project settings. - - :return: HTTP response in dictionary - """ - url = uri_join(self.base_url_v1, 'settings') - response = HttpRequest(self.session.get, url=url, - verify_ssl=self.verify_ssl).make() - return response.json if response else None - - def _log(self, batch: Optional[List[RPRequestLog]]) -> Optional[Tuple[str, ...]]: - url = uri_join(self.base_url_v2, 'log') - if batch: - response = HttpRequest(self.session.post, url, files=RPLogBatch(batch).payload, - verify_ssl=self.verify_ssl).make() - return response.messages - - def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> Optional[Tuple[str, ...]]: - """Send log message to the Report Portal. - - :param time: Time in UTC - :param message: Log message text - :param level: Message's log level - :param attachment: Message's attachments - :param item_id: ID of the RP item the message belongs to - """ - if item_id is NOT_FOUND: - logger.warning("Attempt to log to non-existent item") - return - rp_file = RPFile(**attachment) if attachment else None - rp_log = RPRequestLog(self.launch_uuid, time, rp_file, item_id, level, message) - return self._log(self._log_batcher.append(rp_log)) - - def start(self) -> None: - """Start the client.""" - pass - def start_launch(self, name: str, start_time: str, @@ -603,7 +468,7 @@ def start_launch(self, if not self._skip_analytics: send_event('start_launch', *agent_name_version(attributes)) - self.launch_uuid = response.id + self.__launch_uuid = response.id logger.debug('start_launch - ID: %s', self.launch_uuid) if self.launch_uuid_print and self.print_output: print(f'Report Portal Launch UUID: {self.launch_uuid}', file=self.print_output) @@ -678,9 +543,85 @@ def start_test_item(self, str(response.json)) return item_id - def terminate(self, *_: Any, **__: Any) -> None: - """Call this to terminate the client.""" - pass + def finish_test_item(self, + item_id: str, + end_time: str, + status: str = None, + issue: Optional[Issue] = None, + attributes: Optional[Union[List, Dict]] = None, + description: str = None, + retry: bool = False, + **kwargs: Any) -> Optional[str]: + """Finish suite/case/step/nested step item. + + :param item_id: ID of the test item + :param end_time: The item end time + :param status: Test status. Allowable values: "passed", + "failed", "stopped", "skipped", "interrupted", + "cancelled" or None + :param attributes: Test item attributes(tags). Pairs of key and value. + Override attributes on start + :param description: Test item description. Overrides description + from start request. + :param issue: Issue of the current test item + :param retry: Used to report retry of the test. Allowable values: + "True" or "False" + """ + if item_id is NOT_FOUND or not item_id: + logger.warning('Attempt to finish non-existent item') + return + url = uri_join(self.base_url_v2, 'item', item_id) + request_payload = ItemFinishRequest( + end_time, + self.launch_uuid, + status, + attributes=attributes, + description=description, + is_skipped_an_issue=self.is_skipped_an_issue, + issue=issue, + retry=retry + ).payload + response = HttpRequest(self.session.put, url=url, json=request_payload, + verify_ssl=self.verify_ssl).make() + if not response: + return + self._remove_current_item() + logger.debug('finish_test_item - ID: %s', item_id) + logger.debug('response message: %s', response.message) + return response.message + + def finish_launch(self, + end_time: str, + status: str = None, + attributes: Optional[Union[List, Dict]] = None, + **kwargs: Any) -> Optional[str]: + """Finish launch. + + :param end_time: Launch end time + :param status: Launch status. Can be one of the followings: + PASSED, FAILED, STOPPED, SKIPPED, RESETED, + CANCELLED + :param attributes: Launch attributes + """ + self._log(self._log_batcher.flush()) + if self.launch_uuid is NOT_FOUND or not self.launch_uuid: + logger.warning('Attempt to finish non-existent launch') + return + url = uri_join(self.base_url_v2, 'launch', self.launch_uuid, 'finish') + request_payload = LaunchFinishRequest( + end_time, + status=status, + attributes=attributes, + description=kwargs.get('description') + ).payload + response = HttpRequest(self.session.put, url=url, json=request_payload, + verify_ssl=self.verify_ssl, + name='Finish Launch').make() + if not response: + return + logger.debug('finish_launch - ID: %s', self.launch_uuid) + logger.debug('response message: %s', response.message) + return response.message def update_test_item(self, item_uuid: str, attributes: Optional[Union[List, Dict]] = None, description: Optional[str] = None) -> Optional[str]: @@ -704,6 +645,105 @@ def update_test_item(self, item_uuid: str, attributes: Optional[Union[List, Dict logger.debug('update_test_item - Item: %s', item_id) return response.message + def _log(self, batch: Optional[List[RPRequestLog]]) -> Optional[Tuple[str, ...]]: + if batch: + url = uri_join(self.base_url_v2, 'log') + response = HttpRequest(self.session.post, url, files=RPLogBatch(batch).payload, + verify_ssl=self.verify_ssl).make() + return response.messages + + def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, + attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> Optional[Tuple[str, ...]]: + """Send log message to the Report Portal. + + :param time: Time in UTC + :param message: Log message text + :param level: Message's log level + :param attachment: Message's attachments + :param item_id: ID of the RP item the message belongs to + """ + if item_id is NOT_FOUND: + logger.warning("Attempt to log to non-existent item") + return + rp_file = RPFile(**attachment) if attachment else None + rp_log = RPRequestLog(self.launch_uuid, time, rp_file, item_id, level, message) + return self._log(self._log_batcher.append(rp_log)) + + def get_item_id_by_uuid(self, uuid: str) -> Optional[str]: + """Get test item ID by the given UUID. + + :param uuid: UUID returned on the item start + :return: Test item ID + """ + url = uri_join(self.base_url_v1, 'item', 'uuid', uuid) + response = HttpRequest(self.session.get, url=url, + verify_ssl=self.verify_ssl).make() + return response.id if response else None + + def get_launch_info(self) -> Optional[Dict]: + """Get the current launch information. + + :return dict: Launch information in dictionary + """ + if self.launch_uuid is None: + return {} + url = uri_join(self.base_url_v1, 'launch', 'uuid', self.launch_uuid) + logger.debug('get_launch_info - ID: %s', self.launch_uuid) + response = HttpRequest(self.session.get, url=url, + verify_ssl=self.verify_ssl).make() + if not response: + return + launch_info = None + if response.is_success: + launch_info = response.json + logger.debug( + 'get_launch_info - Launch info: %s', response.json) + else: + logger.warning('get_launch_info - Launch info: ' + 'Failed to fetch launch ID from the API.') + return launch_info + + def get_launch_ui_id(self) -> Optional[int]: + """Get UI ID of the current launch. + + :return: UI ID of the given launch. None if UI ID has not been found. + """ + launch_info = self.get_launch_info() + return launch_info.get('id') if launch_info else None + + def get_launch_ui_url(self) -> Optional[str]: + """Get UI URL of the current launch. + + :return: launch URL or all launches URL. + """ + launch_info = self.get_launch_info() + ui_id = launch_info.get('id') if launch_info else None + if not ui_id: + return + mode = launch_info.get('mode') if launch_info else None + if not mode: + mode = self.mode + + launch_type = 'launches' if mode.upper() == 'DEFAULT' else 'userdebug' + + path = 'ui/#{project_name}/{launch_type}/all/{launch_id}'.format( + project_name=self.__project.lower(), launch_type=launch_type, + launch_id=ui_id) + url = uri_join(self.__endpoint, path) + logger.debug('get_launch_ui_url - UUID: %s', self.launch_uuid) + return url + + def get_project_settings(self) -> Optional[Dict]: + """Get project settings. + + :return: HTTP response in dictionary + """ + url = uri_join(self.base_url_v1, 'settings') + response = HttpRequest(self.session.get, url=url, + verify_ssl=self.verify_ssl).make() + return response.json if response else None + + def _add_current_item(self, item: str) -> None: """Add the last item from the self._items queue.""" self._item_stack.put(item) @@ -726,8 +766,8 @@ def clone(self) -> 'RPClient': :rtype: RPClient """ cloned = RPClient( - endpoint=self.endpoint, - project=self.project, + endpoint=self.__endpoint, + project=self.__project, api_key=self.api_key, log_batch_size=self.log_batch_size, is_skipped_an_issue=self.is_skipped_an_issue, @@ -745,6 +785,14 @@ def clone(self) -> 'RPClient': cloned._add_current_item(current_item) return cloned + def start(self) -> None: + """Start the client.""" + pass + + def terminate(self, *_: Any, **__: Any) -> None: + """Call this to terminate the client.""" + pass + def __getstate__(self) -> Dict[str, Any]: """Control object pickling and return object fields as Dictionary. diff --git a/reportportal_client/steps/__init__.py b/reportportal_client/steps/__init__.py index 0689a31b..faf03b88 100644 --- a/reportportal_client/steps/__init__.py +++ b/reportportal_client/steps/__init__.py @@ -45,7 +45,7 @@ def test_my_nested_step(): from functools import wraps -from reportportal_client._local import current +from reportportal_client import current from reportportal_client.helpers import get_function_params, timestamp NESTED_STEP_ITEMS = ('step', 'scenario', 'before_class', 'before_groups', @@ -157,6 +157,7 @@ def __call__(self, func): :param func: function reference """ + @wraps(func) def wrapper(*args, **kwargs): __tracebackhide__ = True diff --git a/reportportal_client/steps/__init__.pyi b/reportportal_client/steps/__init__.pyi index 2a583751..9ace6093 100644 --- a/reportportal_client/steps/__init__.pyi +++ b/reportportal_client/steps/__init__.pyi @@ -11,42 +11,42 @@ # See the License for the specific language governing permissions and # limitations under the License -from typing import Text, Optional, Dict, Any, Callable, Union +from typing import Optional, Dict, Any, Callable, Union -from reportportal_client.aio.client import RPClient as AsyncRPClient -from reportportal_client.client import RPClient +from reportportal_client import RP +from reportportal_client.aio import Task class StepReporter: - client: RPClient = ... + client: RP = ... - def __init__(self, rp_client: Union[RPClient, AsyncRPClient]) -> None: ... + def __init__(self, rp_client: RP) -> None: ... def start_nested_step(self, - name: Text, - start_time: Text, + name: str, + start_time: str, parameters: Dict = ..., - **kwargs: Any) -> Text: ... + **kwargs: Any) -> Union[Optional[str], Task[str]]: ... def finish_nested_step(self, - item_id: Text, - end_time: Text, - status: Text, + item_id: str, + end_time: str, + status: str, **kwargs: Any) -> None: ... class Step: - name: Text = ... + name: str = ... params: Dict = ... - status: Text = ... - client: Optional[RPClient] = ... - __item_id: Optional[Text] = ... + status: str = ... + client: Optional[RP] = ... + __item_id: Union[Optional[str], Task[str]] = ... def __init__(self, - name: Text, + name: str, params: Dict, - status: Text, - rp_client: Optional[RPClient]) -> None: ... + status: str, + rp_client: Optional[RP]) -> None: ... def __enter__(self) -> None: ... @@ -55,7 +55,7 @@ class Step: def __call__(self, func: Callable) -> Callable: ... -def step(name_source: Union[Callable, Text], +def step(name_source: Union[Callable, str], params: Dict = ..., - status: Text = ..., - rp_client: RPClient = ...) -> None: ... + status: str = ..., + rp_client: RP = ...) -> None: ... diff --git a/tests/test_client.py b/tests/test_client.py index cec79f4a..99c2b0cb 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -57,7 +57,7 @@ def invalid_response(*args, **kwargs): ) def test_connection_errors(rp_client, requests_method, client_method, client_params): - rp_client.launch_uuid = 'test_launch_id' + rp_client._RPClient_launch_uuid = 'test_launch_id' getattr(rp_client.session, requests_method).side_effect = connection_error result = getattr(rp_client, client_method)(*client_params) assert result is None @@ -84,7 +84,7 @@ def test_connection_errors(rp_client, requests_method, client_method, ) def test_invalid_responses(rp_client, requests_method, client_method, client_params): - rp_client.launch_uuid = 'test_launch_id' + rp_client._RPClient_launch_uuid = 'test_launch_id' getattr(rp_client.session, requests_method).side_effect = invalid_response result = getattr(rp_client, client_method)(*client_params) assert result is None @@ -107,8 +107,8 @@ def test_invalid_responses(rp_client, requests_method, client_method, ] ) def test_launch_url_get(rp_client, launch_mode, project_name, expected_url): - rp_client.launch_uuid = 'test_launch_id' - rp_client.project = project_name + rp_client._RPClient_launch_uuid = 'test_launch_id' + rp_client.__project = project_name response = mock.Mock() response.is_success = True From 7b8fb85d4587cedc32a28a63805d09a91a8b27e8 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 25 Sep 2023 14:45:27 +0300 Subject: [PATCH 136/268] Fix tests --- tests/test_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 99c2b0cb..cc1f5f52 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -57,7 +57,7 @@ def invalid_response(*args, **kwargs): ) def test_connection_errors(rp_client, requests_method, client_method, client_params): - rp_client._RPClient_launch_uuid = 'test_launch_id' + rp_client._RPClient__launch_uuid = 'test_launch_id' getattr(rp_client.session, requests_method).side_effect = connection_error result = getattr(rp_client, client_method)(*client_params) assert result is None @@ -84,7 +84,7 @@ def test_connection_errors(rp_client, requests_method, client_method, ) def test_invalid_responses(rp_client, requests_method, client_method, client_params): - rp_client._RPClient_launch_uuid = 'test_launch_id' + rp_client._RPClient__launch_uuid = 'test_launch_id' getattr(rp_client.session, requests_method).side_effect = invalid_response result = getattr(rp_client, client_method)(*client_params) assert result is None @@ -107,8 +107,8 @@ def test_invalid_responses(rp_client, requests_method, client_method, ] ) def test_launch_url_get(rp_client, launch_mode, project_name, expected_url): - rp_client._RPClient_launch_uuid = 'test_launch_id' - rp_client.__project = project_name + rp_client._RPClient__launch_uuid = 'test_launch_id' + rp_client._RPClient__project = project_name response = mock.Mock() response.is_success = True From 47dc18cb90f39dbdd08427365131414d0c39e72b Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 25 Sep 2023 14:53:34 +0300 Subject: [PATCH 137/268] Add getters --- reportportal_client/aio/client.py | 16 +++++++++++----- reportportal_client/client.py | 4 ++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 159737ae..19c80ce8 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -465,6 +465,8 @@ class AsyncRPClient(RP): def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[str] = None, client: Optional[Client] = None, **kwargs: Any) -> None: set_current(self) + self.__endpoint = endpoint + self.__project = project self.__step_reporter = StepReporter(self) self._item_stack = LifoQueue() self._log_batcher = LogBatcher() @@ -473,7 +475,7 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[str] = else: self.__client = Client(endpoint, project, **kwargs) if launch_uuid: - self.launch_uuid = launch_uuid + self.__launch_uuid = launch_uuid self.use_own_launch = False else: self.use_own_launch = True @@ -482,9 +484,13 @@ def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[str] = def launch_uuid(self) -> Optional[str]: return self.__launch_uuid - @launch_uuid.setter - def launch_uuid(self, value: Optional[str]) -> None: - self.__launch_uuid = value + @property + def endpoint(self) -> str: + return self.__endpoint + + @property + def project(self) -> str: + return self.__project @property def step_reporter(self) -> StepReporter: @@ -503,7 +509,7 @@ async def start_launch(self, launch_uuid = await self.__client.start_launch(name, start_time, description=description, attributes=attributes, rerun=rerun, rerun_of=rerun_of, **kwargs) - self.launch_uuid = launch_uuid + self.__launch_uuid = launch_uuid return launch_uuid async def start_test_item(self, diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 64c4f916..ad309931 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -448,6 +448,8 @@ def start_launch(self, :param rerun_of: For rerun mode specifies which launch will be re-run. Should be used with the 'rerun' option. """ + if not self.use_own_launch: + return self.launch_uuid url = uri_join(self.base_url_v2, 'launch') request_payload = LaunchStartRequest( name=name, @@ -604,6 +606,8 @@ def finish_launch(self, :param attributes: Launch attributes """ self._log(self._log_batcher.flush()) + if not self.use_own_launch: + return "" if self.launch_uuid is NOT_FOUND or not self.launch_uuid: logger.warning('Attempt to finish non-existent launch') return From 5ba1a79faf97ca1790d23710a8f1d8bd77fe4a2a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 25 Sep 2023 16:37:28 +0300 Subject: [PATCH 138/268] Fix typing --- reportportal_client/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index 7728d5db..3caf613e 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -30,7 +30,7 @@ _T = TypeVar('_T') -class LifoQueue(Generic[_T], queue.LifoQueue[_T]): +class LifoQueue(Generic[_T], queue.LifoQueue): """This Queue adds 'last' method to original queue.LifoQueue. Unlike 'get' method in queue.LifoQueue the 'last' method do not remove an entity from the queue. From 940435dd5a73c666338dc5bbe2da6f786cad5d85 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 25 Sep 2023 16:43:57 +0300 Subject: [PATCH 139/268] Fix use_own_launch property --- reportportal_client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index ad309931..9c64cc8c 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -391,7 +391,7 @@ def __init__( stacklevel=2 ) self.__launch_uuid = launch_id - self.use_own_launch = bool(self.__launch_uuid) + self.use_own_launch = not bool(self.__launch_uuid) self.log_batch_size = log_batch_size self.log_batch_payload_size = log_batch_payload_size self._log_batcher = log_batcher or LogBatcher(self.log_batch_size, self.log_batch_payload_size) From ffafa275f48750df6a17f3de59b34045c3313b8a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 25 Sep 2023 16:45:40 +0300 Subject: [PATCH 140/268] Fix tests --- tests/steps/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/steps/conftest.py b/tests/steps/conftest.py index f700044d..eade2687 100644 --- a/tests/steps/conftest.py +++ b/tests/steps/conftest.py @@ -23,5 +23,4 @@ def rp_client(): client = RPClient('http://endpoint', 'project', 'api_key') client.session = mock.Mock() - client.step_reporter = StepReporter(client) return client From 97afbc92cc1d6136c2c2d88a39bf85960cec922a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 25 Sep 2023 18:25:42 +0300 Subject: [PATCH 141/268] Pydocs and type hinting fixes --- reportportal_client/client.py | 89 ++++++++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 23 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 9c64cc8c..514d3a4e 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -19,7 +19,7 @@ import warnings from abc import abstractmethod from os import getenv -from typing import Union, Tuple, List, Dict, Any, Optional, TextIO +from typing import Union, Tuple, Any, Optional, TextIO, List, Dict import requests from requests.adapters import HTTPAdapter, Retry, DEFAULT_RETRIES @@ -49,6 +49,7 @@ class RP(metaclass=AbstractBaseClass): This abstract class serves as common interface for different Report Portal clients. It's implemented to ease migration from version to version and to ensure that each particular client has the same methods. """ + __metaclass__ = AbstractBaseClass @property @@ -77,11 +78,19 @@ def launch_id(self) -> Optional[Union[str, Task[str]]]: @property @abstractmethod def endpoint(self) -> str: + """Return current base URL. + + :return: base URL string + """ raise NotImplementedError('"endpoint" property is not implemented!') @property @abstractmethod def project(self) -> str: + """Return current Project name. + + :return: Project name string + """ raise NotImplementedError('"project" property is not implemented!') @property @@ -98,7 +107,7 @@ def start_launch(self, name: str, start_time: str, description: Optional[str] = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Union[Optional[str], Task[str]]: @@ -122,8 +131,8 @@ def start_test_item(self, item_type: str, *, description: Optional[str] = None, - attributes: Optional[List[Dict]] = None, - parameters: Optional[Dict] = None, + attributes: Optional[List[dict]] = None, + parameters: Optional[dict] = None, parent_item_id: Union[Optional[str], Task[str]] = None, has_stats: bool = True, code_ref: Optional[str] = None, @@ -160,7 +169,7 @@ def finish_test_item(self, *, status: str = None, issue: Optional[Issue] = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, description: str = None, retry: bool = False, **kwargs: Any) -> Union[Optional[str], Task[str]]: @@ -185,7 +194,7 @@ def finish_test_item(self, def finish_launch(self, end_time: str, status: str = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, **kwargs: Any) -> Union[Optional[str], Task[str]]: """Finish current launch. @@ -200,7 +209,7 @@ def finish_launch(self, @abstractmethod def update_test_item(self, item_uuid: Union[Optional[str], Task[str]], - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, description: Optional[str] = None) -> Union[Optional[str], Task[str]]: """Update existing test item at the Report Portal. @@ -213,27 +222,56 @@ def update_test_item(self, @abstractmethod def get_launch_info(self) -> Union[Optional[dict], Task[str]]: + """Get the current launch information. + + :returns Launch information in dictionary + """ raise NotImplementedError('"get_launch_info" method is not implemented!') @abstractmethod def get_item_id_by_uuid(self, item_uuid: Union[str, Task[str]]) -> Optional[str]: + """Get test item ID by the given UUID. + + :param item_uuid: UUID returned on the item start + :returns Test item ID + """ raise NotImplementedError('"get_item_id_by_uuid" method is not implemented!') @abstractmethod def get_launch_ui_id(self) -> Optional[int]: + """Get UI ID of the current launch. + + :returns UI ID of the given launch. None if UI ID has not been found. + """ raise NotImplementedError('"get_launch_ui_id" method is not implemented!') @abstractmethod def get_launch_ui_url(self) -> Optional[str]: + """Get full quality URL of the current launch. + + :returns launch URL string + """ raise NotImplementedError('"get_launch_ui_id" method is not implemented!') @abstractmethod - def get_project_settings(self) -> Optional[Dict]: + def get_project_settings(self) -> Union[Optional[dict], Task[dict]]: + """Get project settings. + + :returns HTTP response in dictionary + """ raise NotImplementedError('"get_project_settings" method is not implemented!') @abstractmethod def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, item_id: Union[Optional[str], Task[str]] = None) -> None: + attachment: Optional[dict] = None, item_id: Union[Optional[str], Task[str]] = None) -> None: + """Send log message to the Report Portal. + + :param datetime: Time in UTC + :param message: Log message text + :param level: Message's log level + :param attachment: Message's attachments + :param item_id: ID of the RP item the message belongs to + """ raise NotImplementedError('"log" method is not implemented!') def start(self) -> None: @@ -260,7 +298,11 @@ def current_item(self) -> Optional[Union[str, Task[str]]]: raise NotImplementedError('"current_item" method is not implemented!') @abstractmethod - def clone(self) -> 'RPClient': + def clone(self) -> 'RP': + """Clone the client object, set current Item ID as cloned root Item ID. + + :returns: Cloned client object + """ raise NotImplementedError('"clone" method is not implemented!') @@ -274,6 +316,7 @@ class RPClient(RP): NOTICE: the class is not thread-safe, use new class instance for every new thread to avoid request/response messing and other issues. """ + api_v1: str api_v2: str base_url_v1: str @@ -434,7 +477,7 @@ def start_launch(self, name: str, start_time: str, description: Optional[str] = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Optional[str]: @@ -482,7 +525,7 @@ def start_test_item(self, item_type: str, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, - parameters: Optional[Dict] = None, + parameters: Optional[dict] = None, parent_item_id: Optional[str] = None, has_stats: bool = True, code_ref: Optional[str] = None, @@ -550,7 +593,7 @@ def finish_test_item(self, end_time: str, status: str = None, issue: Optional[Issue] = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, description: str = None, retry: bool = False, **kwargs: Any) -> Optional[str]: @@ -595,7 +638,7 @@ def finish_test_item(self, def finish_launch(self, end_time: str, status: str = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, **kwargs: Any) -> Optional[str]: """Finish launch. @@ -627,7 +670,7 @@ def finish_launch(self, logger.debug('response message: %s', response.message) return response.message - def update_test_item(self, item_uuid: str, attributes: Optional[Union[List, Dict]] = None, + def update_test_item(self, item_uuid: str, attributes: Optional[Union[list, dict]] = None, description: Optional[str] = None) -> Optional[str]: """Update existing test item at the Report Portal. @@ -657,7 +700,7 @@ def _log(self, batch: Optional[List[RPRequestLog]]) -> Optional[Tuple[str, ...]] return response.messages def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, item_id: Optional[str] = None) -> Optional[Tuple[str, ...]]: + attachment: Optional[dict] = None, item_id: Optional[str] = None) -> Optional[Tuple[str, ...]]: """Send log message to the Report Portal. :param time: Time in UTC @@ -673,21 +716,21 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, rp_log = RPRequestLog(self.launch_uuid, time, rp_file, item_id, level, message) return self._log(self._log_batcher.append(rp_log)) - def get_item_id_by_uuid(self, uuid: str) -> Optional[str]: + def get_item_id_by_uuid(self, item_uuid: str) -> Optional[str]: """Get test item ID by the given UUID. - :param uuid: UUID returned on the item start - :return: Test item ID + :param item_uuid: UUID returned on the item start + :returns Test item ID """ - url = uri_join(self.base_url_v1, 'item', 'uuid', uuid) + url = uri_join(self.base_url_v1, 'item', 'uuid', item_uuid) response = HttpRequest(self.session.get, url=url, verify_ssl=self.verify_ssl).make() return response.id if response else None - def get_launch_info(self) -> Optional[Dict]: + def get_launch_info(self) -> Optional[dict]: """Get the current launch information. - :return dict: Launch information in dictionary + :returns Launch information in dictionary """ if self.launch_uuid is None: return {} @@ -737,7 +780,7 @@ def get_launch_ui_url(self) -> Optional[str]: logger.debug('get_launch_ui_url - UUID: %s', self.launch_uuid) return url - def get_project_settings(self) -> Optional[Dict]: + def get_project_settings(self) -> Optional[dict]: """Get project settings. :return: HTTP response in dictionary From 65dabc0f458432516d9b80a070c2a520130f649a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 25 Sep 2023 21:01:40 +0300 Subject: [PATCH 142/268] Pydocs add --- reportportal_client/client.py | 51 +++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 514d3a4e..b8c3af85 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -57,7 +57,7 @@ class RP(metaclass=AbstractBaseClass): def launch_uuid(self) -> Optional[Union[str, Task[str]]]: """Return current launch UUID. - :return: UUID string + :returns UUID string """ raise NotImplementedError('"launch_uuid" property is not implemented!') @@ -65,7 +65,7 @@ def launch_uuid(self) -> Optional[Union[str, Task[str]]]: def launch_id(self) -> Optional[Union[str, Task[str]]]: """Return current launch UUID. - :return: UUID string + :returns UUID string """ warnings.warn( message='`launch_id` property is deprecated since 5.5.0 and will be subject for removing in the' @@ -80,7 +80,7 @@ def launch_id(self) -> Optional[Union[str, Task[str]]]: def endpoint(self) -> str: """Return current base URL. - :return: base URL string + :returns base URL string """ raise NotImplementedError('"endpoint" property is not implemented!') @@ -89,7 +89,7 @@ def endpoint(self) -> str: def project(self) -> str: """Return current Project name. - :return: Project name string + :returns Project name string """ raise NotImplementedError('"project" property is not implemented!') @@ -98,7 +98,7 @@ def project(self) -> str: def step_reporter(self) -> StepReporter: """Return StepReporter object for the current launch. - :return: StepReporter to report steps + :returns StepReporter to report steps """ raise NotImplementedError('"step_reporter" property is not implemented!') @@ -313,8 +313,6 @@ class RPClient(RP): official to make calls to Report Portal. It handles HTTP request and response bodies generation and serialization, connection retries and log batching. - NOTICE: the class is not thread-safe, use new class instance for every new - thread to avoid request/response messing and other issues. """ api_v1: str @@ -344,18 +342,34 @@ class RPClient(RP): @property def launch_uuid(self) -> Optional[str]: + """Return current launch UUID. + + :returns UUID string + """ return self.__launch_uuid @property def endpoint(self) -> str: + """Return current base URL. + + :returns base URL string + """ return self.__endpoint @property def project(self) -> str: + """Return current Project name. + + :returns Project name string + """ return self.__project @property def step_reporter(self) -> StepReporter: + """Return StepReporter object for the current launch. + + :returns StepReporter to report steps + """ return self.__step_reporter def __init_session(self) -> None: @@ -761,7 +775,7 @@ def get_launch_ui_id(self) -> Optional[int]: def get_launch_ui_url(self) -> Optional[str]: """Get UI URL of the current launch. - :return: launch URL or all launches URL. + :returns launch URL or all launches URL. """ launch_info = self.get_launch_info() ui_id = launch_info.get('id') if launch_info else None @@ -783,27 +797,32 @@ def get_launch_ui_url(self) -> Optional[str]: def get_project_settings(self) -> Optional[dict]: """Get project settings. - :return: HTTP response in dictionary + :returns HTTP response in dictionary """ url = uri_join(self.base_url_v1, 'settings') response = HttpRequest(self.session.get, url=url, verify_ssl=self.verify_ssl).make() return response.json if response else None - def _add_current_item(self, item: str) -> None: """Add the last item from the self._items queue.""" self._item_stack.put(item) def _remove_current_item(self) -> Optional[str]: - """Remove the last item from the self._items queue.""" + """Remove the last item from the self._items queue. + + :returns Item UUID string + """ try: return self._item_stack.get(timeout=0) except queue.Empty: return def current_item(self) -> Optional[str]: - """Retrieve the last item reported by the client.""" + """Retrieve the last item reported by the client. + + :returns Item UUID string + """ return self._item_stack.last() def clone(self) -> 'RPClient': @@ -832,14 +851,6 @@ def clone(self) -> 'RPClient': cloned._add_current_item(current_item) return cloned - def start(self) -> None: - """Start the client.""" - pass - - def terminate(self, *_: Any, **__: Any) -> None: - """Call this to terminate the client.""" - pass - def __getstate__(self) -> Dict[str, Any]: """Control object pickling and return object fields as Dictionary. From 7aee6ebc80e0491a657dd945b7d97a896b2c5efd Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 25 Sep 2023 21:10:21 +0300 Subject: [PATCH 143/268] Unify ReportPortal name --- README.md | 2 +- reportportal_client/_local/__init__.py | 10 +++++----- reportportal_client/aio/client.py | 4 ++-- reportportal_client/client.py | 24 ++++++++++++------------ reportportal_client/core/rp_requests.py | 8 ++++---- reportportal_client/core/worker.py | 2 +- reportportal_client/helpers.py | 2 +- reportportal_client/logs/__init__.py | 6 +++--- reportportal_client/logs/batcher.py | 4 ++++ reportportal_client/logs/log_manager.py | 2 +- reportportal_client/steps/__init__.py | 18 +++++++++--------- setup.py | 4 ++-- tests/test_client.py | 8 ++++---- 13 files changed, 49 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 196d2c51..70b5662a 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ from reportportal_client.helpers import timestamp endpoint = "http://docker.local:8080" project = "default" -# You can get UUID from user profile page in the Report Portal. +# You can get UUID from user profile page in the ReportPortal. api_key = "1adf271d-505f-44a8-ad71-0afbdf8c83bd" launch_name = "Test launch" launch_doc = "Testing logging with attachment." diff --git a/reportportal_client/_local/__init__.py b/reportportal_client/_local/__init__.py index 82cc8238..9cef441e 100644 --- a/reportportal_client/_local/__init__.py +++ b/reportportal_client/_local/__init__.py @@ -1,4 +1,4 @@ -"""Report Portal client context storing and retrieving module.""" +"""ReportPortal client context storing and retrieving module.""" # Copyright (c) 2022 EPAM Systems # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,17 +19,17 @@ def current(): - """Return current Report Portal client.""" + """Return current ReportPortal client.""" if hasattr(__INSTANCES, 'current'): return __INSTANCES.current def set_current(client): - """Save Report Portal client as current. + """Save ReportPortal client as current. - The method is not intended to use used by users. Report Portal client calls + The method is not intended to use used by users. ReportPortal client calls it itself when new client is created. - :param client: Report Portal client + :param client: ReportPortal client """ __INSTANCES.current = client diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 19c80ce8..5a6a1265 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -1,4 +1,4 @@ -"""This module contains asynchronous implementation of Report Portal Client.""" +"""This module contains asynchronous implementation of ReportPortal Client.""" # Copyright (c) 2023 EPAM Systems # Licensed under the Apache License, Version 2.0 (the "License"); @@ -235,7 +235,7 @@ async def start_launch(self, launch_uuid = await response.id logger.debug(f'start_launch - ID: {launch_uuid}') if self.launch_uuid_print and self.print_output: - print(f'Report Portal Launch UUID: {launch_uuid}', file=self.print_output) + print(f'ReportPortal Launch UUID: {launch_uuid}', file=self.print_output) return launch_uuid async def start_test_item(self, diff --git a/reportportal_client/client.py b/reportportal_client/client.py index b8c3af85..dbcf7209 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -1,4 +1,4 @@ -"""This module contains Report Portal Client interface and synchronous implementation class.""" +"""This module contains ReportPortal Client interface and synchronous implementation class.""" # Copyright (c) 2023 EPAM Systems # Licensed under the Apache License, Version 2.0 (the "License"); @@ -44,9 +44,9 @@ class RP(metaclass=AbstractBaseClass): - """Common interface for Report Portal clients. + """Common interface for ReportPortal clients. - This abstract class serves as common interface for different Report Portal clients. It's implemented to + This abstract class serves as common interface for different ReportPortal clients. It's implemented to ease migration from version to version and to ensure that each particular client has the same methods. """ @@ -211,7 +211,7 @@ def update_test_item(self, item_uuid: Union[Optional[str], Task[str]], attributes: Optional[Union[list, dict]] = None, description: Optional[str] = None) -> Union[Optional[str], Task[str]]: - """Update existing test item at the Report Portal. + """Update existing test item at the ReportPortal. :param item_uuid: Test item UUID returned on the item start :param attributes: Test item attributes: [{'key': 'k_name', 'value': 'k_value'}, ...] @@ -264,7 +264,7 @@ def get_project_settings(self) -> Union[Optional[dict], Task[dict]]: @abstractmethod def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[dict] = None, item_id: Union[Optional[str], Task[str]] = None) -> None: - """Send log message to the Report Portal. + """Send log message to the ReportPortal. :param datetime: Time in UTC :param message: Log message text @@ -307,10 +307,10 @@ def clone(self) -> 'RP': class RPClient(RP): - """Report portal client. + """ReportPortal client. - The class is supposed to use by Report Portal agents: both custom and - official to make calls to Report Portal. It handles HTTP request and + The class is supposed to use by ReportPortal agents: both custom and + official to make calls to ReportPortal. It handles HTTP request and response bodies generation and serialization, connection retries and log batching. """ @@ -410,7 +410,7 @@ def __init__( ) -> None: """Initialize required attributes. - :param endpoint: Endpoint of the report portal service + :param endpoint: Endpoint of the ReportPortal service :param project: Project name to report to :param api_key: Authorization API key :param log_batch_size: Option to set the maximum number of @@ -530,7 +530,7 @@ def start_launch(self, self.__launch_uuid = response.id logger.debug('start_launch - ID: %s', self.launch_uuid) if self.launch_uuid_print and self.print_output: - print(f'Report Portal Launch UUID: {self.launch_uuid}', file=self.print_output) + print(f'ReportPortal Launch UUID: {self.launch_uuid}', file=self.print_output) return self.launch_uuid def start_test_item(self, @@ -686,7 +686,7 @@ def finish_launch(self, def update_test_item(self, item_uuid: str, attributes: Optional[Union[list, dict]] = None, description: Optional[str] = None) -> Optional[str]: - """Update existing test item at the Report Portal. + """Update existing test item at the ReportPortal. :param str item_uuid: Test item UUID returned on the item start :param str description: Test item description @@ -715,7 +715,7 @@ def _log(self, batch: Optional[List[RPRequestLog]]) -> Optional[Tuple[str, ...]] def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[dict] = None, item_id: Optional[str] = None) -> Optional[Tuple[str, ...]]: - """Send log message to the Report Portal. + """Send log message to the ReportPortal. :param time: Time in UTC :param message: Log message text diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 16f16e8c..904cb274 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -106,12 +106,12 @@ def priority(self, value: Priority) -> None: self._priority = value def make(self) -> Optional[RPResponse]: - """Make HTTP request to the Report Portal API.""" + """Make HTTP request to the ReportPortal API.""" try: return RPResponse(self.session_method(self.url, data=self.data, json=self.json, files=self.files, verify=self.verify_ssl, timeout=self.http_timeout)) except (KeyError, IOError, ValueError, TypeError) as exc: - logger.warning("Report Portal %s request failed", self.name, exc_info=exc) + logger.warning("ReportPortal %s request failed", self.name, exc_info=exc) class AsyncHttpRequest(HttpRequest): @@ -135,7 +135,7 @@ def __init__(self, super().__init__(session_method=session_method, url=url, data=data, json=json, name=name) async def make(self) -> Optional[AsyncRPResponse]: - """Make HTTP request to the Report Portal API.""" + """Make HTTP request to the ReportPortal API.""" url = await await_if_necessary(self.url) if not url: return @@ -144,7 +144,7 @@ async def make(self) -> Optional[AsyncRPResponse]: try: return AsyncRPResponse(await self.session_method(url, data=data, json=json)) except (KeyError, IOError, ValueError, TypeError) as exc: - logger.warning("Report Portal %s request failed", self.name, exc_info=exc) + logger.warning("ReportPortal %s request failed", self.name, exc_info=exc) class RPRequestBase(metaclass=AbstractBaseClass): diff --git a/reportportal_client/core/worker.py b/reportportal_client/core/worker.py index 2c245008..a52aa74c 100644 --- a/reportportal_client/core/worker.py +++ b/reportportal_client/core/worker.py @@ -55,7 +55,7 @@ def __lt__(self, other): class APIWorker(object): - """Worker that makes HTTP requests to the Report Portal.""" + """Worker that makes HTTP requests to the ReportPortal.""" def __init__(self, task_queue): """Initialize instance attributes.""" diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index 3caf613e..74ade370 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -55,7 +55,7 @@ def dict_to_payload(dictionary: dict) -> List[dict]: """Convert incoming dictionary to the list of dictionaries. This function transforms the given dictionary of tags/attributes into - the format required by the Report Portal API. Also, we add the system + the format required by the ReportPortal API. Also, we add the system key to every tag/attribute that indicates that the key should be hidden from the user in UI. :param dictionary: Dictionary containing tags/attributes diff --git a/reportportal_client/logs/__init__.py b/reportportal_client/logs/__init__.py index a511b61b..ca76188c 100644 --- a/reportportal_client/logs/__init__.py +++ b/reportportal_client/logs/__init__.py @@ -10,7 +10,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License -"""Report portal logging handling module.""" +"""ReportPortal logging handling module.""" import logging import sys @@ -102,8 +102,8 @@ def __init__(self, level=logging.NOTSET, filter_client_logs=False, :param level: level of logging :param filter_client_logs: if True throw away logs emitted by a ReportPortal client - :param endpoint: Report Portal endpoint URL, used to filter - out urllib3 logs, mutes Report Portal HTTP logs if set, optional + :param endpoint: ReportPortal endpoint URL, used to filter + out urllib3 logs, mutes ReportPortal HTTP logs if set, optional parameter :param ignored_record_names: a tuple of record names which will be filtered out by the handler (with startswith method) diff --git a/reportportal_client/logs/batcher.py b/reportportal_client/logs/batcher.py index 9ddbd7ec..f068da8f 100644 --- a/reportportal_client/logs/batcher.py +++ b/reportportal_client/logs/batcher.py @@ -24,6 +24,10 @@ class LogBatcher(Generic[T_co]): + """ + + + """ entry_num: int payload_limit: int _lock: threading.Lock diff --git a/reportportal_client/logs/log_manager.py b/reportportal_client/logs/log_manager.py index 8e32c0dc..2c892785 100644 --- a/reportportal_client/logs/log_manager.py +++ b/reportportal_client/logs/log_manager.py @@ -39,7 +39,7 @@ def __init__(self, rp_url, session, api_version, launch_id, project_name, max_payload_size=MAX_LOG_BATCH_PAYLOAD_SIZE): """Initialize instance attributes. - :param rp_url: Report portal URL + :param rp_url: ReportPortal URL :param session: HTTP Session object :param api_version: RP API version :param launch_id: Parent launch UUID diff --git a/reportportal_client/steps/__init__.py b/reportportal_client/steps/__init__.py index faf03b88..a45e2350 100644 --- a/reportportal_client/steps/__init__.py +++ b/reportportal_client/steps/__init__.py @@ -1,6 +1,6 @@ -"""Report portal Nested Step handling module. +"""ReportPortal Nested Step handling module. -The module for handling and reporting Report Portal Nested Steps inside python +The module for handling and reporting ReportPortal Nested Steps inside python test frameworks. Import 'step' function to use it as decorator or together with 'with' keyword to divide your tests on smaller steps. @@ -61,7 +61,7 @@ class StepReporter: def __init__(self, rp_client): """Initialize required attributes. - :param rp_client: Report Portal client which will be used to report + :param rp_client: ReportPortal client which will be used to report steps """ self.client = rp_client @@ -71,7 +71,7 @@ def start_nested_step(self, start_time, parameters=None, **kwargs): - """Start Nested Step on Report Portal. + """Start Nested Step on ReportPortal. :param name: Nested Step name :param start_time: Nested Step start time @@ -90,7 +90,7 @@ def finish_nested_step(self, end_time, status=None, **kwargs): - """Finish a Nested Step on Report Portal. + """Finish a Nested Step on ReportPortal. :param item_id: Nested Step item ID :param end_time: Nested Step finish time @@ -109,7 +109,7 @@ def __init__(self, name, params, status, rp_client): :param params: Nested Step parameters :param status: Nested Step status which will be reported on successful step finish - :param rp_client: Report Portal client which will be used to report + :param rp_client: ReportPortal client which will be used to report the step """ self.name = name @@ -173,16 +173,16 @@ def wrapper(*args, **kwargs): def step(name_source, params=None, status='PASSED', rp_client=None): """Nested step report function. - Create a Nested Step inside a test method on Report Portal. + Create a Nested Step inside a test method on ReportPortal. :param name_source: a function or string which will be used as step's name :param params: nested step parameters which will be reported as the first step entry. If 'name_source' is a function reference and this parameter is not specified, they will be taken from the function. :param status: the status which will be reported after the step - passed. Can be any of legal Report Portal statuses. + passed. Can be any of legal ReportPortal statuses. E.G.: PASSED, WARN, INFO, etc. Default value is PASSED - :param rp_client: overrides Report Portal client which will be used in + :param rp_client: overrides ReportPortal client which will be used in step reporting :return: a step context object """ diff --git a/setup.py b/setup.py index a1d82a3e..bca59839 100644 --- a/setup.py +++ b/setup.py @@ -30,10 +30,10 @@ def read_file(fname): 'reportportal_client.services': TYPE_STUBS, }, version=__version__, - description='Python client for Report Portal v5.', + description='Python client for ReportPortal v5.', long_description=read_file('README.md'), long_description_content_type='text/markdown', - author='Report Portal Team', + author='ReportPortal Team', author_email='support@reportportal.io', url='https://github.com/reportportal/client-Python', download_url=('https://github.com/reportportal/client-Python/' diff --git a/tests/test_client.py b/tests/test_client.py index cc1f5f52..3382c199 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -206,7 +206,7 @@ def test_launch_uuid_print(): client.session = mock.Mock() client._skip_analytics = True client.start_launch('Test Launch', timestamp()) - assert 'Report Portal Launch UUID: ' in str_io.getvalue() + assert 'ReportPortal Launch UUID: ' in str_io.getvalue() def test_no_launch_uuid_print(): @@ -216,7 +216,7 @@ def test_no_launch_uuid_print(): client.session = mock.Mock() client._skip_analytics = True client.start_launch('Test Launch', timestamp()) - assert 'Report Portal Launch UUID: ' not in str_io.getvalue() + assert 'ReportPortal Launch UUID: ' not in str_io.getvalue() @mock.patch('reportportal_client.client.sys.stdout', new_callable=StringIO) @@ -227,7 +227,7 @@ def test_launch_uuid_print_default_io(mock_stdout): client._skip_analytics = True client.start_launch('Test Launch', timestamp()) - assert 'Report Portal Launch UUID: ' in mock_stdout.getvalue() + assert 'ReportPortal Launch UUID: ' in mock_stdout.getvalue() @mock.patch('reportportal_client.client.sys.stdout', new_callable=StringIO) @@ -238,4 +238,4 @@ def test_launch_uuid_print_default_print(mock_stdout): client._skip_analytics = True client.start_launch('Test Launch', timestamp()) - assert 'Report Portal Launch UUID: ' not in mock_stdout.getvalue() + assert 'ReportPortal Launch UUID: ' not in mock_stdout.getvalue() From aaf83613919f8db88974447aea00d03b32e104fc Mon Sep 17 00:00:00 2001 From: "reportportal.io" Date: Mon, 25 Sep 2023 19:47:25 +0000 Subject: [PATCH 144/268] Changelog update --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8ca9d2d..5ad54b9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] + +## [5.4.1] ### Changed - Unified ReportPortal product naming, by @HardNorth - `RPClient` internal item stack implementation changed to `LifoQueue` to maintain concurrency better, by @HardNorth From 7f25d4b3951459781eb9abff0012c180db552ca0 Mon Sep 17 00:00:00 2001 From: "reportportal.io" Date: Mon, 25 Sep 2023 19:47:26 +0000 Subject: [PATCH 145/268] Version update --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 890d811e..0386b5da 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup, find_packages -__version__ = '5.4.1' +__version__ = '5.4.2' TYPE_STUBS = ['*.pyi'] From 79c9b9ed91dc9b6ec648a38e473bec18714eb3ad Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 25 Sep 2023 23:25:25 +0300 Subject: [PATCH 146/268] flake8 fixes --- reportportal_client/core/rp_issues.py | 1 + reportportal_client/static/errors.py | 1 + tests/steps/conftest.py | 1 - 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/reportportal_client/core/rp_issues.py b/reportportal_client/core/rp_issues.py index f6e4f7ac..d6350fe0 100644 --- a/reportportal_client/core/rp_issues.py +++ b/reportportal_client/core/rp_issues.py @@ -39,6 +39,7 @@ } """ + class Issue: """This class represents an issue that can be attached to test result.""" diff --git a/reportportal_client/static/errors.py b/reportportal_client/static/errors.py index 6e36d298..3c95870d 100644 --- a/reportportal_client/static/errors.py +++ b/reportportal_client/static/errors.py @@ -13,6 +13,7 @@ """This module includes exceptions used by the client.""" + class RPExceptionBase(Exception): """General exception for the package.""" diff --git a/tests/steps/conftest.py b/tests/steps/conftest.py index eade2687..d3852d70 100644 --- a/tests/steps/conftest.py +++ b/tests/steps/conftest.py @@ -16,7 +16,6 @@ from pytest import fixture from reportportal_client.client import RPClient -from reportportal_client.steps import StepReporter @fixture From d646a9891be5f163576fad88661a07a7ded64580 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 26 Sep 2023 11:37:43 +0300 Subject: [PATCH 147/268] Add pydocs --- reportportal_client/logs/__init__.py | 6 ++--- reportportal_client/logs/batcher.py | 33 ++++++++++++++++++---------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/reportportal_client/logs/__init__.py b/reportportal_client/logs/__init__.py index 0297668c..aed7e797 100644 --- a/reportportal_client/logs/__init__.py +++ b/reportportal_client/logs/__init__.py @@ -20,10 +20,10 @@ # noinspection PyProtectedMember from reportportal_client._local import current, set_current -from reportportal_client.helpers import timestamp +from reportportal_client.helpers import timestamp, TYPICAL_MULTIPART_FOOTER_LENGTH -MAX_LOG_BATCH_SIZE = 20 -MAX_LOG_BATCH_PAYLOAD_SIZE = 65000000 +MAX_LOG_BATCH_SIZE: int = 20 +MAX_LOG_BATCH_PAYLOAD_SIZE: int = int((64 * 1024 * 1024) * 0.98) - TYPICAL_MULTIPART_FOOTER_LENGTH class RPLogger(logging.getLoggerClass()): diff --git a/reportportal_client/logs/batcher.py b/reportportal_client/logs/batcher.py index 0222fa55..1dd18268 100644 --- a/reportportal_client/logs/batcher.py +++ b/reportportal_client/logs/batcher.py @@ -11,11 +11,12 @@ # See the License for the specific language governing permissions and # limitations under the License +"""This module contains a helper class to automate packaging separate Log entries into log batches.""" + import logging import threading from typing import List, Optional, TypeVar, Generic -from reportportal_client import helpers from reportportal_client.core.rp_requests import RPRequestLog, AsyncRPRequestLog from reportportal_client.logs import MAX_LOG_BATCH_SIZE, MAX_LOG_BATCH_PAYLOAD_SIZE @@ -25,9 +26,10 @@ class LogBatcher(Generic[T_co]): - """ - + """Log packaging class to automate compiling separate Log entries into log batches. + The class accepts the maximum number of log entries in desired batches and maximum batch size to conform + with maximum request size limits, configured on servers. The class implementation is thread-safe. """ entry_num: int payload_limit: int @@ -35,12 +37,17 @@ class LogBatcher(Generic[T_co]): _batch: List[T_co] _payload_size: int - def __init__(self, entry_num=MAX_LOG_BATCH_SIZE, payload_limit=MAX_LOG_BATCH_PAYLOAD_SIZE): + def __init__(self, entry_num=MAX_LOG_BATCH_SIZE, payload_limit=MAX_LOG_BATCH_PAYLOAD_SIZE) -> None: + """Initialize the batcher instance with empty batch and specific limits. + + :param entry_num: maximum numbers of entries in a Log batch + :param payload_limit: maximum batch size in bytes + """ self.entry_num = entry_num self.payload_limit = payload_limit self._lock = threading.Lock() self._batch = [] - self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + self._payload_size = 0 def _append(self, size: int, log_req: T_co) -> Optional[List[T_co]]: with self._lock: @@ -48,33 +55,37 @@ def _append(self, size: int, log_req: T_co) -> Optional[List[T_co]]: if len(self._batch) > 0: batch = self._batch self._batch = [log_req] - self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + self._payload_size = 0 return batch self._batch.append(log_req) self._payload_size += size if len(self._batch) >= self.entry_num: batch = self._batch self._batch = [] - self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + self._payload_size = 0 return batch def append(self, log_req: RPRequestLog) -> Optional[List[RPRequestLog]]: """Add a log request object to internal batch and return the batch if it's full. - :param log_req: log request object - :return ready to send batch or None + :param log_req: log request object + :return: a batch or None """ return self._append(log_req.multipart_size, log_req) async def append_async(self, log_req: AsyncRPRequestLog) -> Optional[List[AsyncRPRequestLog]]: """Add a log request object to internal batch and return the batch if it's full. - :param log_req: log request object - :return ready to send batch or None + :param log_req: log request object + :return: a batch or None """ return self._append(await log_req.multipart_size, log_req) def flush(self) -> Optional[List[T_co]]: + """Immediately return everything what's left in the internal batch. + + :return: a batch or None + """ with self._lock: if len(self._batch) > 0: batch = self._batch From d95e56aa78ca5b8f95a4f8d59e8ebf757877a847 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 26 Sep 2023 11:46:16 +0300 Subject: [PATCH 148/268] Add pydocs --- reportportal_client/logs/batcher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reportportal_client/logs/batcher.py b/reportportal_client/logs/batcher.py index 1dd18268..915d190a 100644 --- a/reportportal_client/logs/batcher.py +++ b/reportportal_client/logs/batcher.py @@ -31,6 +31,7 @@ class LogBatcher(Generic[T_co]): The class accepts the maximum number of log entries in desired batches and maximum batch size to conform with maximum request size limits, configured on servers. The class implementation is thread-safe. """ + entry_num: int payload_limit: int _lock: threading.Lock From 2e50a5359844a71c4fa93195d8a13186197582cf Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 26 Sep 2023 11:48:50 +0300 Subject: [PATCH 149/268] Remove redundant package data --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index bca59839..f418ec46 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,6 @@ def read_file(fname): name='reportportal-client', packages=find_packages(exclude=('tests', 'tests.*')), package_data={ - 'reportportal_client': TYPE_STUBS, 'reportportal_client.steps': TYPE_STUBS, 'reportportal_client.core': TYPE_STUBS, 'reportportal_client.logs': TYPE_STUBS, From 98d89d484b1dcaf1d3352b3e4261307ddccdf71e Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 26 Sep 2023 12:01:02 +0300 Subject: [PATCH 150/268] Unify 'returns' keyword --- reportportal_client/aio/client.py | 8 ++--- reportportal_client/client.py | 56 +++++++++++++++---------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index fadc8930..c181ee98 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -432,7 +432,7 @@ async def log_batch(self, log_batch: Optional[List[AsyncRPRequestLog]]) -> Tuple def clone(self) -> 'Client': """Clone the client object, set current Item ID as cloned item ID. - :returns: Cloned client object + :return: Cloned client object :rtype: AsyncRPClient """ cloned = Client( @@ -625,7 +625,7 @@ async def log(self, time: str, message: str, level: Optional[Union[int, str]] = def clone(self) -> 'AsyncRPClient': """Clone the client object, set current Item ID as cloned item ID. - :returns: Cloned client object + :return: Cloned client object :rtype: AsyncRPClient """ cloned_client = self.__client.clone() @@ -943,7 +943,7 @@ def finish_tasks(self): def clone(self) -> 'ThreadedRPClient': """Clone the client object, set current Item ID as cloned item ID. - :returns: Cloned client object + :return: Cloned client object :rtype: ThreadedRPClient """ cloned_client = self.client.clone() @@ -1023,7 +1023,7 @@ def finish_tasks(self) -> None: def clone(self) -> 'BatchedRPClient': """Clone the client object, set current Item ID as cloned item ID. - :returns: Cloned client object + :return: Cloned client object :rtype: BatchedRPClient """ cloned_client = self.client.clone() diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 1078576c..d5fc1c69 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -57,7 +57,7 @@ class RP(metaclass=AbstractBaseClass): def launch_uuid(self) -> Optional[Union[str, Task[str]]]: """Return current launch UUID. - :returns UUID string + :return: UUID string """ raise NotImplementedError('"launch_uuid" property is not implemented!') @@ -65,7 +65,7 @@ def launch_uuid(self) -> Optional[Union[str, Task[str]]]: def launch_id(self) -> Optional[Union[str, Task[str]]]: """Return current launch UUID. - :returns UUID string + :return: UUID string """ warnings.warn( message='`launch_id` property is deprecated since 5.5.0 and will be subject for removing in the' @@ -80,7 +80,7 @@ def launch_id(self) -> Optional[Union[str, Task[str]]]: def endpoint(self) -> str: """Return current base URL. - :returns base URL string + :return: base URL string """ raise NotImplementedError('"endpoint" property is not implemented!') @@ -89,7 +89,7 @@ def endpoint(self) -> str: def project(self) -> str: """Return current Project name. - :returns Project name string + :return: Project name string """ raise NotImplementedError('"project" property is not implemented!') @@ -98,7 +98,7 @@ def project(self) -> str: def step_reporter(self) -> StepReporter: """Return StepReporter object for the current launch. - :returns StepReporter to report steps + :return: StepReporter to report steps """ raise NotImplementedError('"step_reporter" property is not implemented!') @@ -120,7 +120,7 @@ def start_launch(self, :param rerun: Start launch in rerun mode :param rerun_of: For rerun mode specifies which launch will be re-run. Should be used with the 'rerun' option. - :returns Launch UUID + :return: Launch UUID """ raise NotImplementedError('"start_launch" method is not implemented!') @@ -158,7 +158,7 @@ def start_test_item(self, :param retry: Used to report retry of the test. Allowable values: "True" or "False" :param test_case_id: A unique ID of the current step - :returns Test Item UUID + :return: Test Item UUID """ raise NotImplementedError('"start_test_item" method is not implemented!') @@ -186,7 +186,7 @@ def finish_test_item(self, from start request. :param retry: Used to report retry of the test. Allowable values: "True" or "False" - :returns Response message + :return: Response message """ raise NotImplementedError('"finish_test_item" method is not implemented!') @@ -202,7 +202,7 @@ def finish_launch(self, :param status: Launch status. Can be one of the followings: PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED :param attributes: Launch attributes - :returns Response message + :return: Response message """ raise NotImplementedError('"finish_launch" method is not implemented!') @@ -216,7 +216,7 @@ def update_test_item(self, :param item_uuid: Test item UUID returned on the item start :param attributes: Test item attributes: [{'key': 'k_name', 'value': 'k_value'}, ...] :param description: Test item description - :returns Response message + :return: Response message """ raise NotImplementedError('"update_test_item" method is not implemented!') @@ -224,7 +224,7 @@ def update_test_item(self, def get_launch_info(self) -> Union[Optional[dict], Task[str]]: """Get the current launch information. - :returns Launch information in dictionary + :return: Launch information in dictionary """ raise NotImplementedError('"get_launch_info" method is not implemented!') @@ -233,7 +233,7 @@ def get_item_id_by_uuid(self, item_uuid: Union[str, Task[str]]) -> Optional[str] """Get test item ID by the given UUID. :param item_uuid: UUID returned on the item start - :returns Test item ID + :return: Test item ID """ raise NotImplementedError('"get_item_id_by_uuid" method is not implemented!') @@ -241,7 +241,7 @@ def get_item_id_by_uuid(self, item_uuid: Union[str, Task[str]]) -> Optional[str] def get_launch_ui_id(self) -> Optional[int]: """Get UI ID of the current launch. - :returns UI ID of the given launch. None if UI ID has not been found. + :return: UI ID of the given launch. None if UI ID has not been found. """ raise NotImplementedError('"get_launch_ui_id" method is not implemented!') @@ -249,7 +249,7 @@ def get_launch_ui_id(self) -> Optional[int]: def get_launch_ui_url(self) -> Optional[str]: """Get full quality URL of the current launch. - :returns launch URL string + :return: launch URL string """ raise NotImplementedError('"get_launch_ui_id" method is not implemented!') @@ -257,7 +257,7 @@ def get_launch_ui_url(self) -> Optional[str]: def get_project_settings(self) -> Union[Optional[dict], Task[dict]]: """Get project settings. - :returns HTTP response in dictionary + :return: HTTP response in dictionary """ raise NotImplementedError('"get_project_settings" method is not implemented!') @@ -301,7 +301,7 @@ def current_item(self) -> Optional[Union[str, Task[str]]]: def clone(self) -> 'RP': """Clone the client object, set current Item ID as cloned root Item ID. - :returns: Cloned client object + :return: Cloned client object """ raise NotImplementedError('"clone" method is not implemented!') @@ -344,7 +344,7 @@ class RPClient(RP): def launch_uuid(self) -> Optional[str]: """Return current launch UUID. - :returns UUID string + :return: UUID string """ return self.__launch_uuid @@ -352,7 +352,7 @@ def launch_uuid(self) -> Optional[str]: def endpoint(self) -> str: """Return current base URL. - :returns base URL string + :return: base URL string """ return self.__endpoint @@ -360,7 +360,7 @@ def endpoint(self) -> str: def project(self) -> str: """Return current Project name. - :returns Project name string + :return: Project name string """ return self.__project @@ -368,7 +368,7 @@ def project(self) -> str: def step_reporter(self) -> StepReporter: """Return StepReporter object for the current launch. - :returns StepReporter to report steps + :return: StepReporter to report steps """ return self.__step_reporter @@ -734,7 +734,7 @@ def get_item_id_by_uuid(self, item_uuid: str) -> Optional[str]: """Get test item ID by the given UUID. :param item_uuid: UUID returned on the item start - :returns Test item ID + :return: Test item ID """ url = uri_join(self.base_url_v1, 'item', 'uuid', item_uuid) response = HttpRequest(self.session.get, url=url, @@ -744,7 +744,7 @@ def get_item_id_by_uuid(self, item_uuid: str) -> Optional[str]: def get_launch_info(self) -> Optional[dict]: """Get the current launch information. - :returns Launch information in dictionary + :return: Launch information in dictionary """ if self.launch_uuid is None: return {} @@ -775,7 +775,7 @@ def get_launch_ui_id(self) -> Optional[int]: def get_launch_ui_url(self) -> Optional[str]: """Get UI URL of the current launch. - :returns launch URL or all launches URL. + :return: launch URL or all launches URL. """ launch_info = self.get_launch_info() ui_id = launch_info.get('id') if launch_info else None @@ -797,7 +797,7 @@ def get_launch_ui_url(self) -> Optional[str]: def get_project_settings(self) -> Optional[dict]: """Get project settings. - :returns HTTP response in dictionary + :return: HTTP response in dictionary """ url = uri_join(self.base_url_v1, 'settings') response = HttpRequest(self.session.get, url=url, @@ -811,7 +811,7 @@ def _add_current_item(self, item: str) -> None: def _remove_current_item(self) -> Optional[str]: """Remove the last item from the self._items queue. - :returns Item UUID string + :return: Item UUID string """ try: return self._item_stack.get(timeout=0) @@ -821,14 +821,14 @@ def _remove_current_item(self) -> Optional[str]: def current_item(self) -> Optional[str]: """Retrieve the last item reported by the client. - :returns Item UUID string + :return: Item UUID string """ return self._item_stack.last() def clone(self) -> 'RPClient': """Clone the client object, set current Item ID as cloned item ID. - :returns: Cloned client object + :return: Cloned client object :rtype: RPClient """ cloned = RPClient( @@ -854,7 +854,7 @@ def clone(self) -> 'RPClient': def __getstate__(self) -> Dict[str, Any]: """Control object pickling and return object fields as Dictionary. - :returns: object state dictionary + :return: object state dictionary :rtype: dict """ state = self.__dict__.copy() From 0fdf4780028debf5bda8648d4fb3704bd3f63375 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 26 Sep 2023 12:56:50 +0300 Subject: [PATCH 151/268] Update pydocs --- reportportal_client/core/rp_requests.py | 202 +++++++++++++++++------- 1 file changed, 144 insertions(+), 58 deletions(-) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 3467ee97..24b2c26d 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License -"""This module includes classes representing RP API requests. +"""This module includes classes representing ReportPortal API requests. Detailed information about requests wrapped up in that module can be found by the following link: @@ -22,7 +22,7 @@ import json as json_converter import logging from dataclasses import dataclass -from typing import Callable, Text, Optional, Union, List, Tuple, Any, TypeVar +from typing import Callable, Optional, Union, List, Tuple, Any, TypeVar import aiohttp @@ -46,7 +46,7 @@ class HttpRequest: - """This model stores attributes related to RP HTTP requests.""" + """This model stores attributes related to ReportPortal HTTP requests.""" session_method: Callable url: Any @@ -66,19 +66,18 @@ def __init__(self, files: Optional[Any] = None, verify_ssl: Optional[Union[bool, str]] = None, http_timeout: Union[float, Tuple[float, float]] = (10, 10), - name: Optional[Text] = None) -> None: + name: Optional[str] = None) -> None: """Initialize instance attributes. :param session_method: Method of the requests.Session instance :param url: Request URL - :param data: Dictionary, list of tuples, bytes, or file-like - object to send in the body of the request + :param data: Dictionary, list of tuples, bytes, or file-like object to send in the body of + the request :param json: JSON to be sent in the body of the request - :param files Dictionary for multipart encoding upload. + :param files Dictionary for multipart encoding upload :param verify_ssl: Is SSL certificate verification required - :param http_timeout: a float in seconds for connect and read - timeout. Use a Tuple to specific connect and - read separately. + :param http_timeout: a float in seconds for connect and read timeout. Use a Tuple to specific + connect and read separately :param name: request name """ self.data = data @@ -91,22 +90,38 @@ def __init__(self, self.name = name self._priority = DEFAULT_PRIORITY - def __lt__(self, other) -> bool: - """Priority protocol for the PriorityQueue.""" + def __lt__(self, other: 'HttpRequest') -> bool: + """Priority protocol for the PriorityQueue. + + :param other: another object to compare + :return: if current object is less than other + """ return self.priority < other.priority @property def priority(self) -> Priority: - """Get the priority of the request.""" + """Get the priority of the request. + + :return: the object priority + """ return self._priority @priority.setter def priority(self, value: Priority) -> None: - """Set the priority of the request.""" + """Set the priority of the request. + + :param value: the object priority + """ self._priority = value def make(self) -> Optional[RPResponse]: - """Make HTTP request to the ReportPortal API.""" + """Make HTTP request to the ReportPortal API. + + The method catches any request preparation error to not fail reporting. Since we are reporting tool + and should not fail tests. + + :return: wrapped HTTP response or None in case of failure + """ try: return RPResponse(self.session_method(self.url, data=self.data, json=self.json, files=self.files, verify=self.verify_ssl, timeout=self.http_timeout)) @@ -115,27 +130,33 @@ def make(self) -> Optional[RPResponse]: class AsyncHttpRequest(HttpRequest): - """This model stores attributes related to RP HTTP requests.""" + """This model stores attributes related to asynchronous ReportPortal HTTP requests.""" def __init__(self, session_method: Callable, url: Any, data: Optional[Any] = None, json: Optional[Any] = None, - name: Optional[Text] = None) -> None: + name: Optional[str] = None) -> None: """Initialize instance attributes. :param session_method: Method of the requests.Session instance - :param url: Request URL - :param data: Dictionary, list of tuples, bytes, or file-like object to send in the body of - the request - :param json: JSON to be sent in the body of the request + :param url: Request URL or async coroutine returning the URL + :param data: Dictionary, list of tuples, bytes, or file-like object, or async coroutine to + send in the body of the request + :param json: JSON to be sent in the body of the request as Dictionary or async coroutine :param name: request name """ super().__init__(session_method=session_method, url=url, data=data, json=json, name=name) async def make(self) -> Optional[AsyncRPResponse]: - """Make HTTP request to the ReportPortal API.""" + """Asynchronously make HTTP request to the ReportPortal API. + + The method catches any request preparation error to not fail reporting. Since we are reporting tool + and should not fail tests. + + :return: wrapped HTTP response or None in case of failure + """ url = await await_if_necessary(self.url) if not url: return @@ -148,22 +169,30 @@ async def make(self) -> Optional[AsyncRPResponse]: class RPRequestBase(metaclass=AbstractBaseClass): - """Base class for the rest of the RP request models.""" + """Base class for specific ReportPortal request models. + + Its main function to provide interface of 'payload' method which is used to generate HTTP request payload. + """ __metaclass__ = AbstractBaseClass + @property @abstractmethod def payload(self) -> dict: - """Abstract interface for getting HTTP request payload.""" + """Abstract interface for getting HTTP request payload. + + :return: JSON representation in the form of a Dictionary + """ raise NotImplementedError('Payload interface is not implemented!') @dataclass(frozen=True) class LaunchStartRequest(RPRequestBase): - """RP start launch request model. + """ReportPortal start launch request model. https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#start-launch """ + name: str start_time: str attributes: Optional[Union[list, dict]] = None @@ -175,7 +204,10 @@ class LaunchStartRequest(RPRequestBase): @property def payload(self) -> dict: - """Get HTTP payload for the request.""" + """Get HTTP payload for the request. + + :return: JSON representation in the form of a Dictionary + """ my_attributes = self.attributes if my_attributes and isinstance(self.attributes, dict): my_attributes = dict_to_payload(self.attributes) @@ -195,19 +227,22 @@ def payload(self) -> dict: @dataclass(frozen=True) class LaunchFinishRequest(RPRequestBase): - """RP finish launch request model. + """ReportPortal finish launch request model. https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#finish-launch """ end_time: str - status: Optional[Text] = None + status: Optional[str] = None attributes: Optional[Union[list, dict]] = None description: Optional[str] = None @property def payload(self) -> dict: - """Get HTTP payload for the request.""" + """Get HTTP payload for the request. + + :return: JSON representation in the form of a Dictionary + """ my_attributes = self.attributes if my_attributes and isinstance(self.attributes, dict): my_attributes = dict_to_payload(self.attributes) @@ -221,24 +256,25 @@ def payload(self) -> dict: @dataclass(frozen=True) class ItemStartRequest(RPRequestBase): - """RP start test item request model. + """ReportPortal start test item request model. https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#start-rootsuite-item """ + name: str start_time: str type_: str launch_uuid: Any attributes: Optional[Union[list, dict]] - code_ref: Optional[Text] - description: Optional[Text] + code_ref: Optional[str] + description: Optional[str] has_stats: bool parameters: Optional[Union[list, dict]] retry: bool - test_case_id: Optional[Text] + test_case_id: Optional[str] @staticmethod - def create_request(**kwargs) -> dict: + def _create_request(**kwargs) -> dict: request = { 'codeRef': kwargs.get('code_ref'), 'description': kwargs.get('description'), @@ -262,32 +298,44 @@ def create_request(**kwargs) -> dict: @property def payload(self) -> dict: - """Get HTTP payload for the request.""" + """Get HTTP payload for the request. + + :return: JSON representation in the form of a Dictionary + """ data = self.__dict__.copy() data['type'] = data.pop('type_') - return ItemStartRequest.create_request(**data) + return ItemStartRequest._create_request(**data) class AsyncItemStartRequest(ItemStartRequest): + """ReportPortal start test item request asynchronous model. + + https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#start-rootsuite-item + """ def __int__(self, *args, **kwargs) -> None: + """Initialize an instance of the request with attributes.""" super.__init__(*args, **kwargs) @property async def payload(self) -> dict: - """Get HTTP payload for the request.""" + """Get HTTP payload for the request. + + :return: JSON representation in the form of a Dictionary + """ data = self.__dict__.copy() data['type'] = data.pop('type_') data['launch_uuid'] = await await_if_necessary(data.pop('launch_uuid')) - return ItemStartRequest.create_request(**data) + return ItemStartRequest._create_request(**data) @dataclass(frozen=True) class ItemFinishRequest(RPRequestBase): - """RP finish test item request model. + """ReportPortal finish test item request model. https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#finish-child-item """ + end_time: str launch_uuid: Any status: str @@ -298,7 +346,7 @@ class ItemFinishRequest(RPRequestBase): retry: bool @staticmethod - def create_request(**kwargs) -> dict: + def _create_request(**kwargs) -> dict: request = { 'description': kwargs.get('description'), 'endTime': kwargs['end_time'], @@ -324,29 +372,41 @@ def create_request(**kwargs) -> dict: @property def payload(self) -> dict: - """Get HTTP payload for the request.""" - return ItemFinishRequest.create_request(**self.__dict__) + """Get HTTP payload for the request. + + :return: JSON representation in the form of a Dictionary + """ + return ItemFinishRequest._create_request(**self.__dict__) class AsyncItemFinishRequest(ItemFinishRequest): + """ReportPortal finish test item request asynchronous model. + + https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#finish-child-item + """ def __int__(self, *args, **kwargs) -> None: + """Initialize an instance of the request with attributes.""" super.__init__(*args, **kwargs) @property async def payload(self) -> dict: - """Get HTTP payload for the request.""" + """Get HTTP payload for the request. + + :return: JSON representation in the form of a Dictionary + """ data = self.__dict__.copy() data['launch_uuid'] = await await_if_necessary(data.pop('launch_uuid')) - return ItemFinishRequest.create_request(**data) + return ItemFinishRequest._create_request(**data) @dataclass(frozen=True) class RPRequestLog(RPRequestBase): - """RP log save request model. + """ReportPortal log save request model. https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#save-single-log-without-attachment """ + launch_uuid: Any time: str file: Optional[RPFile] = None @@ -355,7 +415,7 @@ class RPRequestLog(RPRequestBase): message: Optional[str] = None @staticmethod - def create_request(**kwargs) -> dict: + def _create_request(**kwargs) -> dict: request = { 'launchUuid': kwargs['launch_uuid'], 'level': kwargs['level'], @@ -370,8 +430,11 @@ def create_request(**kwargs) -> dict: @property def payload(self) -> dict: - """Get HTTP payload for the request.""" - return RPRequestLog.create_request(**self.__dict__) + """Get HTTP payload for the request. + + :return: JSON representation in the form of a Dictionary + """ + return RPRequestLog._create_request(**self.__dict__) @staticmethod def _multipart_size(payload: dict, file: Optional[RPFile]): @@ -381,36 +444,51 @@ def _multipart_size(payload: dict, file: Optional[RPFile]): @property def multipart_size(self) -> int: - """Calculate request size how it would transfer in Multipart HTTP.""" + """Calculate request size how it would be transfer in Multipart HTTP. + + :return: estimate request size + """ return RPRequestLog._multipart_size(self.payload, self.file) class AsyncRPRequestLog(RPRequestLog): + """ReportPortal log save request asynchronous model. + + https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#save-single-log-without-attachment + """ def __int__(self, *args, **kwargs) -> None: + """Initialize an instance of the request with attributes.""" super.__init__(*args, **kwargs) @property async def payload(self) -> dict: - """Get HTTP payload for the request.""" + """Get HTTP payload for the request. + + :return: JSON representation in the form of a Dictionary + """ data = self.__dict__.copy() uuids = await asyncio.gather(await_if_necessary(data.pop('launch_uuid')), await_if_necessary(data.pop('item_uuid'))) data['launch_uuid'] = uuids[0] data['item_uuid'] = uuids[1] - return RPRequestLog.create_request(**data) + return RPRequestLog._create_request(**data) @property async def multipart_size(self) -> int: - """Calculate request size how it would transfer in Multipart HTTP.""" + """Calculate request size how it would be transfer in Multipart HTTP. + + :return: estimate request size + """ return RPRequestLog._multipart_size(await self.payload, self.file) class RPLogBatch(RPRequestBase): - """RP log save batches with attachments request model. + """ReportPortal log save batches with attachments request model. https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#batch-save-logs """ + default_content: str log_reqs: List[Union[RPRequestLog, AsyncRPRequestLog]] priority: Priority @@ -474,8 +552,13 @@ def payload(self) -> List[Tuple[str, tuple]]: class AsyncRPLogBatch(RPLogBatch): + """ReportPortal log save batches with attachments request asynchronous model. + + https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md#batch-save-logs + """ def __int__(self, *args, **kwargs) -> None: + """Initialize an instance of the request with attributes.""" super.__init__(*args, **kwargs) async def __get_request_part(self) -> List[dict]: @@ -484,12 +567,15 @@ async def __get_request_part(self) -> List[dict]: @property async def payload(self) -> aiohttp.MultipartWriter: - """Get HTTP payload for the request.""" + """Get HTTP payload for the request. + + :return: Multipart request object capable to send with AIOHTTP + """ json_payload = aiohttp.JsonPayload(await self.__get_request_part()) json_payload.set_content_disposition('form-data', name='json_request_part') - mpwriter = aiohttp.MultipartWriter('form-data') - mpwriter.append_payload(json_payload) + mp_writer = aiohttp.MultipartWriter('form-data') + mp_writer.append_payload(json_payload) for _, file in self._get_files(): file_payload = aiohttp.Payload(file[1], content_type=file[2], filename=file[0]) - mpwriter.append_payload(file_payload) - return mpwriter + mp_writer.append_payload(file_payload) + return mp_writer From 47dc7d580af3512f7ad86a1326403039eb6ab9fb Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 26 Sep 2023 15:03:32 +0300 Subject: [PATCH 152/268] Update pydocs --- reportportal_client/helpers.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index cb4ec1d7..af77e105 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -259,6 +259,11 @@ def calculate_file_part_size(file: Optional[RPFile]) -> int: def agent_name_version(attributes: Optional[Union[List, Dict]] = None) -> Tuple[Optional[str], Optional[str]]: + """Extract Agent name and version from given Launch attributes. + + :param attributes: Launch attributes as they provided to Start Launch call + :return: Tuple of (agent name, version) + """ agent_name, agent_version = None, None agent_attribute = [a for a in attributes if a.get('key') == 'agent'] if attributes else [] if len(agent_attribute) > 0 and agent_attribute[0].get('value'): @@ -266,7 +271,12 @@ def agent_name_version(attributes: Optional[Union[List, Dict]] = None) -> Tuple[ return agent_name, agent_version -async def await_if_necessary(obj: Optional[Any]) -> Any: +async def await_if_necessary(obj: Optional[Any]) -> Optional[Any]: + """Await Coroutine, Feature or coroutine Function if given argument is one of them, or return immediately. + + :param obj: value, Coroutine, Feature or coroutine Function + :return: result which was returned by Coroutine, Feature or coroutine Function + """ if obj: if asyncio.isfuture(obj) or asyncio.iscoroutine(obj): return await obj From b21c76f3996e737f3506a3d07d97872c2985ba78 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 26 Sep 2023 15:17:51 +0300 Subject: [PATCH 153/268] Add exception handling in case of connection errors, update pydocs. --- reportportal_client/services/statistics.py | 28 +++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/reportportal_client/services/statistics.py b/reportportal_client/services/statistics.py index 625e9937..aaf7d646 100644 --- a/reportportal_client/services/statistics.py +++ b/reportportal_client/services/statistics.py @@ -13,13 +13,13 @@ """This module sends statistics events to a statistics service.""" -import ssl -import certifi import logging +import ssl from platform import python_version from typing import Optional, Tuple import aiohttp +import certifi import requests from pkg_resources import get_distribution @@ -48,7 +48,14 @@ def _get_platform_info() -> str: return 'Python ' + python_version() -def get_payload(event_name: str, agent_name: Optional[str], agent_version: Optional[str]) -> dict: +def _get_payload(event_name: str, agent_name: Optional[str], agent_version: Optional[str]) -> dict: + """Format Statistics service request as it should be sent. + + :param event_name: name of the event as it will be displayed + :param agent_name: current Agent name + :param agent_version: current Agent version + :return: JSON representation of the request as Dictionary + """ client_name, client_version = _get_client_info() request_params = { 'client_name': client_name, @@ -87,14 +94,14 @@ def send_event(event_name: str, agent_name: Optional[str], agent_version: Option 'api_secret': KEY } try: - return requests.post(url=ENDPOINT, json=get_payload(event_name, agent_name, agent_version), + return requests.post(url=ENDPOINT, json=_get_payload(event_name, agent_name, agent_version), headers=headers, params=query_params) except requests.exceptions.RequestException as err: logger.debug('Failed to send data to Statistics service: %s', str(err)) async def async_send_event(event_name: str, agent_name: Optional[str], - agent_version: Optional[str]) -> aiohttp.ClientResponse: + agent_version: Optional[str]) -> Optional[aiohttp.ClientResponse]: """Send an event to statistics service. Use client and agent versions with their names. @@ -110,8 +117,13 @@ async def async_send_event(event_name: str, agent_name: Optional[str], } ssl_context = ssl.create_default_context(cafile=certifi.where()) async with aiohttp.ClientSession() as session: - result = await session.post(url=ENDPOINT, json=get_payload(event_name, agent_name, agent_version), - headers=headers, params=query_params, ssl=ssl_context) + try: + result = await session.post(url=ENDPOINT, + json=_get_payload(event_name, agent_name, agent_version), + headers=headers, params=query_params, ssl=ssl_context) + except aiohttp.ClientError as exc: + logger.debug('Failed to send data to Statistics service: connection error', exc) + return if not result.ok: - logger.debug('Failed to send data to Statistics service: %s', result.reason) + logger.debug(f'Failed to send data to Statistics service: {result.reason}') return result From ec5f8dccb18201e900dbb06e105bf3627b4f9d28 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 26 Sep 2023 16:55:11 +0300 Subject: [PATCH 154/268] Update pydocs. --- reportportal_client/core/rp_responses.py | 68 +++++++++++++++++------- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index 58f35d15..f399f416 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License -"""This module contains models for the RP response objects. +"""This module contains models for the ReportPortal response objects. Detailed information about responses wrapped up in that module can be found by the following link: @@ -35,18 +35,19 @@ def _iter_json_messages(json: Any) -> Generator[str, None, None]: return data = json.get('responses', [json]) for chunk in data: - message = chunk.get('message', chunk.get('error_code')) + message = chunk.get('message', chunk.get('error_code', NOT_FOUND)) if message: yield message class RPResponse: - """Class representing RP API response.""" + """Class representing ReportPortal API response.""" + _resp: Response __json: Any def __init__(self, data: Response) -> None: - """Initialize instance attributes. + """Initialize an instance with attributes. :param data: requests.Response object """ @@ -55,19 +56,28 @@ def __init__(self, data: Response) -> None: @property def id(self) -> Optional[str]: - """Get value of the 'id' key.""" + """Get value of the 'id' key in the response. + + :return: ID as string or NOT_FOUND, or None if the response is not JSON + """ if self.json is None: return return self.json.get('id', NOT_FOUND) @property def is_success(self) -> bool: - """Check if response to API has been successful.""" + """Check if response to API has been successful. + + :return: is response successful + """ return self._resp.ok @property def json(self) -> Any: - """Get the response in dictionary.""" + """Get the response in Dictionary or List. + + :return: JSON represented as Dictionary or List, or None if the response is not JSON + """ if self.__json is NOT_SET: try: self.__json = self._resp.json() @@ -77,35 +87,45 @@ def json(self) -> Any: @property def message(self) -> Optional[str]: - """Get value of the 'message' key.""" + """Get value of the 'message' key in the response. + + :return: message as string or NOT_FOUND, or None if the response is not JSON + """ if self.json is None: return return self.json.get('message') @property def messages(self) -> Optional[Tuple[str, ...]]: - """Get list of messages received.""" + """Get list of messages received in the response. + + :return: a variable size tuple of strings or NOT_FOUND, or None if the response is not JSON + """ if self.json is None: return return tuple(_iter_json_messages(self.json)) class AsyncRPResponse: - """Class representing RP API response.""" + """Class representing ReportPortal API asynchronous response.""" + _resp: ClientResponse __json: Any def __init__(self, data: ClientResponse) -> None: - """Initialize instance attributes. + """Initialize an instance with attributes. - :param data: requests.Response object + :param data: aiohttp.ClientResponse object """ self._resp = data self.__json = NOT_SET @property async def id(self) -> Optional[str]: - """Get value of the 'id' key.""" + """Get value of the 'id' key in the response. + + :return: ID as string or NOT_FOUND, or None if the response is not JSON + """ json = await self.json if json is None: return @@ -113,12 +133,18 @@ async def id(self) -> Optional[str]: @property def is_success(self) -> bool: - """Check if response to API has been successful.""" + """Check if response to API has been successful. + + :return: is response successful + """ return self._resp.ok @property async def json(self) -> Any: - """Get the response in dictionary.""" + """Get the response in Dictionary or List. + + :return: JSON represented as Dictionary or List, or None if the response is not JSON + """ if self.__json is NOT_SET: try: self.__json = await self._resp.json() @@ -128,15 +154,21 @@ async def json(self) -> Any: @property async def message(self) -> Optional[str]: - """Get value of the 'message' key.""" + """Get value of the 'message' key in the response. + + :return: message as string or NOT_FOUND, or None if the response is not JSON + """ json = await self.json if json is None: return - return json.get('message') + return json.get('message', NOT_FOUND) @property async def messages(self) -> Optional[Tuple[str, ...]]: - """Get list of messages received.""" + """Get list of messages received in the response. + + :return: a variable size tuple of strings or NOT_FOUND, or None if the response is not JSON + """ json = await self.json if json is None: return From 5fc8b64e090e5d57c0517884bf2492997ec3fdb4 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 26 Sep 2023 17:53:02 +0300 Subject: [PATCH 155/268] Update pydocs. --- reportportal_client/aio/http.py | 24 ++++++++++++++++++++++++ reportportal_client/core/rp_requests.py | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py index 80ca827c..533620bd 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/aio/http.py @@ -11,6 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License +"""This module designed to help with asynchronous HTTP request/response handling.""" + import asyncio import sys from typing import Coroutine @@ -26,12 +28,16 @@ class RetryClass(int, Enum): + """Enum contains error types and their retry delay multiply factor as values.""" + SERVER_ERROR = 1 CONNECTION_ERROR = 2 THROTTLING = 3 class RetryingClientSession(ClientSession): + """Class extends aiohttp.ClientSession functionality with request retry logic.""" + __retry_number: int __retry_delay: float @@ -42,6 +48,17 @@ def __init__( base_retry_delay: float = DEFAULT_RETRY_DELAY, **kwargs ): + """Initialize an instance of the session with arguments. + + To obtain the full list of arguments please see aiohttp.ClientSession.__init__() method. This class + just bypass everything to the base method, except two local arguments 'max_retry_number' and + 'base_retry_delay'. + + :param max_retry_number: the maximum number of the request retries if it was unsuccessful + :param base_retry_delay: base value for retry delay, determine how much time the class will wait after + an error. Real value highly depends on Retry Class and Retry attempt number, + since retries are performed in exponential delay manner + """ super().__init__(*args, **kwargs) self.__retry_number = max_retry_number self.__retry_delay = base_retry_delay @@ -61,6 +78,13 @@ async def _request( *args, **kwargs ) -> ClientResponse: + """Make a request and retry if necessary. + + The method overrides aiohttp.ClientSession._request() method and bypass all arguments to it, so + please refer it for detailed argument description. The method retries requests depending on error + class and retry number. For no-retry errors, such as 400 Bad Request it just returns result, for cases + where it's reasonable to retry it does it in exponential manner. + """ result = None exceptions = [] for i in range(self.__retry_number + 1): # add one for the first attempt, which is not a retry diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 24b2c26d..2020916b 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -67,7 +67,7 @@ def __init__(self, verify_ssl: Optional[Union[bool, str]] = None, http_timeout: Union[float, Tuple[float, float]] = (10, 10), name: Optional[str] = None) -> None: - """Initialize instance attributes. + """Initialize an instance of the request with attributes. :param session_method: Method of the requests.Session instance :param url: Request URL @@ -138,7 +138,7 @@ def __init__(self, data: Optional[Any] = None, json: Optional[Any] = None, name: Optional[str] = None) -> None: - """Initialize instance attributes. + """Initialize an instance of the request with attributes. :param session_method: Method of the requests.Session instance :param url: Request URL or async coroutine returning the URL From 27d78f8045f40b5d5b5ecd544bbb634c0d4d9f28 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 26 Sep 2023 22:45:52 +0300 Subject: [PATCH 156/268] Update pydocs. And import fixes. --- reportportal_client/aio/__init__.py | 18 +-- reportportal_client/aio/client.py | 25 ++-- reportportal_client/aio/tasks.py | 140 ++++++++++++++++----- reportportal_client/client.py | 31 +++-- reportportal_client/{static => }/errors.py | 0 5 files changed, 147 insertions(+), 67 deletions(-) rename reportportal_client/{static => }/errors.py (100%) diff --git a/reportportal_client/aio/__init__.py b/reportportal_client/aio/__init__.py index fa047c58..52742af9 100644 --- a/reportportal_client/aio/__init__.py +++ b/reportportal_client/aio/__init__.py @@ -11,18 +11,17 @@ # See the License for the specific language governing permissions and # limitations under the License -from reportportal_client.aio.tasks import (Task, TriggerTaskList, BatchedTask, BatchedTaskFactory, +from reportportal_client.aio.tasks import (Task, TriggerTaskBatcher, BatchedTask, BatchedTaskFactory, ThreadedTask, ThreadedTaskFactory, BlockingOperationError, - BackgroundTaskList, DEFAULT_TASK_TRIGGER_NUM, + BackgroundTaskBatcher, DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL) - -DEFAULT_TASK_TIMEOUT: float = 60.0 -DEFAULT_SHUTDOWN_TIMEOUT: float = 120.0 +from reportportal_client.aio.client import (ThreadedRPClient, BatchedRPClient, AsyncRPClient, + DEFAULT_TASK_TIMEOUT, DEFAULT_SHUTDOWN_TIMEOUT) __all__ = [ 'Task', - 'TriggerTaskList', - 'BackgroundTaskList', + 'TriggerTaskBatcher', + 'BackgroundTaskBatcher', 'BatchedTask', 'BatchedTaskFactory', 'ThreadedTask', @@ -31,5 +30,8 @@ 'DEFAULT_TASK_TIMEOUT', 'DEFAULT_SHUTDOWN_TIMEOUT', 'DEFAULT_TASK_TRIGGER_NUM', - 'DEFAULT_TASK_TRIGGER_INTERVAL' + 'DEFAULT_TASK_TRIGGER_INTERVAL', + 'AsyncRPClient', + 'BatchedRPClient', + 'ThreadedRPClient' ] diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index c181ee98..878f6125 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -29,9 +29,9 @@ from reportportal_client import RP # noinspection PyProtectedMember from reportportal_client._local import set_current -from reportportal_client.aio import (Task, BatchedTaskFactory, ThreadedTaskFactory, DEFAULT_TASK_TIMEOUT, - DEFAULT_SHUTDOWN_TIMEOUT, DEFAULT_TASK_TRIGGER_NUM, TriggerTaskList, - DEFAULT_TASK_TRIGGER_INTERVAL, BackgroundTaskList) +from reportportal_client.aio.tasks import (Task, BatchedTaskFactory, ThreadedTaskFactory, TriggerTaskBatcher, + BackgroundTaskBatcher, DEFAULT_TASK_TRIGGER_NUM, + DEFAULT_TASK_TRIGGER_INTERVAL) from reportportal_client.aio.http import RetryingClientSession from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, @@ -54,6 +54,9 @@ _T = TypeVar('_T') +DEFAULT_TASK_TIMEOUT: float = 60.0 +DEFAULT_SHUTDOWN_TIMEOUT: float = 120.0 + class Client: api_v1: str @@ -882,7 +885,7 @@ async def _close(self): class ThreadedRPClient(_RPClient): _loop: Optional[asyncio.AbstractEventLoop] - __task_list: BackgroundTaskList[Task[_T]] + __task_list: BackgroundTaskBatcher[Task[_T]] __task_mutex: threading.RLock __thread: Optional[threading.Thread] @@ -894,21 +897,21 @@ def __init__( launch_uuid: Optional[Task[str]] = None, client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, - task_list: Optional[BackgroundTaskList[Task[_T]]] = None, + task_list: Optional[BackgroundTaskBatcher[Task[_T]]] = None, task_mutex: Optional[threading.RLock] = None, loop: Optional[asyncio.AbstractEventLoop] = None, **kwargs: Any ) -> None: super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, **kwargs) - self.__task_list = task_list or BackgroundTaskList() + self.__task_list = task_list or BackgroundTaskBatcher() self.__task_mutex = task_mutex or threading.RLock() self.__thread = None if loop: self._loop = loop else: self._loop = asyncio.new_event_loop() - self._loop.set_task_factory(ThreadedTaskFactory(self._loop, self._task_timeout)) + self._loop.set_task_factory(ThreadedTaskFactory(self._task_timeout)) self.__heartbeat() self.__thread = threading.Thread(target=self._loop.run_forever, name='RP-Async-Client', daemon=True) @@ -966,7 +969,7 @@ def clone(self) -> 'ThreadedRPClient': class BatchedRPClient(_RPClient): _loop: asyncio.AbstractEventLoop - __task_list: TriggerTaskList[Task[_T]] + __task_list: TriggerTaskBatcher[Task[_T]] __task_mutex: threading.RLock __last_run_time: float __trigger_num: int @@ -979,7 +982,7 @@ def __init__( launch_uuid: Optional[Task[str]] = None, client: Optional[Client] = None, log_batcher: Optional[LogBatcher] = None, - task_list: Optional[TriggerTaskList] = None, + task_list: Optional[TriggerTaskBatcher] = None, task_mutex: Optional[threading.RLock] = None, loop: Optional[asyncio.AbstractEventLoop] = None, trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, @@ -988,14 +991,14 @@ def __init__( ) -> None: super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, **kwargs) - self.__task_list = task_list or TriggerTaskList() + self.__task_list = task_list or TriggerTaskBatcher() self.__task_mutex = task_mutex or threading.RLock() self.__last_run_time = datetime.time() if loop: self._loop = loop else: self._loop = asyncio.new_event_loop() - self._loop.set_task_factory(BatchedTaskFactory(self._loop)) + self._loop.set_task_factory(BatchedTaskFactory()) self.__trigger_num = trigger_num self.__trigger_interval = trigger_interval diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index fd15d77d..aa3b69cd 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -11,6 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License +"""This module contains customized asynchronous Tasks and Task Factories for the ReportPortal client.""" + import asyncio import sys import time @@ -30,6 +32,12 @@ class BlockingOperationError(RuntimeError): class Task(Generic[_T], asyncio.Task, metaclass=AbstractBaseClass): + """Base class for ReportPortal client tasks. + + Its main function to provide interface of 'blocking_result' method which is used to block current Thread + until the result computation. + """ + __metaclass__ = AbstractBaseClass def __init__( @@ -39,14 +47,44 @@ def __init__( loop: asyncio.AbstractEventLoop, name: Optional[str] = None ) -> None: + """Initialize an instance of the Task. + + :param coro: Future, Coroutine or a Generator of these objects, which will be executed + :param loop: Event Loop which will be used to execute the Task + :param name: the name of the task + """ super().__init__(coro, loop=loop, name=name) @abstractmethod def blocking_result(self) -> _T: + """Block current Thread and wait for the task result. + + :return: execution result or raise an error + """ raise NotImplementedError('"blocking_result" method is not implemented!') + def __repr__(self) -> str: + """Return the result's repr function output if the Task is completed, or the Task's if not. + + :return: canonical string representation of the result or the current Task + """ + if self.done(): + return repr(self.result()) + return super().__repr__() + + def __str__(self): + """Return the result's str function output if the Task is completed, or the Task's if not. + + :return: string object from the result or from the current Task + """ + if self.done(): + return str(self.result()) + return super().__str__() + class BatchedTask(Generic[_T], Task[_T]): + """Represents a Task which uses the current Thread to execute itself.""" + __loop: asyncio.AbstractEventLoop def __init__( @@ -56,26 +94,28 @@ def __init__( loop: asyncio.AbstractEventLoop, name: Optional[str] = None ) -> None: + """Initialize an instance of the Task. + + :param coro: Future, Coroutine or a Generator of these objects, which will be executed + :param loop: Event Loop which will be used to execute the Task + :param name: the name of the task + """ super().__init__(coro, loop=loop, name=name) self.__loop = loop def blocking_result(self) -> _T: + """Use current Thread to execute the Task and return the result if not yet executed. + + :return: execution result or raise an error, or return immediately if already executed + """ if self.done(): return self.result() return self.__loop.run_until_complete(self) - def __repr__(self) -> str: - if self.done(): - return repr(self.result()) - return super().__repr__() - - def __str__(self): - if self.done(): - return str(self.result()) - return super().__str__() - class ThreadedTask(Generic[_T], Task[_T]): + """Represents a Task which runs is a separate Thread.""" + __loop: asyncio.AbstractEventLoop __wait_timeout: float @@ -87,11 +127,21 @@ def __init__( loop: asyncio.AbstractEventLoop, name: Optional[str] = None ) -> None: + """Initialize an instance of the Task. + + :param coro: Future, Coroutine or a Generator of these objects, which will be executed + :param loop: Event Loop which will be used to execute the Task + :param name: the name of the task + """ super().__init__(coro, loop=loop, name=name) self.__loop = loop self.__wait_timeout = wait_timeout def blocking_result(self) -> _T: + """Pause current Thread until the Task completion and return the result if not yet executed. + + :return: execution result or raise an error, or return immediately if already executed + """ if self.done(): return self.result() if not self.__loop.is_running() or self.__loop.is_closed(): @@ -104,22 +154,9 @@ def blocking_result(self) -> _T: raise BlockingOperationError('Timed out waiting for the task execution') return self.result() - def __repr__(self) -> str: - if self.done(): - return repr(self.result()) - return super().__repr__() - - def __str__(self): - if self.done(): - return str(self.result()) - return super().__str__() - class BatchedTaskFactory: - __loop: asyncio.AbstractEventLoop - - def __init__(self, loop: asyncio.AbstractEventLoop): - self.__loop = loop + """Factory protocol which creates Batched Tasks.""" def __call__( self, @@ -127,15 +164,22 @@ def __call__( factory: Union[Coroutine[Any, Any, _T], Generator[Any, None, _T]], **_ ) -> Task[_T]: - return BatchedTask(factory, loop=self.__loop) + """Create Batched Task in appropriate Event Loop. + + :param loop: Event Loop which will be used to execute the Task + :param factory: Future, Coroutine or a Generator of these objects, which will be executed + """ + return BatchedTask(factory, loop=loop) class ThreadedTaskFactory: - __loop: asyncio.AbstractEventLoop __wait_timeout: float - def __init__(self, loop: asyncio.AbstractEventLoop, wait_timeout: float): - self.__loop = loop + def __init__(self, wait_timeout: float): + """Initialize an instance of the Factory. + + :param wait_timeout: Task wait timeout in case of blocking result get + """ self.__wait_timeout = wait_timeout def __call__( @@ -144,10 +188,17 @@ def __call__( factory: Union[Coroutine[Any, Any, _T], Generator[Any, None, _T]], **_ ) -> Task[_T]: - return ThreadedTask(factory, self.__wait_timeout, loop=self.__loop) + """Create Threaded Task in appropriate Event Loop. + + :param loop: Event Loop which will be used to execute the Task + :param factory: Future, Coroutine or a Generator of these objects, which will be executed + """ + return ThreadedTask(factory, self.__wait_timeout, loop=loop) -class TriggerTaskList(Generic[_T]): +class TriggerTaskBatcher(Generic[_T]): + """Batching class which compile its batches by object number or by passed time.""" + __task_list: List[_T] __last_run_time: float __trigger_num: int @@ -155,7 +206,12 @@ class TriggerTaskList(Generic[_T]): def __init__(self, trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, - trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL): + trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL) -> None: + """Initialize an instance of the Batcher. + + :param trigger_num: object number threshold which triggers batch return and reset + :param trigger_interval: amount of time after which return and reset batch + """ self.__task_list = [] self.__last_run_time = time.time() self.__trigger_num = trigger_num @@ -173,6 +229,11 @@ def __ready_to_run(self) -> bool: return False def append(self, value: _T) -> Optional[List[_T]]: + """Add an object to internal batch and return the batch if it's triggered. + + :param value: an object to add to the batch + :return: a batch or None + """ self.__task_list.append(value) if self.__ready_to_run(): tasks = self.__task_list @@ -180,16 +241,23 @@ def append(self, value: _T) -> Optional[List[_T]]: return tasks def flush(self) -> Optional[List[_T]]: + """Immediately return everything what's left in the internal batch. + + :return: a batch or None + """ if len(self.__task_list) > 0: tasks = self.__task_list self.__task_list = [] return tasks -class BackgroundTaskList(Generic[_T]): +class BackgroundTaskBatcher(Generic[_T]): + """Batching class which collects Tasks into internal batch and removes when they complete.""" + __task_list: List[_T] def __init__(self): + """Initialize an instance of the Batcher.""" self.__task_list = [] def __remove_finished(self): @@ -201,10 +269,18 @@ def __remove_finished(self): self.__task_list = self.__task_list[i + 1:] def append(self, value: _T) -> None: + """Add an object to internal batch. + + :param value: an object to add to the batch + """ self.__remove_finished() self.__task_list.append(value) def flush(self) -> Optional[List[_T]]: + """Immediately return everything what's left unfinished in the internal batch. + + :return: a batch or None + """ self.__remove_finished() if len(self.__task_list) > 0: tasks = self.__task_list diff --git a/reportportal_client/client.py b/reportportal_client/client.py index d5fc1c69..edb9d603 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -26,7 +26,6 @@ # noinspection PyProtectedMember from reportportal_client._local import set_current -from reportportal_client.aio import Task from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (HttpRequest, ItemStartRequest, ItemFinishRequest, RPFile, LaunchStartRequest, LaunchFinishRequest, RPRequestLog, @@ -54,7 +53,7 @@ class RP(metaclass=AbstractBaseClass): @property @abstractmethod - def launch_uuid(self) -> Optional[Union[str, Task[str]]]: + def launch_uuid(self) -> Optional[str]: """Return current launch UUID. :return: UUID string @@ -62,7 +61,7 @@ def launch_uuid(self) -> Optional[Union[str, Task[str]]]: raise NotImplementedError('"launch_uuid" property is not implemented!') @property - def launch_id(self) -> Optional[Union[str, Task[str]]]: + def launch_id(self) -> Optional[str]: """Return current launch UUID. :return: UUID string @@ -110,7 +109,7 @@ def start_launch(self, attributes: Optional[Union[list, dict]] = None, rerun: bool = False, rerun_of: Optional[str] = None, - **kwargs) -> Union[Optional[str], Task[str]]: + **kwargs) -> Optional[str]: """Start a new launch with the given parameters. :param name: Launch name @@ -133,12 +132,12 @@ def start_test_item(self, description: Optional[str] = None, attributes: Optional[List[dict]] = None, parameters: Optional[dict] = None, - parent_item_id: Union[Optional[str], Task[str]] = None, + parent_item_id: Optional[str] = None, has_stats: bool = True, code_ref: Optional[str] = None, retry: bool = False, test_case_id: Optional[str] = None, - **kwargs: Any) -> Union[Optional[str], Task[str]]: + **kwargs: Any) -> Optional[str]: """Start case/step/nested step item. :param name: Name of the test item @@ -164,7 +163,7 @@ def start_test_item(self, @abstractmethod def finish_test_item(self, - item_id: Union[str, Task[str]], + item_id: str, end_time: str, *, status: str = None, @@ -172,7 +171,7 @@ def finish_test_item(self, attributes: Optional[Union[list, dict]] = None, description: str = None, retry: bool = False, - **kwargs: Any) -> Union[Optional[str], Task[str]]: + **kwargs: Any) -> Optional[str]: """Finish suite/case/step/nested step item. :param item_id: ID of the test item @@ -195,7 +194,7 @@ def finish_launch(self, end_time: str, status: str = None, attributes: Optional[Union[list, dict]] = None, - **kwargs: Any) -> Union[Optional[str], Task[str]]: + **kwargs: Any) -> Optional[str]: """Finish current launch. :param end_time: Launch end time @@ -208,9 +207,9 @@ def finish_launch(self, @abstractmethod def update_test_item(self, - item_uuid: Union[Optional[str], Task[str]], + item_uuid: Optional[str], attributes: Optional[Union[list, dict]] = None, - description: Optional[str] = None) -> Union[Optional[str], Task[str]]: + description: Optional[str] = None) -> Optional[str]: """Update existing test item at the ReportPortal. :param item_uuid: Test item UUID returned on the item start @@ -221,7 +220,7 @@ def update_test_item(self, raise NotImplementedError('"update_test_item" method is not implemented!') @abstractmethod - def get_launch_info(self) -> Union[Optional[dict], Task[str]]: + def get_launch_info(self) -> Optional[dict]: """Get the current launch information. :return: Launch information in dictionary @@ -229,7 +228,7 @@ def get_launch_info(self) -> Union[Optional[dict], Task[str]]: raise NotImplementedError('"get_launch_info" method is not implemented!') @abstractmethod - def get_item_id_by_uuid(self, item_uuid: Union[str, Task[str]]) -> Optional[str]: + def get_item_id_by_uuid(self, item_uuid: str) -> Optional[str]: """Get test item ID by the given UUID. :param item_uuid: UUID returned on the item start @@ -254,7 +253,7 @@ def get_launch_ui_url(self) -> Optional[str]: raise NotImplementedError('"get_launch_ui_id" method is not implemented!') @abstractmethod - def get_project_settings(self) -> Union[Optional[dict], Task[dict]]: + def get_project_settings(self) -> Optional[dict]: """Get project settings. :return: HTTP response in dictionary @@ -263,7 +262,7 @@ def get_project_settings(self) -> Union[Optional[dict], Task[dict]]: @abstractmethod def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[dict] = None, item_id: Union[Optional[str], Task[str]] = None) -> None: + attachment: Optional[dict] = None, item_id: Optional[str] = None) -> None: """Send log message to the ReportPortal. :param datetime: Time in UTC @@ -293,7 +292,7 @@ def terminate(self, *_: Any, **__: Any) -> None: ) @abstractmethod - def current_item(self) -> Optional[Union[str, Task[str]]]: + def current_item(self) -> Optional[str]: """Retrieve the last item reported by the client.""" raise NotImplementedError('"current_item" method is not implemented!') diff --git a/reportportal_client/static/errors.py b/reportportal_client/errors.py similarity index 100% rename from reportportal_client/static/errors.py rename to reportportal_client/errors.py From 3c5274619153891690c68052f5a35b2754f63212 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 26 Sep 2023 22:49:56 +0300 Subject: [PATCH 157/268] Fix typing --- reportportal_client/aio/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 878f6125..d9a37eb3 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -79,7 +79,7 @@ class Client: print_output: Optional[TextIO] _skip_analytics: str __session: Optional[aiohttp.ClientSession] - __stat_task: Optional[asyncio.Task[aiohttp.ClientResponse]] + __stat_task: Optional[asyncio.Task] def __init__( self, From 61252931ca4d0cc07a1cc823ef24e8ebba08ea56 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 26 Sep 2023 22:54:11 +0300 Subject: [PATCH 158/268] Add pydocs --- reportportal_client/aio/__init__.py | 2 ++ reportportal_client/aio/tasks.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/reportportal_client/aio/__init__.py b/reportportal_client/aio/__init__.py index 52742af9..88060612 100644 --- a/reportportal_client/aio/__init__.py +++ b/reportportal_client/aio/__init__.py @@ -11,6 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License +"""Common package for Asynchronous I/O clients and utilities.""" + from reportportal_client.aio.tasks import (Task, TriggerTaskBatcher, BatchedTask, BatchedTaskFactory, ThreadedTask, ThreadedTaskFactory, BlockingOperationError, BackgroundTaskBatcher, DEFAULT_TASK_TRIGGER_NUM, diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index aa3b69cd..3f81552d 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -173,6 +173,8 @@ def __call__( class ThreadedTaskFactory: + """Factory protocol which creates Threaded Tasks.""" + __wait_timeout: float def __init__(self, wait_timeout: float): From 095a6c5aaff70ecf73e71e58fb9f4e09c7545bf2 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 27 Sep 2023 13:43:45 +0300 Subject: [PATCH 159/268] Update pydocs --- reportportal_client/aio/client.py | 163 +++++++++++++++++++++--------- reportportal_client/aio/http.py | 2 +- reportportal_client/client.py | 51 +++++----- 3 files changed, 139 insertions(+), 77 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index d9a37eb3..7e740056 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License -"""This module contains asynchronous implementation of ReportPortal Client.""" +"""This module contains asynchronous implementations of ReportPortal Client.""" import asyncio import logging @@ -29,10 +29,10 @@ from reportportal_client import RP # noinspection PyProtectedMember from reportportal_client._local import set_current +from reportportal_client.aio.http import RetryingClientSession from reportportal_client.aio.tasks import (Task, BatchedTaskFactory, ThreadedTaskFactory, TriggerTaskBatcher, BackgroundTaskBatcher, DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL) -from reportportal_client.aio.http import RetryingClientSession from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, AsyncItemFinishRequest, LaunchFinishRequest, RPFile, @@ -59,14 +59,19 @@ class Client: + """Stateless asynchronous ReportPortal Client. + + This class intentionally made to not store any data or context from ReportPortal. It provides basic + reporting and data read functions in asynchronous manner. Use it whenever you want to handle item IDs, log + batches, future tasks on your own. + """ + api_v1: str api_v2: str base_url_v1: str base_url_v2: str endpoint: str is_skipped_an_issue: bool - log_batch_size: int - log_batch_payload_size: int project: str api_key: str verify_ssl: Union[bool, str] @@ -75,8 +80,8 @@ class Client: http_timeout: Union[float, Tuple[float, float]] keepalive_timeout: Optional[float] mode: str - launch_uuid_print: Optional[bool] - print_output: Optional[TextIO] + launch_uuid_print: bool + print_output: TextIO _skip_analytics: str __session: Optional[aiohttp.ClientSession] __stat_task: Optional[asyncio.Task] @@ -87,27 +92,40 @@ def __init__( project: str, *, api_key: str = None, - log_batch_size: int = 20, is_skipped_an_issue: bool = True, verify_ssl: Union[bool, str] = True, retries: int = None, max_pool_size: int = 50, http_timeout: Union[float, Tuple[float, float]] = (10, 10), keepalive_timeout: Optional[float] = None, - log_batch_payload_size: int = MAX_LOG_BATCH_PAYLOAD_SIZE, mode: str = 'DEFAULT', launch_uuid_print: bool = False, print_output: Optional[TextIO] = None, **kwargs: Any ) -> None: + """Initialize the class instance with arguments. + + :param endpoint: Endpoint of the ReportPortal service. + :param project: Project name to report to. + :param api_key: Authorization API key. + :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the + server side. + :param verify_ssl: Option to skip ssl verification. + :param retries: Number of retry attempts to make in case of connection / server errors. + :param max_pool_size: Option to set the maximum number of connections to save the pool. + :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to + specific connect and read separately. + :param keepalive_timeout: Maximum amount of idle time in seconds before force connection closing. + :param mode: Launch mode, all Launches started by the client will be in that mode. + :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. + :param print_output: Set output stream for Launch UUID printing. + """ self.api_v1, self.api_v2 = 'v1', 'v2' self.endpoint = endpoint self.project = project self.base_url_v1 = root_uri_join(f'api/{self.api_v1}', self.project) self.base_url_v2 = root_uri_join(f'api/{self.api_v2}', self.project) self.is_skipped_an_issue = is_skipped_an_issue - self.log_batch_size = log_batch_size - self.log_batch_payload_size = log_batch_payload_size self.verify_ssl = verify_ssl self.retries = retries self.max_pool_size = max_pool_size @@ -124,9 +142,8 @@ def __init__( if not self.api_key: if 'token' in kwargs: warnings.warn( - message='Argument `token` is deprecated since 5.3.5 and ' - 'will be subject for removing in the next major ' - 'version. Use `api_key` argument instead.', + message='Argument `token` is deprecated since 5.3.5 and will be subject for removing in ' + 'the next major version. Use `api_key` argument instead.', category=DeprecationWarning, stacklevel=2 ) @@ -134,10 +151,9 @@ def __init__( if not self.api_key: warnings.warn( - message='Argument `api_key` is `None` or empty string, ' - 'that is not supposed to happen because Report ' - 'Portal is usually requires an authorization key. ' - 'Please check your code.', + message='Argument `api_key` is `None` or empty string, that is not supposed to happen ' + 'because Report Portal is usually requires an authorization key. Please check ' + 'your code.', category=RuntimeWarning, stacklevel=2 ) @@ -155,27 +171,34 @@ def session(self) -> aiohttp.ClientSession: else: ssl_config = ssl.create_default_context(cafile=certifi.where()) - params = { + connection_params = { 'ssl': ssl_config, 'limit': self.max_pool_size } if self.keepalive_timeout: - params['keepalive_timeout'] = self.keepalive_timeout - connector = aiohttp.TCPConnector(**params) + connection_params['keepalive_timeout'] = self.keepalive_timeout + connector = aiohttp.TCPConnector(**connection_params) + + headers = {} + if self.api_key: + headers['Authorization'] = f'Bearer {self.api_key}' + + session_params = { + headers: headers, + connector: connector + } - timeout = None if self.http_timeout: if type(self.http_timeout) == tuple: connect_timeout, read_timeout = self.http_timeout else: connect_timeout, read_timeout = self.http_timeout, self.http_timeout - timeout = aiohttp.ClientTimeout(connect=connect_timeout, sock_read=read_timeout) + session_params['timeout'] = aiohttp.ClientTimeout(connect=connect_timeout, sock_read=read_timeout) - headers = {} - if self.api_key: - headers['Authorization'] = f'Bearer {self.api_key}' - self.__session = RetryingClientSession(self.endpoint, connector=connector, headers=headers, - timeout=timeout) + if self.retries: + session_params['max_retry_number'] = self.retries + + self.__session = RetryingClientSession(self.endpoint, **session_params) return self.__session async def close(self): @@ -442,14 +465,12 @@ def clone(self) -> 'Client': endpoint=self.endpoint, project=self.project, api_key=self.api_key, - log_batch_size=self.log_batch_size, is_skipped_an_issue=self.is_skipped_an_issue, verify_ssl=self.verify_ssl, retries=self.retries, max_pool_size=self.max_pool_size, http_timeout=self.http_timeout, keepalive_timeout=self.keepalive_timeout, - log_batch_payload_size=self.log_batch_payload_size, mode=self.mode, launch_uuid_print=self.launch_uuid_print, print_output=self.print_output @@ -458,6 +479,8 @@ def clone(self) -> 'Client': class AsyncRPClient(RP): + log_batch_size: int + log_batch_payload_limit: int _item_stack: LifoQueue _log_batcher: LogBatcher __client: Client @@ -465,24 +488,6 @@ class AsyncRPClient(RP): __step_reporter: StepReporter use_own_launch: bool - def __init__(self, endpoint: str, project: str, *, launch_uuid: Optional[str] = None, - client: Optional[Client] = None, **kwargs: Any) -> None: - set_current(self) - self.__endpoint = endpoint - self.__project = project - self.__step_reporter = StepReporter(self) - self._item_stack = LifoQueue() - self._log_batcher = LogBatcher() - if client: - self.__client = client - else: - self.__client = Client(endpoint, project, **kwargs) - if launch_uuid: - self.__launch_uuid = launch_uuid - self.use_own_launch = False - else: - self.use_own_launch = True - @property def launch_uuid(self) -> Optional[str]: return self.__launch_uuid @@ -499,6 +504,61 @@ def project(self) -> str: def step_reporter(self) -> StepReporter: return self.__step_reporter + def __init__( + self, + endpoint: str, + project: str, + *, + client: Optional[Client] = None, + launch_uuid: Optional[str] = None, + log_batch_size: int = 20, + log_batch_payload_limit: int = MAX_LOG_BATCH_PAYLOAD_SIZE, + log_batcher: Optional[LogBatcher] = None, + **kwargs: Any + ) -> None: + """Initialize the class instance with arguments. + + :param endpoint: Endpoint of the ReportPortal service. + :param project: Project name to report to. + :param api_key: Authorization API key. + :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the + server side. + :param verify_ssl: Option to skip ssl verification. + :param retries: Number of retry attempts to make in case of connection / server errors. + :param max_pool_size: Option to set the maximum number of connections to save the pool. + :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to + specific connect and read separately. + :param keepalive_timeout: Maximum amount of idle time in seconds before force connection closing. + :param mode: Launch mode, all Launches started by the client will be in that mode. + :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. + :param print_output: Set output stream for Launch UUID printing. + :param client: ReportPortal async Client instance to use. If set, all above arguments + will be ignored. + :param launch_uuid: A launch UUID to use instead of starting own one. + :param log_batch_size: Option to set the maximum number of logs that can be processed in one + batch. + :param log_batch_payload_limit: maximum size in bytes of logs that can be processed in one batch + :param log_batcher: ReportPortal log batcher instance to use. If set, 'log_batch' + arguments above will be ignored. + """ + self.__endpoint = endpoint + self.__project = project + self.__step_reporter = StepReporter(self) + self._item_stack = LifoQueue() + self.log_batch_size = log_batch_size + self.log_batch_payload_limit = log_batch_payload_limit + self._log_batcher = log_batcher or LogBatcher(log_batch_size, log_batch_payload_limit) + if client: + self.__client = client + else: + self.__client = Client(endpoint, project, **kwargs) + if launch_uuid: + self.__launch_uuid = launch_uuid + self.use_own_launch = False + else: + self.use_own_launch = True + set_current(self) + async def start_launch(self, name: str, start_time: str, @@ -634,10 +694,13 @@ def clone(self) -> 'AsyncRPClient': cloned_client = self.__client.clone() # noinspection PyTypeChecker cloned = AsyncRPClient( - endpoint=None, - project=None, + endpoint=self.endpoint, + project=self.project, client=cloned_client, - launch_uuid=self.launch_uuid + launch_uuid=self.launch_uuid, + log_batch_size=self.log_batch_size, + log_batch_payload_limit=self.log_batch_payload_limit, + log_batcher=self._log_batcher ) current_item = self.current_item() if current_item: diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py index 533620bd..7150ea8b 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/aio/http.py @@ -24,7 +24,7 @@ DEFAULT_RETRY_NUMBER: int = 5 DEFAULT_RETRY_DELAY: float = 0.005 THROTTLING_STATUSES: set = {425, 429} -RETRY_STATUSES: set = {408, 500, 502, 503, 507}.union(THROTTLING_STATUSES) +RETRY_STATUSES: set = {408, 500, 502, 503, 504, 507}.union(THROTTLING_STATUSES) class RetryClass(int, Enum): diff --git a/reportportal_client/client.py b/reportportal_client/client.py index edb9d603..7275f47a 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -407,25 +407,26 @@ def __init__( log_batcher: Optional[LogBatcher[RPRequestLog]] = None, **kwargs: Any ) -> None: - """Initialize required attributes. - - :param endpoint: Endpoint of the ReportPortal service - :param project: Project name to report to - :param api_key: Authorization API key - :param log_batch_size: Option to set the maximum number of - logs that can be processed in one batch - :param is_skipped_an_issue: Option to mark skipped tests as not - 'To Investigate' items on the server - side - :param verify_ssl: Option to skip ssl verification - :param max_pool_size: Option to set the maximum number of - connections to save the pool. - :param launch_uuid: a launch UUID to use instead of starting own one - :param http_timeout: a float in seconds for connect and read - timeout. Use a Tuple to specific connect - and read separately. - :param log_batch_payload_size: maximum size in bytes of logs that can - be processed in one batch + """Initialize the class instance with arguments. + + :param endpoint: Endpoint of the ReportPortal service. + :param project: Project name to report to. + :param api_key: Authorization API key. + :param log_batch_size: Option to set the maximum number of logs that can be processed in one + batch. + :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the + server side. + :param verify_ssl: Option to skip ssl verification. + :param retries: Number of retry attempts to make in case of connection / server errors. + :param max_pool_size: Option to set the maximum number of connections to save the pool. + :param launch_uuid: A launch UUID to use instead of starting own one. + :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to + specific connect and read separately. + :param log_batch_payload_size: Maximum size in bytes of logs that can be processed in one batch. + :param mode: Launch mode, all Launches started by the client will be in that mode. + :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. + :param print_output: Set output stream for Launch UUID printing. + :param log_batcher: Use existing LogBatcher instance instead of creation of own one. """ set_current(self) self.api_v1, self.api_v2 = 'v1', 'v2' @@ -466,9 +467,8 @@ def __init__( if not self.api_key: if 'token' in kwargs: warnings.warn( - message='Argument `token` is deprecated since 5.3.5 and ' - 'will be subject for removing in the next major ' - 'version. Use `api_key` argument instead.', + message='Argument `token` is deprecated since 5.3.5 and will be subject for removing in ' + 'the next major version. Use `api_key` argument instead.', category=DeprecationWarning, stacklevel=2 ) @@ -476,10 +476,9 @@ def __init__( if not self.api_key: warnings.warn( - message='Argument `api_key` is `None` or empty string, ' - 'that is not supposed to happen because Report ' - 'Portal is usually requires an authorization key. ' - 'Please check your code.', + message='Argument `api_key` is `None` or empty string, that is not supposed to happen ' + 'because Report Portal is usually requires an authorization key. Please check ' + 'your code.', category=RuntimeWarning, stacklevel=2 ) From 47b66e9b4dbd11742e544a254b3cdabf0c41dc41 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 27 Sep 2023 17:24:24 +0300 Subject: [PATCH 160/268] Update pydocs --- reportportal_client/aio/client.py | 138 +++++++++++++++++++++++------- reportportal_client/client.py | 99 ++++++++++----------- 2 files changed, 153 insertions(+), 84 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 7e740056..7cf66419 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -160,6 +160,10 @@ def __init__( @property def session(self) -> aiohttp.ClientSession: + """Return aiohttp.ClientSession class instance, initialize it if necessary. + + :return: aiohttp.ClientSession instance + """ if self.__session: return self.__session @@ -201,7 +205,8 @@ def session(self) -> aiohttp.ClientSession: self.__session = RetryingClientSession(self.endpoint, **session_params) return self.__session - async def close(self): + async def close(self) -> None: + """Gracefully close internal aiohttp.ClientSession class instance and reset it.""" if self.__session: await self.__session.close() self.__session = None @@ -229,15 +234,16 @@ async def start_launch(self, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Optional[str]: - """Start a new launch with the given parameters. + """Start a new launch with the given arguments. :param name: Launch name :param start_time: Launch start time :param description: Launch description :param attributes: Launch attributes :param rerun: Start launch in rerun mode - :param rerun_of: For rerun mode specifies which launch will be - re-run. Should be used with the 'rerun' option. + :param rerun_of: For rerun mode specifies which launch will be re-run. Should be used with the + 'rerun' option. + :return: Launch UUID if successfully started or None """ url = root_uri_join(self.base_url_v2, 'launch') request_payload = LaunchStartRequest( @@ -247,7 +253,7 @@ async def start_launch(self, description=description, mode=self.mode, rerun=rerun, - rerun_of=rerun_of or kwargs.get('rerunOf') + rerun_of=rerun_of ).payload response = await AsyncHttpRequest(self.session.post, url=url, json=request_payload).make() @@ -270,15 +276,34 @@ async def start_test_item(self, start_time: str, item_type: str, *, + parent_item_id: Optional[Union[str, Task[str]]] = None, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, parameters: Optional[Dict] = None, - parent_item_id: Optional[Union[str, Task[str]]] = None, - has_stats: bool = True, code_ref: Optional[str] = None, - retry: bool = False, test_case_id: Optional[str] = None, + has_stats: bool = True, + retry: bool = False, **_: Any) -> Optional[str]: + """Start test case/step/nested step item. + + :param launch_uuid: A launch UUID where to start the Test Item. + :param name: Name of the Test Item. + :param start_time: The Item start time. + :param item_type: Type of the Test Item. Allowed values: + "suite", "story", "test", "scenario", "step", "before_class", "before_groups", + "before_method", "before_suite", "before_test", "after_class", "after_groups", + "after_method", "after_suite", "after_test". + :param parent_item_id: A UUID of a parent SUITE / STEP + :param description: The Item description + :param attributes: Test Item attributes + :param parameters: Set of parameters (for parametrized test items) + :param code_ref: Physical location of the Test Item + :param test_case_id: A unique ID of the current step + :param has_stats: Set to False if test item is a nested step + :param retry: Used to report retry of the test. Allowed values: "True" or "False" + :return: Test Item UUID if successfully started or None + """ if parent_item_id: url = self.__get_item_url(parent_item_id) else: @@ -313,11 +338,25 @@ async def finish_test_item(self, end_time: str, *, status: str = None, - issue: Optional[Issue] = None, - attributes: Optional[Union[List, Dict]] = None, description: str = None, + attributes: Optional[Union[List, Dict]] = None, + issue: Optional[Issue] = None, retry: bool = False, **kwargs: Any) -> Optional[str]: + """Finish test case/step/nested step item. + + :param launch_uuid: A launch UUID where to finish the Test Item. + :param item_id: ID of the Test Item. + :param end_time: The Item end time. + :param status: Test status. Allowed values: + PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED, INFO, WARN or None. + :param description: Test Item description. Overrides description from start request. + :param attributes: Test Item attributes(tags). Pairs of key and value. These attributes override + attributes on start Test Item call. + :param issue: Issue which will be attached to the current Item. + :param retry: Used to report retry of the test. Allowed values: "True" or "False". + :return: Response message. + """ url = self.__get_item_url(item_id) request_payload = AsyncItemFinishRequest( end_time, @@ -344,6 +383,15 @@ async def finish_launch(self, status: str = None, attributes: Optional[Union[List, Dict]] = None, **kwargs: Any) -> Optional[str]: + """Finish a launch. + + :param launch_uuid: A launch UUID to finish. + :param end_time: Launch end time. + :param status: Launch status. Can be one of the followings: + PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED. + :param attributes: Launch attributes. These attributes override attributes on Start Launch call. + :return: Response message. + """ url = self.__get_launch_url(launch_uuid) request_payload = LaunchFinishRequest( end_time, @@ -365,6 +413,13 @@ async def update_test_item(self, *, attributes: Optional[Union[List, Dict]] = None, description: Optional[str] = None) -> Optional[str]: + """Update existing Test Item at the ReportPortal. + + :param item_uuid: Test Item UUID returned on the item start. + :param attributes: Test Item attributes: [{'key': 'k_name', 'value': 'k_value'}, ...]. + :param description: Test Item description. + :return: Response message. + """ data = { 'description': description, 'attributes': verify_value_length(attributes), @@ -377,23 +432,6 @@ async def update_test_item(self, logger.debug('update_test_item - Item: %s', item_id) return await response.message - async def __get_item_uuid_url(self, item_uuid_future: Union[str, Task[str]]) -> Optional[str]: - item_uuid = await await_if_necessary(item_uuid_future) - if item_uuid is NOT_FOUND: - logger.warning('Attempt to make request for non-existent UUID.') - return - return root_uri_join(self.base_url_v1, 'item', 'uuid', item_uuid) - - async def get_item_id_by_uuid(self, item_uuid_future: Union[str, Task[str]]) -> Optional[str]: - """Get test Item ID by the given Item UUID. - - :param item_uuid_future: Str or Task UUID returned on the Item start - :return: Test item ID - """ - url = self.__get_item_uuid_url(item_uuid_future) - response = await AsyncHttpRequest(self.session.get, url=url).make() - return response.id if response else None - async def __get_launch_uuid_url(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[str]: launch_uuid = await await_if_necessary(launch_uuid_future) if launch_uuid is NOT_FOUND: @@ -405,8 +443,8 @@ async def __get_launch_uuid_url(self, launch_uuid_future: Union[str, Task[str]]) async def get_launch_info(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[Dict]: """Get the launch information by Launch UUID. - :param launch_uuid_future: Str or Task UUID returned on the Launch start - :return dict: Launch information in dictionary + :param launch_uuid_future: Str or Task UUID returned on the Launch start. + :return: Launch information in dictionary. """ url = self.__get_launch_uuid_url(launch_uuid_future) response = await AsyncHttpRequest(self.session.get, url=url).make() @@ -420,15 +458,42 @@ async def get_launch_info(self, launch_uuid_future: Union[str, Task[str]]) -> Op launch_info = {} return launch_info + async def __get_item_uuid_url(self, item_uuid_future: Union[str, Task[str]]) -> Optional[str]: + item_uuid = await await_if_necessary(item_uuid_future) + if item_uuid is NOT_FOUND: + logger.warning('Attempt to make request for non-existent UUID.') + return + return root_uri_join(self.base_url_v1, 'item', 'uuid', item_uuid) + + async def get_item_id_by_uuid(self, item_uuid_future: Union[str, Task[str]]) -> Optional[str]: + """Get Test Item ID by the given Item UUID. + + :param item_uuid_future: Str or Task UUID returned on the Item start. + :return: Test Item ID. + """ + url = self.__get_item_uuid_url(item_uuid_future) + response = await AsyncHttpRequest(self.session.get, url=url).make() + return response.id if response else None + async def get_launch_ui_id(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[int]: + """Get Launch ID of the given Launch. + + :param launch_uuid_future: Str or Task UUID returned on the Launch start. + :return: Launch ID of the Launch. None if not found. + """ launch_info = await self.get_launch_info(launch_uuid_future) return launch_info.get('id') if launch_info else None async def get_launch_ui_url(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[str]: + """Get full quality URL of the given launch. + + :param launch_uuid_future: Str or Task UUID returned on the Launch start. + :return: Launch URL string. + """ launch_uuid = await await_if_necessary(launch_uuid_future) launch_info = await self.get_launch_info(launch_uuid) - ui_id = launch_info.get('id') if launch_info else None - if not ui_id: + launch_id = launch_info.get('id') if launch_info else None + if not launch_id: return mode = launch_info.get('mode') if launch_info else None if not mode: @@ -438,17 +503,26 @@ async def get_launch_ui_url(self, launch_uuid_future: Union[str, Task[str]]) -> path = 'ui/#{project_name}/{launch_type}/all/{launch_id}'.format( project_name=self.project.lower(), launch_type=launch_type, - launch_id=ui_id) + launch_id=launch_id) url = root_uri_join(self.endpoint, path) logger.debug('get_launch_ui_url - ID: %s', launch_uuid) return url async def get_project_settings(self) -> Optional[Dict]: + """Get settings of the current project. + + :return: Settings response in Dictionary. + """ url = root_uri_join(self.base_url_v1, 'settings') response = await AsyncHttpRequest(self.session.get, url=url).make() return await response.json if response else None async def log_batch(self, log_batch: Optional[List[AsyncRPRequestLog]]) -> Tuple[str, ...]: + """Send batch logging message to the ReportPortal. + + :param log_batch: A list of log message objects. + :return: Completion message tuple of variable size (depending on request size). + """ url = root_uri_join(self.base_url_v2, 'log') if log_batch: response = await AsyncHttpRequest(self.session.post, url=url, diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 7275f47a..abd1bf75 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -128,7 +128,6 @@ def start_test_item(self, name: str, start_time: str, item_type: str, - *, description: Optional[str] = None, attributes: Optional[List[dict]] = None, parameters: Optional[dict] = None, @@ -140,22 +139,19 @@ def start_test_item(self, **kwargs: Any) -> Optional[str]: """Start case/step/nested step item. - :param name: Name of the test item - :param start_time: The item start time - :param item_type: Type of the test item. Allowable values: - "suite", "story", "test", "scenario", "step", - "before_class", "before_groups", - "before_method", "before_suite", - "before_test", "after_class", "after_groups", - "after_method", "after_suite", "after_test" + :param name: Name of the Test Item. + :param start_time: The Item start time. + :param item_type: Type of the Test Item. Allowed values: + "suite", "story", "test", "scenario", "step", "before_class", "before_groups", + "before_method", "before_suite", "before_test", "after_class", "after_groups", + "after_method", "after_suite", "after_test". :param description: The item description :param attributes: Test item attributes :param parameters: Set of parameters (for parametrized test items) :param parent_item_id: An ID of a parent SUITE / STEP :param has_stats: Set to False if test item is nested step :param code_ref: Physical location of the test item - :param retry: Used to report retry of the test. Allowable - values: "True" or "False" + :param retry: Used to report retry of the test. Allowed values: "True" or "False" :param test_case_id: A unique ID of the current step :return: Test Item UUID """ @@ -165,7 +161,6 @@ def start_test_item(self, def finish_test_item(self, item_id: str, end_time: str, - *, status: str = None, issue: Optional[Issue] = None, attributes: Optional[Union[list, dict]] = None, @@ -174,17 +169,15 @@ def finish_test_item(self, **kwargs: Any) -> Optional[str]: """Finish suite/case/step/nested step item. - :param item_id: ID of the test item - :param end_time: The item end time - :param status: Test status. Allowable values: - PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED, INFO, WARN or None - :param issue: Issue which will be attached to the current item - :param attributes: Test item attributes(tags). Pairs of key and value. - Override attributes on start - :param description: Test item description. Overrides description - from start request. - :param retry: Used to report retry of the test. Allowable values: - "True" or "False" + :param item_id: ID of the Test Item. + :param end_time: The Item end time. + :param status: Test status. Allowed values: + PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED, INFO, WARN or None. + :param issue: Issue which will be attached to the current Item. + :param attributes: Test Item attributes(tags). Pairs of key and value. These are override attributes + on start Test Item call. + :param description: Test Item description. Overrides description from start request. + :param retry: Used to report retry of the test. Allowed values: "True" or "False". :return: Response message """ raise NotImplementedError('"finish_test_item" method is not implemented!') @@ -197,11 +190,11 @@ def finish_launch(self, **kwargs: Any) -> Optional[str]: """Finish current launch. - :param end_time: Launch end time + :param end_time: Launch end time. :param status: Launch status. Can be one of the followings: - PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED - :param attributes: Launch attributes - :return: Response message + PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED. + :param attributes: Launch attributes. These attributes override attributes on Start Launch call. + :return: Response message. """ raise NotImplementedError('"finish_launch" method is not implemented!') @@ -210,12 +203,12 @@ def update_test_item(self, item_uuid: Optional[str], attributes: Optional[Union[list, dict]] = None, description: Optional[str] = None) -> Optional[str]: - """Update existing test item at the ReportPortal. + """Update existing Test Item at the ReportPortal. - :param item_uuid: Test item UUID returned on the item start - :param attributes: Test item attributes: [{'key': 'k_name', 'value': 'k_value'}, ...] - :param description: Test item description - :return: Response message + :param item_uuid: Test Item UUID returned on the item start. + :param attributes: Test Item attributes: [{'key': 'k_name', 'value': 'k_value'}, ...]. + :param description: Test Item description. + :return: Response message. """ raise NotImplementedError('"update_test_item" method is not implemented!') @@ -229,18 +222,18 @@ def get_launch_info(self) -> Optional[dict]: @abstractmethod def get_item_id_by_uuid(self, item_uuid: str) -> Optional[str]: - """Get test item ID by the given UUID. + """Get Test Item ID by the given UUID. - :param item_uuid: UUID returned on the item start - :return: Test item ID + :param item_uuid: UUID returned on the Item start. + :return: Test Item ID. """ raise NotImplementedError('"get_item_id_by_uuid" method is not implemented!') @abstractmethod def get_launch_ui_id(self) -> Optional[int]: - """Get UI ID of the current launch. + """Get Launch ID of the current launch. - :return: UI ID of the given launch. None if UI ID has not been found. + :return: Launch ID of the Launch. None if not found. """ raise NotImplementedError('"get_launch_ui_id" method is not implemented!') @@ -248,28 +241,31 @@ def get_launch_ui_id(self) -> Optional[int]: def get_launch_ui_url(self) -> Optional[str]: """Get full quality URL of the current launch. - :return: launch URL string + :return: Launch URL string. """ raise NotImplementedError('"get_launch_ui_id" method is not implemented!') @abstractmethod def get_project_settings(self) -> Optional[dict]: - """Get project settings. + """Get settings of the current project. - :return: HTTP response in dictionary + :return: Settings response in Dictionary. """ raise NotImplementedError('"get_project_settings" method is not implemented!') @abstractmethod - def log(self, datetime: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[dict] = None, item_id: Optional[str] = None) -> None: + def log(self, + datetime: str, message: str, + level: Optional[Union[int, str]] = None, + attachment: Optional[dict] = None, + item_id: Optional[str] = None) -> None: """Send log message to the ReportPortal. - :param datetime: Time in UTC - :param message: Log message text - :param level: Message's log level - :param attachment: Message's attachments - :param item_id: ID of the RP item the message belongs to + :param datetime: Time in UTC. + :param message: Log message text. + :param level: Message's log level. + :param attachment: Message's attachments. + :param item_id: UUID of the ReportPortal Item the message belongs to. """ raise NotImplementedError('"log" method is not implemented!') @@ -493,7 +489,7 @@ def start_launch(self, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Optional[str]: - """Start a new launch with the given parameters. + """Start a new launch with the given arguments. :param name: Launch name :param start_time: Launch start time @@ -502,6 +498,7 @@ def start_launch(self, :param rerun: Start launch in rerun mode :param rerun_of: For rerun mode specifies which launch will be re-run. Should be used with the 'rerun' option. + :return: Launch UUID if successfully started or None """ if not self.use_own_launch: return self.launch_uuid @@ -513,11 +510,9 @@ def start_launch(self, description=description, mode=self.mode, rerun=rerun, - rerun_of=rerun_of or kwargs.get('rerunOf') + rerun_of=rerun_of ).payload - response = HttpRequest(self.session.post, - url=url, - json=request_payload, + response = HttpRequest(self.session.post, url=url, json=request_payload, verify_ssl=self.verify_ssl).make() if not response: return From 11b1ebb19ecedef5a066bb21d3872c2ba3adbfe5 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 27 Sep 2023 17:40:24 +0300 Subject: [PATCH 161/268] Update pydocs --- reportportal_client/aio/client.py | 44 +++++++++++++++++++++++++++++++ reportportal_client/client.py | 14 +++++----- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 7cf66419..85424afd 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -562,20 +562,44 @@ class AsyncRPClient(RP): __step_reporter: StepReporter use_own_launch: bool + @property + def client(self) -> Client: + """Return current Client instance. + + :return: Client instance. + """ + return self.__client + @property def launch_uuid(self) -> Optional[str]: + """Return current Launch UUID. + + :return: UUID string. + """ return self.__launch_uuid @property def endpoint(self) -> str: + """Return current base URL. + + :return: base URL string. + """ return self.__endpoint @property def project(self) -> str: + """Return current Project name. + + :return: Project name string. + """ return self.__project @property def step_reporter(self) -> StepReporter: + """Return StepReporter object for the current launch. + + :return: StepReporter to report steps. + """ return self.__step_reporter def __init__( @@ -798,22 +822,42 @@ class _RPClient(RP, metaclass=AbstractBaseClass): @property def client(self) -> Client: + """Return current Client instance. + + :return: Client instance. + """ return self.__client @property def launch_uuid(self) -> Optional[Task[str]]: + """Return current Launch UUID. + + :return: UUID string. + """ return self.__launch_uuid @property def endpoint(self) -> str: + """Return current base URL. + + :return: base URL string. + """ return self.__endpoint @property def project(self) -> str: + """Return current Project name. + + :return: Project name string. + """ return self.__project @property def step_reporter(self) -> StepReporter: + """Return StepReporter object for the current launch. + + :return: StepReporter to report steps. + """ return self.__step_reporter def __init__( diff --git a/reportportal_client/client.py b/reportportal_client/client.py index abd1bf75..d9ef410d 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -54,17 +54,17 @@ class RP(metaclass=AbstractBaseClass): @property @abstractmethod def launch_uuid(self) -> Optional[str]: - """Return current launch UUID. + """Return current Launch UUID. - :return: UUID string + :return: UUID string. """ raise NotImplementedError('"launch_uuid" property is not implemented!') @property def launch_id(self) -> Optional[str]: - """Return current launch UUID. + """Return current Launch UUID. - :return: UUID string + :return: UUID string. """ warnings.warn( message='`launch_id` property is deprecated since 5.5.0 and will be subject for removing in the' @@ -79,7 +79,7 @@ def launch_id(self) -> Optional[str]: def endpoint(self) -> str: """Return current base URL. - :return: base URL string + :return: base URL string. """ raise NotImplementedError('"endpoint" property is not implemented!') @@ -88,7 +88,7 @@ def endpoint(self) -> str: def project(self) -> str: """Return current Project name. - :return: Project name string + :return: Project name string. """ raise NotImplementedError('"project" property is not implemented!') @@ -97,7 +97,7 @@ def project(self) -> str: def step_reporter(self) -> StepReporter: """Return StepReporter object for the current launch. - :return: StepReporter to report steps + :return: StepReporter to report steps. """ raise NotImplementedError('"step_reporter" property is not implemented!') From cb59f2a68b608e3fbfbc216e79857b473ba7d174 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 28 Sep 2023 14:10:14 +0300 Subject: [PATCH 162/268] Update pydocs --- reportportal_client/aio/client.py | 190 +++++++++++++++++++++++++----- reportportal_client/client.py | 165 +++++++++++++------------- 2 files changed, 240 insertions(+), 115 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 85424afd..576032b6 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -162,7 +162,7 @@ def __init__( def session(self) -> aiohttp.ClientSession: """Return aiohttp.ClientSession class instance, initialize it if necessary. - :return: aiohttp.ClientSession instance + :return: aiohttp.ClientSession instance. """ if self.__session: return self.__session @@ -234,16 +234,16 @@ async def start_launch(self, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Optional[str]: - """Start a new launch with the given arguments. + """Start a new Launch with the given arguments. - :param name: Launch name - :param start_time: Launch start time - :param description: Launch description - :param attributes: Launch attributes - :param rerun: Start launch in rerun mode + :param name: Launch name. + :param start_time: Launch start time. + :param description: Launch description. + :param attributes: Launch attributes. + :param rerun: Start launch in rerun mode. :param rerun_of: For rerun mode specifies which launch will be re-run. Should be used with the 'rerun' option. - :return: Launch UUID if successfully started or None + :return: Launch UUID if successfully started or None. """ url = root_uri_join(self.base_url_v2, 'launch') request_payload = LaunchStartRequest( @@ -285,7 +285,7 @@ async def start_test_item(self, has_stats: bool = True, retry: bool = False, **_: Any) -> Optional[str]: - """Start test case/step/nested step item. + """Start Test Case/Suite/Step/Nested Step Item. :param launch_uuid: A launch UUID where to start the Test Item. :param name: Name of the Test Item. @@ -294,15 +294,15 @@ async def start_test_item(self, "suite", "story", "test", "scenario", "step", "before_class", "before_groups", "before_method", "before_suite", "before_test", "after_class", "after_groups", "after_method", "after_suite", "after_test". - :param parent_item_id: A UUID of a parent SUITE / STEP - :param description: The Item description - :param attributes: Test Item attributes - :param parameters: Set of parameters (for parametrized test items) - :param code_ref: Physical location of the Test Item - :param test_case_id: A unique ID of the current step - :param has_stats: Set to False if test item is a nested step - :param retry: Used to report retry of the test. Allowed values: "True" or "False" - :return: Test Item UUID if successfully started or None + :param parent_item_id: A UUID of a parent SUITE / STEP. + :param description: The Item description. + :param attributes: Test Item attributes. + :param parameters: Set of parameters (for parametrized Test Items). + :param code_ref: Physical location of the Test Item. + :param test_case_id: A unique ID of the current Step. + :param has_stats: Set to False if test item is a Nested Step. + :param retry: Used to report retry of the test. Allowed values: "True" or "False". + :return: Test Item UUID if successfully started or None. """ if parent_item_id: url = self.__get_item_url(parent_item_id) @@ -343,7 +343,7 @@ async def finish_test_item(self, issue: Optional[Issue] = None, retry: bool = False, **kwargs: Any) -> Optional[str]: - """Finish test case/step/nested step item. + """Finish Test Suite/Case/Step/Nested Step Item. :param launch_uuid: A launch UUID where to finish the Test Item. :param item_id: ID of the Test Item. @@ -383,14 +383,14 @@ async def finish_launch(self, status: str = None, attributes: Optional[Union[List, Dict]] = None, **kwargs: Any) -> Optional[str]: - """Finish a launch. + """Finish a Launch. - :param launch_uuid: A launch UUID to finish. + :param launch_uuid: A Launch UUID to finish. :param end_time: Launch end time. :param status: Launch status. Can be one of the followings: PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED. :param attributes: Launch attributes. These attributes override attributes on Start Launch call. - :return: Response message. + :return: Response message or None. """ url = self.__get_launch_url(launch_uuid) request_payload = LaunchFinishRequest( @@ -418,7 +418,7 @@ async def update_test_item(self, :param item_uuid: Test Item UUID returned on the item start. :param attributes: Test Item attributes: [{'key': 'k_name', 'value': 'k_value'}, ...]. :param description: Test Item description. - :return: Response message. + :return: Response message or None. """ data = { 'description': description, @@ -441,7 +441,7 @@ async def __get_launch_uuid_url(self, launch_uuid_future: Union[str, Task[str]]) return root_uri_join(self.base_url_v1, 'launch', 'uuid', launch_uuid) async def get_launch_info(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[Dict]: - """Get the launch information by Launch UUID. + """Get Launch information by Launch UUID. :param launch_uuid_future: Str or Task UUID returned on the Launch start. :return: Launch information in dictionary. @@ -553,6 +553,13 @@ def clone(self) -> 'Client': class AsyncRPClient(RP): + """Stateful asynchronous ReportPortal Client. + + This class implements common RP client interface but all its methods are async, so it capable to use in + asynchronous ReportPortal agents. It handles HTTP request and response bodies generation and + serialization, connection retries and log batching. + """ + log_batch_size: int log_batch_payload_limit: int _item_stack: LifoQueue @@ -665,6 +672,17 @@ async def start_launch(self, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Optional[str]: + """Start a new Launch with the given arguments. + + :param name: Launch name. + :param start_time: Launch start time. + :param description: Launch description. + :param attributes: Launch attributes. + :param rerun: Start launch in rerun mode. + :param rerun_of: For rerun mode specifies which launch will be re-run. Should be used with the + 'rerun' option. + :return: Launch UUID if successfully started or None. + """ if not self.use_own_launch: return self.launch_uuid launch_uuid = await self.__client.start_launch(name, start_time, description=description, @@ -677,7 +695,6 @@ async def start_test_item(self, name: str, start_time: str, item_type: str, - *, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, parameters: Optional[Dict] = None, @@ -687,6 +704,24 @@ async def start_test_item(self, retry: bool = False, test_case_id: Optional[str] = None, **kwargs: Any) -> Optional[str]: + """Start Test Case/Suite/Step/Nested Step Item. + + :param name: Name of the Test Item. + :param start_time: The Item start time. + :param item_type: Type of the Test Item. Allowed values: + "suite", "story", "test", "scenario", "step", "before_class", "before_groups", + "before_method", "before_suite", "before_test", "after_class", "after_groups", + "after_method", "after_suite", "after_test". + :param description: The Item description. + :param attributes: Test Item attributes. + :param parameters: Set of parameters (for parametrized Test Items). + :param parent_item_id: A UUID of a parent SUITE / STEP. + :param has_stats: Set to False if test item is a Nested Step. + :param code_ref: Physical location of the Test Item. + :param retry: Used to report retry of the test. Allowed values: "True" or "False". + :param test_case_id: A unique ID of the current Step. + :return: Test Item UUID if successfully started or None. + """ item_id = await self.__client.start_test_item(self.launch_uuid, name, start_time, item_type, description=description, attributes=attributes, parameters=parameters, parent_item_id=parent_item_id, @@ -700,13 +735,25 @@ async def start_test_item(self, async def finish_test_item(self, item_id: str, end_time: str, - *, status: str = None, issue: Optional[Issue] = None, attributes: Optional[Union[List, Dict]] = None, description: str = None, retry: bool = False, **kwargs: Any) -> Optional[str]: + """Finish Test Suite/Case/Step/Nested Step Item. + + :param item_id: ID of the Test Item. + :param end_time: The Item end time. + :param status: Test status. Allowed values: + PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED, INFO, WARN or None. + :param issue: Issue which will be attached to the current Item. + :param attributes: Test Item attributes(tags). Pairs of key and value. These attributes override + attributes on start Test Item call. + :param description: Test Item description. Overrides description from start request. + :param retry: Used to report retry of the test. Allowed values: "True" or "False". + :return: Response message. + """ result = await self.__client.finish_test_item(self.launch_uuid, item_id, end_time, status=status, issue=issue, attributes=attributes, description=description, @@ -719,6 +766,14 @@ async def finish_launch(self, status: str = None, attributes: Optional[Union[List, Dict]] = None, **kwargs: Any) -> Optional[str]: + """Finish a Launch. + + :param end_time: Launch end time. + :param status: Launch status. Can be one of the followings: + PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED. + :param attributes: Launch attributes. These attributes override attributes on Start Launch call. + :return: Response message or None. + """ await self.__client.log_batch(self._log_batcher.flush()) if not self.use_own_launch: return "" @@ -728,8 +783,19 @@ async def finish_launch(self, await self.__client.close() return result - async def update_test_item(self, item_uuid: str, attributes: Optional[Union[List, Dict]] = None, - description: Optional[str] = None) -> Optional[str]: + async def update_test_item( + self, + item_uuid: str, + attributes: Optional[Union[List, Dict]] = None, + description: Optional[str] = None + ) -> Optional[str]: + """Update existing Test Item at the ReportPortal. + + :param item_uuid: Test Item UUID returned on the item start. + :param attributes: Test Item attributes: [{'key': 'k_name', 'value': 'k_value'}, ...]. + :param description: Test Item description. + :return: Response message or None. + """ return await self.__client.update_test_item(item_uuid, attributes=attributes, description=description) def _add_current_item(self, item: str) -> None: @@ -745,6 +811,10 @@ def current_item(self) -> Optional[str]: return self._item_stack.last() async def get_launch_info(self) -> Optional[dict]: + """Get current Launch information. + + :return: Launch information in dictionary. + """ if not self.launch_uuid: return {} return await self.__client.get_launch_info(self.launch_uuid) @@ -807,6 +877,8 @@ def clone(self) -> 'AsyncRPClient': class _RPClient(RP, metaclass=AbstractBaseClass): + """Base class for different synchronous to asynchronous client implementations.""" + __metaclass__ = AbstractBaseClass _item_stack: LifoQueue @@ -930,6 +1002,17 @@ def start_launch(self, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Task[str]: + """Start a new Launch with the given arguments. + + :param name: Launch name. + :param start_time: Launch start time. + :param description: Launch description. + :param attributes: Launch attributes. + :param rerun: Start launch in rerun mode. + :param rerun_of: For rerun mode specifies which launch will be re-run. Should be used with the + 'rerun' option. + :return: Launch UUID if successfully started or None. + """ if not self.use_own_launch: return self.launch_uuid launch_uuid_coro = self.__client.start_launch(name, start_time, description=description, @@ -942,7 +1025,6 @@ def start_test_item(self, name: str, start_time: str, item_type: str, - *, description: Optional[str] = None, attributes: Optional[List[Dict]] = None, parameters: Optional[Dict] = None, @@ -952,7 +1034,24 @@ def start_test_item(self, retry: bool = False, test_case_id: Optional[str] = None, **kwargs: Any) -> Task[str]: + """Start Test Case/Suite/Step/Nested Step Item. + :param name: Name of the Test Item. + :param start_time: The Item start time. + :param item_type: Type of the Test Item. Allowed values: + "suite", "story", "test", "scenario", "step", "before_class", "before_groups", + "before_method", "before_suite", "before_test", "after_class", "after_groups", + "after_method", "after_suite", "after_test". + :param description: The Item description. + :param attributes: Test Item attributes. + :param parameters: Set of parameters (for parametrized Test Items). + :param parent_item_id: A UUID of a parent SUITE / STEP. + :param has_stats: Set to False if test item is a Nested Step. + :param code_ref: Physical location of the Test Item. + :param retry: Used to report retry of the test. Allowed values: "True" or "False". + :param test_case_id: A unique ID of the current Step. + :return: Test Item UUID if successfully started or None. + """ item_id_coro = self.__client.start_test_item(self.launch_uuid, name, start_time, item_type, description=description, attributes=attributes, parameters=parameters, parent_item_id=parent_item_id, @@ -965,13 +1064,25 @@ def start_test_item(self, def finish_test_item(self, item_id: Task[str], end_time: str, - *, status: str = None, issue: Optional[Issue] = None, attributes: Optional[Union[List, Dict]] = None, description: str = None, retry: bool = False, **kwargs: Any) -> Task[str]: + """Finish Test Suite/Case/Step/Nested Step Item. + + :param item_id: ID of the Test Item. + :param end_time: The Item end time. + :param status: Test status. Allowed values: + PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED, INFO, WARN or None. + :param issue: Issue which will be attached to the current Item. + :param attributes: Test Item attributes(tags). Pairs of key and value. These attributes override + attributes on start Test Item call. + :param description: Test Item description. Overrides description from start request. + :param retry: Used to report retry of the test. Allowed values: "True" or "False". + :return: Response message. + """ result_coro = self.__client.finish_test_item(self.launch_uuid, item_id, end_time, status=status, issue=issue, attributes=attributes, description=description, @@ -985,6 +1096,14 @@ def finish_launch(self, status: str = None, attributes: Optional[Union[List, Dict]] = None, **kwargs: Any) -> Task[str]: + """Finish a Launch. + + :param end_time: Launch end time. + :param status: Launch status. Can be one of the followings: + PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED. + :param attributes: Launch attributes. These attributes override attributes on Start Launch call. + :return: Response message or None. + """ self.create_task(self.__client.log_batch(self._log_batcher.flush())) if self.use_own_launch: result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, @@ -1000,12 +1119,23 @@ def update_test_item(self, item_uuid: Task[str], attributes: Optional[Union[List, Dict]] = None, description: Optional[str] = None) -> Task: + """Update existing Test Item at the ReportPortal. + + :param item_uuid: Test Item UUID returned on the item start. + :param attributes: Test Item attributes: [{'key': 'k_name', 'value': 'k_value'}, ...]. + :param description: Test Item description. + :return: Response message or None. + """ result_coro = self.__client.update_test_item(item_uuid, attributes=attributes, description=description) result_task = self.create_task(result_coro) return result_task def get_launch_info(self) -> Task[dict]: + """Get current Launch information. + + :return: Launch information in dictionary. + """ if not self.launch_uuid: return self.create_task(self.__empty_dict()) result_coro = self.__client.get_launch_info(self.launch_uuid) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index d9ef410d..87be98bb 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -110,16 +110,16 @@ def start_launch(self, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Optional[str]: - """Start a new launch with the given parameters. + """Start a new Launch with the given arguments. - :param name: Launch name - :param start_time: Launch start time - :param description: Launch description - :param attributes: Launch attributes - :param rerun: Start launch in rerun mode - :param rerun_of: For rerun mode specifies which launch will be - re-run. Should be used with the 'rerun' option. - :return: Launch UUID + :param name: Launch name. + :param start_time: Launch start time. + :param description: Launch description. + :param attributes: Launch attributes. + :param rerun: Start launch in rerun mode. + :param rerun_of: For rerun mode specifies which launch will be re-run. Should be used with the + 'rerun' option. + :return: Launch UUID if successfully started or None. """ raise NotImplementedError('"start_launch" method is not implemented!') @@ -137,7 +137,7 @@ def start_test_item(self, retry: bool = False, test_case_id: Optional[str] = None, **kwargs: Any) -> Optional[str]: - """Start case/step/nested step item. + """Start Test Case/Suite/Step/Nested Step Item. :param name: Name of the Test Item. :param start_time: The Item start time. @@ -145,15 +145,15 @@ def start_test_item(self, "suite", "story", "test", "scenario", "step", "before_class", "before_groups", "before_method", "before_suite", "before_test", "after_class", "after_groups", "after_method", "after_suite", "after_test". - :param description: The item description - :param attributes: Test item attributes - :param parameters: Set of parameters (for parametrized test items) - :param parent_item_id: An ID of a parent SUITE / STEP - :param has_stats: Set to False if test item is nested step - :param code_ref: Physical location of the test item - :param retry: Used to report retry of the test. Allowed values: "True" or "False" - :param test_case_id: A unique ID of the current step - :return: Test Item UUID + :param description: The Item description. + :param attributes: Test Item attributes. + :param parameters: Set of parameters (for parametrized Test Items). + :param parent_item_id: A UUID of a parent SUITE / STEP. + :param has_stats: Set to False if test item is a Nested Step. + :param code_ref: Physical location of the Test Item. + :param retry: Used to report retry of the test. Allowed values: "True" or "False". + :param test_case_id: A unique ID of the current Step. + :return: Test Item UUID if successfully started or None. """ raise NotImplementedError('"start_test_item" method is not implemented!') @@ -167,18 +167,18 @@ def finish_test_item(self, description: str = None, retry: bool = False, **kwargs: Any) -> Optional[str]: - """Finish suite/case/step/nested step item. + """Finish Test Suite/Case/Step/Nested Step Item. :param item_id: ID of the Test Item. :param end_time: The Item end time. :param status: Test status. Allowed values: PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED, INFO, WARN or None. :param issue: Issue which will be attached to the current Item. - :param attributes: Test Item attributes(tags). Pairs of key and value. These are override attributes - on start Test Item call. + :param attributes: Test Item attributes(tags). Pairs of key and value. These attributes override + attributes on start Test Item call. :param description: Test Item description. Overrides description from start request. :param retry: Used to report retry of the test. Allowed values: "True" or "False". - :return: Response message + :return: Response message. """ raise NotImplementedError('"finish_test_item" method is not implemented!') @@ -188,13 +188,13 @@ def finish_launch(self, status: str = None, attributes: Optional[Union[list, dict]] = None, **kwargs: Any) -> Optional[str]: - """Finish current launch. + """Finish a Launch. - :param end_time: Launch end time. - :param status: Launch status. Can be one of the followings: - PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED. - :param attributes: Launch attributes. These attributes override attributes on Start Launch call. - :return: Response message. + :param end_time: Launch end time. + :param status: Launch status. Can be one of the followings: + PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED. + :param attributes: Launch attributes. These attributes override attributes on Start Launch call. + :return: Response message or None. """ raise NotImplementedError('"finish_launch" method is not implemented!') @@ -208,15 +208,15 @@ def update_test_item(self, :param item_uuid: Test Item UUID returned on the item start. :param attributes: Test Item attributes: [{'key': 'k_name', 'value': 'k_value'}, ...]. :param description: Test Item description. - :return: Response message. + :return: Response message or None. """ raise NotImplementedError('"update_test_item" method is not implemented!') @abstractmethod def get_launch_info(self) -> Optional[dict]: - """Get the current launch information. + """Get current Launch information. - :return: Launch information in dictionary + :return: Launch information in dictionary. """ raise NotImplementedError('"get_launch_info" method is not implemented!') @@ -304,10 +304,9 @@ def clone(self) -> 'RP': class RPClient(RP): """ReportPortal client. - The class is supposed to use by ReportPortal agents: both custom and - official to make calls to ReportPortal. It handles HTTP request and - response bodies generation and serialization, connection retries and log - batching. + The class is supposed to use by ReportPortal agents: both custom and official, to make calls to + ReportPortal. It handles HTTP request and response bodies generation and serialization, connection retries + and log batching. """ api_v1: str @@ -489,16 +488,16 @@ def start_launch(self, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Optional[str]: - """Start a new launch with the given arguments. + """Start a new Launch with the given arguments. - :param name: Launch name - :param start_time: Launch start time - :param description: Launch description - :param attributes: Launch attributes - :param rerun: Start launch in rerun mode - :param rerun_of: For rerun mode specifies which launch will be - re-run. Should be used with the 'rerun' option. - :return: Launch UUID if successfully started or None + :param name: Launch name. + :param start_time: Launch start time. + :param description: Launch description. + :param attributes: Launch attributes. + :param rerun: Start launch in rerun mode. + :param rerun_of: For rerun mode specifies which launch will be re-run. Should be used with the + 'rerun' option. + :return: Launch UUID if successfully started or None. """ if not self.use_own_launch: return self.launch_uuid @@ -539,25 +538,23 @@ def start_test_item(self, retry: bool = False, test_case_id: Optional[str] = None, **_: Any) -> Optional[str]: - """Start case/step/nested step item. - - :param name: Name of the test item - :param start_time: The item start time - :param item_type: Type of the test item. Allowable values: - "suite", "story", "test", "scenario", "step", - "before_class", "before_groups", - "before_method", "before_suite", - "before_test", "after_class", "after_groups", - "after_method", "after_suite", "after_test" - :param attributes: Test item attributes - :param code_ref: Physical location of the test item - :param description: The item description - :param has_stats: Set to False if test item is nested step - :param parameters: Set of parameters (for parametrized test items) - :param parent_item_id: An ID of a parent SUITE / STEP - :param retry: Used to report retry of the test. Allowable - values: "True" or "False" - :param test_case_id: A unique ID of the current step + """Start Test Case/Suite/Step/Nested Step Item. + + :param name: Name of the Test Item. + :param start_time: The Item start time. + :param item_type: Type of the Test Item. Allowed values: + "suite", "story", "test", "scenario", "step", "before_class", "before_groups", + "before_method", "before_suite", "before_test", "after_class", "after_groups", + "after_method", "after_suite", "after_test". + :param description: The Item description. + :param attributes: Test Item attributes. + :param parameters: Set of parameters (for parametrized Test Items). + :param parent_item_id: A UUID of a parent SUITE / STEP. + :param has_stats: Set to False if test item is a Nested Step. + :param code_ref: Physical location of the Test Item. + :param retry: Used to report retry of the test. Allowed values: "True" or "False". + :param test_case_id: A unique ID of the current Step. + :return: Test Item UUID if successfully started or None. """ if parent_item_id is NOT_FOUND: logger.warning('Attempt to start item for non-existent parent item.') @@ -604,20 +601,18 @@ def finish_test_item(self, description: str = None, retry: bool = False, **kwargs: Any) -> Optional[str]: - """Finish suite/case/step/nested step item. - - :param item_id: ID of the test item - :param end_time: The item end time - :param status: Test status. Allowable values: "passed", - "failed", "stopped", "skipped", "interrupted", - "cancelled" or None - :param attributes: Test item attributes(tags). Pairs of key and value. - Override attributes on start - :param description: Test item description. Overrides description - from start request. - :param issue: Issue of the current test item - :param retry: Used to report retry of the test. Allowable values: - "True" or "False" + """Finish Test Suite/Case/Step/Nested Step Item. + + :param item_id: ID of the Test Item. + :param end_time: The Item end time. + :param status: Test status. Allowed values: + PASSED, FAILED, STOPPED, SKIPPED, INTERRUPTED, CANCELLED, INFO, WARN or None. + :param issue: Issue which will be attached to the current Item. + :param attributes: Test Item attributes(tags). Pairs of key and value. These attributes override + attributes on start Test Item call. + :param description: Test Item description. Overrides description from start request. + :param retry: Used to report retry of the test. Allowed values: "True" or "False". + :return: Response message. """ if item_id is NOT_FOUND or not item_id: logger.warning('Attempt to finish non-existent item') @@ -679,12 +674,12 @@ def finish_launch(self, def update_test_item(self, item_uuid: str, attributes: Optional[Union[list, dict]] = None, description: Optional[str] = None) -> Optional[str]: - """Update existing test item at the ReportPortal. + """Update existing Test Item at the ReportPortal. - :param str item_uuid: Test item UUID returned on the item start - :param str description: Test item description - :param list attributes: Test item attributes - [{'key': 'k_name', 'value': 'k_value'}, ...] + :param item_uuid: Test Item UUID returned on the item start. + :param attributes: Test Item attributes: [{'key': 'k_name', 'value': 'k_value'}, ...]. + :param description: Test Item description. + :return: Response message or None. """ data = { 'description': description, @@ -735,9 +730,9 @@ def get_item_id_by_uuid(self, item_uuid: str) -> Optional[str]: return response.id if response else None def get_launch_info(self) -> Optional[dict]: - """Get the current launch information. + """Get current Launch information. - :return: Launch information in dictionary + :return: Launch information in dictionary. """ if self.launch_uuid is None: return {} From 3ca6e9756d4c532138113ca12af25e1a9e6f3f35 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 28 Sep 2023 15:14:51 +0300 Subject: [PATCH 163/268] Update pydocs --- reportportal_client/aio/client.py | 133 +++++++++++++++++++++++------- reportportal_client/client.py | 86 +++++++++++-------- 2 files changed, 156 insertions(+), 63 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 576032b6..1986e1a2 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -485,7 +485,7 @@ async def get_launch_ui_id(self, launch_uuid_future: Union[str, Task[str]]) -> O return launch_info.get('id') if launch_info else None async def get_launch_ui_url(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[str]: - """Get full quality URL of the given launch. + """Get full quality URL of the given Launch. :param launch_uuid_future: Str or Task UUID returned on the Launch start. :return: Launch URL string. @@ -509,7 +509,7 @@ async def get_launch_ui_url(self, launch_uuid_future: Union[str, Task[str]]) -> return url async def get_project_settings(self) -> Optional[Dict]: - """Get settings of the current project. + """Get settings of the current Project. :return: Settings response in Dictionary. """ @@ -807,7 +807,10 @@ def _remove_current_item(self) -> Optional[str]: return self._item_stack.get() def current_item(self) -> Optional[str]: - """Retrieve the last item reported by the client.""" + """Retrieve the last Item reported by the client (based on the internal FILO queue). + + :return: Item UUID string. + """ return self._item_stack.last() async def get_launch_info(self) -> Optional[dict]: @@ -820,31 +823,57 @@ async def get_launch_info(self) -> Optional[dict]: return await self.__client.get_launch_info(self.launch_uuid) async def get_item_id_by_uuid(self, item_uuid: str) -> Optional[str]: + """Get Test Item ID by the given Item UUID. + + :param item_uuid: String UUID returned on the Item start. + :return: Test Item ID. + """ return await self.__client.get_item_id_by_uuid(item_uuid) async def get_launch_ui_id(self) -> Optional[int]: + """Get Launch ID of the current Launch. + + :return: Launch ID of the Launch. None if not found. + """ if not self.launch_uuid: return return await self.__client.get_launch_ui_id(self.launch_uuid) async def get_launch_ui_url(self) -> Optional[str]: + """Get full quality URL of the current Launch. + + :return: Launch URL string. + """ if not self.launch_uuid: return return await self.__client.get_launch_ui_url(self.launch_uuid) async def get_project_settings(self) -> Optional[Dict]: - return await self.__client.get_project_settings() + """Get settings of the current Project. - async def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, - item_id: Optional[str] = None) -> Optional[Tuple[str, ...]]: - """Log message. Can be added to test item in any state. + :return: Settings response in Dictionary. + """ + return await self.__client.get_project_settings() - :param time: Log time - :param message: Log message - :param level: Log level - :param attachment: Attachments(images,files,etc.) - :param item_id: Parent item UUID + async def log( + self, + time: str, + message: str, + level: Optional[Union[int, str]] = None, + attachment: Optional[Dict] = None, + item_id: Optional[str] = None + ) -> Optional[Tuple[str, ...]]: + """Send Log message to the ReportPortal and attach it to a Test Item or Launch. + + This method stores Log messages in internal batch and sent it when batch is full, so not every method + call will return any response. + + :param time: Time in UTC. + :param message: Log message text. + :param level: Message's Log level. + :param attachment: Message's attachments(images,files,etc.). + :param item_id: UUID of the ReportPortal Item the message belongs to. + :return: Response message Tuple if Log message batch was sent or None. """ if item_id is NOT_FOUND: logger.warning("Attempt to log to non-existent item") @@ -854,10 +883,10 @@ async def log(self, time: str, message: str, level: Optional[Union[int, str]] = return await self.__client.log_batch(await self._log_batcher.append_async(rp_log)) def clone(self) -> 'AsyncRPClient': - """Clone the client object, set current Item ID as cloned item ID. + """Clone the Client object, set current Item ID as cloned Item ID. :return: Cloned client object - :rtype: AsyncRPClient + :rtype: AsyncRPClient. """ cloned_client = self.__client.clone() # noinspection PyTypeChecker @@ -967,22 +996,37 @@ def __init__( @abstractmethod def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: + """Create a Task from given Coroutine. + + :param coro: Coroutine which will be used for the Task creation. + :return: Task instance. + """ raise NotImplementedError('"create_task" method is not implemented!') @abstractmethod def finish_tasks(self) -> None: + """Ensure all pending Tasks are finished, block current Thread if necessary.""" raise NotImplementedError('"create_task" method is not implemented!') def _add_current_item(self, item: Task[_T]) -> None: - """Add the last item from the self._items queue.""" + """Add the last Item to the internal FILO queue. + + :param item: Future Task of the Item UUID. + """ self._item_stack.put(item) def _remove_current_item(self) -> Task[_T]: - """Remove the last item from the self._items queue.""" + """Remove the last Item from the internal FILO queue. + + :return: Future Task of the Item UUID. + """ return self._item_stack.get() def current_item(self) -> Task[_T]: - """Retrieve the last item reported by the client.""" + """Retrieve the last Item reported by the client (based on the internal FILO queue). + + :return: Future Task of the Item UUID. + """ return self._item_stack.last() async def __empty_str(self): @@ -1143,11 +1187,20 @@ def get_launch_info(self) -> Task[dict]: return result_task def get_item_id_by_uuid(self, item_uuid_future: Task[str]) -> Task[str]: + """Get Test Item ID by the given Item UUID. + + :param item_uuid_future: Str or Task UUID returned on the Item start. + :return: Test Item ID. + """ result_coro = self.__client.get_item_id_by_uuid(item_uuid_future) result_task = self.create_task(result_coro) return result_task def get_launch_ui_id(self) -> Task[int]: + """Get Launch ID of the current Launch. + + :return: Launch ID of the Launch. None if not found. + """ if not self.launch_uuid: return self.create_task(self.__int_value()) result_coro = self.__client.get_launch_ui_id(self.launch_uuid) @@ -1155,6 +1208,10 @@ def get_launch_ui_id(self) -> Task[int]: return result_task def get_launch_ui_url(self) -> Task[str]: + """Get full quality URL of the current Launch. + + :return: Launch URL string. + """ if not self.launch_uuid: return self.create_task(self.__empty_str()) result_coro = self.__client.get_launch_ui_url(self.launch_uuid) @@ -1162,6 +1219,10 @@ def get_launch_ui_url(self) -> Task[str]: return result_task def get_project_settings(self) -> Task[dict]: + """Get settings of the current Project. + + :return: Settings response in Dictionary. + """ result_coro = self.__client.get_project_settings() result_task = self.create_task(result_coro) return result_task @@ -1174,13 +1235,17 @@ async def _log(self, log_rq: AsyncRPRequestLog) -> Optional[Tuple[str, ...]]: def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[Dict] = None, item_id: Optional[Task[str]] = None) -> None: - """Log message. Can be added to test item in any state. + """Send Log message to the ReportPortal and attach it to a Test Item or Launch. + + This method stores Log messages in internal batch and sent it when batch is full, so not every method + call will return any response. - :param time: Log time - :param message: Log message - :param level: Log level - :param attachment: Attachments(images,files,etc.) - :param item_id: Parent item UUID + :param time: Time in UTC. + :param message: Log message text. + :param level: Message's Log level. + :param attachment: Message's attachments(images,files,etc.). + :param item_id: UUID of the ReportPortal Item the message belongs to. + :return: Response message Tuple if Log message batch was sent or None. """ if item_id is NOT_FOUND: logger.warning("Attempt to log to non-existent item") @@ -1234,6 +1299,11 @@ def __heartbeat(self): self._loop.call_at(self._loop.time() + 0.1, self.__heartbeat) def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: + """Create a Task from given Coroutine. + + :param coro: Coroutine which will be used for the Task creation. + :return: Task instance. + """ if not getattr(self, '_loop', None): return result = self._loop.create_task(coro) @@ -1242,6 +1312,7 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: return result def finish_tasks(self): + """Ensure all pending Tasks are finished, block current Thread if necessary.""" shutdown_start_time = datetime.time() with self.__task_mutex: tasks = self.__task_list.flush() @@ -1255,9 +1326,9 @@ def finish_tasks(self): self._loop.create_task(self._close()).blocking_result() def clone(self) -> 'ThreadedRPClient': - """Clone the client object, set current Item ID as cloned item ID. + """Clone the Client object, set current Item ID as cloned Item ID. - :return: Cloned client object + :return: Cloned client object. :rtype: ThreadedRPClient """ cloned_client = self.client.clone() @@ -1314,6 +1385,11 @@ def __init__( self.__trigger_interval = trigger_interval def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: + """Create a Task from given Coroutine. + + :param coro: Coroutine which will be used for the Task creation. + :return: Task instance. + """ if not getattr(self, '_loop', None): return result = self._loop.create_task(coro) @@ -1324,6 +1400,7 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: return result def finish_tasks(self) -> None: + """Ensure all pending Tasks are finished, block current Thread if necessary.""" with self.__task_mutex: tasks = self.__task_list.flush() if tasks: @@ -1335,9 +1412,9 @@ def finish_tasks(self) -> None: self._loop.run_until_complete(self._close()) def clone(self) -> 'BatchedRPClient': - """Clone the client object, set current Item ID as cloned item ID. + """Clone the Client object, set current Item ID as cloned Item ID. - :return: Cloned client object + :return: Cloned client object. :rtype: BatchedRPClient """ cloned_client = self.client.clone() diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 87be98bb..7d66ed33 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -222,16 +222,16 @@ def get_launch_info(self) -> Optional[dict]: @abstractmethod def get_item_id_by_uuid(self, item_uuid: str) -> Optional[str]: - """Get Test Item ID by the given UUID. + """Get Test Item ID by the given Item UUID. - :param item_uuid: UUID returned on the Item start. + :param item_uuid: String UUID returned on the Item start. :return: Test Item ID. """ raise NotImplementedError('"get_item_id_by_uuid" method is not implemented!') @abstractmethod def get_launch_ui_id(self) -> Optional[int]: - """Get Launch ID of the current launch. + """Get Launch ID of the current Launch. :return: Launch ID of the Launch. None if not found. """ @@ -239,7 +239,7 @@ def get_launch_ui_id(self) -> Optional[int]: @abstractmethod def get_launch_ui_url(self) -> Optional[str]: - """Get full quality URL of the current launch. + """Get full quality URL of the current Launch. :return: Launch URL string. """ @@ -247,7 +247,7 @@ def get_launch_ui_url(self) -> Optional[str]: @abstractmethod def get_project_settings(self) -> Optional[dict]: - """Get settings of the current project. + """Get settings of the current Project. :return: Settings response in Dictionary. """ @@ -255,17 +255,21 @@ def get_project_settings(self) -> Optional[dict]: @abstractmethod def log(self, - datetime: str, message: str, + time: str, message: str, level: Optional[Union[int, str]] = None, attachment: Optional[dict] = None, - item_id: Optional[str] = None) -> None: - """Send log message to the ReportPortal. + item_id: Optional[str] = None) -> Optional[Tuple[str, ...]]: + """Send Log message to the ReportPortal and attach it to a Test Item or Launch. - :param datetime: Time in UTC. + This method stores Log messages in internal batch and sent it when batch is full, so not every method + call will return any response. + + :param time: Time in UTC. :param message: Log message text. - :param level: Message's log level. - :param attachment: Message's attachments. + :param level: Message's Log level. + :param attachment: Message's attachments(images,files,etc.). :param item_id: UUID of the ReportPortal Item the message belongs to. + :return: Response message Tuple if Log message batch was sent or None. """ raise NotImplementedError('"log" method is not implemented!') @@ -289,14 +293,18 @@ def terminate(self, *_: Any, **__: Any) -> None: @abstractmethod def current_item(self) -> Optional[str]: - """Retrieve the last item reported by the client.""" + """Retrieve the last Item reported by the client (based on the internal FILO queue). + + :return: Item UUID string. + """ raise NotImplementedError('"current_item" method is not implemented!') @abstractmethod def clone(self) -> 'RP': - """Clone the client object, set current Item ID as cloned root Item ID. + """Clone the Client object, set current Item ID as cloned Item ID. - :return: Cloned client object + :return: Cloned client object. + :rtype: RP """ raise NotImplementedError('"clone" method is not implemented!') @@ -701,15 +709,23 @@ def _log(self, batch: Optional[List[RPRequestLog]]) -> Optional[Tuple[str, ...]] verify_ssl=self.verify_ssl).make() return response.messages - def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[dict] = None, item_id: Optional[str] = None) -> Optional[Tuple[str, ...]]: - """Send log message to the ReportPortal. + def log(self, + time: str, + message: str, + level: Optional[Union[int, str]] = None, + attachment: Optional[dict] = None, + item_id: Optional[str] = None) -> Optional[Tuple[str, ...]]: + """Send Log message to the ReportPortal and attach it to a Test Item or Launch. + + This method stores Log messages in internal batch and sent it when batch is full, so not every method + call will return any response. - :param time: Time in UTC - :param message: Log message text - :param level: Message's log level - :param attachment: Message's attachments - :param item_id: ID of the RP item the message belongs to + :param time: Time in UTC. + :param message: Log message text. + :param level: Message's Log level. + :param attachment: Message's attachments(images,files,etc.). + :param item_id: UUID of the ReportPortal Item the message belongs to. + :return: Response message Tuple if Log message batch was sent or None. """ if item_id is NOT_FOUND: logger.warning("Attempt to log to non-existent item") @@ -719,10 +735,10 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, return self._log(self._log_batcher.append(rp_log)) def get_item_id_by_uuid(self, item_uuid: str) -> Optional[str]: - """Get test item ID by the given UUID. + """Get Test Item ID by the given Item UUID. - :param item_uuid: UUID returned on the item start - :return: Test item ID + :param item_uuid: String UUID returned on the Item start. + :return: Test Item ID. """ url = uri_join(self.base_url_v1, 'item', 'uuid', item_uuid) response = HttpRequest(self.session.get, url=url, @@ -753,17 +769,17 @@ def get_launch_info(self) -> Optional[dict]: return launch_info def get_launch_ui_id(self) -> Optional[int]: - """Get UI ID of the current launch. + """Get Launch ID of the current Launch. - :return: UI ID of the given launch. None if UI ID has not been found. + :return: Launch ID of the Launch. None if not found. """ launch_info = self.get_launch_info() return launch_info.get('id') if launch_info else None def get_launch_ui_url(self) -> Optional[str]: - """Get UI URL of the current launch. + """Get full quality URL of the current Launch. - :return: launch URL or all launches URL. + :return: Launch URL string. """ launch_info = self.get_launch_info() ui_id = launch_info.get('id') if launch_info else None @@ -783,9 +799,9 @@ def get_launch_ui_url(self) -> Optional[str]: return url def get_project_settings(self) -> Optional[dict]: - """Get project settings. + """Get settings of the current Project. - :return: HTTP response in dictionary + :return: Settings response in Dictionary. """ url = uri_join(self.base_url_v1, 'settings') response = HttpRequest(self.session.get, url=url, @@ -807,16 +823,16 @@ def _remove_current_item(self) -> Optional[str]: return def current_item(self) -> Optional[str]: - """Retrieve the last item reported by the client. + """Retrieve the last item reported by the client (based on the internal FILO queue). - :return: Item UUID string + :return: Item UUID string. """ return self._item_stack.last() def clone(self) -> 'RPClient': - """Clone the client object, set current Item ID as cloned item ID. + """Clone the Client object, set current Item ID as cloned Item ID. - :return: Cloned client object + :return: Cloned client object. :rtype: RPClient """ cloned = RPClient( From ac76d445dd8c698775e098ea56157905892798a4 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 28 Sep 2023 17:35:45 +0300 Subject: [PATCH 164/268] Update pydocs --- reportportal_client/aio/__init__.py | 8 +- reportportal_client/aio/client.py | 200 ++++++++++++++++++++++++---- reportportal_client/aio/tasks.py | 4 +- 3 files changed, 178 insertions(+), 34 deletions(-) diff --git a/reportportal_client/aio/__init__.py b/reportportal_client/aio/__init__.py index 88060612..df490be6 100644 --- a/reportportal_client/aio/__init__.py +++ b/reportportal_client/aio/__init__.py @@ -13,17 +13,17 @@ """Common package for Asynchronous I/O clients and utilities.""" +from reportportal_client.aio.client import (ThreadedRPClient, BatchedRPClient, AsyncRPClient, + DEFAULT_TASK_TIMEOUT, DEFAULT_SHUTDOWN_TIMEOUT) from reportportal_client.aio.tasks import (Task, TriggerTaskBatcher, BatchedTask, BatchedTaskFactory, ThreadedTask, ThreadedTaskFactory, BlockingOperationError, - BackgroundTaskBatcher, DEFAULT_TASK_TRIGGER_NUM, + BackgroundTaskList, DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL) -from reportportal_client.aio.client import (ThreadedRPClient, BatchedRPClient, AsyncRPClient, - DEFAULT_TASK_TIMEOUT, DEFAULT_SHUTDOWN_TIMEOUT) __all__ = [ 'Task', 'TriggerTaskBatcher', - 'BackgroundTaskBatcher', + 'BackgroundTaskList', 'BatchedTask', 'BatchedTaskFactory', 'ThreadedTask', diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 1986e1a2..0fc9938c 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -31,7 +31,7 @@ from reportportal_client._local import set_current from reportportal_client.aio.http import RetryingClientSession from reportportal_client.aio.tasks import (Task, BatchedTaskFactory, ThreadedTaskFactory, TriggerTaskBatcher, - BackgroundTaskBatcher, DEFAULT_TASK_TRIGGER_NUM, + BackgroundTaskList, DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL) from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, @@ -553,7 +553,7 @@ def clone(self) -> 'Client': class AsyncRPClient(RP): - """Stateful asynchronous ReportPortal Client. + """Asynchronous ReportPortal Client. This class implements common RP client interface but all its methods are async, so it capable to use in asynchronous ReportPortal agents. It handles HTTP request and response bodies generation and @@ -910,10 +910,10 @@ class _RPClient(RP, metaclass=AbstractBaseClass): __metaclass__ = AbstractBaseClass + log_batch_size: int + log_batch_payload_limit: int _item_stack: LifoQueue _log_batcher: LogBatcher - _shutdown_timeout: float - _task_timeout: float __client: Client __launch_uuid: Optional[Task[str]] __endpoint: str @@ -966,32 +966,61 @@ def __init__( endpoint: str, project: str, *, - launch_uuid: Optional[Task[str]] = None, client: Optional[Client] = None, + launch_uuid: Optional[Task[str]] = None, + log_batch_size: int = 20, + log_batch_payload_limit: int = MAX_LOG_BATCH_PAYLOAD_SIZE, log_batcher: Optional[LogBatcher] = None, - task_timeout: float = DEFAULT_TASK_TIMEOUT, - shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, **kwargs: Any ) -> None: + """Initialize the class instance with arguments. + + :param endpoint: Endpoint of the ReportPortal service. + :param project: Project name to report to. + :param api_key: Authorization API key. + :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the + server side. + :param verify_ssl: Option to skip ssl verification. + :param retries: Number of retry attempts to make in case of connection / server errors. + :param max_pool_size: Option to set the maximum number of connections to save the pool. + :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to + specific connect and read separately. + :param keepalive_timeout: Maximum amount of idle time in seconds before force connection closing. + :param mode: Launch mode, all Launches started by the client will be in that mode. + :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. + :param print_output: Set output stream for Launch UUID printing. + :param client: ReportPortal async Client instance to use. If set, all above arguments + will be ignored. + :param launch_uuid: A launch UUID to use instead of starting own one. + :param log_batch_size: Option to set the maximum number of logs that can be processed in one + batch. + :param log_batch_payload_limit: maximum size in bytes of logs that can be processed in one batch + :param log_batcher: ReportPortal log batcher instance to use. If set, 'log_batch' + arguments above will be ignored. + """ self.__endpoint = endpoint self.__project = project self.__step_reporter = StepReporter(self) self._item_stack = LifoQueue() - self._shutdown_timeout = shutdown_timeout - self._task_timeout = task_timeout + + self.log_batch_size = log_batch_size + self.log_batch_payload_limit = log_batch_payload_limit if log_batcher: self._log_batcher = log_batcher else: - self._log_batcher = LogBatcher() + self._log_batcher = LogBatcher(log_batch_size, log_batch_payload_limit) + if client: self.__client = client else: self.__client = Client(endpoint, project, **kwargs) + if launch_uuid: self.__launch_uuid = launch_uuid self.use_own_launch = False else: self.use_own_launch = True + set_current(self) @abstractmethod @@ -1260,8 +1289,17 @@ async def _close(self): class ThreadedRPClient(_RPClient): + """Synchronous-asynchronous ReportPortal Client which uses background Thread to execute async coroutines. + + This class implements common RP client interface, so it capable to use in synchronous ReportPortal Agents + if you want to achieve async performance level with synchronous code. It handles HTTP request and response + bodies generation and serialization, connection retries and log batching. + """ + + _task_timeout: float + _shutdown_timeout: float _loop: Optional[asyncio.AbstractEventLoop] - __task_list: BackgroundTaskBatcher[Task[_T]] + __task_list: BackgroundTaskList[Task[_T]] __task_mutex: threading.RLock __thread: Optional[threading.Thread] @@ -1270,17 +1308,59 @@ def __init__( endpoint: str, project: str, *, - launch_uuid: Optional[Task[str]] = None, - client: Optional[Client] = None, - log_batcher: Optional[LogBatcher] = None, - task_list: Optional[BackgroundTaskBatcher[Task[_T]]] = None, + task_timeout: float = DEFAULT_TASK_TIMEOUT, + shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, + task_list: Optional[BackgroundTaskList[Task[_T]]] = None, task_mutex: Optional[threading.RLock] = None, loop: Optional[asyncio.AbstractEventLoop] = None, **kwargs: Any ) -> None: - super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, - **kwargs) - self.__task_list = task_list or BackgroundTaskBatcher() + """Initialize the class instance with arguments. + + :param endpoint: Endpoint of the ReportPortal service. + :param project: Project name to report to. + :param api_key: Authorization API key. + :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the + server side. + :param verify_ssl: Option to skip ssl verification. + :param retries: Number of retry attempts to make in case of connection / server errors. + :param max_pool_size: Option to set the maximum number of connections to save the pool. + :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to + specific connect and read separately. + :param keepalive_timeout: Maximum amount of idle time in seconds before force connection closing. + :param mode: Launch mode, all Launches started by the client will be in that mode. + :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. + :param print_output: Set output stream for Launch UUID printing. + :param client: ReportPortal async Client instance to use. If set, all above arguments + will be ignored. + :param launch_uuid: A launch UUID to use instead of starting own one. + :param log_batch_size: Option to set the maximum number of logs that can be processed in one + batch. + :param log_batch_payload_limit: maximum size in bytes of logs that can be processed in one batch + :param log_batcher: ReportPortal log batcher instance to use. If set, 'log_batch' + arguments above will be ignored. + :param task_timeout: Time limit in seconds for a Task processing. + :param shutdown_timeout: Time limit in seconds for shutting down internal Tasks. + :param task_list: Thread-safe Task list to have one task storage for multiple Clients + which guarantees their processing on Launch finish. The Client creates + own Task list if this argument is None. + :param task_mutex: Mutex object which is responsible for synchronization of the passed + task_list. The Client creates own one if this argument is None. + :param loop: Event Loop which is used to process Tasks. The Client creates own one + if this argument is None. + """ + super().__init__(endpoint, project, **kwargs) + self._task_timeout = task_timeout + self._shutdown_timeout = shutdown_timeout + if task_list: + if not task_mutex: + warnings.warn( + '"task_list" argument is set, but not "task_mutex". This usually indicates ' + 'invalid use, since "task_mutex" is used to synchronize on "task_list".', + RuntimeWarning, + 2 + ) + self.__task_list = task_list or BackgroundTaskList() self.__task_mutex = task_mutex or threading.RLock() self.__thread = None if loop: @@ -1338,7 +1418,11 @@ def clone(self) -> 'ThreadedRPClient': project=None, launch_uuid=self.launch_uuid, client=cloned_client, + log_batch_size=self.log_batch_size, + log_batch_payload_limit=self.log_batch_payload_limit, log_batcher=self._log_batcher, + task_timeout=self._task_timeout, + shutdown_timeout=self._shutdown_timeout, task_mutex=self.__task_mutex, task_list=self.__task_list, loop=self._loop @@ -1350,6 +1434,15 @@ def clone(self) -> 'ThreadedRPClient': class BatchedRPClient(_RPClient): + """Synchronous-asynchronous ReportPortal Client which uses the same Thread to execute async coroutines. + + This class implements common RP client interface, so it capable to use in synchronous ReportPortal Agents + if you want to achieve async performance level with synchronous code. It handles HTTP request and response + bodies generation and serialization, connection retries and log batching. + """ + + _task_timeout: float + _shutdown_timeout: float _loop: asyncio.AbstractEventLoop __task_list: TriggerTaskBatcher[Task[_T]] __task_mutex: threading.RLock @@ -1358,12 +1451,12 @@ class BatchedRPClient(_RPClient): __trigger_interval: float def __init__( - self, endpoint: str, + self, + endpoint: str, project: str, *, - launch_uuid: Optional[Task[str]] = None, - client: Optional[Client] = None, - log_batcher: Optional[LogBatcher] = None, + task_timeout: float = DEFAULT_TASK_TIMEOUT, + shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, task_list: Optional[TriggerTaskBatcher] = None, task_mutex: Optional[threading.RLock] = None, loop: Optional[asyncio.AbstractEventLoop] = None, @@ -1371,9 +1464,56 @@ def __init__( trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL, **kwargs: Any ) -> None: - super().__init__(endpoint, project, launch_uuid=launch_uuid, client=client, log_batcher=log_batcher, - **kwargs) - self.__task_list = task_list or TriggerTaskBatcher() + """Initialize the class instance with arguments. + + :param endpoint: Endpoint of the ReportPortal service. + :param project: Project name to report to. + :param api_key: Authorization API key. + :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the + server side. + :param verify_ssl: Option to skip ssl verification. + :param retries: Number of retry attempts to make in case of connection / server errors. + :param max_pool_size: Option to set the maximum number of connections to save the pool. + :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to + specific connect and read separately. + :param keepalive_timeout: Maximum amount of idle time in seconds before force connection closing. + :param mode: Launch mode, all Launches started by the client will be in that mode. + :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. + :param print_output: Set output stream for Launch UUID printing. + :param client: ReportPortal async Client instance to use. If set, all above arguments + will be ignored. + :param launch_uuid: A launch UUID to use instead of starting own one. + :param log_batch_size: Option to set the maximum number of logs that can be processed in one + batch. + :param log_batch_payload_limit: maximum size in bytes of logs that can be processed in one batch + :param log_batcher: ReportPortal log batcher instance to use. If set, 'log_batch' + arguments above will be ignored. + :param task_timeout: Time limit in seconds for a Task processing. + :param shutdown_timeout: Time limit in seconds for shutting down internal Tasks. + :param task_list: Batching Task list to have one task storage for multiple Clients + which guarantees their processing on Launch finish. The Client creates + own Task list if this argument is None. + :param task_mutex: Mutex object which is responsible for synchronization of the passed + task_list. The Client creates own one if this argument is None. + :param loop: Event Loop which is used to process Tasks. The Client creates own one + if this argument is None. + :param trigger_num: Number of tasks which triggers Task batch execution. + :param trigger_interval: Time limit which triggers Task batch execution. + """ + super().__init__(endpoint, project, **kwargs) + self._task_timeout = task_timeout + self._shutdown_timeout = shutdown_timeout + self.__trigger_num = trigger_num + self.__trigger_interval = trigger_interval + if task_list: + if not task_mutex: + warnings.warn( + '"task_list" argument is set, but not "task_mutex". This usually indicates ' + 'invalid use, since "task_mutex" is used to synchronize on "task_list".', + RuntimeWarning, + 2 + ) + self.__task_list = task_list or TriggerTaskBatcher(trigger_num, trigger_interval) self.__task_mutex = task_mutex or threading.RLock() self.__last_run_time = datetime.time() if loop: @@ -1381,8 +1521,6 @@ def __init__( else: self._loop = asyncio.new_event_loop() self._loop.set_task_factory(BatchedTaskFactory()) - self.__trigger_num = trigger_num - self.__trigger_interval = trigger_interval def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: """Create a Task from given Coroutine. @@ -1424,10 +1562,16 @@ def clone(self) -> 'BatchedRPClient': project=None, launch_uuid=self.launch_uuid, client=cloned_client, + log_batch_size=self.log_batch_size, + log_batch_payload_limit=self.log_batch_payload_limit, log_batcher=self._log_batcher, + task_timeout=self._task_timeout, + shutdown_timeout=self._shutdown_timeout, task_list=self.__task_list, task_mutex=self.__task_mutex, - loop=self._loop + loop=self._loop, + trigger_num=self.__trigger_num, + trigger_interval=self.__trigger_interval ) current_item = self.current_item() if current_item: diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index 3f81552d..a173b45c 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -253,8 +253,8 @@ def flush(self) -> Optional[List[_T]]: return tasks -class BackgroundTaskBatcher(Generic[_T]): - """Batching class which collects Tasks into internal batch and removes when they complete.""" +class BackgroundTaskList(Generic[_T]): + """Task list class which collects Tasks into internal batch and removes when they complete.""" __task_list: List[_T] From bdb81c45723def9c4727f66b09ab1ee6a29b4186 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 28 Sep 2023 20:14:40 +0300 Subject: [PATCH 165/268] Fix session config --- reportportal_client/aio/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 0fc9938c..b07d498e 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -188,8 +188,8 @@ def session(self) -> aiohttp.ClientSession: headers['Authorization'] = f'Bearer {self.api_key}' session_params = { - headers: headers, - connector: connector + 'headers': headers, + 'connector': connector } if self.http_timeout: From 22418a48752a1da0a1be3758e99348bdfb66b53d Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 28 Sep 2023 22:35:14 +0300 Subject: [PATCH 166/268] Add more tests --- tests/aio/test_http.py | 94 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 11 deletions(-) diff --git a/tests/aio/test_http.py b/tests/aio/test_http.py index 96300c0d..6b72d347 100644 --- a/tests/aio/test_http.py +++ b/tests/aio/test_http.py @@ -25,8 +25,16 @@ HTTP_TIMEOUT_TIME = 1.2 -class TimeoutHttpHandler(http.server.BaseHTTPRequestHandler): +class OkHttpHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write('{}\n\n'.encode("utf-8")) + self.wfile.flush() + +class TimeoutHttpHandler(http.server.BaseHTTPRequestHandler): def do_GET(self): time.sleep(HTTP_TIMEOUT_TIME) self.send_response(200) @@ -36,6 +44,27 @@ def do_GET(self): self.wfile.flush() +class ResetHttpHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.wfile.close() + + +class ErrorHttpHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(500, 'Internal Server Error') + self.end_headers() + self.wfile.write('Internal Server Error\n\n'.encode("utf-8")) + self.wfile.flush() + + +class ThrottlingHttpHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(429, 'Throttling') + self.end_headers() + self.wfile.write('Throttling\n\n'.encode("utf-8")) + self.wfile.flush() + + SERVER_PORT = 8000 SERVER_ADDRESS = ('', SERVER_PORT) SERVER_CLASS = socketserver.TCPServer @@ -52,26 +81,69 @@ def get_http_server(server_class=SERVER_CLASS, server_address=SERVER_ADDRESS, @pytest.mark.skipif(sys.version_info < (3, 8), reason="the test requires Python 3.8 or higher") +@pytest.mark.parametrize( + 'server_class, port, expected_delay, timeout_seconds, is_exception', + [ + (TimeoutHttpHandler, 8001, 14.4, 1.0, True), + (ResetHttpHandler, 8002, 8.4, 1.0, True), + (ErrorHttpHandler, 8003, 1.1, 1.0, False), + (ThrottlingHttpHandler, 8004, 27.93, 1.0, False), + ] +) +@pytest.mark.asyncio +async def test_retry_on_request_error(server_class, port, expected_delay, timeout_seconds, is_exception): + retry_number = 5 + timeout = aiohttp.ClientTimeout(connect=timeout_seconds, sock_read=timeout_seconds) + connector = aiohttp.TCPConnector(force_close=True) + session = RetryingClientSession(f'http://localhost:{port}', timeout=timeout, + max_retry_number=retry_number, base_retry_delay=0.01, connector=connector) + parent_request = super(type(session), session)._request + async_mock = mock.AsyncMock() + async_mock.side_effect = parent_request + exception = None + result = None + with get_http_server(server_handler=server_class, server_address=('', port)): + with mock.patch('reportportal_client.aio.http.ClientSession._request', async_mock): + async with session: + start_time = time.time() + try: + result = await session.get('/') + except Exception as exc: + exception = exc + total_time = time.time() - start_time + if is_exception: + assert exception is not None + else: + assert exception is None + assert not result.ok + assert async_mock.call_count == 1 + retry_number + assert total_time > expected_delay + assert total_time < expected_delay * 1.5 + + @pytest.mark.asyncio -async def test_retry_on_request_timeout(): +async def test_no_retry_on_ok_request(): + retry_number = 5 + port = 8000 timeout = aiohttp.ClientTimeout(connect=1, sock_read=1) - session = RetryingClientSession('http://localhost:8000', timeout=timeout, max_retry_number=5, - base_retry_delay=0.01) + connector = aiohttp.TCPConnector(force_close=True) + session = RetryingClientSession(f'http://localhost:{port}', timeout=timeout, + max_retry_number=retry_number, base_retry_delay=0.01, connector=connector) parent_request = super(type(session), session)._request async_mock = mock.AsyncMock() async_mock.side_effect = parent_request exception = None - with get_http_server(server_handler=TimeoutHttpHandler): + result = None + with get_http_server(server_handler=OkHttpHandler, server_address=('', port)): with mock.patch('reportportal_client.aio.http.ClientSession._request', async_mock): async with session: start_time = time.time() try: - await session.get('/') + result = await session.get('/') except Exception as exc: exception = exc total_time = time.time() - start_time - retries_and_delays = 6 + 0.02 + 0.4 + 8 - assert exception is not None - assert async_mock.call_count == 6 - assert total_time > retries_and_delays - assert total_time < retries_and_delays * 1.5 + assert exception is None + assert result.ok + assert async_mock.call_count == 1 + assert total_time < 1 From a938efdc3e36da22b26c43f4745fc50b0998c600 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 28 Sep 2023 22:43:43 +0300 Subject: [PATCH 167/268] Ignore tests for Python 3.7 --- tests/aio/test_http.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/aio/test_http.py b/tests/aio/test_http.py index 6b72d347..982db42d 100644 --- a/tests/aio/test_http.py +++ b/tests/aio/test_http.py @@ -80,7 +80,7 @@ def get_http_server(server_class=SERVER_CLASS, server_address=SERVER_ADDRESS, @pytest.mark.skipif(sys.version_info < (3, 8), - reason="the test requires Python 3.8 or higher") + reason="the test requires AsyncMock which was introduced in Python 3.8") @pytest.mark.parametrize( 'server_class, port, expected_delay, timeout_seconds, is_exception', [ @@ -121,6 +121,8 @@ async def test_retry_on_request_error(server_class, port, expected_delay, timeou assert total_time < expected_delay * 1.5 +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") @pytest.mark.asyncio async def test_no_retry_on_ok_request(): retry_number = 5 From 73870cc7b8e48cc94e599ab0733786c5ec1c47cf Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 28 Sep 2023 23:12:38 +0300 Subject: [PATCH 168/268] Add more tests --- tests/aio/test_http.py | 71 ++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/tests/aio/test_http.py b/tests/aio/test_http.py index 982db42d..74a642ef 100644 --- a/tests/aio/test_http.py +++ b/tests/aio/test_http.py @@ -79,24 +79,12 @@ def get_http_server(server_class=SERVER_CLASS, server_address=SERVER_ADDRESS, return httpd -@pytest.mark.skipif(sys.version_info < (3, 8), - reason="the test requires AsyncMock which was introduced in Python 3.8") -@pytest.mark.parametrize( - 'server_class, port, expected_delay, timeout_seconds, is_exception', - [ - (TimeoutHttpHandler, 8001, 14.4, 1.0, True), - (ResetHttpHandler, 8002, 8.4, 1.0, True), - (ErrorHttpHandler, 8003, 1.1, 1.0, False), - (ThrottlingHttpHandler, 8004, 27.93, 1.0, False), - ] -) -@pytest.mark.asyncio -async def test_retry_on_request_error(server_class, port, expected_delay, timeout_seconds, is_exception): - retry_number = 5 +async def execute_http_request(port, retry_number, server_class, timeout_seconds, protocol='http'): timeout = aiohttp.ClientTimeout(connect=timeout_seconds, sock_read=timeout_seconds) connector = aiohttp.TCPConnector(force_close=True) - session = RetryingClientSession(f'http://localhost:{port}', timeout=timeout, + session = RetryingClientSession(f'{protocol}://localhost:{port}', timeout=timeout, max_retry_number=retry_number, base_retry_delay=0.01, connector=connector) + # noinspection PyProtectedMember parent_request = super(type(session), session)._request async_mock = mock.AsyncMock() async_mock.side_effect = parent_request @@ -111,6 +99,25 @@ async def test_retry_on_request_error(server_class, port, expected_delay, timeou except Exception as exc: exception = exc total_time = time.time() - start_time + return async_mock, exception, result, total_time + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@pytest.mark.parametrize( + 'server_class, port, expected_delay, timeout_seconds, is_exception', + [ + (TimeoutHttpHandler, 8001, 14.4, 1.0, True), + (ResetHttpHandler, 8002, 8.4, 1.0, True), + (ErrorHttpHandler, 8003, 1.1, 1.0, False), + (ThrottlingHttpHandler, 8004, 27.93, 1.0, False), + ] +) +@pytest.mark.asyncio +async def test_retry_on_request_error(server_class, port, expected_delay, timeout_seconds, is_exception): + retry_number = 5 + async_mock, exception, result, total_time = await execute_http_request(port, retry_number, server_class, + timeout_seconds) if is_exception: assert exception is not None else: @@ -127,25 +134,23 @@ async def test_retry_on_request_error(server_class, port, expected_delay, timeou async def test_no_retry_on_ok_request(): retry_number = 5 port = 8000 - timeout = aiohttp.ClientTimeout(connect=1, sock_read=1) - connector = aiohttp.TCPConnector(force_close=True) - session = RetryingClientSession(f'http://localhost:{port}', timeout=timeout, - max_retry_number=retry_number, base_retry_delay=0.01, connector=connector) - parent_request = super(type(session), session)._request - async_mock = mock.AsyncMock() - async_mock.side_effect = parent_request - exception = None - result = None - with get_http_server(server_handler=OkHttpHandler, server_address=('', port)): - with mock.patch('reportportal_client.aio.http.ClientSession._request', async_mock): - async with session: - start_time = time.time() - try: - result = await session.get('/') - except Exception as exc: - exception = exc - total_time = time.time() - start_time + async_mock, exception, result, total_time = await execute_http_request(port, retry_number, OkHttpHandler, + 1) assert exception is None assert result.ok assert async_mock.call_count == 1 assert total_time < 1 + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@pytest.mark.asyncio +async def test_no_retry_on_not_retryable_error(): + retry_number = 5 + port = 8005 + async_mock, exception, result, total_time = await execute_http_request(port, retry_number, OkHttpHandler, + 1, protocol='https') + assert exception is not None + assert result is None + assert async_mock.call_count == 1 + assert total_time < 1 From 00ddcc186cd606a67cbae8d01e8e0dcc946d6100 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 29 Sep 2023 12:05:41 +0300 Subject: [PATCH 169/268] Add get/set state methods --- reportportal_client/aio/client.py | 36 +++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index b07d498e..873f00eb 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -83,7 +83,7 @@ class Client: launch_uuid_print: bool print_output: TextIO _skip_analytics: str - __session: Optional[aiohttp.ClientSession] + _session: Optional[aiohttp.ClientSession] __stat_task: Optional[asyncio.Task] def __init__( @@ -135,7 +135,7 @@ def __init__( self._skip_analytics = getenv('AGENT_NO_ANALYTICS') self.launch_uuid_print = launch_uuid_print self.print_output = print_output or sys.stdout - self.__session = None + self._session = None self.__stat_task = None self.api_key = api_key @@ -164,8 +164,8 @@ def session(self) -> aiohttp.ClientSession: :return: aiohttp.ClientSession instance. """ - if self.__session: - return self.__session + if self._session: + return self._session ssl_config = self.verify_ssl if ssl_config: @@ -202,14 +202,14 @@ def session(self) -> aiohttp.ClientSession: if self.retries: session_params['max_retry_number'] = self.retries - self.__session = RetryingClientSession(self.endpoint, **session_params) - return self.__session + self._session = RetryingClientSession(self.endpoint, **session_params) + return self._session async def close(self) -> None: """Gracefully close internal aiohttp.ClientSession class instance and reset it.""" - if self.__session: - await self.__session.close() - self.__session = None + if self._session: + await self._session.close() + self._session = None async def __get_item_url(self, item_id_future: Union[str, Task[str]]) -> Optional[str]: item_id = await await_if_necessary(item_id_future) @@ -551,6 +551,24 @@ def clone(self) -> 'Client': ) return cloned + def __getstate__(self) -> Dict[str, Any]: + """Control object pickling and return object fields as Dictionary. + + :return: object state dictionary + :rtype: dict + """ + state = self.__dict__.copy() + # Don't pickle 'session' field, since it contains unpickling 'socket' + del state['_session'] + return state + + def __setstate__(self, state: Dict[str, Any]) -> None: + """Control object pickling, receives object state as Dictionary. + + :param dict state: object state dictionary + """ + self.__dict__.update(state) + class AsyncRPClient(RP): """Asynchronous ReportPortal Client. From d67f6b39f1077e57344c950e344043daf75007cd Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 29 Sep 2023 16:30:10 +0300 Subject: [PATCH 170/268] Fixes and tests for pickling --- reportportal_client/__init__.py | 3 +- reportportal_client/aio/client.py | 168 ++++++++++++++++++---------- reportportal_client/client.py | 22 +++- reportportal_client/helpers.py | 28 ++++- reportportal_client/logs/batcher.py | 21 +++- tests/aio/test_client.py | 43 +++++++ tests/test_client.py | 16 ++- 7 files changed, 234 insertions(+), 67 deletions(-) create mode 100644 tests/aio/test_client.py diff --git a/reportportal_client/__init__.py b/reportportal_client/__init__.py index 8ee6b918..e8e95dc8 100644 --- a/reportportal_client/__init__.py +++ b/reportportal_client/__init__.py @@ -16,13 +16,14 @@ # noinspection PyProtectedMember from reportportal_client._local import current from reportportal_client.logs import RPLogger, RPLogHandler -from reportportal_client.client import RP, RPClient +from reportportal_client.client import RP, RPClient, OutputType from reportportal_client.steps import step __all__ = [ 'current', 'RP', 'RPClient', + 'OutputType', 'RPLogger', 'RPLogHandler', 'step', diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 873f00eb..5ca4a91f 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -16,17 +16,16 @@ import asyncio import logging import ssl -import sys import threading import time as datetime import warnings from os import getenv -from typing import Union, Tuple, List, Dict, Any, Optional, TextIO, Coroutine, TypeVar +from typing import Union, Tuple, List, Dict, Any, Optional, Coroutine, TypeVar import aiohttp import certifi -from reportportal_client import RP +from reportportal_client import RP, OutputType # noinspection PyProtectedMember from reportportal_client._local import set_current from reportportal_client.aio.http import RetryingClientSession @@ -81,7 +80,7 @@ class Client: keepalive_timeout: Optional[float] mode: str launch_uuid_print: bool - print_output: TextIO + print_output: OutputType _skip_analytics: str _session: Optional[aiohttp.ClientSession] __stat_task: Optional[asyncio.Task] @@ -100,7 +99,7 @@ def __init__( keepalive_timeout: Optional[float] = None, mode: str = 'DEFAULT', launch_uuid_print: bool = False, - print_output: Optional[TextIO] = None, + print_output: OutputType = OutputType.STDOUT, **kwargs: Any ) -> None: """Initialize the class instance with arguments. @@ -134,7 +133,7 @@ def __init__( self.mode = mode self._skip_analytics = getenv('AGENT_NO_ANALYTICS') self.launch_uuid_print = launch_uuid_print - self.print_output = print_output or sys.stdout + self.print_output = print_output self._session = None self.__stat_task = None @@ -267,7 +266,7 @@ async def start_launch(self, launch_uuid = await response.id logger.debug(f'start_launch - ID: {launch_uuid}') if self.launch_uuid_print and self.print_output: - print(f'ReportPortal Launch UUID: {launch_uuid}', file=self.print_output) + print(f'ReportPortal Launch UUID: {launch_uuid}', file=self.print_output.get_output()) return launch_uuid async def start_test_item(self, @@ -1317,9 +1316,39 @@ class ThreadedRPClient(_RPClient): _task_timeout: float _shutdown_timeout: float _loop: Optional[asyncio.AbstractEventLoop] + _task_mutex: threading.RLock + _thread: Optional[threading.Thread] __task_list: BackgroundTaskList[Task[_T]] - __task_mutex: threading.RLock - __thread: Optional[threading.Thread] + + def __init_task_list(self, task_list: Optional[BackgroundTaskList[Task[_T]]] = None, + task_mutex: Optional[threading.RLock] = None): + if task_list: + if not task_mutex: + warnings.warn( + '"task_list" argument is set, but not "task_mutex". This usually indicates ' + 'invalid use, since "task_mutex" is used to synchronize on "task_list".', + RuntimeWarning, + 3 + ) + self.__task_list = task_list or BackgroundTaskList() + self._task_mutex = task_mutex or threading.RLock() + + def __heartbeat(self): + # We operate on our own loop with daemon thread, so we will exit in any way when main thread exit, + # so we can iterate forever + self._loop.call_at(self._loop.time() + 0.1, self.__heartbeat) + + def __init_loop(self, loop: Optional[asyncio.AbstractEventLoop] = None): + self._thread = None + if loop: + self._loop = loop + else: + self._loop = asyncio.new_event_loop() + self._loop.set_task_factory(ThreadedTaskFactory(self._task_timeout)) + self.__heartbeat() + self._thread = threading.Thread(target=self._loop.run_forever, name='RP-Async-Client', + daemon=True) + self._thread.start() def __init__( self, @@ -1370,31 +1399,8 @@ def __init__( super().__init__(endpoint, project, **kwargs) self._task_timeout = task_timeout self._shutdown_timeout = shutdown_timeout - if task_list: - if not task_mutex: - warnings.warn( - '"task_list" argument is set, but not "task_mutex". This usually indicates ' - 'invalid use, since "task_mutex" is used to synchronize on "task_list".', - RuntimeWarning, - 2 - ) - self.__task_list = task_list or BackgroundTaskList() - self.__task_mutex = task_mutex or threading.RLock() - self.__thread = None - if loop: - self._loop = loop - else: - self._loop = asyncio.new_event_loop() - self._loop.set_task_factory(ThreadedTaskFactory(self._task_timeout)) - self.__heartbeat() - self.__thread = threading.Thread(target=self._loop.run_forever, name='RP-Async-Client', - daemon=True) - self.__thread.start() - - def __heartbeat(self): - # We operate on our own loop with daemon thread, so we will exit in any way when main thread exit, - # so we can iterate forever - self._loop.call_at(self._loop.time() + 0.1, self.__heartbeat) + self.__init_task_list(task_list, task_mutex) + self.__init_loop(loop) def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: """Create a Task from given Coroutine. @@ -1405,14 +1411,14 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: if not getattr(self, '_loop', None): return result = self._loop.create_task(coro) - with self.__task_mutex: + with self._task_mutex: self.__task_list.append(result) return result def finish_tasks(self): """Ensure all pending Tasks are finished, block current Thread if necessary.""" shutdown_start_time = datetime.time() - with self.__task_mutex: + with self._task_mutex: tasks = self.__task_list.flush() for task in tasks: task.blocking_result() @@ -1441,7 +1447,7 @@ def clone(self) -> 'ThreadedRPClient': log_batcher=self._log_batcher, task_timeout=self._task_timeout, shutdown_timeout=self._shutdown_timeout, - task_mutex=self.__task_mutex, + task_mutex=self._task_mutex, task_list=self.__task_list, loop=self._loop ) @@ -1450,6 +1456,28 @@ def clone(self) -> 'ThreadedRPClient': cloned._add_current_item(current_item) return cloned + def __getstate__(self) -> Dict[str, Any]: + """Control object pickling and return object fields as Dictionary. + + :return: object state dictionary + :rtype: dict + """ + state = self.__dict__.copy() + # Don't pickle 'session' field, since it contains unpickling 'socket' + del state['_task_mutex'] + del state['_loop'] + del state['_thread'] + return state + + def __setstate__(self, state: Dict[str, Any]) -> None: + """Control object pickling, receives object state as Dictionary. + + :param dict state: object state dictionary + """ + self.__dict__.update(state) + self.__init_task_list(self.__task_list, threading.RLock()) + self.__init_loop() + class BatchedRPClient(_RPClient): """Synchronous-asynchronous ReportPortal Client which uses the same Thread to execute async coroutines. @@ -1462,12 +1490,32 @@ class BatchedRPClient(_RPClient): _task_timeout: float _shutdown_timeout: float _loop: asyncio.AbstractEventLoop + _task_mutex: threading.RLock __task_list: TriggerTaskBatcher[Task[_T]] - __task_mutex: threading.RLock __last_run_time: float __trigger_num: int __trigger_interval: float + def __init_task_list(self, task_list: Optional[TriggerTaskBatcher[Task[_T]]] = None, + task_mutex: Optional[threading.RLock] = None): + if task_list: + if not task_mutex: + warnings.warn( + '"task_list" argument is set, but not "task_mutex". This usually indicates ' + 'invalid use, since "task_mutex" is used to synchronize on "task_list".', + RuntimeWarning, + 3 + ) + self.__task_list = task_list or TriggerTaskBatcher(self.__trigger_num, self.__trigger_interval) + self._task_mutex = task_mutex or threading.RLock() + + def __init_loop(self, loop: Optional[asyncio.AbstractEventLoop] = None): + if loop: + self._loop = loop + else: + self._loop = asyncio.new_event_loop() + self._loop.set_task_factory(BatchedTaskFactory()) + def __init__( self, endpoint: str, @@ -1523,22 +1571,9 @@ def __init__( self._shutdown_timeout = shutdown_timeout self.__trigger_num = trigger_num self.__trigger_interval = trigger_interval - if task_list: - if not task_mutex: - warnings.warn( - '"task_list" argument is set, but not "task_mutex". This usually indicates ' - 'invalid use, since "task_mutex" is used to synchronize on "task_list".', - RuntimeWarning, - 2 - ) - self.__task_list = task_list or TriggerTaskBatcher(trigger_num, trigger_interval) - self.__task_mutex = task_mutex or threading.RLock() + self.__init_task_list(task_list, task_mutex) self.__last_run_time = datetime.time() - if loop: - self._loop = loop - else: - self._loop = asyncio.new_event_loop() - self._loop.set_task_factory(BatchedTaskFactory()) + self.__init_loop(loop) def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: """Create a Task from given Coroutine. @@ -1549,7 +1584,7 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: if not getattr(self, '_loop', None): return result = self._loop.create_task(coro) - with self.__task_mutex: + with self._task_mutex: tasks = self.__task_list.append(result) if tasks: self._loop.run_until_complete(asyncio.wait(tasks, timeout=self._task_timeout)) @@ -1557,7 +1592,7 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: def finish_tasks(self) -> None: """Ensure all pending Tasks are finished, block current Thread if necessary.""" - with self.__task_mutex: + with self._task_mutex: tasks = self.__task_list.flush() if tasks: self._loop.run_until_complete(asyncio.wait(tasks, timeout=self._shutdown_timeout)) @@ -1586,7 +1621,7 @@ def clone(self) -> 'BatchedRPClient': task_timeout=self._task_timeout, shutdown_timeout=self._shutdown_timeout, task_list=self.__task_list, - task_mutex=self.__task_mutex, + task_mutex=self._task_mutex, loop=self._loop, trigger_num=self.__trigger_num, trigger_interval=self.__trigger_interval @@ -1595,3 +1630,24 @@ def clone(self) -> 'BatchedRPClient': if current_item: cloned._add_current_item(current_item) return cloned + + def __getstate__(self) -> Dict[str, Any]: + """Control object pickling and return object fields as Dictionary. + + :return: object state dictionary + :rtype: dict + """ + state = self.__dict__.copy() + # Don't pickle 'session' field, since it contains unpickling 'socket' + del state['_task_mutex'] + del state['_loop'] + return state + + def __setstate__(self, state: Dict[str, Any]) -> None: + """Control object pickling, receives object state as Dictionary. + + :param dict state: object state dictionary + """ + self.__dict__.update(state) + self.__init_task_list(self.__task_list, threading.RLock()) + self.__init_loop() diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 7d66ed33..7ccfea2b 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -21,6 +21,7 @@ from os import getenv from typing import Union, Tuple, Any, Optional, TextIO, List, Dict +import aenum import requests from requests.adapters import HTTPAdapter, Retry, DEFAULT_RETRIES @@ -42,6 +43,17 @@ logger.addHandler(logging.NullHandler()) +class OutputType(aenum.Enum): + STDOUT = aenum.auto() + STDERR = aenum.auto() + + def get_output(self) -> Optional[TextIO]: + if self == OutputType.STDOUT: + return sys.stdout + if self == OutputType.STDERR: + return sys.stderr + + class RP(metaclass=AbstractBaseClass): """Common interface for ReportPortal clients. @@ -337,7 +349,7 @@ class RPClient(RP): __step_reporter: StepReporter mode: str launch_uuid_print: Optional[bool] - print_output: Optional[TextIO] + print_output: OutputType _skip_analytics: str _item_stack: LifoQueue _log_batcher: LogBatcher[RPRequestLog] @@ -406,7 +418,7 @@ def __init__( log_batch_payload_size: int = MAX_LOG_BATCH_PAYLOAD_SIZE, mode: str = 'DEFAULT', launch_uuid_print: bool = False, - print_output: Optional[TextIO] = None, + print_output: OutputType = OutputType.STDOUT, log_batcher: Optional[LogBatcher[RPRequestLog]] = None, **kwargs: Any ) -> None: @@ -464,7 +476,7 @@ def __init__( self.mode = mode self._skip_analytics = getenv('AGENT_NO_ANALYTICS') self.launch_uuid_print = launch_uuid_print - self.print_output = print_output or sys.stdout + self.print_output = print_output self.api_key = api_key if not self.api_key: @@ -530,7 +542,7 @@ def start_launch(self, self.__launch_uuid = response.id logger.debug('start_launch - ID: %s', self.launch_uuid) if self.launch_uuid_print and self.print_output: - print(f'ReportPortal Launch UUID: {self.launch_uuid}', file=self.print_output) + print(f'ReportPortal Launch UUID: {self.launch_uuid}', file=self.print_output.get_output()) return self.launch_uuid def start_test_item(self, @@ -864,8 +876,6 @@ def __getstate__(self) -> Dict[str, Any]: state = self.__dict__.copy() # Don't pickle 'session' field, since it contains unpickling 'socket' del state['session'] - # Don't pickle '_log_manager' field, since it uses 'session' field - del state['_log_manager'] return state def __setstate__(self, state: Dict[str, Any]) -> None: diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index af77e105..84b5f974 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -17,9 +17,10 @@ import inspect import json import logging +import queue +import threading import time import uuid -import queue from platform import machine, processor, system from typing import Optional, Any, List, Dict, Callable, Tuple, Union, TypeVar, Generic @@ -47,6 +48,31 @@ def last(self) -> _T: if self._qsize(): return self.queue[-1] + def __getstate__(self) -> Dict[str, Any]: + """Control object pickling and return object fields as Dictionary. + + :return: object state dictionary + :rtype: dict + """ + state = self.__dict__.copy() + # Don't pickle 'session' field, since it contains unpickling 'socket' + del state['mutex'] + del state['not_empty'] + del state['not_full'] + del state['all_tasks_done'] + return state + + def __setstate__(self, state: Dict[str, Any]) -> None: + """Control object pickling, receives object state as Dictionary. + + :param dict state: object state dictionary + """ + self.__dict__.update(state) + self.mutex = threading.Lock() + self.not_empty = threading.Condition(self.mutex) + self.not_full = threading.Condition(self.mutex) + self.all_tasks_done = threading.Condition(self.mutex) + def generate_uuid() -> str: """Generate UUID.""" diff --git a/reportportal_client/logs/batcher.py b/reportportal_client/logs/batcher.py index 915d190a..1297625f 100644 --- a/reportportal_client/logs/batcher.py +++ b/reportportal_client/logs/batcher.py @@ -15,7 +15,7 @@ import logging import threading -from typing import List, Optional, TypeVar, Generic +from typing import List, Optional, TypeVar, Generic, Dict, Any from reportportal_client.core.rp_requests import RPRequestLog, AsyncRPRequestLog from reportportal_client.logs import MAX_LOG_BATCH_SIZE, MAX_LOG_BATCH_PAYLOAD_SIZE @@ -92,3 +92,22 @@ def flush(self) -> Optional[List[T_co]]: batch = self._batch self._batch = [] return batch + + def __getstate__(self) -> Dict[str, Any]: + """Control object pickling and return object fields as Dictionary. + + :return: object state dictionary + :rtype: dict + """ + state = self.__dict__.copy() + # Don't pickle 'session' field, since it contains unpickling 'socket' + del state['_lock'] + return state + + def __setstate__(self, state: Dict[str, Any]) -> None: + """Control object pickling, receives object state as Dictionary. + + :param dict state: object state dictionary + """ + self.__dict__.update(state) + self._lock = threading.Lock() diff --git a/tests/aio/test_client.py b/tests/aio/test_client.py new file mode 100644 index 00000000..03837125 --- /dev/null +++ b/tests/aio/test_client.py @@ -0,0 +1,43 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pickle +from reportportal_client.aio.client import Client, AsyncRPClient, ThreadedRPClient, BatchedRPClient + + +def test_client_pickling(): + client = Client('http://localhost:8080', 'default_personal', api_key='test_key') + pickled_client = pickle.dumps(client) + unpickled_client = pickle.loads(pickled_client) + assert unpickled_client is not None + + +def test_async_rp_client_pickling(): + client = AsyncRPClient('http://localhost:8080', 'default_personal', api_key='test_key') + pickled_client = pickle.dumps(client) + unpickled_client = pickle.loads(pickled_client) + assert unpickled_client is not None + + +def test_threaded_rp_client_pickling(): + client = ThreadedRPClient('http://localhost:8080', 'default_personal', api_key='test_key') + pickled_client = pickle.dumps(client) + unpickled_client = pickle.loads(pickled_client) + assert unpickled_client is not None + + +def test_batched_rp_client_pickling(): + client = BatchedRPClient('http://localhost:8080', 'default_personal', api_key='test_key') + pickled_client = pickle.dumps(client) + unpickled_client = pickle.loads(pickled_client) + assert unpickled_client is not None diff --git a/tests/test_client.py b/tests/test_client.py index 3382c199..c2ea9dc7 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -10,6 +10,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License +import pickle from io import StringIO from unittest import mock @@ -201,8 +202,10 @@ def test_empty_api_key_argument(warn): def test_launch_uuid_print(): str_io = StringIO() + output_mock = mock.Mock() + output_mock.get_output.side_effect = lambda: str_io client = RPClient(endpoint='http://endpoint', project='project', - api_key='test', launch_uuid_print=True, print_output=str_io) + api_key='test', launch_uuid_print=True, print_output=output_mock) client.session = mock.Mock() client._skip_analytics = True client.start_launch('Test Launch', timestamp()) @@ -211,8 +214,10 @@ def test_launch_uuid_print(): def test_no_launch_uuid_print(): str_io = StringIO() + output_mock = mock.Mock() + output_mock.get_output.side_effect = lambda: str_io client = RPClient(endpoint='http://endpoint', project='project', - api_key='test', launch_uuid_print=False, print_output=str_io) + api_key='test', launch_uuid_print=False, print_output=output_mock) client.session = mock.Mock() client._skip_analytics = True client.start_launch('Test Launch', timestamp()) @@ -239,3 +244,10 @@ def test_launch_uuid_print_default_print(mock_stdout): client.start_launch('Test Launch', timestamp()) assert 'ReportPortal Launch UUID: ' not in mock_stdout.getvalue() + + +def test_client_pickling(): + client = RPClient('http://localhost:8080', 'default_personal', api_key='test_key') + pickled_client = pickle.dumps(client) + unpickled_client = pickle.loads(pickled_client) + assert unpickled_client is not None From 8004c78bf86e60b49359bbf5617fc649de809734 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 29 Sep 2023 16:37:43 +0300 Subject: [PATCH 171/268] CHANGELOG.md update --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ad54b9e..07c40cbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # Changelog ## [Unreleased] +### Added +- `RP` class in `reportportal_client.client` module as common interface for all ReportPortal clients, by @HardNorth +- `reportportal_client.aio` with asynchronous clients and auxiliary classes, by @HardNorth +- Dependency on `aiohttp` and `certifi`, by @HardNorth +### Changed +- RPClient class does not use separate Thread for log processing anymore, by @HardNorth +### Removed +- Dependency on `six`, by @HardNorth ## [5.4.1] ### Changed From effce03d0abe5d67c9e21dd45add5cc423bd459f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 29 Sep 2023 16:58:09 +0300 Subject: [PATCH 172/268] Implement own LifoQueue --- reportportal_client/client.py | 2 +- reportportal_client/helpers.py | 53 ++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 7ccfea2b..79c71668 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -830,7 +830,7 @@ def _remove_current_item(self) -> Optional[str]: :return: Item UUID string """ try: - return self._item_stack.get(timeout=0) + return self._item_stack.get() except queue.Empty: return diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index 84b5f974..197645c5 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -33,20 +33,47 @@ _T = TypeVar('_T') -class LifoQueue(Generic[_T], queue.LifoQueue): - """This Queue adds 'last' method to original queue.LifoQueue. +class LifoQueue(Generic[_T]): + """Primitive thread-safe Last-in-first-out queue implementation.""" - Unlike 'get' method in queue.LifoQueue the 'last' method do not remove an entity from the queue. - """ + _lock: threading.Lock() + __items: List[_T] + + def __init__(self): + """Initialize the queue instance.""" + self._lock = threading.Lock() + self.__items = [] + + def put(self, element: _T) -> None: + """Add an element to the queue.""" + with self._lock: + self.__items.append(element) + + def get(self) -> Optional[_T]: + """Return and remove the last element from the queue. + + :return: The last element in the queue. + """ + result = None + with self._lock: + if len(self.__items) > 0: + result = self.__items[-1] + self.__items = self.__items[:-1] + return result def last(self) -> _T: """Return the last element from the queue, but does not remove it. - :return: the last element in the queue + :return: The last element in the queue. """ - with self.mutex: - if self._qsize(): - return self.queue[-1] + with self._lock: + if len(self.__items) > 0: + return self.__items[-1] + + def qsize(self): + """Return the queue size.""" + with self._lock: + return len(self.__items) def __getstate__(self) -> Dict[str, Any]: """Control object pickling and return object fields as Dictionary. @@ -56,10 +83,7 @@ def __getstate__(self) -> Dict[str, Any]: """ state = self.__dict__.copy() # Don't pickle 'session' field, since it contains unpickling 'socket' - del state['mutex'] - del state['not_empty'] - del state['not_full'] - del state['all_tasks_done'] + del state['_lock'] return state def __setstate__(self, state: Dict[str, Any]) -> None: @@ -68,10 +92,7 @@ def __setstate__(self, state: Dict[str, Any]) -> None: :param dict state: object state dictionary """ self.__dict__.update(state) - self.mutex = threading.Lock() - self.not_empty = threading.Condition(self.mutex) - self.not_full = threading.Condition(self.mutex) - self.all_tasks_done = threading.Condition(self.mutex) + self._lock = threading.Lock() def generate_uuid() -> str: From 1383551b899fdab942ea7a3e53bb4b1e738bd126 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 29 Sep 2023 17:03:55 +0300 Subject: [PATCH 173/268] Update pydocs --- reportportal_client/client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 79c71668..1057c076 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -44,10 +44,13 @@ class OutputType(aenum.Enum): + """Enum of possible print output types.""" + STDOUT = aenum.auto() STDERR = aenum.auto() def get_output(self) -> Optional[TextIO]: + """Return TextIO based on the current type.""" if self == OutputType.STDOUT: return sys.stdout if self == OutputType.STDERR: From 21686fcbd54eaf8a1bec2c56101e7ebeebd76501 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 29 Sep 2023 17:16:51 +0300 Subject: [PATCH 174/268] Fix flake8 --- reportportal_client/helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index 197645c5..02d328d6 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -17,7 +17,6 @@ import inspect import json import logging -import queue import threading import time import uuid From bcd58193c9f0b79eca4ef64122e1dc9b0b3af97f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 2 Oct 2023 13:14:13 +0300 Subject: [PATCH 175/268] Remove redundant package files --- tests/__init__.py | 1 - tests/core/__init__.py | 14 -------------- tests/logs/__init__.py | 14 -------------- 3 files changed, 29 deletions(-) delete mode 100644 tests/__init__.py delete mode 100644 tests/core/__init__.py delete mode 100644 tests/logs/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 65b0b270..00000000 --- a/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""This package contains unit tests for the project.""" diff --git a/tests/core/__init__.py b/tests/core/__init__.py deleted file mode 100644 index 30e3faf9..00000000 --- a/tests/core/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""This package contains unit tests for core module.""" - -# Copyright (c) 2022 EPAM Systems -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License diff --git a/tests/logs/__init__.py b/tests/logs/__init__.py deleted file mode 100644 index c72fbd80..00000000 --- a/tests/logs/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""This package contains unit tests for logging.""" - -# Copyright (c) 2022 EPAM Systems -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License From 5d50669b0c65a1b4c59a7711826ed108a9cb1a13 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 2 Oct 2023 13:14:24 +0300 Subject: [PATCH 176/268] Add 'test_async_send_event' --- tests/{ => services}/test_statistics.py | 96 ++++++++++++++++++------- 1 file changed, 70 insertions(+), 26 deletions(-) rename tests/{ => services}/test_statistics.py (51%) diff --git a/tests/test_statistics.py b/tests/services/test_statistics.py similarity index 51% rename from tests/test_statistics.py rename to tests/services/test_statistics.py index dcfb74f9..25cc2d03 100644 --- a/tests/test_statistics.py +++ b/tests/services/test_statistics.py @@ -12,16 +12,44 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License -# noinspection PyUnresolvedReferences +import sys from unittest import mock +import pytest from requests.exceptions import RequestException from reportportal_client.services.constants import ENDPOINT, CLIENT_INFO -from reportportal_client.services.statistics import send_event +from reportportal_client.services.statistics import send_event, async_send_event EVENT_NAME = 'start_launch' +EXPECTED_CL_VERSION, EXPECTED_CL_NAME = '5.0.4', 'reportportal-client' +AGENT_VERSION, AGENT_NAME = '5.0.5', 'pytest-reportportal' +EXPECTED_HEADERS = {'User-Agent': 'python-requests'} +EXPECTED_AIO_HEADERS = {'User-Agent': 'python-aiohttp'} +EXPECTED_DATA = { + 'client_id': '555', + 'events': [{ + 'name': EVENT_NAME, + 'params': { + 'client_name': EXPECTED_CL_NAME, + 'client_version': EXPECTED_CL_VERSION, + 'interpreter': 'Python 3.6.6', + 'agent_name': AGENT_NAME, + 'agent_version': AGENT_VERSION, + } + }] +} +MID, KEY = CLIENT_INFO.split(':') +EXPECTED_PARAMS = {'measurement_id': MID, 'api_secret': KEY} @mock.patch('reportportal_client.services.statistics.get_client_id', @@ -36,32 +64,13 @@ def test_send_event(mocked_distribution, mocked_requests): :param mocked_distribution: Mocked get_distribution() function :param mocked_requests: Mocked requests module """ - expected_cl_version, expected_cl_name = '5.0.4', 'reportportal-client' - agent_version, agent_name = '5.0.5', 'pytest-reportportal' - mocked_distribution.return_value.version = expected_cl_version - mocked_distribution.return_value.project_name = expected_cl_name + mocked_distribution.return_value.version = EXPECTED_CL_VERSION + mocked_distribution.return_value.project_name = EXPECTED_CL_NAME - expected_headers = {'User-Agent': 'python-requests'} - - expected_data = { - 'client_id': '555', - 'events': [{ - 'name': EVENT_NAME, - 'params': { - 'client_name': expected_cl_name, - 'client_version': expected_cl_version, - 'interpreter': 'Python 3.6.6', - 'agent_name': agent_name, - 'agent_version': agent_version, - } - }] - } - mid, key = CLIENT_INFO.split(':') - expected_params = {'measurement_id': mid, 'api_secret': key} - send_event(EVENT_NAME, agent_name, agent_version) + send_event(EVENT_NAME, AGENT_NAME, AGENT_VERSION) mocked_requests.assert_called_with( - url=ENDPOINT, json=expected_data, headers=expected_headers, - params=expected_params) + url=ENDPOINT, json=EXPECTED_DATA, headers=EXPECTED_HEADERS, + params=EXPECTED_PARAMS) @mock.patch('reportportal_client.services.statistics.get_client_id', @@ -98,3 +107,38 @@ def test_same_client_id(mocked_distribution, mocked_requests): result2 = args_list[1][1]['json']['client_id'] assert result1 == result2 + + +MOCKED_AIOHTTP = mock.AsyncMock() + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@mock.patch('reportportal_client.services.statistics.get_client_id', + mock.Mock(return_value='555')) +@mock.patch('reportportal_client.services.statistics.aiohttp.ClientSession.post', MOCKED_AIOHTTP) +@mock.patch('reportportal_client.services.statistics.get_distribution') +@mock.patch('reportportal_client.services.statistics.python_version', + mock.Mock(return_value='3.6.6')) +@pytest.mark.asyncio +async def test_async_send_event(mocked_distribution): + """Test functionality of the send_event() function. + + :param mocked_distribution: Mocked get_distribution() function + """ + mocked_distribution.return_value.version = EXPECTED_CL_VERSION + mocked_distribution.return_value.project_name = EXPECTED_CL_NAME + + await async_send_event(EVENT_NAME, AGENT_NAME, AGENT_VERSION) + assert len(MOCKED_AIOHTTP.call_args_list) == 1 + args, kwargs = MOCKED_AIOHTTP.call_args_list[0] + assert len(args) == 0 + kwargs_keys = ['headers', 'url', 'json', 'params', 'ssl'] + for key in kwargs_keys: + assert key in kwargs + assert len(kwargs_keys) == len(kwargs) + assert kwargs['headers'] == EXPECTED_AIO_HEADERS + assert kwargs['url'] == ENDPOINT + assert kwargs['json'] == EXPECTED_DATA + assert kwargs['params'] == EXPECTED_PARAMS + assert kwargs['ssl'] is not None From a09b2ffca09c245c0e4c0e09e3ecc1dd284500fb Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 2 Oct 2023 13:15:25 +0300 Subject: [PATCH 177/268] Fix for Python 3.7 --- tests/services/test_statistics.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/services/test_statistics.py b/tests/services/test_statistics.py index 25cc2d03..53a2890d 100644 --- a/tests/services/test_statistics.py +++ b/tests/services/test_statistics.py @@ -109,7 +109,9 @@ def test_same_client_id(mocked_distribution, mocked_requests): assert result1 == result2 -MOCKED_AIOHTTP = mock.AsyncMock() +MOCKED_AIOHTTP = None +if not sys.version_info < (3, 8): + MOCKED_AIOHTTP = mock.AsyncMock() @pytest.mark.skipif(sys.version_info < (3, 8), From c8c40811798f435bda544f734bb5fe1e3c9cd5a7 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 2 Oct 2023 13:36:38 +0300 Subject: [PATCH 178/268] Rename test module --- tests/aio/{test_client.py => test_aio_client.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/aio/{test_client.py => test_aio_client.py} (100%) diff --git a/tests/aio/test_client.py b/tests/aio/test_aio_client.py similarity index 100% rename from tests/aio/test_client.py rename to tests/aio/test_aio_client.py From b7203b18991d4b035be79bc7459751e0a02ba0a9 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 2 Oct 2023 13:40:28 +0300 Subject: [PATCH 179/268] Move code style checks on Python 3.8 --- .github/workflows/tests.yml | 2 +- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 46036112..2e7d9d44 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,7 +54,7 @@ jobs: run: tox - name: Upload coverage to Codecov - if: matrix.python-version == 3.7 && success() + if: matrix.python-version == 3.8 && success() uses: codecov/codecov-action@v3 with: files: coverage.xml diff --git a/tox.ini b/tox.ini index d82c5674..2f8dad03 100644 --- a/tox.ini +++ b/tox.ini @@ -25,8 +25,8 @@ commands = pre-commit run --all-files --show-diff-on-failure [gh-actions] python = - 3.7: pep, py37 - 3.8: py38 + 3.7: py37 + 3.8: pep, py38 3.9: py39 3.10: py310 3.11: py311 From c62a571b7f417bc2f58daf67a2a8e11bb84ad65f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 2 Oct 2023 13:57:40 +0300 Subject: [PATCH 180/268] Refactor aio client tests --- tests/aio/test_aio_client.py | 24 ++---------------------- tests/aio/test_async_client.py | 23 +++++++++++++++++++++++ tests/aio/test_batched_client.py | 22 ++++++++++++++++++++++ tests/aio/test_threaded_client.py | 22 ++++++++++++++++++++++ 4 files changed, 69 insertions(+), 22 deletions(-) create mode 100644 tests/aio/test_async_client.py create mode 100644 tests/aio/test_batched_client.py create mode 100644 tests/aio/test_threaded_client.py diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 03837125..236f8ac5 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -12,7 +12,8 @@ # limitations under the License import pickle -from reportportal_client.aio.client import Client, AsyncRPClient, ThreadedRPClient, BatchedRPClient + +from reportportal_client.aio.client import Client def test_client_pickling(): @@ -20,24 +21,3 @@ def test_client_pickling(): pickled_client = pickle.dumps(client) unpickled_client = pickle.loads(pickled_client) assert unpickled_client is not None - - -def test_async_rp_client_pickling(): - client = AsyncRPClient('http://localhost:8080', 'default_personal', api_key='test_key') - pickled_client = pickle.dumps(client) - unpickled_client = pickle.loads(pickled_client) - assert unpickled_client is not None - - -def test_threaded_rp_client_pickling(): - client = ThreadedRPClient('http://localhost:8080', 'default_personal', api_key='test_key') - pickled_client = pickle.dumps(client) - unpickled_client = pickle.loads(pickled_client) - assert unpickled_client is not None - - -def test_batched_rp_client_pickling(): - client = BatchedRPClient('http://localhost:8080', 'default_personal', api_key='test_key') - pickled_client = pickle.dumps(client) - unpickled_client = pickle.loads(pickled_client) - assert unpickled_client is not None diff --git a/tests/aio/test_async_client.py b/tests/aio/test_async_client.py new file mode 100644 index 00000000..10b76a4a --- /dev/null +++ b/tests/aio/test_async_client.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pickle + +from aio import AsyncRPClient + + +def test_async_rp_client_pickling(): + client = AsyncRPClient('http://localhost:8080', 'default_personal', api_key='test_key') + pickled_client = pickle.dumps(client) + unpickled_client = pickle.loads(pickled_client) + assert unpickled_client is not None diff --git a/tests/aio/test_batched_client.py b/tests/aio/test_batched_client.py new file mode 100644 index 00000000..572141cf --- /dev/null +++ b/tests/aio/test_batched_client.py @@ -0,0 +1,22 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +import pickle + +from aio import BatchedRPClient + + +def test_batched_rp_client_pickling(): + client = BatchedRPClient('http://localhost:8080', 'default_personal', api_key='test_key') + pickled_client = pickle.dumps(client) + unpickled_client = pickle.loads(pickled_client) + assert unpickled_client is not None diff --git a/tests/aio/test_threaded_client.py b/tests/aio/test_threaded_client.py new file mode 100644 index 00000000..7749cbbd --- /dev/null +++ b/tests/aio/test_threaded_client.py @@ -0,0 +1,22 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +import pickle + +from aio import ThreadedRPClient + + +def test_threaded_rp_client_pickling(): + client = ThreadedRPClient('http://localhost:8080', 'default_personal', api_key='test_key') + pickled_client = pickle.dumps(client) + unpickled_client = pickle.loads(pickled_client) + assert unpickled_client is not None From c2af6b086796ac5e59f2ea03fd4bb95720746daa Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 2 Oct 2023 15:29:27 +0300 Subject: [PATCH 181/268] Update retries settings logic and add tests --- reportportal_client/aio/client.py | 14 ++++++++++---- tests/aio/test_aio_client.py | 25 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 5ca4a91f..591bf2a4 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -45,7 +45,7 @@ AbstractBaseClass, abstractmethod ) -from reportportal_client.static.defines import NOT_FOUND +from reportportal_client.static.defines import NOT_FOUND, NOT_SET from reportportal_client.steps import StepReporter logger = logging.getLogger(__name__) @@ -93,7 +93,7 @@ def __init__( api_key: str = None, is_skipped_an_issue: bool = True, verify_ssl: Union[bool, str] = True, - retries: int = None, + retries: int = NOT_SET, max_pool_size: int = 50, http_timeout: Union[float, Tuple[float, float]] = (10, 10), keepalive_timeout: Optional[float] = None, @@ -198,10 +198,16 @@ def session(self) -> aiohttp.ClientSession: connect_timeout, read_timeout = self.http_timeout, self.http_timeout session_params['timeout'] = aiohttp.ClientTimeout(connect=connect_timeout, sock_read=read_timeout) - if self.retries: + retries_set = self.retries is not NOT_SET and self.retries and self.retries > 0 + use_retries = self.retries is NOT_SET or (self.retries and self.retries > 0) + + if retries_set: session_params['max_retry_number'] = self.retries - self._session = RetryingClientSession(self.endpoint, **session_params) + if use_retries: + self._session = RetryingClientSession(self.endpoint, **session_params) + else: + self._session = aiohttp.ClientSession(self.endpoint, **session_params) return self._session async def close(self) -> None: diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 236f8ac5..c657bc09 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -12,8 +12,14 @@ # limitations under the License import pickle +from unittest import mock + +import aiohttp +import pytest from reportportal_client.aio.client import Client +from reportportal_client.aio.http import RetryingClientSession, DEFAULT_RETRY_NUMBER +from reportportal_client.static.defines import NOT_SET def test_client_pickling(): @@ -21,3 +27,22 @@ def test_client_pickling(): pickled_client = pickle.dumps(client) unpickled_client = pickle.loads(pickled_client) assert unpickled_client is not None + + +@pytest.mark.parametrize( + 'retry_num, expected_class, expected_param', + [ + (1, RetryingClientSession, 1), + (0, aiohttp.ClientSession, NOT_SET), + (-1, aiohttp.ClientSession, NOT_SET), + (None, aiohttp.ClientSession, NOT_SET), + (NOT_SET, RetryingClientSession, DEFAULT_RETRY_NUMBER) + ] +) +def test_retries_param(retry_num, expected_class, expected_param): + client = Client('http://localhost:8080', 'default_personal', api_key='test_key', + retries=retry_num) + session = client.session + assert isinstance(session, expected_class) + if expected_param is not NOT_SET: + assert getattr(session, f'_RetryingClientSession__retry_number') == expected_param From 5b4460e9153c123c818e680783e7491472379f07 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 2 Oct 2023 16:02:15 +0300 Subject: [PATCH 182/268] Add timeout argument tests --- reportportal_client/aio/client.py | 6 ++--- tests/aio/test_aio_client.py | 40 ++++++++++++++++++++++++++++--- tests/services/test_statistics.py | 6 ++--- 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 591bf2a4..b72ab3dd 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -74,9 +74,9 @@ class Client: project: str api_key: str verify_ssl: Union[bool, str] - retries: int + retries: Optional[int] max_pool_size: int - http_timeout: Union[float, Tuple[float, float]] + http_timeout: Optional[Union[float, Tuple[float, float]]] keepalive_timeout: Optional[float] mode: str launch_uuid_print: bool @@ -95,7 +95,7 @@ def __init__( verify_ssl: Union[bool, str] = True, retries: int = NOT_SET, max_pool_size: int = 50, - http_timeout: Union[float, Tuple[float, float]] = (10, 10), + http_timeout: Optional[Union[float, Tuple[float, float]]] = (10, 10), keepalive_timeout: Optional[float] = None, mode: str = 'DEFAULT', launch_uuid_print: bool = False, diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index c657bc09..80181465 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -21,9 +21,13 @@ from reportportal_client.aio.http import RetryingClientSession, DEFAULT_RETRY_NUMBER from reportportal_client.static.defines import NOT_SET +ENDPOINT = 'http://localhost:8080' +PROJECT = 'default_personal' +API_KEY = 'test_key' + def test_client_pickling(): - client = Client('http://localhost:8080', 'default_personal', api_key='test_key') + client = Client(ENDPOINT, PROJECT, api_key=API_KEY) pickled_client = pickle.dumps(client) unpickled_client = pickle.loads(pickled_client) assert unpickled_client is not None @@ -40,9 +44,39 @@ def test_client_pickling(): ] ) def test_retries_param(retry_num, expected_class, expected_param): - client = Client('http://localhost:8080', 'default_personal', api_key='test_key', - retries=retry_num) + client = Client(ENDPOINT, PROJECT, api_key=API_KEY, retries=retry_num) session = client.session assert isinstance(session, expected_class) if expected_param is not NOT_SET: assert getattr(session, f'_RetryingClientSession__retry_number') == expected_param + + +@pytest.mark.parametrize( + 'timeout_param, expected_connect_param, expected_sock_read_param', + [ + ((15, 17), 15, 17), + (21, 21, 21), + (None, None, None) + ] +) +@mock.patch('reportportal_client.aio.client.RetryingClientSession') +def test_timeout_param(mocked_session, timeout_param, expected_connect_param, expected_sock_read_param): + client = Client(ENDPOINT, PROJECT, api_key=API_KEY, http_timeout=timeout_param) + session = client.session + assert session is not None + assert len(mocked_session.call_args_list) == 1 + args, kwargs = mocked_session.call_args_list[0] + assert len(args) == 1 and args[0] == ENDPOINT + expected_kwargs_keys = ['headers', 'connector'] + if timeout_param: + expected_kwargs_keys.append('timeout') + for key in expected_kwargs_keys: + assert key in kwargs + assert len(expected_kwargs_keys) == len(kwargs) + assert kwargs['headers'] == {'Authorization': f'Bearer {API_KEY}'} + assert kwargs['connector'] is not None + if timeout_param: + assert kwargs['timeout'] is not None + assert isinstance(kwargs['timeout'], aiohttp.ClientTimeout) + assert kwargs['timeout'].connect == expected_connect_param + assert kwargs['timeout'].sock_read == expected_sock_read_param diff --git a/tests/services/test_statistics.py b/tests/services/test_statistics.py index 53a2890d..c5827b33 100644 --- a/tests/services/test_statistics.py +++ b/tests/services/test_statistics.py @@ -135,10 +135,10 @@ async def test_async_send_event(mocked_distribution): assert len(MOCKED_AIOHTTP.call_args_list) == 1 args, kwargs = MOCKED_AIOHTTP.call_args_list[0] assert len(args) == 0 - kwargs_keys = ['headers', 'url', 'json', 'params', 'ssl'] - for key in kwargs_keys: + expected_kwargs_keys = ['headers', 'url', 'json', 'params', 'ssl'] + for key in expected_kwargs_keys: assert key in kwargs - assert len(kwargs_keys) == len(kwargs) + assert len(expected_kwargs_keys) == len(kwargs) assert kwargs['headers'] == EXPECTED_AIO_HEADERS assert kwargs['url'] == ENDPOINT assert kwargs['json'] == EXPECTED_DATA From cbf34ebb525e878949e5b20d468990aa3f2388b5 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 2 Oct 2023 16:04:56 +0300 Subject: [PATCH 183/268] Fix tests --- tests/aio/test_async_client.py | 2 +- tests/aio/test_batched_client.py | 2 +- tests/aio/test_threaded_client.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/aio/test_async_client.py b/tests/aio/test_async_client.py index 10b76a4a..9c3b9d10 100644 --- a/tests/aio/test_async_client.py +++ b/tests/aio/test_async_client.py @@ -13,7 +13,7 @@ import pickle -from aio import AsyncRPClient +from reportportal_client.aio import AsyncRPClient def test_async_rp_client_pickling(): diff --git a/tests/aio/test_batched_client.py b/tests/aio/test_batched_client.py index 572141cf..55e1a714 100644 --- a/tests/aio/test_batched_client.py +++ b/tests/aio/test_batched_client.py @@ -12,7 +12,7 @@ # limitations under the License import pickle -from aio import BatchedRPClient +from reportportal_client.aio import BatchedRPClient def test_batched_rp_client_pickling(): diff --git a/tests/aio/test_threaded_client.py b/tests/aio/test_threaded_client.py index 7749cbbd..3082c6e3 100644 --- a/tests/aio/test_threaded_client.py +++ b/tests/aio/test_threaded_client.py @@ -12,7 +12,7 @@ # limitations under the License import pickle -from aio import ThreadedRPClient +from reportportal_client.aio import ThreadedRPClient def test_threaded_rp_client_pickling(): From c45dd145ca8e6b9e61360f03d99559f3fba0d6d6 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 2 Oct 2023 16:13:07 +0300 Subject: [PATCH 184/268] Fix tests --- tests/aio/test_aio_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 80181465..fc58724d 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -48,7 +48,7 @@ def test_retries_param(retry_num, expected_class, expected_param): session = client.session assert isinstance(session, expected_class) if expected_param is not NOT_SET: - assert getattr(session, f'_RetryingClientSession__retry_number') == expected_param + assert getattr(session, '_RetryingClientSession__retry_number') == expected_param @pytest.mark.parametrize( From 684b8eac66f4353ac9aa40cd0f93e56c41e30c50 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 3 Oct 2023 13:26:55 +0300 Subject: [PATCH 185/268] Add clone test --- tests/aio/test_aio_client.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index fc58724d..66e99288 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -17,6 +17,7 @@ import aiohttp import pytest +from reportportal_client import OutputType from reportportal_client.aio.client import Client from reportportal_client.aio.http import RetryingClientSession, DEFAULT_RETRY_NUMBER from reportportal_client.static.defines import NOT_SET @@ -80,3 +81,26 @@ def test_timeout_param(mocked_session, timeout_param, expected_connect_param, ex assert isinstance(kwargs['timeout'], aiohttp.ClientTimeout) assert kwargs['timeout'].connect == expected_connect_param assert kwargs['timeout'].sock_read == expected_sock_read_param + + +def test_clone(): + args = ['http://endpoint', 'project'] + kwargs = {'api_key': 'api_key', 'is_skipped_an_issue': False, 'verify_ssl': False, 'retries': 5, + 'max_pool_size': 30, 'http_timeout': (30, 30), 'keepalive_timeout': 25, 'mode': 'DEBUG', + 'launch_uuid_print': True, 'print_output': OutputType.STDERR} + client = Client(*args, **kwargs) + cloned = client.clone() + assert cloned is not None and client is not cloned + assert cloned.endpoint == args[0] and cloned.project == args[1] + assert ( + cloned.api_key == kwargs['api_key'] + and cloned.is_skipped_an_issue == kwargs['is_skipped_an_issue'] + and cloned.verify_ssl == kwargs['verify_ssl'] + and cloned.retries == kwargs['retries'] + and cloned.max_pool_size == kwargs['max_pool_size'] + and cloned.http_timeout == kwargs['http_timeout'] + and cloned.keepalive_timeout == kwargs['keepalive_timeout'] + and cloned.mode == kwargs['mode'] + and cloned.launch_uuid_print == kwargs['launch_uuid_print'] + and cloned.print_output == kwargs['print_output'] + ) From b3cb4331a004de5be13ae515aeeb9421d225d08f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 3 Oct 2023 13:27:16 +0300 Subject: [PATCH 186/268] Correct pydoc line length --- reportportal_client/aio/client.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index b72ab3dd..9652b9d4 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -652,11 +652,13 @@ def __init__( :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the server side. :param verify_ssl: Option to skip ssl verification. - :param retries: Number of retry attempts to make in case of connection / server errors. + :param retries: Number of retry attempts to make in case of connection / server + errors. :param max_pool_size: Option to set the maximum number of connections to save the pool. :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to specific connect and read separately. - :param keepalive_timeout: Maximum amount of idle time in seconds before force connection closing. + :param keepalive_timeout: Maximum amount of idle time in seconds before force connection + closing. :param mode: Launch mode, all Launches started by the client will be in that mode. :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. :param print_output: Set output stream for Launch UUID printing. @@ -1004,11 +1006,13 @@ def __init__( :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the server side. :param verify_ssl: Option to skip ssl verification. - :param retries: Number of retry attempts to make in case of connection / server errors. + :param retries: Number of retry attempts to make in case of connection / server + errors. :param max_pool_size: Option to set the maximum number of connections to save the pool. :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to specific connect and read separately. - :param keepalive_timeout: Maximum amount of idle time in seconds before force connection closing. + :param keepalive_timeout: Maximum amount of idle time in seconds before force connection + closing. :param mode: Launch mode, all Launches started by the client will be in that mode. :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. :param print_output: Set output stream for Launch UUID printing. @@ -1376,11 +1380,13 @@ def __init__( :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the server side. :param verify_ssl: Option to skip ssl verification. - :param retries: Number of retry attempts to make in case of connection / server errors. + :param retries: Number of retry attempts to make in case of connection / server + errors. :param max_pool_size: Option to set the maximum number of connections to save the pool. :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to specific connect and read separately. - :param keepalive_timeout: Maximum amount of idle time in seconds before force connection closing. + :param keepalive_timeout: Maximum amount of idle time in seconds before force connection + closing. :param mode: Launch mode, all Launches started by the client will be in that mode. :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. :param print_output: Set output stream for Launch UUID printing. @@ -1544,11 +1550,13 @@ def __init__( :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the server side. :param verify_ssl: Option to skip ssl verification. - :param retries: Number of retry attempts to make in case of connection / server errors. + :param retries: Number of retry attempts to make in case of connection / server + errors. :param max_pool_size: Option to set the maximum number of connections to save the pool. :param http_timeout: A float in seconds for connect and read timeout. Use a Tuple to specific connect and read separately. - :param keepalive_timeout: Maximum amount of idle time in seconds before force connection closing. + :param keepalive_timeout: Maximum amount of idle time in seconds before force connection + closing. :param mode: Launch mode, all Launches started by the client will be in that mode. :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. :param print_output: Set output stream for Launch UUID printing. From 13487ddb1d09974213cdd1f146ec754c662c2558 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 3 Oct 2023 13:44:36 +0300 Subject: [PATCH 187/268] Add clone test --- tests/aio/test_async_client.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/aio/test_async_client.py b/tests/aio/test_async_client.py index 9c3b9d10..6f5c2e29 100644 --- a/tests/aio/test_async_client.py +++ b/tests/aio/test_async_client.py @@ -21,3 +21,37 @@ def test_async_rp_client_pickling(): pickled_client = pickle.dumps(client) unpickled_client = pickle.loads(pickled_client) assert unpickled_client is not None + + +def test_clone(): + args = ['http://endpoint', 'project'] + kwargs = {'api_key': 'api_key1', 'launch_uuid': 'launch_uuid', 'log_batch_size': 30, + 'log_batch_payload_limit': 30 * 1024 * 1024} + async_client = AsyncRPClient(*args, **kwargs) + async_client._add_current_item('test-321') + async_client._add_current_item('test-322') + client = async_client.client + step_reporter = async_client.step_reporter + cloned = async_client.clone() + assert ( + cloned is not None + and async_client is not cloned + and cloned.client is not None + and cloned.client is not client + and cloned.step_reporter is not None + and cloned.step_reporter is not step_reporter + ) + assert ( + cloned.endpoint == args[0] + and cloned.project == args[1] + and cloned.client.endpoint == args[0] + and cloned.client.project == args[1] + ) + assert ( + cloned.client.api_key == kwargs['api_key'] + and cloned.launch_uuid == kwargs['launch_uuid'] + and cloned.log_batch_size == kwargs['log_batch_size'] + and cloned.log_batch_payload_limit == kwargs['log_batch_payload_limit'] + ) + assert cloned._item_stack.qsize() == 1 \ + and async_client.current_item() == cloned.current_item() From 290319f9d4acdfcb06b05326d7195daa80c1a9f5 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 3 Oct 2023 21:37:49 +0300 Subject: [PATCH 188/268] Add clone test --- reportportal_client/aio/client.py | 44 ++++++++++++++--------------- tests/aio/test_batched_client.py | 47 +++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 22 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 9652b9d4..eca2b2d1 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -1499,14 +1499,14 @@ class BatchedRPClient(_RPClient): bodies generation and serialization, connection retries and log batching. """ - _task_timeout: float - _shutdown_timeout: float + task_timeout: float + shutdown_timeout: float + trigger_num: int + trigger_interval: float _loop: asyncio.AbstractEventLoop _task_mutex: threading.RLock - __task_list: TriggerTaskBatcher[Task[_T]] + _task_list: TriggerTaskBatcher[Task[_T]] __last_run_time: float - __trigger_num: int - __trigger_interval: float def __init_task_list(self, task_list: Optional[TriggerTaskBatcher[Task[_T]]] = None, task_mutex: Optional[threading.RLock] = None): @@ -1518,7 +1518,7 @@ def __init_task_list(self, task_list: Optional[TriggerTaskBatcher[Task[_T]]] = N RuntimeWarning, 3 ) - self.__task_list = task_list or TriggerTaskBatcher(self.__trigger_num, self.__trigger_interval) + self._task_list = task_list or TriggerTaskBatcher(self.trigger_num, self.trigger_interval) self._task_mutex = task_mutex or threading.RLock() def __init_loop(self, loop: Optional[asyncio.AbstractEventLoop] = None): @@ -1581,10 +1581,10 @@ def __init__( :param trigger_interval: Time limit which triggers Task batch execution. """ super().__init__(endpoint, project, **kwargs) - self._task_timeout = task_timeout - self._shutdown_timeout = shutdown_timeout - self.__trigger_num = trigger_num - self.__trigger_interval = trigger_interval + self.task_timeout = task_timeout + self.shutdown_timeout = shutdown_timeout + self.trigger_num = trigger_num + self.trigger_interval = trigger_interval self.__init_task_list(task_list, task_mutex) self.__last_run_time = datetime.time() self.__init_loop(loop) @@ -1599,17 +1599,17 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: return result = self._loop.create_task(coro) with self._task_mutex: - tasks = self.__task_list.append(result) + tasks = self._task_list.append(result) if tasks: - self._loop.run_until_complete(asyncio.wait(tasks, timeout=self._task_timeout)) + self._loop.run_until_complete(asyncio.wait(tasks, timeout=self.task_timeout)) return result def finish_tasks(self) -> None: """Ensure all pending Tasks are finished, block current Thread if necessary.""" with self._task_mutex: - tasks = self.__task_list.flush() + tasks = self._task_list.flush() if tasks: - self._loop.run_until_complete(asyncio.wait(tasks, timeout=self._shutdown_timeout)) + self._loop.run_until_complete(asyncio.wait(tasks, timeout=self.shutdown_timeout)) logs = self._log_batcher.flush() if logs: log_task = self._loop.create_task(self._log_batch(logs)) @@ -1625,20 +1625,20 @@ def clone(self) -> 'BatchedRPClient': cloned_client = self.client.clone() # noinspection PyTypeChecker cloned = BatchedRPClient( - endpoint=None, - project=None, + endpoint=self.endpoint, + project=self.project, launch_uuid=self.launch_uuid, client=cloned_client, log_batch_size=self.log_batch_size, log_batch_payload_limit=self.log_batch_payload_limit, log_batcher=self._log_batcher, - task_timeout=self._task_timeout, - shutdown_timeout=self._shutdown_timeout, - task_list=self.__task_list, + task_timeout=self.task_timeout, + shutdown_timeout=self.shutdown_timeout, + task_list=self._task_list, task_mutex=self._task_mutex, loop=self._loop, - trigger_num=self.__trigger_num, - trigger_interval=self.__trigger_interval + trigger_num=self.trigger_num, + trigger_interval=self.trigger_interval ) current_item = self.current_item() if current_item: @@ -1663,5 +1663,5 @@ def __setstate__(self, state: Dict[str, Any]) -> None: :param dict state: object state dictionary """ self.__dict__.update(state) - self.__init_task_list(self.__task_list, threading.RLock()) + self.__init_task_list(self._task_list, threading.RLock()) self.__init_loop() diff --git a/tests/aio/test_batched_client.py b/tests/aio/test_batched_client.py index 55e1a714..0d6b07be 100644 --- a/tests/aio/test_batched_client.py +++ b/tests/aio/test_batched_client.py @@ -12,6 +12,7 @@ # limitations under the License import pickle +from reportportal_client.aio import BatchedTask from reportportal_client.aio import BatchedRPClient @@ -20,3 +21,49 @@ def test_batched_rp_client_pickling(): pickled_client = pickle.dumps(client) unpickled_client = pickle.loads(pickled_client) assert unpickled_client is not None + + +async def __empty_string(): + return "" + + +def test_clone(): + args = ['http://endpoint', 'project'] + kwargs = {'api_key': 'api_key1', 'launch_uuid': 'launch_uuid', 'log_batch_size': 30, + 'log_batch_payload_limit': 30 * 1024 * 1024, 'task_timeout': 63, 'shutdown_timeout': 123, + 'trigger_num': 25, 'trigger_interval': 3} + async_client = BatchedRPClient(*args, **kwargs) + async_client._add_current_item(BatchedTask(__empty_string(), loop=async_client._loop, name='test-321')) + async_client._add_current_item(BatchedTask(__empty_string(), loop=async_client._loop, name='test-322')) + client = async_client.client + step_reporter = async_client.step_reporter + cloned = async_client.clone() + assert ( + cloned is not None + and async_client is not cloned + and cloned.client is not None + and cloned.client is not client + and cloned.step_reporter is not None + and cloned.step_reporter is not step_reporter + and cloned._task_list is async_client._task_list + and cloned._task_mutex is async_client._task_mutex + and cloned._loop is async_client._loop + ) + assert ( + cloned.endpoint == args[0] + and cloned.project == args[1] + and cloned.client.endpoint == args[0] + and cloned.client.project == args[1] + ) + assert ( + cloned.client.api_key == kwargs['api_key'] + and cloned.launch_uuid == kwargs['launch_uuid'] + and cloned.log_batch_size == kwargs['log_batch_size'] + and cloned.log_batch_payload_limit == kwargs['log_batch_payload_limit'] + and cloned.task_timeout == kwargs['task_timeout'] + and cloned.shutdown_timeout == kwargs['shutdown_timeout'] + and cloned.trigger_num == kwargs['trigger_num'] + and cloned.trigger_interval == kwargs['trigger_interval'] + ) + assert cloned._item_stack.qsize() == 1 \ + and async_client.current_item() == cloned.current_item() From 9577a0fd48bd04ab86674f57673b26c8b3fdfce7 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 3 Oct 2023 22:12:06 +0300 Subject: [PATCH 189/268] Add clone test --- reportportal_client/aio/client.py | 34 +++++++++++----------- tests/aio/test_threaded_client.py | 47 ++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 18 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index eca2b2d1..dd89c10f 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -1323,12 +1323,12 @@ class ThreadedRPClient(_RPClient): bodies generation and serialization, connection retries and log batching. """ - _task_timeout: float - _shutdown_timeout: float - _loop: Optional[asyncio.AbstractEventLoop] + task_timeout: float + shutdown_timeout: float + _task_list: BackgroundTaskList[Task[_T]] _task_mutex: threading.RLock + _loop: Optional[asyncio.AbstractEventLoop] _thread: Optional[threading.Thread] - __task_list: BackgroundTaskList[Task[_T]] def __init_task_list(self, task_list: Optional[BackgroundTaskList[Task[_T]]] = None, task_mutex: Optional[threading.RLock] = None): @@ -1340,7 +1340,7 @@ def __init_task_list(self, task_list: Optional[BackgroundTaskList[Task[_T]]] = N RuntimeWarning, 3 ) - self.__task_list = task_list or BackgroundTaskList() + self._task_list = task_list or BackgroundTaskList() self._task_mutex = task_mutex or threading.RLock() def __heartbeat(self): @@ -1354,7 +1354,7 @@ def __init_loop(self, loop: Optional[asyncio.AbstractEventLoop] = None): self._loop = loop else: self._loop = asyncio.new_event_loop() - self._loop.set_task_factory(ThreadedTaskFactory(self._task_timeout)) + self._loop.set_task_factory(ThreadedTaskFactory(self.task_timeout)) self.__heartbeat() self._thread = threading.Thread(target=self._loop.run_forever, name='RP-Async-Client', daemon=True) @@ -1409,8 +1409,8 @@ def __init__( if this argument is None. """ super().__init__(endpoint, project, **kwargs) - self._task_timeout = task_timeout - self._shutdown_timeout = shutdown_timeout + self.task_timeout = task_timeout + self.shutdown_timeout = shutdown_timeout self.__init_task_list(task_list, task_mutex) self.__init_loop(loop) @@ -1424,17 +1424,17 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: return result = self._loop.create_task(coro) with self._task_mutex: - self.__task_list.append(result) + self._task_list.append(result) return result def finish_tasks(self): """Ensure all pending Tasks are finished, block current Thread if necessary.""" shutdown_start_time = datetime.time() with self._task_mutex: - tasks = self.__task_list.flush() + tasks = self._task_list.flush() for task in tasks: task.blocking_result() - if datetime.time() - shutdown_start_time >= self._shutdown_timeout: + if datetime.time() - shutdown_start_time >= self.shutdown_timeout: break logs = self._log_batcher.flush() if logs: @@ -1450,17 +1450,17 @@ def clone(self) -> 'ThreadedRPClient': cloned_client = self.client.clone() # noinspection PyTypeChecker cloned = ThreadedRPClient( - endpoint=None, - project=None, + endpoint=self.endpoint, + project=self.project, launch_uuid=self.launch_uuid, client=cloned_client, log_batch_size=self.log_batch_size, log_batch_payload_limit=self.log_batch_payload_limit, log_batcher=self._log_batcher, - task_timeout=self._task_timeout, - shutdown_timeout=self._shutdown_timeout, + task_timeout=self.task_timeout, + shutdown_timeout=self.shutdown_timeout, task_mutex=self._task_mutex, - task_list=self.__task_list, + task_list=self._task_list, loop=self._loop ) current_item = self.current_item() @@ -1487,7 +1487,7 @@ def __setstate__(self, state: Dict[str, Any]) -> None: :param dict state: object state dictionary """ self.__dict__.update(state) - self.__init_task_list(self.__task_list, threading.RLock()) + self.__init_task_list(self._task_list, threading.RLock()) self.__init_loop() diff --git a/tests/aio/test_threaded_client.py b/tests/aio/test_threaded_client.py index 3082c6e3..c8d64300 100644 --- a/tests/aio/test_threaded_client.py +++ b/tests/aio/test_threaded_client.py @@ -12,7 +12,7 @@ # limitations under the License import pickle -from reportportal_client.aio import ThreadedRPClient +from reportportal_client.aio import ThreadedRPClient, ThreadedTask def test_threaded_rp_client_pickling(): @@ -20,3 +20,48 @@ def test_threaded_rp_client_pickling(): pickled_client = pickle.dumps(client) unpickled_client = pickle.loads(pickled_client) assert unpickled_client is not None + + +async def __empty_string(): + return "" + + +def test_clone(): + args = ['http://endpoint', 'project'] + kwargs = {'api_key': 'api_key1', 'launch_uuid': 'launch_uuid', 'log_batch_size': 30, + 'log_batch_payload_limit': 30 * 1024 * 1024, 'task_timeout': 63, 'shutdown_timeout': 123} + async_client = ThreadedRPClient(*args, **kwargs) + async_client._add_current_item(ThreadedTask(__empty_string(), 64, loop=async_client._loop, + name='test-321')) + async_client._add_current_item(ThreadedTask(__empty_string(), 64, loop=async_client._loop, + name='test-322')) + client = async_client.client + step_reporter = async_client.step_reporter + cloned = async_client.clone() + assert ( + cloned is not None + and async_client is not cloned + and cloned.client is not None + and cloned.client is not client + and cloned.step_reporter is not None + and cloned.step_reporter is not step_reporter + and cloned._task_list is async_client._task_list + and cloned._task_mutex is async_client._task_mutex + and cloned._loop is async_client._loop + ) + assert ( + cloned.endpoint == args[0] + and cloned.project == args[1] + and cloned.client.endpoint == args[0] + and cloned.client.project == args[1] + ) + assert ( + cloned.client.api_key == kwargs['api_key'] + and cloned.launch_uuid == kwargs['launch_uuid'] + and cloned.log_batch_size == kwargs['log_batch_size'] + and cloned.log_batch_payload_limit == kwargs['log_batch_payload_limit'] + and cloned.task_timeout == kwargs['task_timeout'] + and cloned.shutdown_timeout == kwargs['shutdown_timeout'] + ) + assert cloned._item_stack.qsize() == 1 \ + and async_client.current_item() == cloned.current_item() From ac3f5d51c8e6341992f74bb7c1a5f65650e27c9c Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 3 Oct 2023 22:29:19 +0300 Subject: [PATCH 190/268] Fix for Python 3.7 --- reportportal_client/aio/tasks.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index a173b45c..aadd666f 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -100,7 +100,10 @@ def __init__( :param loop: Event Loop which will be used to execute the Task :param name: the name of the task """ - super().__init__(coro, loop=loop, name=name) + if sys.version_info < (3, 7): + super().__init__(coro, loop=loop) + else: + super().__init__(coro, loop=loop, name=name) self.__loop = loop def blocking_result(self) -> _T: @@ -133,7 +136,10 @@ def __init__( :param loop: Event Loop which will be used to execute the Task :param name: the name of the task """ - super().__init__(coro, loop=loop, name=name) + if sys.version_info < (3, 7): + super().__init__(coro, loop=loop) + else: + super().__init__(coro, loop=loop, name=name) self.__loop = loop self.__wait_timeout = wait_timeout From 5f9b2fca99e6054b3ddff1125a25f4d47dff13c3 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 3 Oct 2023 22:30:35 +0300 Subject: [PATCH 191/268] Fix for Python 3.7 --- reportportal_client/aio/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index aadd666f..fc89d95a 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -100,7 +100,7 @@ def __init__( :param loop: Event Loop which will be used to execute the Task :param name: the name of the task """ - if sys.version_info < (3, 7): + if sys.version_info < (3, 8): super().__init__(coro, loop=loop) else: super().__init__(coro, loop=loop, name=name) @@ -136,7 +136,7 @@ def __init__( :param loop: Event Loop which will be used to execute the Task :param name: the name of the task """ - if sys.version_info < (3, 7): + if sys.version_info < (3, 8): super().__init__(coro, loop=loop) else: super().__init__(coro, loop=loop, name=name) From 1c94489b5a4550b81ad04100d8794d03689b174a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 3 Oct 2023 23:14:39 +0300 Subject: [PATCH 192/268] Use 'importlib.metadata' for package data extraction for Python versions starting 3.8 --- reportportal_client/services/statistics.py | 14 +++++++-- tests/services/test_statistics.py | 34 +++++++--------------- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/reportportal_client/services/statistics.py b/reportportal_client/services/statistics.py index aaf7d646..c1fec324 100644 --- a/reportportal_client/services/statistics.py +++ b/reportportal_client/services/statistics.py @@ -13,6 +13,7 @@ """This module sends statistics events to a statistics service.""" +import sys import logging import ssl from platform import python_version @@ -21,7 +22,6 @@ import aiohttp import certifi import requests -from pkg_resources import get_distribution from reportportal_client.services.client_id import get_client_id from reportportal_client.services.constants import CLIENT_INFO, ENDPOINT @@ -36,8 +36,16 @@ def _get_client_info() -> Tuple[str, str]: :return: ('reportportal-client', '5.0.4') """ - client = get_distribution('reportportal-client') - return client.project_name, client.version + if sys.version_info < (3, 8): + from pkg_resources import get_distribution + client = get_distribution('reportportal-client') + name, version = client.project_name, client.version + else: + # noinspection PyCompatibility + from importlib.metadata import distribution + client = distribution('reportportal-client') + name, version = client.name.lower(), client.version + return name, version def _get_platform_info() -> str: diff --git a/tests/services/test_statistics.py b/tests/services/test_statistics.py index c5827b33..5253378f 100644 --- a/tests/services/test_statistics.py +++ b/tests/services/test_statistics.py @@ -21,6 +21,7 @@ # See the License for the specific language governing permissions and # limitations under the License +import re import sys from unittest import mock @@ -30,8 +31,13 @@ from reportportal_client.services.constants import ENDPOINT, CLIENT_INFO from reportportal_client.services.statistics import send_event, async_send_event +VERSION_VAR = '__version__' EVENT_NAME = 'start_launch' -EXPECTED_CL_VERSION, EXPECTED_CL_NAME = '5.0.4', 'reportportal-client' +with open('setup.py') as f: + EXPECTED_CL_VERSION = next( + map(lambda l: re.sub(f'^\\s*{VERSION_VAR}\\s*=\\s*[\'"]([^\'"]+)[\'"]', '\\g<1>', l), + filter(lambda x: VERSION_VAR in x, f.read().splitlines()))) +EXPECTED_CL_NAME = 'reportportal-client' AGENT_VERSION, AGENT_NAME = '5.0.5', 'pytest-reportportal' EXPECTED_HEADERS = {'User-Agent': 'python-requests'} EXPECTED_AIO_HEADERS = {'User-Agent': 'python-aiohttp'} @@ -55,17 +61,13 @@ @mock.patch('reportportal_client.services.statistics.get_client_id', mock.Mock(return_value='555')) @mock.patch('reportportal_client.services.statistics.requests.post') -@mock.patch('reportportal_client.services.statistics.get_distribution') @mock.patch('reportportal_client.services.statistics.python_version', mock.Mock(return_value='3.6.6')) -def test_send_event(mocked_distribution, mocked_requests): +def test_send_event(mocked_requests): """Test functionality of the send_event() function. - :param mocked_distribution: Mocked get_distribution() function :param mocked_requests: Mocked requests module """ - mocked_distribution.return_value.version = EXPECTED_CL_VERSION - mocked_distribution.return_value.project_name = EXPECTED_CL_NAME send_event(EVENT_NAME, AGENT_NAME, AGENT_VERSION) mocked_requests.assert_called_with( @@ -77,27 +79,20 @@ def test_send_event(mocked_distribution, mocked_requests): mock.Mock(return_value='555')) @mock.patch('reportportal_client.services.statistics.requests.post', mock.Mock(side_effect=RequestException)) -@mock.patch('reportportal_client.services.statistics.get_distribution', - mock.Mock()) def test_send_event_raises(): """Test that the send_event() does not raise exceptions.""" send_event(EVENT_NAME, 'pytest-reportportal', '5.0.5') @mock.patch('reportportal_client.services.statistics.requests.post') -@mock.patch('reportportal_client.services.statistics.get_distribution') @mock.patch('reportportal_client.services.statistics.python_version', mock.Mock(return_value='3.6.6')) -def test_same_client_id(mocked_distribution, mocked_requests): +def test_same_client_id(mocked_requests): """Test functionality of the send_event() function. - :param mocked_distribution: Mocked get_distribution() function :param mocked_requests: Mocked requests module """ - expected_cl_version, expected_cl_name = '5.0.4', 'reportportal-client' agent_version, agent_name = '5.0.5', 'pytest-reportportal' - mocked_distribution.return_value.version = expected_cl_version - mocked_distribution.return_value.project_name = expected_cl_name send_event(EVENT_NAME, agent_name, agent_version) send_event(EVENT_NAME, agent_name, agent_version) @@ -119,18 +114,11 @@ def test_same_client_id(mocked_distribution, mocked_requests): @mock.patch('reportportal_client.services.statistics.get_client_id', mock.Mock(return_value='555')) @mock.patch('reportportal_client.services.statistics.aiohttp.ClientSession.post', MOCKED_AIOHTTP) -@mock.patch('reportportal_client.services.statistics.get_distribution') @mock.patch('reportportal_client.services.statistics.python_version', mock.Mock(return_value='3.6.6')) @pytest.mark.asyncio -async def test_async_send_event(mocked_distribution): - """Test functionality of the send_event() function. - - :param mocked_distribution: Mocked get_distribution() function - """ - mocked_distribution.return_value.version = EXPECTED_CL_VERSION - mocked_distribution.return_value.project_name = EXPECTED_CL_NAME - +async def test_async_send_event(): + """Test functionality of the send_event() function.""" await async_send_event(EVENT_NAME, AGENT_NAME, AGENT_VERSION) assert len(MOCKED_AIOHTTP.call_args_list) == 1 args, kwargs = MOCKED_AIOHTTP.call_args_list[0] From 769e6098da6031bdfa549e87fd43bdee2fd912a8 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 3 Oct 2023 23:16:08 +0300 Subject: [PATCH 193/268] CHANGELOG.md update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07c40cbc..f6fa4a05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Dependency on `aiohttp` and `certifi`, by @HardNorth ### Changed - RPClient class does not use separate Thread for log processing anymore, by @HardNorth +- Use `importlib.metadata` package for distribution data extraction for Python versions starting 3.8, by @HardNorth ### Removed - Dependency on `six`, by @HardNorth From 5925cc7cc362271de71be425bd8a06d988c2df3d Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 3 Oct 2023 23:24:24 +0300 Subject: [PATCH 194/268] Fix for Python 3.7 --- reportportal_client/aio/tasks.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index fc89d95a..28f772fc 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -40,6 +40,8 @@ class Task(Generic[_T], asyncio.Task, metaclass=AbstractBaseClass): __metaclass__ = AbstractBaseClass + name: Optional[str] + def __init__( self, coro: Union[Generator[Future, None, _T], Awaitable[_T]], @@ -53,7 +55,11 @@ def __init__( :param loop: Event Loop which will be used to execute the Task :param name: the name of the task """ - super().__init__(coro, loop=loop, name=name) + self.name = name + if sys.version_info < (3, 8): + super().__init__(coro, loop=loop) + else: + super().__init__(coro, loop=loop, name=name) @abstractmethod def blocking_result(self) -> _T: @@ -100,10 +106,7 @@ def __init__( :param loop: Event Loop which will be used to execute the Task :param name: the name of the task """ - if sys.version_info < (3, 8): - super().__init__(coro, loop=loop) - else: - super().__init__(coro, loop=loop, name=name) + super().__init__(coro, loop=loop, name=name) self.__loop = loop def blocking_result(self) -> _T: @@ -136,10 +139,7 @@ def __init__( :param loop: Event Loop which will be used to execute the Task :param name: the name of the task """ - if sys.version_info < (3, 8): - super().__init__(coro, loop=loop) - else: - super().__init__(coro, loop=loop, name=name) + super().__init__(coro, loop=loop, name=name) self.__loop = loop self.__wait_timeout = wait_timeout From e0c1e1734d6b845cabdfb94684f4e3130fee43fe Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 3 Oct 2023 23:33:03 +0300 Subject: [PATCH 195/268] A try to fix tests for python 3.8, 3.9 --- reportportal_client/services/statistics.py | 2 +- tests/test_client.py | 23 ++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/reportportal_client/services/statistics.py b/reportportal_client/services/statistics.py index c1fec324..36130ae2 100644 --- a/reportportal_client/services/statistics.py +++ b/reportportal_client/services/statistics.py @@ -44,7 +44,7 @@ def _get_client_info() -> Tuple[str, str]: # noinspection PyCompatibility from importlib.metadata import distribution client = distribution('reportportal-client') - name, version = client.name.lower(), client.version + name, version = client.metadata['Name'], client.metadata['Version'] return name, version diff --git a/tests/test_client.py b/tests/test_client.py index c2ea9dc7..7927218f 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -156,16 +156,19 @@ def test_clone(): cloned = client.clone() assert cloned is not None and client is not cloned assert cloned.endpoint == args[0] and cloned.project == args[1] - assert cloned.api_key == kwargs[ - 'api_key'] and cloned.log_batch_size == kwargs[ - 'log_batch_size'] and cloned.is_skipped_an_issue == kwargs[ - 'is_skipped_an_issue'] and cloned.verify_ssl == kwargs[ - 'verify_ssl'] and cloned.retries == kwargs[ - 'retries'] and cloned.max_pool_size == kwargs[ - 'max_pool_size'] and cloned.launch_id == kwargs[ - 'launch_id'] and cloned.http_timeout == kwargs[ - 'http_timeout'] and cloned.log_batch_payload_size == kwargs[ - 'log_batch_payload_size'] and cloned.mode == kwargs['mode'] + assert ( + cloned.api_key == kwargs['api_key'] + and cloned.log_batch_size == kwargs['log_batch_size'] + and cloned.is_skipped_an_issue == kwargs['is_skipped_an_issue'] + and cloned.verify_ssl == kwargs['verify_ssl'] + and cloned.retries == kwargs['retries'] + and cloned.max_pool_size == kwargs['max_pool_size'] + and cloned.launch_uuid == kwargs['launch_id'] + and cloned.launch_id == kwargs['launch_id'] + and cloned.http_timeout == kwargs['http_timeout'] + and cloned.log_batch_payload_size == kwargs['log_batch_payload_size'] + and cloned.mode == kwargs['mode'] + ) assert cloned._item_stack.qsize() == 1 \ and client.current_item() == cloned.current_item() From e3035f36d7f595de85843a82852fbc6a5af8024f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 3 Oct 2023 23:46:06 +0300 Subject: [PATCH 196/268] Client.session method made async --- reportportal_client/aio/client.py | 21 ++++++++++----------- tests/aio/test_aio_client.py | 10 ++++++---- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index dd89c10f..b7139566 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -157,8 +157,7 @@ def __init__( stacklevel=2 ) - @property - def session(self) -> aiohttp.ClientSession: + async def session(self) -> aiohttp.ClientSession: """Return aiohttp.ClientSession class instance, initialize it if necessary. :return: aiohttp.ClientSession instance. @@ -261,7 +260,7 @@ async def start_launch(self, rerun_of=rerun_of ).payload - response = await AsyncHttpRequest(self.session.post, url=url, json=request_payload).make() + response = await AsyncHttpRequest((await self.session()).post, url=url, json=request_payload).make() if not response: return @@ -327,7 +326,7 @@ async def start_test_item(self, test_case_id=test_case_id ).payload - response = await AsyncHttpRequest(self.session.post, url=url, json=request_payload).make() + response = await AsyncHttpRequest((await self.session()).post, url=url, json=request_payload).make() if not response: return item_id = await response.id @@ -373,7 +372,7 @@ async def finish_test_item(self, issue=issue, retry=retry ).payload - response = await AsyncHttpRequest(self.session.put, url=url, json=request_payload).make() + response = await AsyncHttpRequest((await self.session()).put, url=url, json=request_payload).make() if not response: return message = await response.message @@ -404,7 +403,7 @@ async def finish_launch(self, attributes=attributes, description=kwargs.get('description') ).payload - response = await AsyncHttpRequest(self.session.put, url=url, json=request_payload, + response = await AsyncHttpRequest((await self.session()).put, url=url, json=request_payload, name='Finish Launch').make() if not response: return @@ -431,7 +430,7 @@ async def update_test_item(self, } item_id = await self.get_item_id_by_uuid(item_uuid) url = root_uri_join(self.base_url_v1, 'item', item_id, 'update') - response = await AsyncHttpRequest(self.session.put, url=url, json=data).make() + response = await AsyncHttpRequest((await self.session()).put, url=url, json=data).make() if not response: return logger.debug('update_test_item - Item: %s', item_id) @@ -452,7 +451,7 @@ async def get_launch_info(self, launch_uuid_future: Union[str, Task[str]]) -> Op :return: Launch information in dictionary. """ url = self.__get_launch_uuid_url(launch_uuid_future) - response = await AsyncHttpRequest(self.session.get, url=url).make() + response = await AsyncHttpRequest((await self.session()).get, url=url).make() if not response: return if response.is_success: @@ -477,7 +476,7 @@ async def get_item_id_by_uuid(self, item_uuid_future: Union[str, Task[str]]) -> :return: Test Item ID. """ url = self.__get_item_uuid_url(item_uuid_future) - response = await AsyncHttpRequest(self.session.get, url=url).make() + response = await AsyncHttpRequest((await self.session()).get, url=url).make() return response.id if response else None async def get_launch_ui_id(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[int]: @@ -519,7 +518,7 @@ async def get_project_settings(self) -> Optional[Dict]: :return: Settings response in Dictionary. """ url = root_uri_join(self.base_url_v1, 'settings') - response = await AsyncHttpRequest(self.session.get, url=url).make() + response = await AsyncHttpRequest((await self.session()).get, url=url).make() return await response.json if response else None async def log_batch(self, log_batch: Optional[List[AsyncRPRequestLog]]) -> Tuple[str, ...]: @@ -530,7 +529,7 @@ async def log_batch(self, log_batch: Optional[List[AsyncRPRequestLog]]) -> Tuple """ url = root_uri_join(self.base_url_v2, 'log') if log_batch: - response = await AsyncHttpRequest(self.session.post, url=url, + response = await AsyncHttpRequest((await self.session()).post, url=url, data=AsyncRPLogBatch(log_batch).payload).make() return await response.messages diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 66e99288..e107749c 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -44,9 +44,10 @@ def test_client_pickling(): (NOT_SET, RetryingClientSession, DEFAULT_RETRY_NUMBER) ] ) -def test_retries_param(retry_num, expected_class, expected_param): +@pytest.mark.asyncio +async def test_retries_param(retry_num, expected_class, expected_param): client = Client(ENDPOINT, PROJECT, api_key=API_KEY, retries=retry_num) - session = client.session + session = await client.session() assert isinstance(session, expected_class) if expected_param is not NOT_SET: assert getattr(session, '_RetryingClientSession__retry_number') == expected_param @@ -61,9 +62,10 @@ def test_retries_param(retry_num, expected_class, expected_param): ] ) @mock.patch('reportportal_client.aio.client.RetryingClientSession') -def test_timeout_param(mocked_session, timeout_param, expected_connect_param, expected_sock_read_param): +@pytest.mark.asyncio +async def test_timeout_param(mocked_session, timeout_param, expected_connect_param, expected_sock_read_param): client = Client(ENDPOINT, PROJECT, api_key=API_KEY, http_timeout=timeout_param) - session = client.session + session = await client.session() assert session is not None assert len(mocked_session.call_args_list) == 1 args, kwargs = mocked_session.call_args_list[0] From 54f5be869161c404acf5a01fd1bee52f8fe73787 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 00:09:22 +0300 Subject: [PATCH 197/268] Add test skip --- tests/aio/test_aio_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index e107749c..444a77e4 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -10,7 +10,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License - +import sys import pickle from unittest import mock @@ -53,6 +53,9 @@ async def test_retries_param(retry_num, expected_class, expected_param): assert getattr(session, '_RetryingClientSession__retry_number') == expected_param +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="For some reasons this does not work on Python 3.7 on Ubuntu, " + "but works on my Mac. Unfortunately GHA use Python 3.7 on Ubuntu.") @pytest.mark.parametrize( 'timeout_param, expected_connect_param, expected_sock_read_param', [ From 0ae3df1ff1cc5e7bd6dedd99dd2d1d47fc47268f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 13:18:57 +0300 Subject: [PATCH 198/268] Add Python 3.12 to supported versions --- .github/workflows/tests.yml | 2 +- setup.py | 1 + tox.ini | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2e7d9d44..6cbcf93a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ] + python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12' ] steps: - name: Checkout repository uses: actions/checkout@v3 diff --git a/setup.py b/setup.py index f418ec46..44d73b62 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ def read_file(fname): 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ], install_requires=read_file('requirements.txt').splitlines(), ) diff --git a/tox.ini b/tox.ini index 2f8dad03..fbfbada4 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ envlist = py39 py310 py311 + py312 [testenv] deps = @@ -30,3 +31,4 @@ python = 3.9: py39 3.10: py310 3.11: py311 + 3.12: py312 From 0ea9b9aa64986dc31560acb849bc39b250d03564 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 13:23:40 +0300 Subject: [PATCH 199/268] A try to fix build for Python 3.12 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6c8a44c8..b0367162 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ aenum requests>=2.28.0 -aiohttp>=3.8.3 +aiohttp>=3.8.5 certifi>=2023.7.22 From a8c244369e406b37d37c52b63bc8c71f82ab7855 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 13:28:05 +0300 Subject: [PATCH 200/268] Revert "A try to fix build for Python 3.12" This reverts commit 0ea9b9aa64986dc31560acb849bc39b250d03564. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b0367162..6c8a44c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ aenum requests>=2.28.0 -aiohttp>=3.8.5 +aiohttp>=3.8.3 certifi>=2023.7.22 From f83f573cae4bd5a8c79da1e88046a71fa8ed0ddb Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 13:28:05 +0300 Subject: [PATCH 201/268] Revert "Add Python 3.12 to supported versions" This reverts commit 0ae3df1ff1cc5e7bd6dedd99dd2d1d47fc47268f. --- .github/workflows/tests.yml | 2 +- setup.py | 1 - tox.ini | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6cbcf93a..2e7d9d44 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12' ] + python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ] steps: - name: Checkout repository uses: actions/checkout@v3 diff --git a/setup.py b/setup.py index 44d73b62..f418ec46 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,6 @@ def read_file(fname): 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', ], install_requires=read_file('requirements.txt').splitlines(), ) diff --git a/tox.ini b/tox.ini index fbfbada4..2f8dad03 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,6 @@ envlist = py39 py310 py311 - py312 [testenv] deps = @@ -31,4 +30,3 @@ python = 3.9: py39 3.10: py310 3.11: py311 - 3.12: py312 From 43035f5953158374f21d12104b9388cf14790f27 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 14:17:06 +0300 Subject: [PATCH 202/268] Add launch url tests --- reportportal_client/aio/client.py | 4 ++-- tests/aio/test_aio_client.py | 33 +++++++++++++++++++++++++++++++ tests/conftest.py | 9 +++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index b7139566..3db99ed0 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -37,7 +37,7 @@ AsyncItemFinishRequest, LaunchFinishRequest, RPFile, AsyncRPRequestLog, AsyncRPLogBatch) from reportportal_client.helpers import (root_uri_join, verify_value_length, await_if_necessary, - agent_name_version, LifoQueue) + agent_name_version, LifoQueue, uri_join) from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE from reportportal_client.logs.batcher import LogBatcher from reportportal_client.services.statistics import async_send_event @@ -508,7 +508,7 @@ async def get_launch_ui_url(self, launch_uuid_future: Union[str, Task[str]]) -> path = 'ui/#{project_name}/{launch_type}/all/{launch_id}'.format( project_name=self.project.lower(), launch_type=launch_type, launch_id=launch_id) - url = root_uri_join(self.endpoint, path) + url = uri_join(self.endpoint, path) logger.debug('get_launch_ui_url - ID: %s', launch_uuid) return url diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 444a77e4..cd440697 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -109,3 +109,36 @@ def test_clone(): and cloned.launch_uuid_print == kwargs['launch_uuid_print'] and cloned.print_output == kwargs['print_output'] ) + + +LAUNCH_ID = 333 +EXPECTED_DEFAULT_URL = 'http://endpoint/ui/#project/launches/all/' + str( + LAUNCH_ID) +EXPECTED_DEBUG_URL = 'http://endpoint/ui/#project/userdebug/all/' + str( + LAUNCH_ID) + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@pytest.mark.parametrize( + 'launch_mode, project_name, expected_url', + [ + ('DEFAULT', "project", EXPECTED_DEFAULT_URL), + ('DEBUG', "project", EXPECTED_DEBUG_URL), + ('DEFAULT', "PROJECT", EXPECTED_DEFAULT_URL), + ('debug', "PROJECT", EXPECTED_DEBUG_URL) + ] +) +@pytest.mark.asyncio +async def test_launch_url_get(aio_client, launch_mode, project_name, expected_url): + aio_client.project = project_name + response = mock.AsyncMock() + response.is_success = True + response.json.side_effect = lambda: {'mode': launch_mode, 'id': LAUNCH_ID} + + async def get_call(*args, **kwargs): + return response + + (await aio_client.session()).get.side_effect = get_call + + assert await (aio_client.get_launch_ui_url('test_launch_uuid')) == expected_url diff --git a/tests/conftest.py b/tests/conftest.py index 79d1402d..0d0cc2b3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ from pytest import fixture from reportportal_client.client import RPClient +from reportportal_client.aio.client import Client @fixture() @@ -33,3 +34,11 @@ def rp_client(): client = RPClient('http://endpoint', 'project', 'api_key') client.session = mock.Mock() return client + + +@fixture +def aio_client(): + """Prepare instance of the Client for testing.""" + client = Client('http://endpoint', 'project', api_key='api_key') + client._session = mock.AsyncMock() + return client From ecefaa5b59f6e39dcfcdfdeeee70045bdf0a05f0 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 14:22:26 +0300 Subject: [PATCH 203/268] Remove unused code --- reportportal_client/helpers.py | 13 ------------- tests/test_helpers.py | 6 ------ 2 files changed, 19 deletions(-) diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index 02d328d6..049d89cf 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -160,19 +160,6 @@ def get_launch_sys_attrs() -> Dict[str, str]: } -def get_package_version(package_name) -> str: - """Get version of the given package. - - :param package_name: Name of the package - :return: Version of the package - """ - try: - package_version = get_distribution(package_name).version - except DistributionNotFound: - package_version = 'not found' - return package_version - - def verify_value_length(attributes: List[dict]) -> List[dict]: """Verify length of the attribute value. diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 3c71e144..40f5a868 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -18,7 +18,6 @@ from reportportal_client.helpers import ( gen_attributes, get_launch_sys_attrs, - get_package_version, verify_value_length ) @@ -60,11 +59,6 @@ def test_get_launch_sys_attrs_docker(): assert result['cpu'] == 'unknown' -def test_get_package_version(): - """Test for the get_package_version() function-helper.""" - assert get_package_version('noname') == 'not found' - - def test_verify_value_length(): """Test for validate verify_value_length() function.""" inputl = [{'key': 'tn', 'value': 'v' * 130}, [1, 2], From 8279add2c3ad7c644a3937e9a5a809b7fd79bee9 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 14:36:01 +0300 Subject: [PATCH 204/268] Remove unused code --- reportportal_client/helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index 049d89cf..5284f7c8 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -23,8 +23,6 @@ from platform import machine, processor, system from typing import Optional, Any, List, Dict, Callable, Tuple, Union, TypeVar, Generic -from pkg_resources import DistributionNotFound, get_distribution - from reportportal_client.core.rp_file import RPFile from reportportal_client.static.defines import ATTRIBUTE_LENGTH_LIMIT From 1d3f8152fab2b34b63beaed1e1e289791e36fe75 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 16:39:04 +0300 Subject: [PATCH 205/268] Add debug printing --- tests/logs/test_rp_log_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/logs/test_rp_log_handler.py b/tests/logs/test_rp_log_handler.py index d2b3e165..1c1dedfb 100644 --- a/tests/logs/test_rp_log_handler.py +++ b/tests/logs/test_rp_log_handler.py @@ -96,7 +96,8 @@ def test_emit_custom_level(mocked_handle): log_handler = RPLogHandler() log_handler.emit(record) assert mock_client.log.call_count == 1 - call_kwargs = mock_client.log.call_args[1] + call_args, call_kwargs = mock_client.log.call_args + print("\n" + str(call_args) + "," + str(call_kwargs)) assert call_kwargs['level'] == 'WARN' From f905945797b4236935ae4336180f5e0316ac8426 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 17:12:51 +0300 Subject: [PATCH 206/268] Fix warning which affects other tests --- tests/aio/test_batched_client.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/aio/test_batched_client.py b/tests/aio/test_batched_client.py index 0d6b07be..33b8f8b5 100644 --- a/tests/aio/test_batched_client.py +++ b/tests/aio/test_batched_client.py @@ -33,8 +33,12 @@ def test_clone(): 'log_batch_payload_limit': 30 * 1024 * 1024, 'task_timeout': 63, 'shutdown_timeout': 123, 'trigger_num': 25, 'trigger_interval': 3} async_client = BatchedRPClient(*args, **kwargs) - async_client._add_current_item(BatchedTask(__empty_string(), loop=async_client._loop, name='test-321')) - async_client._add_current_item(BatchedTask(__empty_string(), loop=async_client._loop, name='test-322')) + task1 = async_client.create_task(__empty_string()) + task2 = async_client.create_task(__empty_string()) + task1.blocking_result() + task2.blocking_result() + async_client._add_current_item(task1) + async_client._add_current_item(task2) client = async_client.client step_reporter = async_client.step_reporter cloned = async_client.clone() From 7929da70fa4f9fbdfe1ca31265618fa5ff0c9525 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 17:16:22 +0300 Subject: [PATCH 207/268] Fix flake8 --- tests/aio/test_batched_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/aio/test_batched_client.py b/tests/aio/test_batched_client.py index 33b8f8b5..bfce3c5b 100644 --- a/tests/aio/test_batched_client.py +++ b/tests/aio/test_batched_client.py @@ -12,7 +12,6 @@ # limitations under the License import pickle -from reportportal_client.aio import BatchedTask from reportportal_client.aio import BatchedRPClient From 90a033d19737b9de4b9da8389cf666fe6c91a3c0 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 17:51:29 +0300 Subject: [PATCH 208/268] Refactor RetryingClientSession class --- reportportal_client/aio/client.py | 5 +-- reportportal_client/aio/http.py | 52 ++++++++++++++++++++++--------- tests/aio/test_http.py | 14 +++------ 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 3db99ed0..9ebb235c 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -82,7 +82,7 @@ class Client: launch_uuid_print: bool print_output: OutputType _skip_analytics: str - _session: Optional[aiohttp.ClientSession] + _session: Optional[RetryingClientSession] __stat_task: Optional[asyncio.Task] def __init__( @@ -157,7 +157,7 @@ def __init__( stacklevel=2 ) - async def session(self) -> aiohttp.ClientSession: + async def session(self) -> RetryingClientSession: """Return aiohttp.ClientSession class instance, initialize it if necessary. :return: aiohttp.ClientSession instance. @@ -206,6 +206,7 @@ async def session(self) -> aiohttp.ClientSession: if use_retries: self._session = RetryingClientSession(self.endpoint, **session_params) else: + # noinspection PyTypeChecker self._session = aiohttp.ClientSession(self.endpoint, **session_params) return self._session diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py index 7150ea8b..3a3786a0 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/aio/http.py @@ -15,7 +15,8 @@ import asyncio import sys -from typing import Coroutine +from types import TracebackType +from typing import Coroutine, Any, Optional, Type from aenum import Enum from aiohttp import ClientSession, ClientResponse, ServerConnectionError, \ @@ -35,9 +36,10 @@ class RetryClass(int, Enum): THROTTLING = 3 -class RetryingClientSession(ClientSession): - """Class extends aiohttp.ClientSession functionality with request retry logic.""" +class RetryingClientSession: + """Class uses aiohttp.ClientSession.request method and adds request retry logic.""" + _client: ClientSession __retry_number: int __retry_delay: float @@ -59,7 +61,7 @@ def __init__( an error. Real value highly depends on Retry Class and Retry attempt number, since retries are performed in exponential delay manner """ - super().__init__(*args, **kwargs) + self._client = ClientSession(*args, **kwargs) self.__retry_number = max_retry_number self.__retry_delay = base_retry_delay @@ -73,24 +75,22 @@ def __sleep(self, retry_num: int, retry_factor: int) -> Coroutine: else: return self.__nothing() - async def _request( - self, - *args, - **kwargs + async def request( + self, method: str, url: str, **kwargs: Any ) -> ClientResponse: """Make a request and retry if necessary. - The method overrides aiohttp.ClientSession._request() method and bypass all arguments to it, so - please refer it for detailed argument description. The method retries requests depending on error - class and retry number. For no-retry errors, such as 400 Bad Request it just returns result, for cases - where it's reasonable to retry it does it in exponential manner. - """ + The method overrides aiohttp.ClientSession._request() method and bypass all arguments to it, so + please refer it for detailed argument description. The method retries requests depending on error + class and retry number. For no-retry errors, such as 400 Bad Request it just returns result, for cases + where it's reasonable to retry it does it in exponential manner. + """ result = None exceptions = [] for i in range(self.__retry_number + 1): # add one for the first attempt, which is not a retry retry_factor = None try: - result = await super()._request(*args, **kwargs) + result = await self._client.request(method, url, **kwargs) except Exception as exc: exceptions.append(exc) if isinstance(exc, ServerConnectionError) or isinstance(exc, ClientResponseError): @@ -125,3 +125,27 @@ class and retry number. For no-retry errors, such as 400 Bad Request it just ret else: raise exceptions[0] return result + + def get(self, url: str, *, allow_redirects: bool = True, + **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]: + return self.request('GET', url, allow_redirects=allow_redirects, **kwargs) + + def post(self, url: str, *, data: Any = None, **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]: + return self.request('POST', url, data=data, **kwargs) + + def put(self, url: str, *, data: Any = None, **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]: + return self.request('PUT', url, data=data, **kwargs) + + async def close(self) -> Coroutine: + return self._client.close() + + async def __aenter__(self) -> "RetryingClientSession": + return self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + await self.close() diff --git a/tests/aio/test_http.py b/tests/aio/test_http.py index 74a642ef..62785eea 100644 --- a/tests/aio/test_http.py +++ b/tests/aio/test_http.py @@ -12,12 +12,12 @@ # limitations under the License import http.server import socketserver -import sys import threading import time from unittest import mock import aiohttp +# noinspection PyPackageRequirements import pytest from reportportal_client.aio.http import RetryingClientSession @@ -85,13 +85,13 @@ async def execute_http_request(port, retry_number, server_class, timeout_seconds session = RetryingClientSession(f'{protocol}://localhost:{port}', timeout=timeout, max_retry_number=retry_number, base_retry_delay=0.01, connector=connector) # noinspection PyProtectedMember - parent_request = super(type(session), session)._request - async_mock = mock.AsyncMock() + parent_request = session._client.request + async_mock = mock.Mock() async_mock.side_effect = parent_request exception = None result = None with get_http_server(server_handler=server_class, server_address=('', port)): - with mock.patch('reportportal_client.aio.http.ClientSession._request', async_mock): + with mock.patch('reportportal_client.aio.http.ClientSession.request', async_mock): async with session: start_time = time.time() try: @@ -102,8 +102,6 @@ async def execute_http_request(port, retry_number, server_class, timeout_seconds return async_mock, exception, result, total_time -@pytest.mark.skipif(sys.version_info < (3, 8), - reason="the test requires AsyncMock which was introduced in Python 3.8") @pytest.mark.parametrize( 'server_class, port, expected_delay, timeout_seconds, is_exception', [ @@ -128,8 +126,6 @@ async def test_retry_on_request_error(server_class, port, expected_delay, timeou assert total_time < expected_delay * 1.5 -@pytest.mark.skipif(sys.version_info < (3, 8), - reason="the test requires AsyncMock which was introduced in Python 3.8") @pytest.mark.asyncio async def test_no_retry_on_ok_request(): retry_number = 5 @@ -142,8 +138,6 @@ async def test_no_retry_on_ok_request(): assert total_time < 1 -@pytest.mark.skipif(sys.version_info < (3, 8), - reason="the test requires AsyncMock which was introduced in Python 3.8") @pytest.mark.asyncio async def test_no_retry_on_not_retryable_error(): retry_number = 5 From e91bb1432eeabe3449aaf19149ad173576c85e7a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 18:02:51 +0300 Subject: [PATCH 209/268] Fix docstrings and close method --- reportportal_client/aio/http.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py index 3a3786a0..3114b8d7 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/aio/http.py @@ -80,11 +80,11 @@ async def request( ) -> ClientResponse: """Make a request and retry if necessary. - The method overrides aiohttp.ClientSession._request() method and bypass all arguments to it, so - please refer it for detailed argument description. The method retries requests depending on error - class and retry number. For no-retry errors, such as 400 Bad Request it just returns result, for cases - where it's reasonable to retry it does it in exponential manner. - """ + The method uses aiohttp.ClientSession.request() method and bypass all arguments to it, so + please refer it for detailed argument description. The method retries requests depending on error + class and retry number. For no-retry errors, such as 400 Bad Request it just returns result, for cases + where it's reasonable to retry it does it in exponential manner. + """ result = None exceptions = [] for i in range(self.__retry_number + 1): # add one for the first attempt, which is not a retry @@ -128,18 +128,23 @@ class and retry number. For no-retry errors, such as 400 Bad Request it just ret def get(self, url: str, *, allow_redirects: bool = True, **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]: + """Perform HTTP GET request.""" return self.request('GET', url, allow_redirects=allow_redirects, **kwargs) def post(self, url: str, *, data: Any = None, **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]: + """Perform HTTP POST request.""" return self.request('POST', url, data=data, **kwargs) def put(self, url: str, *, data: Any = None, **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]: + """Perform HTTP PUT request.""" return self.request('PUT', url, data=data, **kwargs) - async def close(self) -> Coroutine: + def close(self) -> Coroutine: + """Gracefully close internal aiohttp.ClientSession class instance.""" return self._client.close() async def __aenter__(self) -> "RetryingClientSession": + """Auxiliary method which controls what `async with` construction does on block enter.""" return self async def __aexit__( @@ -148,4 +153,5 @@ async def __aexit__( exc_val: Optional[BaseException], exc_tb: Optional[TracebackType], ) -> None: + """Auxiliary method which controls what `async with` construction does on block exit.""" await self.close() From 0d597e9359a8c5a0d3dfa20a67cf96d77254782d Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 4 Oct 2023 18:09:24 +0300 Subject: [PATCH 210/268] Remove debug print --- tests/logs/test_rp_log_handler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/logs/test_rp_log_handler.py b/tests/logs/test_rp_log_handler.py index 1c1dedfb..01357456 100644 --- a/tests/logs/test_rp_log_handler.py +++ b/tests/logs/test_rp_log_handler.py @@ -97,7 +97,6 @@ def test_emit_custom_level(mocked_handle): log_handler.emit(record) assert mock_client.log.call_count == 1 call_args, call_kwargs = mock_client.log.call_args - print("\n" + str(call_args) + "," + str(call_kwargs)) assert call_kwargs['level'] == 'WARN' From 446387e32313f1446469b84316d87d0b88cb3847 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 5 Oct 2023 12:00:32 +0300 Subject: [PATCH 211/268] Update http module --- reportportal_client/aio/http.py | 14 +++++++------- tests/aio/test_http.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py index 3114b8d7..a3e81ca4 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/aio/http.py @@ -16,7 +16,7 @@ import asyncio import sys from types import TracebackType -from typing import Coroutine, Any, Optional, Type +from typing import Coroutine, Any, Optional, Type, Callable from aenum import Enum from aiohttp import ClientSession, ClientResponse, ServerConnectionError, \ @@ -75,8 +75,8 @@ def __sleep(self, retry_num: int, retry_factor: int) -> Coroutine: else: return self.__nothing() - async def request( - self, method: str, url: str, **kwargs: Any + async def __request( + self, method: Callable, url, **kwargs: Any ) -> ClientResponse: """Make a request and retry if necessary. @@ -90,7 +90,7 @@ class and retry number. For no-retry errors, such as 400 Bad Request it just ret for i in range(self.__retry_number + 1): # add one for the first attempt, which is not a retry retry_factor = None try: - result = await self._client.request(method, url, **kwargs) + result = await method(url, **kwargs) except Exception as exc: exceptions.append(exc) if isinstance(exc, ServerConnectionError) or isinstance(exc, ClientResponseError): @@ -129,15 +129,15 @@ class and retry number. For no-retry errors, such as 400 Bad Request it just ret def get(self, url: str, *, allow_redirects: bool = True, **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]: """Perform HTTP GET request.""" - return self.request('GET', url, allow_redirects=allow_redirects, **kwargs) + return self.__request(self._client.get, url, allow_redirects=allow_redirects, **kwargs) def post(self, url: str, *, data: Any = None, **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]: """Perform HTTP POST request.""" - return self.request('POST', url, data=data, **kwargs) + return self.__request(self._client.post, url, data=data, **kwargs) def put(self, url: str, *, data: Any = None, **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]: """Perform HTTP PUT request.""" - return self.request('PUT', url, data=data, **kwargs) + return self.__request(self._client.put, url, data=data, **kwargs) def close(self) -> Coroutine: """Gracefully close internal aiohttp.ClientSession class instance.""" diff --git a/tests/aio/test_http.py b/tests/aio/test_http.py index 62785eea..a2048a8a 100644 --- a/tests/aio/test_http.py +++ b/tests/aio/test_http.py @@ -85,13 +85,13 @@ async def execute_http_request(port, retry_number, server_class, timeout_seconds session = RetryingClientSession(f'{protocol}://localhost:{port}', timeout=timeout, max_retry_number=retry_number, base_retry_delay=0.01, connector=connector) # noinspection PyProtectedMember - parent_request = session._client.request + parent_request = session._client.get async_mock = mock.Mock() async_mock.side_effect = parent_request exception = None result = None with get_http_server(server_handler=server_class, server_address=('', port)): - with mock.patch('reportportal_client.aio.http.ClientSession.request', async_mock): + with mock.patch('reportportal_client.aio.http.ClientSession.get', async_mock): async with session: start_time = time.time() try: From d996fe095e20b6b2a87653b729c7d1826429768d Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 5 Oct 2023 12:02:01 +0300 Subject: [PATCH 212/268] Update http module --- reportportal_client/aio/http.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/reportportal_client/aio/http.py b/reportportal_client/aio/http.py index a3e81ca4..8acaa5bf 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/aio/http.py @@ -80,10 +80,9 @@ async def __request( ) -> ClientResponse: """Make a request and retry if necessary. - The method uses aiohttp.ClientSession.request() method and bypass all arguments to it, so - please refer it for detailed argument description. The method retries requests depending on error - class and retry number. For no-retry errors, such as 400 Bad Request it just returns result, for cases - where it's reasonable to retry it does it in exponential manner. + The method retries requests depending on error class and retry number. For no-retry errors, such as + 400 Bad Request it just returns result, for cases where it's reasonable to retry it does it in + exponential manner. """ result = None exceptions = [] From f4bdd095687a00f43189abd1ccd62582b0f3a4a0 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 5 Oct 2023 12:30:50 +0300 Subject: [PATCH 213/268] Refactor package parameters get --- reportportal_client/helpers.py | 46 +++++++++++++++++++++- reportportal_client/services/statistics.py | 12 +----- tests/conftest.py | 2 +- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index 5284f7c8..b3a38ed5 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -17,12 +17,15 @@ import inspect import json import logging +import sys import threading import time import uuid from platform import machine, processor, system from typing import Optional, Any, List, Dict, Callable, Tuple, Union, TypeVar, Generic +from pkg_resources import DistributionNotFound + from reportportal_client.core.rp_file import RPFile from reportportal_client.static.defines import ATTRIBUTE_LENGTH_LIMIT @@ -158,7 +161,48 @@ def get_launch_sys_attrs() -> Dict[str, str]: } -def verify_value_length(attributes: List[dict]) -> List[dict]: +def get_package_parameters(package_name: str, parameters: List[str] = None) -> List[Optional[str]]: + """Get parameters of the given package. + + :param package_name: Name of the package. + :param parameters: Wanted parameters. + :return: Parameter List. + """ + result = [] + if not parameters: + return result + + if sys.version_info < (3, 8): + from pkg_resources import get_distribution + try: + package_info = get_distribution(package_name) + except DistributionNotFound: + return [None] * len(parameters) + for param in parameters: + if param.lower() == 'name': + result.append(getattr(package_info, 'project_name', None)) + else: + # noinspection PyCompatibility + from importlib.metadata import distribution, PackageNotFoundError + try: + package_info = distribution(package_name) + except PackageNotFoundError: + return [None] * len(parameters) + for param in parameters: + result.append(package_info.metadata[param.lower()[:1].upper() + param.lower()[1:]]) + return result + + +def get_package_version(package_name: str) -> Optional[str]: + """Get version of the given package. + + :param package_name: Name of the package + :return: Version of the package + """ + return get_package_parameters(package_name, ['version'])[0] + + +def verify_value_length(attributes): """Verify length of the attribute value. The length of the attribute value should have size from '1' to '128'. diff --git a/reportportal_client/services/statistics.py b/reportportal_client/services/statistics.py index 36130ae2..0a3b0d9f 100644 --- a/reportportal_client/services/statistics.py +++ b/reportportal_client/services/statistics.py @@ -13,7 +13,6 @@ """This module sends statistics events to a statistics service.""" -import sys import logging import ssl from platform import python_version @@ -23,6 +22,7 @@ import certifi import requests +from reportportal_client.helpers import get_package_parameters from reportportal_client.services.client_id import get_client_id from reportportal_client.services.constants import CLIENT_INFO, ENDPOINT @@ -36,15 +36,7 @@ def _get_client_info() -> Tuple[str, str]: :return: ('reportportal-client', '5.0.4') """ - if sys.version_info < (3, 8): - from pkg_resources import get_distribution - client = get_distribution('reportportal-client') - name, version = client.project_name, client.version - else: - # noinspection PyCompatibility - from importlib.metadata import distribution - client = distribution('reportportal-client') - name, version = client.metadata['Name'], client.metadata['Version'] + name, version = get_package_parameters('reportportal-client', ['name', 'version']) return name, version diff --git a/tests/conftest.py b/tests/conftest.py index 0d0cc2b3..9430de11 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,7 @@ def inner(ret_code, ret_value): :param ret_code: Return code for the response :param ret_value: Return value for the response - :return: Mocked Responce object with the given parameters + :return: Mocked Response object with the given parameters """ with mock.patch('requests.Response') as resp: resp.status_code = ret_code From 65a139db0470c856b067c404228a687cd210b3bc Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 5 Oct 2023 12:39:10 +0300 Subject: [PATCH 214/268] Fix for Python 3.7 --- reportportal_client/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index b3a38ed5..0e6e4048 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -180,7 +180,8 @@ def get_package_parameters(package_name: str, parameters: List[str] = None) -> L return [None] * len(parameters) for param in parameters: if param.lower() == 'name': - result.append(getattr(package_info, 'project_name', None)) + param = 'project_name' + result.append(getattr(package_info, param, None)) else: # noinspection PyCompatibility from importlib.metadata import distribution, PackageNotFoundError From efefdde980d7925dfc58e53ef9df508ff3177ba8 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 5 Oct 2023 12:46:08 +0300 Subject: [PATCH 215/268] Fix warnings for Python versions > 3.7 --- reportportal_client/helpers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index 0e6e4048..b319b1ae 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -24,8 +24,6 @@ from platform import machine, processor, system from typing import Optional, Any, List, Dict, Callable, Tuple, Union, TypeVar, Generic -from pkg_resources import DistributionNotFound - from reportportal_client.core.rp_file import RPFile from reportportal_client.static.defines import ATTRIBUTE_LENGTH_LIMIT @@ -173,7 +171,7 @@ def get_package_parameters(package_name: str, parameters: List[str] = None) -> L return result if sys.version_info < (3, 8): - from pkg_resources import get_distribution + from pkg_resources import get_distribution, DistributionNotFound try: package_info = get_distribution(package_name) except DistributionNotFound: From 5909f21f1eee09517d9973e9fc3659fa8c103180 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 5 Oct 2023 12:51:01 +0300 Subject: [PATCH 216/268] Fix coverage --- tests/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..5ff17af2 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""Package which contains all ReportPortal client tests.""" From 3a97f2792d247254f7665ee92e5e332c80c86ecc Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 5 Oct 2023 14:18:38 +0300 Subject: [PATCH 217/268] Add a comment --- tests/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/__init__.py b/tests/__init__.py index 5ff17af2..758c5345 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -12,3 +12,4 @@ # limitations under the License """Package which contains all ReportPortal client tests.""" +# This file is here because otherwise code coverage is not working. But pytest does not require package files. From 2cbbf73e38a8c145fca4ac7a1da48b6988d51648 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 5 Oct 2023 15:40:24 +0300 Subject: [PATCH 218/268] Update clone test --- tests/aio/test_threaded_client.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/aio/test_threaded_client.py b/tests/aio/test_threaded_client.py index c8d64300..11088c89 100644 --- a/tests/aio/test_threaded_client.py +++ b/tests/aio/test_threaded_client.py @@ -31,10 +31,12 @@ def test_clone(): kwargs = {'api_key': 'api_key1', 'launch_uuid': 'launch_uuid', 'log_batch_size': 30, 'log_batch_payload_limit': 30 * 1024 * 1024, 'task_timeout': 63, 'shutdown_timeout': 123} async_client = ThreadedRPClient(*args, **kwargs) - async_client._add_current_item(ThreadedTask(__empty_string(), 64, loop=async_client._loop, - name='test-321')) - async_client._add_current_item(ThreadedTask(__empty_string(), 64, loop=async_client._loop, - name='test-322')) + task1 = async_client.create_task(__empty_string()) + task2 = async_client.create_task(__empty_string()) + task1.blocking_result() + task2.blocking_result() + async_client._add_current_item(task1) + async_client._add_current_item(task2) client = async_client.client step_reporter = async_client.step_reporter cloned = async_client.clone() From 686141964b357c177d45ff041899fefddad9f7f1 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 5 Oct 2023 16:58:18 +0300 Subject: [PATCH 219/268] Refactoring, hide some classes from outsiders --- reportportal_client/__init__.py | 2 +- reportportal_client/_internal/__init__.py | 14 ++ reportportal_client/_internal/aio/__init__.py | 12 + .../{ => _internal}/aio/http.py | 8 + reportportal_client/_internal/aio/tasks.py | 233 ++++++++++++++++++ .../{_local => _internal/local}/__init__.py | 0 .../{_local => _internal/local}/__init__.pyi | 0 .../_internal/logs/__init__.py | 12 + .../{ => _internal}/logs/batcher.py | 0 .../{ => _internal}/services/__init__.py | 0 .../{ => _internal}/services/client_id.py | 0 .../{ => _internal}/services/client_id.pyi | 0 .../{ => _internal}/services/constants.py | 0 .../{ => _internal}/services/constants.pyi | 0 .../{ => _internal}/services/statistics.py | 4 +- .../{ => _internal}/static/__init__.py | 0 .../{ => _internal}/static/abstract.py | 0 .../{ => _internal}/static/defines.py | 2 +- reportportal_client/aio/__init__.py | 14 +- reportportal_client/aio/client.py | 32 ++- reportportal_client/aio/tasks.py | 216 +--------------- reportportal_client/client.py | 14 +- reportportal_client/core/rp_file.pyi | 3 + reportportal_client/core/rp_issues.pyi | 7 + reportportal_client/core/rp_requests.py | 14 +- reportportal_client/core/rp_responses.py | 3 +- reportportal_client/core/worker.py | 10 +- reportportal_client/core/worker.pyi | 3 +- reportportal_client/helpers.py | 3 +- reportportal_client/logs/__init__.py | 2 +- reportportal_client/logs/log_manager.py | 12 +- reportportal_client/logs/log_manager.pyi | 3 - reportportal_client/steps/__init__.py | 3 +- tests/{ => _internal}/aio/test_http.py | 3 +- .../{ => _internal}/logs/test_log_manager.py | 6 +- .../services}/test_client_id.py | 14 +- .../services/test_statistics.py | 7 +- tests/aio/test_aio_client.py | 9 +- tests/aio/test_threaded_client.py | 2 +- tests/conftest.py | 15 +- tests/logs/test_rp_file.py | 1 + tests/logs/test_rp_log_handler.py | 2 +- tests/logs/test_rp_logger.py | 5 +- tests/steps/conftest.py | 1 + tests/steps/test_steps.py | 3 +- tests/test_client.py | 2 + tests/test_helpers.py | 4 +- 47 files changed, 419 insertions(+), 281 deletions(-) create mode 100644 reportportal_client/_internal/__init__.py create mode 100644 reportportal_client/_internal/aio/__init__.py rename reportportal_client/{ => _internal}/aio/http.py (94%) create mode 100644 reportportal_client/_internal/aio/tasks.py rename reportportal_client/{_local => _internal/local}/__init__.py (100%) rename reportportal_client/{_local => _internal/local}/__init__.pyi (100%) create mode 100644 reportportal_client/_internal/logs/__init__.py rename reportportal_client/{ => _internal}/logs/batcher.py (100%) rename reportportal_client/{ => _internal}/services/__init__.py (100%) rename reportportal_client/{ => _internal}/services/client_id.py (100%) rename reportportal_client/{ => _internal}/services/client_id.pyi (100%) rename reportportal_client/{ => _internal}/services/constants.py (100%) rename reportportal_client/{ => _internal}/services/constants.pyi (100%) rename reportportal_client/{ => _internal}/services/statistics.py (96%) rename reportportal_client/{ => _internal}/static/__init__.py (100%) rename reportportal_client/{ => _internal}/static/abstract.py (100%) rename reportportal_client/{ => _internal}/static/defines.py (97%) rename tests/{ => _internal}/aio/test_http.py (98%) rename tests/{ => _internal}/logs/test_log_manager.py (98%) rename tests/{ => _internal/services}/test_client_id.py (83%) rename tests/{ => _internal}/services/test_statistics.py (94%) diff --git a/reportportal_client/__init__.py b/reportportal_client/__init__.py index e8e95dc8..a3933a67 100644 --- a/reportportal_client/__init__.py +++ b/reportportal_client/__init__.py @@ -14,7 +14,7 @@ """This package is the base package for ReportPortal client.""" # noinspection PyProtectedMember -from reportportal_client._local import current +from reportportal_client._internal.local import current from reportportal_client.logs import RPLogger, RPLogHandler from reportportal_client.client import RP, RPClient, OutputType from reportportal_client.steps import step diff --git a/reportportal_client/_internal/__init__.py b/reportportal_client/_internal/__init__.py new file mode 100644 index 00000000..c04681ca --- /dev/null +++ b/reportportal_client/_internal/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""ReportPortal client internal API. No warnings before breaking changes. No backward compatibility.""" diff --git a/reportportal_client/_internal/aio/__init__.py b/reportportal_client/_internal/aio/__init__.py new file mode 100644 index 00000000..1c768643 --- /dev/null +++ b/reportportal_client/_internal/aio/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License diff --git a/reportportal_client/aio/http.py b/reportportal_client/_internal/aio/http.py similarity index 94% rename from reportportal_client/aio/http.py rename to reportportal_client/_internal/aio/http.py index 8acaa5bf..c9c6d344 100644 --- a/reportportal_client/aio/http.py +++ b/reportportal_client/_internal/aio/http.py @@ -10,6 +10,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License """This module designed to help with asynchronous HTTP request/response handling.""" diff --git a/reportportal_client/_internal/aio/tasks.py b/reportportal_client/_internal/aio/tasks.py new file mode 100644 index 00000000..03194932 --- /dev/null +++ b/reportportal_client/_internal/aio/tasks.py @@ -0,0 +1,233 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module contains customized asynchronous Tasks and Task Factories for the ReportPortal client.""" + +import asyncio +import sys +import time +from asyncio import Future +from typing import Optional, List, TypeVar, Generic, Union, Generator, Awaitable, Coroutine, Any + +from reportportal_client.aio.client import DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL +from reportportal_client.aio.tasks import Task, BlockingOperationError + +_T = TypeVar('_T') + + +class BatchedTask(Generic[_T], Task[_T]): + """Represents a Task which uses the current Thread to execute itself.""" + + __loop: asyncio.AbstractEventLoop + + def __init__( + self, + coro: Union[Generator[Future, None, _T], Awaitable[_T]], + *, + loop: asyncio.AbstractEventLoop, + name: Optional[str] = None + ) -> None: + """Initialize an instance of the Task. + + :param coro: Future, Coroutine or a Generator of these objects, which will be executed + :param loop: Event Loop which will be used to execute the Task + :param name: the name of the task + """ + super().__init__(coro, loop=loop, name=name) + self.__loop = loop + + def blocking_result(self) -> _T: + """Use current Thread to execute the Task and return the result if not yet executed. + + :return: execution result or raise an error, or return immediately if already executed + """ + if self.done(): + return self.result() + return self.__loop.run_until_complete(self) + + +class ThreadedTask(Generic[_T], Task[_T]): + """Represents a Task which runs is a separate Thread.""" + + __loop: asyncio.AbstractEventLoop + __wait_timeout: float + + def __init__( + self, + coro: Union[Generator[Future, None, _T], Awaitable[_T]], + wait_timeout: float, + *, + loop: asyncio.AbstractEventLoop, + name: Optional[str] = None + ) -> None: + """Initialize an instance of the Task. + + :param coro: Future, Coroutine or a Generator of these objects, which will be executed + :param loop: Event Loop which will be used to execute the Task + :param name: the name of the task + """ + super().__init__(coro, loop=loop, name=name) + self.__loop = loop + self.__wait_timeout = wait_timeout + + def blocking_result(self) -> _T: + """Pause current Thread until the Task completion and return the result if not yet executed. + + :return: execution result or raise an error, or return immediately if already executed + """ + if self.done(): + return self.result() + if not self.__loop.is_running() or self.__loop.is_closed(): + raise BlockingOperationError('Running loop is not alive') + start_time = time.time() + sleep_time = sys.getswitchinterval() + while not self.done() and time.time() - start_time < self.__wait_timeout: + time.sleep(sleep_time) + if not self.done(): + raise BlockingOperationError('Timed out waiting for the task execution') + return self.result() + + +class BatchedTaskFactory: + """Factory protocol which creates Batched Tasks.""" + + def __call__( + self, + loop: asyncio.AbstractEventLoop, + factory: Union[Coroutine[Any, Any, _T], Generator[Any, None, _T]], + **_ + ) -> Task[_T]: + """Create Batched Task in appropriate Event Loop. + + :param loop: Event Loop which will be used to execute the Task + :param factory: Future, Coroutine or a Generator of these objects, which will be executed + """ + return BatchedTask(factory, loop=loop) + + +class ThreadedTaskFactory: + """Factory protocol which creates Threaded Tasks.""" + + __wait_timeout: float + + def __init__(self, wait_timeout: float): + """Initialize an instance of the Factory. + + :param wait_timeout: Task wait timeout in case of blocking result get + """ + self.__wait_timeout = wait_timeout + + def __call__( + self, + loop: asyncio.AbstractEventLoop, + factory: Union[Coroutine[Any, Any, _T], Generator[Any, None, _T]], + **_ + ) -> Task[_T]: + """Create Threaded Task in appropriate Event Loop. + + :param loop: Event Loop which will be used to execute the Task + :param factory: Future, Coroutine or a Generator of these objects, which will be executed + """ + return ThreadedTask(factory, self.__wait_timeout, loop=loop) + + +class TriggerTaskBatcher(Generic[_T]): + """Batching class which compile its batches by object number or by passed time.""" + + __task_list: List[_T] + __last_run_time: float + __trigger_num: int + __trigger_interval: float + + def __init__(self, + trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, + trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL) -> None: + """Initialize an instance of the Batcher. + + :param trigger_num: object number threshold which triggers batch return and reset + :param trigger_interval: amount of time after which return and reset batch + """ + self.__task_list = [] + self.__last_run_time = time.time() + self.__trigger_num = trigger_num + self.__trigger_interval = trigger_interval + + def __ready_to_run(self) -> bool: + current_time = time.time() + last_time = self.__last_run_time + if len(self.__task_list) <= 0: + return False + if (len(self.__task_list) >= self.__trigger_num + or current_time - last_time >= self.__trigger_interval): + self.__last_run_time = current_time + return True + return False + + def append(self, value: _T) -> Optional[List[_T]]: + """Add an object to internal batch and return the batch if it's triggered. + + :param value: an object to add to the batch + :return: a batch or None + """ + self.__task_list.append(value) + if self.__ready_to_run(): + tasks = self.__task_list + self.__task_list = [] + return tasks + + def flush(self) -> Optional[List[_T]]: + """Immediately return everything what's left in the internal batch. + + :return: a batch or None + """ + if len(self.__task_list) > 0: + tasks = self.__task_list + self.__task_list = [] + return tasks + + +class BackgroundTaskList(Generic[_T]): + """Task list class which collects Tasks into internal batch and removes when they complete.""" + + __task_list: List[_T] + + def __init__(self): + """Initialize an instance of the Batcher.""" + self.__task_list = [] + + def __remove_finished(self): + i = -1 + for task in self.__task_list: + if not task.done(): + break + i += 1 + self.__task_list = self.__task_list[i + 1:] + + def append(self, value: _T) -> None: + """Add an object to internal batch. + + :param value: an object to add to the batch + """ + self.__remove_finished() + self.__task_list.append(value) + + def flush(self) -> Optional[List[_T]]: + """Immediately return everything what's left unfinished in the internal batch. + + :return: a batch or None + """ + self.__remove_finished() + if len(self.__task_list) > 0: + tasks = self.__task_list + self.__task_list = [] + return tasks diff --git a/reportportal_client/_local/__init__.py b/reportportal_client/_internal/local/__init__.py similarity index 100% rename from reportportal_client/_local/__init__.py rename to reportportal_client/_internal/local/__init__.py diff --git a/reportportal_client/_local/__init__.pyi b/reportportal_client/_internal/local/__init__.pyi similarity index 100% rename from reportportal_client/_local/__init__.pyi rename to reportportal_client/_internal/local/__init__.pyi diff --git a/reportportal_client/_internal/logs/__init__.py b/reportportal_client/_internal/logs/__init__.py new file mode 100644 index 00000000..1c768643 --- /dev/null +++ b/reportportal_client/_internal/logs/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License diff --git a/reportportal_client/logs/batcher.py b/reportportal_client/_internal/logs/batcher.py similarity index 100% rename from reportportal_client/logs/batcher.py rename to reportportal_client/_internal/logs/batcher.py diff --git a/reportportal_client/services/__init__.py b/reportportal_client/_internal/services/__init__.py similarity index 100% rename from reportportal_client/services/__init__.py rename to reportportal_client/_internal/services/__init__.py diff --git a/reportportal_client/services/client_id.py b/reportportal_client/_internal/services/client_id.py similarity index 100% rename from reportportal_client/services/client_id.py rename to reportportal_client/_internal/services/client_id.py diff --git a/reportportal_client/services/client_id.pyi b/reportportal_client/_internal/services/client_id.pyi similarity index 100% rename from reportportal_client/services/client_id.pyi rename to reportportal_client/_internal/services/client_id.pyi diff --git a/reportportal_client/services/constants.py b/reportportal_client/_internal/services/constants.py similarity index 100% rename from reportportal_client/services/constants.py rename to reportportal_client/_internal/services/constants.py diff --git a/reportportal_client/services/constants.pyi b/reportportal_client/_internal/services/constants.pyi similarity index 100% rename from reportportal_client/services/constants.pyi rename to reportportal_client/_internal/services/constants.pyi diff --git a/reportportal_client/services/statistics.py b/reportportal_client/_internal/services/statistics.py similarity index 96% rename from reportportal_client/services/statistics.py rename to reportportal_client/_internal/services/statistics.py index 0a3b0d9f..46fcf28a 100644 --- a/reportportal_client/services/statistics.py +++ b/reportportal_client/_internal/services/statistics.py @@ -23,8 +23,8 @@ import requests from reportportal_client.helpers import get_package_parameters -from reportportal_client.services.client_id import get_client_id -from reportportal_client.services.constants import CLIENT_INFO, ENDPOINT +from reportportal_client._internal.services.client_id import get_client_id +from reportportal_client._internal.services.constants import CLIENT_INFO, ENDPOINT logger = logging.getLogger(__name__) diff --git a/reportportal_client/static/__init__.py b/reportportal_client/_internal/static/__init__.py similarity index 100% rename from reportportal_client/static/__init__.py rename to reportportal_client/_internal/static/__init__.py diff --git a/reportportal_client/static/abstract.py b/reportportal_client/_internal/static/abstract.py similarity index 100% rename from reportportal_client/static/abstract.py rename to reportportal_client/_internal/static/abstract.py diff --git a/reportportal_client/static/defines.py b/reportportal_client/_internal/static/defines.py similarity index 97% rename from reportportal_client/static/defines.py rename to reportportal_client/_internal/static/defines.py index cd3565c3..62385394 100644 --- a/reportportal_client/static/defines.py +++ b/reportportal_client/_internal/static/defines.py @@ -68,7 +68,7 @@ class ItemStartType(str, enum.Enum): class Priority(enum.IntEnum): - """Generic enum for various operations prioritization.""" + """Generic enum for various operations' prioritization.""" PRIORITY_IMMEDIATE = 0x0 PRIORITY_HIGH = 0x1 diff --git a/reportportal_client/aio/__init__.py b/reportportal_client/aio/__init__.py index df490be6..fd5f103b 100644 --- a/reportportal_client/aio/__init__.py +++ b/reportportal_client/aio/__init__.py @@ -14,20 +14,12 @@ """Common package for Asynchronous I/O clients and utilities.""" from reportportal_client.aio.client import (ThreadedRPClient, BatchedRPClient, AsyncRPClient, - DEFAULT_TASK_TIMEOUT, DEFAULT_SHUTDOWN_TIMEOUT) -from reportportal_client.aio.tasks import (Task, TriggerTaskBatcher, BatchedTask, BatchedTaskFactory, - ThreadedTask, ThreadedTaskFactory, BlockingOperationError, - BackgroundTaskList, DEFAULT_TASK_TRIGGER_NUM, - DEFAULT_TASK_TRIGGER_INTERVAL) + DEFAULT_TASK_TIMEOUT, DEFAULT_SHUTDOWN_TIMEOUT, + DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL) +from reportportal_client.aio.tasks import Task, BlockingOperationError __all__ = [ 'Task', - 'TriggerTaskBatcher', - 'BackgroundTaskList', - 'BatchedTask', - 'BatchedTaskFactory', - 'ThreadedTask', - 'ThreadedTaskFactory', 'BlockingOperationError', 'DEFAULT_TASK_TIMEOUT', 'DEFAULT_SHUTDOWN_TIMEOUT', diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 9ebb235c..9eec682f 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -27,11 +27,24 @@ from reportportal_client import RP, OutputType # noinspection PyProtectedMember -from reportportal_client._local import set_current -from reportportal_client.aio.http import RetryingClientSession -from reportportal_client.aio.tasks import (Task, BatchedTaskFactory, ThreadedTaskFactory, TriggerTaskBatcher, - BackgroundTaskList, DEFAULT_TASK_TRIGGER_NUM, - DEFAULT_TASK_TRIGGER_INTERVAL) +from reportportal_client._internal.aio.http import RetryingClientSession +# noinspection PyProtectedMember +from reportportal_client._internal.aio.tasks import (BatchedTaskFactory, ThreadedTaskFactory, + TriggerTaskBatcher, BackgroundTaskList) +# noinspection PyProtectedMember +from reportportal_client._internal.local import set_current +# noinspection PyProtectedMember +from reportportal_client._internal.logs.batcher import LogBatcher +# noinspection PyProtectedMember +from reportportal_client._internal.services.statistics import async_send_event +# noinspection PyProtectedMember +from reportportal_client._internal.static.abstract import ( + AbstractBaseClass, + abstractmethod +) +# noinspection PyProtectedMember +from reportportal_client._internal.static.defines import NOT_FOUND, NOT_SET +from reportportal_client.aio.tasks import Task from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, AsyncItemFinishRequest, LaunchFinishRequest, RPFile, @@ -39,13 +52,6 @@ from reportportal_client.helpers import (root_uri_join, verify_value_length, await_if_necessary, agent_name_version, LifoQueue, uri_join) from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE -from reportportal_client.logs.batcher import LogBatcher -from reportportal_client.services.statistics import async_send_event -from reportportal_client.static.abstract import ( - AbstractBaseClass, - abstractmethod -) -from reportportal_client.static.defines import NOT_FOUND, NOT_SET from reportportal_client.steps import StepReporter logger = logging.getLogger(__name__) @@ -55,6 +61,8 @@ DEFAULT_TASK_TIMEOUT: float = 60.0 DEFAULT_SHUTDOWN_TIMEOUT: float = 120.0 +DEFAULT_TASK_TRIGGER_NUM: int = 10 +DEFAULT_TASK_TRIGGER_INTERVAL: float = 1.0 class Client: diff --git a/reportportal_client/aio/tasks.py b/reportportal_client/aio/tasks.py index 28f772fc..0afc77ac 100644 --- a/reportportal_client/aio/tasks.py +++ b/reportportal_client/aio/tasks.py @@ -15,16 +15,14 @@ import asyncio import sys -import time from abc import abstractmethod from asyncio import Future -from typing import TypeVar, Generic, Union, Generator, Awaitable, Optional, Coroutine, Any, List +from typing import TypeVar, Generic, Union, Generator, Awaitable, Optional -from reportportal_client.static.abstract import AbstractBaseClass +# noinspection PyProtectedMember +from reportportal_client._internal.static.abstract import AbstractBaseClass _T = TypeVar('_T') -DEFAULT_TASK_TRIGGER_NUM: int = 10 -DEFAULT_TASK_TRIGGER_INTERVAL: float = 1.0 class BlockingOperationError(RuntimeError): @@ -86,211 +84,3 @@ def __str__(self): if self.done(): return str(self.result()) return super().__str__() - - -class BatchedTask(Generic[_T], Task[_T]): - """Represents a Task which uses the current Thread to execute itself.""" - - __loop: asyncio.AbstractEventLoop - - def __init__( - self, - coro: Union[Generator[Future, None, _T], Awaitable[_T]], - *, - loop: asyncio.AbstractEventLoop, - name: Optional[str] = None - ) -> None: - """Initialize an instance of the Task. - - :param coro: Future, Coroutine or a Generator of these objects, which will be executed - :param loop: Event Loop which will be used to execute the Task - :param name: the name of the task - """ - super().__init__(coro, loop=loop, name=name) - self.__loop = loop - - def blocking_result(self) -> _T: - """Use current Thread to execute the Task and return the result if not yet executed. - - :return: execution result or raise an error, or return immediately if already executed - """ - if self.done(): - return self.result() - return self.__loop.run_until_complete(self) - - -class ThreadedTask(Generic[_T], Task[_T]): - """Represents a Task which runs is a separate Thread.""" - - __loop: asyncio.AbstractEventLoop - __wait_timeout: float - - def __init__( - self, - coro: Union[Generator[Future, None, _T], Awaitable[_T]], - wait_timeout: float, - *, - loop: asyncio.AbstractEventLoop, - name: Optional[str] = None - ) -> None: - """Initialize an instance of the Task. - - :param coro: Future, Coroutine or a Generator of these objects, which will be executed - :param loop: Event Loop which will be used to execute the Task - :param name: the name of the task - """ - super().__init__(coro, loop=loop, name=name) - self.__loop = loop - self.__wait_timeout = wait_timeout - - def blocking_result(self) -> _T: - """Pause current Thread until the Task completion and return the result if not yet executed. - - :return: execution result or raise an error, or return immediately if already executed - """ - if self.done(): - return self.result() - if not self.__loop.is_running() or self.__loop.is_closed(): - raise BlockingOperationError('Running loop is not alive') - start_time = time.time() - sleep_time = sys.getswitchinterval() - while not self.done() and time.time() - start_time < self.__wait_timeout: - time.sleep(sleep_time) - if not self.done(): - raise BlockingOperationError('Timed out waiting for the task execution') - return self.result() - - -class BatchedTaskFactory: - """Factory protocol which creates Batched Tasks.""" - - def __call__( - self, - loop: asyncio.AbstractEventLoop, - factory: Union[Coroutine[Any, Any, _T], Generator[Any, None, _T]], - **_ - ) -> Task[_T]: - """Create Batched Task in appropriate Event Loop. - - :param loop: Event Loop which will be used to execute the Task - :param factory: Future, Coroutine or a Generator of these objects, which will be executed - """ - return BatchedTask(factory, loop=loop) - - -class ThreadedTaskFactory: - """Factory protocol which creates Threaded Tasks.""" - - __wait_timeout: float - - def __init__(self, wait_timeout: float): - """Initialize an instance of the Factory. - - :param wait_timeout: Task wait timeout in case of blocking result get - """ - self.__wait_timeout = wait_timeout - - def __call__( - self, - loop: asyncio.AbstractEventLoop, - factory: Union[Coroutine[Any, Any, _T], Generator[Any, None, _T]], - **_ - ) -> Task[_T]: - """Create Threaded Task in appropriate Event Loop. - - :param loop: Event Loop which will be used to execute the Task - :param factory: Future, Coroutine or a Generator of these objects, which will be executed - """ - return ThreadedTask(factory, self.__wait_timeout, loop=loop) - - -class TriggerTaskBatcher(Generic[_T]): - """Batching class which compile its batches by object number or by passed time.""" - - __task_list: List[_T] - __last_run_time: float - __trigger_num: int - __trigger_interval: float - - def __init__(self, - trigger_num: int = DEFAULT_TASK_TRIGGER_NUM, - trigger_interval: float = DEFAULT_TASK_TRIGGER_INTERVAL) -> None: - """Initialize an instance of the Batcher. - - :param trigger_num: object number threshold which triggers batch return and reset - :param trigger_interval: amount of time after which return and reset batch - """ - self.__task_list = [] - self.__last_run_time = time.time() - self.__trigger_num = trigger_num - self.__trigger_interval = trigger_interval - - def __ready_to_run(self) -> bool: - current_time = time.time() - last_time = self.__last_run_time - if len(self.__task_list) <= 0: - return False - if (len(self.__task_list) >= self.__trigger_num - or current_time - last_time >= self.__trigger_interval): - self.__last_run_time = current_time - return True - return False - - def append(self, value: _T) -> Optional[List[_T]]: - """Add an object to internal batch and return the batch if it's triggered. - - :param value: an object to add to the batch - :return: a batch or None - """ - self.__task_list.append(value) - if self.__ready_to_run(): - tasks = self.__task_list - self.__task_list = [] - return tasks - - def flush(self) -> Optional[List[_T]]: - """Immediately return everything what's left in the internal batch. - - :return: a batch or None - """ - if len(self.__task_list) > 0: - tasks = self.__task_list - self.__task_list = [] - return tasks - - -class BackgroundTaskList(Generic[_T]): - """Task list class which collects Tasks into internal batch and removes when they complete.""" - - __task_list: List[_T] - - def __init__(self): - """Initialize an instance of the Batcher.""" - self.__task_list = [] - - def __remove_finished(self): - i = -1 - for task in self.__task_list: - if not task.done(): - break - i += 1 - self.__task_list = self.__task_list[i + 1:] - - def append(self, value: _T) -> None: - """Add an object to internal batch. - - :param value: an object to add to the batch - """ - self.__remove_finished() - self.__task_list.append(value) - - def flush(self) -> Optional[List[_T]]: - """Immediately return everything what's left unfinished in the internal batch. - - :return: a batch or None - """ - self.__remove_finished() - if len(self.__task_list) > 0: - tasks = self.__task_list - self.__task_list = [] - return tasks diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 1057c076..e6002d49 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -26,17 +26,21 @@ from requests.adapters import HTTPAdapter, Retry, DEFAULT_RETRIES # noinspection PyProtectedMember -from reportportal_client._local import set_current +from reportportal_client._internal.local import set_current +# noinspection PyProtectedMember +from reportportal_client._internal.logs.batcher import LogBatcher +# noinspection PyProtectedMember +from reportportal_client._internal.services.statistics import send_event +# noinspection PyProtectedMember +from reportportal_client._internal.static.abstract import AbstractBaseClass +# noinspection PyProtectedMember +from reportportal_client._internal.static.defines import NOT_FOUND from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (HttpRequest, ItemStartRequest, ItemFinishRequest, RPFile, LaunchStartRequest, LaunchFinishRequest, RPRequestLog, RPLogBatch) from reportportal_client.helpers import uri_join, verify_value_length, agent_name_version, LifoQueue from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE -from reportportal_client.logs.batcher import LogBatcher -from reportportal_client.services.statistics import send_event -from reportportal_client.static.abstract import AbstractBaseClass -from reportportal_client.static.defines import NOT_FOUND from reportportal_client.steps import StepReporter logger = logging.getLogger(__name__) diff --git a/reportportal_client/core/rp_file.pyi b/reportportal_client/core/rp_file.pyi index 1d08b606..b16db04d 100644 --- a/reportportal_client/core/rp_file.pyi +++ b/reportportal_client/core/rp_file.pyi @@ -13,13 +13,16 @@ from typing import Any, Dict, Optional, Text + class RPFile: content: Any = ... content_type: Text = ... name: Text = ... + def __init__(self, name: Optional[Text], content: Any, content_type: Optional[Text]) -> None: ... + @property def payload(self) -> Dict: ... diff --git a/reportportal_client/core/rp_issues.pyi b/reportportal_client/core/rp_issues.pyi index de58bae9..a7257ad9 100644 --- a/reportportal_client/core/rp_issues.pyi +++ b/reportportal_client/core/rp_issues.pyi @@ -13,32 +13,39 @@ from typing import Dict, List, Optional, Text + class Issue: _external_issues: List = ... auto_analyzed: bool = ... comment: Text = ... ignore_analyzer: bool = ... issue_type: Text = ... + def __init__(self, issue_type: Text, comment: Optional[Text] = ..., auto_analyzed: Optional[bool] = ..., ignore_analyzer: Optional[bool] = ...) -> None: ... + def external_issue_add(self, issue: ExternalIssue) -> None: ... + @property def payload(self) -> Dict: ... + class ExternalIssue: bts_url: Text = ... bts_project: Text = ... submit_date: Text = ... ticket_id: Text = ... url: Text = ... + def __init__(self, bts_url: Optional[Text] = ..., bts_project: Optional[Text] = ..., submit_date: Optional[Text] = ..., ticket_id: Optional[Text] = ..., url: Optional[Text] = ...) -> None: ... + @property def payload(self) -> Dict: ... diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 2020916b..6bd66912 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -27,19 +27,21 @@ import aiohttp from reportportal_client import helpers -from reportportal_client.core.rp_file import RPFile -from reportportal_client.core.rp_issues import Issue -from reportportal_client.core.rp_responses import RPResponse, AsyncRPResponse -from reportportal_client.helpers import dict_to_payload, await_if_necessary -from reportportal_client.static.abstract import ( +# noinspection PyProtectedMember +from reportportal_client._internal.static.abstract import ( AbstractBaseClass, abstractmethod ) -from reportportal_client.static.defines import ( +# noinspection PyProtectedMember +from reportportal_client._internal.static.defines import ( DEFAULT_PRIORITY, LOW_PRIORITY, RP_LOG_LEVELS, Priority ) +from reportportal_client.core.rp_file import RPFile +from reportportal_client.core.rp_issues import Issue +from reportportal_client.core.rp_responses import RPResponse, AsyncRPResponse +from reportportal_client.helpers import dict_to_payload, await_if_necessary logger = logging.getLogger(__name__) T = TypeVar("T") diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index f399f416..891336b3 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -25,7 +25,8 @@ from aiohttp import ClientResponse from requests import Response -from reportportal_client.static.defines import NOT_FOUND, NOT_SET +# noinspection PyProtectedMember +from reportportal_client._internal.static.defines import NOT_FOUND, NOT_SET logger = logging.getLogger(__name__) diff --git a/reportportal_client/core/worker.py b/reportportal_client/core/worker.py index f7b62ca1..c9af496f 100644 --- a/reportportal_client/core/worker.py +++ b/reportportal_client/core/worker.py @@ -16,11 +16,13 @@ import logging import queue import threading +import warnings from threading import current_thread, Thread from aenum import auto, Enum, unique -from reportportal_client.static.defines import Priority +# noinspection PyProtectedMember +from reportportal_client._internal.static.defines import Priority logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -59,6 +61,12 @@ class APIWorker(object): def __init__(self, task_queue): """Initialize instance attributes.""" + warnings.warn( + message='`APIWorker` class is deprecated since 5.5.0 and will be subject for removing in the' + ' next major version.', + category=DeprecationWarning, + stacklevel=2 + ) self._queue = task_queue self._thread = None self._stop_lock = threading.Condition() diff --git a/reportportal_client/core/worker.pyi b/reportportal_client/core/worker.pyi index 036c59bd..8bf5d897 100644 --- a/reportportal_client/core/worker.pyi +++ b/reportportal_client/core/worker.pyi @@ -19,8 +19,9 @@ from typing import Any, Optional, Text, Union from aenum import Enum +# noinspection PyProtectedMember +from reportportal_client._internal.static.defines import Priority from reportportal_client.core.rp_requests import RPRequestBase as RPRequest, HttpRequest -from static.defines import Priority logger: Logger THREAD_TIMEOUT: int diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index b319b1ae..b6b0a39c 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -25,7 +25,8 @@ from typing import Optional, Any, List, Dict, Callable, Tuple, Union, TypeVar, Generic from reportportal_client.core.rp_file import RPFile -from reportportal_client.static.defines import ATTRIBUTE_LENGTH_LIMIT +# noinspection PyProtectedMember +from reportportal_client._internal.static.defines import ATTRIBUTE_LENGTH_LIMIT logger: logging.Logger = logging.getLogger(__name__) _T = TypeVar('_T') diff --git a/reportportal_client/logs/__init__.py b/reportportal_client/logs/__init__.py index aed7e797..c5c7a700 100644 --- a/reportportal_client/logs/__init__.py +++ b/reportportal_client/logs/__init__.py @@ -19,7 +19,7 @@ from urllib.parse import urlparse # noinspection PyProtectedMember -from reportportal_client._local import current, set_current +from reportportal_client._internal.local import current, set_current from reportportal_client.helpers import timestamp, TYPICAL_MULTIPART_FOOTER_LENGTH MAX_LOG_BATCH_SIZE: int = 20 diff --git a/reportportal_client/logs/log_manager.py b/reportportal_client/logs/log_manager.py index f6a2a226..0d12bb77 100644 --- a/reportportal_client/logs/log_manager.py +++ b/reportportal_client/logs/log_manager.py @@ -15,9 +15,12 @@ import logging import queue +import warnings from threading import Lock from reportportal_client import helpers +# noinspection PyProtectedMember +from reportportal_client._internal.static.defines import NOT_FOUND from reportportal_client.core.rp_requests import ( HttpRequest, RPFile, @@ -26,12 +29,11 @@ ) from reportportal_client.core.worker import APIWorker from reportportal_client.logs import MAX_LOG_BATCH_SIZE, MAX_LOG_BATCH_PAYLOAD_SIZE -from reportportal_client.static.defines import NOT_FOUND logger = logging.getLogger(__name__) -class LogManager(object): +class LogManager: """Manager of the log items.""" def __init__(self, rp_url, session, api_version, launch_id, project_name, @@ -51,6 +53,12 @@ def __init__(self, rp_url, session, api_version, launch_id, project_name, :param max_payload_size: maximum size in bytes of logs that can be processed in one batch """ + warnings.warn( + message='`LogManager` class is deprecated since 5.5.0 and will be subject for removing in the' + ' next major version.', + category=DeprecationWarning, + stacklevel=2 + ) self._lock = Lock() self._batch = [] self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH diff --git a/reportportal_client/logs/log_manager.pyi b/reportportal_client/logs/log_manager.pyi index 31af263a..5feb42ab 100644 --- a/reportportal_client/logs/log_manager.pyi +++ b/reportportal_client/logs/log_manager.pyi @@ -23,9 +23,6 @@ from reportportal_client.core.worker import APIWorker as APIWorker logger: Logger -MAX_LOG_BATCH_SIZE: int -MAX_LOG_BATCH_PAYLOAD_SIZE: int - class LogManager: _lock: Lock = ... diff --git a/reportportal_client/steps/__init__.py b/reportportal_client/steps/__init__.py index a99f47f2..b51be5d7 100644 --- a/reportportal_client/steps/__init__.py +++ b/reportportal_client/steps/__init__.py @@ -137,8 +137,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): """Exit the runtime context related to this object.""" - # Cannot call _local.current() early since it will be initialized - # before client put something in there + # Cannot call local.current() early since it will be initialized before client put something in there rp_client = self.client or current() if not rp_client: return diff --git a/tests/aio/test_http.py b/tests/_internal/aio/test_http.py similarity index 98% rename from tests/aio/test_http.py rename to tests/_internal/aio/test_http.py index a2048a8a..d279a8d3 100644 --- a/tests/aio/test_http.py +++ b/tests/_internal/aio/test_http.py @@ -20,7 +20,8 @@ # noinspection PyPackageRequirements import pytest -from reportportal_client.aio.http import RetryingClientSession +# noinspection PyProtectedMember +from reportportal_client._internal.aio.http import RetryingClientSession HTTP_TIMEOUT_TIME = 1.2 diff --git a/tests/logs/test_log_manager.py b/tests/_internal/logs/test_log_manager.py similarity index 98% rename from tests/logs/test_log_manager.py rename to tests/_internal/logs/test_log_manager.py index ce617af9..5267fb11 100644 --- a/tests/logs/test_log_manager.py +++ b/tests/_internal/logs/test_log_manager.py @@ -11,14 +11,14 @@ # See the License for the specific language governing permissions and # limitations under the License -import os import json +import os from unittest import mock from reportportal_client import helpers from reportportal_client.core.rp_requests import HttpRequest -from reportportal_client.logs.log_manager import LogManager, \ - MAX_LOG_BATCH_PAYLOAD_SIZE +from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE +from reportportal_client.logs.log_manager import LogManager RP_URL = 'http://docker.local:8080' API_VERSION = 'v2' diff --git a/tests/test_client_id.py b/tests/_internal/services/test_client_id.py similarity index 83% rename from tests/test_client_id.py rename to tests/_internal/services/test_client_id.py index 291437ed..915d381c 100644 --- a/tests/test_client_id.py +++ b/tests/_internal/services/test_client_id.py @@ -10,13 +10,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License import os import re from uuid import UUID, uuid4 -from reportportal_client.services.client_id import get_client_id -from reportportal_client.services.constants import RP_PROPERTIES_FILE_PATH +# noinspection PyProtectedMember +from reportportal_client._internal.services.client_id import get_client_id +# noinspection PyProtectedMember +from reportportal_client._internal.services.constants import RP_PROPERTIES_FILE_PATH def test_get_client_id_should_return_the_id_for_two_calls(): diff --git a/tests/services/test_statistics.py b/tests/_internal/services/test_statistics.py similarity index 94% rename from tests/services/test_statistics.py rename to tests/_internal/services/test_statistics.py index 5253378f..d27966d0 100644 --- a/tests/services/test_statistics.py +++ b/tests/_internal/services/test_statistics.py @@ -25,11 +25,14 @@ import sys from unittest import mock +# noinspection PyPackageRequirements import pytest from requests.exceptions import RequestException -from reportportal_client.services.constants import ENDPOINT, CLIENT_INFO -from reportportal_client.services.statistics import send_event, async_send_event +# noinspection PyProtectedMember +from reportportal_client._internal.services.constants import ENDPOINT, CLIENT_INFO +# noinspection PyProtectedMember +from reportportal_client._internal.services.statistics import send_event, async_send_event VERSION_VAR = '__version__' EVENT_NAME = 'start_launch' diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index cd440697..0f45ecd9 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -10,17 +10,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License -import sys import pickle +import sys from unittest import mock import aiohttp +# noinspection PyPackageRequirements import pytest from reportportal_client import OutputType +# noinspection PyProtectedMember +from reportportal_client._internal.aio.http import RetryingClientSession, DEFAULT_RETRY_NUMBER +# noinspection PyProtectedMember +from reportportal_client._internal.static.defines import NOT_SET from reportportal_client.aio.client import Client -from reportportal_client.aio.http import RetryingClientSession, DEFAULT_RETRY_NUMBER -from reportportal_client.static.defines import NOT_SET ENDPOINT = 'http://localhost:8080' PROJECT = 'default_personal' diff --git a/tests/aio/test_threaded_client.py b/tests/aio/test_threaded_client.py index 11088c89..196549fb 100644 --- a/tests/aio/test_threaded_client.py +++ b/tests/aio/test_threaded_client.py @@ -12,7 +12,7 @@ # limitations under the License import pickle -from reportportal_client.aio import ThreadedRPClient, ThreadedTask +from reportportal_client.aio import ThreadedRPClient def test_threaded_rp_client_pickling(): diff --git a/tests/conftest.py b/tests/conftest.py index 9430de11..f5c768b0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,16 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + """This module contains common Pytest fixtures and hooks for unit tests.""" from unittest import mock @@ -9,7 +22,7 @@ from reportportal_client.aio.client import Client -@fixture() +@fixture def response(): """Cook up a mock for the Response with specific arguments.""" diff --git a/tests/logs/test_rp_file.py b/tests/logs/test_rp_file.py index 4af5e60f..b846a31c 100644 --- a/tests/logs/test_rp_file.py +++ b/tests/logs/test_rp_file.py @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License +# noinspection PyPackageRequirements import pytest from reportportal_client.core.rp_file import RPFile diff --git a/tests/logs/test_rp_log_handler.py b/tests/logs/test_rp_log_handler.py index 01357456..1f275a78 100644 --- a/tests/logs/test_rp_log_handler.py +++ b/tests/logs/test_rp_log_handler.py @@ -18,7 +18,7 @@ import pytest # noinspection PyProtectedMember -from reportportal_client._local import set_current +from reportportal_client._internal.local import set_current from reportportal_client.logs import RPLogHandler, RPLogger diff --git a/tests/logs/test_rp_logger.py b/tests/logs/test_rp_logger.py index 685481f5..a6beefb1 100644 --- a/tests/logs/test_rp_logger.py +++ b/tests/logs/test_rp_logger.py @@ -10,15 +10,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License + import inspect import logging import sys from logging import LogRecord from unittest import mock +# noinspection PyPackageRequirements import pytest -from reportportal_client._local import set_current +# noinspection PyProtectedMember +from reportportal_client._internal.local import set_current from reportportal_client.logs import RPLogger, RPLogHandler diff --git a/tests/steps/conftest.py b/tests/steps/conftest.py index d3852d70..ec86929e 100644 --- a/tests/steps/conftest.py +++ b/tests/steps/conftest.py @@ -13,6 +13,7 @@ from unittest import mock +# noinspection PyPackageRequirements from pytest import fixture from reportportal_client.client import RPClient diff --git a/tests/steps/test_steps.py b/tests/steps/test_steps.py index 9cd9afc5..8384ec28 100644 --- a/tests/steps/test_steps.py +++ b/tests/steps/test_steps.py @@ -15,7 +15,8 @@ from unittest import mock from reportportal_client import step -from reportportal_client._local import set_current +# noinspection PyProtectedMember +from reportportal_client._internal.local import set_current NESTED_STEP_NAME = 'test nested step' PARENT_STEP_ID = '123-123-1234-123' diff --git a/tests/test_client.py b/tests/test_client.py index 7927218f..643c96c2 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -10,10 +10,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License + import pickle from io import StringIO from unittest import mock +# noinspection PyPackageRequirements import pytest from requests import Response from requests.exceptions import ReadTimeout diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 40f5a868..dbba942f 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,5 +1,3 @@ -"""This script contains unit tests for the helpers script.""" - # Copyright (c) 2022 EPAM Systems # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License +"""This script contains unit tests for the helpers script.""" + from unittest import mock from reportportal_client.helpers import ( From d093b51e411312172eb607f78dac676702757876 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 5 Oct 2023 17:02:36 +0300 Subject: [PATCH 220/268] Fix build --- reportportal_client/_internal/aio/tasks.py | 5 ++++- reportportal_client/aio/client.py | 5 ++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/reportportal_client/_internal/aio/tasks.py b/reportportal_client/_internal/aio/tasks.py index 03194932..c5e4b237 100644 --- a/reportportal_client/_internal/aio/tasks.py +++ b/reportportal_client/_internal/aio/tasks.py @@ -19,12 +19,15 @@ from asyncio import Future from typing import Optional, List, TypeVar, Generic, Union, Generator, Awaitable, Coroutine, Any -from reportportal_client.aio.client import DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL from reportportal_client.aio.tasks import Task, BlockingOperationError _T = TypeVar('_T') +DEFAULT_TASK_TRIGGER_NUM: int = 10 +DEFAULT_TASK_TRIGGER_INTERVAL: float = 1.0 + + class BatchedTask(Generic[_T], Task[_T]): """Represents a Task which uses the current Thread to execute itself.""" diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 9eec682f..2c3929d5 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -30,7 +30,8 @@ from reportportal_client._internal.aio.http import RetryingClientSession # noinspection PyProtectedMember from reportportal_client._internal.aio.tasks import (BatchedTaskFactory, ThreadedTaskFactory, - TriggerTaskBatcher, BackgroundTaskList) + TriggerTaskBatcher, BackgroundTaskList, + DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL) # noinspection PyProtectedMember from reportportal_client._internal.local import set_current # noinspection PyProtectedMember @@ -61,8 +62,6 @@ DEFAULT_TASK_TIMEOUT: float = 60.0 DEFAULT_SHUTDOWN_TIMEOUT: float = 120.0 -DEFAULT_TASK_TRIGGER_NUM: int = 10 -DEFAULT_TASK_TRIGGER_INTERVAL: float = 1.0 class Client: From 7c5324cb6cbb61f49e0d3a2b129e9556eafde031 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 5 Oct 2023 17:06:55 +0300 Subject: [PATCH 221/268] Fix tests --- tests/_internal/aio/test_http.py | 10 +++++++++- tests/_internal/services/test_statistics.py | 20 ++++++++++---------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/_internal/aio/test_http.py b/tests/_internal/aio/test_http.py index d279a8d3..eda0fab9 100644 --- a/tests/_internal/aio/test_http.py +++ b/tests/_internal/aio/test_http.py @@ -10,6 +10,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License import http.server import socketserver import threading @@ -92,7 +100,7 @@ async def execute_http_request(port, retry_number, server_class, timeout_seconds exception = None result = None with get_http_server(server_handler=server_class, server_address=('', port)): - with mock.patch('reportportal_client.aio.http.ClientSession.get', async_mock): + with mock.patch('reportportal_client._internal.aio.http.ClientSession.get', async_mock): async with session: start_time = time.time() try: diff --git a/tests/_internal/services/test_statistics.py b/tests/_internal/services/test_statistics.py index d27966d0..abda1f59 100644 --- a/tests/_internal/services/test_statistics.py +++ b/tests/_internal/services/test_statistics.py @@ -61,10 +61,10 @@ EXPECTED_PARAMS = {'measurement_id': MID, 'api_secret': KEY} -@mock.patch('reportportal_client.services.statistics.get_client_id', +@mock.patch('reportportal_client._internal.services.statistics.get_client_id', mock.Mock(return_value='555')) -@mock.patch('reportportal_client.services.statistics.requests.post') -@mock.patch('reportportal_client.services.statistics.python_version', +@mock.patch('reportportal_client._internal.services.statistics.requests.post') +@mock.patch('reportportal_client._internal.services.statistics.python_version', mock.Mock(return_value='3.6.6')) def test_send_event(mocked_requests): """Test functionality of the send_event() function. @@ -78,17 +78,17 @@ def test_send_event(mocked_requests): params=EXPECTED_PARAMS) -@mock.patch('reportportal_client.services.statistics.get_client_id', +@mock.patch('reportportal_client._internal.services.statistics.get_client_id', mock.Mock(return_value='555')) -@mock.patch('reportportal_client.services.statistics.requests.post', +@mock.patch('reportportal_client._internal.services.statistics.requests.post', mock.Mock(side_effect=RequestException)) def test_send_event_raises(): """Test that the send_event() does not raise exceptions.""" send_event(EVENT_NAME, 'pytest-reportportal', '5.0.5') -@mock.patch('reportportal_client.services.statistics.requests.post') -@mock.patch('reportportal_client.services.statistics.python_version', +@mock.patch('reportportal_client._internal.services.statistics.requests.post') +@mock.patch('reportportal_client._internal.services.statistics.python_version', mock.Mock(return_value='3.6.6')) def test_same_client_id(mocked_requests): """Test functionality of the send_event() function. @@ -114,10 +114,10 @@ def test_same_client_id(mocked_requests): @pytest.mark.skipif(sys.version_info < (3, 8), reason="the test requires AsyncMock which was introduced in Python 3.8") -@mock.patch('reportportal_client.services.statistics.get_client_id', +@mock.patch('reportportal_client._internal.services.statistics.get_client_id', mock.Mock(return_value='555')) -@mock.patch('reportportal_client.services.statistics.aiohttp.ClientSession.post', MOCKED_AIOHTTP) -@mock.patch('reportportal_client.services.statistics.python_version', +@mock.patch('reportportal_client._internal.services.statistics.aiohttp.ClientSession.post', MOCKED_AIOHTTP) +@mock.patch('reportportal_client._internal.services.statistics.python_version', mock.Mock(return_value='3.6.6')) @pytest.mark.asyncio async def test_async_send_event(): From 429b1f282c6c4c8c6960a728b4e48021e9199d3f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 6 Oct 2023 14:36:13 +0300 Subject: [PATCH 222/268] Refactoring of client closing --- reportportal_client/__init__.py | 3 +- .../_internal/local/__init__.py | 7 +- .../_internal/local/__init__.pyi | 22 ----- reportportal_client/aio/client.py | 35 ++++---- reportportal_client/client.py | 89 +++++++++++-------- 5 files changed, 78 insertions(+), 78 deletions(-) delete mode 100644 reportportal_client/_internal/local/__init__.pyi diff --git a/reportportal_client/__init__.py b/reportportal_client/__init__.py index a3933a67..e5427ff0 100644 --- a/reportportal_client/__init__.py +++ b/reportportal_client/__init__.py @@ -14,13 +14,14 @@ """This package is the base package for ReportPortal client.""" # noinspection PyProtectedMember -from reportportal_client._internal.local import current +from reportportal_client._internal.local import current, set_current from reportportal_client.logs import RPLogger, RPLogHandler from reportportal_client.client import RP, RPClient, OutputType from reportportal_client.steps import step __all__ = [ 'current', + 'set_current', 'RP', 'RPClient', 'OutputType', diff --git a/reportportal_client/_internal/local/__init__.py b/reportportal_client/_internal/local/__init__.py index 6f3186d5..6a52a0b0 100644 --- a/reportportal_client/_internal/local/__init__.py +++ b/reportportal_client/_internal/local/__init__.py @@ -14,17 +14,20 @@ """ReportPortal client context storing and retrieving module.""" from threading import local +from typing import Optional + +from reportportal_client import RP __INSTANCES = local() -def current(): +def current() -> Optional[RP]: """Return current ReportPortal client.""" if hasattr(__INSTANCES, 'current'): return __INSTANCES.current -def set_current(client): +def set_current(client: Optional[RP]) -> None: """Save ReportPortal client as current. The method is not intended to use used by users. ReportPortal client calls diff --git a/reportportal_client/_internal/local/__init__.pyi b/reportportal_client/_internal/local/__init__.pyi deleted file mode 100644 index a5d72a7e..00000000 --- a/reportportal_client/_internal/local/__init__.pyi +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) 2022 EPAM Systems -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License - -from typing import Optional - -from reportportal_client import RP - - -def current() -> Optional[RP]: ... - - -def set_current(client: Optional[RP]) -> None: ... diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 2c3929d5..68512f98 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -806,13 +806,13 @@ async def finish_launch(self, :param attributes: Launch attributes. These attributes override attributes on Start Launch call. :return: Response message or None. """ - await self.__client.log_batch(self._log_batcher.flush()) - if not self.use_own_launch: - return "" - result = await self.__client.finish_launch(self.launch_uuid, end_time, status=status, - attributes=attributes, - **kwargs) - await self.__client.close() + if self.use_own_launch: + result = await self.__client.finish_launch(self.launch_uuid, end_time, status=status, + attributes=attributes, + **kwargs) + else: + result = "" + await self.close() return result async def update_test_item( @@ -936,6 +936,10 @@ def clone(self) -> 'AsyncRPClient': cloned._add_current_item(current_item) return cloned + async def close(self) -> None: + await self.__client.log_batch(self._log_batcher.flush()) + await self.__client.close() + class _RPClient(RP, metaclass=AbstractBaseClass): """Base class for different synchronous to asynchronous client implementations.""" @@ -1066,11 +1070,6 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: """ raise NotImplementedError('"create_task" method is not implemented!') - @abstractmethod - def finish_tasks(self) -> None: - """Ensure all pending Tasks are finished, block current Thread if necessary.""" - raise NotImplementedError('"create_task" method is not implemented!') - def _add_current_item(self, item: Task[_T]) -> None: """Add the last Item to the internal FILO queue. @@ -1219,7 +1218,7 @@ def finish_launch(self, result_coro = self.__empty_str() result_task = self.create_task(result_coro) - self.finish_tasks() + self.close() return result_task def update_test_item(self, @@ -1446,7 +1445,6 @@ def finish_tasks(self): logs = self._log_batcher.flush() if logs: self._loop.create_task(self._log_batch(logs)).blocking_result() - self._loop.create_task(self._close()).blocking_result() def clone(self) -> 'ThreadedRPClient': """Clone the Client object, set current Item ID as cloned Item ID. @@ -1475,6 +1473,10 @@ def clone(self) -> 'ThreadedRPClient': cloned._add_current_item(current_item) return cloned + def close(self) -> None: + self.finish_tasks() + self._loop.create_task(self._close()).blocking_result() + def __getstate__(self) -> Dict[str, Any]: """Control object pickling and return object fields as Dictionary. @@ -1621,7 +1623,6 @@ def finish_tasks(self) -> None: if logs: log_task = self._loop.create_task(self._log_batch(logs)) self._loop.run_until_complete(log_task) - self._loop.run_until_complete(self._close()) def clone(self) -> 'BatchedRPClient': """Clone the Client object, set current Item ID as cloned Item ID. @@ -1652,6 +1653,10 @@ def clone(self) -> 'BatchedRPClient': cloned._add_current_item(current_item) return cloned + def close(self) -> None: + self.finish_tasks() + self._loop.run_until_complete(self._close()) + def __getstate__(self) -> Dict[str, Any]: """Control object pickling and return object fields as Dictionary. diff --git a/reportportal_client/client.py b/reportportal_client/client.py index e6002d49..b7d0b099 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -292,23 +292,6 @@ def log(self, """ raise NotImplementedError('"log" method is not implemented!') - def start(self) -> None: - """Start the client.""" - warnings.warn( - message='`start` method is deprecated since 5.5.0 and will be subject for removing in the' - ' next major version. There is no any necessity to call this method anymore.', - category=DeprecationWarning, - stacklevel=2 - ) - - def terminate(self, *_: Any, **__: Any) -> None: - """Call this to terminate the client.""" - warnings.warn( - message='`terminate` method is deprecated since 5.5.0 and will be subject for removing in the' - ' next major version. There is no any necessity to call this method anymore.', - category=DeprecationWarning, - stacklevel=2 - ) @abstractmethod def current_item(self) -> Optional[str]: @@ -327,6 +310,30 @@ def clone(self) -> 'RP': """ raise NotImplementedError('"clone" method is not implemented!') + @abstractmethod + def close(self) -> None: + """Close current client connections and flush batches.""" + raise NotImplementedError('"clone" method is not implemented!') + + def start(self) -> None: + """Start the client.""" + warnings.warn( + message='`start` method is deprecated since 5.5.0 and will be subject for removing in the' + ' next major version. There is no any necessity to call this method anymore.', + category=DeprecationWarning, + stacklevel=2 + ) + + def terminate(self, *_: Any, **__: Any) -> None: + """Call this to terminate the client.""" + warnings.warn( + message='`terminate` method is deprecated since 5.5.0 and will be subject for removing in the' + ' next major version. There is no any necessity to call this method anymore.', + category=DeprecationWarning, + stacklevel=2 + ) + self.close() + class RPClient(RP): """ReportPortal client. @@ -677,27 +684,29 @@ def finish_launch(self, CANCELLED :param attributes: Launch attributes """ - self._log(self._log_batcher.flush()) - if not self.use_own_launch: - return "" - if self.launch_uuid is NOT_FOUND or not self.launch_uuid: - logger.warning('Attempt to finish non-existent launch') - return - url = uri_join(self.base_url_v2, 'launch', self.launch_uuid, 'finish') - request_payload = LaunchFinishRequest( - end_time, - status=status, - attributes=attributes, - description=kwargs.get('description') - ).payload - response = HttpRequest(self.session.put, url=url, json=request_payload, - verify_ssl=self.verify_ssl, - name='Finish Launch').make() - if not response: - return - logger.debug('finish_launch - ID: %s', self.launch_uuid) - logger.debug('response message: %s', response.message) - return response.message + if self.use_own_launch: + if self.launch_uuid is NOT_FOUND or not self.launch_uuid: + logger.warning('Attempt to finish non-existent launch') + return + url = uri_join(self.base_url_v2, 'launch', self.launch_uuid, 'finish') + request_payload = LaunchFinishRequest( + end_time, + status=status, + attributes=attributes, + description=kwargs.get('description') + ).payload + response = HttpRequest(self.session.put, url=url, json=request_payload, + verify_ssl=self.verify_ssl, + name='Finish Launch').make() + if not response: + return + logger.debug('finish_launch - ID: %s', self.launch_uuid) + logger.debug('response message: %s', response.message) + message = response.message + else: + message = "" + self.close() + return message def update_test_item(self, item_uuid: str, attributes: Optional[Union[list, dict]] = None, description: Optional[str] = None) -> Optional[str]: @@ -874,6 +883,10 @@ def clone(self) -> 'RPClient': cloned._add_current_item(current_item) return cloned + def close(self) -> None: + self._log(self._log_batcher.flush()) + self.session.close() + def __getstate__(self) -> Dict[str, Any]: """Control object pickling and return object fields as Dictionary. From af9ab51207d60137825347f6e18eae73dbfca91a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 6 Oct 2023 14:38:55 +0300 Subject: [PATCH 223/268] fix build --- reportportal_client/_internal/local/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/_internal/local/__init__.py b/reportportal_client/_internal/local/__init__.py index 6a52a0b0..e92ebdbb 100644 --- a/reportportal_client/_internal/local/__init__.py +++ b/reportportal_client/_internal/local/__init__.py @@ -16,7 +16,7 @@ from threading import local from typing import Optional -from reportportal_client import RP +from reportportal_client.client import RP __INSTANCES = local() From 57c0d00bdc89af3f5f565ed342d6bcd8cd2247e2 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 6 Oct 2023 16:45:51 +0300 Subject: [PATCH 224/268] A try to fix build --- reportportal_client/_internal/local/__init__.py | 2 +- reportportal_client/aio/client.py | 4 +--- reportportal_client/client.py | 2 +- reportportal_client/logs/__init__.py | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/reportportal_client/_internal/local/__init__.py b/reportportal_client/_internal/local/__init__.py index e92ebdbb..6a52a0b0 100644 --- a/reportportal_client/_internal/local/__init__.py +++ b/reportportal_client/_internal/local/__init__.py @@ -16,7 +16,7 @@ from threading import local from typing import Optional -from reportportal_client.client import RP +from reportportal_client import RP __INSTANCES = local() diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 68512f98..f5055a84 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -25,7 +25,7 @@ import aiohttp import certifi -from reportportal_client import RP, OutputType +from reportportal_client import RP, OutputType, set_current # noinspection PyProtectedMember from reportportal_client._internal.aio.http import RetryingClientSession # noinspection PyProtectedMember @@ -33,8 +33,6 @@ TriggerTaskBatcher, BackgroundTaskList, DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL) # noinspection PyProtectedMember -from reportportal_client._internal.local import set_current -# noinspection PyProtectedMember from reportportal_client._internal.logs.batcher import LogBatcher # noinspection PyProtectedMember from reportportal_client._internal.services.statistics import async_send_event diff --git a/reportportal_client/client.py b/reportportal_client/client.py index b7d0b099..54ad7b51 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -26,7 +26,7 @@ from requests.adapters import HTTPAdapter, Retry, DEFAULT_RETRIES # noinspection PyProtectedMember -from reportportal_client._internal.local import set_current +from reportportal_client import set_current # noinspection PyProtectedMember from reportportal_client._internal.logs.batcher import LogBatcher # noinspection PyProtectedMember diff --git a/reportportal_client/logs/__init__.py b/reportportal_client/logs/__init__.py index c5c7a700..c496c112 100644 --- a/reportportal_client/logs/__init__.py +++ b/reportportal_client/logs/__init__.py @@ -19,7 +19,7 @@ from urllib.parse import urlparse # noinspection PyProtectedMember -from reportportal_client._internal.local import current, set_current +from reportportal_client import current, set_current from reportportal_client.helpers import timestamp, TYPICAL_MULTIPART_FOOTER_LENGTH MAX_LOG_BATCH_SIZE: int = 20 From a7c1d2c21b04377f1d37546c4c9574e1eda371fb Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 6 Oct 2023 17:01:14 +0300 Subject: [PATCH 225/268] Fix build --- reportportal_client/_internal/local/__init__.py | 8 ++++---- reportportal_client/client.py | 2 +- reportportal_client/logs/__init__.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/reportportal_client/_internal/local/__init__.py b/reportportal_client/_internal/local/__init__.py index 6a52a0b0..3a617506 100644 --- a/reportportal_client/_internal/local/__init__.py +++ b/reportportal_client/_internal/local/__init__.py @@ -14,20 +14,20 @@ """ReportPortal client context storing and retrieving module.""" from threading import local -from typing import Optional +from typing import Optional, TypeVar -from reportportal_client import RP +T_co = TypeVar('T_co', bound='RP', covariant=True) __INSTANCES = local() -def current() -> Optional[RP]: +def current() -> Optional[T_co]: """Return current ReportPortal client.""" if hasattr(__INSTANCES, 'current'): return __INSTANCES.current -def set_current(client: Optional[RP]) -> None: +def set_current(client: Optional[T_co]) -> None: """Save ReportPortal client as current. The method is not intended to use used by users. ReportPortal client calls diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 54ad7b51..b7d0b099 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -26,7 +26,7 @@ from requests.adapters import HTTPAdapter, Retry, DEFAULT_RETRIES # noinspection PyProtectedMember -from reportportal_client import set_current +from reportportal_client._internal.local import set_current # noinspection PyProtectedMember from reportportal_client._internal.logs.batcher import LogBatcher # noinspection PyProtectedMember diff --git a/reportportal_client/logs/__init__.py b/reportportal_client/logs/__init__.py index c496c112..c5c7a700 100644 --- a/reportportal_client/logs/__init__.py +++ b/reportportal_client/logs/__init__.py @@ -19,7 +19,7 @@ from urllib.parse import urlparse # noinspection PyProtectedMember -from reportportal_client import current, set_current +from reportportal_client._internal.local import current, set_current from reportportal_client.helpers import timestamp, TYPICAL_MULTIPART_FOOTER_LENGTH MAX_LOG_BATCH_SIZE: int = 20 From bbe95ab7ad30e917a23d87db2a019b540c3781cb Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 6 Oct 2023 17:07:40 +0300 Subject: [PATCH 226/268] Fix flake8 --- reportportal_client/aio/client.py | 3 +++ reportportal_client/client.py | 1 + 2 files changed, 4 insertions(+) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index f5055a84..eced0a1b 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -935,6 +935,7 @@ def clone(self) -> 'AsyncRPClient': return cloned async def close(self) -> None: + """Close current client connections and flush batches.""" await self.__client.log_batch(self._log_batcher.flush()) await self.__client.close() @@ -1472,6 +1473,7 @@ def clone(self) -> 'ThreadedRPClient': return cloned def close(self) -> None: + """Close current client connections and flush batches.""" self.finish_tasks() self._loop.create_task(self._close()).blocking_result() @@ -1652,6 +1654,7 @@ def clone(self) -> 'BatchedRPClient': return cloned def close(self) -> None: + """Close current client connections and flush batches.""" self.finish_tasks() self._loop.run_until_complete(self._close()) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index b7d0b099..d8dad57f 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -884,6 +884,7 @@ def clone(self) -> 'RPClient': return cloned def close(self) -> None: + """Close current client connections and flush batches.""" self._log(self._log_batcher.flush()) self.session.close() From 853e5fd5a2e6cd3ead9fdb91af9abeae4bfcceb6 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 6 Oct 2023 17:41:49 +0300 Subject: [PATCH 227/268] Fix flake8 --- .../_internal/local/__init__.py | 7 ++---- .../_internal/local/__init__.pyi | 22 +++++++++++++++++++ reportportal_client/aio/client.py | 4 +++- reportportal_client/client.py | 1 - reportportal_client/steps/__init__.py | 3 ++- reportportal_client/steps/__init__.pyi | 2 +- 6 files changed, 30 insertions(+), 9 deletions(-) create mode 100644 reportportal_client/_internal/local/__init__.pyi diff --git a/reportportal_client/_internal/local/__init__.py b/reportportal_client/_internal/local/__init__.py index 3a617506..6f3186d5 100644 --- a/reportportal_client/_internal/local/__init__.py +++ b/reportportal_client/_internal/local/__init__.py @@ -14,20 +14,17 @@ """ReportPortal client context storing and retrieving module.""" from threading import local -from typing import Optional, TypeVar - -T_co = TypeVar('T_co', bound='RP', covariant=True) __INSTANCES = local() -def current() -> Optional[T_co]: +def current(): """Return current ReportPortal client.""" if hasattr(__INSTANCES, 'current'): return __INSTANCES.current -def set_current(client: Optional[T_co]) -> None: +def set_current(client): """Save ReportPortal client as current. The method is not intended to use used by users. ReportPortal client calls diff --git a/reportportal_client/_internal/local/__init__.pyi b/reportportal_client/_internal/local/__init__.pyi new file mode 100644 index 00000000..a5d72a7e --- /dev/null +++ b/reportportal_client/_internal/local/__init__.pyi @@ -0,0 +1,22 @@ +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +from typing import Optional + +from reportportal_client import RP + + +def current() -> Optional[RP]: ... + + +def set_current(client: Optional[RP]) -> None: ... diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index eced0a1b..a5e2306b 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -25,10 +25,12 @@ import aiohttp import certifi -from reportportal_client import RP, OutputType, set_current +from reportportal_client import RP, OutputType # noinspection PyProtectedMember from reportportal_client._internal.aio.http import RetryingClientSession # noinspection PyProtectedMember +from reportportal_client._internal.local import set_current +# noinspection PyProtectedMember from reportportal_client._internal.aio.tasks import (BatchedTaskFactory, ThreadedTaskFactory, TriggerTaskBatcher, BackgroundTaskList, DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index d8dad57f..e6e3a702 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -292,7 +292,6 @@ def log(self, """ raise NotImplementedError('"log" method is not implemented!') - @abstractmethod def current_item(self) -> Optional[str]: """Retrieve the last Item reported by the client (based on the internal FILO queue). diff --git a/reportportal_client/steps/__init__.py b/reportportal_client/steps/__init__.py index b51be5d7..b482395e 100644 --- a/reportportal_client/steps/__init__.py +++ b/reportportal_client/steps/__init__.py @@ -44,7 +44,8 @@ def test_my_nested_step(): """ from functools import wraps -from reportportal_client import current +# noinspection PyProtectedMember +from reportportal_client._internal.local import current from reportportal_client.helpers import get_function_params, timestamp NESTED_STEP_ITEMS = ('step', 'scenario', 'before_class', 'before_groups', diff --git a/reportportal_client/steps/__init__.pyi b/reportportal_client/steps/__init__.pyi index 9ace6093..9c63b3a3 100644 --- a/reportportal_client/steps/__init__.pyi +++ b/reportportal_client/steps/__init__.pyi @@ -13,7 +13,7 @@ from typing import Optional, Dict, Any, Callable, Union -from reportportal_client import RP +from reportportal_client.client import RP from reportportal_client.aio import Task From 2f9a8f64a0073a38671bd270098ba30d780ca186 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 6 Oct 2023 17:58:39 +0300 Subject: [PATCH 228/268] Fix task completion logic --- reportportal_client/aio/client.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index a5e2306b..91a44296 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -1439,10 +1439,11 @@ def finish_tasks(self): shutdown_start_time = datetime.time() with self._task_mutex: tasks = self._task_list.flush() - for task in tasks: - task.blocking_result() - if datetime.time() - shutdown_start_time >= self.shutdown_timeout: - break + if tasks: + for task in tasks: + task.blocking_result() + if datetime.time() - shutdown_start_time >= self.shutdown_timeout: + break logs = self._log_batcher.flush() if logs: self._loop.create_task(self._log_batch(logs)).blocking_result() From bbc2a18ce596ac2e08e3192b93286f0e436f4c63 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 6 Oct 2023 18:31:05 +0300 Subject: [PATCH 229/268] Update connection closing logic --- reportportal_client/aio/client.py | 17 ++++++++++------- reportportal_client/client.py | 6 +++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 91a44296..5f76ded3 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -812,6 +812,7 @@ async def finish_launch(self, **kwargs) else: result = "" + await self.__client.log_batch(self._log_batcher.flush()) await self.close() return result @@ -937,8 +938,7 @@ def clone(self) -> 'AsyncRPClient': return cloned async def close(self) -> None: - """Close current client connections and flush batches.""" - await self.__client.log_batch(self._log_batcher.flush()) + """Close current client connections.""" await self.__client.close() @@ -1071,6 +1071,11 @@ def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: """ raise NotImplementedError('"create_task" method is not implemented!') + @abstractmethod + def finish_tasks(self) -> None: + """Ensure all pending Tasks are finished, block current Thread if necessary.""" + raise NotImplementedError('"create_task" method is not implemented!') + def _add_current_item(self, item: Task[_T]) -> None: """Add the last Item to the internal FILO queue. @@ -1219,7 +1224,7 @@ def finish_launch(self, result_coro = self.__empty_str() result_task = self.create_task(result_coro) - self.close() + self.finish_tasks() return result_task def update_test_item(self, @@ -1476,8 +1481,7 @@ def clone(self) -> 'ThreadedRPClient': return cloned def close(self) -> None: - """Close current client connections and flush batches.""" - self.finish_tasks() + """Close current client connections.""" self._loop.create_task(self._close()).blocking_result() def __getstate__(self) -> Dict[str, Any]: @@ -1657,8 +1661,7 @@ def clone(self) -> 'BatchedRPClient': return cloned def close(self) -> None: - """Close current client connections and flush batches.""" - self.finish_tasks() + """Close current client connections.""" self._loop.run_until_complete(self._close()) def __getstate__(self) -> Dict[str, Any]: diff --git a/reportportal_client/client.py b/reportportal_client/client.py index e6e3a702..ba8ea78a 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -311,7 +311,7 @@ def clone(self) -> 'RP': @abstractmethod def close(self) -> None: - """Close current client connections and flush batches.""" + """Close current client connections.""" raise NotImplementedError('"clone" method is not implemented!') def start(self) -> None: @@ -704,6 +704,7 @@ def finish_launch(self, message = response.message else: message = "" + self._log(self._log_batcher.flush()) self.close() return message @@ -883,8 +884,7 @@ def clone(self) -> 'RPClient': return cloned def close(self) -> None: - """Close current client connections and flush batches.""" - self._log(self._log_batcher.flush()) + """Close current client connections.""" self.session.close() def __getstate__(self) -> Dict[str, Any]: From 9b6c008b68af9a9209722e6097d6205dee4ebff9 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 9 Oct 2023 14:00:17 +0300 Subject: [PATCH 230/268] Refactor client closing logic --- reportportal_client/aio/client.py | 38 ++++++++++++------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 5f76ded3..b58998ff 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -949,13 +949,14 @@ class _RPClient(RP, metaclass=AbstractBaseClass): log_batch_size: int log_batch_payload_limit: int + own_launch: bool + own_client: bool _item_stack: LifoQueue _log_batcher: LogBatcher __client: Client __launch_uuid: Optional[Task[str]] __endpoint: str __project: str - use_own_launch: bool __step_reporter: StepReporter @property @@ -1044,21 +1045,20 @@ def __init__( self.log_batch_size = log_batch_size self.log_batch_payload_limit = log_batch_payload_limit - if log_batcher: - self._log_batcher = log_batcher - else: - self._log_batcher = LogBatcher(log_batch_size, log_batch_payload_limit) + self._log_batcher = log_batcher or LogBatcher(log_batch_size, log_batch_payload_limit) if client: self.__client = client + self.own_client = False else: self.__client = Client(endpoint, project, **kwargs) + self.own_client = False if launch_uuid: self.__launch_uuid = launch_uuid - self.use_own_launch = False + self.own_launch = False else: - self.use_own_launch = True + self.own_launch = True set_current(self) @@ -1125,7 +1125,7 @@ def start_launch(self, 'rerun' option. :return: Launch UUID if successfully started or None. """ - if not self.use_own_launch: + if not self.own_launch: return self.launch_uuid launch_uuid_coro = self.__client.start_launch(name, start_time, description=description, attributes=attributes, rerun=rerun, rerun_of=rerun_of, @@ -1217,7 +1217,7 @@ def finish_launch(self, :return: Response message or None. """ self.create_task(self.__client.log_batch(self._log_batcher.flush())) - if self.use_own_launch: + if self.own_launch: result_coro = self.__client.finish_launch(self.launch_uuid, end_time, status=status, attributes=attributes, **kwargs) else: @@ -1323,8 +1323,10 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, self.create_task(self._log(rp_log)) return None - async def _close(self): - await self.__client.close() + def close(self) -> None: + """Close current client connections.""" + if self.own_client: + self.create_task(self.__client.close()).blocking_result() class ThreadedRPClient(_RPClient): @@ -1459,13 +1461,12 @@ def clone(self) -> 'ThreadedRPClient': :return: Cloned client object. :rtype: ThreadedRPClient """ - cloned_client = self.client.clone() # noinspection PyTypeChecker cloned = ThreadedRPClient( endpoint=self.endpoint, project=self.project, launch_uuid=self.launch_uuid, - client=cloned_client, + client=self.client, log_batch_size=self.log_batch_size, log_batch_payload_limit=self.log_batch_payload_limit, log_batcher=self._log_batcher, @@ -1480,10 +1481,6 @@ def clone(self) -> 'ThreadedRPClient': cloned._add_current_item(current_item) return cloned - def close(self) -> None: - """Close current client connections.""" - self._loop.create_task(self._close()).blocking_result() - def __getstate__(self) -> Dict[str, Any]: """Control object pickling and return object fields as Dictionary. @@ -1637,13 +1634,12 @@ def clone(self) -> 'BatchedRPClient': :return: Cloned client object. :rtype: BatchedRPClient """ - cloned_client = self.client.clone() # noinspection PyTypeChecker cloned = BatchedRPClient( endpoint=self.endpoint, project=self.project, launch_uuid=self.launch_uuid, - client=cloned_client, + client=self.client, log_batch_size=self.log_batch_size, log_batch_payload_limit=self.log_batch_payload_limit, log_batcher=self._log_batcher, @@ -1660,10 +1656,6 @@ def clone(self) -> 'BatchedRPClient': cloned._add_current_item(current_item) return cloned - def close(self) -> None: - """Close current client connections.""" - self._loop.run_until_complete(self._close()) - def __getstate__(self) -> Dict[str, Any]: """Control object pickling and return object fields as Dictionary. From 3641521f7bb5ae5a24168c891581cefef78084bb Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 9 Oct 2023 16:47:25 +0300 Subject: [PATCH 231/268] Add SSL tests --- MANIFEST.in | 1 + reportportal_client/aio/client.py | 31 ++++------------ setup.py | 2 +- test_res/root.pem | 31 ++++++++++++++++ tests/aio/test_aio_client.py | 60 ++++++++++++++++++++++++++++--- 5 files changed, 94 insertions(+), 31 deletions(-) create mode 100644 test_res/root.pem diff --git a/MANIFEST.in b/MANIFEST.in index 14236ba2..4fd17dd1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include MANIFEST.in include README.md CONTRIBUTING.rst requirements.txt +exclude test_res/* diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index b58998ff..06b21537 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -143,26 +143,7 @@ def __init__( self.print_output = print_output self._session = None self.__stat_task = None - self.api_key = api_key - if not self.api_key: - if 'token' in kwargs: - warnings.warn( - message='Argument `token` is deprecated since 5.3.5 and will be subject for removing in ' - 'the next major version. Use `api_key` argument instead.', - category=DeprecationWarning, - stacklevel=2 - ) - self.api_key = kwargs['token'] - - if not self.api_key: - warnings.warn( - message='Argument `api_key` is `None` or empty string, that is not supposed to happen ' - 'because Report Portal is usually requires an authorization key. Please check ' - 'your code.', - category=RuntimeWarning, - stacklevel=2 - ) async def session(self) -> RetryingClientSession: """Return aiohttp.ClientSession class instance, initialize it if necessary. @@ -172,13 +153,13 @@ async def session(self) -> RetryingClientSession: if self._session: return self._session - ssl_config = self.verify_ssl - if ssl_config: - if type(ssl_config) == str: - sl_config = ssl.create_default_context() - sl_config.load_cert_chain(ssl_config) + if self.verify_ssl is None or (type(self.verify_ssl) == bool and not self.verify_ssl): + ssl_config = False + else: + if type(self.verify_ssl) == str: + ssl_config = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=self.verify_ssl) else: - ssl_config = ssl.create_default_context(cafile=certifi.where()) + ssl_config = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=certifi.where()) connection_params = { 'ssl': ssl_config, diff --git a/setup.py b/setup.py index f418ec46..272b6ac5 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def read_file(fname): setup( name='reportportal-client', - packages=find_packages(exclude=('tests', 'tests.*')), + packages=find_packages(exclude=['tests', 'test_res']), package_data={ 'reportportal_client.steps': TYPE_STUBS, 'reportportal_client.core': TYPE_STUBS, diff --git a/test_res/root.pem b/test_res/root.pem new file mode 100644 index 00000000..b85c8037 --- /dev/null +++ b/test_res/root.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw +TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh +cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4 +WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu +ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY +MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc +h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+ +0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U +A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW +T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH +B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC +B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv +KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn +OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn +jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw +qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI +rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV +HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq +hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL +ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ +3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK +NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5 +ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur +TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC +jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc +oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq +4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA +mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d +emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= +-----END CERTIFICATE----- diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 0f45ecd9..899de476 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -10,8 +10,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License +import os import pickle import sys +from ssl import SSLContext from unittest import mock import aiohttp @@ -115,10 +117,8 @@ def test_clone(): LAUNCH_ID = 333 -EXPECTED_DEFAULT_URL = 'http://endpoint/ui/#project/launches/all/' + str( - LAUNCH_ID) -EXPECTED_DEBUG_URL = 'http://endpoint/ui/#project/userdebug/all/' + str( - LAUNCH_ID) +EXPECTED_DEFAULT_URL = f'http://endpoint/ui/#project/launches/all/{LAUNCH_ID}' +EXPECTED_DEBUG_URL = f'http://endpoint/ui/#project/userdebug/all/{LAUNCH_ID}' @pytest.mark.skipif(sys.version_info < (3, 8), @@ -133,7 +133,7 @@ def test_clone(): ] ) @pytest.mark.asyncio -async def test_launch_url_get(aio_client, launch_mode, project_name, expected_url): +async def test_launch_url_get(aio_client, launch_mode: str, project_name: str, expected_url: str): aio_client.project = project_name response = mock.AsyncMock() response.is_success = True @@ -145,3 +145,53 @@ async def get_call(*args, **kwargs): (await aio_client.session()).get.side_effect = get_call assert await (aio_client.get_launch_ui_url('test_launch_uuid')) == expected_url + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@pytest.mark.parametrize('default', [True, False]) +@mock.patch('reportportal_client.aio.client.aiohttp.TCPConnector') +@pytest.mark.asyncio +async def test_verify_ssl_default(connector_mock: mock.Mock, default: bool): + if default: + client = Client('http://endpoint', 'project', api_key='api_key') + else: + client = Client('http://endpoint', 'project', api_key='api_key', verify_ssl=True) + await client.session() + connector_mock.assert_called_once() + _, kwargs = connector_mock.call_args_list[0] + ssl_context: SSLContext = kwargs.get('ssl', None) + assert ssl_context is not None and isinstance(ssl_context, SSLContext) + assert len(ssl_context.get_ca_certs()) > 0 + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@pytest.mark.parametrize('param_value', [False, None]) +@mock.patch('reportportal_client.aio.client.aiohttp.TCPConnector') +@pytest.mark.asyncio +async def test_verify_ssl_off(connector_mock: mock.Mock, param_value): + client = Client('http://endpoint', 'project', api_key='api_key', verify_ssl=param_value) + await client.session() + connector_mock.assert_called_once() + _, kwargs = connector_mock.call_args_list[0] + ssl_context: SSLContext = kwargs.get('ssl', None) + assert ssl_context is not None and isinstance(ssl_context, bool) and not ssl_context + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@mock.patch('reportportal_client.aio.client.aiohttp.TCPConnector') +@pytest.mark.asyncio +async def test_verify_ssl_str(connector_mock: mock.Mock): + client = Client('http://endpoint', 'project', api_key='api_key', + verify_ssl=os.path.join(os.getcwd(), 'test_res/root.pem')) + await client.session() + connector_mock.assert_called_once() + _, kwargs = connector_mock.call_args_list[0] + ssl_context: SSLContext = kwargs.get('ssl', None) + assert ssl_context is not None and isinstance(ssl_context, SSLContext) + assert len(ssl_context.get_ca_certs()) == 1 + certificate = ssl_context.get_ca_certs()[0] + assert certificate['subject'][1] == (('organizationName', 'Internet Security Research Group'),) + assert certificate['notAfter'] == 'Jun 4 11:04:38 2035 GMT' From c233e110d75cbfa239461ac6a387caed96ff01f9 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 9 Oct 2023 16:49:19 +0300 Subject: [PATCH 232/268] Enable SSL tests for python 3.7 --- tests/aio/test_aio_client.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 899de476..18b67624 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -147,8 +147,6 @@ async def get_call(*args, **kwargs): assert await (aio_client.get_launch_ui_url('test_launch_uuid')) == expected_url -@pytest.mark.skipif(sys.version_info < (3, 8), - reason="the test requires AsyncMock which was introduced in Python 3.8") @pytest.mark.parametrize('default', [True, False]) @mock.patch('reportportal_client.aio.client.aiohttp.TCPConnector') @pytest.mark.asyncio @@ -165,8 +163,6 @@ async def test_verify_ssl_default(connector_mock: mock.Mock, default: bool): assert len(ssl_context.get_ca_certs()) > 0 -@pytest.mark.skipif(sys.version_info < (3, 8), - reason="the test requires AsyncMock which was introduced in Python 3.8") @pytest.mark.parametrize('param_value', [False, None]) @mock.patch('reportportal_client.aio.client.aiohttp.TCPConnector') @pytest.mark.asyncio @@ -179,8 +175,6 @@ async def test_verify_ssl_off(connector_mock: mock.Mock, param_value): assert ssl_context is not None and isinstance(ssl_context, bool) and not ssl_context -@pytest.mark.skipif(sys.version_info < (3, 8), - reason="the test requires AsyncMock which was introduced in Python 3.8") @mock.patch('reportportal_client.aio.client.aiohttp.TCPConnector') @pytest.mark.asyncio async def test_verify_ssl_str(connector_mock: mock.Mock): From 6dea905da2b6acf5a41d09e71dd7792a603d83bc Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 9 Oct 2023 16:53:06 +0300 Subject: [PATCH 233/268] Fix tests --- tests/aio/test_batched_client.py | 2 +- tests/aio/test_threaded_client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/aio/test_batched_client.py b/tests/aio/test_batched_client.py index bfce3c5b..a9502f0b 100644 --- a/tests/aio/test_batched_client.py +++ b/tests/aio/test_batched_client.py @@ -45,7 +45,7 @@ def test_clone(): cloned is not None and async_client is not cloned and cloned.client is not None - and cloned.client is not client + and cloned.client is client and cloned.step_reporter is not None and cloned.step_reporter is not step_reporter and cloned._task_list is async_client._task_list diff --git a/tests/aio/test_threaded_client.py b/tests/aio/test_threaded_client.py index 196549fb..caf26949 100644 --- a/tests/aio/test_threaded_client.py +++ b/tests/aio/test_threaded_client.py @@ -44,7 +44,7 @@ def test_clone(): cloned is not None and async_client is not cloned and cloned.client is not None - and cloned.client is not client + and cloned.client is client and cloned.step_reporter is not None and cloned.step_reporter is not step_reporter and cloned._task_list is async_client._task_list From 265e0d8010d62cc8322f9f1f759c0364ded57cf4 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 9 Oct 2023 16:58:26 +0300 Subject: [PATCH 234/268] Disable tests for Python 3.7 --- tests/aio/test_aio_client.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 18b67624..1ab87a56 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -147,6 +147,9 @@ async def get_call(*args, **kwargs): assert await (aio_client.get_launch_ui_url('test_launch_uuid')) == expected_url +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="For some reasons this does not work on Python 3.7 on Ubuntu, " + "but works on my Mac. Unfortunately GHA use Python 3.7 on Ubuntu.") @pytest.mark.parametrize('default', [True, False]) @mock.patch('reportportal_client.aio.client.aiohttp.TCPConnector') @pytest.mark.asyncio @@ -163,6 +166,9 @@ async def test_verify_ssl_default(connector_mock: mock.Mock, default: bool): assert len(ssl_context.get_ca_certs()) > 0 +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="For some reasons this does not work on Python 3.7 on Ubuntu, " + "but works on my Mac. Unfortunately GHA use Python 3.7 on Ubuntu.") @pytest.mark.parametrize('param_value', [False, None]) @mock.patch('reportportal_client.aio.client.aiohttp.TCPConnector') @pytest.mark.asyncio @@ -175,6 +181,9 @@ async def test_verify_ssl_off(connector_mock: mock.Mock, param_value): assert ssl_context is not None and isinstance(ssl_context, bool) and not ssl_context +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="For some reasons this does not work on Python 3.7 on Ubuntu, " + "but works on my Mac. Unfortunately GHA use Python 3.7 on Ubuntu.") @mock.patch('reportportal_client.aio.client.aiohttp.TCPConnector') @pytest.mark.asyncio async def test_verify_ssl_str(connector_mock: mock.Mock): @@ -189,3 +198,4 @@ async def test_verify_ssl_str(connector_mock: mock.Mock): certificate = ssl_context.get_ca_certs()[0] assert certificate['subject'][1] == (('organizationName', 'Internet Security Research Group'),) assert certificate['notAfter'] == 'Jun 4 11:04:38 2035 GMT' + From a0e78d1aae0d5bfb95c5ae68da60a4f205126737 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Mon, 9 Oct 2023 18:17:32 +0300 Subject: [PATCH 235/268] Add close method test --- tests/aio/test_aio_client.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 1ab87a56..abf04b66 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -199,3 +199,29 @@ async def test_verify_ssl_str(connector_mock: mock.Mock): assert certificate['subject'][1] == (('organizationName', 'Internet Security Research Group'),) assert certificate['notAfter'] == 'Jun 4 11:04:38 2035 GMT' + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="For some reasons this does not work on Python 3.7 on Ubuntu, " + "but works on my Mac. Unfortunately GHA use Python 3.7 on Ubuntu.") +@mock.patch('reportportal_client.aio.client.aiohttp.TCPConnector') +@pytest.mark.asyncio +async def test_keepalive_timeout(connector_mock: mock.Mock): + keepalive_timeout = 33 + client = Client('http://endpoint', 'project', api_key='api_key', + keepalive_timeout=keepalive_timeout) + await client.session() + connector_mock.assert_called_once() + _, kwargs = connector_mock.call_args_list[0] + timeout = kwargs.get('keepalive_timeout', None) + assert timeout is not None and timeout == keepalive_timeout + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@pytest.mark.asyncio +async def test_close(aio_client: Client): + # noinspection PyTypeChecker + session: mock.AsyncMock = await aio_client.session() + await (aio_client.close()) + assert aio_client._session is None + session.close.assert_awaited_once() From 7e772bd21360d40d54778c727fde43a3c7e2ed7b Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 10:41:11 +0300 Subject: [PATCH 236/268] Type corrections --- reportportal_client/aio/client.py | 46 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 06b21537..796d3aa6 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -223,7 +223,7 @@ async def start_launch(self, start_time: str, *, description: Optional[str] = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Optional[str]: @@ -271,8 +271,8 @@ async def start_test_item(self, *, parent_item_id: Optional[Union[str, Task[str]]] = None, description: Optional[str] = None, - attributes: Optional[List[Dict]] = None, - parameters: Optional[Dict] = None, + attributes: Optional[List[dict]] = None, + parameters: Optional[dict] = None, code_ref: Optional[str] = None, test_case_id: Optional[str] = None, has_stats: bool = True, @@ -332,7 +332,7 @@ async def finish_test_item(self, *, status: str = None, description: str = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, issue: Optional[Issue] = None, retry: bool = False, **kwargs: Any) -> Optional[str]: @@ -374,7 +374,7 @@ async def finish_launch(self, end_time: str, *, status: str = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, **kwargs: Any) -> Optional[str]: """Finish a Launch. @@ -404,7 +404,7 @@ async def finish_launch(self, async def update_test_item(self, item_uuid: Union[str, Task[str]], *, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, description: Optional[str] = None) -> Optional[str]: """Update existing Test Item at the ReportPortal. @@ -433,7 +433,7 @@ async def __get_launch_uuid_url(self, launch_uuid_future: Union[str, Task[str]]) logger.debug('get_launch_info - ID: %s', launch_uuid) return root_uri_join(self.base_url_v1, 'launch', 'uuid', launch_uuid) - async def get_launch_info(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[Dict]: + async def get_launch_info(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[dict]: """Get Launch information by Launch UUID. :param launch_uuid_future: Str or Task UUID returned on the Launch start. @@ -501,7 +501,7 @@ async def get_launch_ui_url(self, launch_uuid_future: Union[str, Task[str]]) -> logger.debug('get_launch_ui_url - ID: %s', launch_uuid) return url - async def get_project_settings(self) -> Optional[Dict]: + async def get_project_settings(self) -> Optional[dict]: """Get settings of the current Project. :return: Settings response in Dictionary. @@ -681,7 +681,7 @@ async def start_launch(self, name: str, start_time: str, description: Optional[str] = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Optional[str]: @@ -709,8 +709,8 @@ async def start_test_item(self, start_time: str, item_type: str, description: Optional[str] = None, - attributes: Optional[List[Dict]] = None, - parameters: Optional[Dict] = None, + attributes: Optional[List[dict]] = None, + parameters: Optional[dict] = None, parent_item_id: Optional[str] = None, has_stats: bool = True, code_ref: Optional[str] = None, @@ -750,7 +750,7 @@ async def finish_test_item(self, end_time: str, status: str = None, issue: Optional[Issue] = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, description: str = None, retry: bool = False, **kwargs: Any) -> Optional[str]: @@ -777,7 +777,7 @@ async def finish_test_item(self, async def finish_launch(self, end_time: str, status: str = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, **kwargs: Any) -> Optional[str]: """Finish a Launch. @@ -800,7 +800,7 @@ async def finish_launch(self, async def update_test_item( self, item_uuid: str, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, description: Optional[str] = None ) -> Optional[str]: """Update existing Test Item at the ReportPortal. @@ -862,7 +862,7 @@ async def get_launch_ui_url(self) -> Optional[str]: return return await self.__client.get_launch_ui_url(self.launch_uuid) - async def get_project_settings(self) -> Optional[Dict]: + async def get_project_settings(self) -> Optional[dict]: """Get settings of the current Project. :return: Settings response in Dictionary. @@ -874,7 +874,7 @@ async def log( time: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, + attachment: Optional[dict] = None, item_id: Optional[str] = None ) -> Optional[Tuple[str, ...]]: """Send Log message to the ReportPortal and attach it to a Test Item or Launch. @@ -1091,7 +1091,7 @@ def start_launch(self, name: str, start_time: str, description: Optional[str] = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, rerun: bool = False, rerun_of: Optional[str] = None, **kwargs) -> Task[str]: @@ -1119,8 +1119,8 @@ def start_test_item(self, start_time: str, item_type: str, description: Optional[str] = None, - attributes: Optional[List[Dict]] = None, - parameters: Optional[Dict] = None, + attributes: Optional[List[dict]] = None, + parameters: Optional[dict] = None, parent_item_id: Optional[Task[str]] = None, has_stats: bool = True, code_ref: Optional[str] = None, @@ -1159,7 +1159,7 @@ def finish_test_item(self, end_time: str, status: str = None, issue: Optional[Issue] = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, description: str = None, retry: bool = False, **kwargs: Any) -> Task[str]: @@ -1187,7 +1187,7 @@ def finish_test_item(self, def finish_launch(self, end_time: str, status: str = None, - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, **kwargs: Any) -> Task[str]: """Finish a Launch. @@ -1210,7 +1210,7 @@ def finish_launch(self, def update_test_item(self, item_uuid: Task[str], - attributes: Optional[Union[List, Dict]] = None, + attributes: Optional[Union[list, dict]] = None, description: Optional[str] = None) -> Task: """Update existing Test Item at the ReportPortal. @@ -1283,7 +1283,7 @@ async def _log(self, log_rq: AsyncRPRequestLog) -> Optional[Tuple[str, ...]]: return await self._log_batch(await self._log_batcher.append_async(log_rq)) def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[Dict] = None, item_id: Optional[Task[str]] = None) -> None: + attachment: Optional[dict] = None, item_id: Optional[Task[str]] = None) -> None: """Send Log message to the ReportPortal and attach it to a Test Item or Launch. This method stores Log messages in internal batch and sent it when batch is full, so not every method From 548e985399bce8598b269c81a6220454b116e604 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 12:43:06 +0300 Subject: [PATCH 237/268] Add start launch tests --- reportportal_client/helpers.py | 7 ++- tests/aio/test_aio_client.py | 82 ++++++++++++++++++++++++++++++++++ tests/conftest.py | 2 + 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index b6b0a39c..43cc6951 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -333,14 +333,17 @@ def calculate_file_part_size(file: Optional[RPFile]) -> int: return size -def agent_name_version(attributes: Optional[Union[List, Dict]] = None) -> Tuple[Optional[str], Optional[str]]: +def agent_name_version(attributes: Optional[Union[list, dict]] = None) -> Tuple[Optional[str], Optional[str]]: """Extract Agent name and version from given Launch attributes. :param attributes: Launch attributes as they provided to Start Launch call :return: Tuple of (agent name, version) """ + my_attributes = attributes + if isinstance(my_attributes, dict): + my_attributes = dict_to_payload(my_attributes) agent_name, agent_version = None, None - agent_attribute = [a for a in attributes if a.get('key') == 'agent'] if attributes else [] + agent_attribute = [a for a in my_attributes if a.get('key') == 'agent'] if my_attributes else [] if len(agent_attribute) > 0 and agent_attribute[0].get('value'): agent_name, agent_version = agent_attribute[0]['value'].split('|') return agent_name, agent_version diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index abf04b66..62d71f9f 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -14,6 +14,7 @@ import pickle import sys from ssl import SSLContext +from typing import List from unittest import mock import aiohttp @@ -30,6 +31,8 @@ ENDPOINT = 'http://localhost:8080' PROJECT = 'default_personal' API_KEY = 'test_key' +RESPONSE_ID = 'test_launch_uuid' +RETURN_JSON = {'id': RESPONSE_ID} def test_client_pickling(): @@ -225,3 +228,82 @@ async def test_close(aio_client: Client): await (aio_client.close()) assert aio_client._session is None session.close.assert_awaited_once() + + +def mock_basic_post_response(session: mock.AsyncMock): + return_object = mock.AsyncMock() + return_object.json.return_value = RETURN_JSON + session.post.return_value = return_object + + +def verify_attributes(expected_attributes: dict, actual_attributes: List[dict]): + if expected_attributes is None: + assert actual_attributes is None + return + else: + assert actual_attributes is not None + assert len(actual_attributes) == len(expected_attributes.items()) + for attribute in actual_attributes: + if 'key' in attribute: + assert expected_attributes.get(attribute.get('key')) == attribute.get('value') + assert attribute.get('system') is False + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@pytest.mark.asyncio +async def test_start_launch(aio_client: Client): + # noinspection PyTypeChecker + session: mock.AsyncMock = await aio_client.session() + mock_basic_post_response(session) + + launch_name = 'Test Launch' + start_time = str(1696921416000) + description = 'Test Launch description' + attributes = {'attribute_key': 'attribute_value'} + rerun_of = 'test_prent_launch_uuid' + result = await aio_client.start_launch(launch_name, start_time, description=description, + attributes=attributes, rerun=True, rerun_of=rerun_of) + + assert result == RESPONSE_ID + session.post.assert_called_once() + call_args = session.post.call_args_list[0] + assert '/api/v2/project/launch' == call_args[0][0] + kwargs = call_args[1] + assert kwargs.get('data') is None + actual_json = kwargs.get('json') + assert actual_json is not None + assert actual_json.get('rerun') is True + assert actual_json.get('rerunOf') == rerun_of + assert actual_json.get('description') == description + assert actual_json.get('startTime') == start_time + actual_attributes = actual_json.get('attributes') + verify_attributes(attributes, actual_attributes) + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@mock.patch('reportportal_client.aio.client.async_send_event') +@pytest.mark.asyncio +async def test_start_launch_event_send(async_send_event): + # noinspection PyTypeChecker + session = mock.AsyncMock() + client = Client('http://endpoint', 'project', api_key='api_key') + client._session = session + mock_basic_post_response(session) + + launch_name = 'Test Launch' + start_time = str(1696921416000) + agent_name = 'pytest-reportportal' + agent_version = '5.0.4' + attributes = {'agent': f'{agent_name}|{agent_version}'} + await client.start_launch(launch_name, start_time, attributes=attributes) + async_send_event.assert_called_once() + call_args = async_send_event.call_args_list[0] + args = call_args[0] + kwargs = call_args[1] + assert len(args) == 3 + assert args[0] == 'start_launch' + assert args[1] == agent_name + assert args[2] == agent_version + assert len(kwargs.items()) == 0 diff --git a/tests/conftest.py b/tests/conftest.py index f5c768b0..503cfdd6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,6 +46,7 @@ def rp_client(): """Prepare instance of the RPClient for testing.""" client = RPClient('http://endpoint', 'project', 'api_key') client.session = mock.Mock() + client._skip_analytics = True return client @@ -54,4 +55,5 @@ def aio_client(): """Prepare instance of the Client for testing.""" client = Client('http://endpoint', 'project', api_key='api_key') client._session = mock.AsyncMock() + client._skip_analytics = True return client From 8fc3a7395e93c71301cf54fbbfb7e965ce79103a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 12:46:44 +0300 Subject: [PATCH 238/268] Fix build --- tests/aio/test_aio_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 62d71f9f..56b3c323 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -230,7 +230,7 @@ async def test_close(aio_client: Client): session.close.assert_awaited_once() -def mock_basic_post_response(session: mock.AsyncMock): +def mock_basic_post_response(session): return_object = mock.AsyncMock() return_object.json.return_value = RETURN_JSON session.post.return_value = return_object From 1e1a2063e01800ccaf6dd3724f7002ba755da9a6 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 13:08:16 +0300 Subject: [PATCH 239/268] A try to fix tests --- tests/aio/test_aio_client.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 56b3c323..b9513303 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -281,11 +281,17 @@ async def test_start_launch(aio_client: Client): verify_attributes(attributes, actual_attributes) +if sys.version_info >= (3, 8): + ASYNC_STAT_MOCK = mock.AsyncMock() +else: + ASYNC_STAT_MOCK = None + + @pytest.mark.skipif(sys.version_info < (3, 8), reason="the test requires AsyncMock which was introduced in Python 3.8") -@mock.patch('reportportal_client.aio.client.async_send_event') +@mock.patch('reportportal_client.aio.client.async_send_event', ASYNC_STAT_MOCK) @pytest.mark.asyncio -async def test_start_launch_event_send(async_send_event): +async def test_start_launch_event_send(): # noinspection PyTypeChecker session = mock.AsyncMock() client = Client('http://endpoint', 'project', api_key='api_key') @@ -298,8 +304,12 @@ async def test_start_launch_event_send(async_send_event): agent_version = '5.0.4' attributes = {'agent': f'{agent_name}|{agent_version}'} await client.start_launch(launch_name, start_time, attributes=attributes) - async_send_event.assert_called_once() - call_args = async_send_event.call_args_list[0] + # noinspection PyUnresolvedReferences + stat_task = client._Client__stat_task + assert stat_task is not None + await stat_task + ASYNC_STAT_MOCK.assert_called_once() + call_args = ASYNC_STAT_MOCK.call_args_list[0] args = call_args[0] kwargs = call_args[1] assert len(args) == 3 From 27a2042a1d55f780b5d62d9f2bbd38d3354bdb24 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 13:14:26 +0300 Subject: [PATCH 240/268] A try to fix tests --- tests/aio/test_aio_client.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index b9513303..dd55de87 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -281,20 +281,15 @@ async def test_start_launch(aio_client: Client): verify_attributes(attributes, actual_attributes) -if sys.version_info >= (3, 8): - ASYNC_STAT_MOCK = mock.AsyncMock() -else: - ASYNC_STAT_MOCK = None - - @pytest.mark.skipif(sys.version_info < (3, 8), reason="the test requires AsyncMock which was introduced in Python 3.8") -@mock.patch('reportportal_client.aio.client.async_send_event', ASYNC_STAT_MOCK) +@mock.patch('reportportal_client.aio.client.async_send_event') @pytest.mark.asyncio -async def test_start_launch_event_send(): +async def test_start_launch_event_send(async_send_event): # noinspection PyTypeChecker session = mock.AsyncMock() client = Client('http://endpoint', 'project', api_key='api_key') + client._skip_analytics = False client._session = session mock_basic_post_response(session) @@ -304,12 +299,8 @@ async def test_start_launch_event_send(): agent_version = '5.0.4' attributes = {'agent': f'{agent_name}|{agent_version}'} await client.start_launch(launch_name, start_time, attributes=attributes) - # noinspection PyUnresolvedReferences - stat_task = client._Client__stat_task - assert stat_task is not None - await stat_task - ASYNC_STAT_MOCK.assert_called_once() - call_args = ASYNC_STAT_MOCK.call_args_list[0] + async_send_event.assert_called_once() + call_args = async_send_event.call_args_list[0] args = call_args[0] kwargs = call_args[1] assert len(args) == 3 From a73bae266d24c7dcae5235fe00a086ec4647f75a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 13:23:43 +0300 Subject: [PATCH 241/268] Launch print tests --- tests/aio/test_aio_client.py | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index dd55de87..3ed5e19e 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -13,6 +13,7 @@ import os import pickle import sys +from io import StringIO from ssl import SSLContext from typing import List from unittest import mock @@ -27,6 +28,7 @@ # noinspection PyProtectedMember from reportportal_client._internal.static.defines import NOT_SET from reportportal_client.aio.client import Client +from reportportal_client.helpers import timestamp ENDPOINT = 'http://localhost:8080' PROJECT = 'default_personal' @@ -308,3 +310,59 @@ async def test_start_launch_event_send(async_send_event): assert args[1] == agent_name assert args[2] == agent_version assert len(kwargs.items()) == 0 + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@pytest.mark.asyncio +async def test_launch_uuid_print(): + str_io = StringIO() + output_mock = mock.Mock() + output_mock.get_output.side_effect = lambda: str_io + client = Client(endpoint='http://endpoint', project='project', + api_key='test', launch_uuid_print=True, print_output=output_mock) + client._session = mock.AsyncMock() + client._skip_analytics = True + await client.start_launch('Test Launch', timestamp()) + assert 'ReportPortal Launch UUID: ' in str_io.getvalue() + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@pytest.mark.asyncio +async def test_no_launch_uuid_print(): + str_io = StringIO() + output_mock = mock.Mock() + output_mock.get_output.side_effect = lambda: str_io + client = Client(endpoint='http://endpoint', project='project', + api_key='test', launch_uuid_print=False, print_output=output_mock) + client._session = mock.AsyncMock() + client._skip_analytics = True + await client.start_launch('Test Launch', timestamp()) + assert 'ReportPortal Launch UUID: ' not in str_io.getvalue() + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@pytest.mark.asyncio +@mock.patch('reportportal_client.client.sys.stdout', new_callable=StringIO) +async def test_launch_uuid_print_default_io(mock_stdout): + client = Client(endpoint='http://endpoint', project='project', + api_key='test', launch_uuid_print=True) + client._session = mock.AsyncMock() + client._skip_analytics = True + await client.start_launch('Test Launch', timestamp()) + assert 'ReportPortal Launch UUID: ' in mock_stdout.getvalue() + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@pytest.mark.asyncio +@mock.patch('reportportal_client.client.sys.stdout', new_callable=StringIO) +async def test_launch_uuid_print_default_print(mock_stdout): + client = Client(endpoint='http://endpoint', project='project', + api_key='test') + client._session = mock.AsyncMock() + client._skip_analytics = True + await client.start_launch('Test Launch', timestamp()) + assert 'ReportPortal Launch UUID: ' not in mock_stdout.getvalue() From 6808b70e19d332e54d12f7cdfdb5b5d71b6b20a8 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 13:26:41 +0300 Subject: [PATCH 242/268] Format fix --- tests/aio/test_aio_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 3ed5e19e..34c722bd 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -335,7 +335,7 @@ async def test_no_launch_uuid_print(): output_mock = mock.Mock() output_mock.get_output.side_effect = lambda: str_io client = Client(endpoint='http://endpoint', project='project', - api_key='test', launch_uuid_print=False, print_output=output_mock) + api_key='test', launch_uuid_print=False, print_output=output_mock) client._session = mock.AsyncMock() client._skip_analytics = True await client.start_launch('Test Launch', timestamp()) @@ -348,7 +348,7 @@ async def test_no_launch_uuid_print(): @mock.patch('reportportal_client.client.sys.stdout', new_callable=StringIO) async def test_launch_uuid_print_default_io(mock_stdout): client = Client(endpoint='http://endpoint', project='project', - api_key='test', launch_uuid_print=True) + api_key='test', launch_uuid_print=True) client._session = mock.AsyncMock() client._skip_analytics = True await client.start_launch('Test Launch', timestamp()) @@ -361,7 +361,7 @@ async def test_launch_uuid_print_default_io(mock_stdout): @mock.patch('reportportal_client.client.sys.stdout', new_callable=StringIO) async def test_launch_uuid_print_default_print(mock_stdout): client = Client(endpoint='http://endpoint', project='project', - api_key='test') + api_key='test') client._session = mock.AsyncMock() client._skip_analytics = True await client.start_launch('Test Launch', timestamp()) From 3ac1ba0998ab1c452af0b244ead546bf027f73bb Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 16:54:51 +0300 Subject: [PATCH 243/268] Add connection error tests --- reportportal_client/aio/client.py | 8 ++--- tests/aio/test_aio_client.py | 49 +++++++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 796d3aa6..b2518d4d 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -29,12 +29,12 @@ # noinspection PyProtectedMember from reportportal_client._internal.aio.http import RetryingClientSession # noinspection PyProtectedMember -from reportportal_client._internal.local import set_current -# noinspection PyProtectedMember from reportportal_client._internal.aio.tasks import (BatchedTaskFactory, ThreadedTaskFactory, TriggerTaskBatcher, BackgroundTaskList, DEFAULT_TASK_TRIGGER_NUM, DEFAULT_TASK_TRIGGER_INTERVAL) # noinspection PyProtectedMember +from reportportal_client._internal.local import set_current +# noinspection PyProtectedMember from reportportal_client._internal.logs.batcher import LogBatcher # noinspection PyProtectedMember from reportportal_client._internal.services.statistics import async_send_event @@ -443,12 +443,12 @@ async def get_launch_info(self, launch_uuid_future: Union[str, Task[str]]) -> Op response = await AsyncHttpRequest((await self.session()).get, url=url).make() if not response: return + launch_info = None if response.is_success: launch_info = await response.json logger.debug('get_launch_info - Launch info: %s', launch_info) else: logger.warning('get_launch_info - Launch info: Failed to fetch launch ID from the API.') - launch_info = {} return launch_info async def __get_item_uuid_url(self, item_uuid_future: Union[str, Task[str]]) -> Optional[str]: @@ -466,7 +466,7 @@ async def get_item_id_by_uuid(self, item_uuid_future: Union[str, Task[str]]) -> """ url = self.__get_item_uuid_url(item_uuid_future) response = await AsyncHttpRequest((await self.session()).get, url=url).make() - return response.id if response else None + return await response.id if response else None async def get_launch_ui_id(self, launch_uuid_future: Union[str, Task[str]]) -> Optional[int]: """Get Launch ID of the given Launch. diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 34c722bd..b63f3aeb 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -14,6 +14,7 @@ import pickle import sys from io import StringIO +from json import JSONDecodeError from ssl import SSLContext from typing import List from unittest import mock @@ -21,6 +22,7 @@ import aiohttp # noinspection PyPackageRequirements import pytest +from aiohttp import ServerTimeoutError from reportportal_client import OutputType # noinspection PyProtectedMember @@ -318,7 +320,7 @@ async def test_start_launch_event_send(async_send_event): async def test_launch_uuid_print(): str_io = StringIO() output_mock = mock.Mock() - output_mock.get_output.side_effect = lambda: str_io + output_mock.get_output.return_value = str_io client = Client(endpoint='http://endpoint', project='project', api_key='test', launch_uuid_print=True, print_output=output_mock) client._session = mock.AsyncMock() @@ -333,7 +335,7 @@ async def test_launch_uuid_print(): async def test_no_launch_uuid_print(): str_io = StringIO() output_mock = mock.Mock() - output_mock.get_output.side_effect = lambda: str_io + output_mock.get_output.return_value = str_io client = Client(endpoint='http://endpoint', project='project', api_key='test', launch_uuid_print=False, print_output=output_mock) client._session = mock.AsyncMock() @@ -366,3 +368,46 @@ async def test_launch_uuid_print_default_print(mock_stdout): client._skip_analytics = True await client.start_launch('Test Launch', timestamp()) assert 'ReportPortal Launch UUID: ' not in mock_stdout.getvalue() + + +def connection_error(*args, **kwargs): + raise ServerTimeoutError() + + +def json_error(*args, **kwargs): + raise JSONDecodeError('invalid Json', '502 Gateway Timeout', 0) + + +def response_error(*args, **kwargs): + result = mock.AsyncMock() + result.ok = False + result.json.side_effect = json_error + result.status_code = 502 + return result + + +@pytest.mark.parametrize( + 'requests_method, client_method, client_params', + [ + ('post', 'start_launch', ['Test Launch', timestamp()]), + ('put', 'finish_launch', ['launch_uuid', timestamp()]), + ('post', 'start_test_item', ['launch_uuid', 'Test Item', timestamp(), 'STEP']), + ('put', 'finish_test_item', ['launch_uuid', 'test_item_id', timestamp()]), + ('put', 'update_test_item', ['test_item_id']), + ('get', 'get_item_id_by_uuid', ['test_item_uuid']), + ('get', 'get_launch_info', ['launch_uuid']), + ('get', 'get_launch_ui_id', ['launch_uuid']), + ('get', 'get_launch_ui_url', ['launch_uuid']), + ('get', 'get_project_settings', []) + ] +) +@pytest.mark.asyncio +async def test_connection_errors(aio_client, requests_method, client_method, + client_params): + getattr(await aio_client.session(), requests_method).side_effect = connection_error + result = await getattr(aio_client, client_method)(*client_params) + assert result is None + + getattr(await aio_client.session(), requests_method).side_effect = response_error + result = await getattr(aio_client, client_method)(*client_params) + assert result is None From efb029d33408b8d9af59f99cc817bd8dd146f969 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 17:09:14 +0300 Subject: [PATCH 244/268] Fix tests --- tests/aio/test_aio_client.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index b63f3aeb..73d12554 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -22,7 +22,7 @@ import aiohttp # noinspection PyPackageRequirements import pytest -from aiohttp import ServerTimeoutError +from aiohttp import ServerTimeoutError, ServerConnectionError from reportportal_client import OutputType # noinspection PyProtectedMember @@ -371,7 +371,7 @@ async def test_launch_uuid_print_default_print(mock_stdout): def connection_error(*args, **kwargs): - raise ServerTimeoutError() + raise ServerConnectionError() def json_error(*args, **kwargs): @@ -405,8 +405,10 @@ def response_error(*args, **kwargs): async def test_connection_errors(aio_client, requests_method, client_method, client_params): getattr(await aio_client.session(), requests_method).side_effect = connection_error - result = await getattr(aio_client, client_method)(*client_params) - assert result is None + try: + await getattr(aio_client, client_method)(*client_params) + except Exception as e: + assert type(e) == ServerConnectionError getattr(await aio_client.session(), requests_method).side_effect = response_error result = await getattr(aio_client, client_method)(*client_params) From 5fe8fd307c7a7dd46bb553562b853214ef7e5555 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 17:15:37 +0300 Subject: [PATCH 245/268] Fix for Python 3.7 --- tests/aio/test_aio_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 73d12554..6d644a58 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -386,6 +386,8 @@ def response_error(*args, **kwargs): return result +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") @pytest.mark.parametrize( 'requests_method, client_method, client_params', [ From a483310cd1e68cb6b3f420309ac262ed2928fa8e Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 17:18:37 +0300 Subject: [PATCH 246/268] Fix tests --- tests/aio/test_aio_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 6d644a58..a202a162 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -22,7 +22,7 @@ import aiohttp # noinspection PyPackageRequirements import pytest -from aiohttp import ServerTimeoutError, ServerConnectionError +from aiohttp import ServerConnectionError from reportportal_client import OutputType # noinspection PyProtectedMember From 01f6f943628d6c942db92c0a0087f5b6d2d69933 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 17:47:49 +0300 Subject: [PATCH 247/268] Add more tests --- tests/aio/test_aio_client.py | 41 ++++++++++++++++++++++++++++++++++-- tests/test_client.py | 23 ++------------------ 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index a202a162..9d535fa2 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -24,6 +24,7 @@ import pytest from aiohttp import ServerConnectionError +from core.rp_requests import AsyncRPRequestLog from reportportal_client import OutputType # noinspection PyProtectedMember from reportportal_client._internal.aio.http import RetryingClientSession, DEFAULT_RETRY_NUMBER @@ -289,7 +290,7 @@ async def test_start_launch(aio_client: Client): reason="the test requires AsyncMock which was introduced in Python 3.8") @mock.patch('reportportal_client.aio.client.async_send_event') @pytest.mark.asyncio -async def test_start_launch_event_send(async_send_event): +async def test_start_launch_statistics_send(async_send_event): # noinspection PyTypeChecker session = mock.AsyncMock() client = Client('http://endpoint', 'project', api_key='api_key') @@ -314,6 +315,28 @@ async def test_start_launch_event_send(async_send_event): assert len(kwargs.items()) == 0 +@pytest.mark.skipif(sys.version_info < (3, 8), + reason="the test requires AsyncMock which was introduced in Python 3.8") +@mock.patch('reportportal_client.aio.client.getenv') +@mock.patch('reportportal_client.aio.client.async_send_event') +@pytest.mark.asyncio +async def test_start_launch_no_statistics_send(async_send_event: mock.AsyncMock, getenv): + getenv.return_value = '1' + # noinspection PyTypeChecker + session = mock.AsyncMock() + client = Client('http://endpoint', 'project', api_key='api_key') + client._session = session + mock_basic_post_response(session) + + launch_name = 'Test Launch' + start_time = str(1696921416000) + agent_name = 'pytest-reportportal' + agent_version = '5.0.4' + attributes = {'agent': f'{agent_name}|{agent_version}'} + await client.start_launch(launch_name, start_time, attributes=attributes) + async_send_event.assert_not_called() + + @pytest.mark.skipif(sys.version_info < (3, 8), reason="the test requires AsyncMock which was introduced in Python 3.8") @pytest.mark.asyncio @@ -386,6 +409,14 @@ def response_error(*args, **kwargs): return result +def invalid_response(*args, **kwargs): + result = mock.AsyncMock() + result.ok = True + result.json.side_effect = json_error + result.status_code = 200 + return result + + @pytest.mark.skipif(sys.version_info < (3, 8), reason="the test requires AsyncMock which was introduced in Python 3.8") @pytest.mark.parametrize( @@ -400,7 +431,8 @@ def response_error(*args, **kwargs): ('get', 'get_launch_info', ['launch_uuid']), ('get', 'get_launch_ui_id', ['launch_uuid']), ('get', 'get_launch_ui_url', ['launch_uuid']), - ('get', 'get_project_settings', []) + ('get', 'get_project_settings', []), + ('post', 'log_batch', [[AsyncRPRequestLog('launch_uuid', timestamp(), item_uuid='test_item_uuid')]]) ] ) @pytest.mark.asyncio @@ -410,8 +442,13 @@ async def test_connection_errors(aio_client, requests_method, client_method, try: await getattr(aio_client, client_method)(*client_params) except Exception as e: + # On this level we pass all errors through by design assert type(e) == ServerConnectionError getattr(await aio_client.session(), requests_method).side_effect = response_error result = await getattr(aio_client, client_method)(*client_params) assert result is None + + getattr(await aio_client.session(), requests_method).side_effect = invalid_response + result = await getattr(aio_client, client_method)(*client_params) + assert result is None diff --git a/tests/test_client.py b/tests/test_client.py index 643c96c2..d4a1f3b0 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -38,8 +38,8 @@ def response_error(*args, **kwargs): def invalid_response(*args, **kwargs): result = Response() result._content = \ - '405 Not Allowed' - result.status_code = 405 + 'Hello World!' + result.status_code = 200 return result @@ -69,25 +69,6 @@ def test_connection_errors(rp_client, requests_method, client_method, result = getattr(rp_client, client_method)(*client_params) assert result is None - -@pytest.mark.parametrize( - 'requests_method, client_method, client_params', - [ - ('put', 'finish_launch', [timestamp()]), - ('put', 'finish_test_item', ['test_item_id', timestamp()]), - ('get', 'get_item_id_by_uuid', ['test_item_uuid']), - ('get', 'get_launch_info', []), - ('get', 'get_launch_ui_id', []), - ('get', 'get_launch_ui_url', []), - ('get', 'get_project_settings', []), - ('post', 'start_launch', ['Test Launch', timestamp()]), - ('post', 'start_test_item', ['Test Item', timestamp(), 'STEP']), - ('put', 'update_test_item', ['test_item_id']) - ] -) -def test_invalid_responses(rp_client, requests_method, client_method, - client_params): - rp_client._RPClient__launch_uuid = 'test_launch_id' getattr(rp_client.session, requests_method).side_effect = invalid_response result = getattr(rp_client, client_method)(*client_params) assert result is None From c698f347cca96bbd58c046ae78479a83c90707e3 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 18:03:46 +0300 Subject: [PATCH 248/268] Fix tests --- tests/aio/test_aio_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 9d535fa2..7e354fd9 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -24,13 +24,13 @@ import pytest from aiohttp import ServerConnectionError -from core.rp_requests import AsyncRPRequestLog from reportportal_client import OutputType # noinspection PyProtectedMember from reportportal_client._internal.aio.http import RetryingClientSession, DEFAULT_RETRY_NUMBER # noinspection PyProtectedMember from reportportal_client._internal.static.defines import NOT_SET from reportportal_client.aio.client import Client +from reportportal_client.core.rp_requests import AsyncRPRequestLog from reportportal_client.helpers import timestamp ENDPOINT = 'http://localhost:8080' From 32744f6f5d271f5524a81e334bf9015d8daa9c01 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 18:06:52 +0300 Subject: [PATCH 249/268] Fix tests for Python 3.7 --- tests/aio/test_aio_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 7e354fd9..4100b111 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -255,7 +255,7 @@ def verify_attributes(expected_attributes: dict, actual_attributes: List[dict]): @pytest.mark.skipif(sys.version_info < (3, 8), - reason="the test requires AsyncMock which was introduced in Python 3.8") + reason='the test requires AsyncMock which was introduced in Python 3.8') @pytest.mark.asyncio async def test_start_launch(aio_client: Client): # noinspection PyTypeChecker @@ -287,7 +287,7 @@ async def test_start_launch(aio_client: Client): @pytest.mark.skipif(sys.version_info < (3, 8), - reason="the test requires AsyncMock which was introduced in Python 3.8") + reason='the test requires AsyncMock which was introduced in Python 3.8') @mock.patch('reportportal_client.aio.client.async_send_event') @pytest.mark.asyncio async def test_start_launch_statistics_send(async_send_event): @@ -316,11 +316,11 @@ async def test_start_launch_statistics_send(async_send_event): @pytest.mark.skipif(sys.version_info < (3, 8), - reason="the test requires AsyncMock which was introduced in Python 3.8") + reason='the test requires AsyncMock which was introduced in Python 3.8') @mock.patch('reportportal_client.aio.client.getenv') @mock.patch('reportportal_client.aio.client.async_send_event') @pytest.mark.asyncio -async def test_start_launch_no_statistics_send(async_send_event: mock.AsyncMock, getenv): +async def test_start_launch_no_statistics_send(async_send_event, getenv): getenv.return_value = '1' # noinspection PyTypeChecker session = mock.AsyncMock() From 47ec1f82486474b984885582ceeaab5ba28c2df5 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 10 Oct 2023 22:02:14 +0300 Subject: [PATCH 250/268] Add test_start_test_item --- reportportal_client/client.py | 2 +- tests/aio/test_aio_client.py | 69 ++++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index ba8ea78a..097c316a 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -563,7 +563,7 @@ def start_test_item(self, start_time: str, item_type: str, description: Optional[str] = None, - attributes: Optional[List[Dict]] = None, + attributes: Optional[List[dict]] = None, parameters: Optional[dict] = None, parent_item_id: Optional[str] = None, has_stats: bool = True, diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 4100b111..2b9fc02d 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -31,7 +31,7 @@ from reportportal_client._internal.static.defines import NOT_SET from reportportal_client.aio.client import Client from reportportal_client.core.rp_requests import AsyncRPRequestLog -from reportportal_client.helpers import timestamp +from reportportal_client.helpers import timestamp, dict_to_payload ENDPOINT = 'http://localhost:8080' PROJECT = 'default_personal' @@ -247,7 +247,7 @@ def verify_attributes(expected_attributes: dict, actual_attributes: List[dict]): return else: assert actual_attributes is not None - assert len(actual_attributes) == len(expected_attributes.items()) + assert len(actual_attributes) == len(expected_attributes) for attribute in actual_attributes: if 'key' in attribute: assert expected_attributes.get(attribute.get('key')) == attribute.get('value') @@ -452,3 +452,68 @@ async def test_connection_errors(aio_client, requests_method, client_method, getattr(await aio_client.session(), requests_method).side_effect = invalid_response result = await getattr(aio_client, client_method)(*client_params) assert result is None + + +def verify_parameters(expected_parameters: dict, actual_parameters: List[dict]): + if expected_parameters is None: + assert actual_parameters is None + return + else: + assert actual_parameters is not None + assert len(actual_parameters) == len(expected_parameters) + for attribute in actual_parameters: + assert expected_parameters.get(attribute.get('key')) == attribute.get('value') + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.parametrize( + 'parent_id, expected_uri', + [ + ('test_parent_uuid', '/api/v2/project/item/test_parent_uuid'), + (None, '/api/v2/project/item'), + ] +) +@pytest.mark.asyncio +async def test_start_test_item(aio_client: Client, parent_id, expected_uri): + # noinspection PyTypeChecker + session: mock.AsyncMock = await aio_client.session() + mock_basic_post_response(session) + + launch_uuid = 'test_launch_uuid' + item_name = 'Test Item' + start_time = str(1696921416000) + item_type = 'STEP' + description = 'Test Launch description' + attributes = {'attribute_key': 'attribute_value'} + parameters = {'parameter_key': 'parameter_value'} + code_ref = 'io.reportportal.test' + test_case_id = 'test_prent_launch_uuid[parameter_value]' + result = await aio_client.start_test_item(launch_uuid, item_name, start_time, item_type, + parent_item_id=parent_id, description=description, + attributes=dict_to_payload(attributes), parameters=parameters, + has_stats=False, code_ref=code_ref, test_case_id=test_case_id, + retry=True) + + assert result == RESPONSE_ID + session.post.assert_called_once() + call_args = session.post.call_args_list[0] + assert expected_uri == call_args[0][0] + kwargs = call_args[1] + assert kwargs.get('data') is None + actual_json = kwargs.get('json') + assert actual_json is not None + assert actual_json.get('retry') is True + assert actual_json.get('testCaseId') == test_case_id + assert actual_json.get('codeRef') == code_ref + assert actual_json.get('hasStats') is False + assert actual_json.get('description') == description + assert actual_json.get('parentId') is None + assert actual_json.get('type') == item_type + assert actual_json.get('startTime') == start_time + assert actual_json.get('name') == item_name + assert actual_json.get('launchUuid') == launch_uuid + actual_attributes = actual_json.get('attributes') + verify_attributes(attributes, actual_attributes) + actual_parameters = actual_json.get('parameters') + verify_parameters(parameters, actual_parameters) From e9578c2288951921562dc773b0fb49ce5f6ffd32 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 11 Oct 2023 15:54:27 +0300 Subject: [PATCH 251/268] Update helpers.verify_value_length function --- CHANGELOG.md | 1 + .../_internal/static/defines.py | 3 +- reportportal_client/helpers.py | 67 +++++++++++++------ tests/test_helpers.py | 42 ++++++++++-- 4 files changed, 84 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6fa4a05..6a89e320 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Changed - RPClient class does not use separate Thread for log processing anymore, by @HardNorth - Use `importlib.metadata` package for distribution data extraction for Python versions starting 3.8, by @HardNorth +- `helpers.verify_value_length` function updated to truncate attribute keys also, by @HardNorth ### Removed - Dependency on `six`, by @HardNorth diff --git a/reportportal_client/_internal/static/defines.py b/reportportal_client/_internal/static/defines.py index 62385394..4335e0e8 100644 --- a/reportportal_client/_internal/static/defines.py +++ b/reportportal_client/_internal/static/defines.py @@ -14,6 +14,7 @@ """This module provides RP client static objects and variables.""" import aenum as enum +from reportportal_client.helpers import ATTRIBUTE_LENGTH_LIMIT as ATTRIBUTE_LIMIT RP_LOG_LEVELS = { @@ -76,7 +77,7 @@ class Priority(enum.IntEnum): PRIORITY_LOW = 0x3 -ATTRIBUTE_LENGTH_LIMIT = 128 +ATTRIBUTE_LENGTH_LIMIT = ATTRIBUTE_LIMIT DEFAULT_PRIORITY = Priority.PRIORITY_MEDIUM LOW_PRIORITY = Priority.PRIORITY_LOW NOT_FOUND = _PresenceSentinel() diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index 43cc6951..f563c101 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -25,11 +25,11 @@ from typing import Optional, Any, List, Dict, Callable, Tuple, Union, TypeVar, Generic from reportportal_client.core.rp_file import RPFile -# noinspection PyProtectedMember -from reportportal_client._internal.static.defines import ATTRIBUTE_LENGTH_LIMIT logger: logging.Logger = logging.getLogger(__name__) _T = TypeVar('_T') +ATTRIBUTE_LENGTH_LIMIT: int = 128 +TRUNCATE_REPLACEMENT: str = '...' class LifoQueue(Generic[_T]): @@ -99,7 +99,7 @@ def generate_uuid() -> str: return str(uuid.uuid4()) -def dict_to_payload(dictionary: dict) -> List[dict]: +def dict_to_payload(dictionary: Optional[dict]) -> Optional[List[dict]]: """Convert incoming dictionary to the list of dictionaries. This function transforms the given dictionary of tags/attributes into @@ -109,11 +109,18 @@ def dict_to_payload(dictionary: dict) -> List[dict]: :param dictionary: Dictionary containing tags/attributes :return list: List of tags/attributes in the required format """ - hidden = dictionary.pop('system', False) - return [ - {'key': key, 'value': str(value), 'system': hidden} - for key, value in sorted(dictionary.items()) - ] + if not dictionary: + return dictionary + my_dictionary = dict(dictionary) + + hidden = my_dictionary.pop('system', None) + result = [] + for key, value in sorted(my_dictionary.items()): + attribute = {'key': str(key), 'value': str(value)} + if hidden is not None: + attribute['system'] = hidden + result.append(attribute) + return result def gen_attributes(rp_attributes: List[str]) -> List[Dict[str, str]]: @@ -202,28 +209,46 @@ def get_package_version(package_name: str) -> Optional[str]: return get_package_parameters(package_name, ['version'])[0] -def verify_value_length(attributes): +def truncate_attribute_string(text: str) -> str: + truncation_length = len(TRUNCATE_REPLACEMENT) + if len(text) > ATTRIBUTE_LENGTH_LIMIT and len(text) > truncation_length: + return text[:ATTRIBUTE_LENGTH_LIMIT - truncation_length] + TRUNCATE_REPLACEMENT + return text + + +def verify_value_length(attributes: Optional[Union[List[dict], dict]]) -> Optional[List[dict]]: """Verify length of the attribute value. The length of the attribute value should have size from '1' to '128'. Otherwise, HTTP response will return an error. Example of the input list: [{'key': 'tag_name', 'value': 'tag_value1'}, {'value': 'tag_value2'}] + :param attributes: List of attributes(tags) :return: List of attributes with corrected value length """ - if attributes is not None: - for pair in attributes: - if not isinstance(pair, dict): - continue - attr_value = pair.get('value') - if attr_value is None: - continue - try: - pair['value'] = attr_value[:ATTRIBUTE_LENGTH_LIMIT] - except TypeError: - continue - return attributes + if attributes is None: + return + + my_attributes = attributes + if isinstance(my_attributes, dict): + my_attributes = dict_to_payload(my_attributes) + + result = [] + for pair in my_attributes: + if not isinstance(pair, dict): + continue + attr_value = pair.get('value') + if attr_value is None: + continue + truncated = {} + truncated.update(pair) + result.append(truncated) + attr_key = pair.get('key') + if attr_key: + truncated['key'] = truncate_attribute_string(str(attr_key)) + truncated['value'] = truncate_attribute_string(str(attr_value)) + return result def timestamp() -> str: diff --git a/tests/test_helpers.py b/tests/test_helpers.py index dbba942f..8d243a6f 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -15,10 +15,13 @@ from unittest import mock +# noinspection PyPackageRequirements +import pytest + from reportportal_client.helpers import ( gen_attributes, get_launch_sys_attrs, - verify_value_length + verify_value_length, ATTRIBUTE_LENGTH_LIMIT, TRUNCATE_REPLACEMENT ) @@ -59,10 +62,35 @@ def test_get_launch_sys_attrs_docker(): assert result['cpu'] == 'unknown' -def test_verify_value_length(): +@pytest.mark.parametrize( + 'attributes, expected_attributes', + [ + ({'tn': 'v' * 129}, [{'key': 'tn', 'value': 'v' * ( + ATTRIBUTE_LENGTH_LIMIT - len(TRUNCATE_REPLACEMENT)) + TRUNCATE_REPLACEMENT}]), + ({'tn': 'v' * 128}, [{'key': 'tn', 'value': 'v' * 128}]), + ({'k' * 129: 'v'}, [{'key': 'k' * ( + ATTRIBUTE_LENGTH_LIMIT - len(TRUNCATE_REPLACEMENT)) + TRUNCATE_REPLACEMENT, 'value': 'v'}]), + ({'k' * 128: 'v'}, [{'key': 'k' * 128, 'value': 'v'}]), + ({'tn': 'v' * 128, 'system': True}, [{'key': 'tn', 'value': 'v' * 128, 'system': True}]), + ({'tn': 'v' * 129, 'system': True}, [{'key': 'tn', 'value': 'v' * ( + ATTRIBUTE_LENGTH_LIMIT - len(TRUNCATE_REPLACEMENT)) + TRUNCATE_REPLACEMENT, 'system': True}]), + ({'k' * 129: 'v', 'system': False}, [{'key': 'k' * ( + ATTRIBUTE_LENGTH_LIMIT - len(TRUNCATE_REPLACEMENT)) + TRUNCATE_REPLACEMENT, 'value': 'v', + 'system': False}]), + ([{'key': 'tn', 'value': 'v' * 129}], [{'key': 'tn', 'value': 'v' * ( + ATTRIBUTE_LENGTH_LIMIT - len(TRUNCATE_REPLACEMENT)) + TRUNCATE_REPLACEMENT}]), + ([{'key': 'k' * 129, 'value': 'v'}], [{'key': 'k' * ( + ATTRIBUTE_LENGTH_LIMIT - len(TRUNCATE_REPLACEMENT)) + TRUNCATE_REPLACEMENT, 'value': 'v'}]), + + ] +) +def test_verify_value_length(attributes, expected_attributes): """Test for validate verify_value_length() function.""" - inputl = [{'key': 'tn', 'value': 'v' * 130}, [1, 2], - {'value': 'tv2'}, {'value': 300}] - expected = [{'key': 'tn', 'value': 'v' * 128}, [1, 2], - {'value': 'tv2'}, {'value': 300}] - assert verify_value_length(inputl) == expected + result = verify_value_length(attributes) + assert len(result) == len(expected_attributes) + for i, element in enumerate(result): + expected = expected_attributes[i] + assert len(element) == len(expected) + assert element.get('key') == expected.get('key') + assert element.get('value') == expected.get('value') + assert element.get('system') == expected.get('system') From 0f7f4d67b2ee6c7e3d12b88874761ade753f8248 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 11 Oct 2023 15:54:52 +0300 Subject: [PATCH 252/268] Add more aio client tests --- reportportal_client/aio/client.py | 8 +- reportportal_client/client.py | 4 +- tests/aio/test_aio_client.py | 129 ++++++++++++++++++++++++++++-- 3 files changed, 130 insertions(+), 11 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index b2518d4d..59ae6efa 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -271,7 +271,7 @@ async def start_test_item(self, *, parent_item_id: Optional[Union[str, Task[str]]] = None, description: Optional[str] = None, - attributes: Optional[List[dict]] = None, + attributes: Optional[Union[List[dict], dict]] = None, parameters: Optional[dict] = None, code_ref: Optional[str] = None, test_case_id: Optional[str] = None, @@ -319,7 +319,7 @@ async def start_test_item(self, if not response: return item_id = await response.id - if item_id is NOT_FOUND: + if item_id is NOT_FOUND or item_id is None: logger.warning('start_test_item - invalid response: %s', str(await response.json)) else: logger.debug('start_test_item - ID: %s', item_id) @@ -355,7 +355,7 @@ async def finish_test_item(self, end_time, launch_uuid, status, - attributes=attributes, + attributes=verify_value_length(attributes), description=description, is_skipped_an_issue=self.is_skipped_an_issue, issue=issue, @@ -389,7 +389,7 @@ async def finish_launch(self, request_payload = LaunchFinishRequest( end_time, status=status, - attributes=attributes, + attributes=verify_value_length(attributes), description=kwargs.get('description') ).payload response = await AsyncHttpRequest((await self.session()).put, url=url, json=request_payload, diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 097c316a..9a106679 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -148,7 +148,7 @@ def start_test_item(self, start_time: str, item_type: str, description: Optional[str] = None, - attributes: Optional[List[dict]] = None, + attributes: Optional[Union[List[dict], dict]] = None, parameters: Optional[dict] = None, parent_item_id: Optional[str] = None, has_stats: bool = True, @@ -563,7 +563,7 @@ def start_test_item(self, start_time: str, item_type: str, description: Optional[str] = None, - attributes: Optional[List[dict]] = None, + attributes: Optional[Union[List[dict], dict]] = None, parameters: Optional[dict] = None, parent_item_id: Optional[str] = None, has_stats: bool = True, diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 2b9fc02d..7a40deb0 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -30,14 +30,17 @@ # noinspection PyProtectedMember from reportportal_client._internal.static.defines import NOT_SET from reportportal_client.aio.client import Client +from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import AsyncRPRequestLog -from reportportal_client.helpers import timestamp, dict_to_payload +from reportportal_client.helpers import timestamp ENDPOINT = 'http://localhost:8080' PROJECT = 'default_personal' API_KEY = 'test_key' RESPONSE_ID = 'test_launch_uuid' -RETURN_JSON = {'id': RESPONSE_ID} +RETURN_POST_JSON = {'id': RESPONSE_ID} +RESPONSE_MESSAGE = 'Item finished successfully' +RETURN_PUT_JSON = {'message': RESPONSE_MESSAGE} def test_client_pickling(): @@ -237,7 +240,7 @@ async def test_close(aio_client: Client): def mock_basic_post_response(session): return_object = mock.AsyncMock() - return_object.json.return_value = RETURN_JSON + return_object.json.return_value = RETURN_POST_JSON session.post.return_value = return_object @@ -488,10 +491,10 @@ async def test_start_test_item(aio_client: Client, parent_id, expected_uri): attributes = {'attribute_key': 'attribute_value'} parameters = {'parameter_key': 'parameter_value'} code_ref = 'io.reportportal.test' - test_case_id = 'test_prent_launch_uuid[parameter_value]' + test_case_id = 'io.reportportal.test[parameter_value]' result = await aio_client.start_test_item(launch_uuid, item_name, start_time, item_type, parent_item_id=parent_id, description=description, - attributes=dict_to_payload(attributes), parameters=parameters, + attributes=attributes, parameters=parameters, has_stats=False, code_ref=code_ref, test_case_id=test_case_id, retry=True) @@ -517,3 +520,119 @@ async def test_start_test_item(aio_client: Client, parent_id, expected_uri): verify_attributes(attributes, actual_attributes) actual_parameters = actual_json.get('parameters') verify_parameters(parameters, actual_parameters) + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.asyncio +async def test_start_test_item_default_values(aio_client: Client): + # noinspection PyTypeChecker + session: mock.AsyncMock = await aio_client.session() + mock_basic_post_response(session) + + expected_uri = '/api/v2/project/item' + launch_uuid = 'test_launch_uuid' + item_name = 'Test Item' + start_time = str(1696921416000) + item_type = 'STEP' + result = await aio_client.start_test_item(launch_uuid, item_name, start_time, item_type) + + assert result == RESPONSE_ID + session.post.assert_called_once() + call_args = session.post.call_args_list[0] + assert expected_uri == call_args[0][0] + kwargs = call_args[1] + assert kwargs.get('data') is None + actual_json = kwargs.get('json') + assert actual_json is not None + assert actual_json.get('retry') is False + assert actual_json.get('testCaseId') is None + assert actual_json.get('codeRef') is None + assert actual_json.get('hasStats') is True + assert actual_json.get('description') is None + assert actual_json.get('parentId') is None + assert actual_json.get('type') == item_type + assert actual_json.get('startTime') == start_time + assert actual_json.get('name') == item_name + assert actual_json.get('launchUuid') == launch_uuid + assert actual_json.get('attributes') is None + assert actual_json.get('parameters') is None + + +def mock_basic_put_response(session): + return_object = mock.AsyncMock() + return_object.json.return_value = RETURN_PUT_JSON + session.put.return_value = return_object + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.asyncio +async def test_finish_test_item(aio_client: Client): + # noinspection PyTypeChecker + session: mock.AsyncMock = await aio_client.session() + mock_basic_put_response(session) + + launch_uuid = 'test_launch_uuid' + item_id = 'test_item_uuid' + expected_uri = f'/api/v2/project/item/{item_id}' + end_time = str(1696921416000) + status = 'FAILED' + description = 'Test Launch description' + attributes = {'attribute_key': 'attribute_value'} + issue = Issue('pb001', comment='Horrible bug!') + + result = await aio_client.finish_test_item(launch_uuid, item_id, end_time, status=status, + description=description, attributes=attributes, + issue=issue, retry=True) + assert result == RESPONSE_MESSAGE + session.put.assert_called_once() + call_args = session.put.call_args_list[0] + assert expected_uri == call_args[0][0] + kwargs = call_args[1] + assert kwargs.get('data') is None + actual_json = kwargs.get('json') + assert actual_json is not None + assert actual_json.get('retry') is True + assert actual_json.get('description') == description + assert actual_json.get('launchUuid') == launch_uuid + assert actual_json.get('endTime') == end_time + assert actual_json.get('status') == status + actual_attributes = actual_json.get('attributes') + verify_attributes(attributes, actual_attributes) + actual_issue = actual_json.get('issue') + expected_issue = issue.payload + assert len(actual_issue) == len(expected_issue) + for entry in actual_issue.items(): + assert entry[1] == expected_issue[entry[0]] + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.asyncio +async def test_finish_test_item_default_values(aio_client: Client): + # noinspection PyTypeChecker + session: mock.AsyncMock = await aio_client.session() + mock_basic_put_response(session) + + launch_uuid = 'test_launch_uuid' + item_id = 'test_item_uuid' + expected_uri = f'/api/v2/project/item/{item_id}' + end_time = str(1696921416000) + + result = await aio_client.finish_test_item(launch_uuid, item_id, end_time) + assert result == RESPONSE_MESSAGE + session.put.assert_called_once() + call_args = session.put.call_args_list[0] + assert expected_uri == call_args[0][0] + kwargs = call_args[1] + assert kwargs.get('data') is None + actual_json = kwargs.get('json') + assert actual_json is not None + assert actual_json.get('retry') is False + assert actual_json.get('description') is None + assert actual_json.get('launchUuid') == launch_uuid + assert actual_json.get('endTime') == end_time + assert actual_json.get('status') is None + assert actual_json.get('attributes') is None + assert actual_json.get('issue') is None From fa64a56a11d1ac361a42c3f5dbb3a5afb429e9e0 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 11 Oct 2023 16:10:47 +0300 Subject: [PATCH 253/268] Fix tests --- tests/aio/test_aio_client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 7a40deb0..11f6eb73 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -250,11 +250,12 @@ def verify_attributes(expected_attributes: dict, actual_attributes: List[dict]): return else: assert actual_attributes is not None + hidden = expected_attributes.pop('system', None) assert len(actual_attributes) == len(expected_attributes) for attribute in actual_attributes: if 'key' in attribute: - assert expected_attributes.get(attribute.get('key')) == attribute.get('value') - assert attribute.get('system') is False + assert attribute.get('value') == expected_attributes.get(attribute.get('key')) + assert attribute.get('system') == hidden @pytest.mark.skipif(sys.version_info < (3, 8), @@ -579,7 +580,7 @@ async def test_finish_test_item(aio_client: Client): end_time = str(1696921416000) status = 'FAILED' description = 'Test Launch description' - attributes = {'attribute_key': 'attribute_value'} + attributes = {'attribute_key': 'attribute_value', 'system': True} issue = Issue('pb001', comment='Horrible bug!') result = await aio_client.finish_test_item(launch_uuid, item_id, end_time, status=status, From 9a37e9859c7f852e82e8c22d24e988d400c99227 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 11 Oct 2023 16:16:36 +0300 Subject: [PATCH 254/268] Add pydocs --- reportportal_client/helpers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index f563c101..b7a902e7 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -203,13 +203,18 @@ def get_package_parameters(package_name: str, parameters: List[str] = None) -> L def get_package_version(package_name: str) -> Optional[str]: """Get version of the given package. - :param package_name: Name of the package - :return: Version of the package + :param package_name: Name of the package. + :return: Version of the package. """ return get_package_parameters(package_name, ['version'])[0] def truncate_attribute_string(text: str) -> str: + """Truncate a text if it's longer than allowed. + + :param text: Text to truncate. + :return: Truncated text. + """ truncation_length = len(TRUNCATE_REPLACEMENT) if len(text) > ATTRIBUTE_LENGTH_LIMIT and len(text) > truncation_length: return text[:ATTRIBUTE_LENGTH_LIMIT - truncation_length] + TRUNCATE_REPLACEMENT From 22d1ec68996fdf3e129de0c0790584a86a34650d Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 11 Oct 2023 16:17:37 +0300 Subject: [PATCH 255/268] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a89e320..5ab10cae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ ### Changed - RPClient class does not use separate Thread for log processing anymore, by @HardNorth - Use `importlib.metadata` package for distribution data extraction for Python versions starting 3.8, by @HardNorth -- `helpers.verify_value_length` function updated to truncate attribute keys also, by @HardNorth +- `helpers.verify_value_length` function updated to truncate attribute keys also and reveal attributes were truncated, by @HardNorth ### Removed - Dependency on `six`, by @HardNorth From 25b816b96ca430f99f072408e83c6b95af7df894 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 11 Oct 2023 16:53:45 +0300 Subject: [PATCH 256/268] Add more tests --- reportportal_client/aio/client.py | 12 ++-- tests/aio/test_aio_client.py | 115 +++++++++++++++++++++++++++--- 2 files changed, 112 insertions(+), 15 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 59ae6efa..212bc63a 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -510,7 +510,7 @@ async def get_project_settings(self) -> Optional[dict]: response = await AsyncHttpRequest((await self.session()).get, url=url).make() return await response.json if response else None - async def log_batch(self, log_batch: Optional[List[AsyncRPRequestLog]]) -> Tuple[str, ...]: + async def log_batch(self, log_batch: Optional[List[AsyncRPRequestLog]]) -> Optional[Tuple[str, ...]]: """Send batch logging message to the ReportPortal. :param log_batch: A list of log message objects. @@ -520,6 +520,8 @@ async def log_batch(self, log_batch: Optional[List[AsyncRPRequestLog]]) -> Tuple if log_batch: response = await AsyncHttpRequest((await self.session()).post, url=url, data=AsyncRPLogBatch(log_batch).payload).make() + if not response: + return return await response.messages def clone(self) -> 'Client': @@ -1283,7 +1285,7 @@ async def _log(self, log_rq: AsyncRPRequestLog) -> Optional[Tuple[str, ...]]: return await self._log_batch(await self._log_batcher.append_async(log_rq)) def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, - attachment: Optional[dict] = None, item_id: Optional[Task[str]] = None) -> None: + attachment: Optional[dict] = None, item_id: Optional[Task[str]] = None) -> Task[Tuple[str, ...]]: """Send Log message to the ReportPortal and attach it to a Test Item or Launch. This method stores Log messages in internal batch and sent it when batch is full, so not every method @@ -1296,13 +1298,9 @@ def log(self, time: str, message: str, level: Optional[Union[int, str]] = None, :param item_id: UUID of the ReportPortal Item the message belongs to. :return: Response message Tuple if Log message batch was sent or None. """ - if item_id is NOT_FOUND: - logger.warning("Attempt to log to non-existent item") - return rp_file = RPFile(**attachment) if attachment else None rp_log = AsyncRPRequestLog(self.launch_uuid, time, rp_file, item_id, level, message) - self.create_task(self._log(rp_log)) - return None + return self.create_task(self._log(rp_log)) def close(self) -> None: """Close current client connections.""" diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 11f6eb73..5fec253b 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -16,7 +16,7 @@ from io import StringIO from json import JSONDecodeError from ssl import SSLContext -from typing import List +from typing import List, Optional from unittest import mock import aiohttp @@ -37,10 +37,12 @@ ENDPOINT = 'http://localhost:8080' PROJECT = 'default_personal' API_KEY = 'test_key' -RESPONSE_ID = 'test_launch_uuid' -RETURN_POST_JSON = {'id': RESPONSE_ID} +POST_RESPONSE_ID = 'test_launch_uuid' +RETURN_POST_JSON = {'id': POST_RESPONSE_ID} RESPONSE_MESSAGE = 'Item finished successfully' RETURN_PUT_JSON = {'message': RESPONSE_MESSAGE} +GET_RESPONSE_ID = 'test_item_id' +RETURN_GET_JSON = {'id': GET_RESPONSE_ID} def test_client_pickling(): @@ -244,7 +246,7 @@ def mock_basic_post_response(session): session.post.return_value = return_object -def verify_attributes(expected_attributes: dict, actual_attributes: List[dict]): +def verify_attributes(expected_attributes: Optional[dict], actual_attributes: Optional[List[dict]]): if expected_attributes is None: assert actual_attributes is None return @@ -255,7 +257,7 @@ def verify_attributes(expected_attributes: dict, actual_attributes: List[dict]): for attribute in actual_attributes: if 'key' in attribute: assert attribute.get('value') == expected_attributes.get(attribute.get('key')) - assert attribute.get('system') == hidden + assert attribute.get('system') == hidden @pytest.mark.skipif(sys.version_info < (3, 8), @@ -274,7 +276,7 @@ async def test_start_launch(aio_client: Client): result = await aio_client.start_launch(launch_name, start_time, description=description, attributes=attributes, rerun=True, rerun_of=rerun_of) - assert result == RESPONSE_ID + assert result == POST_RESPONSE_ID session.post.assert_called_once() call_args = session.post.call_args_list[0] assert '/api/v2/project/launch' == call_args[0][0] @@ -421,6 +423,10 @@ def invalid_response(*args, **kwargs): return result +def request_error(*args, **kwargs): + raise ValueError() + + @pytest.mark.skipif(sys.version_info < (3, 8), reason="the test requires AsyncMock which was introduced in Python 3.8") @pytest.mark.parametrize( @@ -457,6 +463,10 @@ async def test_connection_errors(aio_client, requests_method, client_method, result = await getattr(aio_client, client_method)(*client_params) assert result is None + getattr(await aio_client.session(), requests_method).side_effect = request_error + result = await getattr(aio_client, client_method)(*client_params) + assert result is None + def verify_parameters(expected_parameters: dict, actual_parameters: List[dict]): if expected_parameters is None: @@ -499,7 +509,7 @@ async def test_start_test_item(aio_client: Client, parent_id, expected_uri): has_stats=False, code_ref=code_ref, test_case_id=test_case_id, retry=True) - assert result == RESPONSE_ID + assert result == POST_RESPONSE_ID session.post.assert_called_once() call_args = session.post.call_args_list[0] assert expected_uri == call_args[0][0] @@ -538,7 +548,7 @@ async def test_start_test_item_default_values(aio_client: Client): item_type = 'STEP' result = await aio_client.start_test_item(launch_uuid, item_name, start_time, item_type) - assert result == RESPONSE_ID + assert result == POST_RESPONSE_ID session.post.assert_called_once() call_args = session.post.call_args_list[0] assert expected_uri == call_args[0][0] @@ -637,3 +647,92 @@ async def test_finish_test_item_default_values(aio_client: Client): assert actual_json.get('status') is None assert actual_json.get('attributes') is None assert actual_json.get('issue') is None + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.asyncio +async def test_finish_launch(aio_client: Client): + # noinspection PyTypeChecker + session: mock.AsyncMock = await aio_client.session() + mock_basic_put_response(session) + + launch_uuid = 'test_launch_uuid' + expected_uri = f'/api/v2/project/launch/{launch_uuid}/finish' + end_time = str(1696921416000) + status = 'FAILED' + attributes = {'attribute_key': 'attribute_value', 'system': False} + + result = await aio_client.finish_launch(launch_uuid, end_time, status=status, attributes=attributes) + assert result == RESPONSE_MESSAGE + session.put.assert_called_once() + call_args = session.put.call_args_list[0] + assert expected_uri == call_args[0][0] + kwargs = call_args[1] + assert kwargs.get('data') is None + actual_json = kwargs.get('json') + assert actual_json is not None + assert actual_json.get('endTime') == end_time + assert actual_json.get('status') == status + actual_attributes = actual_json.get('attributes') + verify_attributes(attributes, actual_attributes) + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.asyncio +async def test_finish_launch_default_values(aio_client: Client): + # noinspection PyTypeChecker + session: mock.AsyncMock = await aio_client.session() + mock_basic_put_response(session) + + launch_uuid = 'test_launch_uuid' + expected_uri = f'/api/v2/project/launch/{launch_uuid}/finish' + end_time = str(1696921416000) + + result = await aio_client.finish_launch(launch_uuid, end_time) + assert result == RESPONSE_MESSAGE + session.put.assert_called_once() + call_args = session.put.call_args_list[0] + assert expected_uri == call_args[0][0] + kwargs = call_args[1] + assert kwargs.get('data') is None + actual_json = kwargs.get('json') + assert actual_json is not None + assert actual_json.get('endTime') == end_time + assert actual_json.get('status') is None + actual_attributes = actual_json.get('attributes') + verify_attributes(None, actual_attributes) + + +def mock_basic_get_response(session): + return_object = mock.AsyncMock() + return_object.json.return_value = RETURN_GET_JSON + session.get.return_value = return_object + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.asyncio +async def test_update_item(aio_client: Client): + # noinspection PyTypeChecker + session: mock.AsyncMock = await aio_client.session() + mock_basic_put_response(session) + mock_basic_get_response(session) + + item_id = 'test_item_uuid' + expected_uri = f'/api/v1/project/item/{GET_RESPONSE_ID}/update' + description = 'Test Launch description' + attributes = {'attribute_key': 'attribute_value', 'system': True} + + result = await aio_client.update_test_item(item_id, description=description, attributes=attributes) + assert result == RESPONSE_MESSAGE + session.put.assert_called_once() + call_args = session.put.call_args_list[0] + assert expected_uri == call_args[0][0] + kwargs = call_args[1] + assert kwargs.get('data') is None + actual_json = kwargs.get('json') + assert actual_json is not None + actual_attributes = actual_json.get('attributes') + verify_attributes(attributes, actual_attributes) From 85d187a7da5173624e92b1b17de530d565c165e5 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 11 Oct 2023 17:41:41 +0300 Subject: [PATCH 257/268] Add more tests --- reportportal_client/aio/client.py | 4 +--- tests/aio/test_aio_client.py | 36 +++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 212bc63a..88e36f75 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -494,9 +494,7 @@ async def get_launch_ui_url(self, launch_uuid_future: Union[str, Task[str]]) -> launch_type = 'launches' if mode.upper() == 'DEFAULT' else 'userdebug' - path = 'ui/#{project_name}/{launch_type}/all/{launch_id}'.format( - project_name=self.project.lower(), launch_type=launch_type, - launch_id=launch_id) + path = f'ui/#{self.project.lower()}/{launch_type}/all/{launch_id}' url = uri_join(self.endpoint, path) logger.debug('get_launch_ui_url - ID: %s', launch_uuid) return url diff --git a/tests/aio/test_aio_client.py b/tests/aio/test_aio_client.py index 5fec253b..bcc264ac 100644 --- a/tests/aio/test_aio_client.py +++ b/tests/aio/test_aio_client.py @@ -736,3 +736,39 @@ async def test_update_item(aio_client: Client): assert actual_json is not None actual_attributes = actual_json.get('attributes') verify_attributes(attributes, actual_attributes) + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.asyncio +async def test_get_item_id_by_uuid(aio_client: Client): + # noinspection PyTypeChecker + session: mock.AsyncMock = await aio_client.session() + mock_basic_get_response(session) + + item_id = 'test_item_uuid' + expected_uri = f'/api/v1/project/item/uuid/{item_id}' + + result = await aio_client.get_item_id_by_uuid(item_id) + assert result == GET_RESPONSE_ID + session.get.assert_called_once() + call_args = session.get.call_args_list[0] + assert expected_uri == call_args[0][0] + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.asyncio +async def test_get_launch_ui_url(aio_client: Client): + # noinspection PyTypeChecker + session: mock.AsyncMock = await aio_client.session() + mock_basic_get_response(session) + + launch_id = 'test_launch_uuid' + expected_uri = f'/api/v1/project/launch/uuid/{launch_id}' + + result = await aio_client.get_launch_ui_url(launch_id) + assert result == f'http://endpoint/ui/#project/launches/all/{GET_RESPONSE_ID}' + session.get.assert_called_once() + call_args = session.get.call_args_list[0] + assert expected_uri == call_args[0][0] From eda0c708ae6fe944d7151e03270795fd8424a65d Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 11 Oct 2023 18:11:21 +0300 Subject: [PATCH 258/268] Add more tests --- tests/aio/test_async_client.py | 42 ++++++++++++++++++++++++++++++++++ tests/conftest.py | 10 +++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/tests/aio/test_async_client.py b/tests/aio/test_async_client.py index 6f5c2e29..70a742e5 100644 --- a/tests/aio/test_async_client.py +++ b/tests/aio/test_async_client.py @@ -12,6 +12,11 @@ # limitations under the License import pickle +import sys +from unittest import mock + +# noinspection PyPackageRequirements +import pytest from reportportal_client.aio import AsyncRPClient @@ -55,3 +60,40 @@ def test_clone(): ) assert cloned._item_stack.qsize() == 1 \ and async_client.current_item() == cloned.current_item() + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.parametrize( + 'launch_uuid', + [ + 'test_launch_uuid', + None, + ] +) +@pytest.mark.asyncio +async def test_start_launch(launch_uuid): + aio_client = mock.AsyncMock() + client = AsyncRPClient('http://endpoint', 'project', api_key='api_key', + client=aio_client, launch_uuid=launch_uuid) + launch_name = 'Test Launch' + start_time = str(1696921416000) + description = 'Test Launch description' + attributes = {'attribute_key': 'attribute_value'} + rerun = True + rerun_of = 'test_prent_launch_uuid' + result = await client.start_launch(launch_name, start_time, description=description, + attributes=attributes, rerun=rerun, rerun_of=rerun_of) + if launch_uuid: + assert result == launch_uuid + aio_client.start_launch.assert_not_called() + else: + assert result is not None + aio_client.start_launch.assert_called_once() + args, kwargs = aio_client.start_launch.call_args_list[0] + assert args[0] == launch_name + assert args[1] == start_time + assert kwargs.get('description') == description + assert kwargs.get('attributes') == attributes + assert kwargs.get('rerun') == rerun + assert kwargs.get('rerun_of') == rerun_of diff --git a/tests/conftest.py b/tests/conftest.py index 503cfdd6..1572a2d6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,7 +19,7 @@ from pytest import fixture from reportportal_client.client import RPClient -from reportportal_client.aio.client import Client +from reportportal_client.aio.client import Client, AsyncRPClient @fixture @@ -57,3 +57,11 @@ def aio_client(): client._session = mock.AsyncMock() client._skip_analytics = True return client + + +@fixture +def async_client(): + """Prepare instance of the AsyncRPClient for testing.""" + client = AsyncRPClient('http://endpoint', 'project', api_key='api_key', + client=mock.AsyncMock()) + return client From 2042520bf5541996d9ad507d1d71f690602d7dbb Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 12 Oct 2023 13:12:43 +0300 Subject: [PATCH 259/268] Add more tests --- tests/aio/test_async_client.py | 90 ++++++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 20 deletions(-) diff --git a/tests/aio/test_async_client.py b/tests/aio/test_async_client.py index 70a742e5..4ea884bc 100644 --- a/tests/aio/test_async_client.py +++ b/tests/aio/test_async_client.py @@ -19,6 +19,7 @@ import pytest from reportportal_client.aio import AsyncRPClient +from reportportal_client.helpers import timestamp def test_async_rp_client_pickling(): @@ -64,18 +65,11 @@ def test_clone(): @pytest.mark.skipif(sys.version_info < (3, 8), reason='the test requires AsyncMock which was introduced in Python 3.8') -@pytest.mark.parametrize( - 'launch_uuid', - [ - 'test_launch_uuid', - None, - ] -) @pytest.mark.asyncio -async def test_start_launch(launch_uuid): +async def test_start_launch(): aio_client = mock.AsyncMock() client = AsyncRPClient('http://endpoint', 'project', api_key='api_key', - client=aio_client, launch_uuid=launch_uuid) + client=aio_client) launch_name = 'Test Launch' start_time = str(1696921416000) description = 'Test Launch description' @@ -84,16 +78,72 @@ async def test_start_launch(launch_uuid): rerun_of = 'test_prent_launch_uuid' result = await client.start_launch(launch_name, start_time, description=description, attributes=attributes, rerun=rerun, rerun_of=rerun_of) - if launch_uuid: - assert result == launch_uuid + + assert result is not None + assert client.launch_uuid == result + aio_client.start_launch.assert_called_once() + args, kwargs = aio_client.start_launch.call_args_list[0] + assert args[0] == launch_name + assert args[1] == start_time + assert kwargs.get('description') == description + assert kwargs.get('attributes') == attributes + assert kwargs.get('rerun') == rerun + assert kwargs.get('rerun_of') == rerun_of + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.parametrize( + 'launch_uuid, method, params', + [ + ('test_launch_uuid', 'start_test_item', ['Test Item', timestamp(), 'STEP']), + ('test_launch_uuid', 'finish_test_item', ['test_item_id', timestamp()]), + ('test_launch_uuid', 'get_launch_info', []), + ('test_launch_uuid', 'get_launch_ui_id', []), + ('test_launch_uuid', 'get_launch_ui_url', []), + ('test_launch_uuid', 'log', [timestamp(), 'Test message']), + (None, 'start_test_item', ['Test Item', timestamp(), 'STEP']), + (None, 'finish_test_item', ['test_item_id', timestamp()]), + (None, 'get_launch_info', []), + (None, 'get_launch_ui_id', []), + (None, 'get_launch_ui_url', []), + (None, 'log', [timestamp(), 'Test message']), + ] +) +@pytest.mark.asyncio +async def test_launch_uuid_usage(launch_uuid, method, params): + started_launch_uuid = 'new_test_launch_uuid' + aio_client = mock.AsyncMock() + aio_client.start_launch.return_value = started_launch_uuid + client = AsyncRPClient('http://endpoint', 'project', api_key='api_key', + client=aio_client, launch_uuid=launch_uuid, log_batch_size=1) + + actual_launch_uuid = await client.start_launch('Test Launch', timestamp()) + await getattr(client, method)(*params) + + if launch_uuid is None: + aio_client.start_launch.assert_called_once() + assert actual_launch_uuid == started_launch_uuid + assert client.launch_uuid == started_launch_uuid + else: aio_client.start_launch.assert_not_called() + assert actual_launch_uuid == launch_uuid + assert client.launch_uuid == launch_uuid + assert client.launch_uuid == actual_launch_uuid + + if method == 'log': + getattr(aio_client, 'log_batch').assert_called_once() + args, kwargs = getattr(aio_client, 'log_batch').call_args_list[0] + batch = args[0] + assert isinstance(batch, list) + assert len(batch) == 1 + log = batch[0] + assert log.launch_uuid == actual_launch_uuid + assert log.time == params[0] + assert log.message == params[1] else: - assert result is not None - aio_client.start_launch.assert_called_once() - args, kwargs = aio_client.start_launch.call_args_list[0] - assert args[0] == launch_name - assert args[1] == start_time - assert kwargs.get('description') == description - assert kwargs.get('attributes') == attributes - assert kwargs.get('rerun') == rerun - assert kwargs.get('rerun_of') == rerun_of + getattr(aio_client, method).assert_called_once() + args, kwargs = getattr(aio_client, method).call_args_list[0] + assert args[0] == actual_launch_uuid + for i, param in enumerate(params): + assert args[i + 1] == param From 291eff5d4d7954a3a319e473c5f3c88b327fabab Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 12 Oct 2023 13:14:47 +0300 Subject: [PATCH 260/268] Add more tests --- tests/aio/test_async_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/aio/test_async_client.py b/tests/aio/test_async_client.py index 4ea884bc..35f9c402 100644 --- a/tests/aio/test_async_client.py +++ b/tests/aio/test_async_client.py @@ -117,7 +117,6 @@ async def test_launch_uuid_usage(launch_uuid, method, params): aio_client.start_launch.return_value = started_launch_uuid client = AsyncRPClient('http://endpoint', 'project', api_key='api_key', client=aio_client, launch_uuid=launch_uuid, log_batch_size=1) - actual_launch_uuid = await client.start_launch('Test Launch', timestamp()) await getattr(client, method)(*params) From 6f268b1929743ecec4b08af39115381224f1464b Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 12 Oct 2023 14:32:46 +0300 Subject: [PATCH 261/268] Update test --- tests/aio/test_async_client.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/aio/test_async_client.py b/tests/aio/test_async_client.py index 35f9c402..a55c0bf2 100644 --- a/tests/aio/test_async_client.py +++ b/tests/aio/test_async_client.py @@ -119,19 +119,24 @@ async def test_launch_uuid_usage(launch_uuid, method, params): client=aio_client, launch_uuid=launch_uuid, log_batch_size=1) actual_launch_uuid = await client.start_launch('Test Launch', timestamp()) await getattr(client, method)(*params) + finish_launch_message = await client.finish_launch(timestamp()) if launch_uuid is None: aio_client.start_launch.assert_called_once() assert actual_launch_uuid == started_launch_uuid assert client.launch_uuid == started_launch_uuid + aio_client.finish_launch.assert_called_once() + assert finish_launch_message else: aio_client.start_launch.assert_not_called() assert actual_launch_uuid == launch_uuid assert client.launch_uuid == launch_uuid + aio_client.finish_launch.assert_not_called() + assert finish_launch_message == '' assert client.launch_uuid == actual_launch_uuid if method == 'log': - getattr(aio_client, 'log_batch').assert_called_once() + assert len(getattr(aio_client, 'log_batch').call_args_list) == 2 args, kwargs = getattr(aio_client, 'log_batch').call_args_list[0] batch = args[0] assert isinstance(batch, list) From 3476496992143a81db6eb48faa254aa7c9875921 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 12 Oct 2023 14:59:16 +0300 Subject: [PATCH 262/268] Add more tests --- tests/aio/test_async_client.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/aio/test_async_client.py b/tests/aio/test_async_client.py index a55c0bf2..aa79c8cd 100644 --- a/tests/aio/test_async_client.py +++ b/tests/aio/test_async_client.py @@ -151,3 +151,23 @@ async def test_launch_uuid_usage(launch_uuid, method, params): assert args[0] == actual_launch_uuid for i, param in enumerate(params): assert args[i + 1] == param + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.asyncio +async def test_start_item_tracking(async_client: AsyncRPClient): + aio_client = async_client.client + + started_launch_uuid = 'new_test_launch_uuid' + aio_client.start_launch.return_value = started_launch_uuid + test_item_id = 'test_item_uuid' + aio_client.start_test_item.return_value = test_item_id + + await async_client.start_launch('Test Launch', timestamp()) + actual_item_id = await async_client.start_test_item('Test Item Name', timestamp(), 'STEP') + assert actual_item_id == test_item_id + assert async_client.current_item() == test_item_id + + await async_client.finish_test_item(actual_item_id, timestamp()) + assert async_client.current_item() is None From 7479384fd03cfad885ae529f2490c92e1a431db2 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 12 Oct 2023 15:38:10 +0300 Subject: [PATCH 263/268] Add more tests --- reportportal_client/aio/client.py | 20 ++++++++- tests/aio/test_batched_client.py | 69 ++++++++++++++++++++++++++++++- tests/aio/test_threaded_client.py | 2 +- 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index 88e36f75..eaf7ac42 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -1351,6 +1351,9 @@ def __init_loop(self, loop: Optional[asyncio.AbstractEventLoop] = None): daemon=True) self._thread.start() + async def __return_value(self, value): + return value + def __init__( self, endpoint: str, @@ -1358,6 +1361,7 @@ def __init__( *, task_timeout: float = DEFAULT_TASK_TIMEOUT, shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, + launch_uuid: Optional[Union[str, Task[str]]] = None, task_list: Optional[BackgroundTaskList[Task[_T]]] = None, task_mutex: Optional[threading.RLock] = None, loop: Optional[asyncio.AbstractEventLoop] = None, @@ -1399,11 +1403,15 @@ def __init__( :param loop: Event Loop which is used to process Tasks. The Client creates own one if this argument is None. """ - super().__init__(endpoint, project, **kwargs) self.task_timeout = task_timeout self.shutdown_timeout = shutdown_timeout self.__init_task_list(task_list, task_mutex) self.__init_loop(loop) + if type(launch_uuid) == str: + super().__init__(endpoint, project, + launch_uuid=self.create_task(self.__return_value(launch_uuid)), **kwargs) + else: + super().__init__(endpoint, project, launch_uuid=launch_uuid, **kwargs) def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: """Create a Task from given Coroutine. @@ -1518,6 +1526,9 @@ def __init_loop(self, loop: Optional[asyncio.AbstractEventLoop] = None): self._loop = asyncio.new_event_loop() self._loop.set_task_factory(BatchedTaskFactory()) + async def __return_value(self, value): + return value + def __init__( self, endpoint: str, @@ -1525,6 +1536,7 @@ def __init__( *, task_timeout: float = DEFAULT_TASK_TIMEOUT, shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT, + launch_uuid: Optional[Union[str, Task[str]]] = None, task_list: Optional[TriggerTaskBatcher] = None, task_mutex: Optional[threading.RLock] = None, loop: Optional[asyncio.AbstractEventLoop] = None, @@ -1570,7 +1582,6 @@ def __init__( :param trigger_num: Number of tasks which triggers Task batch execution. :param trigger_interval: Time limit which triggers Task batch execution. """ - super().__init__(endpoint, project, **kwargs) self.task_timeout = task_timeout self.shutdown_timeout = shutdown_timeout self.trigger_num = trigger_num @@ -1578,6 +1589,11 @@ def __init__( self.__init_task_list(task_list, task_mutex) self.__last_run_time = datetime.time() self.__init_loop(loop) + if type(launch_uuid) == str: + super().__init__(endpoint, project, + launch_uuid=self.create_task(self.__return_value(launch_uuid)), **kwargs) + else: + super().__init__(endpoint, project, launch_uuid=launch_uuid, **kwargs) def create_task(self, coro: Coroutine[Any, Any, _T]) -> Optional[Task[_T]]: """Create a Task from given Coroutine. diff --git a/tests/aio/test_batched_client.py b/tests/aio/test_batched_client.py index a9502f0b..9962a117 100644 --- a/tests/aio/test_batched_client.py +++ b/tests/aio/test_batched_client.py @@ -11,8 +11,14 @@ # See the License for the specific language governing permissions and # limitations under the License import pickle +import sys +from unittest import mock + +# noinspection PyPackageRequirements +import pytest from reportportal_client.aio import BatchedRPClient +from reportportal_client.helpers import timestamp def test_batched_rp_client_pickling(): @@ -60,7 +66,7 @@ def test_clone(): ) assert ( cloned.client.api_key == kwargs['api_key'] - and cloned.launch_uuid == kwargs['launch_uuid'] + and cloned.launch_uuid.blocking_result() == kwargs['launch_uuid'] and cloned.log_batch_size == kwargs['log_batch_size'] and cloned.log_batch_payload_limit == kwargs['log_batch_payload_limit'] and cloned.task_timeout == kwargs['task_timeout'] @@ -70,3 +76,64 @@ def test_clone(): ) assert cloned._item_stack.qsize() == 1 \ and async_client.current_item() == cloned.current_item() + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.parametrize( + 'launch_uuid, method, params', + [ + ('test_launch_uuid', 'start_test_item', ['Test Item', timestamp(), 'STEP']), + ('test_launch_uuid', 'finish_test_item', ['test_item_id', timestamp()]), + ('test_launch_uuid', 'get_launch_info', []), + ('test_launch_uuid', 'get_launch_ui_id', []), + ('test_launch_uuid', 'get_launch_ui_url', []), + ('test_launch_uuid', 'log', [timestamp(), 'Test message']), + (None, 'start_test_item', ['Test Item', timestamp(), 'STEP']), + (None, 'finish_test_item', ['test_item_id', timestamp()]), + (None, 'get_launch_info', []), + (None, 'get_launch_ui_id', []), + (None, 'get_launch_ui_url', []), + (None, 'log', [timestamp(), 'Test message']), + ] +) +def test_launch_uuid_usage(launch_uuid, method, params): + started_launch_uuid = 'new_test_launch_uuid' + aio_client = mock.AsyncMock() + aio_client.start_launch.return_value = started_launch_uuid + client = BatchedRPClient('http://endpoint', 'project', api_key='api_key', + client=aio_client, launch_uuid=launch_uuid, log_batch_size=1) + actual_launch_uuid = (client.start_launch('Test Launch', timestamp())).blocking_result() + getattr(client, method)(*params).blocking_result() + finish_launch_message = (client.finish_launch(timestamp())).blocking_result() + + if launch_uuid is None: + aio_client.start_launch.assert_called_once() + assert actual_launch_uuid == started_launch_uuid + assert client.launch_uuid.blocking_result() == started_launch_uuid + aio_client.finish_launch.assert_called_once() + assert finish_launch_message + else: + aio_client.start_launch.assert_not_called() + assert actual_launch_uuid == launch_uuid + assert client.launch_uuid.blocking_result() == launch_uuid + aio_client.finish_launch.assert_not_called() + assert finish_launch_message == '' + assert client.launch_uuid.blocking_result() == actual_launch_uuid + + if method == 'log': + assert len(getattr(aio_client, 'log_batch').call_args_list) == 2 + args, kwargs = getattr(aio_client, 'log_batch').call_args_list[0] + batch = args[0] + assert isinstance(batch, list) + assert len(batch) == 1 + log = batch[0] + assert log.launch_uuid.blocking_result() == actual_launch_uuid + assert log.time == params[0] + assert log.message == params[1] + else: + getattr(aio_client, method).assert_called_once() + args, kwargs = getattr(aio_client, method).call_args_list[0] + assert args[0].blocking_result() == actual_launch_uuid + for i, param in enumerate(params): + assert args[i + 1] == param diff --git a/tests/aio/test_threaded_client.py b/tests/aio/test_threaded_client.py index caf26949..cc0334c7 100644 --- a/tests/aio/test_threaded_client.py +++ b/tests/aio/test_threaded_client.py @@ -59,7 +59,7 @@ def test_clone(): ) assert ( cloned.client.api_key == kwargs['api_key'] - and cloned.launch_uuid == kwargs['launch_uuid'] + and cloned.launch_uuid.blocking_result() == kwargs['launch_uuid'] and cloned.log_batch_size == kwargs['log_batch_size'] and cloned.log_batch_payload_limit == kwargs['log_batch_payload_limit'] and cloned.task_timeout == kwargs['task_timeout'] From b79b66468f25c7a1613e308951ca971b4a208132 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 12 Oct 2023 15:40:43 +0300 Subject: [PATCH 264/268] Add more tests --- tests/aio/test_threaded_client.py | 66 +++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/aio/test_threaded_client.py b/tests/aio/test_threaded_client.py index cc0334c7..10a9500c 100644 --- a/tests/aio/test_threaded_client.py +++ b/tests/aio/test_threaded_client.py @@ -11,8 +11,13 @@ # See the License for the specific language governing permissions and # limitations under the License import pickle +import sys +from unittest import mock + +import pytest from reportportal_client.aio import ThreadedRPClient +from reportportal_client.helpers import timestamp def test_threaded_rp_client_pickling(): @@ -67,3 +72,64 @@ def test_clone(): ) assert cloned._item_stack.qsize() == 1 \ and async_client.current_item() == cloned.current_item() + + +@pytest.mark.skipif(sys.version_info < (3, 8), + reason='the test requires AsyncMock which was introduced in Python 3.8') +@pytest.mark.parametrize( + 'launch_uuid, method, params', + [ + ('test_launch_uuid', 'start_test_item', ['Test Item', timestamp(), 'STEP']), + ('test_launch_uuid', 'finish_test_item', ['test_item_id', timestamp()]), + ('test_launch_uuid', 'get_launch_info', []), + ('test_launch_uuid', 'get_launch_ui_id', []), + ('test_launch_uuid', 'get_launch_ui_url', []), + ('test_launch_uuid', 'log', [timestamp(), 'Test message']), + (None, 'start_test_item', ['Test Item', timestamp(), 'STEP']), + (None, 'finish_test_item', ['test_item_id', timestamp()]), + (None, 'get_launch_info', []), + (None, 'get_launch_ui_id', []), + (None, 'get_launch_ui_url', []), + (None, 'log', [timestamp(), 'Test message']), + ] +) +def test_launch_uuid_usage(launch_uuid, method, params): + started_launch_uuid = 'new_test_launch_uuid' + aio_client = mock.AsyncMock() + aio_client.start_launch.return_value = started_launch_uuid + client = ThreadedRPClient('http://endpoint', 'project', api_key='api_key', + client=aio_client, launch_uuid=launch_uuid, log_batch_size=1) + actual_launch_uuid = (client.start_launch('Test Launch', timestamp())).blocking_result() + getattr(client, method)(*params).blocking_result() + finish_launch_message = (client.finish_launch(timestamp())).blocking_result() + + if launch_uuid is None: + aio_client.start_launch.assert_called_once() + assert actual_launch_uuid == started_launch_uuid + assert client.launch_uuid.blocking_result() == started_launch_uuid + aio_client.finish_launch.assert_called_once() + assert finish_launch_message + else: + aio_client.start_launch.assert_not_called() + assert actual_launch_uuid == launch_uuid + assert client.launch_uuid.blocking_result() == launch_uuid + aio_client.finish_launch.assert_not_called() + assert finish_launch_message == '' + assert client.launch_uuid.blocking_result() == actual_launch_uuid + + if method == 'log': + assert len(getattr(aio_client, 'log_batch').call_args_list) == 2 + args, kwargs = getattr(aio_client, 'log_batch').call_args_list[0] + batch = args[0] + assert isinstance(batch, list) + assert len(batch) == 1 + log = batch[0] + assert log.launch_uuid.blocking_result() == actual_launch_uuid + assert log.time == params[0] + assert log.message == params[1] + else: + getattr(aio_client, method).assert_called_once() + args, kwargs = getattr(aio_client, method).call_args_list[0] + assert args[0].blocking_result() == actual_launch_uuid + for i, param in enumerate(params): + assert args[i + 1] == param From 979e7f398ba8dd3e73af627eab550d20f3c2828c Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 12 Oct 2023 16:50:35 +0300 Subject: [PATCH 265/268] Add more tests --- reportportal_client/_internal/logs/batcher.py | 3 +- tests/_internal/logs/test_log_batcher.py | 104 ++++++++++++++++++ .../{_internal => }/logs/test_log_manager.py | 0 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 tests/_internal/logs/test_log_batcher.py rename tests/{_internal => }/logs/test_log_manager.py (100%) diff --git a/reportportal_client/_internal/logs/batcher.py b/reportportal_client/_internal/logs/batcher.py index 1297625f..bffd5668 100644 --- a/reportportal_client/_internal/logs/batcher.py +++ b/reportportal_client/_internal/logs/batcher.py @@ -56,7 +56,7 @@ def _append(self, size: int, log_req: T_co) -> Optional[List[T_co]]: if len(self._batch) > 0: batch = self._batch self._batch = [log_req] - self._payload_size = 0 + self._payload_size = size return batch self._batch.append(log_req) self._payload_size += size @@ -91,6 +91,7 @@ def flush(self) -> Optional[List[T_co]]: if len(self._batch) > 0: batch = self._batch self._batch = [] + self._payload_size = 0 return batch def __getstate__(self) -> Dict[str, Any]: diff --git a/tests/_internal/logs/test_log_batcher.py b/tests/_internal/logs/test_log_batcher.py new file mode 100644 index 00000000..625554d2 --- /dev/null +++ b/tests/_internal/logs/test_log_batcher.py @@ -0,0 +1,104 @@ +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import os + +from reportportal_client import helpers +# noinspection PyProtectedMember +from reportportal_client._internal.logs.batcher import LogBatcher +from reportportal_client.core.rp_file import RPFile +from reportportal_client.core.rp_requests import RPRequestLog +from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE + +TEST_LAUNCH_ID = 'test_launch_uuid' +TEST_ITEM_ID = 'test_item_id' +PROJECT_NAME = 'test_project' +TEST_MASSAGE = 'test_message' +TEST_LEVEL = 'DEBUG' +TEST_BATCH_SIZE = 5 +TEST_ATTACHMENT_NAME = 'test_file.bin' +TEST_ATTACHMENT_TYPE = 'application/zip' + + +def test_log_batch_send_by_length(): + log_batcher = LogBatcher(entry_num=TEST_BATCH_SIZE) + + for i in range(TEST_BATCH_SIZE): + result = log_batcher.append( + RPRequestLog(launch_uuid=TEST_LAUNCH_ID, time=helpers.timestamp(), message=TEST_MASSAGE, + level=TEST_LEVEL, item_uuid=TEST_ITEM_ID)) + if i < TEST_BATCH_SIZE - 1: + assert result is None + assert len(log_batcher._batch) == i + 1 + assert log_batcher._payload_size > 0 + else: + assert len(result) == TEST_BATCH_SIZE + assert len(log_batcher._batch) == 0 + assert log_batcher._payload_size == 0 + + +def test_log_batch_send_by_flush(): + log_batcher = LogBatcher(entry_num=TEST_BATCH_SIZE) + + for _ in range(TEST_BATCH_SIZE - 1): + log_batcher.append( + RPRequestLog(launch_uuid=TEST_LAUNCH_ID, time=helpers.timestamp(), message=TEST_MASSAGE, + level=TEST_LEVEL, item_uuid=TEST_ITEM_ID)) + result = log_batcher.flush() + + assert len(result) == TEST_BATCH_SIZE - 1 + assert len(log_batcher._batch) == 0 + assert log_batcher._payload_size == 0 + + +def test_log_batch_send_by_size(): + log_batcher = LogBatcher(entry_num=TEST_BATCH_SIZE) + + random_byte_array = bytearray(os.urandom(MAX_LOG_BATCH_PAYLOAD_SIZE)) + binary_result = log_batcher.append( + RPRequestLog(launch_uuid=TEST_LAUNCH_ID, time=helpers.timestamp(), message=TEST_MASSAGE, + level=TEST_LEVEL, item_uuid=TEST_ITEM_ID, + file=RPFile(name=TEST_ATTACHMENT_NAME, content=random_byte_array, + content_type=TEST_ATTACHMENT_TYPE))) + message_result = log_batcher.append( + RPRequestLog(launch_uuid=TEST_LAUNCH_ID, time=helpers.timestamp(), message=TEST_MASSAGE, + level=TEST_LEVEL, item_uuid=TEST_ITEM_ID)) + + assert binary_result is None + assert message_result is not None + assert len(message_result) == 1 + assert message_result[0].file is not None + assert log_batcher._payload_size > 0 + assert len(log_batcher._batch) == 1 + + +def test_log_batch_triggers_previous_request_to_send(): + log_batcher = LogBatcher(entry_num=TEST_BATCH_SIZE) + + random_byte_array = bytearray(os.urandom(MAX_LOG_BATCH_PAYLOAD_SIZE)) + + message_result = log_batcher.append( + RPRequestLog(launch_uuid=TEST_LAUNCH_ID, time=helpers.timestamp(), message=TEST_MASSAGE, + level=TEST_LEVEL, item_uuid=TEST_ITEM_ID)) + + binary_result = log_batcher.append( + RPRequestLog(launch_uuid=TEST_LAUNCH_ID, time=helpers.timestamp(), message=TEST_MASSAGE, + level=TEST_LEVEL, item_uuid=TEST_ITEM_ID, + file=RPFile(name=TEST_ATTACHMENT_NAME, content=random_byte_array, + content_type=TEST_ATTACHMENT_TYPE))) + + assert binary_result is not None + assert message_result is None + assert len(binary_result) == 1 + assert binary_result[0].file is None + assert MAX_LOG_BATCH_PAYLOAD_SIZE < log_batcher._payload_size < MAX_LOG_BATCH_PAYLOAD_SIZE * 1.1 diff --git a/tests/_internal/logs/test_log_manager.py b/tests/logs/test_log_manager.py similarity index 100% rename from tests/_internal/logs/test_log_manager.py rename to tests/logs/test_log_manager.py From ad146e632cb051191dfad2ac153e24de91d48884 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 13 Oct 2023 14:13:09 +0300 Subject: [PATCH 266/268] Add factory function --- reportportal_client/__init__.py | 93 ++++++++++++++++++++++++++++++- reportportal_client/aio/client.py | 2 +- reportportal_client/client.py | 2 +- tests/test_client_factory.py | 32 +++++++++++ 4 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 tests/test_client_factory.py diff --git a/reportportal_client/__init__.py b/reportportal_client/__init__.py index e5427ff0..bd2b4f5a 100644 --- a/reportportal_client/__init__.py +++ b/reportportal_client/__init__.py @@ -12,18 +12,109 @@ # limitations under the License """This package is the base package for ReportPortal client.""" +import warnings +from typing import Optional, Any, Union + +import aenum # noinspection PyProtectedMember from reportportal_client._internal.local import current, set_current -from reportportal_client.logs import RPLogger, RPLogHandler +from reportportal_client.aio.client import AsyncRPClient, BatchedRPClient, ThreadedRPClient from reportportal_client.client import RP, RPClient, OutputType +from reportportal_client.logs import RPLogger, RPLogHandler from reportportal_client.steps import step + +class ClientType(aenum.Enum): + """Enum of possible type of ReportPortal clients.""" + + SYNC = aenum.auto() + ASYNC = aenum.auto() + ASYNC_THREAD = aenum.auto() + ASYNC_BATCHED = aenum.auto() + + +# noinspection PyIncorrectDocstring +def create_client( + client_type: ClientType, + endpoint: str, + project: str, + *, + api_key: str = None, + **kwargs: Any +) -> Optional[RP]: + """Create and ReportPortal Client based on the type and arguments provided. + + :param client_type: Type of the Client to create. + :type client_type: ClientType + :param endpoint: Endpoint of the ReportPortal service. + :type endpoint: str + :param project: Project name to report to. + :type project: str + :param api_key: Authorization API key. + :type api_key: str + :param launch_uuid: A launch UUID to use instead of starting own one. + :type launch_uuid: str + :param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the server + side. + :type is_skipped_an_issue: bool + :param verify_ssl: Option to skip ssl verification. + :type verify_ssl: Union[str, bool] + :param retries: Number of retry attempts to make in case of connection / server + errors. + :type retries: int + :param max_pool_size: Option to set the maximum number of connections to save the pool. + :type max_pool_size: int + :param http_timeout : A float in seconds for connect and read timeout. Use a Tuple to + specific connect and read separately. + :type http_timeout: Tuple[float, float] + :param mode: Launch mode, all Launches started by the client will be in that mode. + :type mode: str + :param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout. + :type launch_uuid_print: bool + :param print_output: Set output stream for Launch UUID printing. + :type print_output: OutputType + :param log_batch_size: Option to set the maximum number of logs that can be processed in one + batch. + :type log_batch_size: int + :param log_batch_payload_limit: Maximum size in bytes of logs that can be processed in one batch. + :type log_batch_payload_limit: int + :param keepalive_timeout: For Async Clients only. Maximum amount of idle time in seconds before + force connection closing. + :type keepalive_timeout: int + :param task_timeout: For Async Threaded and Batched Clients only. Time limit in seconds for a + Task processing. + :type task_timeout: float + :param shutdown_timeout: For Async Threaded and Batched Clients only. Time limit in seconds for + shutting down internal Tasks. + :type shutdown_timeout: float + :param trigger_num: For Async Batched Client only. Number of tasks which triggers Task batch + execution. + :param trigger_interval: For Async Batched Client only. Time limit which triggers Task batch + execution. + :return: ReportPortal Client instance. + """ + if client_type is ClientType.SYNC: + return RPClient(endpoint, project, api_key=api_key, **kwargs) + if client_type is ClientType.ASYNC: + return AsyncRPClient(endpoint, project, api_key=api_key, **kwargs) + if client_type is ClientType.ASYNC_THREAD: + return ThreadedRPClient(endpoint, project, api_key=api_key, **kwargs) + if client_type is ClientType.ASYNC_BATCHED: + return BatchedRPClient(endpoint, project, api_key=api_key, **kwargs) + warnings.warn(f'Unknown ReportPortal Client type requested: {client_type}', RuntimeWarning, stacklevel=2) + + __all__ = [ + 'ClientType', + 'create_client', 'current', 'set_current', 'RP', 'RPClient', + 'AsyncRPClient', + 'BatchedRPClient', + 'ThreadedRPClient', 'OutputType', 'RPLogger', 'RPLogHandler', diff --git a/reportportal_client/aio/client.py b/reportportal_client/aio/client.py index eaf7ac42..2a5dca21 100644 --- a/reportportal_client/aio/client.py +++ b/reportportal_client/aio/client.py @@ -25,7 +25,6 @@ import aiohttp import certifi -from reportportal_client import RP, OutputType # noinspection PyProtectedMember from reportportal_client._internal.aio.http import RetryingClientSession # noinspection PyProtectedMember @@ -46,6 +45,7 @@ # noinspection PyProtectedMember from reportportal_client._internal.static.defines import NOT_FOUND, NOT_SET from reportportal_client.aio.tasks import Task +from reportportal_client.client import RP, OutputType from reportportal_client.core.rp_issues import Issue from reportportal_client.core.rp_requests import (LaunchStartRequest, AsyncHttpRequest, AsyncItemStartRequest, AsyncItemFinishRequest, LaunchFinishRequest, RPFile, diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 9a106679..1b5b73a1 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -423,7 +423,7 @@ def __init__( api_key: str = None, log_batch_size: int = 20, is_skipped_an_issue: bool = True, - verify_ssl: bool = True, + verify_ssl: Union[bool, str] = True, retries: int = None, max_pool_size: int = 50, launch_uuid: str = None, diff --git a/tests/test_client_factory.py b/tests/test_client_factory.py new file mode 100644 index 00000000..5b0461bb --- /dev/null +++ b/tests/test_client_factory.py @@ -0,0 +1,32 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# noinspection PyPackageRequirements +import pytest + +from reportportal_client import (create_client, ClientType, RPClient, AsyncRPClient, ThreadedRPClient, + BatchedRPClient) + + +@pytest.mark.parametrize( + 'requested_type, expected_type', + [ + (ClientType.SYNC, RPClient), + (ClientType.ASYNC, AsyncRPClient), + (ClientType.ASYNC_THREAD, ThreadedRPClient), + (ClientType.ASYNC_BATCHED, BatchedRPClient), + ] +) +def test_client_factory_types(requested_type: ClientType, expected_type): + result = create_client(requested_type, 'http://endpoint', 'default_personal') + assert isinstance(result, expected_type) From 5bd01d2576096cfa73a1346d5d56e29837176302 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 13 Oct 2023 14:20:11 +0300 Subject: [PATCH 267/268] Fix flake8 --- reportportal_client/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/reportportal_client/__init__.py b/reportportal_client/__init__.py index bd2b4f5a..13057877 100644 --- a/reportportal_client/__init__.py +++ b/reportportal_client/__init__.py @@ -12,8 +12,8 @@ # limitations under the License """This package is the base package for ReportPortal client.""" +import typing import warnings -from typing import Optional, Any, Union import aenum @@ -41,8 +41,8 @@ def create_client( project: str, *, api_key: str = None, - **kwargs: Any -) -> Optional[RP]: + **kwargs: typing.Any +) -> typing.Optional[RP]: """Create and ReportPortal Client based on the type and arguments provided. :param client_type: Type of the Client to create. @@ -59,7 +59,7 @@ def create_client( side. :type is_skipped_an_issue: bool :param verify_ssl: Option to skip ssl verification. - :type verify_ssl: Union[str, bool] + :type verify_ssl: typing.Union[bool, str] :param retries: Number of retry attempts to make in case of connection / server errors. :type retries: int From 117ca910e83d1a168cbacc6fc9a80bd10e812094 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Fri, 13 Oct 2023 14:22:03 +0300 Subject: [PATCH 268/268] Add missed types --- reportportal_client/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/reportportal_client/__init__.py b/reportportal_client/__init__.py index 13057877..d0bb0e03 100644 --- a/reportportal_client/__init__.py +++ b/reportportal_client/__init__.py @@ -90,8 +90,10 @@ def create_client( :type shutdown_timeout: float :param trigger_num: For Async Batched Client only. Number of tasks which triggers Task batch execution. + :type trigger_num: int :param trigger_interval: For Async Batched Client only. Time limit which triggers Task batch execution. + :type trigger_interval: float :return: ReportPortal Client instance. """ if client_type is ClientType.SYNC: