From a6d15f55c8a796fc0879f27aa0fc71a019494e52 Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Thu, 20 Jul 2023 11:15:20 +0200 Subject: [PATCH 01/25] draft of production runs, production run, quantities and metrics --- src/enlyze/models.py | 102 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/src/enlyze/models.py b/src/enlyze/models.py index 121105c..9628caa 100644 --- a/src/enlyze/models.py +++ b/src/enlyze/models.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import asdict, dataclass from datetime import date, datetime, timezone from enum import Enum from itertools import chain @@ -197,3 +197,103 @@ def to_dataframe(self, use_display_names: bool = False) -> pandas.DataFrame: ) df.index = pandas.to_datetime(df.index, utc=True, format="ISO8601") return df + + +@dataclass(frozen=True) +class Quantities: + """Container of computed or synced quantities produced within a production run.""" + + #: The physical unit of the quantities. + unit: Optional[str] + + #: The total quantity of product produced within the production run. This is the sum of scrap and yield. + total: Optional[float] + + #: The quantity of product produced during the production run that has met the quality criteria. + yield_: Optional[float] + + #: The quantity of scrap produced during the production run. + scrap: Optional[float] + + +@dataclass(frozen=True) +class Metrics: + """Container of computed (productivity) metrics for a production run.""" + + #: Productivity percentage of the production run calculated by the ENLYZE Platform. + productivity_percentage: Optional[float] + + #: The aggregate unproductive time in seconds due to scrap production, downtimes and producing slower than the golden run. + productivity_timeloss: Optional[int] + + #: Availability percentage of the production run calculated by the ENLYZE Platform. + availability_percentage: Optional[float] + + #: The number of seconds lost due to downtimes. + availability_timeloss: Optional[int] + + #: Performance percentage of the production run calculated by the ENLYZE Platform. + performance_percentage: Optional[float] + + #: The number of seconds lost due to lower throughput compared to the golden run. + performance_timeloss: Optional[int] + + #: Quality percentage of the production run calculated by the ENLYZE Platform. + quality_percentage: Optional[float] + + #: The number of seconds lost due to producing scrap. + quality_timeloss: Optional[int] + + +@dataclass(frozen=True) +class ProductionRun: + """Representation of an :ref:`production run ` in the ENLYZE platform. + + Contains details about the production run. + + """ + + #: The UUID of the appliance the production run was exeucted on. + appliance: UUID + + #: The average throughput of the production run excluding downtimes. + average_throughput: Optional[float] + + #: The identifier of the production order. + production_order: str + + #: The identifier of the product that was produced. + product: str + + #: The begin of the production run. + begin: datetime + + #: The end of the production run. + end: Optional[datetime] + + #: Quantities produced during the production run. + quantities: Optional[Quantities] + + #: Productivity metrics of the production run + metrics: Optional[Metrics] + + +class ProductionRuns(list): + """Representation of multiple :ref:`production runs `""" + + def to_dataframe(self) -> pandas.DataFrame: + """Convert production runs into :py:class:`pandas.DataFrame` + + Each row in the data frame will represent one production run. + The ``begin`` and ``end`` of every production run will be + represented as :ref:`timezone-aware ` + :py:class:`datetime.datetime` localized in UTC. + + :returns: DataFrame with production runsthat consists of + + """ + + df = pandas.json_normalize([asdict(run) for run in self]) + df.begin = pandas.to_datetime(df.begin, utc=True, format="ISO8601") + df.end = pandas.to_datetime(df.end, utc=True, format="ISO8601") + return df From 293332adccfe8516d2356c421a6d3223d31ec57c Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Thu, 20 Jul 2023 12:33:43 +0200 Subject: [PATCH 02/25] remove production run from docstrings --- src/enlyze/models.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/enlyze/models.py b/src/enlyze/models.py index 9628caa..8142fcd 100644 --- a/src/enlyze/models.py +++ b/src/enlyze/models.py @@ -201,44 +201,45 @@ def to_dataframe(self, use_display_names: bool = False) -> pandas.DataFrame: @dataclass(frozen=True) class Quantities: - """Container of computed or synced quantities produced within a production run.""" + """Container of computed or synced quantities produced.""" #: The physical unit of the quantities. unit: Optional[str] - #: The total quantity of product produced within the production run. This is the sum of scrap and yield. + #: The total quantity of product produced. This is the sum of scrap and yield. total: Optional[float] - #: The quantity of product produced during the production run that has met the quality criteria. + #: The quantity of product produced that has met the quality criteria. yield_: Optional[float] - #: The quantity of scrap produced during the production run. + #: The quantity of scrap produced. scrap: Optional[float] @dataclass(frozen=True) class Metrics: - """Container of computed (productivity) metrics for a production run.""" + """Container of computed (productivity) metrics.""" - #: Productivity percentage of the production run calculated by the ENLYZE Platform. + #: Productivity percentage calculated by the ENLYZE Platform. productivity_percentage: Optional[float] - #: The aggregate unproductive time in seconds due to scrap production, downtimes and producing slower than the golden run. + #: The aggregate unproductive time in seconds due to scrap production, + # downtimes and producing slower than the golden run. productivity_timeloss: Optional[int] - #: Availability percentage of the production run calculated by the ENLYZE Platform. + #: Availability percentage calculated by the ENLYZE Platform. availability_percentage: Optional[float] #: The number of seconds lost due to downtimes. availability_timeloss: Optional[int] - #: Performance percentage of the production run calculated by the ENLYZE Platform. + #: Performance percentage calculated by the ENLYZE Platform. performance_percentage: Optional[float] #: The number of seconds lost due to lower throughput compared to the golden run. performance_timeloss: Optional[int] - #: Quality percentage of the production run calculated by the ENLYZE Platform. + #: Quality percentage calculated by the ENLYZE Platform. quality_percentage: Optional[float] #: The number of seconds lost due to producing scrap. From cd9db4f3b91fc5f4bd9d009c3723fe54a766bc12 Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Thu, 20 Jul 2023 12:34:02 +0200 Subject: [PATCH 03/25] set default for Metrics and Quantities --- src/enlyze/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/enlyze/models.py b/src/enlyze/models.py index 8142fcd..af7b867 100644 --- a/src/enlyze/models.py +++ b/src/enlyze/models.py @@ -273,10 +273,10 @@ class ProductionRun: end: Optional[datetime] #: Quantities produced during the production run. - quantities: Optional[Quantities] + quantities: Optional[Quantities] = Quantities() #: Productivity metrics of the production run - metrics: Optional[Metrics] + metrics: Optional[Metrics] = Metrics() class ProductionRuns(list): From 88257fd7a2f4a1486ec026ace6dd1bcd674d557e Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Thu, 20 Jul 2023 12:41:32 +0200 Subject: [PATCH 04/25] run pyproject-fmt --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 77410f3..196db6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,11 @@ readme = "README.rst" license = {text = "MIT"} authors = [{name = "ENLYZE GmbH", email = "hello@enlyze.com"},] requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] dynamic = [ 'version', ] From af984abac72539083490e9897f5d43c78a9499a9 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Thu, 20 Jul 2023 15:26:00 +0200 Subject: [PATCH 05/25] Align modelling of production runs after remodelling of API (#9) --- docs/models.rst | 12 +++++++ src/enlyze/models.py | 81 +++++++++++++++++++++----------------------- 2 files changed, 51 insertions(+), 42 deletions(-) diff --git a/docs/models.rst b/docs/models.rst index 07d0fdf..cfb091e 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -24,3 +24,15 @@ Data models .. autoclass:: ResamplingMethod() :members: :undoc-members: + +.. autoclass:: ProductionRun() + :members: + :undoc-members: + +.. autoclass:: OEEScore() + :members: + :undoc-members: + +.. autoclass:: Quantity() + :members: + :undoc-members: diff --git a/src/enlyze/models.py b/src/enlyze/models.py index af7b867..4937edd 100644 --- a/src/enlyze/models.py +++ b/src/enlyze/models.py @@ -1,5 +1,5 @@ from dataclasses import asdict, dataclass -from datetime import date, datetime, timezone +from datetime import date, datetime, timedelta, timezone from enum import Enum from itertools import chain from typing import Any, Iterator, Optional, Sequence @@ -200,55 +200,37 @@ def to_dataframe(self, use_display_names: bool = False) -> pandas.DataFrame: @dataclass(frozen=True) -class Quantities: - """Container of computed or synced quantities produced.""" +class OEEScore: + """Individual Overall Equipment Effectiveness (OEE) score - #: The physical unit of the quantities. - unit: Optional[str] + This is calculated by the ENLYZE Platform based on a combination of real + machine data and production order booking information provided by the + customer. - #: The total quantity of product produced. This is the sum of scrap and yield. - total: Optional[float] + For more information, please check out https://www.oee.com + """ - #: The quantity of product produced that has met the quality criteria. - yield_: Optional[float] + #: The score is expressed as a ratio between 0 and 1.0, with 1.0 meaning 100 %. + score: float - #: The quantity of scrap produced. - scrap: Optional[float] + #: Unproductive time due to non-ideal production. + time_loss: timedelta @dataclass(frozen=True) -class Metrics: - """Container of computed (productivity) metrics.""" - - #: Productivity percentage calculated by the ENLYZE Platform. - productivity_percentage: Optional[float] - - #: The aggregate unproductive time in seconds due to scrap production, - # downtimes and producing slower than the golden run. - productivity_timeloss: Optional[int] - - #: Availability percentage calculated by the ENLYZE Platform. - availability_percentage: Optional[float] - - #: The number of seconds lost due to downtimes. - availability_timeloss: Optional[int] - - #: Performance percentage calculated by the ENLYZE Platform. - performance_percentage: Optional[float] +class Quantity: + """Representation of a physical quantity""" - #: The number of seconds lost due to lower throughput compared to the golden run. - performance_timeloss: Optional[int] + #: Physical unit of quantity + unit: str - #: Quality percentage calculated by the ENLYZE Platform. - quality_percentage: Optional[float] - - #: The number of seconds lost due to producing scrap. - quality_timeloss: Optional[int] + #: The quantity expressed in `unit` + value: float @dataclass(frozen=True) class ProductionRun: - """Representation of an :ref:`production run ` in the ENLYZE platform. + """Representation of a production run in the ENLYZE platform. Contains details about the production run. @@ -272,14 +254,29 @@ class ProductionRun: #: The end of the production run. end: Optional[datetime] - #: Quantities produced during the production run. - quantities: Optional[Quantities] = Quantities() + #: This is the sum of scrap and yield. + quantity_total: Optional[Quantity] + + #: The amount of product produced that doesn't meet quality criteria. + quantity_scrap: Optional[Quantity] + + #: The amount of product produced that can be sold. + quantity_yield: Optional[Quantity] + + #: OEE component that reflects when the appliance did not produce. + availability: Optional[OEEScore] + + #: OEE component that reflects how fast the appliance has run. + performance: Optional[OEEScore] + + #: OEE component that reflects how much defects have been produced. + quality: Optional[OEEScore] - #: Productivity metrics of the production run - metrics: Optional[Metrics] = Metrics() + #: Aggregate OEE score that comprises availability, performance and quality. + productivity: Optional[OEEScore] -class ProductionRuns(list): +class ProductionRuns(list[ProductionRun]): """Representation of multiple :ref:`production runs `""" def to_dataframe(self) -> pandas.DataFrame: From 4a10d7b34c9e6a66c60ee790cb8dadca74b5b4fe Mon Sep 17 00:00:00 2001 From: Daniel Roussel Date: Thu, 20 Jul 2023 15:38:32 +0200 Subject: [PATCH 06/25] docs: add concepts for production order, production run and product --- docs/concepts.rst | 50 ++++++++++++++++++++++++++++++++++++++++++++ src/enlyze/models.py | 7 ++++--- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 7981aa8..622b052 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -32,3 +32,53 @@ Variable A *variable* represents a process measure of one :ref:`appliance ` of which timeseries data is captured and stored in the ENLYZE platform. One appliance may have many variables, whereas one variable is only associated with one appliance. + + +.. _production_order: + +Production Order +---------------- + +A *production order* represents the goal of producing a certain quantity of a +given :ref:`product `. Production orders are usually created in an ERP +system or MES of the customer and then synchronized into the ENLYZE platform. +They are referenced by an identifier which oftentimes is a short combination of +numbers and/or characters, like FA23000123 or 332554. + +In the ENLYZE platform, a production order always encompasses the production of +one single :ref:`product ` on one single :ref:`appliance `. + + +.. _production_run: + +Production Run +-------------- + +A *production run* represents the real allocation of a :ref:`production order +` on an :ref:`appliance `. Usually, the operator of +the appliance uses some kind of interface to log the time when a certain +production order has been worked on. A production order oftentimes is not +allocated continously on an appliance due to interruptions like a breakdown or +weekend. Each of these individual, continous allocations is called a *production +run* in the ENLYZE platform. It always has a beginning and, if it's not still +running, it also has an end. The ENLYZE platform calculates different aggregates +from the timeseries data of the appliance for each production run. + + +.. _product: + +Product +------- + +A *product* is the output of the production process which is executed by an +:ref:`appliance `, driven by a :ref:`production order +`. In the real world, an appliance might have some additonal +outputs, but only the main output (the product) modelled in the ENLYZE platform. +Similarly to the production order, a product is referenced by an identifier +originating from a system of the customer, which is synchronized into the ENLYZE +platform. + +During the integration into the ENLYZE platform, the product identifier is +chosen in such a way that :ref:`production runs ` of the same +product are comparable with one another. This is the foundation for constantly +improving the production of recurring products over time. diff --git a/src/enlyze/models.py b/src/enlyze/models.py index 4937edd..66f6bae 100644 --- a/src/enlyze/models.py +++ b/src/enlyze/models.py @@ -230,7 +230,8 @@ class Quantity: @dataclass(frozen=True) class ProductionRun: - """Representation of a production run in the ENLYZE platform. + """Representation of a :ref:`production run ` in the ENLYZE + platform. Contains details about the production run. @@ -277,7 +278,7 @@ class ProductionRun: class ProductionRuns(list[ProductionRun]): - """Representation of multiple :ref:`production runs `""" + """Representation of multiple :ref:`production runs `.""" def to_dataframe(self) -> pandas.DataFrame: """Convert production runs into :py:class:`pandas.DataFrame` @@ -287,7 +288,7 @@ def to_dataframe(self) -> pandas.DataFrame: represented as :ref:`timezone-aware ` :py:class:`datetime.datetime` localized in UTC. - :returns: DataFrame with production runsthat consists of + :returns: DataFrame with production runs """ From 21c9cf78d3338ceefb01f1d6a482a30b75fb4970 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Fri, 21 Jul 2023 08:50:38 +0200 Subject: [PATCH 07/25] Update docs/concepts.rst Co-authored-by: Deniz Saner --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 622b052..236bbbe 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -75,7 +75,7 @@ A *product* is the output of the production process which is executed by an `. In the real world, an appliance might have some additonal outputs, but only the main output (the product) modelled in the ENLYZE platform. Similarly to the production order, a product is referenced by an identifier -originating from a system of the customer, which is synchronized into the ENLYZE +originating from a customer system, which is synchronized into the ENLYZE platform. During the integration into the ENLYZE platform, the product identifier is From 9cba52cddb5095815bade6ec42ed2d7583fe8d42 Mon Sep 17 00:00:00 2001 From: Daniel Krebs Date: Fri, 21 Jul 2023 08:50:50 +0200 Subject: [PATCH 08/25] Update docs/concepts.rst --- docs/concepts.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 236bbbe..58932cb 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -40,8 +40,8 @@ Production Order ---------------- A *production order* represents the goal of producing a certain quantity of a -given :ref:`product `. Production orders are usually created in an ERP -system or MES of the customer and then synchronized into the ENLYZE platform. +given :ref:`product `. Production orders are usually created in an external ERP +system or MES and then synchronized into the ENLYZE platform. They are referenced by an identifier which oftentimes is a short combination of numbers and/or characters, like FA23000123 or 332554. From 9dbcdb299da3357646a3230443f06b8f720bc772 Mon Sep 17 00:00:00 2001 From: Daniel Roussel Date: Fri, 21 Jul 2023 09:10:41 +0200 Subject: [PATCH 09/25] add "This is how operators are told how much of which product to produce." --- docs/concepts.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 58932cb..7a17a12 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -40,10 +40,11 @@ Production Order ---------------- A *production order* represents the goal of producing a certain quantity of a -given :ref:`product `. Production orders are usually created in an external ERP -system or MES and then synchronized into the ENLYZE platform. -They are referenced by an identifier which oftentimes is a short combination of -numbers and/or characters, like FA23000123 or 332554. +given :ref:`product `. This is how operators are told how much of which +product to produce. Production orders are usually created in an external ERP +system or MES and then synchronized into the ENLYZE platform. They are +referenced by an identifier which oftentimes is a short combination of numbers +and/or characters, like FA23000123 or 332554. In the ENLYZE platform, a production order always encompasses the production of one single :ref:`product ` on one single :ref:`appliance `. From 2c8948bcfbff9ddcc29d47ec0435abf5e4afc30d Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Tue, 10 Oct 2023 16:05:29 +0200 Subject: [PATCH 10/25] Update docs/concepts.rst Co-authored-by: Daniel Krebs --- docs/concepts.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 7a17a12..a81b3ea 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -57,8 +57,8 @@ Production Run A *production run* represents the real allocation of a :ref:`production order ` on an :ref:`appliance `. Usually, the operator of -the appliance uses some kind of interface to log the time when a certain -production order has been worked on. A production order oftentimes is not +the appliance uses an interface to log the time when a certain +production order has been worked on. For instance, this could be the appliance's HMI or a tablet computer next to it. In German, this is often referred to as "Betriebsdatenerfassung (BDE)". A production order oftentimes is not allocated continously on an appliance due to interruptions like a breakdown or weekend. Each of these individual, continous allocations is called a *production run* in the ENLYZE platform. It always has a beginning and, if it's not still From 2ac763f9c4066afa906f98f13ca225cb20e9233e Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Fri, 22 Sep 2023 11:16:09 +0200 Subject: [PATCH 11/25] add missing "next" field to response mocks --- tests/enlyze/timeseries_api/test_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/enlyze/timeseries_api/test_client.py b/tests/enlyze/timeseries_api/test_client.py index 4805d93..9026218 100644 --- a/tests/enlyze/timeseries_api/test_client.py +++ b/tests/enlyze/timeseries_api/test_client.py @@ -101,26 +101,26 @@ def test_timeseries_api_get_paginated_raises_invalid_pagination_schema( @respx.mock def test_timeseries_api_get_paginated_raises_invalid_data_schema(client): - respx.get("").respond(json={"data": [42, 1337]}) + respx.get("").respond(json={"data": [42, 1337], "next": None}) with pytest.raises(EnlyzeError, match="API returned an unparsable"): list(client.get_paginated("", TimeseriesApiModel)) @respx.mock def test_timeseries_api_get_paginated_data_empty(client, string_model): - respx.get("").respond(json={"data": []}) + respx.get("").respond(json={"data": [], "next": None}) assert list(client.get_paginated("", string_model)) == [] @respx.mock def test_timeseries_api_get_paginated_single_page(client, string_model): - respx.get("").respond(json={"data": ["a", "b"]}) + respx.get("").respond(json={"data": ["a", "b"], "next": None}) assert list(client.get_paginated("", string_model)) == ["a", "b"] @respx.mock def test_timeseries_api_get_paginated_multi_page(client, string_model): - respx.get("", params="offset=1").respond(json={"data": ["z"]}) + respx.get("", params="offset=1").respond(json={"data": ["z"], "next": None}) respx.get("").mock( side_effect=lambda request: httpx.Response( 200, From 3915e4fe4c05310f8e58b81a1d6a47c8983b559e Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Fri, 22 Sep 2023 20:47:38 +0200 Subject: [PATCH 12/25] exclude pydantic members from docs this was causing spell and link checks to fail --- docs/timeseries_api/models.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/timeseries_api/models.rst b/docs/timeseries_api/models.rst index aa5025b..c897d38 100644 --- a/docs/timeseries_api/models.rst +++ b/docs/timeseries_api/models.rst @@ -8,19 +8,23 @@ Models .. autoclass:: Site() :members: :undoc-members: + :exclude-members: model_config, model_fields :show-inheritance: .. autoclass:: Appliance() :members: :undoc-members: + :exclude-members: model_config, model_fields :show-inheritance: .. autoclass:: Variable() :members: :undoc-members: + :exclude-members: model_config, model_fields :show-inheritance: .. autoclass:: TimeseriesData() :members: :undoc-members: + :exclude-members: model_config, model_fields :show-inheritance: From 42368c329bce8e5ff5df09b6319f8054dd700ce2 Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Fri, 29 Sep 2023 14:28:53 +0200 Subject: [PATCH 13/25] Implement `ApiBaseClient` and derive `TimeseriesApiClient` from it (#14) :point_up: This PR is the first in a row in an effort to expose production runs via the ENLYZE SDK. At the moment, we only expose time series data via the SDK and interaction with the timeseries service is contained in the `TimeseriesApiClient`. However, the new production runs API introduces a different pagination pattern, which makes it inconvenient to generalize the `TimeseriesApiClient` to a `EnlyzePlatformApiClient` :tm:. Thus, this and subsequent PRs are building on top of the assumption that we will have separate API Clients for the timeseries and production runs API. This first PR introduces an `ApiBaseClient` class, which implement generalized `get` and `get_paginated` methods. The latter calls two abstract methods, `_has_more` and `_next_page_call_args` that must be implemented by API clients deriving from this class. --------- Co-authored-by: Daniel Krebs --- docs/api_clients/base.rst | 17 ++ docs/api_clients/index.rst | 8 + docs/api_clients/timeseries/client.rst | 11 + .../timeseries}/index.rst | 0 .../timeseries}/models.rst | 2 +- docs/index.rst | 2 +- docs/spelling_wordlist.txt | 2 + docs/timeseries_api/client.rst | 7 - .../__init__.py | 0 src/enlyze/api_clients/base.py | 220 ++++++++++++++ .../api_clients/timeseries}/__init__.py | 0 src/enlyze/api_clients/timeseries/client.py | 61 ++++ .../timeseries}/models.py | 5 +- src/enlyze/client.py | 4 +- src/enlyze/timeseries_api/client.py | 146 --------- tests/conftest.py | 18 ++ tests/enlyze/api_clients/__init__.py | 0 tests/enlyze/api_clients/test_base.py | 282 ++++++++++++++++++ .../enlyze/api_clients/timeseries/__init__.py | 0 .../api_clients/timeseries/test_client.py | 114 +++++++ tests/enlyze/test_client.py | 4 +- tests/enlyze/timeseries_api/test_client.py | 131 -------- 22 files changed, 741 insertions(+), 293 deletions(-) create mode 100644 docs/api_clients/base.rst create mode 100644 docs/api_clients/index.rst create mode 100644 docs/api_clients/timeseries/client.rst rename docs/{timeseries_api => api_clients/timeseries}/index.rst (100%) rename docs/{timeseries_api => api_clients/timeseries}/models.rst (91%) delete mode 100644 docs/timeseries_api/client.rst rename src/enlyze/{timeseries_api => api_clients}/__init__.py (100%) create mode 100644 src/enlyze/api_clients/base.py rename {tests/enlyze/timeseries_api => src/enlyze/api_clients/timeseries}/__init__.py (100%) create mode 100644 src/enlyze/api_clients/timeseries/client.py rename src/enlyze/{timeseries_api => api_clients/timeseries}/models.py (96%) delete mode 100644 src/enlyze/timeseries_api/client.py create mode 100644 tests/enlyze/api_clients/__init__.py create mode 100644 tests/enlyze/api_clients/test_base.py create mode 100644 tests/enlyze/api_clients/timeseries/__init__.py create mode 100644 tests/enlyze/api_clients/timeseries/test_client.py delete mode 100644 tests/enlyze/timeseries_api/test_client.py diff --git a/docs/api_clients/base.rst b/docs/api_clients/base.rst new file mode 100644 index 0000000..1f7ef06 --- /dev/null +++ b/docs/api_clients/base.rst @@ -0,0 +1,17 @@ +Base Client +=========== + +.. currentmodule:: enlyze.api_clients.base + +.. autoclass:: M + +.. autoclass:: R + +.. autoclass:: ApiBaseModel + +.. autoclass:: PaginatedResponseBaseModel + +.. autoclass:: ApiBaseClient + :members: + :private-members: + :undoc-members: diff --git a/docs/api_clients/index.rst b/docs/api_clients/index.rst new file mode 100644 index 0000000..ddd6e48 --- /dev/null +++ b/docs/api_clients/index.rst @@ -0,0 +1,8 @@ +API Clients +=========== + +.. toctree:: + :maxdepth: 1 + + base + timeseries/index diff --git a/docs/api_clients/timeseries/client.rst b/docs/api_clients/timeseries/client.rst new file mode 100644 index 0000000..fb84a40 --- /dev/null +++ b/docs/api_clients/timeseries/client.rst @@ -0,0 +1,11 @@ +Timeseries API Client +===================== + +.. currentmodule:: enlyze.api_clients.timeseries.client + +.. autoclass:: _PaginatedResponse() + :members: + :exclude-members: model_config, model_fields + +.. autoclass:: TimeseriesApiClient() + :members: get, get_paginated diff --git a/docs/timeseries_api/index.rst b/docs/api_clients/timeseries/index.rst similarity index 100% rename from docs/timeseries_api/index.rst rename to docs/api_clients/timeseries/index.rst diff --git a/docs/timeseries_api/models.rst b/docs/api_clients/timeseries/models.rst similarity index 91% rename from docs/timeseries_api/models.rst rename to docs/api_clients/timeseries/models.rst index c897d38..19d57ee 100644 --- a/docs/timeseries_api/models.rst +++ b/docs/api_clients/timeseries/models.rst @@ -1,7 +1,7 @@ Models ====== -.. currentmodule:: enlyze.timeseries_api.models +.. currentmodule:: enlyze.api_clients.timeseries.models .. autoclass:: TimeseriesApiModel() diff --git a/docs/index.rst b/docs/index.rst index 75e100f..08dccf7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,4 +19,4 @@ User's Guide models errors constants - timeseries_api/index + api_clients/index diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index c9ed809..6f65e78 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -6,6 +6,7 @@ UUID VariableDataType enlyze entrypoint +iterable noqa quickstart resample @@ -13,6 +14,7 @@ resampled resampling str timeseries +url uuid virtualenv whitespace diff --git a/docs/timeseries_api/client.rst b/docs/timeseries_api/client.rst deleted file mode 100644 index 7a9f6a9..0000000 --- a/docs/timeseries_api/client.rst +++ /dev/null @@ -1,7 +0,0 @@ -Client -====== - -.. currentmodule:: enlyze.timeseries_api.client - -.. autoclass:: TimeseriesApiClient() - :members: diff --git a/src/enlyze/timeseries_api/__init__.py b/src/enlyze/api_clients/__init__.py similarity index 100% rename from src/enlyze/timeseries_api/__init__.py rename to src/enlyze/api_clients/__init__.py diff --git a/src/enlyze/api_clients/base.py b/src/enlyze/api_clients/base.py new file mode 100644 index 0000000..2fffff5 --- /dev/null +++ b/src/enlyze/api_clients/base.py @@ -0,0 +1,220 @@ +import json +from abc import ABC, abstractmethod +from collections.abc import Iterator +from functools import cache +from http import HTTPStatus +from typing import Any, Generic, TypeVar + +import httpx +from pydantic import BaseModel, ValidationError + +from enlyze.auth import TokenAuth +from enlyze.constants import ENLYZE_BASE_URL, HTTPX_TIMEOUT +from enlyze.errors import EnlyzeError, InvalidTokenError + + +class ApiBaseModel(BaseModel): + """Base class for ENLYZE platform API object models using pydantic + + All objects received from ENLYZE platform APIs are passed into models that derive + from this class and thus use pydantic for schema definition and validation. + + """ + + +class PaginatedResponseBaseModel(BaseModel): + """Base class for paginated ENLYZE platform API responses using pydantic.""" + + data: Any + + +#: TypeVar("M", bound=ApiBaseModel): Type variable serving as a parameter +# for API response model classes. +M = TypeVar("M", bound=ApiBaseModel) + + +#: TypeVar("R", bound=PaginatedResponseBaseModel) Type variable serving as a parameter +# for paginated response models. +R = TypeVar("R", bound=PaginatedResponseBaseModel) + + +class ApiBaseClient(ABC, Generic[R]): + """Client base class encapsulating all interaction with all ENLYZE platform APIs. + + :param token: API token for the ENLYZE platform + :param base_url: Base URL of the ENLYZE platform + :param timeout: Global timeout for HTTP requests sent to the ENLYZE platform APIs + + """ + + PaginatedResponseModel: type[R] + + def __init__( + self, + token: str, + *, + base_url: str | httpx.URL = ENLYZE_BASE_URL, + timeout: float = HTTPX_TIMEOUT, + ): + self._client = httpx.Client( + auth=TokenAuth(token), + base_url=httpx.URL(base_url), + timeout=timeout, + ) + + @cache + def _full_url(self, api_path: str) -> str: + """Construct full URL from relative URL""" + return str(self._client.build_request("", api_path).url) + + def get(self, api_path: str, **kwargs: Any) -> Any: + """Wraps :meth:`httpx.Client.get` with defensive error handling + + :param api_path: Relative URL path inside the API name space (or a full URL) + + :raises: :exc:`~enlyze.errors.EnlyzeError` on request failure + + :raises: :exc:`~enlyze.errors.EnlyzeError` on non-2xx status code + + :raises: :exc:`~enlyze.errors.EnlyzeError` on non-JSON payload + + :returns: JSON payload of the response as Python object + + """ + + try: + response = self._client.get(api_path, **kwargs) + except Exception as e: + raise EnlyzeError( + "Couldn't read from the ENLYZE platform API " + f"(GET {self._full_url(api_path)})", + ) from e + + try: + response.raise_for_status() + except httpx.HTTPStatusError as e: + if e.response.status_code in ( + HTTPStatus.UNAUTHORIZED, + HTTPStatus.FORBIDDEN, + ): + raise InvalidTokenError + else: + raise EnlyzeError( + f"ENLYZE platform API returned error {response.status_code}" + f" (GET {self._full_url(api_path)})" + ) from e + + try: + return response.json() + except json.JSONDecodeError as e: + raise EnlyzeError( + "ENLYZE platform API didn't return a valid JSON object " + f"(GET {self._full_url(api_path)})", + ) from e + + def _transform_paginated_response_data(self, data: Any) -> Any: + """Transform paginated response data. Returns ``data`` by default. + + :param data: Response data from a paginated response + + :returns: An iterable of transformed data + + """ + return data + + @abstractmethod + def _has_more(self, paginated_response: R) -> bool: + """Indicates there is more data to fetch from the server. + + :param paginated_response: A paginated response model deriving from + :class:`PaginatedResponseBaseModel`. + + """ + + @abstractmethod + def _next_page_call_args( + self, + *, + url: str, + params: dict[str, Any], + paginated_response: R, + **kwargs: Any, + ) -> tuple[str, dict[str, Any], dict[str, Any]]: + r"""Compute call arguments for the next page. + + :param url: The URL used to fetch the current page + :param params: URL query parameters of the current page + :param paginated_response: A paginated response model deriving from + :class:`~enlyze.api_clients.base.PaginatedResponseBaseModel` + :param \**kwargs: Keyword arguments passed into + :py:meth:`~enlyze.api_clients.base.ApiBaseClient.get_paginated` + + :returns: A tuple of comprised of the URL, query parameters and keyword + arguments to fetch the next page + + """ + + def get_paginated( + self, api_path: str, model: type[M], **kwargs: Any + ) -> Iterator[M]: + """Retrieve objects from a paginated ENLYZE platform API endpoint via HTTP GET. + + To add pagination capabilities to an API client deriving from this class, two + abstract methods need to be implemented, + :py:meth:`~enlyze.api_clients.base.ApiBaseClient._has_more` and + :py:meth:`~enlyze.api_clients.base.ApiBaseClient._next_page_call_args`. + Optionally, API clients may transform page data by overriding + :py:meth:`~enlyze.api_clients.base.ApiBaseClient._transform_paginated_response_data`, + which by default returns the unmodified page data. + + :param api_path: Relative URL path inside the API name space + :param model: API response model class deriving from + :class:`~enlyze.api_clients.base.ApiBaseModel` + + :raises: :exc:`~enlyze.errors.EnlyzeError` on invalid pagination schema + + :raises: :exc:`~enlyze.errors.EnlyzeError` on invalid data schema + + :raises: see :py:meth:`get` for more errors raised by this method + + :returns: Instances of ``model`` retrieved from the ``api_path`` endpoint + + """ + + url = api_path + params = kwargs.pop("params", {}) + + while True: + response_body = self.get(url, params=params, **kwargs) + try: + paginated_response = self.PaginatedResponseModel.parse_obj( + response_body + ) + except ValidationError as e: + raise EnlyzeError( + f"Paginated response expected (GET {self._full_url(url)})" + ) from e + + page_data = paginated_response.data + if not page_data: + break + + page_data = self._transform_paginated_response_data(page_data) + + for elem in page_data: + try: + yield model.parse_obj(elem) + except ValidationError as e: + raise EnlyzeError( + f"ENLYZE platform API returned an unparsable {model.__name__} " + f"object (GET {self._full_url(api_path)})" + ) from e + if not self._has_more(paginated_response): + break + + url, params, kwargs = self._next_page_call_args( + url=url, + params=params, + paginated_response=paginated_response, + **kwargs, + ) diff --git a/tests/enlyze/timeseries_api/__init__.py b/src/enlyze/api_clients/timeseries/__init__.py similarity index 100% rename from tests/enlyze/timeseries_api/__init__.py rename to src/enlyze/api_clients/timeseries/__init__.py diff --git a/src/enlyze/api_clients/timeseries/client.py b/src/enlyze/api_clients/timeseries/client.py new file mode 100644 index 0000000..a4824a4 --- /dev/null +++ b/src/enlyze/api_clients/timeseries/client.py @@ -0,0 +1,61 @@ +from typing import Any, Tuple + +import httpx +from pydantic import AnyUrl + +from enlyze.api_clients.base import ApiBaseClient, PaginatedResponseBaseModel +from enlyze.constants import ENLYZE_BASE_URL, TIMESERIES_API_SUB_PATH + + +class _PaginatedResponse(PaginatedResponseBaseModel): + next: AnyUrl | None + data: list[Any] | dict[str, Any] + + +class TimeseriesApiClient(ApiBaseClient[_PaginatedResponse]): + """Client class encapsulating all interaction with the Timeseries API + + :param token: API token for the ENLYZE platform + :param base_url: Base URL of the ENLYZE platform + :param timeout: Global timeout for all HTTP requests sent to the Timeseries API + + """ + + PaginatedResponseModel = _PaginatedResponse + + def __init__( + self, + token: str, + *, + base_url: str | httpx.URL = ENLYZE_BASE_URL, + **kwargs: Any, + ): + super().__init__( + token, base_url=httpx.URL(base_url).join(TIMESERIES_API_SUB_PATH), **kwargs + ) + + def _transform_paginated_response_data( + self, paginated_response_data: list[Any] | dict[str, Any] + ) -> list[dict[str, Any]]: + # The timeseries endpoint's response data field is a mapping. + # Because get_paginated assumes the ``data`` field to be a list, + # we wrap it into a list. + return ( + paginated_response_data + if isinstance(paginated_response_data, list) + else [paginated_response_data] + ) + + def _has_more(self, paginated_response: _PaginatedResponse) -> bool: + return paginated_response.next is not None + + def _next_page_call_args( + self, + *, + url: str, + params: dict[str, Any], + paginated_response: _PaginatedResponse, + **kwargs: Any, + ) -> Tuple[str, dict[str, Any], dict[str, Any]]: + next_url = str(paginated_response.next) + return (next_url, params, kwargs) diff --git a/src/enlyze/timeseries_api/models.py b/src/enlyze/api_clients/timeseries/models.py similarity index 96% rename from src/enlyze/timeseries_api/models.py rename to src/enlyze/api_clients/timeseries/models.py index df922f7..5f84820 100644 --- a/src/enlyze/timeseries_api/models.py +++ b/src/enlyze/api_clients/timeseries/models.py @@ -2,12 +2,11 @@ from typing import Any, Optional, Sequence from uuid import UUID -from pydantic import BaseModel - import enlyze.models as user_models +from enlyze.api_clients.base import ApiBaseModel -class TimeseriesApiModel(BaseModel): +class TimeseriesApiModel(ApiBaseModel): """Base class for Timeseries API object models using pydantic All objects received from the Timeseries API are passed into models that derive from diff --git a/src/enlyze/client.py b/src/enlyze/client.py index 59d5842..4d866ad 100644 --- a/src/enlyze/client.py +++ b/src/enlyze/client.py @@ -3,11 +3,11 @@ from typing import Iterator, Mapping, Optional, Sequence from uuid import UUID +import enlyze.api_clients.timeseries.models as api_models import enlyze.models as user_models -import enlyze.timeseries_api.models as api_models +from enlyze.api_clients.timeseries.client import TimeseriesApiClient from enlyze.constants import VARIABLE_UUID_AND_RESAMPLING_METHOD_SEPARATOR from enlyze.errors import EnlyzeError -from enlyze.timeseries_api.client import TimeseriesApiClient from enlyze.validators import ( validate_resampling_interval, validate_resampling_method_for_data_type, diff --git a/src/enlyze/timeseries_api/client.py b/src/enlyze/timeseries_api/client.py deleted file mode 100644 index ec54fe9..0000000 --- a/src/enlyze/timeseries_api/client.py +++ /dev/null @@ -1,146 +0,0 @@ -import json -from collections.abc import Iterator -from functools import cache -from http import HTTPStatus -from typing import Any, Type, TypeVar - -import httpx -from pydantic import AnyUrl, BaseModel, ValidationError - -from enlyze.auth import TokenAuth -from enlyze.constants import ENLYZE_BASE_URL, HTTPX_TIMEOUT, TIMESERIES_API_SUB_PATH -from enlyze.errors import EnlyzeError, InvalidTokenError -from enlyze.timeseries_api.models import TimeseriesApiModel - -T = TypeVar("T", bound=TimeseriesApiModel) - - -class _PaginatedResponse(BaseModel): - next: AnyUrl | None - data: list[Any] | dict[str, Any] - - -class TimeseriesApiClient: - """Client class encapsulating all interaction with the Timeseries API - - :param token: API token for the ENLYZE platform - :param base_url: Base URL of the ENLYZE platform - :param timeout: Global timeout for all HTTP requests sent to the Timeseries API - - """ - - def __init__( - self, - token: str, - *, - base_url: str | httpx.URL = ENLYZE_BASE_URL, - timeout: float = HTTPX_TIMEOUT, - ): - self._client = httpx.Client( - auth=TokenAuth(token), - base_url=httpx.URL(base_url).join(TIMESERIES_API_SUB_PATH), - timeout=timeout, - ) - - @cache - def _full_url(self, api_path: str) -> str: - """Construct full URL from relative URL""" - return str(self._client.build_request("", api_path).url) - - def get(self, api_path: str, **kwargs: Any) -> Any: - """Wraps :meth:`httpx.Client.get` with defensive error handling - - :param api_path: Relative URL path inside the Timeseries API (or a full URL) - - :raises: :exc:`~enlyze.errors.EnlyzeError` on request failure - - :raises: :exc:`~enlyze.errors.EnlyzeError` on non-2xx status code - - :raises: :exc:`~enlyze.errors.EnlyzeError` on non-JSON payload - - :returns: JSON payload of the response as Python object - - """ - - try: - response = self._client.get(api_path, **kwargs) - except Exception as e: - raise EnlyzeError( - "Couldn't read from the Timeseries API " - f"(GET {self._full_url(api_path)})", - ) from e - - try: - response.raise_for_status() - except httpx.HTTPStatusError as e: - if e.response.status_code in ( - HTTPStatus.UNAUTHORIZED, - HTTPStatus.FORBIDDEN, - ): - raise InvalidTokenError - else: - raise EnlyzeError( - f"Timeseries API returned error {response.status_code}" - f" (GET {self._full_url(api_path)})" - ) from e - - try: - return response.json() - except json.JSONDecodeError as e: - raise EnlyzeError( - "Timeseries API didn't return a valid JSON object " - f"(GET {self._full_url(api_path)})", - ) from e - - def get_paginated( - self, api_path: str, model: Type[T], **kwargs: Any - ) -> Iterator[T]: - """Retrieve objects from paginated Timeseries API endpoint via HTTP GET - - :param api_path: Relative URL path inside the Timeseries API - :param model: Class derived from - :class:`~enlyze.timeseries_api.models.TimeseriesApiModel` - - :raises: :exc:`~enlyze.errors.EnlyzeError` on invalid pagination schema - - :raises: :exc:`~enlyze.errors.EnlyzeError` on invalid data schema - - :raises: see :py:meth:`get` for more errors raised by this method - - :returns: Instances of ``model`` retrieved from the ``api_path`` endpoint - - """ - - url = api_path - - while True: - response_body = self.get(url, **kwargs) - - try: - paginated_response = _PaginatedResponse.parse_obj(response_body) - except ValidationError as e: - raise EnlyzeError( - f"Paginated response expected (GET {self._full_url(url)})" - ) from e - - page_data = paginated_response.data - if not page_data: - break - - # if `data` is a list we assume there are multiple objects inside. - # if `data` is a dict then we treat it as only one object - page_data = page_data if isinstance(page_data, list) else [page_data] - - for elem in page_data: - try: - yield model.parse_obj(elem) - except ValidationError as e: - raise EnlyzeError( - f"Timeseries API returned an unparsable {model.__name__} " - f"object (GET {self._full_url(api_path)})" - ) from e - - if not paginated_response.next: - break - - url = str(paginated_response.next) diff --git a/tests/conftest.py b/tests/conftest.py index 69f117a..1b54475 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,13 @@ import os from datetime import datetime, timezone +from unittest.mock import patch import hypothesis +import pytest from hypothesis import strategies as st +from enlyze.api_clients.base import ApiBaseModel + hypothesis.settings.register_profile("ci", deadline=None) hypothesis.settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default")) @@ -17,3 +21,17 @@ max_value=datetime.utcnow().replace(hour=0), timezones=st.just(timezone.utc), ) + + +@pytest.fixture +def auth_token(): + return "some-token" + + +@pytest.fixture +def string_model(): + with patch( + "enlyze.api_clients.base.ApiBaseModel.parse_obj", + side_effect=lambda o: str(o), + ): + yield ApiBaseModel diff --git a/tests/enlyze/api_clients/__init__.py b/tests/enlyze/api_clients/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/enlyze/api_clients/test_base.py b/tests/enlyze/api_clients/test_base.py new file mode 100644 index 0000000..9394d4a --- /dev/null +++ b/tests/enlyze/api_clients/test_base.py @@ -0,0 +1,282 @@ +import string +from unittest.mock import MagicMock, call, patch + +import httpx +import pytest +import respx +from hypothesis import given +from hypothesis import strategies as st + +from enlyze.api_clients.base import ( + ApiBaseClient, + ApiBaseModel, + PaginatedResponseBaseModel, +) +from enlyze.constants import ENLYZE_BASE_URL +from enlyze.errors import EnlyzeError, InvalidTokenError + + +class Metadata(ApiBaseModel): + has_more: bool + next_cursor: int | None = None + + +class PaginatedResponseModel(PaginatedResponseBaseModel): + metadata: Metadata + data: list + + +def _transform_paginated_data_integers(data: list) -> list: + return [n * n for n in data] + + +@pytest.fixture +def last_page_metadata(): + return Metadata(has_more=False, next_cursor=None) + + +@pytest.fixture +def next_page_metadata(): + return Metadata(has_more=True, next_cursor=100) + + +@pytest.fixture +def empty_paginated_response(last_page_metadata): + return PaginatedResponseModel(data=[], metadata=last_page_metadata) + + +@pytest.fixture +def response_data_integers(): + return list(range(20)) + + +@pytest.fixture +def paginated_response_with_next_page(response_data_integers, next_page_metadata): + return PaginatedResponseModel( + data=response_data_integers, metadata=next_page_metadata + ) + + +@pytest.fixture +def paginated_response_no_next_page(response_data_integers, last_page_metadata): + return PaginatedResponseModel( + data=response_data_integers, metadata=last_page_metadata + ) + + +@pytest.fixture +def base_client(auth_token, string_model): + mock_has_more = MagicMock() + mock_transform_paginated_response_data = MagicMock(side_effect=lambda e: e) + mock_next_page_call_args = MagicMock() + with patch.multiple( + ApiBaseClient, + __abstractmethods__=set(), + _has_more=mock_has_more, + _next_page_call_args=mock_next_page_call_args, + _transform_paginated_response_data=mock_transform_paginated_response_data, + ): + client = ApiBaseClient[PaginatedResponseModel](token=auth_token) + client.PaginatedResponseModel = PaginatedResponseModel + yield client + + +@given( + token=st.text(string.printable, min_size=1), +) +@respx.mock +def test_token_auth(token): + with patch.multiple(ApiBaseClient, __abstractmethods__=set()): + client = ApiBaseClient(token=token) + + route_is_authenticated = respx.get( + "", + headers__contains={"Authorization": f"Token {token}"}, + ).respond(json={}) + + client.get("") + assert route_is_authenticated.called + + +@respx.mock +def test_base_url(base_client): + endpoint = "some-endpoint" + + route = respx.get( + httpx.URL(ENLYZE_BASE_URL).join(endpoint), + ).respond(json={}) + + base_client.get(endpoint) + assert route.called + + +@respx.mock +def test_get_raises_cannot_read(base_client): + with pytest.raises(EnlyzeError, match="Couldn't read"): + respx.get("").mock(side_effect=Exception("oops")) + base_client.get("") + + +@respx.mock +def test_get_raises_on_error(base_client): + with pytest.raises(EnlyzeError, match="returned error 404"): + respx.get("").respond(404) + base_client.get("") + + +@respx.mock +def test_get_raises_invalid_token_error_not_authenticated(base_client): + with pytest.raises(InvalidTokenError): + respx.get("").respond(403) + base_client.get("") + + +@respx.mock +def test_get_raises_non_json(base_client): + with pytest.raises(EnlyzeError, match="didn't return a valid JSON object"): + respx.get("").respond(200, json=None) + base_client.get("") + + +@respx.mock +def test_get_paginated_single_page( + base_client, string_model, paginated_response_no_next_page +): + endpoint = "https://irrelevant-url.com" + params = {"params": {"param1": "value1"}} + expected_data = [ + string_model.parse_obj(e) for e in paginated_response_no_next_page.data + ] + + mock_has_more = base_client._has_more + mock_has_more.return_value = False + route = respx.get(endpoint, params=params).respond( + 200, json=paginated_response_no_next_page.dict() + ) + + data = list(base_client.get_paginated(endpoint, ApiBaseModel, params=params)) + + assert route.called + assert route.call_count == 1 + assert expected_data == data + mock_has_more.assert_called_once_with(paginated_response_no_next_page) + + +@respx.mock +def test_get_paginated_multi_page( + base_client, + paginated_response_with_next_page, + paginated_response_no_next_page, + string_model, +): + endpoint = "https://irrelevant-url.com" + initial_params = {"irrelevant": "values"} + expected_data = [ + string_model.parse_obj(e) + for e in [ + *paginated_response_with_next_page.data, + *paginated_response_no_next_page.data, + ] + ] + + mock_has_more = base_client._has_more + mock_has_more.side_effect = [True, False] + + mock_next_page_call_args = base_client._next_page_call_args + mock_next_page_call_args.return_value = (endpoint, {}, {}) + + route = respx.get(endpoint) + route.side_effect = [ + httpx.Response(200, json=paginated_response_with_next_page.dict()), + httpx.Response(200, json=paginated_response_no_next_page.dict()), + ] + + data = list( + base_client.get_paginated(endpoint, ApiBaseModel, params=initial_params) + ) + + assert route.called + assert route.call_count == 2 + assert data == expected_data + mock_has_more.assert_has_calls( + [ + call(paginated_response_with_next_page), + call(paginated_response_no_next_page), + ] + ) + mock_next_page_call_args.assert_called_once_with( + url=endpoint, + params=initial_params, + paginated_response=paginated_response_with_next_page, + ) + + +@pytest.mark.parametrize( + "invalid_payload", + [ + "not a paginated response", + {"data": "something but not a list"}, + ], +) +@respx.mock +def test_get_paginated_raises_invalid_pagination_schema( + base_client, + invalid_payload, +): + with pytest.raises(EnlyzeError, match="Paginated response expected"): + respx.get("").respond(json=invalid_payload) + next( + base_client.get_paginated( + "", + ApiBaseModel, + ) + ) + + +@respx.mock +def test_get_paginated_raises_enlyze_error( + base_client, string_model, paginated_response_no_next_page +): + # most straightforward way to raise a pydantic.ValidationError + # https://github.com/pydantic/pydantic/discussions/6459 + string_model.parse_obj.side_effect = lambda _: Metadata() + respx.get("").respond(200, json=paginated_response_no_next_page.dict()) + + with pytest.raises(EnlyzeError, match="ENLYZE platform API returned an unparsable"): + next(base_client.get_paginated("", string_model)) + + +@respx.mock +def test_get_paginated_transform_paginated_data( + base_client, paginated_response_no_next_page, string_model +): + base_client._has_more.return_value = False + base_client._transform_paginated_response_data.side_effect = ( + _transform_paginated_data_integers + ) + expected_data = [ + string_model.parse_obj(e) + for e in _transform_paginated_data_integers( + paginated_response_no_next_page.data + ) + ] + + route = respx.get("").respond(200, json=paginated_response_no_next_page.dict()) + + data = list(base_client.get_paginated("", ApiBaseModel)) + + base_client._transform_paginated_response_data.assert_called_once_with( + paginated_response_no_next_page.data + ) + + assert route.called + assert route.call_count == 1 + assert data == expected_data + + +def test_transform_paginated_data_returns_unmutated_element_by_default(auth_token): + with patch.multiple(ApiBaseClient, __abstractmethods__=set()): + client = ApiBaseClient(auth_token) + data = [1, 2, 3] + value = client._transform_paginated_response_data(data) + assert data == value diff --git a/tests/enlyze/api_clients/timeseries/__init__.py b/tests/enlyze/api_clients/timeseries/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/enlyze/api_clients/timeseries/test_client.py b/tests/enlyze/api_clients/timeseries/test_client.py new file mode 100644 index 0000000..87823a7 --- /dev/null +++ b/tests/enlyze/api_clients/timeseries/test_client.py @@ -0,0 +1,114 @@ +import httpx +import pytest +import respx + +from enlyze.api_clients.timeseries.client import TimeseriesApiClient, _PaginatedResponse +from enlyze.constants import TIMESERIES_API_SUB_PATH + + +@pytest.fixture +def endpoint(): + return "https://my-endpoint.com" + + +@pytest.fixture +def response_data_list() -> list: + return [1, 2, 3] + + +@pytest.fixture +def response_data_dict() -> dict: + return {"some": "dictionary"} + + +@pytest.fixture +def transformed_data_dict(response_data_dict) -> list[dict]: + return [response_data_dict] + + +@pytest.fixture +def paginated_response_no_next_page(): + return _PaginatedResponse(data=[], next=None) + + +@pytest.fixture +def paginated_response_with_next_page(endpoint): + return _PaginatedResponse( + data=[], + next=f"{endpoint}?offset=1337", + ) + + +@pytest.fixture +def timeseries_client(auth_token): + return TimeseriesApiClient(token=auth_token) + + +def test_timeseries_api_appends_sub_path(auth_token): + base_url = "https://some-base-url.com" + expected = str(httpx.URL(base_url).join(TIMESERIES_API_SUB_PATH)) + client = TimeseriesApiClient(token=auth_token, base_url=base_url) + assert client._full_url("") == expected + + +@pytest.mark.parametrize( + ("response_fixture", "expected_has_more"), + ( + ("paginated_response_no_next_page", False), + ("paginated_response_with_next_page", True), + ), +) +def test_has_more(request, response_fixture, expected_has_more, timeseries_client): + response = request.getfixturevalue(response_fixture) + assert timeseries_client._has_more(response) == expected_has_more + + +@pytest.mark.parametrize( + ("data_fixture", "expected_fixture"), + ( + ("response_data_list", "response_data_list"), + ("response_data_dict", "transformed_data_dict"), + ), +) +def test_get_paginated_transform_paginated_data( + request, timeseries_client, data_fixture, expected_fixture +): + data = request.getfixturevalue(data_fixture) + expected = request.getfixturevalue(expected_fixture) + assert timeseries_client._transform_paginated_response_data(data) == expected + + +def test_next_page_call_args( + timeseries_client, endpoint, paginated_response_with_next_page +): + params = {"some": "param"} + kwargs = {"some": "kwarg"} + url = endpoint + next_url, next_params, next_kwargs = timeseries_client._next_page_call_args( + url=url, + params=params, + paginated_response=paginated_response_with_next_page, + **kwargs, + ) + assert next_url == str(paginated_response_with_next_page.next) + assert next_params == params + assert next_kwargs == kwargs + + +@respx.mock +def test_timeseries_api_get_paginated_single_page(timeseries_client, string_model): + respx.get("").respond(json={"data": ["a", "b"], "next": None}) + assert list(timeseries_client.get_paginated("", string_model)) == ["a", "b"] + + +@respx.mock +def test_timeseries_api_get_paginated_multi_page(timeseries_client, string_model): + respx.get("", params="offset=1").respond(json={"data": ["z"], "next": None}) + respx.get("").mock( + side_effect=lambda request: httpx.Response( + 200, + json={"data": ["x", "y"], "next": str(request.url.join("?offset=1"))}, + ) + ) + + assert list(timeseries_client.get_paginated("", string_model)) == ["x", "y", "z"] diff --git a/tests/enlyze/test_client.py b/tests/enlyze/test_client.py index bc2a136..bc648f0 100644 --- a/tests/enlyze/test_client.py +++ b/tests/enlyze/test_client.py @@ -6,12 +6,12 @@ from hypothesis import given from hypothesis import strategies as st +import enlyze.api_clients.timeseries.models as api_models import enlyze.models as user_models -import enlyze.timeseries_api.models as api_models +from enlyze.api_clients.timeseries.client import _PaginatedResponse from enlyze.client import EnlyzeClient from enlyze.constants import ENLYZE_BASE_URL, TIMESERIES_API_SUB_PATH from enlyze.errors import EnlyzeError -from enlyze.timeseries_api.client import _PaginatedResponse from tests.conftest import ( datetime_before_today_strategy, datetime_today_until_now_strategy, diff --git a/tests/enlyze/timeseries_api/test_client.py b/tests/enlyze/timeseries_api/test_client.py deleted file mode 100644 index 9026218..0000000 --- a/tests/enlyze/timeseries_api/test_client.py +++ /dev/null @@ -1,131 +0,0 @@ -import string -from unittest import mock - -import httpx -import pytest -import respx -from hypothesis import given -from hypothesis import strategies as st - -from enlyze.constants import ENLYZE_BASE_URL, TIMESERIES_API_SUB_PATH -from enlyze.errors import EnlyzeError, InvalidTokenError -from enlyze.timeseries_api.client import TimeseriesApiClient -from enlyze.timeseries_api.models import TimeseriesApiModel - - -@pytest.fixture -def string_model(): - with mock.patch( - "enlyze.timeseries_api.models.TimeseriesApiModel.parse_obj", - side_effect=lambda o: str(o), - ): - yield TimeseriesApiModel - - -@pytest.fixture -def client(): - return TimeseriesApiClient(token="some token") - - -@given( - token=st.text(string.printable, min_size=1), -) -@respx.mock -def test_timeseries_api_token_auth(token): - client = TimeseriesApiClient(token=token) - - route_is_authenticated = respx.get( - "", - headers__contains={"Authorization": f"Token {token}"}, - ).respond(json={}) - - client.get("") - assert route_is_authenticated.called - - -@respx.mock -def test_timeseries_api_client_base_url(client): - endpoint = "some-endpoint" - - route = respx.get( - httpx.URL(ENLYZE_BASE_URL).join(TIMESERIES_API_SUB_PATH).join(endpoint), - ).respond(json={}) - - client.get(endpoint) - assert route.called - - -@respx.mock -def test_timeseries_api_get_raises_cannot_read(client): - with pytest.raises(EnlyzeError, match="Couldn't read"): - respx.get("").mock(side_effect=Exception("oops")) - client.get("") - - -@respx.mock -def test_timeseries_api_get_raises_on_error(client): - with pytest.raises(EnlyzeError, match="returned error 404"): - respx.get("").respond(404) - client.get("") - - -@respx.mock -def test_timeseries_api_get_raises_invalid_token_error_not_authenticated(client): - with pytest.raises(InvalidTokenError): - respx.get("").respond(403) - client.get("") - - -@respx.mock -def test_timeseries_api_get_raises_non_json(client): - with pytest.raises(EnlyzeError, match="didn't return a valid JSON object"): - respx.get("").respond(200, json=None) - client.get("") - - -@pytest.mark.parametrize( - "invalid_payload", - [ - "not a paginated response", - {"data": "something but not a list"}, - ], -) -@respx.mock -def test_timeseries_api_get_paginated_raises_invalid_pagination_schema( - client, string_model, invalid_payload -): - with pytest.raises(EnlyzeError, match="Paginated response expected"): - respx.get("").respond(json=invalid_payload) - next(client.get_paginated("", string_model)) - - -@respx.mock -def test_timeseries_api_get_paginated_raises_invalid_data_schema(client): - respx.get("").respond(json={"data": [42, 1337], "next": None}) - with pytest.raises(EnlyzeError, match="API returned an unparsable"): - list(client.get_paginated("", TimeseriesApiModel)) - - -@respx.mock -def test_timeseries_api_get_paginated_data_empty(client, string_model): - respx.get("").respond(json={"data": [], "next": None}) - assert list(client.get_paginated("", string_model)) == [] - - -@respx.mock -def test_timeseries_api_get_paginated_single_page(client, string_model): - respx.get("").respond(json={"data": ["a", "b"], "next": None}) - assert list(client.get_paginated("", string_model)) == ["a", "b"] - - -@respx.mock -def test_timeseries_api_get_paginated_multi_page(client, string_model): - respx.get("", params="offset=1").respond(json={"data": ["z"], "next": None}) - respx.get("").mock( - side_effect=lambda request: httpx.Response( - 200, - json={"data": ["x", "y"], "next": str(request.url.join("?offset=1"))}, - ) - ) - - assert list(client.get_paginated("", string_model)) == ["x", "y", "z"] From c5784d5ced18b642dfbb36006628fca1d49e3cbf Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Fri, 29 Sep 2023 16:42:01 +0200 Subject: [PATCH 14/25] Add `ProductionRunsApiClient` (#15) Building up on #14 , this PR introduces the `ProductionRunsApiClient`, which instruments the Production Runs API. A subsequent PR will then expose a `ProductionRun` user model via the `EnlyzeClient`. --------- Co-authored-by: Daniel Krebs --- docs/api_clients/index.rst | 1 + docs/api_clients/production_runs/client.rst | 9 ++ docs/api_clients/production_runs/index.rst | 7 ++ .../api_clients/production_runs/__init__.py | 0 .../api_clients/production_runs/client.py | 51 ++++++++ src/enlyze/constants.py | 3 + tests/conftest.py | 12 -- tests/enlyze/api_clients/conftest.py | 19 +++ .../api_clients/production_runs/__init__.py | 0 .../production_runs/test_client.py | 118 ++++++++++++++++++ .../api_clients/timeseries/test_client.py | 5 - 11 files changed, 208 insertions(+), 17 deletions(-) create mode 100644 docs/api_clients/production_runs/client.rst create mode 100644 docs/api_clients/production_runs/index.rst create mode 100644 src/enlyze/api_clients/production_runs/__init__.py create mode 100644 src/enlyze/api_clients/production_runs/client.py create mode 100644 tests/enlyze/api_clients/conftest.py create mode 100644 tests/enlyze/api_clients/production_runs/__init__.py create mode 100644 tests/enlyze/api_clients/production_runs/test_client.py diff --git a/docs/api_clients/index.rst b/docs/api_clients/index.rst index ddd6e48..91ec8e1 100644 --- a/docs/api_clients/index.rst +++ b/docs/api_clients/index.rst @@ -6,3 +6,4 @@ API Clients base timeseries/index + production_runs/index diff --git a/docs/api_clients/production_runs/client.rst b/docs/api_clients/production_runs/client.rst new file mode 100644 index 0000000..2c5f92f --- /dev/null +++ b/docs/api_clients/production_runs/client.rst @@ -0,0 +1,9 @@ +Production Runs API Client +========================== + +.. currentmodule:: enlyze.api_clients.production_runs.client + +.. autoclass:: _PaginatedResponse + +.. autoclass:: ProductionRunsApiClient() + :members: diff --git a/docs/api_clients/production_runs/index.rst b/docs/api_clients/production_runs/index.rst new file mode 100644 index 0000000..a4aa667 --- /dev/null +++ b/docs/api_clients/production_runs/index.rst @@ -0,0 +1,7 @@ +Production Runs API +=================== + +.. toctree:: + :maxdepth: 1 + + client diff --git a/src/enlyze/api_clients/production_runs/__init__.py b/src/enlyze/api_clients/production_runs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/enlyze/api_clients/production_runs/client.py b/src/enlyze/api_clients/production_runs/client.py new file mode 100644 index 0000000..73ab06f --- /dev/null +++ b/src/enlyze/api_clients/production_runs/client.py @@ -0,0 +1,51 @@ +from typing import Any + +import httpx +from pydantic import BaseModel + +from enlyze.api_clients.base import ApiBaseClient, PaginatedResponseBaseModel +from enlyze.constants import ENLYZE_BASE_URL, PRODUCTION_RUNS_API_SUB_PATH + + +class _Metadata(BaseModel): + next_cursor: int | None + has_more: bool + + +class _PaginatedResponse(PaginatedResponseBaseModel): + metadata: _Metadata + data: list[dict[str, Any]] + + +class ProductionRunsApiClient(ApiBaseClient[_PaginatedResponse]): + """Client class encapsulating all interaction with the Production Runs API + + :param token: API token for the ENLYZE platform + :param base_url: Base URL of the ENLYZE platform + :param timeout: Global timeout for all HTTP requests sent to the Production Runs API + + """ + + PaginatedResponseModel = _PaginatedResponse + + def __init__( + self, token: str, *, base_url: str | httpx.URL = ENLYZE_BASE_URL, **kwargs: Any + ): + super().__init__( + token, + base_url=httpx.URL(base_url).join(PRODUCTION_RUNS_API_SUB_PATH), + **kwargs, + ) + + def _has_more(self, paginated_response: _PaginatedResponse) -> bool: + return paginated_response.metadata.has_more + + def _next_page_call_args( + self, + url: str, + params: dict[str, Any], + paginated_response: _PaginatedResponse, + **kwargs: Any, + ) -> tuple[str, dict[str, Any], dict[str, Any]]: + next_params = {**params, "cursor": paginated_response.metadata.next_cursor} + return (url, next_params, kwargs) diff --git a/src/enlyze/constants.py b/src/enlyze/constants.py index 6abf60c..ac632c9 100644 --- a/src/enlyze/constants.py +++ b/src/enlyze/constants.py @@ -4,6 +4,9 @@ #: URL sub-path where the Timeseries API is deployed on the ENLYZE platform. TIMESERIES_API_SUB_PATH = "timeseries-service/v1/" +#: URL sub-path where the Production Runs API is deployed on the ENLYZE platform. +PRODUCTION_RUNS_API_SUB_PATH = "production-runs/v1/" + #: HTTP timeout for requests to the Timeseries API. #: #: Reference: https://www.python-httpx.org/advanced/#timeout-configuration diff --git a/tests/conftest.py b/tests/conftest.py index 1b54475..af8ef29 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,10 @@ import os from datetime import datetime, timezone -from unittest.mock import patch import hypothesis import pytest from hypothesis import strategies as st -from enlyze.api_clients.base import ApiBaseModel - hypothesis.settings.register_profile("ci", deadline=None) hypothesis.settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default")) @@ -26,12 +23,3 @@ @pytest.fixture def auth_token(): return "some-token" - - -@pytest.fixture -def string_model(): - with patch( - "enlyze.api_clients.base.ApiBaseModel.parse_obj", - side_effect=lambda o: str(o), - ): - yield ApiBaseModel diff --git a/tests/enlyze/api_clients/conftest.py b/tests/enlyze/api_clients/conftest.py new file mode 100644 index 0000000..8ab937b --- /dev/null +++ b/tests/enlyze/api_clients/conftest.py @@ -0,0 +1,19 @@ +from unittest.mock import patch + +import pytest + +from enlyze.api_clients.base import ApiBaseModel + + +@pytest.fixture +def string_model(): + with patch( + "enlyze.api_clients.base.ApiBaseModel.parse_obj", + side_effect=lambda o: str(o), + ): + yield ApiBaseModel + + +@pytest.fixture +def endpoint(): + return "https://my-endpoint.com" diff --git a/tests/enlyze/api_clients/production_runs/__init__.py b/tests/enlyze/api_clients/production_runs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/enlyze/api_clients/production_runs/test_client.py b/tests/enlyze/api_clients/production_runs/test_client.py new file mode 100644 index 0000000..2e2f675 --- /dev/null +++ b/tests/enlyze/api_clients/production_runs/test_client.py @@ -0,0 +1,118 @@ +import httpx +import pytest +import respx + +from enlyze.api_clients.production_runs.client import ( + ProductionRunsApiClient, + _Metadata, + _PaginatedResponse, +) +from enlyze.constants import ENLYZE_BASE_URL, PRODUCTION_RUNS_API_SUB_PATH + + +@pytest.fixture +def metadata_last_page(): + return _Metadata(has_more=False, next_cursor=None) + + +@pytest.fixture +def metadata_next_page(): + return _Metadata(has_more=True, next_cursor=1337) + + +@pytest.fixture +def response_data(): + return [{"id": i, "name": f"row-{i}"} for i in range(10)] + + +@pytest.fixture +def paginated_response_no_next_page(response_data, metadata_last_page): + return _PaginatedResponse(data=response_data, metadata=metadata_last_page) + + +@pytest.fixture +def paginated_response_with_next_page(response_data, metadata_next_page): + return _PaginatedResponse(data=response_data, metadata=metadata_next_page) + + +@pytest.fixture +def production_runs_client(auth_token): + return ProductionRunsApiClient(token=auth_token) + + +def test_timeseries_api_appends_sub_path(auth_token): + base_url = ENLYZE_BASE_URL + expected = str(httpx.URL(base_url).join(PRODUCTION_RUNS_API_SUB_PATH)) + client = ProductionRunsApiClient(token=auth_token, base_url=base_url) + assert client._full_url("") == expected + + +@pytest.mark.parametrize( + ("response_fixture", "expected_has_more"), + ( + ("paginated_response_no_next_page", False), + ("paginated_response_with_next_page", True), + ), +) +def test_has_more(request, response_fixture, expected_has_more, production_runs_client): + response = request.getfixturevalue(response_fixture) + assert production_runs_client._has_more(response) == expected_has_more + + +def test_next_page_call_args( + production_runs_client, endpoint, paginated_response_with_next_page +): + params = {"some": "param"} + kwargs = {"some": "kwarg"} + url = endpoint + next_url, next_params, next_kwargs = production_runs_client._next_page_call_args( + url=url, + params=params, + paginated_response=paginated_response_with_next_page, + **kwargs, + ) + assert next_url == url + assert next_params == { + **params, + "cursor": paginated_response_with_next_page.metadata.next_cursor, + } + assert next_kwargs == kwargs + + +@respx.mock +def test_timeseries_api_get_paginated_single_page( + production_runs_client, string_model, paginated_response_no_next_page +): + expected_data = [ + string_model.parse_obj(e) for e in paginated_response_no_next_page.data + ] + respx.get("").respond(json=paginated_response_no_next_page.dict()) + assert list(production_runs_client.get_paginated("", string_model)) == expected_data + + +@respx.mock +def test_timeseries_api_get_paginated_multi_page( + production_runs_client, + string_model, + paginated_response_with_next_page, + paginated_response_no_next_page, +): + expected_data = [ + string_model.parse_obj(e) + for e in [ + *paginated_response_no_next_page.data, + *paginated_response_with_next_page.data, + ] + ] + next_cursor = paginated_response_with_next_page.metadata.next_cursor + respx.get("", params=f"cursor={next_cursor}").respond( + 200, json=paginated_response_no_next_page.dict() + ) + respx.get("").mock( + side_effect=lambda request: httpx.Response( + 200, + json=paginated_response_with_next_page.dict(), + ) + ) + + assert list(production_runs_client.get_paginated("", string_model)) == expected_data diff --git a/tests/enlyze/api_clients/timeseries/test_client.py b/tests/enlyze/api_clients/timeseries/test_client.py index 87823a7..8cfe9ac 100644 --- a/tests/enlyze/api_clients/timeseries/test_client.py +++ b/tests/enlyze/api_clients/timeseries/test_client.py @@ -6,11 +6,6 @@ from enlyze.constants import TIMESERIES_API_SUB_PATH -@pytest.fixture -def endpoint(): - return "https://my-endpoint.com" - - @pytest.fixture def response_data_list() -> list: return [1, 2, 3] From a650dc4bb84b09a6cfa3597db51282061ff20a97 Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Tue, 10 Oct 2023 16:28:35 +0200 Subject: [PATCH 15/25] Revert "Align modelling of production runs after remodelling of API (#9)" This reverts commit af984abac72539083490e9897f5d43c78a9499a9. --- docs/models.rst | 12 ----- src/enlyze/models.py | 101 +------------------------------------------ 2 files changed, 1 insertion(+), 112 deletions(-) diff --git a/docs/models.rst b/docs/models.rst index cfb091e..07d0fdf 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -24,15 +24,3 @@ Data models .. autoclass:: ResamplingMethod() :members: :undoc-members: - -.. autoclass:: ProductionRun() - :members: - :undoc-members: - -.. autoclass:: OEEScore() - :members: - :undoc-members: - -.. autoclass:: Quantity() - :members: - :undoc-members: diff --git a/src/enlyze/models.py b/src/enlyze/models.py index 66f6bae..85f3ae6 100644 --- a/src/enlyze/models.py +++ b/src/enlyze/models.py @@ -1,5 +1,5 @@ from dataclasses import asdict, dataclass -from datetime import date, datetime, timedelta, timezone +from datetime import date, datetime, timezone from enum import Enum from itertools import chain from typing import Any, Iterator, Optional, Sequence @@ -197,102 +197,3 @@ def to_dataframe(self, use_display_names: bool = False) -> pandas.DataFrame: ) df.index = pandas.to_datetime(df.index, utc=True, format="ISO8601") return df - - -@dataclass(frozen=True) -class OEEScore: - """Individual Overall Equipment Effectiveness (OEE) score - - This is calculated by the ENLYZE Platform based on a combination of real - machine data and production order booking information provided by the - customer. - - For more information, please check out https://www.oee.com - """ - - #: The score is expressed as a ratio between 0 and 1.0, with 1.0 meaning 100 %. - score: float - - #: Unproductive time due to non-ideal production. - time_loss: timedelta - - -@dataclass(frozen=True) -class Quantity: - """Representation of a physical quantity""" - - #: Physical unit of quantity - unit: str - - #: The quantity expressed in `unit` - value: float - - -@dataclass(frozen=True) -class ProductionRun: - """Representation of a :ref:`production run ` in the ENLYZE - platform. - - Contains details about the production run. - - """ - - #: The UUID of the appliance the production run was exeucted on. - appliance: UUID - - #: The average throughput of the production run excluding downtimes. - average_throughput: Optional[float] - - #: The identifier of the production order. - production_order: str - - #: The identifier of the product that was produced. - product: str - - #: The begin of the production run. - begin: datetime - - #: The end of the production run. - end: Optional[datetime] - - #: This is the sum of scrap and yield. - quantity_total: Optional[Quantity] - - #: The amount of product produced that doesn't meet quality criteria. - quantity_scrap: Optional[Quantity] - - #: The amount of product produced that can be sold. - quantity_yield: Optional[Quantity] - - #: OEE component that reflects when the appliance did not produce. - availability: Optional[OEEScore] - - #: OEE component that reflects how fast the appliance has run. - performance: Optional[OEEScore] - - #: OEE component that reflects how much defects have been produced. - quality: Optional[OEEScore] - - #: Aggregate OEE score that comprises availability, performance and quality. - productivity: Optional[OEEScore] - - -class ProductionRuns(list[ProductionRun]): - """Representation of multiple :ref:`production runs `.""" - - def to_dataframe(self) -> pandas.DataFrame: - """Convert production runs into :py:class:`pandas.DataFrame` - - Each row in the data frame will represent one production run. - The ``begin`` and ``end`` of every production run will be - represented as :ref:`timezone-aware ` - :py:class:`datetime.datetime` localized in UTC. - - :returns: DataFrame with production runs - - """ - - df = pandas.json_normalize([asdict(run) for run in self]) - df.begin = pandas.to_datetime(df.begin, utc=True, format="ISO8601") - df.end = pandas.to_datetime(df.end, utc=True, format="ISO8601") - return df From 75246bd36697cf065b87e60c027c32e25cf01589 Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Tue, 10 Oct 2023 16:29:18 +0200 Subject: [PATCH 16/25] Revert "draft of production runs, production run, quantities and metrics" This reverts commit a6d15f55c8a796fc0879f27aa0fc71a019494e52. --- src/enlyze/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/enlyze/models.py b/src/enlyze/models.py index 85f3ae6..121105c 100644 --- a/src/enlyze/models.py +++ b/src/enlyze/models.py @@ -1,4 +1,4 @@ -from dataclasses import asdict, dataclass +from dataclasses import dataclass from datetime import date, datetime, timezone from enum import Enum from itertools import chain From 0448296cb283ae7fea0ce49d77f7d2173416ee41 Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Tue, 10 Oct 2023 16:58:38 +0200 Subject: [PATCH 17/25] add Betriebsdatenerfassung to spelling wordlist --- docs/spelling_wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 6f65e78..ad561c6 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -1,3 +1,4 @@ +Betriebsdatenerfassung CNC JSON ResamplingMethod From e952cd2d60b72d614ab37b487fb5263b7655f331 Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Tue, 10 Oct 2023 16:59:02 +0200 Subject: [PATCH 18/25] rephrase concepts --- docs/concepts.rst | 57 ++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index a81b3ea..222d9af 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -33,38 +33,36 @@ A *variable* represents a process measure of one :ref:`appliance ` of timeseries data is captured and stored in the ENLYZE platform. One appliance may have many variables, whereas one variable is only associated with one appliance. - .. _production_order: Production Order ---------------- -A *production order* represents the goal of producing a certain quantity of a -given :ref:`product `. This is how operators are told how much of which -product to produce. Production orders are usually created in an external ERP -system or MES and then synchronized into the ENLYZE platform. They are -referenced by an identifier which oftentimes is a short combination of numbers -and/or characters, like FA23000123 or 332554. - -In the ENLYZE platform, a production order always encompasses the production of -one single :ref:`product ` on one single :ref:`appliance `. +A *production order* represents the goal of producing a certain quantity of a given +:ref:`product `. This is how operators know how much of which product to +produce. Production orders are usually created in an external system such as an ERP or +MES and then synchronized into the ENLYZE platform. They are referenced by an identifier +which oftentimes is a short combination of numbers and/or characters, like FA23000123 or +332554. +In the ENLYZE platform, a production order always encompasses the production of one +single :ref:`product ` on one single :ref:`appliance ` within or +more :ref:`production runs `. .. _production_run: Production Run -------------- -A *production run* represents the real allocation of a :ref:`production order -` on an :ref:`appliance `. Usually, the operator of -the appliance uses an interface to log the time when a certain -production order has been worked on. For instance, this could be the appliance's HMI or a tablet computer next to it. In German, this is often referred to as "Betriebsdatenerfassung (BDE)". A production order oftentimes is not -allocated continously on an appliance due to interruptions like a breakdown or -weekend. Each of these individual, continous allocations is called a *production -run* in the ENLYZE platform. It always has a beginning and, if it's not still -running, it also has an end. The ENLYZE platform calculates different aggregates -from the timeseries data of the appliance for each production run. +A *production run* is a time frame within a machine was producing a :ref:`product +` on an :ref:`appliance ` in order to complete a :ref:`production +order `. A production run always has a beginning and, if it's not +still running, it also has an end. +Usually, the operator of the appliance uses an interface to log the time when a certain +production order has been worked on. For instance, this could be the appliance's HMI or +a tablet computer next to it. In German, this is often referred to as +"Betriebsdatenerfassung (BDE)". .. _product: @@ -72,14 +70,13 @@ Product ------- A *product* is the output of the production process which is executed by an -:ref:`appliance `, driven by a :ref:`production order -`. In the real world, an appliance might have some additonal -outputs, but only the main output (the product) modelled in the ENLYZE platform. -Similarly to the production order, a product is referenced by an identifier -originating from a customer system, which is synchronized into the ENLYZE -platform. - -During the integration into the ENLYZE platform, the product identifier is -chosen in such a way that :ref:`production runs ` of the same -product are comparable with one another. This is the foundation for constantly -improving the production of recurring products over time. +:ref:`appliance `, driven by a :ref:`production order `. In +the real world, an appliance might have some additional outputs, but only the main +output (the product) modeled in the ENLYZE platform. Similarly to the production order, +a product is referenced by an identifier originating from a customer system, which is +synchronized into the ENLYZE platform. + +During the integration into the ENLYZE platform, the product identifier is chosen in +such a way that :ref:`production runs ` of the same product are +comparable with one another. This is the foundation for constantly improving the +production of recurring products over time. From 3cd10baef3f3236c1811c1d8571385cc2ca6b660 Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Tue, 10 Oct 2023 17:30:32 +0200 Subject: [PATCH 19/25] Update docs/concepts.rst Co-authored-by: Daniel Krebs --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 222d9af..1bd253a 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -73,7 +73,7 @@ A *product* is the output of the production process which is executed by an :ref:`appliance `, driven by a :ref:`production order `. In the real world, an appliance might have some additional outputs, but only the main output (the product) modeled in the ENLYZE platform. Similarly to the production order, -a product is referenced by an identifier originating from a customer system, which is +a product is referenced by an identifier originating from a customer's system, that gets synchronized into the ENLYZE platform. During the integration into the ENLYZE platform, the product identifier is chosen in From 4569019cced32546a631fea146687c740e6b0056 Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Tue, 10 Oct 2023 17:30:42 +0200 Subject: [PATCH 20/25] Update docs/concepts.rst Co-authored-by: Daniel Krebs --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 1bd253a..39ec17e 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -72,7 +72,7 @@ Product A *product* is the output of the production process which is executed by an :ref:`appliance `, driven by a :ref:`production order `. In the real world, an appliance might have some additional outputs, but only the main -output (the product) modeled in the ENLYZE platform. Similarly to the production order, +output (the product) is modeled in the ENLYZE platform. Similarly to the production order, a product is referenced by an identifier originating from a customer's system, that gets synchronized into the ENLYZE platform. From 1f91c680948357cc172f671cf9fd637974d5ea84 Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Tue, 10 Oct 2023 17:30:49 +0200 Subject: [PATCH 21/25] Update docs/concepts.rst Co-authored-by: Daniel Krebs --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 39ec17e..f0da9ed 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -62,7 +62,7 @@ still running, it also has an end. Usually, the operator of the appliance uses an interface to log the time when a certain production order has been worked on. For instance, this could be the appliance's HMI or a tablet computer next to it. In German, this is often referred to as -"Betriebsdatenerfassung (BDE)". +*Betriebsdatenerfassung* (BDE). It is common, that a production order is not completed in one go, but is interrupted several times for very different reasons, like a breakdown of the appliance or a public holiday. These interruptions lead to the creation of multiple production runs for a single production order. .. _product: From c2345af252686f43332be8cd50138074da54333d Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Tue, 10 Oct 2023 17:30:56 +0200 Subject: [PATCH 22/25] Update docs/concepts.rst Co-authored-by: Daniel Krebs --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index f0da9ed..9a958d7 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -54,7 +54,7 @@ more :ref:`production runs `. Production Run -------------- -A *production run* is a time frame within a machine was producing a :ref:`product +A *production run* is a time frame within which a machine was producing a :ref:`product ` on an :ref:`appliance ` in order to complete a :ref:`production order `. A production run always has a beginning and, if it's not still running, it also has an end. From 32b36ea673221b4cd809827bfdbf8e23ad18b626 Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Tue, 10 Oct 2023 17:31:10 +0200 Subject: [PATCH 23/25] Update docs/concepts.rst Co-authored-by: Daniel Krebs --- docs/concepts.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 9a958d7..edc47fb 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -46,8 +46,8 @@ which oftentimes is a short combination of numbers and/or characters, like FA230 332554. In the ENLYZE platform, a production order always encompasses the production of one -single :ref:`product ` on one single :ref:`appliance ` within or -more :ref:`production runs `. +single :ref:`product ` on one single :ref:`appliance ` within one +or more :ref:`production runs `. .. _production_run: From 3312288e2141325ffc70f2ae4ac420eb88e99410 Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Tue, 10 Oct 2023 17:31:18 +0200 Subject: [PATCH 24/25] Update docs/concepts.rst Co-authored-by: Daniel Krebs --- docs/concepts.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index edc47fb..7dee8ae 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -42,8 +42,7 @@ A *production order* represents the goal of producing a certain quantity of a gi :ref:`product `. This is how operators know how much of which product to produce. Production orders are usually created in an external system such as an ERP or MES and then synchronized into the ENLYZE platform. They are referenced by an identifier -which oftentimes is a short combination of numbers and/or characters, like FA23000123 or -332554. +which oftentimes is a short combination of numbers and/or characters, like FA23000123. In the ENLYZE platform, a production order always encompasses the production of one single :ref:`product ` on one single :ref:`appliance ` within one From 77a0a4e65940ddadd2faabe4e9dcbb38d6884f28 Mon Sep 17 00:00:00 2001 From: Deniz Saner Date: Tue, 10 Oct 2023 17:32:41 +0200 Subject: [PATCH 25/25] format docs --- docs/concepts.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 7dee8ae..9f9b54f 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -60,8 +60,11 @@ still running, it also has an end. Usually, the operator of the appliance uses an interface to log the time when a certain production order has been worked on. For instance, this could be the appliance's HMI or -a tablet computer next to it. In German, this is often referred to as -*Betriebsdatenerfassung* (BDE). It is common, that a production order is not completed in one go, but is interrupted several times for very different reasons, like a breakdown of the appliance or a public holiday. These interruptions lead to the creation of multiple production runs for a single production order. +a tablet computer next to it. In German, this is often referred to as *Betriebsdatenerfassung* (BDE). +It is common, that a production order is not completed in one go, but is interrupted +several times for very different reasons, like a breakdown of the appliance or a +public holiday. These interruptions lead to the creation of multiple production runs +for a single production order. .. _product: