From d5290dcbad7da84b7ff252dd3ce5bf927d0de2ce Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 1 Nov 2024 08:51:45 -0400 Subject: [PATCH 01/50] Refactoring Dependencies to drop 3.7 and 3.8 support --- setup.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index a524c862..bb425e10 100644 --- a/setup.py +++ b/setup.py @@ -31,33 +31,27 @@ def find_version(): package_data={"": ["README.md"]}, install_requires=[ "appdirs<2", - "lark-parser<0.7", - "marshmallow<3", - "marshmallow-polyfield<4", + "lark-parser<1", + "marshmallow<4", + "marshmallow-polyfield<6", "packaging", - "pika<=1.2,>=1.0.1", - "pytz<2021", + "pika<=1.4,>=1.0.1", + "pytz", "requests<3", "simplejson<4", "six<2", "wrapt", "yapconf>=0.3.7", ], - extras_require={ - ':python_version=="2.7"': ["futures", "funcsigs", "pathlib"], - ':python_version<"3.4"': ["enum34"], - ':python_version<"3.5"': ["typing"], - }, classifiers=[ "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Software Development :: Libraries :: Python Modules", ], ) From a149fae1edc50d88a935bbad31b13a846577313a Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 1 Nov 2024 08:53:29 -0400 Subject: [PATCH 02/50] Update pr actions to include 3.12 and 3.13 --- .github/workflows/pr-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-actions.yml b/.github/workflows/pr-actions.yml index 6477ba5c..a8bd8106 100644 --- a/.github/workflows/pr-actions.yml +++ b/.github/workflows/pr-actions.yml @@ -37,7 +37,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] os: ['ubuntu-latest'] name: PyTests OS ${{ matrix.os }} - Python ${{ matrix.python-version }} steps: From 5fdaca3b2243ccbf5ef4f2b22efa5729d1e123b6 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 1 Nov 2024 09:13:13 -0400 Subject: [PATCH 03/50] fixing UTC usage --- brewtils/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brewtils/schemas.py b/brewtils/schemas.py index ccaeee87..1d31c199 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -7,8 +7,8 @@ import marshmallow import simplejson from marshmallow import Schema, fields, post_load, pre_load -from marshmallow.utils import UTC from marshmallow_polyfield import PolyField +from pytz import UTC __all__ = [ "SystemSchema", From 083c24595ccf81fb2ba8428fc3bbb7d66db6d954 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Mon, 4 Nov 2024 06:45:05 -0500 Subject: [PATCH 04/50] Migrate to new date time field --- brewtils/schemas.py | 106 ++++++++++++++++++++------------------------ 1 file changed, 47 insertions(+), 59 deletions(-) diff --git a/brewtils/schemas.py b/brewtils/schemas.py index 1d31c199..9df24335 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -1,14 +1,11 @@ # -*- coding: utf-8 -*- -import calendar -import datetime from functools import partial import marshmallow import simplejson from marshmallow import Schema, fields, post_load, pre_load from marshmallow_polyfield import PolyField -from pytz import UTC __all__ = [ "SystemSchema", @@ -94,45 +91,6 @@ def __init__(self, type_field="payload_type", allowed_types=None, **kwargs): ) -class DateTime(fields.DateTime): - """Class that adds methods for (de)serializing DateTime fields as an epoch""" - - def __init__(self, format="epoch", **kwargs): - self.DATEFORMAT_SERIALIZATION_FUNCS["epoch"] = self.to_epoch - self.DATEFORMAT_DESERIALIZATION_FUNCS["epoch"] = self.from_epoch - super(DateTime, self).__init__(format=format, **kwargs) - - @staticmethod - def to_epoch(dt, localtime=False): - # If already in epoch form just return it - if isinstance(dt, int): - return dt - - if localtime and dt.tzinfo is not None: - localized = dt - else: - if dt.tzinfo is None: - localized = UTC.localize(dt) - else: - localized = dt.astimezone(UTC) - return (calendar.timegm(localized.timetuple()) * 1000) + int( - localized.microsecond / 1000 - ) - - @staticmethod - def from_epoch(epoch): - # If already in datetime form just return it - if isinstance(epoch, datetime.datetime): - return epoch - - # utcfromtimestamp will correctly parse milliseconds in Python 3, - # but in Python 2 we need to help it - seconds, millis = divmod(epoch, 1000) - return datetime.datetime.utcfromtimestamp(seconds).replace( - microsecond=millis * 1000 - ) - - class BaseSchema(Schema): class Meta: version_nums = marshmallow.__version__.split(".") @@ -256,7 +214,9 @@ class FileSchema(BaseSchema): owner = fields.Raw(allow_none=True) job = fields.Nested("JobSchema", allow_none=True) request = fields.Nested("RequestSchema", allow_none=True) - updated_at = DateTime(allow_none=True, format="epoch", example="1500065932000") + updated_at = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) file_name = fields.Str(allow_none=True) file_size = fields.Int(allow_none=False) chunks = fields.Dict(allow_none=True) @@ -325,10 +285,14 @@ class RequestSchema(RequestTemplateSchema): hidden = fields.Boolean(allow_none=True) status = fields.Str(allow_none=True) error_class = fields.Str(allow_none=True) - created_at = DateTime(allow_none=True, format="epoch", example="1500065932000") - updated_at = DateTime(allow_none=True, format="epoch", example="1500065932000") - status_updated_at = DateTime( - allow_none=True, format="epoch", example="1500065932000" + created_at = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) + updated_at = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) + status_updated_at = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" ) has_parent = fields.Bool(allow_none=True) requester = fields.String(allow_none=True) @@ -337,12 +301,16 @@ class RequestSchema(RequestTemplateSchema): class StatusHistorySchema(BaseSchema): - heartbeat = DateTime(allow_none=True, format="epoch", example="1500065932000") + heartbeat = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) status = fields.Str(allow_none=True) class StatusInfoSchema(BaseSchema): - heartbeat = DateTime(allow_none=True, format="epoch", example="1500065932000") + heartbeat = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) history = fields.Nested("StatusHistorySchema", many=True, allow_none=True) @@ -395,7 +363,9 @@ class EventSchema(BaseSchema): namespace = fields.Str(allow_none=True) garden = fields.Str(allow_none=True) metadata = fields.Dict(allow_none=True) - timestamp = DateTime(allow_none=True, format="epoch", example="1500065932000") + timestamp = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) payload_type = fields.Str(allow_none=True) payload = ModelField(allow_none=True, type_field="payload_type") @@ -417,13 +387,19 @@ class QueueSchema(BaseSchema): class UserTokenSchema(BaseSchema): id = fields.Str(allow_none=True) uuid = fields.Str(allow_none=True) - issued_at = DateTime(allow_none=True, format="epoch", example="1500065932000") - expires_at = DateTime(allow_none=True, format="epoch", example="1500065932000") + issued_at = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) + expires_at = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) username = fields.Str(allow_none=True) class DateTriggerSchema(BaseSchema): - run_date = DateTime(allow_none=True, format="epoch", example="1500065932000") + run_date = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) timezone = fields.Str(allow_none=True) @@ -433,8 +409,12 @@ class IntervalTriggerSchema(BaseSchema): hours = fields.Int(allow_none=True) minutes = fields.Int(allow_none=True) seconds = fields.Int(allow_none=True) - start_date = DateTime(allow_none=True, format="epoch", example="1500065932000") - end_date = DateTime(allow_none=True, format="epoch", example="1500065932000") + start_date = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) + end_date = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) timezone = fields.Str(allow_none=True) jitter = fields.Int(allow_none=True) reschedule_on_finish = fields.Bool(allow_none=True) @@ -449,8 +429,12 @@ class CronTriggerSchema(BaseSchema): hour = fields.Str(allow_none=True) minute = fields.Str(allow_none=True) second = fields.Str(allow_none=True) - start_date = DateTime(allow_none=True, format="epoch", example="1500065932000") - end_date = DateTime(allow_none=True, format="epoch", example="1500065932000") + start_date = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) + end_date = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) timezone = fields.Str(allow_none=True) jitter = fields.Int(allow_none=True) @@ -509,7 +493,9 @@ class JobSchema(BaseSchema): request_template = fields.Nested("RequestTemplateSchema", allow_none=True) misfire_grace_time = fields.Int(allow_none=True) coalesce = fields.Bool(allow_none=True) - next_run_time = DateTime(allow_none=True, format="epoch", example="1500065932000") + next_run_time = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) success_count = fields.Int(allow_none=True) error_count = fields.Int(allow_none=True) canceled_count = fields.Int(allow_none=True) @@ -622,7 +608,9 @@ class TopicSchema(BaseSchema): class ReplicationSchema(BaseSchema): id = fields.Str(allow_none=True) replication_id = fields.Str(allow_none=True) - expires_at = DateTime(allow_none=True, format="epoch", example="1500065932000") + expires_at = fields.DateTime( + allow_none=True, format="timestamp", example="1500065932000" + ) class UserSchema(BaseSchema): From 621e13cbbc33f990beb2208e0f55f6fdd98d7068 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:20:51 -0500 Subject: [PATCH 05/50] Fixed Marshmallow --- brewtils/schema_parser.py | 8 +-- brewtils/schemas.py | 138 +++++++++++++++---------------------- setup.py | 2 +- test/schema_parser_test.py | 10 +-- test/schema_test.py | 27 +------- 5 files changed, 67 insertions(+), 118 deletions(-) diff --git a/brewtils/schema_parser.py b/brewtils/schema_parser.py index 86e7a22c..6853a700 100644 --- a/brewtils/schema_parser.py +++ b/brewtils/schema_parser.py @@ -370,9 +370,9 @@ def parse_job_ids(cls, job_id_json, from_string=False, **kwargs): schema = brewtils.schemas.JobExportInputSchema(**kwargs) if from_string: - return schema.loads(job_id_json).data + return schema.loads(job_id_json) else: - return schema.load(job_id_json).data + return schema.load(job_id_json) @classmethod def parse_garden(cls, garden, from_string=False, **kwargs): @@ -561,7 +561,7 @@ def parse( schema.context["models"] = cls._models - return schema.loads(data).data if from_string else schema.load(data).data + return schema.loads(data) if from_string else schema.load(data) # Serialization methods @classmethod @@ -1162,7 +1162,7 @@ def serialize( schema = getattr(brewtils.schemas, schema_name)(**kwargs) - return schema.dumps(model).data if to_string else schema.dump(model).data + return schema.dumps(model) if to_string else schema.dump(model) # Explicitly force to_string to False so only original call returns a string multiple = [ diff --git a/brewtils/schemas.py b/brewtils/schemas.py index 9df24335..cdd24c71 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -2,8 +2,6 @@ from functools import partial -import marshmallow -import simplejson from marshmallow import Schema, fields, post_load, pre_load from marshmallow_polyfield import PolyField @@ -92,18 +90,15 @@ def __init__(self, type_field="payload_type", allowed_types=None, **kwargs): class BaseSchema(Schema): - class Meta: - version_nums = marshmallow.__version__.split(".") - if int(version_nums[0]) <= 2 and int(version_nums[1]) < 17: # pragma: no cover - json_module = simplejson - else: - render_module = simplejson - - def __init__(self, strict=True, **kwargs): - super(BaseSchema, self).__init__(strict=strict, **kwargs) + # class Meta: + # version_nums = marshmallow.__version__.split(".") + # if int(version_nums[0]) <= 2 and int(version_nums[1]) < 17: # pragma: no cover + # json_module = simplejson + # else: + # render_module = simplejson @post_load - def make_object(self, data): + def make_object(self, data, **_): try: model_class = self.context["models"][self.__class__.__name__] except KeyError: @@ -123,8 +118,8 @@ def get_attribute_names(cls): class ChoicesSchema(BaseSchema): type = fields.Str(allow_none=True) display = fields.Str(allow_none=True) - value = fields.Raw(allow_none=True, many=True) - strict = fields.Bool(allow_none=True, default=False) + value = fields.List(fields.Raw, allow_none=True) + strict = fields.Bool(allow_none=True, dump_default=False) details = fields.Dict(allow_none=True) @@ -136,8 +131,8 @@ class ParameterSchema(BaseSchema): optional = fields.Bool(allow_none=True) default = fields.Raw(allow_none=True) description = fields.Str(allow_none=True) - choices = fields.Nested("ChoicesSchema", allow_none=True, many=False) - parameters = fields.Nested("self", many=True, allow_none=True) + choices = fields.Nested(lambda: ChoicesSchema, allow_none=True) + parameters = fields.List(fields.Nested(lambda: ParameterSchema), allow_none=True) nullable = fields.Bool(allow_none=True) maximum = fields.Int(allow_none=True) minimum = fields.Int(allow_none=True) @@ -149,7 +144,7 @@ class ParameterSchema(BaseSchema): class CommandSchema(BaseSchema): name = fields.Str(allow_none=True) description = fields.Str(allow_none=True) - parameters = fields.Nested("ParameterSchema", many=True) + parameters = fields.List(fields.Nested(lambda: ParameterSchema()), allow_none=True) command_type = fields.Str(allow_none=True) output_type = fields.Str(allow_none=True) schema = fields.Dict(allow_none=True) @@ -168,7 +163,7 @@ class InstanceSchema(BaseSchema): name = fields.Str(allow_none=True) description = fields.Str(allow_none=True) status = fields.Str(allow_none=True) - status_info = fields.Nested("StatusInfoSchema", allow_none=True) + status_info = fields.Nested(lambda: StatusInfoSchema(), allow_none=True) queue_type = fields.Str(allow_none=True) queue_info = fields.Dict(allow_none=True) icon_name = fields.Str(allow_none=True) @@ -182,8 +177,8 @@ class SystemSchema(BaseSchema): version = fields.Str(allow_none=True) max_instances = fields.Integer(allow_none=True) icon_name = fields.Str(allow_none=True) - instances = fields.Nested("InstanceSchema", many=True, allow_none=True) - commands = fields.Nested("CommandSchema", many=True, allow_none=True) + instances = fields.List(fields.Nested(lambda: InstanceSchema()), allow_none=True) + commands = fields.List(fields.Nested(lambda: CommandSchema()), allow_none=True) display_name = fields.Str(allow_none=True) metadata = fields.Dict(allow_none=True) namespace = fields.Str(allow_none=True) @@ -214,9 +209,7 @@ class FileSchema(BaseSchema): owner = fields.Raw(allow_none=True) job = fields.Nested("JobSchema", allow_none=True) request = fields.Nested("RequestSchema", allow_none=True) - updated_at = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) + updated_at = fields.DateTime(allow_none=True, format="timestamp_ms") file_name = fields.Str(allow_none=True) file_size = fields.Int(allow_none=False) chunks = fields.Dict(allow_none=True) @@ -277,23 +270,28 @@ class RequestTemplateSchema(BaseSchema): class RequestSchema(RequestTemplateSchema): id = fields.Str(allow_none=True) is_event = fields.Bool(allow_none=True) - parent = fields.Nested("self", exclude=("children",), allow_none=True) - children = fields.Nested( - "self", exclude=("parent", "children"), many=True, default=None, allow_none=True + parent = fields.Nested( + lambda: RequestSchema(exclude=("children",)), allow_none=True + ) + children = fields.List( + fields.Nested( + lambda: RequestSchema( + exclude=( + "parent", + "children", + ) + ) + ), + dump_default=None, + allow_none=True, ) output = fields.Str(allow_none=True) hidden = fields.Boolean(allow_none=True) status = fields.Str(allow_none=True) error_class = fields.Str(allow_none=True) - created_at = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) - updated_at = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) - status_updated_at = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) + created_at = fields.DateTime(allow_none=True, format="timestamp_ms") + updated_at = fields.DateTime(allow_none=True, format="timestamp_ms") + status_updated_at = fields.DateTime(allow_none=True, format="timestamp_ms") has_parent = fields.Bool(allow_none=True) requester = fields.String(allow_none=True) source_garden = fields.String(allow_none=True) @@ -301,17 +299,13 @@ class RequestSchema(RequestTemplateSchema): class StatusHistorySchema(BaseSchema): - heartbeat = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) + heartbeat = fields.DateTime(allow_none=True, format="timestamp_ms") status = fields.Str(allow_none=True) class StatusInfoSchema(BaseSchema): - heartbeat = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) - history = fields.Nested("StatusHistorySchema", many=True, allow_none=True) + heartbeat = fields.DateTime(allow_none=True, format="timestamp_ms") + history = fields.List(fields.Nested(lambda: StatusHistorySchema()), allow_none=True) class PatchSchema(BaseSchema): @@ -320,7 +314,7 @@ class PatchSchema(BaseSchema): value = fields.Raw(allow_none=True) @pre_load(pass_many=True) - def unwrap_envelope(self, data, many): + def unwrap_envelope(self, data, many, **_): """Helper function for parsing the different patch formats. This exists because previously multiple patches serialized like:: @@ -363,9 +357,7 @@ class EventSchema(BaseSchema): namespace = fields.Str(allow_none=True) garden = fields.Str(allow_none=True) metadata = fields.Dict(allow_none=True) - timestamp = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) + timestamp = fields.DateTime(allow_none=True, format="timestamp_ms") payload_type = fields.Str(allow_none=True) payload = ModelField(allow_none=True, type_field="payload_type") @@ -387,19 +379,13 @@ class QueueSchema(BaseSchema): class UserTokenSchema(BaseSchema): id = fields.Str(allow_none=True) uuid = fields.Str(allow_none=True) - issued_at = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) - expires_at = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) + issued_at = fields.DateTime(allow_none=True, format="timestamp_ms") + expires_at = fields.DateTime(allow_none=True, format="timestamp_ms") username = fields.Str(allow_none=True) class DateTriggerSchema(BaseSchema): - run_date = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) + run_date = fields.DateTime(allow_none=True, format="timestamp_ms") timezone = fields.Str(allow_none=True) @@ -409,12 +395,8 @@ class IntervalTriggerSchema(BaseSchema): hours = fields.Int(allow_none=True) minutes = fields.Int(allow_none=True) seconds = fields.Int(allow_none=True) - start_date = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) - end_date = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) + start_date = fields.DateTime(allow_none=True, format="timestamp_ms") + end_date = fields.DateTime(allow_none=True, format="timestamp_ms") timezone = fields.Str(allow_none=True) jitter = fields.Int(allow_none=True) reschedule_on_finish = fields.Bool(allow_none=True) @@ -429,12 +411,8 @@ class CronTriggerSchema(BaseSchema): hour = fields.Str(allow_none=True) minute = fields.Str(allow_none=True) second = fields.Str(allow_none=True) - start_date = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) - end_date = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) + start_date = fields.DateTime(allow_none=True, format="timestamp_ms") + end_date = fields.DateTime(allow_none=True, format="timestamp_ms") timezone = fields.Str(allow_none=True) jitter = fields.Int(allow_none=True) @@ -452,7 +430,7 @@ class FileTriggerSchema(BaseSchema): class ConnectionSchema(BaseSchema): api = fields.Str(allow_none=True) status = fields.Str(allow_none=True) - status_info = fields.Nested("StatusInfoSchema", allow_none=True) + status_info = fields.Nested(lambda: StatusInfoSchema(), allow_none=True) config = fields.Dict(allow_none=True) @@ -460,20 +438,20 @@ class GardenSchema(BaseSchema): id = fields.Str(allow_none=True) name = fields.Str(allow_none=True) status = fields.Str(allow_none=True) - status_info = fields.Nested("StatusInfoSchema", allow_none=True) + status_info = fields.Nested(lambda: StatusInfoSchema(), allow_none=True) connection_type = fields.Str(allow_none=True) - receiving_connections = fields.Nested( - "ConnectionSchema", many=True, allow_none=True + receiving_connections = fields.List( + fields.Nested(lambda: ConnectionSchema()), allow_none=True ) - publishing_connections = fields.Nested( - "ConnectionSchema", many=True, allow_none=True + publishing_connections = fields.List( + fields.Nested(lambda: ConnectionSchema()), allow_none=True ) namespaces = fields.List(fields.Str(), allow_none=True) - systems = fields.Nested("SystemSchema", many=True, allow_none=True) + systems = fields.List(fields.Nested(lambda: SystemSchema()), allow_none=True) has_parent = fields.Bool(allow_none=True) parent = fields.Str(allow_none=True) - children = fields.Nested( - "self", exclude=("parent"), many=True, default=None, allow_none=True + children = fields.List( + fields.Nested(lambda: GardenSchema(exclude=("parent",))), allow_none=True ) metadata = fields.Dict(allow_none=True) default_user = fields.Str(allow_none=True) @@ -493,9 +471,7 @@ class JobSchema(BaseSchema): request_template = fields.Nested("RequestTemplateSchema", allow_none=True) misfire_grace_time = fields.Int(allow_none=True) coalesce = fields.Bool(allow_none=True) - next_run_time = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) + next_run_time = fields.DateTime(allow_none=True, format="timestamp_ms") success_count = fields.Int(allow_none=True) error_count = fields.Int(allow_none=True) canceled_count = fields.Int(allow_none=True) @@ -608,9 +584,7 @@ class TopicSchema(BaseSchema): class ReplicationSchema(BaseSchema): id = fields.Str(allow_none=True) replication_id = fields.Str(allow_none=True) - expires_at = fields.DateTime( - allow_none=True, format="timestamp", example="1500065932000" - ) + expires_at = fields.DateTime(allow_none=True, format="timestamp_ms") class UserSchema(BaseSchema): diff --git a/setup.py b/setup.py index bb425e10..20292885 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_version(): install_requires=[ "appdirs<2", "lark-parser<1", - "marshmallow<4", + "marshmallow<4,>=3.3", "marshmallow-polyfield<6", "packaging", "pika<=1.4,>=1.0.1", diff --git a/test/schema_parser_test.py b/test/schema_parser_test.py index d49652ee..28e200b9 100644 --- a/test/schema_parser_test.py +++ b/test/schema_parser_test.py @@ -60,11 +60,11 @@ def test_error(self, data, kwargs, error): with pytest.raises(error): SchemaParser.parse_system(data, **kwargs) - def test_non_strict_failure(self, system_dict): - system_dict["name"] = 1234 - value = SchemaParser.parse_system(system_dict, from_string=False, strict=False) - assert value.get("name") is None - assert value["version"] == system_dict["version"] + # def test_non_strict_failure(self, system_dict): + # system_dict["name"] = 1234 + # value = SchemaParser.parse_system(system_dict, from_string=False, strict=False) + # assert value.get("name") is None + # assert value["version"] == system_dict["version"] def test_no_modify(self, system_dict): system_copy = copy.deepcopy(system_dict) diff --git a/test/schema_test.py b/test/schema_test.py index da25bd61..b95f0a6b 100644 --- a/test/schema_test.py +++ b/test/schema_test.py @@ -6,17 +6,15 @@ from pytest_lazyfixture import lazy_fixture from brewtils.models import System +from brewtils.schema_parser import SchemaParser from brewtils.schemas import ( BaseSchema, - DateTime, SystemSchema, _deserialize_model, _serialize_model, model_schema_map, ) -from brewtils.schema_parser import SchemaParser - class TestSchemas(object): def test_make_object(self): @@ -36,29 +34,6 @@ def test_get_attributes(self): class TestFields(object): - @pytest.mark.parametrize( - "dt,localtime,expected", - [ - (lazy_fixture("ts_dt"), False, lazy_fixture("ts_epoch")), - (lazy_fixture("ts_dt"), True, lazy_fixture("ts_epoch")), - (lazy_fixture("ts_dt_eastern"), False, lazy_fixture("ts_epoch_eastern")), - (lazy_fixture("ts_dt_eastern"), True, lazy_fixture("ts_epoch")), - (lazy_fixture("ts_epoch"), False, lazy_fixture("ts_epoch")), - (lazy_fixture("ts_epoch"), True, lazy_fixture("ts_epoch")), - ], - ) - def test_to_epoch(self, dt, localtime, expected): - assert DateTime.to_epoch(dt, localtime) == expected - - @pytest.mark.parametrize( - "epoch,expected", - [ - (lazy_fixture("ts_epoch"), lazy_fixture("ts_dt")), - (lazy_fixture("ts_dt"), lazy_fixture("ts_dt")), - ], - ) - def test_from_epoch(self, epoch, expected): - assert DateTime.from_epoch(epoch) == expected def test_modelfield_serialize_invalid_type(self): with pytest.raises(TypeError): From f0e4b6d25cb93a2287ecacf4f289cea13ebf7a11 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:21:36 -0500 Subject: [PATCH 06/50] Remove Metadata for legacy marshmallow schemas --- brewtils/schemas.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/brewtils/schemas.py b/brewtils/schemas.py index cdd24c71..18dc74cf 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -90,12 +90,6 @@ def __init__(self, type_field="payload_type", allowed_types=None, **kwargs): class BaseSchema(Schema): - # class Meta: - # version_nums = marshmallow.__version__.split(".") - # if int(version_nums[0]) <= 2 and int(version_nums[1]) < 17: # pragma: no cover - # json_module = simplejson - # else: - # render_module = simplejson @post_load def make_object(self, data, **_): From 35698fd466647d7fbee873ed06270452279b2bc0 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:23:53 -0500 Subject: [PATCH 07/50] removed legacy test --- test/schema_parser_test.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/schema_parser_test.py b/test/schema_parser_test.py index 28e200b9..92c44190 100644 --- a/test/schema_parser_test.py +++ b/test/schema_parser_test.py @@ -60,12 +60,6 @@ def test_error(self, data, kwargs, error): with pytest.raises(error): SchemaParser.parse_system(data, **kwargs) - # def test_non_strict_failure(self, system_dict): - # system_dict["name"] = 1234 - # value = SchemaParser.parse_system(system_dict, from_string=False, strict=False) - # assert value.get("name") is None - # assert value["version"] == system_dict["version"] - def test_no_modify(self, system_dict): system_copy = copy.deepcopy(system_dict) SchemaParser().parse_system(system_dict) From 957bf8f7b6d5e1e9851a3ca93d55379e36ce68be Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:09:14 -0500 Subject: [PATCH 08/50] Removing pytz dependency --- brewtils/models.py | 23 ++++++++++++++--------- brewtils/test/fixtures.py | 10 +++++----- setup.py | 1 - test/models_test.py | 28 +++++++++++++++++++--------- 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/brewtils/models.py b/brewtils/models.py index d0b967fa..a2c313f4 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -3,8 +3,8 @@ import copy from datetime import datetime from enum import Enum +from zoneinfo import ZoneInfo -import pytz # noqa # not in requirements file import six # noqa # not in requirements file from brewtils.errors import ModelError, _deprecate @@ -1273,9 +1273,10 @@ def scheduler_attributes(self): @property def scheduler_kwargs(self): - tz = pytz.timezone(self.timezone) - return {"timezone": tz, "run_date": tz.localize(self.run_date)} + tz = ZoneInfo(self.timezone.upper()) + + return {"timezone": tz, "run_date": self.run_date.replace(tzinfo=tz)} class IntervalTrigger(BaseModel): @@ -1332,14 +1333,16 @@ def scheduler_attributes(self): @property def scheduler_kwargs(self): - tz = pytz.timezone(self.timezone) + tz = ZoneInfo(self.timezone.upper()) kwargs = {key: getattr(self, key) for key in self.scheduler_attributes} kwargs.update( { "timezone": tz, - "start_date": tz.localize(self.start_date) if self.start_date else None, - "end_date": tz.localize(self.end_date) if self.end_date else None, + "start_date": ( + self.start_date.replace(tzinfo=tz) if self.start_date else None + ), + "end_date": self.end_date.replace(tzinfo=tz) if self.end_date else None, } ) @@ -1408,14 +1411,16 @@ def scheduler_attributes(self): @property def scheduler_kwargs(self): - tz = pytz.timezone(self.timezone) + tz = ZoneInfo(self.timezone.upper()) kwargs = {key: getattr(self, key) for key in self.scheduler_attributes} kwargs.update( { "timezone": tz, - "start_date": tz.localize(self.start_date) if self.start_date else None, - "end_date": tz.localize(self.end_date) if self.end_date else None, + "start_date": ( + self.start_date.replace(tzinfo=tz) if self.start_date else None + ), + "end_date": self.end_date.replace(tzinfo=tz) if self.end_date else None, } ) diff --git a/brewtils/test/fixtures.py b/brewtils/test/fixtures.py index 367335e7..4ff15853 100644 --- a/brewtils/test/fixtures.py +++ b/brewtils/test/fixtures.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- import copy -from datetime import datetime +from datetime import datetime, timezone +from zoneinfo import ZoneInfo import pytest -import pytz from brewtils.models import ( AliasUserMap, @@ -62,7 +62,7 @@ def ts_epoch(): @pytest.fixture def ts_dt_utc(ts_epoch): """Jan 1, 2016 UTC as timezone-aware datetime.""" - return datetime.fromtimestamp(ts_epoch / 1000, tz=pytz.utc) + return datetime.fromtimestamp(ts_epoch / 1000, tz=timezone.utc) @pytest.fixture @@ -74,7 +74,7 @@ def ts_epoch_eastern(): @pytest.fixture def ts_dt_eastern(): """Jan 1, 2016 US/Eastern as timezone-aware datetime.""" - return datetime(2016, 1, 1, tzinfo=pytz.timezone("US/Eastern")) + return datetime(2016, 1, 1, tzinfo=ZoneInfo("US/Eastern")) @pytest.fixture @@ -92,7 +92,7 @@ def ts_2_epoch(): @pytest.fixture def ts_2_dt_utc(ts_2_epoch): """Feb 2, 2017 UTC as timezone-aware datetime.""" - return datetime.fromtimestamp(ts_2_epoch / 1000, tz=pytz.utc) + return datetime.fromtimestamp(ts_2_epoch / 1000, tz=timezone.utc) @pytest.fixture diff --git a/setup.py b/setup.py index 20292885..9fb6dc87 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,6 @@ def find_version(): "marshmallow-polyfield<6", "packaging", "pika<=1.4,>=1.0.1", - "pytz", "requests<3", "simplejson<4", "six<2", diff --git a/test/models_test.py b/test/models_test.py index 8bfa5443..cab6a494 100644 --- a/test/models_test.py +++ b/test/models_test.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- import warnings +from datetime import timezone +from zoneinfo import ZoneInfo import pytest -import pytz +from pytest_lazyfixture import lazy_fixture + from brewtils.errors import ModelError from brewtils.models import ( Choices, @@ -13,17 +16,16 @@ LoggingConfig, Parameter, PatchOperation, - User, Queue, Request, RequestFile, RequestTemplate, Role, - Subscriber, StatusInfo, + Subscriber, Topic, + User, ) -from pytest_lazyfixture import lazy_fixture @pytest.fixture @@ -567,7 +569,7 @@ def test_repr(self, role): class TestDateTrigger(object): def test_scheduler_kwargs(self, bg_date_trigger, ts_dt_utc): assert bg_date_trigger.scheduler_kwargs == { - "timezone": pytz.utc, + "timezone": ZoneInfo("UTC"), "run_date": ts_dt_utc, } @@ -595,7 +597,7 @@ def test_scheduler_kwargs_default(self): "seconds": None, "start_date": None, "end_date": None, - "timezone": pytz.utc, + "timezone": ZoneInfo("UTC"), "jitter": None, "reschedule_on_finish": None, } @@ -605,7 +607,11 @@ def test_scheduler_kwargs( ): expected = interval_trigger_dict expected.update( - {"timezone": pytz.utc, "start_date": ts_dt_utc, "end_date": ts_2_dt_utc} + { + "timezone": ZoneInfo("UTC"), + "start_date": ts_dt_utc, + "end_date": ts_2_dt_utc, + } ) assert bg_interval_trigger.scheduler_kwargs == expected @@ -623,7 +629,7 @@ def test_scheduler_kwargs_default(self): "second": None, "start_date": None, "end_date": None, - "timezone": pytz.utc, + "timezone": ZoneInfo("UTC"), "jitter": None, } @@ -632,7 +638,11 @@ def test_scheduler_kwargs( ): expected = cron_trigger_dict expected.update( - {"timezone": pytz.utc, "start_date": ts_dt_utc, "end_date": ts_2_dt_utc} + { + "timezone": ZoneInfo("UTC"), + "start_date": ts_dt_utc, + "end_date": ts_2_dt_utc, + } ) assert bg_cron_trigger.scheduler_kwargs == expected From 284a0bf99cb9df3e5e454d69e7bd042494abcc37 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:14:44 -0500 Subject: [PATCH 09/50] replaced utcnow() function --- brewtils/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brewtils/models.py b/brewtils/models.py index a2c313f4..2603dc3d 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -466,7 +466,7 @@ def __init__(self, heartbeat=None, history=None): def set_status_heartbeat(self, status, max_history=None): - self.heartbeat = datetime.utcnow() + self.heartbeat = datetime.now(datetime.timezone.utc) self.history.append( StatusHistory(status=copy.deepcopy(status), heartbeat=self.heartbeat) ) From 53c7b333df31d3cacd3b3124c54848cf813f7d63 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Tue, 5 Nov 2024 06:22:37 -0500 Subject: [PATCH 10/50] Fixed now utc --- brewtils/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brewtils/models.py b/brewtils/models.py index 2603dc3d..46a11c48 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import copy -from datetime import datetime +from datetime import datetime, timezone from enum import Enum from zoneinfo import ZoneInfo @@ -466,7 +466,7 @@ def __init__(self, heartbeat=None, history=None): def set_status_heartbeat(self, status, max_history=None): - self.heartbeat = datetime.now(datetime.timezone.utc) + self.heartbeat = datetime.now(timezone.utc) self.history.append( StatusHistory(status=copy.deepcopy(status), heartbeat=self.heartbeat) ) From abae4ab21dcf49cdc92a9a3562246bb4bc7147b7 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Tue, 5 Nov 2024 12:55:11 -0500 Subject: [PATCH 11/50] Raw fields as many --- brewtils/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brewtils/schemas.py b/brewtils/schemas.py index 18dc74cf..101f0678 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -112,7 +112,7 @@ def get_attribute_names(cls): class ChoicesSchema(BaseSchema): type = fields.Str(allow_none=True) display = fields.Str(allow_none=True) - value = fields.List(fields.Raw, allow_none=True) + value = fields.Raw(allow_none=True, many=True) strict = fields.Bool(allow_none=True, dump_default=False) details = fields.Dict(allow_none=True) From 91d720bec35250d6cc1e41539489d64a98a03dcb Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Wed, 6 Nov 2024 17:14:33 +0000 Subject: [PATCH 12/50] Reverted to legacy datatime object --- brewtils/schemas.py | 62 ++++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/brewtils/schemas.py b/brewtils/schemas.py index 101f0678..a30a07fe 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -2,7 +2,7 @@ from functools import partial -from marshmallow import Schema, fields, post_load, pre_load +from marshmallow import Schema, fields, post_load, pre_load, utils from marshmallow_polyfield import PolyField __all__ = [ @@ -89,6 +89,34 @@ def __init__(self, type_field="payload_type", allowed_types=None, **kwargs): ) +class DateTime(fields.DateTime): + """Class that adds methods for (de)serializing DateTime fields as an epoch + + This is required for going from Mongo Model objects to Marshmallow model Objects + """ + + def __init__(self, format="epoch", **kwargs): + self.SERIALIZATION_FUNCS["epoch"] = self.to_epoch + self.DESERIALIZATION_FUNCS["epoch"] = self.from_epoch + super(DateTime, self).__init__(format=format, **kwargs) + + @staticmethod + def to_epoch(value): + # If already in epoch form just return it + if isinstance(value, int): + return value + + return utils.timestamp_ms(value) + + @staticmethod + def from_epoch(value): + # If already in datetime form just return it + if isinstance(value, datetime.datetime): + return value + + return utils.from_timestamp_ms(value) + + class BaseSchema(Schema): @post_load @@ -203,7 +231,7 @@ class FileSchema(BaseSchema): owner = fields.Raw(allow_none=True) job = fields.Nested("JobSchema", allow_none=True) request = fields.Nested("RequestSchema", allow_none=True) - updated_at = fields.DateTime(allow_none=True, format="timestamp_ms") + updated_at = DateTime(allow_none=True, format="epoch") file_name = fields.Str(allow_none=True) file_size = fields.Int(allow_none=False) chunks = fields.Dict(allow_none=True) @@ -283,9 +311,9 @@ class RequestSchema(RequestTemplateSchema): hidden = fields.Boolean(allow_none=True) status = fields.Str(allow_none=True) error_class = fields.Str(allow_none=True) - created_at = fields.DateTime(allow_none=True, format="timestamp_ms") - updated_at = fields.DateTime(allow_none=True, format="timestamp_ms") - status_updated_at = fields.DateTime(allow_none=True, format="timestamp_ms") + created_at = DateTime(allow_none=True, format="epoch") + updated_at = DateTime(allow_none=True, format="epoch") + status_updated_at = DateTime(allow_none=True, format="epoch") has_parent = fields.Bool(allow_none=True) requester = fields.String(allow_none=True) source_garden = fields.String(allow_none=True) @@ -293,12 +321,12 @@ class RequestSchema(RequestTemplateSchema): class StatusHistorySchema(BaseSchema): - heartbeat = fields.DateTime(allow_none=True, format="timestamp_ms") + heartbeat = DateTime(allow_none=True, format="epoch") status = fields.Str(allow_none=True) class StatusInfoSchema(BaseSchema): - heartbeat = fields.DateTime(allow_none=True, format="timestamp_ms") + heartbeat = DateTime(allow_none=True, format="epoch") history = fields.List(fields.Nested(lambda: StatusHistorySchema()), allow_none=True) @@ -351,7 +379,7 @@ class EventSchema(BaseSchema): namespace = fields.Str(allow_none=True) garden = fields.Str(allow_none=True) metadata = fields.Dict(allow_none=True) - timestamp = fields.DateTime(allow_none=True, format="timestamp_ms") + timestamp = DateTime(allow_none=True, format="epoch") payload_type = fields.Str(allow_none=True) payload = ModelField(allow_none=True, type_field="payload_type") @@ -373,13 +401,13 @@ class QueueSchema(BaseSchema): class UserTokenSchema(BaseSchema): id = fields.Str(allow_none=True) uuid = fields.Str(allow_none=True) - issued_at = fields.DateTime(allow_none=True, format="timestamp_ms") - expires_at = fields.DateTime(allow_none=True, format="timestamp_ms") + issued_at = DateTime(allow_none=True, format="epoch") + expires_at = DateTime(allow_none=True, format="epoch") username = fields.Str(allow_none=True) class DateTriggerSchema(BaseSchema): - run_date = fields.DateTime(allow_none=True, format="timestamp_ms") + run_date = DateTime(allow_none=True, format="epoch") timezone = fields.Str(allow_none=True) @@ -389,8 +417,8 @@ class IntervalTriggerSchema(BaseSchema): hours = fields.Int(allow_none=True) minutes = fields.Int(allow_none=True) seconds = fields.Int(allow_none=True) - start_date = fields.DateTime(allow_none=True, format="timestamp_ms") - end_date = fields.DateTime(allow_none=True, format="timestamp_ms") + start_date = DateTime(allow_none=True, format="epoch") + end_date = DateTime(allow_none=True, format="epoch") timezone = fields.Str(allow_none=True) jitter = fields.Int(allow_none=True) reschedule_on_finish = fields.Bool(allow_none=True) @@ -405,8 +433,8 @@ class CronTriggerSchema(BaseSchema): hour = fields.Str(allow_none=True) minute = fields.Str(allow_none=True) second = fields.Str(allow_none=True) - start_date = fields.DateTime(allow_none=True, format="timestamp_ms") - end_date = fields.DateTime(allow_none=True, format="timestamp_ms") + start_date = DateTime(allow_none=True, format="epoch") + end_date = DateTime(allow_none=True, format="epoch") timezone = fields.Str(allow_none=True) jitter = fields.Int(allow_none=True) @@ -465,7 +493,7 @@ class JobSchema(BaseSchema): request_template = fields.Nested("RequestTemplateSchema", allow_none=True) misfire_grace_time = fields.Int(allow_none=True) coalesce = fields.Bool(allow_none=True) - next_run_time = fields.DateTime(allow_none=True, format="timestamp_ms") + next_run_time = DateTime(allow_none=True, format="epoch") success_count = fields.Int(allow_none=True) error_count = fields.Int(allow_none=True) canceled_count = fields.Int(allow_none=True) @@ -578,7 +606,7 @@ class TopicSchema(BaseSchema): class ReplicationSchema(BaseSchema): id = fields.Str(allow_none=True) replication_id = fields.Str(allow_none=True) - expires_at = fields.DateTime(allow_none=True, format="timestamp_ms") + expires_at = DateTime(allow_none=True, format="epoch") class UserSchema(BaseSchema): From b8d7e69d31daf85ad2a97f19dffcd6efbbe31e02 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Wed, 6 Nov 2024 17:16:06 +0000 Subject: [PATCH 13/50] fixed imports --- brewtils/schemas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/brewtils/schemas.py b/brewtils/schemas.py index a30a07fe..0b1116c0 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import datetime from functools import partial from marshmallow import Schema, fields, post_load, pre_load, utils From 80e0116fb504565fa8db3a2811e1cf9468b53a2b Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Wed, 6 Nov 2024 17:27:35 +0000 Subject: [PATCH 14/50] add float support for epoch --- brewtils/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brewtils/schemas.py b/brewtils/schemas.py index 0b1116c0..52999a4e 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -104,7 +104,7 @@ def __init__(self, format="epoch", **kwargs): @staticmethod def to_epoch(value): # If already in epoch form just return it - if isinstance(value, int): + if isinstance(value, int) or isinstance(value, float): return value return utils.timestamp_ms(value) From fae7b8cb2d5e3c8582134d24061faa2cc92a99ae Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Wed, 13 Nov 2024 08:15:46 -0500 Subject: [PATCH 15/50] Set Timezone --- brewtils/schemas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brewtils/schemas.py b/brewtils/schemas.py index 52999a4e..b0defb53 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -113,9 +113,9 @@ def to_epoch(value): def from_epoch(value): # If already in datetime form just return it if isinstance(value, datetime.datetime): - return value + return value.replace(tzinfo=datetime.timezone.utc) - return utils.from_timestamp_ms(value) + return utils.from_timestamp_ms(value).replace(tzinfo=datetime.timezone.utc) class BaseSchema(Schema): From cc5f8c51c92a4b24f1515f570538efbae51c5057 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Wed, 13 Nov 2024 08:29:04 -0500 Subject: [PATCH 16/50] Adding to_epoch support for timezones --- brewtils/schemas.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/brewtils/schemas.py b/brewtils/schemas.py index b0defb53..0c1237a5 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -107,13 +107,18 @@ def to_epoch(value): if isinstance(value, int) or isinstance(value, float): return value + if value.tzinfo is not None and value.tzinfo is not datetime.timezone.utc: + value = value.replace(tzinfo=datetime.timezone.utc) + return utils.timestamp_ms(value) @staticmethod def from_epoch(value): # If already in datetime form just return it if isinstance(value, datetime.datetime): - return value.replace(tzinfo=datetime.timezone.utc) + if value.tzinfo is None: + return value.replace(tzinfo=datetime.timezone.utc) + return value return utils.from_timestamp_ms(value).replace(tzinfo=datetime.timezone.utc) From 5e83310f4edd63ee8705307b3489da044816f35d Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:22:32 -0500 Subject: [PATCH 17/50] Update fixtures.py --- brewtils/test/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brewtils/test/fixtures.py b/brewtils/test/fixtures.py index 4ff15853..bfeb7153 100644 --- a/brewtils/test/fixtures.py +++ b/brewtils/test/fixtures.py @@ -50,7 +50,7 @@ def system_id(): @pytest.fixture def ts_dt(): """Jan 1, 2016 as a naive datetime.""" - return datetime(2016, 1, 1) + return datetime.fromtimestamp(ts_epoch / 1000, tz=timezone.utc) @pytest.fixture From 4b181472e558cd8db09e4218b47e9f18b3d7f8ce Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:28:17 -0500 Subject: [PATCH 18/50] Update fixtures.py --- brewtils/test/fixtures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brewtils/test/fixtures.py b/brewtils/test/fixtures.py index bfeb7153..b91a3a15 100644 --- a/brewtils/test/fixtures.py +++ b/brewtils/test/fixtures.py @@ -49,8 +49,8 @@ def system_id(): @pytest.fixture def ts_dt(): - """Jan 1, 2016 as a naive datetime.""" - return datetime.fromtimestamp(ts_epoch / 1000, tz=timezone.utc) + """Jan 1, 2016 as a UTC timezone-aware datetime.""" + return datetime(2016, 1, 1, tzinfo=timezone.utc) @pytest.fixture From 3b8730929727f3aca5441517237e7f84283d6509 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 22 Nov 2024 09:51:47 -0500 Subject: [PATCH 19/50] Updating Timezone Defaults --- brewtils/models.py | 6 +++--- brewtils/test/fixtures.py | 14 ++++---------- test/models_test.py | 2 +- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/brewtils/models.py b/brewtils/models.py index 46a11c48..9eb7e257 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -1257,7 +1257,7 @@ def __repr__(self): class DateTrigger(BaseModel): schema = "DateTriggerSchema" - def __init__(self, run_date=None, timezone=None): + def __init__(self, run_date=None, timezone="UTC"): self.run_date = run_date self.timezone = timezone @@ -1291,7 +1291,7 @@ def __init__( seconds=None, start_date=None, end_date=None, - timezone=None, + timezone="UTC", jitter=None, reschedule_on_finish=None, ): @@ -1364,7 +1364,7 @@ def __init__( second=None, start_date=None, end_date=None, - timezone=None, + timezone="UTC", jitter=None, ): self.year = year diff --git a/brewtils/test/fixtures.py b/brewtils/test/fixtures.py index b91a3a15..266d01de 100644 --- a/brewtils/test/fixtures.py +++ b/brewtils/test/fixtures.py @@ -77,12 +77,6 @@ def ts_dt_eastern(): return datetime(2016, 1, 1, tzinfo=ZoneInfo("US/Eastern")) -@pytest.fixture -def ts_2_dt(ts_2_epoch): - """Feb 2, 2017 as a naive datetime.""" - return datetime(2017, 2, 2) - - @pytest.fixture def ts_2_epoch(): """Feb 2, 2017 UTC as epoch milliseconds.""" @@ -814,11 +808,11 @@ def interval_trigger_dict(ts_epoch, ts_2_epoch): @pytest.fixture -def bg_interval_trigger(interval_trigger_dict, ts_dt, ts_2_dt): +def bg_interval_trigger(interval_trigger_dict, ts_dt, ts_2_dt_utc): """An interval trigger as a model.""" dict_copy = copy.deepcopy(interval_trigger_dict) dict_copy["start_date"] = ts_dt - dict_copy["end_date"] = ts_2_dt + dict_copy["end_date"] = ts_2_dt_utc return IntervalTrigger(**dict_copy) @@ -848,11 +842,11 @@ def cron_trigger_dict(ts_epoch, ts_2_epoch): @pytest.fixture -def bg_cron_trigger(cron_trigger_dict, ts_dt, ts_2_dt): +def bg_cron_trigger(cron_trigger_dict, ts_dt, ts_2_dt_utc): """A cron trigger as a model.""" dict_copy = copy.deepcopy(cron_trigger_dict) dict_copy["start_date"] = ts_dt - dict_copy["end_date"] = ts_2_dt + dict_copy["end_date"] = ts_2_dt_utc return CronTrigger(**dict_copy) diff --git a/test/models_test.py b/test/models_test.py index cab6a494..330ba3e9 100644 --- a/test/models_test.py +++ b/test/models_test.py @@ -508,7 +508,7 @@ def test_str(self, bg_event): def test_repr(self, bg_event, bg_request): assert ( repr(bg_event) == "" % bg_request ) From 07cfdca4ae2aa9adc57ab99e384fb29f883b336d Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 22 Nov 2024 09:55:02 -0500 Subject: [PATCH 20/50] Updating Testing --- test/models_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/models_test.py b/test/models_test.py index 330ba3e9..a0694778 100644 --- a/test/models_test.py +++ b/test/models_test.py @@ -657,8 +657,8 @@ def test_scheduler_kwargs( ), ( lazy_fixture("bg_date_trigger"), - "", - "", + "", + "", ), ( lazy_fixture("bg_file_trigger"), From 8161380bd8c9445b362f6bbede9743de44a7a41e Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Tue, 3 Dec 2024 06:41:27 -0500 Subject: [PATCH 21/50] Adding strict feature for parsing --- brewtils/schema_parser.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/brewtils/schema_parser.py b/brewtils/schema_parser.py index 6853a700..9d52039b 100644 --- a/brewtils/schema_parser.py +++ b/brewtils/schema_parser.py @@ -6,6 +6,7 @@ import six # type: ignore from box import Box # type: ignore +from marshmallow import ValidationError import brewtils.models import brewtils.schemas @@ -527,6 +528,7 @@ def parse( data, # type: Optional[Union[str, Dict[str, Any]]] model_class, # type: Any from_string=False, # type: bool + strict=True, # type: bool **kwargs # type: Any ): # type: (...) -> Union[str, Dict[str, Any]] """Convert a JSON string or dictionary into a model object @@ -535,6 +537,7 @@ def parse( data: The raw input model_class: Class object of the desired model type from_string: True if input is a JSON string, False if a dictionary + strict: False if parsing should return back valid sections **kwargs: Additional parameters to be passed to the Schema (e.g. many=True) Returns: @@ -561,7 +564,13 @@ def parse( schema.context["models"] = cls._models - return schema.loads(data) if from_string else schema.load(data) + try: + return schema.loads(data) if from_string else schema.load(data) + except ValidationError as err: + if strict: + raise err + cls.logger.error(err.messages) + return err.valid_data # Serialization methods @classmethod From 4edbb37c125b1b6c36157f3abf1112d035cfd3e8 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:20:44 -0500 Subject: [PATCH 22/50] load valid data --- brewtils/schema_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brewtils/schema_parser.py b/brewtils/schema_parser.py index 9d52039b..d7b3aede 100644 --- a/brewtils/schema_parser.py +++ b/brewtils/schema_parser.py @@ -570,7 +570,7 @@ def parse( if strict: raise err cls.logger.error(err.messages) - return err.valid_data + return schema.load(err.valid_data) # Serialization methods @classmethod From 353148206eb2f82f0fd4ae372fc0f323c1e12dd4 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:34:39 -0500 Subject: [PATCH 23/50] return valid data --- brewtils/schema_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brewtils/schema_parser.py b/brewtils/schema_parser.py index d7b3aede..9d52039b 100644 --- a/brewtils/schema_parser.py +++ b/brewtils/schema_parser.py @@ -570,7 +570,7 @@ def parse( if strict: raise err cls.logger.error(err.messages) - return schema.load(err.valid_data) + return err.valid_data # Serialization methods @classmethod From 17875b0f6b12fb42faff462591a19b00e34640e0 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:37:42 -0500 Subject: [PATCH 24/50] logging --- brewtils/schema_parser.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/brewtils/schema_parser.py b/brewtils/schema_parser.py index 9d52039b..98710701 100644 --- a/brewtils/schema_parser.py +++ b/brewtils/schema_parser.py @@ -570,7 +570,13 @@ def parse( if strict: raise err cls.logger.error(err.messages) - return err.valid_data + try: + cls.logger.error("LOAD") + return schema.load(err.valid_data) + except: + cls.logger.error("LOADS") + return schema.loads(err.valid_data) + # Serialization methods @classmethod From fa87a9b859736445cdfdec0ab5a3fe2940c43cc2 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:58:56 -0500 Subject: [PATCH 25/50] exclude unknown fields --- brewtils/schema_parser.py | 17 ++--------------- brewtils/schemas.py | 3 +++ 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/brewtils/schema_parser.py b/brewtils/schema_parser.py index 98710701..3c42607b 100644 --- a/brewtils/schema_parser.py +++ b/brewtils/schema_parser.py @@ -528,7 +528,6 @@ def parse( data, # type: Optional[Union[str, Dict[str, Any]]] model_class, # type: Any from_string=False, # type: bool - strict=True, # type: bool **kwargs # type: Any ): # type: (...) -> Union[str, Dict[str, Any]] """Convert a JSON string or dictionary into a model object @@ -537,7 +536,6 @@ def parse( data: The raw input model_class: Class object of the desired model type from_string: True if input is a JSON string, False if a dictionary - strict: False if parsing should return back valid sections **kwargs: Additional parameters to be passed to the Schema (e.g. many=True) Returns: @@ -563,19 +561,8 @@ def parse( schema = getattr(brewtils.schemas, model_class.schema)(**kwargs) schema.context["models"] = cls._models - - try: - return schema.loads(data) if from_string else schema.load(data) - except ValidationError as err: - if strict: - raise err - cls.logger.error(err.messages) - try: - cls.logger.error("LOAD") - return schema.load(err.valid_data) - except: - cls.logger.error("LOADS") - return schema.loads(err.valid_data) + + return schema.loads(data) if from_string else schema.load(data) # Serialization methods diff --git a/brewtils/schemas.py b/brewtils/schemas.py index 0c1237a5..794b8d23 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -142,6 +142,9 @@ def get_attribute_names(cls): if isinstance(value, fields.FieldABC) ] + class Meta: + unknown = 'EXCLUDE' + class ChoicesSchema(BaseSchema): type = fields.Str(allow_none=True) From dbe77f487ddaf9aebf753c71094270e933e95a54 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:05:29 -0500 Subject: [PATCH 26/50] exclude unknown fields --- brewtils/schemas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brewtils/schemas.py b/brewtils/schemas.py index 794b8d23..d9cceecf 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -3,7 +3,7 @@ import datetime from functools import partial -from marshmallow import Schema, fields, post_load, pre_load, utils +from marshmallow import Schema, fields, post_load, pre_load, utils, EXCLUDE from marshmallow_polyfield import PolyField __all__ = [ @@ -143,7 +143,7 @@ def get_attribute_names(cls): ] class Meta: - unknown = 'EXCLUDE' + unknown = EXCLUDE class ChoicesSchema(BaseSchema): From 7d074e1564089575a7a3a52a2a7135b8de420cdf Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Wed, 11 Dec 2024 09:20:21 -0500 Subject: [PATCH 27/50] formatting --- brewtils/schema_parser.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/brewtils/schema_parser.py b/brewtils/schema_parser.py index 3c42607b..6853a700 100644 --- a/brewtils/schema_parser.py +++ b/brewtils/schema_parser.py @@ -6,7 +6,6 @@ import six # type: ignore from box import Box # type: ignore -from marshmallow import ValidationError import brewtils.models import brewtils.schemas @@ -561,9 +560,8 @@ def parse( schema = getattr(brewtils.schemas, model_class.schema)(**kwargs) schema.context["models"] = cls._models - + return schema.loads(data) if from_string else schema.load(data) - # Serialization methods @classmethod From a88bffd7d049dd1385e710dfb56f031af86da6d9 Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:19:18 +0000 Subject: [PATCH 28/50] Adding deprecated to command and parameter decorators --- CHANGELOG.rst | 6 ++++++ brewtils/decorators.py | 45 +++++++++++++++++++++++++++++++++++++++ brewtils/models.py | 4 ++++ brewtils/schemas.py | 2 ++ brewtils/test/fixtures.py | 3 +++ 5 files changed, 60 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d152f730..2e7746ac 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Brewtils Changelog ================== +3.29.2 +------ +TBD + +- Added deprecated decorator and deprecated kwarg to command and parameter decorators + 3.29.1 ------ 12/31/24 diff --git a/brewtils/decorators.py b/brewtils/decorators.py index a395893f..34305fe1 100644 --- a/brewtils/decorators.py +++ b/brewtils/decorators.py @@ -14,6 +14,10 @@ from brewtils.errors import PluginParamError, _deprecate from brewtils.models import Command, Parameter, Resolvable +import logging + +logger = logging.getLogger(__name__) + if sys.version_info.major == 2: from funcsigs import Parameter as InspectParameter # noqa from funcsigs import signature @@ -125,6 +129,7 @@ def command( tag=None, # type: str tags=None, # type: Optional[List[str]] allow_any_kwargs=None, # type: Optional[bool] + deprecated=None, # type: Optional[bool] ): """Decorator for specifying Command details @@ -158,6 +163,7 @@ def echo_json(self, message): tags: A list of tags that can be used to filter commands allow_any_kwargs: Flag controlling whether passed kwargs will be restricted to the Command parameters defined. + deprecated: Boolean to indicate if command is deprecated Returns: The decorated function @@ -207,6 +213,7 @@ def echo_json(self, message): metadata=metadata, tags=tags, allow_any_kwargs=allow_any_kwargs, + deprecated=deprecated, ) if output_type is None: @@ -231,6 +238,7 @@ def echo_json(self, message): metadata=metadata, tags=tags, allow_any_kwargs=allow_any_kwargs, + deprecated=deprecated, ) # Python 2 compatibility @@ -261,6 +269,7 @@ def parameter( type_info=None, # type: Optional[dict] is_kwarg=None, # type: Optional[bool] model=None, # type: Optional[Type] + deprecated=None, # type: Optional[bool] ): """Decorator for specifying Parameter details @@ -304,6 +313,7 @@ def echo(self, message): method. model: Class to be used as a model for this parameter. Must be a Python type object, not an instance. + deprecated: Boolean to indicate if this parameter is deprecated Returns: The decorated function @@ -346,6 +356,7 @@ def echo(self, message): type_info=type_info, is_kwarg=is_kwarg, model=model, + deprecated=deprecated, ) new_parameter = Parameter( @@ -366,6 +377,7 @@ def echo(self, message): type_info=type_info, is_kwarg=is_kwarg, model=model, + deprecated=deprecated, ) # Python 2 compatibility @@ -473,6 +485,24 @@ def pre_shutdown(self): return _wrapped +def deprecated(_wrapped=None): + """Decorator for specifying a deprecated command or parameter + + for example:: + + @deprecated + def pre_running(self): + # Run pre-running processing + return + + Args: + _wrapped: The function to decorate. This is handled as a positional argument and + shouldn't be explicitly set. + """ + _wrapped._deprecated = True + return _wrapped + + def startup(_wrapped=None): """Decorator for specifying a function to run before a plugin is running. @@ -551,6 +581,18 @@ def _parse_shutdown_functions(client): return shutdown_functions +def _parse_deprecated(method): + """Get a list of deprecated methods""" + cmd = getattr(method, "_command", Command()) + if cmd.deprecated: + return True + + if callable(method) and getattr(method, "_deprecated", False): + return True + + return False + + def _parse_startup_functions(client): # type: (object) -> List[Callable] """Get a list of callable fields labeled with the startup annotation @@ -679,6 +721,7 @@ def _initialize_command(method): cmd.name = _method_name(method) cmd.display_name = cmd.display_name or _method_name(method) cmd.description = cmd.description or _method_docstring(method) + cmd.deprecated = cmd.deprecated or _parse_deprecated(method) try: base_dir = os.path.dirname(inspect.getfile(method)) @@ -902,6 +945,7 @@ def _initialize_parameter( is_kwarg=None, model=None, method=None, + deprecated=None, ): # type: (...) -> Parameter """Initialize a Parameter @@ -942,6 +986,7 @@ def _initialize_parameter( type_info=type_info, is_kwarg=is_kwarg, model=model, + deprecated=deprecated, ) # Every parameter needs a key, so stop that right here diff --git a/brewtils/models.py b/brewtils/models.py index 635634e4..351b429d 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -143,6 +143,7 @@ def __init__( tags=None, topics=None, allow_any_kwargs=None, + deprecated=None, ): self.name = name self.display_name = display_name or name @@ -159,6 +160,7 @@ def __init__( self.tags = tags or [] self.topics = topics or [] self.allow_any_kwargs = allow_any_kwargs + self.deprecated = deprecated def __str__(self): return self.name @@ -336,6 +338,7 @@ def __init__( regex=None, form_input_type=None, type_info=None, + deprecated=None, is_kwarg=None, model=None, ): @@ -354,6 +357,7 @@ def __init__( self.regex = regex self.form_input_type = form_input_type self.type_info = type_info or {} + self.deprecated = deprecated # These are special - they aren't part of the Parameter "API" (they aren't in # the serialization schema) but we still need them on this model for consistency diff --git a/brewtils/schemas.py b/brewtils/schemas.py index fe3828f5..437d2c83 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -186,6 +186,7 @@ class ParameterSchema(BaseSchema): regex = fields.Str(allow_none=True) form_input_type = fields.Str(allow_none=True) type_info = fields.Dict(allow_none=True) + deprecated = fields.Bool(allow_none=True) class CommandSchema(BaseSchema): @@ -204,6 +205,7 @@ class CommandSchema(BaseSchema): tags = fields.List(fields.Str(), allow_none=True) topics = fields.List(fields.Str(), allow_none=True) allow_any_kwargs = fields.Boolean(allow_none=True) + deprecated = fields.Boolean(allow_none=True) class InstanceSchema(BaseSchema): diff --git a/brewtils/test/fixtures.py b/brewtils/test/fixtures.py index 321e393a..8d085f95 100644 --- a/brewtils/test/fixtures.py +++ b/brewtils/test/fixtures.py @@ -131,6 +131,7 @@ def nested_parameter_dict(): "regex": None, "form_input_type": None, "type_info": {}, + "deprecated": False, } @@ -153,6 +154,7 @@ def parameter_dict(nested_parameter_dict, choices_dict): "regex": ".*", "form_input_type": None, "type_info": {}, + "deprecated": False, } @@ -184,6 +186,7 @@ def command_dict(parameter_dict, system_id): "tags": [], "topics": [], "allow_any_kwargs": False, + "deprecated": False, } From 0086bef3500684cc9561a73b7b593aa2e3f401aa Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Fri, 3 Jan 2025 13:00:34 +0000 Subject: [PATCH 29/50] Use standard library deprecated decorator and use inspect to determine if member of method --- brewtils/decorators.py | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/brewtils/decorators.py b/brewtils/decorators.py index 34305fe1..46dc623a 100644 --- a/brewtils/decorators.py +++ b/brewtils/decorators.py @@ -14,10 +14,6 @@ from brewtils.errors import PluginParamError, _deprecate from brewtils.models import Command, Parameter, Resolvable -import logging - -logger = logging.getLogger(__name__) - if sys.version_info.major == 2: from funcsigs import Parameter as InspectParameter # noqa from funcsigs import signature @@ -485,24 +481,6 @@ def pre_shutdown(self): return _wrapped -def deprecated(_wrapped=None): - """Decorator for specifying a deprecated command or parameter - - for example:: - - @deprecated - def pre_running(self): - # Run pre-running processing - return - - Args: - _wrapped: The function to decorate. This is handled as a positional argument and - shouldn't be explicitly set. - """ - _wrapped._deprecated = True - return _wrapped - - def startup(_wrapped=None): """Decorator for specifying a function to run before a plugin is running. @@ -582,15 +560,8 @@ def _parse_shutdown_functions(client): def _parse_deprecated(method): - """Get a list of deprecated methods""" - cmd = getattr(method, "_command", Command()) - if cmd.deprecated: - return True - - if callable(method) and getattr(method, "_deprecated", False): - return True - - return False + """Determine if method is deprecated""" + return any("__deprecated__" in t for t in inspect.getmembers(method)) def _parse_startup_functions(client): From 7ce03123774d7e1ed679bcd61141d1cc2a620850 Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:37:31 +0000 Subject: [PATCH 30/50] Update logic for deprecated --- brewtils/decorators.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/brewtils/decorators.py b/brewtils/decorators.py index 46dc623a..e6c03bb4 100644 --- a/brewtils/decorators.py +++ b/brewtils/decorators.py @@ -693,6 +693,12 @@ def _initialize_command(method): cmd.display_name = cmd.display_name or _method_name(method) cmd.description = cmd.description or _method_docstring(method) cmd.deprecated = cmd.deprecated or _parse_deprecated(method) + if cmd.deprecated: + cmd.hidden = cmd.deprecated + if cmd.description: + cmd.description = f"(Deprecated) {cmd.description}" + else: + cmd.description = "(Deprecated)" try: base_dir = os.path.dirname(inspect.getfile(method)) @@ -985,6 +991,13 @@ def _initialize_parameter( # Process the raw choices into a Choices object param.choices = process_choices(param.choices) + # Process deprecated + if param.deprecated: + if param.description: + param.description = f"(Deprecated) {param.description}" + else: + param.description = "(Deprecated)" + # Now deal with nested parameters if param.parameters or param.model: if param.model: From 8dff99d6424af3d8f4d60332ef6d7cf954523959 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Thu, 16 Jan 2025 14:11:58 -0500 Subject: [PATCH 31/50] add newer statement --- brewtils/models.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/brewtils/models.py b/brewtils/models.py index 635634e4..f6895ab3 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -812,6 +812,25 @@ def is_ephemeral(self): def is_json(self): return self.output_type and self.output_type.upper() == "JSON" + def is_newer(self, old_request): + if self._status in self.COMPLETED_STATUSES and old_request._status in [ + "CREATED", + "RECEIVED", + "IN_PROGRESS", + ]: + return True + + if self._status == "IN_PROGRESS" and old_request._status in [ + "CREATED", + "RECEIVED", + ]: + return True + + if self.status_updated_at > old_request.status_updated_at: + return True + + return False + class System(BaseModel): schema = "SystemSchema" @@ -1565,6 +1584,13 @@ def __repr__(self): ) ) + def is_newer(self, old_garden): + + if self.status_info.heartbeat > old_garden.status_info.heartbeat: + return True + + return False + class Connection(BaseModel): schema = "ConnectionSchema" From 7803ff23e126565848ddc0520e8777660d5e73b2 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:44:06 -0500 Subject: [PATCH 32/50] Add Comparison checks --- brewtils/models.py | 269 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 255 insertions(+), 14 deletions(-) diff --git a/brewtils/models.py b/brewtils/models.py index f6895ab3..71f19406 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -274,6 +274,31 @@ def __str__(self): def __repr__(self): return "" % (self.name, self.status) + def __eq__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + if not isinstance(other, Instance): + return False + + return self.id == other.id or self.name == other.name + + def __gt__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + if not isinstance(other, Instance): + return False + + if hasattr(other, "status_info") and hasattr(self, "status_info"): + return self.status_info.heartbeat > other.status_info.heartbeat + + if hasattr(self, "status_info") and hasattr(self.status_info, "heartbeat"): + return True + + return False + + def __lt__(self, other): + return not self.__gt__(other) + class Choices(BaseModel): schema = "ChoicesSchema" @@ -458,6 +483,31 @@ def __repr__(self): self.heartbeat, ) + def __eq__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + if not isinstance(other, StatusHistory): + return False + + return self.status == other.status and self.heartbeat == other.heartbeat + + def __gt__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + if not isinstance(other, StatusHistory): + return False + + if hasattr(other, "heartbeat") and hasattr(self, "heartbeat"): + return self.heartbeat > other.heartbeat + + if hasattr(self, "heartbeat"): + return True + + return False + + def __lt__(self, other): + return not self.__gt__(other) + class StatusInfo(BaseModel): schema = "StatusInfoSchema" @@ -485,6 +535,31 @@ def __repr__(self): self.history, ) + def __eq__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + if not isinstance(other, StatusInfo): + return False + + return self.heartbeat == other.heartbeat + + def __gt__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + if not isinstance(other, StatusInfo): + return False + + if hasattr(other, "heartbeat") and hasattr(self, "heartbeat"): + return self.heartbeat > other.heartbeat + + if hasattr(self, "heartbeat"): + return True + + return False + + def __lt__(self, other): + return not self.__gt__(other) + class RequestFile(BaseModel): schema = "RequestFileSchema" @@ -812,25 +887,51 @@ def is_ephemeral(self): def is_json(self): return self.output_type and self.output_type.upper() == "JSON" - def is_newer(self, old_request): - if self._status in self.COMPLETED_STATUSES and old_request._status in [ - "CREATED", - "RECEIVED", - "IN_PROGRESS", - ]: - return True + def __eq__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + if not isinstance(other, Request): + return False + + return self.id == other.id and self._status == other._status + + def __gt__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + if not isinstance(other, Request): + return False + + if self._status != other._status: + + if self._status in self.COMPLETED_STATUSES and other._status in [ + "CREATED", + "RECEIVED", + "IN_PROGRESS", + ]: + return True - if self._status == "IN_PROGRESS" and old_request._status in [ - "CREATED", - "RECEIVED", - ]: - return True + if self._status == "IN_PROGRESS" and other._status in [ + "CREATED", + "RECEIVED", + ]: + return True - if self.status_updated_at > old_request.status_updated_at: - return True + return False + + if hasattr(self, "status_updated_at") and hasattr(other, "status_updated_at"): + return self.status_updated_at > other.status_updated_at + + if hasattr(self, "updated_at") and hasattr(other, "updated_at"): + return self.updated_at > other.updated_at + + if hasattr(self, "created_at") and hasattr(other, "created_at"): + return self.created_at > other.created_at return False + def __lt__(self, other): + return not self.__gt__(other) + class System(BaseModel): schema = "SystemSchema" @@ -883,6 +984,39 @@ def __repr__(self): self.namespace, ) + def __eq__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + if not isinstance(other, System): + return False + + return self.id == other.id or ( + self.name == other.name + and self.namespace == other.namespace + and self.version == other.version + ) + + def __gt__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + if not isinstance(other, System): + return False + + if hasattr(other, "instances") and hasattr(self, "instances"): + + for self_instance in self.instances: + for other_instance in other.instances: + if ( + self_instance == other_instance + and self_instance > other_instance + ): + return True + + return False + + def __lt__(self, other): + return not self.__gt__(other) + @property def instance_names(self): return [i.name for i in self.instances] @@ -1176,6 +1310,40 @@ def __repr__(self): ) ) + def __eq__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + if not isinstance(other, System): + return False + + return ( + self.namespace == other.namespace + and self.garden == other.garden + and self.name == other.name + and self.error == other.error + and self.error_message == other.error_message + and self.payload_type == other.payload_type + and self.payload == other.payload + ) + + def __gt__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + if not isinstance(other, Event): + return False + + if self.timestamp and other.timestamp and self.timestamp > other.timestamp: + return True + + if self.payload and other.payload: + if self.payload == other.payload and self.payload > other.payload: + return True + + return False + + def __lt__(self, other): + return not self.__gt__(other) + class Queue(BaseModel): schema = "QueueSchema" @@ -1591,6 +1759,61 @@ def is_newer(self, old_garden): return False + def __eq__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + + if not isinstance(other, Garden): + return False + + return self.id == other.id or self.name == other.name + + def __gt__(self, other): + # Implemented not for full model to model comparison, but to allow for + # quick comparisons for event logic filtering + + if not isinstance(other, Event): + return False + + if hasattr(self, "status_info") and hasattr(other, "status_info"): + return self.status_info > other.status_info + + if hasattr(other, "receiving_connections") and hasattr( + self, "receiving_connections" + ): + + for self_connection in self.receiving_connections: + for other_connection in other.receiving_connections: + if ( + self_connection == other_connection + and self_connection > other_connection + ): + return True + + if hasattr(other, "publishing_connections") and hasattr( + self, "publishing_connections" + ): + + for self_connection in self.publishing_connections: + for other_connection in other.publishing_connections: + if ( + self_connection == other_connection + and self_connection > other_connection + ): + return True + + if hasattr(other, "systems") and hasattr(self, "systems"): + + for self_system in self.systems: + for other_system in other.systems: + if self_system == other_system and self_system > other_system: + return True + + return False + + def __lt__(self, other): + return not self.__gt__(other) + class Connection(BaseModel): schema = "ConnectionSchema" @@ -1630,6 +1853,24 @@ def __repr__(self): self.config, ) + def __eq__(self, other): + if not isinstance(other, Connection): + return False + + return self.api == other.api + + def __gt__(self, other): + if not isinstance(other, Connection): + return False + + if hasattr(self, "status_info") and hasattr(other, "status_info"): + return self.status_info > other.status_info + + return False + + def __lt__(self, other): + return not self.__gt__(other) + class Operation(BaseModel): schema = "OperationSchema" From f147ef1d640c2d32a9ad4e14732ec7a9595f7194 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Tue, 21 Jan 2025 11:23:23 -0500 Subject: [PATCH 33/50] Changelog Update --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d152f730..c6abb1aa 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Brewtils Changelog ================== +3.29.2 +------ +TBD + +- Added __eq__ and __gt__ support for various models to enable improved event handling in framework + 3.29.1 ------ 12/31/24 From d3798a6e0a22cc068aa2eca70de1c5d5014bfaf1 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Wed, 22 Jan 2025 06:34:34 -0500 Subject: [PATCH 34/50] Updating back to is_newer --- brewtils/models.py | 164 +++++++-------------------------------------- 1 file changed, 25 insertions(+), 139 deletions(-) diff --git a/brewtils/models.py b/brewtils/models.py index 71f19406..05aebc6c 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -274,31 +274,20 @@ def __str__(self): def __repr__(self): return "" % (self.name, self.status) - def __eq__(self, other): - # Implemented not for full model to model comparison, but to allow for - # quick comparisons for event logic filtering - if not isinstance(other, Instance): - return False - - return self.id == other.id or self.name == other.name - - def __gt__(self, other): + def is_newer(self, other): # Implemented not for full model to model comparison, but to allow for # quick comparisons for event logic filtering if not isinstance(other, Instance): return False if hasattr(other, "status_info") and hasattr(self, "status_info"): - return self.status_info.heartbeat > other.status_info.heartbeat + return self.status_info.is_newer(other.status_info) if hasattr(self, "status_info") and hasattr(self.status_info, "heartbeat"): return True return False - def __lt__(self, other): - return not self.__gt__(other) - class Choices(BaseModel): schema = "ChoicesSchema" @@ -483,15 +472,7 @@ def __repr__(self): self.heartbeat, ) - def __eq__(self, other): - # Implemented not for full model to model comparison, but to allow for - # quick comparisons for event logic filtering - if not isinstance(other, StatusHistory): - return False - - return self.status == other.status and self.heartbeat == other.heartbeat - - def __gt__(self, other): + def is_newer(self, other): # Implemented not for full model to model comparison, but to allow for # quick comparisons for event logic filtering if not isinstance(other, StatusHistory): @@ -505,9 +486,6 @@ def __gt__(self, other): return False - def __lt__(self, other): - return not self.__gt__(other) - class StatusInfo(BaseModel): schema = "StatusInfoSchema" @@ -535,15 +513,7 @@ def __repr__(self): self.history, ) - def __eq__(self, other): - # Implemented not for full model to model comparison, but to allow for - # quick comparisons for event logic filtering - if not isinstance(other, StatusInfo): - return False - - return self.heartbeat == other.heartbeat - - def __gt__(self, other): + def is_newer(self, other): # Implemented not for full model to model comparison, but to allow for # quick comparisons for event logic filtering if not isinstance(other, StatusInfo): @@ -557,9 +527,6 @@ def __gt__(self, other): return False - def __lt__(self, other): - return not self.__gt__(other) - class RequestFile(BaseModel): schema = "RequestFileSchema" @@ -887,15 +854,7 @@ def is_ephemeral(self): def is_json(self): return self.output_type and self.output_type.upper() == "JSON" - def __eq__(self, other): - # Implemented not for full model to model comparison, but to allow for - # quick comparisons for event logic filtering - if not isinstance(other, Request): - return False - - return self.id == other.id and self._status == other._status - - def __gt__(self, other): + def is_newer(self, other): # Implemented not for full model to model comparison, but to allow for # quick comparisons for event logic filtering if not isinstance(other, Request): @@ -929,9 +888,6 @@ def __gt__(self, other): return False - def __lt__(self, other): - return not self.__gt__(other) - class System(BaseModel): schema = "SystemSchema" @@ -984,19 +940,7 @@ def __repr__(self): self.namespace, ) - def __eq__(self, other): - # Implemented not for full model to model comparison, but to allow for - # quick comparisons for event logic filtering - if not isinstance(other, System): - return False - - return self.id == other.id or ( - self.name == other.name - and self.namespace == other.namespace - and self.version == other.version - ) - - def __gt__(self, other): + def is_newer(self, other): # Implemented not for full model to model comparison, but to allow for # quick comparisons for event logic filtering if not isinstance(other, System): @@ -1007,16 +951,13 @@ def __gt__(self, other): for self_instance in self.instances: for other_instance in other.instances: if ( - self_instance == other_instance - and self_instance > other_instance - ): + self_instance.id == other_instance.id + or self_instance.name == other_instance.name + ) and self_instance.is_newer(other_instance): return True return False - def __lt__(self, other): - return not self.__gt__(other) - @property def instance_names(self): return [i.name for i in self.instances] @@ -1310,40 +1251,6 @@ def __repr__(self): ) ) - def __eq__(self, other): - # Implemented not for full model to model comparison, but to allow for - # quick comparisons for event logic filtering - if not isinstance(other, System): - return False - - return ( - self.namespace == other.namespace - and self.garden == other.garden - and self.name == other.name - and self.error == other.error - and self.error_message == other.error_message - and self.payload_type == other.payload_type - and self.payload == other.payload - ) - - def __gt__(self, other): - # Implemented not for full model to model comparison, but to allow for - # quick comparisons for event logic filtering - if not isinstance(other, Event): - return False - - if self.timestamp and other.timestamp and self.timestamp > other.timestamp: - return True - - if self.payload and other.payload: - if self.payload == other.payload and self.payload > other.payload: - return True - - return False - - def __lt__(self, other): - return not self.__gt__(other) - class Queue(BaseModel): schema = "QueueSchema" @@ -1752,23 +1659,7 @@ def __repr__(self): ) ) - def is_newer(self, old_garden): - - if self.status_info.heartbeat > old_garden.status_info.heartbeat: - return True - - return False - - def __eq__(self, other): - # Implemented not for full model to model comparison, but to allow for - # quick comparisons for event logic filtering - - if not isinstance(other, Garden): - return False - - return self.id == other.id or self.name == other.name - - def __gt__(self, other): + def is_newer(self, other): # Implemented not for full model to model comparison, but to allow for # quick comparisons for event logic filtering @@ -1776,7 +1667,7 @@ def __gt__(self, other): return False if hasattr(self, "status_info") and hasattr(other, "status_info"): - return self.status_info > other.status_info + return self.status_info.is_newer(other.status_info) if hasattr(other, "receiving_connections") and hasattr( self, "receiving_connections" @@ -1785,8 +1676,8 @@ def __gt__(self, other): for self_connection in self.receiving_connections: for other_connection in other.receiving_connections: if ( - self_connection == other_connection - and self_connection > other_connection + self_connection.api == other_connection.api + and self_connection.is_newer(other_connection) ): return True @@ -1797,8 +1688,8 @@ def __gt__(self, other): for self_connection in self.publishing_connections: for other_connection in other.publishing_connections: if ( - self_connection == other_connection - and self_connection > other_connection + self_connection.api == other_connection.api + and self_connection.is_newer(other_connection) ): return True @@ -1806,14 +1697,18 @@ def __gt__(self, other): for self_system in self.systems: for other_system in other.systems: - if self_system == other_system and self_system > other_system: + if ( + self_system.id == other_system.id + or ( + self_system.namespace == other_system.namespace + and self_system.name == other_system.name + and self_system.version == other_system.version + ) + ) and self_system.is_newer(other_system): return True return False - def __lt__(self, other): - return not self.__gt__(other) - class Connection(BaseModel): schema = "ConnectionSchema" @@ -1853,24 +1748,15 @@ def __repr__(self): self.config, ) - def __eq__(self, other): - if not isinstance(other, Connection): - return False - - return self.api == other.api - - def __gt__(self, other): + def is_newer(self, other): if not isinstance(other, Connection): return False if hasattr(self, "status_info") and hasattr(other, "status_info"): - return self.status_info > other.status_info + return self.status_info.is_newer(other.status_info) return False - def __lt__(self, other): - return not self.__gt__(other) - class Operation(BaseModel): schema = "OperationSchema" From 0bed424c82ab1b3d9a840ed29484cca3908c4395 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Wed, 22 Jan 2025 06:46:44 -0500 Subject: [PATCH 35/50] update change log --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c6abb1aa..a72798ac 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,7 +5,7 @@ Brewtils Changelog ------ TBD -- Added __eq__ and __gt__ support for various models to enable improved event handling in framework +- Added `is_newer` support for various models to enable improved event handling in framework 3.29.1 ------ From f3a7f8e93bc3d43e415854fe3669fd2164ca4ac3 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Wed, 22 Jan 2025 10:04:20 -0500 Subject: [PATCH 36/50] Updating Unit Testing --- brewtils/models.py | 48 ++++++++++++++++--------- test/models_test.py | 85 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 17 deletions(-) diff --git a/brewtils/models.py b/brewtils/models.py index 05aebc6c..09ec7b74 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -280,10 +280,11 @@ def is_newer(self, other): if not isinstance(other, Instance): return False - if hasattr(other, "status_info") and hasattr(self, "status_info"): - return self.status_info.is_newer(other.status_info) - if hasattr(self, "status_info") and hasattr(self.status_info, "heartbeat"): + if hasattr(other, "status_info") and hasattr( + other.status_info, "heartbeat" + ): + return self.status_info.is_newer(other.status_info) return True return False @@ -478,10 +479,9 @@ def is_newer(self, other): if not isinstance(other, StatusHistory): return False - if hasattr(other, "heartbeat") and hasattr(self, "heartbeat"): - return self.heartbeat > other.heartbeat - - if hasattr(self, "heartbeat"): + if hasattr(self, "heartbeat") and self.heartbeat: + if hasattr(other, "heartbeat") and other.heartbeat: + return self.heartbeat > other.heartbeat return True return False @@ -519,10 +519,9 @@ def is_newer(self, other): if not isinstance(other, StatusInfo): return False - if hasattr(other, "heartbeat") and hasattr(self, "heartbeat"): - return self.heartbeat > other.heartbeat - - if hasattr(self, "heartbeat"): + if hasattr(self, "heartbeat") and self.heartbeat: + if hasattr(other, "heartbeat") and other.heartbeat: + return self.heartbeat > other.heartbeat return True return False @@ -875,16 +874,31 @@ def is_newer(self, other): ]: return True + if self._status == "RECEIVED" and other._status == "CREATED": + return True + return False - if hasattr(self, "status_updated_at") and hasattr(other, "status_updated_at"): - return self.status_updated_at > other.status_updated_at + self_newest_timestamp = None + if hasattr(self, "status_updated_at") and self.status_updated_at: + self_newest_timestamp = self.status_updated_at + + if hasattr(self, "updated_at") and self.updated_at: + if not self_newest_timestamp or self.updated_at > self_newest_timestamp: + self_newest_timestamp = self.updated_at + + if hasattr(self, "created_at") and self.created_at: + if not self_newest_timestamp or self.created_at > self_newest_timestamp: + self_newest_timestamp = self.created_at + + if hasattr(other, "status_updated_at") and other.status_updated_at: + return self_newest_timestamp > other.status_updated_at - if hasattr(self, "updated_at") and hasattr(other, "updated_at"): - return self.updated_at > other.updated_at + if hasattr(other, "updated_at") and other.updated_at: + return self_newest_timestamp > other.updated_at - if hasattr(self, "created_at") and hasattr(other, "created_at"): - return self.created_at > other.created_at + if hasattr(other, "created_at") and other.created_at: + return self_newest_timestamp > other.created_at return False diff --git a/test/models_test.py b/test/models_test.py index 8bfa5443..d35a0002 100644 --- a/test/models_test.py +++ b/test/models_test.py @@ -22,6 +22,7 @@ Subscriber, StatusInfo, Topic, + StatusHistory, ) from pytest_lazyfixture import lazy_fixture @@ -127,6 +128,22 @@ def test_repr(self): assert "name" in repr(instance) assert "RUNNING" in repr(instance) + def test_is_newer(self): + instance1 = Instance( + name="name", status="RUNNING", status_info=StatusInfo(heartbeat=1) + ) + instance2 = Instance( + name="name", status="RUNNING", status_info=StatusInfo(heartbeat=2) + ) + instance3 = Instance(name="name", status="RUNNING") + instance4 = Instance(name="name", status="RUNNING") + assert instance2.is_newer(instance1) + assert instance2.is_newer(instance3) + + # Unable to determine, so ensure it never returns True + assert not instance3.is_newer(instance4) + assert not instance4.is_newer(instance3) + class TestChoices(object): def test_str(self): @@ -344,6 +361,36 @@ def test_from_template_overrides(self, bg_request_template): if key != "command_type": assert getattr(request, key) == getattr(bg_request_template, key) + def test_is_newer_status(self): + request_created = Request(status="CREATED") + request_received = Request(status="RECEIVED") + request_in_progress = Request(status="IN_PROGRESS") + request_canceled = Request(status="CANCELED") + request_success = Request(status="SUCCESS") + request_error = Request(status="ERROR") + request_invalid = Request(status="INVALID") + + for completed_request in [ + request_canceled, + request_success, + request_error, + request_invalid, + ]: + assert completed_request.is_newer(request_created) + assert completed_request.is_newer(request_received) + assert completed_request.is_newer(request_in_progress) + + assert request_in_progress.is_newer(request_received) + assert request_in_progress.is_newer(request_created) + + assert request_received.is_newer(request_created) + + def test_is_newer_timestamp(self): + + assert Request(status_updated_at=2).is_newer(Request(status_updated_at=1)) + assert Request(updated_at=2).is_newer(Request(updated_at=1)) + assert Request(created_at=2).is_newer(Request(created_at=1)) + class TestSystem(object): def test_get_command_by_name(self, bg_system, bg_command): @@ -407,6 +454,9 @@ def test_repr(self, bg_system): assert "system" in repr(bg_system) assert "1.0.0" in repr(bg_system) + def test_is_newer(self): + assert False + class TestPatchOperation(object): @pytest.fixture @@ -775,3 +825,38 @@ def test_negative_history(self): status_info.set_status_heartbeat("RUNNING", max_history=-1) assert len(status_info.history) == 10 + + def test_is_newer(self): + status_info = StatusInfo(heartbeat=1) + status_info2 = StatusInfo(heartbeat=2) + + status_info3 = StatusInfo() + status_info4 = StatusInfo() + + assert status_info2.is_newer(status_info) + assert status_info2.is_newer(status_info3) + + assert not status_info3.is_newer(status_info4) + assert not status_info4.is_newer(status_info3) + + +class TestStatusHistory: + + def test_is_newer(self): + + history1 = StatusHistory(status="RUNNING", heartbeat=1) + history2 = StatusHistory(status="RUNNING", heartbeat=2) + history3 = StatusHistory(status="RUNNING") + history4 = StatusHistory(status="RUNNING") + + assert history2.is_newer(history1) + assert history2.is_newer(history3) + + assert not history3.is_newer(history4) + assert not history4.is_newer(history3) + + +class TestConnection: + + def test_is_newer(self): + assert False From e847077e2ea9f32ba4734ee54120737b49ed8ecf Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Wed, 22 Jan 2025 16:29:43 +0000 Subject: [PATCH 37/50] Updating Testing --- brewtils/models.py | 26 +++++++++++++++++--------- test/models_test.py | 38 ++++++++++++++++++++++++++++++++------ 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/brewtils/models.py b/brewtils/models.py index 09ec7b74..677ed408 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -960,17 +960,25 @@ def is_newer(self, other): if not isinstance(other, System): return False - if hasattr(other, "instances") and hasattr(self, "instances"): + self_newest_instance = None - for self_instance in self.instances: - for other_instance in other.instances: - if ( - self_instance.id == other_instance.id - or self_instance.name == other_instance.name - ) and self_instance.is_newer(other_instance): - return True + if hasattr(self, "instances"): + for instance in self.instances: + if not self_newest_instance: + self_newest_instance = instance + elif instance.is_newer(self_newest_instance): + self_newest_instance = instance - return False + if not self_newest_instance: + return False + + if hasattr(other, "instances"): + + for other_instance in other.instances: + if other_instance.is_newer(self_newest_instance): + return False + + return True @property def instance_names(self): diff --git a/test/models_test.py b/test/models_test.py index d35a0002..3b5d83d3 100644 --- a/test/models_test.py +++ b/test/models_test.py @@ -3,28 +3,31 @@ import pytest import pytz +from pytest_lazyfixture import lazy_fixture + from brewtils.errors import ModelError from brewtils.models import ( Choices, Command, + Connection, CronTrigger, Instance, IntervalTrigger, LoggingConfig, Parameter, PatchOperation, - User, Queue, Request, RequestFile, RequestTemplate, Role, - Subscriber, + StatusHistory, StatusInfo, + Subscriber, + System, Topic, - StatusHistory, + User, ) -from pytest_lazyfixture import lazy_fixture @pytest.fixture @@ -455,7 +458,21 @@ def test_repr(self, bg_system): assert "1.0.0" in repr(bg_system) def test_is_newer(self): - assert False + system1 = System(instances=[Instance(status_info=StatusInfo(heartbeat=1))]) + system2 = System(instances=[Instance(status_info=StatusInfo(heartbeat=2))]) + system3 = System( + instances=[ + Instance(status_info=StatusInfo(heartbeat=2)), + Instance(status_info=StatusInfo(heartbeat=3)), + ] + ) + system4 = System(instances=[]) + + assert system2.is_newer(system1) + assert system3.is_newer(system1) + assert system3.is_newer(system2) + + assert not system4.is_newer(system4) class TestPatchOperation(object): @@ -859,4 +876,13 @@ def test_is_newer(self): class TestConnection: def test_is_newer(self): - assert False + connection1 = Connection(status="RUNNING", status_info=StatusInfo(heartbeat=1)) + connection2 = Connection(status="RUNNING", status_info=StatusInfo(heartbeat=2)) + connection3 = Connection(status="RUNNING") + connection4 = Connection(status="RUNNING") + assert connection2.is_newer(connection1) + assert connection2.is_newer(connection3) + + # Unable to determine, so ensure it never returns True + assert not connection3.is_newer(connection4) + assert not connection4.is_newer(connection4) From 76dedb3ecff734134c6b4ac95c3fe08de32e6c13 Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Thu, 23 Jan 2025 10:38:33 +0000 Subject: [PATCH 38/50] Deprecated unit tests --- setup.py | 1 + test/decorators_test.py | 141 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+) diff --git a/setup.py b/setup.py index 2bbbb4a1..8e2594c4 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ def find_version(): "requests<3", "simplejson<4", "six<2", + "typing_extensions>=14.12.2", "wrapt", "yapconf>=0.3.7", ], diff --git a/test/decorators_test.py b/test/decorators_test.py index 69d6c2f5..62438a18 100644 --- a/test/decorators_test.py +++ b/test/decorators_test.py @@ -41,6 +41,11 @@ else: from inspect import signature # noqa +if sys.version_info.major == 3 and sys.version_info.minor >= 13: + from warnings import deprecated # noqa +elif sys.version_info.major == 3 and sys.version_info.minor < 13: + from typing_extensions import deprecated # noqa + @pytest.fixture def cmd(): @@ -697,6 +702,45 @@ def cmd1(foo): assert cmd1._command.display_name == "foo_test" + def test_command_deprecated_docstring(self): + @deprecated("Do not use for this reason") + @command(deprecated=True) + def cmd(): + """Docstring""" + return True + + assert hasattr(cmd, "_command") + + _cmd = _initialize_command(cmd) + + assert _cmd.description == "(Deprecated) Docstring" + assert _cmd.hidden is True + + def test_command_deprecated_description(self): + @command(description="Description", deprecated=True) + def cmd(): + """Docstring""" + return True + + assert hasattr(cmd, "_command") + + _cmd = _initialize_command(cmd) + + assert _cmd.description == "(Deprecated) Description" + assert _cmd.hidden is True + + def test_command_deprecated_no_description(self): + @command(deprecated=True) + def cmd(): + return True + + assert hasattr(cmd, "_command") + + _cmd = _initialize_command(cmd) + + assert _cmd.description == "(Deprecated)" + assert _cmd.hidden is True + class TestParameter(object): """Test parameter decorator @@ -796,6 +840,32 @@ def cmd_bad_1(foo): def cmd_bad_2(foo): return foo + def test_parameter_deprecated(self, basic_param): + @parameter(**basic_param, deprecated=True) + def cmd(foo): + return foo + + assert hasattr(cmd, "parameters") + assert len(cmd.parameters) == 1 + assert ( + _initialize_parameter(cmd.parameters[0]).description + == "(Deprecated) Mutant" + ) + + def test_parameter_deprecated_multiple_param(self, basic_param): + @parameter(**basic_param) + @parameter(key="bar", description="Param2", deprecated=True) + def cmd(foo): + return foo + + assert hasattr(cmd, "parameters") + assert len(cmd.parameters) == 2 + assert _initialize_parameter(cmd.parameters[0]).description == "Mutant" + assert ( + _initialize_parameter(cmd.parameters[1]).description + == "(Deprecated) Param2" + ) + class TestParameters(object): @pytest.fixture(autouse=True) @@ -1081,6 +1151,31 @@ def test_file_defaults(self): """File parameter defaults should be cleared for safety""" assert _initialize_parameter(Parameter(key="f", type="Base64")).default is None + def test_deprecated(self): + """File parameter defaults should be cleared for safety""" + assert ( + _initialize_parameter(Parameter(key="a", type="boolean")).description + is None + ) + assert ( + _initialize_parameter( + Parameter(key="a", type="boolean", deprecated=True) + ).description + == "(Deprecated)" + ) + assert ( + _initialize_parameter( + Parameter(key="a", type="boolean", description="Desc") + ).description + == "Desc" + ) + assert ( + _initialize_parameter( + Parameter(key="a", type="boolean", description="Desc", deprecated=True) + ).description + == "(Deprecated) Desc" + ) + class TestNesting(object): @pytest.fixture def inner(self): @@ -1463,6 +1558,52 @@ def test_plugin_param(self, cmd, parameter_dict): ) +class TestDeprecated(object): + """Test deprecated parameter""" + + def test_deprecated_command_docstring(self): + @deprecated("Do not use for this reason") + @command + def cmd(): + """Docstring""" + return True + + # Initialize this command and make sure the description has been updated + assert hasattr(cmd, "_command") + + _cmd = _initialize_command(cmd) + + assert _cmd.description == "(Deprecated) Docstring" + assert _cmd.hidden is True + + def test_deprecated_command_description(self): + @deprecated("Do not use for this reason") + @command(description="Description") + def cmd(): + """Docstring""" + return True + + assert hasattr(cmd, "_command") + + _cmd = _initialize_command(cmd) + + assert _cmd.description == "(Deprecated) Description" + assert _cmd.hidden is True + + def test_deprecated_command_no_description(self): + @deprecated("Do not use for this reason") + @command() + def cmd(): + return True + + assert hasattr(cmd, "_command") + + _cmd = _initialize_command(cmd) + + assert _cmd.description == "(Deprecated)" + assert _cmd.hidden is True + + class TestShutdown(object): """Test shutdown decorator""" From bfec3ad3a7ed63461ffbf0598edde79f9f320713 Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Thu, 23 Jan 2025 07:01:15 -0500 Subject: [PATCH 39/50] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8e2594c4..6201c7b5 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ def find_version(): "requests<3", "simplejson<4", "six<2", - "typing_extensions>=14.12.2", + "typing_extensions>=4.12.2", "wrapt", "yapconf>=0.3.7", ], From c698512fa3cb6ef6e7f214d1b496f30b968df7b7 Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:16:35 +0000 Subject: [PATCH 40/50] Fix typing_extensions version in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6201c7b5..1ac73ad5 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ def find_version(): "requests<3", "simplejson<4", "six<2", - "typing_extensions>=4.12.2", + "typing_extensions>=4.7", "wrapt", "yapconf>=0.3.7", ], From 7212161ec49da980c6090f40b9b1aa9ebd52c5d8 Mon Sep 17 00:00:00 2001 From: 1maple1 <160027655+1maple1@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:47:44 +0000 Subject: [PATCH 41/50] Add chunk size to File model --- CHANGELOG.rst | 1 + brewtils/models.py | 3 +++ brewtils/rest/easy_client.py | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b7a70590..5852e533 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,7 @@ TBD - Added deprecated decorator and deprecated kwarg to command and parameter decorators - Added `is_newer` support for various models to enable improved event handling in framework +- Added static chunk size to File model 3.29.1 ------ diff --git a/brewtils/models.py b/brewtils/models.py index 26ed5669..c03ea38f 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -554,6 +554,9 @@ def __repr__(self): class File(BaseModel): schema = "FileSchema" + CHUNK_SIZE = 255 * 1024 + MAX_CHUNK_SIZE = 1024 * 1024 * 15 # 15MB + def __init__( self, id=None, # noqa # shadows built-in diff --git a/brewtils/rest/easy_client.py b/brewtils/rest/easy_client.py index 84a5d4da..87596df4 100644 --- a/brewtils/rest/easy_client.py +++ b/brewtils/rest/easy_client.py @@ -25,7 +25,7 @@ WaitExceededError, _deprecate, ) -from brewtils.models import BaseModel, Event, Job, PatchOperation +from brewtils.models import BaseModel, Event, File, Job, PatchOperation from brewtils.rest.client import RestClient from brewtils.schema_parser import SchemaParser @@ -170,7 +170,7 @@ class EasyClient(object): """ _default_file_params = { - "chunk_size": 255 * 1024, + "chunk_size": File.CHUNK_SIZE, } def __init__(self, *args, **kwargs): From 8a0f9bb82b7d611c03352944a6586f2ece703644 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 31 Jan 2025 13:38:54 +0000 Subject: [PATCH 42/50] 3.29.2 --- CHANGELOG.rst | 2 +- brewtils/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5852e533..e433f784 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,7 +3,7 @@ Brewtils Changelog 3.29.2 ------ -TBD +1/31/25 - Added deprecated decorator and deprecated kwarg to command and parameter decorators - Added `is_newer` support for various models to enable improved event handling in framework diff --git a/brewtils/__version__.py b/brewtils/__version__.py index cd11dcf1..ef29b5bb 100644 --- a/brewtils/__version__.py +++ b/brewtils/__version__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -__version__ = "3.29.1" +__version__ = "3.29.2" From 861b79c133e18425fa42372ac413315c73326db3 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 31 Jan 2025 13:44:57 +0000 Subject: [PATCH 43/50] 3.29.2 --- .github/workflows/tag-actions.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tag-actions.yml b/.github/workflows/tag-actions.yml index 12be66aa..57e53727 100644 --- a/.github/workflows/tag-actions.yml +++ b/.github/workflows/tag-actions.yml @@ -25,7 +25,7 @@ jobs: pypi-publish: name: PyPI Publish - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 environment: name: release url: https://pypi.org/p/brewtils @@ -62,7 +62,7 @@ jobs: pypi-verify: name: Verify PyPI Publish - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: pypi-publish steps: @@ -83,7 +83,7 @@ jobs: docker-publish: name: Docker Publish - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: pypi-verify steps: From f7ca2abe6382711420b0de5d37241c688a1ec7c1 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 31 Jan 2025 14:17:30 +0000 Subject: [PATCH 44/50] 3.29.2 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1ac73ad5..e842ff6a 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ def find_version(): version=find_version(), description="Beer-garden plugin and utility library", long_description=readme, + long_description_content_type="text/x-rst", url="https://beer-garden.io/", author="The Beer-garden Team", author_email=" ", From ae42186e6eaf821dd5f16c42b0c16d4eba40eca4 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 31 Jan 2025 14:38:00 +0000 Subject: [PATCH 45/50] 3.29.2 --- .github/workflows/tag-actions.yml | 4 ++++ README.rst | 5 ++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tag-actions.yml b/.github/workflows/tag-actions.yml index 57e53727..7b74a4ac 100644 --- a/.github/workflows/tag-actions.yml +++ b/.github/workflows/tag-actions.yml @@ -9,6 +9,7 @@ jobs: github-release: name: Github Release runs-on: ubuntu-latest + needs: pypi-publish steps: - name: Create Release @@ -57,6 +58,9 @@ jobs: - name: Make Package run: make package + - name: Verify local package + run: twine check dist/* + - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/README.rst b/README.rst index e2f0bc35..93cc6e76 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,6 @@ - -======== +========= Brewtils -======== +========= Brewtils is the Python library for interfacing with Beergarden systems. If you are planning on writing beer-garden plugins, this is the correct library for you. In addition to writing plugins, From 39bf8186a29c7a72b91b60bf5b264e824c223cf7 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 31 Jan 2025 14:46:12 +0000 Subject: [PATCH 46/50] 3.29.2 --- .github/workflows/tag-actions.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tag-actions.yml b/.github/workflows/tag-actions.yml index 7b74a4ac..87455594 100644 --- a/.github/workflows/tag-actions.yml +++ b/.github/workflows/tag-actions.yml @@ -63,6 +63,8 @@ jobs: - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + with: + verify-metadata: false pypi-verify: name: Verify PyPI Publish From cad0a119c843ce90ade39df930a659ecb27faded Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 31 Jan 2025 14:54:48 +0000 Subject: [PATCH 47/50] 3.29.2 --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index e842ff6a..1ac73ad5 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,6 @@ def find_version(): version=find_version(), description="Beer-garden plugin and utility library", long_description=readme, - long_description_content_type="text/x-rst", url="https://beer-garden.io/", author="The Beer-garden Team", author_email=" ", From 4bcbb930223d10454eac7b56f5047bce4103bba1 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 31 Jan 2025 15:58:50 +0000 Subject: [PATCH 48/50] 3.29.3 --- .github/workflows/tag-actions.yml | 2 +- CHANGELOG.rst | 8 +++++++- brewtils/__version__.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tag-actions.yml b/.github/workflows/tag-actions.yml index 87455594..8c32250d 100644 --- a/.github/workflows/tag-actions.yml +++ b/.github/workflows/tag-actions.yml @@ -53,7 +53,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel twine + pip install --upgrade setuptools wheel twine - name: Make Package run: make package diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e433f784..7c6856bd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,13 +1,19 @@ Brewtils Changelog ================== -3.29.2 +3.29.3 ------ 1/31/25 - Added deprecated decorator and deprecated kwarg to command and parameter decorators - Added `is_newer` support for various models to enable improved event handling in framework - Added static chunk size to File model +- Updating Pypi release process + +3.29.3 +------ + +- Failed Pypi Release Process 3.29.1 ------ diff --git a/brewtils/__version__.py b/brewtils/__version__.py index ef29b5bb..022bfb05 100644 --- a/brewtils/__version__.py +++ b/brewtils/__version__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -__version__ = "3.29.2" +__version__ = "3.29.3" From 991610e6393f4f1fc407e4e98b5ba14d74105b7f Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Mon, 3 Feb 2025 09:45:12 -0500 Subject: [PATCH 49/50] Change Log --- CHANGELOG.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7c6856bd..75c1291e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Brewtils Changelog ================== +3.30.0 +------ + +## Major dependency upgrades and dropping 3.6, 3.7, 3.8 Python Support + +- Upgraded all dependencies to latest versions +- Dropped support for Python 3.6, 3.7, and 3.8 + 3.29.3 ------ 1/31/25 From 8692ca3714c6d7630e1bde20474f44e74fd23f14 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Mon, 3 Feb 2025 09:45:19 -0500 Subject: [PATCH 50/50] Change Logs --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 75c1291e..fdeade52 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,7 @@ Brewtils Changelog 3.30.0 ------ +TBD ## Major dependency upgrades and dropping 3.6, 3.7, 3.8 Python Support