From f71049de84f1d0f2f3ad3903ea0d25f84a16949f Mon Sep 17 00:00:00 2001 From: Logan Jones Date: Fri, 24 May 2019 19:49:49 -0400 Subject: [PATCH 1/4] Adding file support to bg-utils --- bg_utils/mongo/models.py | 27 ++++++++++++++++++++++++++- bg_utils/mongo/parser.py | 2 ++ bg_utils/mongo/util.py | 1 + test/unit/init_test.py | 6 +++--- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/bg_utils/mongo/models.py b/bg_utils/mongo/models.py index 2b9b7c5..91ab4c6 100644 --- a/bg_utils/mongo/models.py +++ b/bg_utils/mongo/models.py @@ -27,6 +27,7 @@ StringField, CASCADE, PULL, + FileField, ) from mongoengine.errors import DoesNotExist @@ -39,6 +40,7 @@ Instance as BrewtilsInstance, Parameter as BrewtilsParameter, Request as BrewtilsRequest, + RequestFile as BrewtilsRequestFile, System as BrewtilsSystem, Event as BrewtilsEvent, Principal as BrewtilsPrincipal, @@ -62,6 +64,7 @@ "Role", "RefreshToken", "Job", + "RequestFile", "RequestTemplate", "DateTrigger", "CronTrigger", @@ -131,7 +134,7 @@ def clean(self): class Parameter(EmbeddedDocument, BrewtilsParameter): - """Mongo-Backed BREWMASTER Parameter Object""" + """Mongo-Backed brewtils parameter object""" key = StringField(required=True) type = StringField(required=True, default="Any", choices=BrewtilsParameter.TYPES) @@ -149,6 +152,7 @@ class Parameter(EmbeddedDocument, BrewtilsParameter): required=False, choices=BrewtilsParameter.FORM_INPUT_TYPES ) parameters = ListField(EmbeddedDocumentField("Parameter")) + type_info = DictField(required=False) # If no display name was set, it will default it to the same thing as the key def __init__(self, *args, **kwargs): @@ -260,6 +264,27 @@ def clean(self): ) +class RequestFile(Document, BrewtilsRequestFile): + """Mongo backed request file resource""" + + DEFAULT_CONTENT_TYPE = "application/octet-stream" + STORAGE_ENGINES = ["gridfs"] + + content_type = StringField(required=True, default=DEFAULT_CONTENT_TYPE) + storage_type = StringField(required=True, default="gridfs") + filename = StringField(required=True) + external_link = StringField() + body = FileField() + + @property + def fetch_id(self): + if self.external_link: + return self.external_link + elif self.body is not None: + return self.id + raise LookupError("No fetch id could be found") + + class Request(Document, BrewtilsRequest): """Mongo-Backed BREWMASTER Request Object""" diff --git a/bg_utils/mongo/parser.py b/bg_utils/mongo/parser.py index 15a764c..361de54 100644 --- a/bg_utils/mongo/parser.py +++ b/bg_utils/mongo/parser.py @@ -15,6 +15,7 @@ RefreshToken, Job, RequestTemplate, + RequestFile, DateTrigger, IntervalTrigger, CronTrigger, @@ -34,6 +35,7 @@ class MongoParser(SchemaParser): "CommandSchema": Command, "ParameterSchema": Parameter, "RequestSchema": Request, + "RequestFileSchema": RequestFile, "RequestTemplateSchema": RequestTemplate, "ChoicesSchema": Choices, "EventSchema": Event, diff --git a/bg_utils/mongo/util.py b/bg_utils/mongo/util.py index 436fc1e..975fd9f 100644 --- a/bg_utils/mongo/util.py +++ b/bg_utils/mongo/util.py @@ -111,6 +111,7 @@ def _ensure_roles(): name="bg-plugin", description="Allows actions necessary for plugins to function", permissions=[ + "bg-file-read", "bg-instance-update", "bg-job-create", "bg-job-update", diff --git a/test/unit/init_test.py b/test/unit/init_test.py index 143d42d..df586b2 100644 --- a/test/unit/init_test.py +++ b/test/unit/init_test.py @@ -226,9 +226,9 @@ def test_correctness(self, tmpdir, spec): config_file = os.path.join(str(tmpdir), "config.yaml") logging_config_file = os.path.join(str(tmpdir), "logging.json") - bg_utils.generate_config_file(spec, [ - "-c", config_file, "-l", logging_config_file - ]) + bg_utils.generate_config_file( + spec, ["-c", config_file, "-l", logging_config_file] + ) spec.add_source(label="config_file", source_type="yaml", filename=config_file) config = spec.load_config("config_file") From 6a493af72d4a8badabb08f89b06fda7bef419afd Mon Sep 17 00:00:00 2001 From: Logan Jones Date: Sat, 1 Jun 2019 18:59:40 -0400 Subject: [PATCH 2/4] Removing content_type from request file --- bg_utils/mongo/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bg_utils/mongo/models.py b/bg_utils/mongo/models.py index 91ab4c6..2b4a391 100644 --- a/bg_utils/mongo/models.py +++ b/bg_utils/mongo/models.py @@ -267,10 +267,8 @@ def clean(self): class RequestFile(Document, BrewtilsRequestFile): """Mongo backed request file resource""" - DEFAULT_CONTENT_TYPE = "application/octet-stream" STORAGE_ENGINES = ["gridfs"] - content_type = StringField(required=True, default=DEFAULT_CONTENT_TYPE) storage_type = StringField(required=True, default="gridfs") filename = StringField(required=True) external_link = StringField() From 2f3ff3a97f18d125e9538b6d0cfb1f310275c7ab Mon Sep 17 00:00:00 2001 From: Logan Jones Date: Sat, 1 Jun 2019 19:02:22 -0400 Subject: [PATCH 3/4] Removed unused external link. For now, we only support gridfs, which is all internal. --- bg_utils/mongo/models.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/bg_utils/mongo/models.py b/bg_utils/mongo/models.py index 2b4a391..cc644b1 100644 --- a/bg_utils/mongo/models.py +++ b/bg_utils/mongo/models.py @@ -271,16 +271,7 @@ class RequestFile(Document, BrewtilsRequestFile): storage_type = StringField(required=True, default="gridfs") filename = StringField(required=True) - external_link = StringField() - body = FileField() - - @property - def fetch_id(self): - if self.external_link: - return self.external_link - elif self.body is not None: - return self.id - raise LookupError("No fetch id could be found") + body = FileField(required=True) class Request(Document, BrewtilsRequest): From 75c112cfb6b21f220f3e109302ca7057783499b3 Mon Sep 17 00:00:00 2001 From: Logan Jones Date: Sun, 2 Jun 2019 16:16:00 -0400 Subject: [PATCH 4/4] Adding fields to RequestFile. These fields are necessary to help cleanup old request files and their associated gridfs objects. --- bg_utils/mongo/models.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/bg_utils/mongo/models.py b/bg_utils/mongo/models.py index cc644b1..2e386e5 100644 --- a/bg_utils/mongo/models.py +++ b/bg_utils/mongo/models.py @@ -264,16 +264,6 @@ def clean(self): ) -class RequestFile(Document, BrewtilsRequestFile): - """Mongo backed request file resource""" - - STORAGE_ENGINES = ["gridfs"] - - storage_type = StringField(required=True, default="gridfs") - filename = StringField(required=True) - body = FileField(required=True) - - class Request(Document, BrewtilsRequest): """Mongo-Backed BREWMASTER Request Object""" @@ -423,6 +413,18 @@ def find_or_none(system_id): return None +class RequestFile(Document, BrewtilsRequestFile): + """Mongo backed request file resource""" + + STORAGE_ENGINES = ["gridfs"] + + storage_type = StringField(required=True, default="gridfs") + filename = StringField(required=True) + body = FileField(required=True) + created_at = DateTimeField(default=datetime.datetime.utcnow, required=True) + request = ReferenceField("Request", reverse_delete_rule=CASCADE) + + class System(Document, BrewtilsSystem): """Mongo-Backed BREWMASTER System Object"""