diff --git a/.circleci/config.yml b/.circleci/config.yml index 17222099e3..fa58cfff6e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -414,6 +414,8 @@ jobs: name: Install graphviz command: sudo apt-get install -y xvfb libgraphviz-dev graphviz - run: *wait_for_flowdb + # Ensure that flowmachine's .egg-info is created so that entry points are available + - run: pipenv install --dev - run: name: Run tests command: | diff --git a/flowmachine/Pipfile b/flowmachine/Pipfile index 8a0d692f4c..4219dac53f 100644 --- a/flowmachine/Pipfile +++ b/flowmachine/Pipfile @@ -42,6 +42,7 @@ cachey = "*" approvaltests = "*" watchdog = "*" ipdb = "*" +flowmachine = {editable = true,path = "."} [requires] python_version = "3.7" diff --git a/flowmachine/Pipfile.lock b/flowmachine/Pipfile.lock index fc7bf9c5f9..bb982d494f 100644 --- a/flowmachine/Pipfile.lock +++ b/flowmachine/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7d78fdaa1ac1cca66f1720215da150d549964ca6a5c94193be77db8c9d5e645e" + "sha256": "b4bbf35670805bee1a1b9d66ba71444fa3549c2ab042159b91c969b73055708d" }, "pipfile-spec": 6, "requires": { @@ -359,6 +359,21 @@ } }, "develop": { + "apispec": { + "hashes": [ + "sha256:419d0564b899e182c2af50483ea074db8cb05fee60838be58bb4542095d5c08d", + "sha256:9bf4e51d56c9067c60668b78210ae213894f060f85593dc2ad8805eb7d875a2a" + ], + "version": "==3.3.0" + }, + "apispec-oneofschema": { + "hashes": [ + "sha256:43b0d59951e9dd17f3fbbf762775e73f7fc1e8064248d9a3549067848adfa11b", + "sha256:9ac9ac402c2f31d9c023a99019d59bb30e1e409825c81d32db2e814676f7cf75" + ], + "index": "pypi", + "version": "==2.1.1" + }, "appdirs": { "hashes": [ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", @@ -411,6 +426,14 @@ "index": "pypi", "version": "==19.10b0" }, + "cachetools": { + "hashes": [ + "sha256:1d057645db16ca7fe1f3bd953558897603d6f0b9c51ed9d11eb4d071ec4e2aab", + "sha256:de5d88f87781602201cde465d3afe837546663b168e8b39df67411b0bf10cefc" + ], + "index": "pypi", + "version": "==4.1.0" + }, "cachey": { "hashes": [ "sha256:0310ba8afe52729fa7626325c8d8356a8421c434bf887ac851e58dcf7cf056a6", @@ -520,6 +543,13 @@ ], "version": "==3.0.12" }, + "finist": { + "hashes": [ + "sha256:795fc4d9f73e6e0e1af4c151d577f4a7b31bd591cd7163e1a99bb4a17faf2742" + ], + "index": "pypi", + "version": "==0.1.2" + }, "fiona": { "hashes": [ "sha256:1a432bf9fd56f089256c010da009c90d4a795c531a848132c965052185336600", @@ -536,6 +566,10 @@ ], "version": "==1.8.13.post1" }, + "flowmachine": { + "editable": true, + "path": "." + }, "geojson": { "hashes": [ "sha256:6e4bb7ace4226a45d9c8c8b1348b3fc43540658359f93c3f7e03efa9f15f658a", @@ -552,6 +586,13 @@ "index": "pypi", "version": "==0.7.0" }, + "get-secret-or-env-var": { + "hashes": [ + "sha256:669e85819ac680e980df7161b4a3b98ddd7253c703e8dbf2b16f36dea3214c60" + ], + "index": "pypi", + "version": "==0.0.2" + }, "heapdict": { "hashes": [ "sha256:6065f90933ab1bb7e50db403b90cab653c853690c5992e69294c2de2b253fc92", @@ -646,6 +687,22 @@ ], "version": "==1.2.0" }, + "marshmallow": { + "hashes": [ + "sha256:90854221bbb1498d003a0c3cc9d8390259137551917961c8b5258c64026b2f85", + "sha256:ac2e13b30165501b7d41fc0371b8df35944f5849769d136f20e2c5f6cdc6e665" + ], + "index": "pypi", + "version": "==3.5.1" + }, + "marshmallow-oneofschema": { + "hashes": [ + "sha256:29673bcc415f4a2098e273bf3c8229bc1a33e91378885eac90b06943bd626110", + "sha256:b88daa4f2de249e0a51a617aa09490f896acaddef6f6cf97dda78218c5abb86f" + ], + "index": "pypi", + "version": "==2.0.1" + }, "matplotlib": { "hashes": [ "sha256:2466d4dddeb0f5666fd1e6736cc5287a4f9f7ae6c1a9e0779deff798b28e1d35", @@ -679,6 +736,14 @@ ], "version": "==2.5.0" }, + "networkx": { + "hashes": [ + "sha256:cdfbf698749a5014bf2ed9db4a07a5295df1d3a53bf80bf3cbd61edf9df05fa1", + "sha256:f8f4ff0b6f96e4f9b16af6b84622597b5334bf9cae8cf9b2e42e7985d5c95c64" + ], + "index": "pypi", + "version": "==2.4" + }, "nodeenv": { "hashes": [ "sha256:5b2438f2e42af54ca968dd1b374d14a1194848955187b0e5e4be1f73813a5212" @@ -769,6 +834,29 @@ "markers": "sys_platform != 'win32'", "version": "==4.8.0" }, + "pglast": { + "hashes": [ + "sha256:0eeb140abe24148f5ec771cd6acc7ab5efc84541c18c02144a5a0015d11e915b", + "sha256:100fe6319ac4fb37d53e24f19767ee9556eb59b386c027b84b61306897c896b3", + "sha256:296f2994c136ceb8f52cf98de73ada2da40c06eddd7598207194aa75f8c0b6a7", + "sha256:327dab64fd177eef0eccc10959c352ba22c8ef70f1eaca9f712d58573b326549", + "sha256:3cff2142c06ddaf7b0ff04e6a35a0d4c2989431f052b0fea2c3bc79469b81837", + "sha256:4495b5ad82ac18737df7729e8f360291ad5d5b28478d01dc073af5ea4bf0397a", + "sha256:5aba6a94489526b66c2c690d686e88b73bfd5f601b4d98e8c8c4da378777be59", + "sha256:5d183d2259969f5ad9c81955c1d628f134ad9c3dafc97883eec6ada30eff17fc", + "sha256:70b4e50d355052181f690a12ba1700268b428cc984da75d6598fbf0737571a94", + "sha256:88f74bea8986803c832f82f00da927a8bafad43c9b9f006223c21b34d0678f4b", + "sha256:89dd5f5d4b5b2a1a4cefa483ab234aa3acbbdf0cbd2931a8fae1f7258627a132", + "sha256:a26ba77127b363446955e8a5317b3194defb1c1bb9d2ed5e7d4830fd4f066d97", + "sha256:b59009d3acf412c0cda7a9c8c5d01b256a3d6638dffa52e8c8591ca4b534ea49", + "sha256:bf10846005409cd037c804e278205dad069b20dc2aafeb273c2275cb137671a9", + "sha256:c3578e93bfa81118e095319d454fb5d0a4e826c58fef5675ed9e89fa1a579f65", + "sha256:fcfb5b6844c9350fcafbe30c0251e401b72b8b12cb8a1d39984f3c4083402b31", + "sha256:fee345b584317e4ef301f3242713e0edaeb8530c02c5874d3beb2882ea22fdd9" + ], + "index": "pypi", + "version": "==1.10" + }, "pickleshare": { "hashes": [ "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca", @@ -798,6 +886,42 @@ ], "version": "==3.0.5" }, + "psycopg2-binary": { + "hashes": [ + "sha256:008da3ab51adc70a5f1cfbbe5db3a22607ab030eb44bcecf517ad11a0c2b3cac", + "sha256:07cf82c870ec2d2ce94d18e70c13323c89f2f2a2628cbf1feee700630be2519a", + "sha256:08507efbe532029adee21b8d4c999170a83760d38249936038bd0602327029b5", + "sha256:107d9be3b614e52a192719c6bf32e8813030020ea1d1215daa86ded9a24d8b04", + "sha256:17a0ea0b0eabf07035e5e0d520dabc7950aeb15a17c6d36128ba99b2721b25b1", + "sha256:3286541b9d85a340ee4ed42732d15fc1bb441dc500c97243a768154ab8505bb5", + "sha256:3939cf75fc89c5e9ed836e228c4a63604dff95ad19aed2bbf71d5d04c15ed5ce", + "sha256:40abc319f7f26c042a11658bf3dd3b0b3bceccf883ec1c565d5c909a90204434", + "sha256:51f7823f1b087d2020d8e8c9e6687473d3d239ba9afc162d9b2ab6e80b53f9f9", + "sha256:6bb2dd006a46a4a4ce95201f836194eb6a1e863f69ee5bab506673e0ca767057", + "sha256:702f09d8f77dc4794651f650828791af82f7c2efd8c91ae79e3d9fe4bb7d4c98", + "sha256:7036ccf715925251fac969f4da9ad37e4b7e211b1e920860148a10c0de963522", + "sha256:7b832d76cc65c092abd9505cc670c4e3421fd136fb6ea5b94efbe4c146572505", + "sha256:8f74e631b67482d504d7e9cf364071fc5d54c28e79a093ff402d5f8f81e23bfa", + "sha256:930315ac53dc65cbf52ab6b6d27422611f5fb461d763c531db229c7e1af6c0b3", + "sha256:96d3038f5bd061401996614f65d27a4ecb62d843eb4f48e212e6d129171a721f", + "sha256:a20299ee0ea2f9cca494396ac472d6e636745652a64a418b39522c120fd0a0a4", + "sha256:a34826d6465c2e2bbe9d0605f944f19d2480589f89863ed5f091943be27c9de4", + "sha256:a69970ee896e21db4c57e398646af9edc71c003bc52a3cc77fb150240fefd266", + "sha256:b9a8b391c2b0321e0cd7ec6b4cfcc3dd6349347bd1207d48bcb752aa6c553a66", + "sha256:ba13346ff6d3eb2dca0b6fa0d8a9d999eff3dcd9b55f3a890f12b0b6362b2b38", + "sha256:bb0608694a91db1e230b4a314e8ed00ad07ed0c518f9a69b83af2717e31291a3", + "sha256:c8830b7d5f16fd79d39b21e3d94f247219036b29b30c8270314c46bf8b732389", + "sha256:cac918cd7c4c498a60f5d2a61d4f0a6091c2c9490d81bc805c963444032d0dab", + "sha256:cc30cb900f42c8a246e2cb76539d9726f407330bc244ca7729c41a44e8d807fb", + "sha256:ccdc6a87f32b491129ada4b87a43b1895cf2c20fdb7f98ad979647506ffc41b6", + "sha256:d1a8b01f6a964fec702d6b6dac1f91f2b9f9fe41b310cbb16c7ef1fac82df06d", + "sha256:e004db88e5a75e5fdab1620fb9f90c9598c2a195a594225ac4ed2a6f1c23e162", + "sha256:eb2f43ae3037f1ef5e19339c41cf56947021ac892f668765cd65f8ab9814192e", + "sha256:fa466306fcf6b39b8a61d003123d442b23707d635a5cb05ac4e1b62cc79105cd" + ], + "index": "pypi", + "version": "==2.8.5" + }, "ptyprocess": { "hashes": [ "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0", @@ -899,6 +1023,48 @@ "index": "pypi", "version": "==2.8.1" }, + "python-louvain": { + "hashes": [ + "sha256:e0a50c7a1e87b45410cd2a1d6298268862e55edaa46f90ae6b834bbb62df0951" + ], + "index": "pypi", + "version": "==0.14" + }, + "python-rapidjson": { + "hashes": [ + "sha256:0e129f8c5a098aa7d066fe7ca3a99411010bf3ca1dc17525d4869c77e2dd4351", + "sha256:0fc885de12a35699ed7e4ec9fa56ec791d1d2404269398bf989543e6b324c7fd", + "sha256:12eab9675a75dd14c07ee851429a5465f051aa1ebf66ade093fb15f0dc3159a6", + "sha256:26bfff41f0206f6f1df42c31a4c0d7c8de1e0fb57731707a0e04fa094f8bb7e1", + "sha256:292fdd82d4cc3e81b01b46717b969fbba56d19e70708e545e7d86c70cc11596a", + "sha256:2c3f37212499fab32abc3d53d591231e9fe99668f6a93faf8d17de30cdaf80a3", + "sha256:3eee83134adfec5644821c10f776839ee78f9321c691af0116d8fb4331caeb32", + "sha256:47bb70892452a14729a76e08bb60ba5f9628f5c6ee6a9a0c648fa3c6e543f019", + "sha256:4e00de6ae6a4c0f91a9096fdea79d4c074166578a431ad4141b678d4531be081", + "sha256:541935d6ef72d6e665acbb649144d4f7aa364e54b0d3c6689b5d427bec2b07c2", + "sha256:5647c1df3cd2253d16192795c61aeb7be37c07d007179aaeac014992cddc937c", + "sha256:602904892022bef8b1a1e4dc633c052e400d66652fcf0313167d4e227718629c", + "sha256:680a476d41f95cdc7292d17fcb1fcf54445f08e9ca1c5fba2a6067016b42a859", + "sha256:70fbeacd28582a9be0b1fe521c14a9fa7d62d624616325de9c55dc53001f6715", + "sha256:7321bd3b7c96417f0c3d83c1b40d350e0f4893549881bdb384dadf08034f7975", + "sha256:842c2298f91ecae60925bb133ad5dfb2793dcb909266b88d1d30eaabfd3f5199", + "sha256:887026e57e5a142d31d06087ecee556062d83ee1c4dca81230c8d2a6a4cefc40", + "sha256:99c7896f14ddd7592842da5181ae6b2cf4362159cde7993875b72b93818a4840", + "sha256:ad554220dd5903f67765f69698c9c0be50ad67cc989ed9f406c2c19c73d8f2f3", + "sha256:ad80bd7e4bb15d9705227630037a433e2e2a7982b54b51de2ebabdd1611394a1", + "sha256:b11b523ba060fecadf306eeea597645fed287369971bddac1e5bb76e9bff016d", + "sha256:b8d26ceb35495a885c4dcf8d28946ae2a878dc749d3c0fedbf0df6e6f5d99f06", + "sha256:c31e659b9720358533e2774fac12f5e5c4ea8cd45e6629fc8eccad0d8f421552", + "sha256:c687dd3b2566dd8fbee4d24b484d35e17b4be9edea720a6601fcf65f39f9b1cd", + "sha256:c7d509a8c6192c6e57b5e2c005e8c6dd8d2f988776b3444af42a39ffda38f6a8", + "sha256:c86d6b499603ece6c487cb45d757f7ee1cc8f8a5dc9e1a5548ce54cd763ffe48", + "sha256:dab0d99456537e180da9f311aff70712a165c6c362dc4da5fd766ab53869bb64", + "sha256:f81061d34e1510ab284e0c447e7b68dc6f3989260f57d84b3f335e10433692ca", + "sha256:fc43e5376d8a1243c82e27397011bf03297e94242d45fad2f7b7b1561bf29615" + ], + "index": "pypi", + "version": "==0.9.1" + }, "pytz": { "hashes": [ "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", @@ -957,6 +1123,14 @@ "index": "pypi", "version": "==19.0.0" }, + "redis": { + "hashes": [ + "sha256:0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f", + "sha256:b205cffd05ebfd0a468db74f0eedbff8df1a7bfc47521516ade4692991bb0833" + ], + "index": "pypi", + "version": "==3.4.1" + }, "regex": { "hashes": [ "sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b", @@ -1015,6 +1189,39 @@ ], "version": "==1.14.0" }, + "sqlalchemy": { + "hashes": [ + "sha256:083e383a1dca8384d0ea6378bd182d83c600ed4ff4ec8247d3b2442cf70db1ad", + "sha256:0a690a6486658d03cc6a73536d46e796b6570ac1f8a7ec133f9e28c448b69828", + "sha256:114b6ace30001f056e944cebd46daef38fdb41ebb98f5e5940241a03ed6cad43", + "sha256:128f6179325f7597a46403dde0bf148478f868df44841348dfc8d158e00db1f9", + "sha256:13d48cd8b925b6893a4e59b2dfb3e59a5204fd8c98289aad353af78bd214db49", + "sha256:211a1ce7e825f7142121144bac76f53ac28b12172716a710f4bf3eab477e730b", + "sha256:2dc57ee80b76813759cccd1a7affedf9c4dbe5b065a91fb6092c9d8151d66078", + "sha256:3e625e283eecc15aee5b1ef77203bfb542563fa4a9aa622c7643c7b55438ff49", + "sha256:43078c7ec0457387c79b8d52fff90a7ad352ca4c7aa841c366238c3e2cf52fdf", + "sha256:5b1bf3c2c2dca738235ce08079783ef04f1a7fc5b21cf24adaae77f2da4e73c3", + "sha256:6056b671aeda3fc451382e52ab8a753c0d5f66ef2a5ccc8fa5ba7abd20988b4d", + "sha256:68d78cf4a9dfade2e6cf57c4be19f7b82ed66e67dacf93b32bb390c9bed12749", + "sha256:7025c639ce7e170db845e94006cf5f404e243e6fc00d6c86fa19e8ad8d411880", + "sha256:7224e126c00b8178dfd227bc337ba5e754b197a3867d33b9f30dc0208f773d70", + "sha256:7d98e0785c4cd7ae30b4a451416db71f5724a1839025544b4edbd92e00b91f0f", + "sha256:8d8c21e9d4efef01351bf28513648ceb988031be4159745a7ad1b3e28c8ff68a", + "sha256:bbb545da054e6297242a1bb1ba88e7a8ffb679f518258d66798ec712b82e4e07", + "sha256:d00b393f05dbd4ecd65c989b7f5a81110eae4baea7a6a4cdd94c20a908d1456e", + "sha256:e18752cecaef61031252ca72031d4d6247b3212ebb84748fc5d1a0d2029c23ea" + ], + "index": "pypi", + "version": "==1.3.16" + }, + "structlog": { + "hashes": [ + "sha256:7a48375db6274ed1d0ae6123c486472aa1d0890b08d314d2b016f3aa7f35990b", + "sha256:8a672be150547a93d90a7d74229a29e765be05bd156a35cdcc527ebf68e9af92" + ], + "index": "pypi", + "version": "==20.1.0" + }, "toml": { "hashes": [ "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", diff --git a/flowmachine/flowmachine/core/server/action_handlers.py b/flowmachine/flowmachine/core/server/action_handlers.py index 6e0be3353c..6539777b32 100644 --- a/flowmachine/flowmachine/core/server/action_handlers.py +++ b/flowmachine/flowmachine/core/server/action_handlers.py @@ -32,8 +32,8 @@ from flowmachine.core.query_state import QueryStateMachine, QueryState from flowmachine.utils import convert_dict_keys_to_strings from .exceptions import FlowmachineServerError -from .query_schemas import FlowmachineQuerySchema, GeographySchema -from .query_schemas.flowmachine_query import get_query_schema +from .query_schemas.flowmachine_query import get_query_schema, FlowmachineQuerySchema +from .query_schemas.geography import GeographySchema from .zmq_helpers import ZMQReply __all__ = ["perform_action"] diff --git a/flowmachine/flowmachine/core/server/query_schemas/__init__.py b/flowmachine/flowmachine/core/server/query_schemas/__init__.py index 4ba7bdb6ea..6fbe8159b2 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/__init__.py +++ b/flowmachine/flowmachine/core/server/query_schemas/__init__.py @@ -1,7 +1,3 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from .base_exposed_query import BaseExposedQuery -from .flowmachine_query import FlowmachineQuerySchema -from .geography import GeographySchema diff --git a/flowmachine/flowmachine/core/server/query_schemas/active_at_reference_location_counts.py b/flowmachine/flowmachine/core/server/query_schemas/active_at_reference_location_counts.py index 06fda53a9a..b813d5af6d 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/active_at_reference_location_counts.py +++ b/flowmachine/flowmachine/core/server/query_schemas/active_at_reference_location_counts.py @@ -15,13 +15,13 @@ RedactedActiveAtReferenceLocationCounts, ) -from . import BaseExposedQuery - __all__ = [ "ActiveAtReferenceLocationCountsSchema", "ActiveAtReferenceLocationCountsExposed", ] +from .base_exposed_query import BaseExposedQuery + from .base_schema import BaseSchema from .reference_location import ReferenceLocationSchema diff --git a/flowmachine/flowmachine/core/server/query_schemas/consecutive_trips_od_matrix.py b/flowmachine/flowmachine/core/server/query_schemas/consecutive_trips_od_matrix.py index 9b77695bb7..627b54664a 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/consecutive_trips_od_matrix.py +++ b/flowmachine/flowmachine/core/server/query_schemas/consecutive_trips_od_matrix.py @@ -12,7 +12,7 @@ from flowmachine.features.location.consecutive_trips_od_matrix import ( ConsecutiveTripsODMatrix, ) -from . import BaseExposedQuery +from .base_exposed_query import BaseExposedQuery from .base_schema import BaseSchema from .custom_fields import SubscriberSubset, ISODateTime from .aggregation_unit import AggregationUnit, get_spatial_unit_obj diff --git a/flowmachine/flowmachine/core/server/query_schemas/displacement.py b/flowmachine/flowmachine/core/server/query_schemas/displacement.py index 1e1215a6ce..c6ba5f713f 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/displacement.py +++ b/flowmachine/flowmachine/core/server/query_schemas/displacement.py @@ -4,29 +4,20 @@ from marshmallow import fields from marshmallow.validate import OneOf -from marshmallow_oneofschema import OneOfSchema from flowmachine.features import Displacement from .custom_fields import SubscriberSubset, Statistic, ISODateTime -from .daily_location import DailyLocationSchema -from .modal_location import ModalLocationSchema + from .base_query_with_sampling import ( BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, ) - +from .metric_types import ContinuousMetric +from .reference_location import ReferenceLocationSchema __all__ = ["DisplacementSchema", "DisplacementExposed"] -class InputToDisplacementSchema(OneOfSchema): - type_field = "query_kind" - type_schemas = { - "daily_location": DailyLocationSchema, - "modal_location": ModalLocationSchema, - } - - class DisplacementExposed(BaseExposedQueryWithSampling): def __init__( self, @@ -65,12 +56,12 @@ def _unsampled_query_obj(self): ) -class DisplacementSchema(BaseQueryWithSamplingSchema): +class DisplacementSchema(ContinuousMetric, BaseQueryWithSamplingSchema): query_kind = fields.String(validate=OneOf(["displacement"])) start = ISODateTime(required=True) stop = ISODateTime(required=True) statistic = Statistic() - reference_location = fields.Nested(InputToDisplacementSchema, many=False) + reference_location = fields.Nested(ReferenceLocationSchema, many=False) subscriber_subset = SubscriberSubset() __model__ = DisplacementExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/event_count.py b/flowmachine/flowmachine/core/server/query_schemas/event_count.py index 767f610608..7d44724793 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/event_count.py +++ b/flowmachine/flowmachine/core/server/query_schemas/event_count.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from marshmallow import fields, post_load +from marshmallow import fields from marshmallow.validate import OneOf from flowmachine.features import EventCount @@ -11,6 +11,7 @@ BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, ) +from .metric_types import ContinuousMetric __all__ = ["EventCountSchema", "EventCountExposed"] @@ -53,7 +54,7 @@ def _unsampled_query_obj(self): ) -class EventCountSchema(BaseQueryWithSamplingSchema): +class EventCountSchema(ContinuousMetric, BaseQueryWithSamplingSchema): query_kind = fields.String(validate=OneOf(["event_count"])) start = ISODateTime(required=True) stop = ISODateTime(required=True) diff --git a/flowmachine/flowmachine/core/server/query_schemas/flowable_queries.py b/flowmachine/flowmachine/core/server/query_schemas/flowable_queries.py new file mode 100644 index 0000000000..3698f4c600 --- /dev/null +++ b/flowmachine/flowmachine/core/server/query_schemas/flowable_queries.py @@ -0,0 +1,13 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from flowmachine.core.server.query_schemas.daily_location import DailyLocationSchema +from flowmachine.core.server.query_schemas.modal_location import ModalLocationSchema +from flowmachine.core.server.query_schemas.unique_locations import UniqueLocationsSchema + +flowable_queries = { + "daily_location": DailyLocationSchema, + "modal_location": ModalLocationSchema, + "unique_locations": UniqueLocationsSchema, +} diff --git a/flowmachine/flowmachine/core/server/query_schemas/flowmachine_query.py b/flowmachine/flowmachine/core/server/query_schemas/flowmachine_query.py index 0312ba7ee3..bd5e014d1c 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/flowmachine_query.py +++ b/flowmachine/flowmachine/core/server/query_schemas/flowmachine_query.py @@ -8,64 +8,12 @@ from marshmallow_oneofschema import OneOfSchema -from flowmachine.core.server.query_schemas.joined_spatial_aggregate import ( - JoinedSpatialAggregateSchema, -) -from flowmachine.core.server.query_schemas.spatial_aggregate import ( - SpatialAggregateSchema, -) -from .histogram_aggregate import HistogramAggregateSchema -from .active_at_reference_location_counts import ActiveAtReferenceLocationCountsSchema -from .consecutive_trips_od_matrix import ConsecutiveTripsODMatrixSchema -from .dummy_query import DummyQuerySchema -from .flows import FlowsSchema -from .meaningful_locations import ( - MeaningfulLocationsAggregateSchema, - MeaningfulLocationsBetweenLabelODMatrixSchema, - MeaningfulLocationsBetweenDatesODMatrixSchema, -) - -from .aggregate_network_objects import AggregateNetworkObjectsSchema - -from .geography import GeographySchema -from .location_event_counts import LocationEventCountsSchema -from .trips_od_matrix import TripsODMatrixSchema -from .unique_subscriber_counts import UniqueSubscriberCountsSchema -from .location_introversion import LocationIntroversionSchema -from .total_network_objects import TotalNetworkObjectsSchema -from .dfs_metric_total_amount import DFSTotalMetricAmountSchema -from .unique_visitor_counts import UniqueVisitorCountsSchema -from .unmoving_at_reference_location_counts import ( - UnmovingAtReferenceLocationCountsSchema, -) -from .unmoving_counts import UnmovingCountsSchema +from .util import get_type_schemas_from_entrypoint class FlowmachineQuerySchema(OneOfSchema): type_field = "query_kind" - type_schemas = { - "dummy_query": DummyQuerySchema, - "flows": FlowsSchema, - "meaningful_locations_aggregate": MeaningfulLocationsAggregateSchema, - "meaningful_locations_between_label_od_matrix": MeaningfulLocationsBetweenLabelODMatrixSchema, - "meaningful_locations_between_dates_od_matrix": MeaningfulLocationsBetweenDatesODMatrixSchema, - "geography": GeographySchema, - "location_event_counts": LocationEventCountsSchema, - "unique_subscriber_counts": UniqueSubscriberCountsSchema, - "location_introversion": LocationIntroversionSchema, - "total_network_objects": TotalNetworkObjectsSchema, - "aggregate_network_objects": AggregateNetworkObjectsSchema, - "dfs_metric_total_amount": DFSTotalMetricAmountSchema, - "spatial_aggregate": SpatialAggregateSchema, - "joined_spatial_aggregate": JoinedSpatialAggregateSchema, - "histogram_aggregate": HistogramAggregateSchema, - "active_at_reference_location_counts": ActiveAtReferenceLocationCountsSchema, - "unique_visitor_counts": UniqueVisitorCountsSchema, - "consecutive_trips_od_matrix": ConsecutiveTripsODMatrixSchema, - "unmoving_counts": UnmovingCountsSchema, - "unmoving_at_reference_location_counts": UnmovingAtReferenceLocationCountsSchema, - "trips_od_matrix": TripsODMatrixSchema, - } + type_schemas = get_type_schemas_from_entrypoint("top_level_queries") @lru_cache(maxsize=1) diff --git a/flowmachine/flowmachine/core/server/query_schemas/flows.py b/flowmachine/flowmachine/core/server/query_schemas/flows.py index cbd8d50149..7a66309b25 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/flows.py +++ b/flowmachine/flowmachine/core/server/query_schemas/flows.py @@ -11,21 +11,14 @@ from flowmachine.features.location.redacted_flows import RedactedFlows from .base_exposed_query import BaseExposedQuery from .base_schema import BaseSchema -from .daily_location import DailyLocationSchema -from .modal_location import ModalLocationSchema +from .util import get_type_schemas_from_entrypoint __all__ = ["FlowsSchema", "FlowsExposed"] -from .unique_locations import UniqueLocationsSchema - class InputToFlowsSchema(OneOfSchema): type_field = "query_kind" - type_schemas = { - "daily_location": DailyLocationSchema, - "modal_location": ModalLocationSchema, - "unique_locations": UniqueLocationsSchema, - } + type_schemas = get_type_schemas_from_entrypoint("flowable_queries") class FlowsExposed(BaseExposedQuery): diff --git a/flowmachine/flowmachine/core/server/query_schemas/handset.py b/flowmachine/flowmachine/core/server/query_schemas/handset.py index 005dcdd745..1343b704f7 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/handset.py +++ b/flowmachine/flowmachine/core/server/query_schemas/handset.py @@ -11,6 +11,7 @@ BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, ) +from .metric_types import CategoricalMetric __all__ = ["HandsetSchema", "HandsetExposed"] @@ -53,7 +54,7 @@ def _unsampled_query_obj(self): ) -class HandsetSchema(BaseQueryWithSamplingSchema): +class HandsetSchema(CategoricalMetric, BaseQueryWithSamplingSchema): # query_kind parameter is required here for claims validation query_kind = fields.String(validate=OneOf(["handset"])) start_date = ISODateTime(required=True) diff --git a/flowmachine/flowmachine/core/server/query_schemas/histogram_aggregate.py b/flowmachine/flowmachine/core/server/query_schemas/histogram_aggregate.py index 0f997fda4d..290c3bb86a 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/histogram_aggregate.py +++ b/flowmachine/flowmachine/core/server/query_schemas/histogram_aggregate.py @@ -7,26 +7,11 @@ from marshmallow_oneofschema import OneOfSchema from flowmachine.core.server.query_schemas.custom_fields import Bounds -from flowmachine.core.server.query_schemas.radius_of_gyration import ( - RadiusOfGyrationSchema, -) -from flowmachine.core.server.query_schemas.subscriber_degree import ( - SubscriberDegreeSchema, -) -from flowmachine.core.server.query_schemas.topup_amount import TopUpAmountSchema -from flowmachine.core.server.query_schemas.event_count import EventCountSchema -from flowmachine.core.server.query_schemas.nocturnal_events import NocturnalEventsSchema -from flowmachine.core.server.query_schemas.unique_location_counts import ( - UniqueLocationCountsSchema, -) -from flowmachine.core.server.query_schemas.displacement import DisplacementSchema -from flowmachine.core.server.query_schemas.pareto_interactions import ( - ParetoInteractionsSchema, -) -from flowmachine.core.server.query_schemas.topup_balance import TopUpBalanceSchema + from flowmachine.features import HistogramAggregation from .base_exposed_query import BaseExposedQuery +from .util import get_type_schemas_from_entrypoint __all__ = ["HistogramAggregateSchema", "HistogramAggregateExposed"] @@ -36,17 +21,7 @@ class HistogrammableMetrics(OneOfSchema): type_field = "query_kind" - type_schemas = { - "radius_of_gyration": RadiusOfGyrationSchema, - "unique_location_counts": UniqueLocationCountsSchema, - "topup_balance": TopUpBalanceSchema, - "subscriber_degree": SubscriberDegreeSchema, - "topup_amount": TopUpAmountSchema, - "event_count": EventCountSchema, - "pareto_interactions": ParetoInteractionsSchema, - "nocturnal_events": NocturnalEventsSchema, - "displacement": DisplacementSchema, - } + type_schemas = get_type_schemas_from_entrypoint("histogrammable_queries") class HistogramBins(Schema): diff --git a/flowmachine/flowmachine/core/server/query_schemas/histogrammable_queries.py b/flowmachine/flowmachine/core/server/query_schemas/histogrammable_queries.py new file mode 100644 index 0000000000..8b0808d788 --- /dev/null +++ b/flowmachine/flowmachine/core/server/query_schemas/histogrammable_queries.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from flowmachine.core.server.query_schemas.displacement import DisplacementSchema +from flowmachine.core.server.query_schemas.event_count import EventCountSchema +from flowmachine.core.server.query_schemas.nocturnal_events import NocturnalEventsSchema +from flowmachine.core.server.query_schemas.pareto_interactions import ( + ParetoInteractionsSchema, +) +from flowmachine.core.server.query_schemas.radius_of_gyration import ( + RadiusOfGyrationSchema, +) +from flowmachine.core.server.query_schemas.subscriber_degree import ( + SubscriberDegreeSchema, +) +from flowmachine.core.server.query_schemas.topup_amount import TopUpAmountSchema +from flowmachine.core.server.query_schemas.topup_balance import TopUpBalanceSchema +from flowmachine.core.server.query_schemas.unique_location_counts import ( + UniqueLocationCountsSchema, +) + +histogrammable_queries = { + "radius_of_gyration": RadiusOfGyrationSchema, + "unique_location_counts": UniqueLocationCountsSchema, + "topup_balance": TopUpBalanceSchema, + "subscriber_degree": SubscriberDegreeSchema, + "topup_amount": TopUpAmountSchema, + "event_count": EventCountSchema, + "pareto_interactions": ParetoInteractionsSchema, + "nocturnal_events": NocturnalEventsSchema, + "displacement": DisplacementSchema, +} diff --git a/flowmachine/flowmachine/core/server/query_schemas/joinable_queries.py b/flowmachine/flowmachine/core/server/query_schemas/joinable_queries.py new file mode 100644 index 0000000000..ad53886d04 --- /dev/null +++ b/flowmachine/flowmachine/core/server/query_schemas/joinable_queries.py @@ -0,0 +1,31 @@ +from flowmachine.core.server.query_schemas.radius_of_gyration import ( + RadiusOfGyrationSchema, +) +from flowmachine.core.server.query_schemas.subscriber_degree import ( + SubscriberDegreeSchema, +) +from flowmachine.core.server.query_schemas.topup_amount import TopUpAmountSchema +from flowmachine.core.server.query_schemas.event_count import EventCountSchema +from flowmachine.core.server.query_schemas.handset import HandsetSchema +from flowmachine.core.server.query_schemas.nocturnal_events import NocturnalEventsSchema +from flowmachine.core.server.query_schemas.unique_location_counts import ( + UniqueLocationCountsSchema, +) +from flowmachine.core.server.query_schemas.displacement import DisplacementSchema +from flowmachine.core.server.query_schemas.pareto_interactions import ( + ParetoInteractionsSchema, +) +from flowmachine.core.server.query_schemas.topup_balance import TopUpBalanceSchema + +joinable_queries = { + "radius_of_gyration": RadiusOfGyrationSchema, + "unique_location_counts": UniqueLocationCountsSchema, + "topup_balance": TopUpBalanceSchema, + "subscriber_degree": SubscriberDegreeSchema, + "topup_amount": TopUpAmountSchema, + "event_count": EventCountSchema, + "handset": HandsetSchema, + "pareto_interactions": ParetoInteractionsSchema, + "nocturnal_events": NocturnalEventsSchema, + "displacement": DisplacementSchema, +} diff --git a/flowmachine/flowmachine/core/server/query_schemas/joined_spatial_aggregate.py b/flowmachine/flowmachine/core/server/query_schemas/joined_spatial_aggregate.py index 3d036495ff..ded09abfd7 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/joined_spatial_aggregate.py +++ b/flowmachine/flowmachine/core/server/query_schemas/joined_spatial_aggregate.py @@ -2,31 +2,16 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from marshmallow import fields, pre_load, ValidationError + +from marshmallow import ( + fields, + ValidationError, + validates_schema, +) from marshmallow.validate import OneOf from marshmallow_oneofschema import OneOfSchema -from flowmachine.core.server.query_schemas.radius_of_gyration import ( - RadiusOfGyrationSchema, -) -from flowmachine.core.server.query_schemas.subscriber_degree import ( - SubscriberDegreeSchema, -) -from flowmachine.core.server.query_schemas.topup_amount import TopUpAmountSchema -from flowmachine.core.server.query_schemas.event_count import EventCountSchema -from flowmachine.core.server.query_schemas.handset import HandsetSchema -from flowmachine.core.server.query_schemas.nocturnal_events import NocturnalEventsSchema -from flowmachine.core.server.query_schemas.unique_location_counts import ( - UniqueLocationCountsSchema, -) -from flowmachine.core.server.query_schemas.displacement import DisplacementSchema -from flowmachine.core.server.query_schemas.pareto_interactions import ( - ParetoInteractionsSchema, -) -from flowmachine.core.server.query_schemas.topup_balance import TopUpBalanceSchema -from flowmachine.core.server.query_schemas.spatial_aggregate import ( - InputToSpatialAggregate, -) + from flowmachine.features.location.joined_spatial_aggregate import ( JoinedSpatialAggregate, ) @@ -34,27 +19,17 @@ RedactedJoinedSpatialAggregate, ) from .base_exposed_query import BaseExposedQuery +from .base_schema import BaseSchema +from .reference_location import ReferenceLocationSchema +from .util import get_type_schemas_from_entrypoint __all__ = ["JoinedSpatialAggregateSchema", "JoinedSpatialAggregateExposed"] -from .base_schema import BaseSchema - class JoinableMetrics(OneOfSchema): type_field = "query_kind" - type_schemas = { - "radius_of_gyration": RadiusOfGyrationSchema, - "unique_location_counts": UniqueLocationCountsSchema, - "topup_balance": TopUpBalanceSchema, - "subscriber_degree": SubscriberDegreeSchema, - "topup_amount": TopUpAmountSchema, - "event_count": EventCountSchema, - "handset": HandsetSchema, - "pareto_interactions": ParetoInteractionsSchema, - "nocturnal_events": NocturnalEventsSchema, - "displacement": DisplacementSchema, - } + type_schemas = get_type_schemas_from_entrypoint("joinable_queries") class JoinedSpatialAggregateExposed(BaseExposedQuery): @@ -86,35 +61,23 @@ def _flowmachine_query_obj(self): class JoinedSpatialAggregateSchema(BaseSchema): # query_kind parameter is required here for claims validation query_kind = fields.String(validate=OneOf(["joined_spatial_aggregate"])) - locations = fields.Nested(InputToSpatialAggregate, required=True) + locations = fields.Nested(ReferenceLocationSchema, required=True) metric = fields.Nested(JoinableMetrics, required=True) method = fields.String(validate=OneOf(JoinedSpatialAggregate.allowed_methods)) - @pre_load - def validate_method(self, data, **kwargs): - continuous_metrics = [ - "radius_of_gyration", - "unique_location_counts", - "topup_balance", - "subscriber_degree", - "topup_amount", - "event_count", - "nocturnal_events", - "pareto_interactions", - "displacement", - ] - categorical_metrics = ["handset"] - if data["metric"]["query_kind"] in continuous_metrics: - validate = OneOf( - ["avg", "max", "min", "median", "mode", "stddev", "variance"] - ) - elif data["metric"]["query_kind"] in categorical_metrics: - validate = OneOf(["distr"]) - else: + @validates_schema(pass_original=True) + def validate_method(self, data, original_data, **kwargs): + try: + OneOf( + self._declared_fields["metric"] + .schema.type_schemas[original_data["metric"]["query_kind"]] + .valid_metrics + )(original_data["method"]) + except AttributeError: raise ValidationError( - f"{data['metric']['query_kind']} does not have a valid metric type." + f"{original_data['metric']['query_kind']} does not have a valid metric type." ) - validate(data["method"]) - return data + except ValidationError as exc: + raise ValidationError(exc.messages, "method") __model__ = JoinedSpatialAggregateExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/metric_types.py b/flowmachine/flowmachine/core/server/query_schemas/metric_types.py new file mode 100644 index 0000000000..088cbc186b --- /dev/null +++ b/flowmachine/flowmachine/core/server/query_schemas/metric_types.py @@ -0,0 +1,11 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +class ContinuousMetric: + valid_metrics = ["avg", "max", "min", "median", "mode", "stddev", "variance"] + + +class CategoricalMetric: + valid_metrics = ["distr"] diff --git a/flowmachine/flowmachine/core/server/query_schemas/nocturnal_events.py b/flowmachine/flowmachine/core/server/query_schemas/nocturnal_events.py index cf93727d57..be642817dc 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/nocturnal_events.py +++ b/flowmachine/flowmachine/core/server/query_schemas/nocturnal_events.py @@ -11,6 +11,7 @@ BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, ) +from .metric_types import ContinuousMetric __all__ = ["NocturnalEventsSchema", "NocturnalEventsExposed"] @@ -51,7 +52,7 @@ def _unsampled_query_obj(self): ) -class NocturnalEventsSchema(BaseQueryWithSamplingSchema): +class NocturnalEventsSchema(ContinuousMetric, BaseQueryWithSamplingSchema): query_kind = fields.String(validate=OneOf(["nocturnal_events"])) start = ISODateTime(required=True) stop = ISODateTime(required=True) diff --git a/flowmachine/flowmachine/core/server/query_schemas/pareto_interactions.py b/flowmachine/flowmachine/core/server/query_schemas/pareto_interactions.py index e9f4383cb4..f4a47f9650 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/pareto_interactions.py +++ b/flowmachine/flowmachine/core/server/query_schemas/pareto_interactions.py @@ -11,6 +11,7 @@ BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, ) +from .metric_types import ContinuousMetric __all__ = ["ParetoInteractionsSchema", "ParetoInteractionsExposed"] @@ -44,7 +45,7 @@ def _unsampled_query_obj(self): ) -class ParetoInteractionsSchema(BaseQueryWithSamplingSchema): +class ParetoInteractionsSchema(ContinuousMetric, BaseQueryWithSamplingSchema): query_kind = fields.String(validate=OneOf(["pareto_interactions"])) start = ISODateTime(required=True) stop = ISODateTime(required=True) diff --git a/flowmachine/flowmachine/core/server/query_schemas/radius_of_gyration.py b/flowmachine/flowmachine/core/server/query_schemas/radius_of_gyration.py index 069424ae98..dc548c2c1f 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/radius_of_gyration.py +++ b/flowmachine/flowmachine/core/server/query_schemas/radius_of_gyration.py @@ -11,6 +11,7 @@ BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, ) +from .metric_types import ContinuousMetric __all__ = ["RadiusOfGyrationSchema", "RadiusOfGyrationExposed"] @@ -40,7 +41,7 @@ def _unsampled_query_obj(self): ) -class RadiusOfGyrationSchema(BaseQueryWithSamplingSchema): +class RadiusOfGyrationSchema(ContinuousMetric, BaseQueryWithSamplingSchema): # query_kind parameter is required here for claims validation query_kind = fields.String(validate=OneOf(["radius_of_gyration"])) start_date = ISODateTime(required=True) diff --git a/flowmachine/flowmachine/core/server/query_schemas/reference_location.py b/flowmachine/flowmachine/core/server/query_schemas/reference_location.py index 51bd4e9b94..2b7ab493e9 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/reference_location.py +++ b/flowmachine/flowmachine/core/server/query_schemas/reference_location.py @@ -3,14 +3,9 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from marshmallow_oneofschema import OneOfSchema - -from flowmachine.core.server.query_schemas.daily_location import DailyLocationSchema -from flowmachine.core.server.query_schemas.modal_location import ModalLocationSchema +from flowmachine.core.server.query_schemas.util import get_type_schemas_from_entrypoint class ReferenceLocationSchema(OneOfSchema): type_field = "query_kind" - type_schemas = { - "daily_location": DailyLocationSchema, - "modal_location": ModalLocationSchema, - } + type_schemas = get_type_schemas_from_entrypoint("reference_location_queries") diff --git a/flowmachine/flowmachine/core/server/query_schemas/reference_location_queries.py b/flowmachine/flowmachine/core/server/query_schemas/reference_location_queries.py new file mode 100644 index 0000000000..b43855c73e --- /dev/null +++ b/flowmachine/flowmachine/core/server/query_schemas/reference_location_queries.py @@ -0,0 +1,11 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from flowmachine.core.server.query_schemas.daily_location import DailyLocationSchema +from flowmachine.core.server.query_schemas.modal_location import ModalLocationSchema + +reference_location_queries = { + "daily_location": DailyLocationSchema, + "modal_location": ModalLocationSchema, +} diff --git a/flowmachine/flowmachine/core/server/query_schemas/spatial_aggregate.py b/flowmachine/flowmachine/core/server/query_schemas/spatial_aggregate.py index eb234d90b2..710de124f0 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/spatial_aggregate.py +++ b/flowmachine/flowmachine/core/server/query_schemas/spatial_aggregate.py @@ -2,9 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from marshmallow import fields, post_load +from marshmallow import fields from marshmallow.validate import OneOf -from marshmallow_oneofschema import OneOfSchema from flowmachine.features.location.redacted_spatial_aggregate import ( RedactedSpatialAggregate, @@ -12,24 +11,14 @@ from flowmachine.features.location.spatial_aggregate import SpatialAggregate from .base_exposed_query import BaseExposedQuery from .base_schema import BaseSchema -from .daily_location import DailyLocationSchema -from .modal_location import ModalLocationSchema +from .reference_location import ReferenceLocationSchema __all__ = [ "SpatialAggregateSchema", "SpatialAggregateExposed", - "InputToSpatialAggregate", ] -class InputToSpatialAggregate(OneOfSchema): - type_field = "query_kind" - type_schemas = { - "daily_location": DailyLocationSchema, - "modal_location": ModalLocationSchema, - } - - class SpatialAggregateExposed(BaseExposedQuery): def __init__(self, *, locations): # Note: all input parameters need to be defined as attributes on `self` @@ -54,6 +43,6 @@ def _flowmachine_query_obj(self): class SpatialAggregateSchema(BaseSchema): # query_kind parameter is required here for claims validation query_kind = fields.String(validate=OneOf(["spatial_aggregate"])) - locations = fields.Nested(InputToSpatialAggregate, required=True) + locations = fields.Nested(ReferenceLocationSchema, required=True) __model__ = SpatialAggregateExposed diff --git a/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py b/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py index 4ab014059a..f8e8f53b09 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py +++ b/flowmachine/flowmachine/core/server/query_schemas/subscriber_degree.py @@ -11,6 +11,7 @@ BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, ) +from .metric_types import ContinuousMetric __all__ = ["SubscriberDegreeSchema", "SubscriberDegreeExposed"] @@ -44,7 +45,7 @@ def _unsampled_query_obj(self): ) -class SubscriberDegreeSchema(BaseQueryWithSamplingSchema): +class SubscriberDegreeSchema(ContinuousMetric, BaseQueryWithSamplingSchema): query_kind = fields.String(validate=OneOf(["subscriber_degree"])) start = ISODateTime(required=True) stop = ISODateTime(required=True) diff --git a/flowmachine/flowmachine/core/server/query_schemas/top_level_queries.py b/flowmachine/flowmachine/core/server/query_schemas/top_level_queries.py new file mode 100644 index 0000000000..4c4ad61b0f --- /dev/null +++ b/flowmachine/flowmachine/core/server/query_schemas/top_level_queries.py @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from flowmachine.core.server.query_schemas.active_at_reference_location_counts import ( + ActiveAtReferenceLocationCountsSchema, +) +from flowmachine.core.server.query_schemas.aggregate_network_objects import ( + AggregateNetworkObjectsSchema, +) +from flowmachine.core.server.query_schemas.consecutive_trips_od_matrix import ( + ConsecutiveTripsODMatrixSchema, +) +from flowmachine.core.server.query_schemas.dfs_metric_total_amount import ( + DFSTotalMetricAmountSchema, +) +from flowmachine.core.server.query_schemas.dummy_query import DummyQuerySchema +from flowmachine.core.server.query_schemas.flows import FlowsSchema +from flowmachine.core.server.query_schemas.geography import GeographySchema +from flowmachine.core.server.query_schemas.histogram_aggregate import ( + HistogramAggregateSchema, +) +from flowmachine.core.server.query_schemas.joined_spatial_aggregate import ( + JoinedSpatialAggregateSchema, +) +from flowmachine.core.server.query_schemas.location_event_counts import ( + LocationEventCountsSchema, +) +from flowmachine.core.server.query_schemas.location_introversion import ( + LocationIntroversionSchema, +) +from flowmachine.core.server.query_schemas.meaningful_locations import ( + MeaningfulLocationsAggregateSchema, + MeaningfulLocationsBetweenLabelODMatrixSchema, + MeaningfulLocationsBetweenDatesODMatrixSchema, +) +from flowmachine.core.server.query_schemas.spatial_aggregate import ( + SpatialAggregateSchema, +) +from flowmachine.core.server.query_schemas.total_network_objects import ( + TotalNetworkObjectsSchema, +) +from flowmachine.core.server.query_schemas.trips_od_matrix import TripsODMatrixSchema +from flowmachine.core.server.query_schemas.unique_subscriber_counts import ( + UniqueSubscriberCountsSchema, +) +from flowmachine.core.server.query_schemas.unique_visitor_counts import ( + UniqueVisitorCountsSchema, +) +from flowmachine.core.server.query_schemas.unmoving_at_reference_location_counts import ( + UnmovingAtReferenceLocationCountsSchema, +) +from flowmachine.core.server.query_schemas.unmoving_counts import UnmovingCountsSchema + +top_level_queries = { + "dummy_query": DummyQuerySchema, + "flows": FlowsSchema, + "meaningful_locations_aggregate": MeaningfulLocationsAggregateSchema, + "meaningful_locations_between_label_od_matrix": MeaningfulLocationsBetweenLabelODMatrixSchema, + "meaningful_locations_between_dates_od_matrix": MeaningfulLocationsBetweenDatesODMatrixSchema, + "geography": GeographySchema, + "location_event_counts": LocationEventCountsSchema, + "unique_subscriber_counts": UniqueSubscriberCountsSchema, + "location_introversion": LocationIntroversionSchema, + "total_network_objects": TotalNetworkObjectsSchema, + "aggregate_network_objects": AggregateNetworkObjectsSchema, + "dfs_metric_total_amount": DFSTotalMetricAmountSchema, + "spatial_aggregate": SpatialAggregateSchema, + "joined_spatial_aggregate": JoinedSpatialAggregateSchema, + "histogram_aggregate": HistogramAggregateSchema, + "active_at_reference_location_counts": ActiveAtReferenceLocationCountsSchema, + "unique_visitor_counts": UniqueVisitorCountsSchema, + "consecutive_trips_od_matrix": ConsecutiveTripsODMatrixSchema, + "unmoving_counts": UnmovingCountsSchema, + "unmoving_at_reference_location_counts": UnmovingAtReferenceLocationCountsSchema, + "trips_od_matrix": TripsODMatrixSchema, +} diff --git a/flowmachine/flowmachine/core/server/query_schemas/topup_amount.py b/flowmachine/flowmachine/core/server/query_schemas/topup_amount.py index 3de1812741..5674dca4b2 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/topup_amount.py +++ b/flowmachine/flowmachine/core/server/query_schemas/topup_amount.py @@ -11,6 +11,7 @@ BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, ) +from .metric_types import ContinuousMetric __all__ = ["TopUpAmountSchema", "TopUpAmountExposed"] @@ -44,7 +45,7 @@ def _unsampled_query_obj(self): ) -class TopUpAmountSchema(BaseQueryWithSamplingSchema): +class TopUpAmountSchema(ContinuousMetric, BaseQueryWithSamplingSchema): query_kind = fields.String(validate=OneOf(["topup_amount"])) start = ISODateTime(required=True) stop = ISODateTime(required=True) diff --git a/flowmachine/flowmachine/core/server/query_schemas/topup_balance.py b/flowmachine/flowmachine/core/server/query_schemas/topup_balance.py index 32f8dba010..077f7ab994 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/topup_balance.py +++ b/flowmachine/flowmachine/core/server/query_schemas/topup_balance.py @@ -11,6 +11,7 @@ BaseQueryWithSamplingSchema, BaseExposedQueryWithSampling, ) +from .metric_types import ContinuousMetric __all__ = ["TopUpBalanceSchema", "TopUpBalanceExposed"] @@ -50,7 +51,7 @@ def _unsampled_query_obj(self): ) -class TopUpBalanceSchema(BaseQueryWithSamplingSchema): +class TopUpBalanceSchema(ContinuousMetric, BaseQueryWithSamplingSchema): query_kind = fields.String(validate=OneOf(["topup_balance"])) start_date = ISODateTime(required=True) end_date = ISODateTime(required=True) diff --git a/flowmachine/flowmachine/core/server/query_schemas/trips_od_matrix.py b/flowmachine/flowmachine/core/server/query_schemas/trips_od_matrix.py index 3be42afa61..4f97767456 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/trips_od_matrix.py +++ b/flowmachine/flowmachine/core/server/query_schemas/trips_od_matrix.py @@ -8,7 +8,7 @@ from flowmachine.features.location.redacted_trips_od_matrix import RedactedTripsODMatrix from flowmachine.features.utilities.subscriber_locations import SubscriberLocations from flowmachine.features.location.trips_od_matrix import TripsODMatrix -from . import BaseExposedQuery +from .base_exposed_query import BaseExposedQuery from .base_schema import BaseSchema from .custom_fields import SubscriberSubset, ISODateTime from .aggregation_unit import AggregationUnit, get_spatial_unit_obj diff --git a/flowmachine/flowmachine/core/server/query_schemas/unique_location_counts.py b/flowmachine/flowmachine/core/server/query_schemas/unique_location_counts.py index c534dfc766..3feeee7385 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/unique_location_counts.py +++ b/flowmachine/flowmachine/core/server/query_schemas/unique_location_counts.py @@ -2,14 +2,15 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from marshmallow import fields, post_load +from marshmallow import fields from marshmallow.validate import OneOf from flowmachine.features import UniqueLocationCounts -from . import BaseExposedQuery +from .base_exposed_query import BaseExposedQuery from .base_schema import BaseSchema from .custom_fields import SubscriberSubset, ISODateTime from .aggregation_unit import AggregationUnit, get_spatial_unit_obj +from .metric_types import ContinuousMetric __all__ = ["UniqueLocationCountsSchema", "UniqueLocationCountsExposed"] @@ -48,7 +49,7 @@ def _flowmachine_query_obj(self): ) -class UniqueLocationCountsSchema(BaseSchema): +class UniqueLocationCountsSchema(ContinuousMetric, BaseSchema): query_kind = fields.String(validate=OneOf(["unique_location_counts"])) start_date = ISODateTime(required=True) end_date = ISODateTime(required=True) diff --git a/flowmachine/flowmachine/core/server/query_schemas/unique_visitor_counts.py b/flowmachine/flowmachine/core/server/query_schemas/unique_visitor_counts.py index 4738849116..acec6206e6 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/unique_visitor_counts.py +++ b/flowmachine/flowmachine/core/server/query_schemas/unique_visitor_counts.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from marshmallow import fields, post_load +from marshmallow import fields from marshmallow.validate import OneOf from flowmachine.features.location.unique_visitor_counts import UniqueVisitorCounts @@ -11,7 +11,7 @@ from .unique_subscriber_counts import UniqueSubscriberCountsSchema -from . import BaseExposedQuery +from .base_exposed_query import BaseExposedQuery __all__ = [ "UniqueVisitorCountsSchema", diff --git a/flowmachine/flowmachine/core/server/query_schemas/unmoving_at_reference_location_counts.py b/flowmachine/flowmachine/core/server/query_schemas/unmoving_at_reference_location_counts.py index 3d8784513e..e283b324a1 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/unmoving_at_reference_location_counts.py +++ b/flowmachine/flowmachine/core/server/query_schemas/unmoving_at_reference_location_counts.py @@ -14,7 +14,7 @@ from flowmachine.features.subscriber.unmoving_at_reference_location import ( UnmovingAtReferenceLocation, ) -from . import BaseExposedQuery +from .base_exposed_query import BaseExposedQuery from .base_schema import BaseSchema from .reference_location import ReferenceLocationSchema from .unique_locations import UniqueLocationsSchema diff --git a/flowmachine/flowmachine/core/server/query_schemas/unmoving_counts.py b/flowmachine/flowmachine/core/server/query_schemas/unmoving_counts.py index 807c724547..bbb6069911 100644 --- a/flowmachine/flowmachine/core/server/query_schemas/unmoving_counts.py +++ b/flowmachine/flowmachine/core/server/query_schemas/unmoving_counts.py @@ -10,7 +10,7 @@ ) from flowmachine.features.location.unmoving_counts import UnmovingCounts from flowmachine.features.subscriber.unmoving import Unmoving -from . import BaseExposedQuery +from .base_exposed_query import BaseExposedQuery from .base_schema import BaseSchema from .unique_locations import UniqueLocationsSchema diff --git a/flowmachine/flowmachine/core/server/query_schemas/util.py b/flowmachine/flowmachine/core/server/query_schemas/util.py new file mode 100644 index 0000000000..2b3f5cec39 --- /dev/null +++ b/flowmachine/flowmachine/core/server/query_schemas/util.py @@ -0,0 +1,32 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from typing import Dict, Type +from collections import ChainMap + +import pkg_resources +import structlog + +logger = structlog.get_logger("flowmachine.debug", submodule=__name__) + + +def get_type_schemas_from_entrypoint(entry_point_name: str) -> Dict[str, Type]: + logger.info("Loading type schemas.", entry_point=entry_point_name) + type_schemas = dict() + for entry_point in ( + entry_point + for entry_point in pkg_resources.iter_entry_points("flowkit.queries") + if entry_point.name == entry_point_name + ): + try: + schemas = entry_point.load() + logger.info( + "Loaded type schemas", entry_point=entry_point_name, schemas=schemas + ) + type_schemas = dict(type_schemas, **schemas) + except Exception as exc: + logger.error( + "Failed to load type schema", entry_point=entry_point, exception=exc + ) + return type_schemas diff --git a/flowmachine/flowmachine/core/server/server.py b/flowmachine/flowmachine/core/server/server.py index c49f02c22e..3867a4242d 100644 --- a/flowmachine/flowmachine/core/server/server.py +++ b/flowmachine/flowmachine/core/server/server.py @@ -3,7 +3,6 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import asyncio -from concurrent.futures import Executor from json import JSONDecodeError import traceback @@ -20,7 +19,6 @@ from zmq.asyncio import Context import flowmachine -from flowmachine.core import Query, Connection from flowmachine.core.cache import watch_and_shrink_cache from flowmachine.core.context import get_db, get_executor from flowmachine.utils import convert_dict_keys_to_strings diff --git a/flowmachine/setup.py b/flowmachine/setup.py index 9577836f78..7c5b82780c 100644 --- a/flowmachine/setup.py +++ b/flowmachine/setup.py @@ -58,7 +58,14 @@ def read(filename, parent=None): version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), entry_points={ - "console_scripts": ["flowmachine = flowmachine.core.server.server:main"] + "console_scripts": ["flowmachine = flowmachine.core.server.server:main"], + "flowkit.queries": [ + "top_level_queries = flowmachine.core.server.query_schemas.top_level_queries:top_level_queries", + "histogrammable_queries = flowmachine.core.server.query_schemas.histogrammable_queries:histogrammable_queries", + "joinable_queries = flowmachine.core.server.query_schemas.joinable_queries:joinable_queries", + "flowable_queries = flowmachine.core.server.query_schemas.flowable_queries:flowable_queries", + "reference_location_queries = flowmachine.core.server.query_schemas.reference_location_queries:reference_location_queries", + ], }, description="Digestion program for Call Detail Record (CDR) data.", long_description=readme, diff --git a/flowmachine/tests/server/test_action_handlers.py b/flowmachine/tests/server/test_action_handlers.py index 9696fef8c6..a31925a43b 100644 --- a/flowmachine/tests/server/test_action_handlers.py +++ b/flowmachine/tests/server/test_action_handlers.py @@ -4,7 +4,7 @@ from asyncio import sleep import pytest -from marshmallow import Schema, fields +from marshmallow import Schema import flowmachine from flowmachine import connections @@ -13,8 +13,6 @@ get_redis, get_db, redis_connection, - context, - get_executor, ) from flowmachine.core.query_info_lookup import QueryInfoLookup from flowmachine.core.query_state import QueryState, QueryStateMachine @@ -27,7 +25,9 @@ get_action_handler, ) from flowmachine.core.server.exceptions import FlowmachineServerError -from flowmachine.core.server.query_schemas import FlowmachineQuerySchema +from flowmachine.core.server.query_schemas.flowmachine_query import ( + FlowmachineQuerySchema, +) from flowmachine.core.server.zmq_helpers import ZMQReplyStatus diff --git a/flowmachine/tests/test_query_object_construction.py b/flowmachine/tests/test_query_object_construction.py index 6990062af4..7f1594ee10 100644 --- a/flowmachine/tests/test_query_object_construction.py +++ b/flowmachine/tests/test_query_object_construction.py @@ -6,7 +6,9 @@ import pytest from marshmallow import ValidationError -from flowmachine.core.server.query_schemas import FlowmachineQuerySchema +from flowmachine.core.server.query_schemas.flowmachine_query import ( + FlowmachineQuerySchema, +) def test_construct_query(diff_reporter): diff --git a/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_json_spec.approved.txt b/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_json_spec.approved.txt index 6902ed6153..f38d4f7e42 100644 --- a/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_json_spec.approved.txt +++ b/integration_tests/tests/flowapi_tests/test_api_spec.test_generated_openapi_json_spec.approved.txt @@ -262,7 +262,7 @@ "type": "string" }, "reference_location": { - "$ref": "#/components/schemas/InputToDisplacement" + "$ref": "#/components/schemas/ReferenceLocation" }, "sampling": { "allOf": [ @@ -684,23 +684,6 @@ } ] }, - "InputToDisplacement": { - "discriminator": { - "mapping": { - "daily_location": "#/components/schemas/DailyLocation", - "modal_location": "#/components/schemas/ModalLocation" - }, - "propertyName": "query_kind" - }, - "oneOf": [ - { - "$ref": "#/components/schemas/DailyLocation" - }, - { - "$ref": "#/components/schemas/ModalLocation" - } - ] - }, "InputToFlows": { "discriminator": { "mapping": { @@ -735,23 +718,6 @@ } ] }, - "InputToSpatialAggregate": { - "discriminator": { - "mapping": { - "daily_location": "#/components/schemas/DailyLocation", - "modal_location": "#/components/schemas/ModalLocation" - }, - "propertyName": "query_kind" - }, - "oneOf": [ - { - "$ref": "#/components/schemas/DailyLocation" - }, - { - "$ref": "#/components/schemas/ModalLocation" - } - ] - }, "JoinableMetrics": { "discriminator": { "mapping": { @@ -804,7 +770,7 @@ "JoinedSpatialAggregate": { "properties": { "locations": { - "$ref": "#/components/schemas/InputToSpatialAggregate" + "$ref": "#/components/schemas/ReferenceLocation" }, "method": { "enum": [ @@ -1461,7 +1427,7 @@ "SpatialAggregate": { "properties": { "locations": { - "$ref": "#/components/schemas/InputToSpatialAggregate" + "$ref": "#/components/schemas/ReferenceLocation" }, "query_kind": { "enum": [ diff --git a/integration_tests/tests/flowmachine_server_tests/test_server.test_api_spec_of_flowmachine_query_schemas.approved.txt b/integration_tests/tests/flowmachine_server_tests/test_server.test_api_spec_of_flowmachine_query_schemas.approved.txt index 2c6a553300..f669ce2c53 100644 --- a/integration_tests/tests/flowmachine_server_tests/test_server.test_api_spec_of_flowmachine_query_schemas.approved.txt +++ b/integration_tests/tests/flowmachine_server_tests/test_server.test_api_spec_of_flowmachine_query_schemas.approved.txt @@ -256,7 +256,7 @@ "type": "string" }, "reference_location": { - "$ref": "#/components/schemas/InputToDisplacement" + "$ref": "#/components/schemas/ReferenceLocation" }, "sampling": { "allOf": [ @@ -671,23 +671,6 @@ } ] }, - "InputToDisplacement": { - "discriminator": { - "mapping": { - "daily_location": "#/components/schemas/DailyLocation", - "modal_location": "#/components/schemas/ModalLocation" - }, - "propertyName": "query_kind" - }, - "oneOf": [ - { - "$ref": "#/components/schemas/DailyLocation" - }, - { - "$ref": "#/components/schemas/ModalLocation" - } - ] - }, "InputToFlows": { "discriminator": { "mapping": { @@ -722,23 +705,6 @@ } ] }, - "InputToSpatialAggregate": { - "discriminator": { - "mapping": { - "daily_location": "#/components/schemas/DailyLocation", - "modal_location": "#/components/schemas/ModalLocation" - }, - "propertyName": "query_kind" - }, - "oneOf": [ - { - "$ref": "#/components/schemas/DailyLocation" - }, - { - "$ref": "#/components/schemas/ModalLocation" - } - ] - }, "JoinableMetrics": { "discriminator": { "mapping": { @@ -791,7 +757,7 @@ "JoinedSpatialAggregate": { "properties": { "locations": { - "$ref": "#/components/schemas/InputToSpatialAggregate" + "$ref": "#/components/schemas/ReferenceLocation" }, "method": { "enum": [ @@ -1439,7 +1405,7 @@ "SpatialAggregate": { "properties": { "locations": { - "$ref": "#/components/schemas/InputToSpatialAggregate" + "$ref": "#/components/schemas/ReferenceLocation" }, "query_kind": { "enum": [