diff --git a/CITATION.cff b/CITATION.cff index 77cec1f..4ff83c3 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,7 +2,7 @@ # Visit https://bit.ly/cffinit to generate yours today! cff-version: 1.2.0 -title: EOSCF-Content-Based-RS +title: EOSCF-Autocompletion Suggestion message: >- If you use this software, please cite it using the metadata from this file. diff --git a/Dockerfile-autocompletion b/Dockerfile-autocompletion index 041f986..0e019d2 100644 --- a/Dockerfile-autocompletion +++ b/Dockerfile-autocompletion @@ -8,4 +8,4 @@ RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt COPY . /app -CMD ["python3.9", "recommendation_system_app.py", "--config_file", "api/config/backend-auto-prod.yaml"] +CMD ["python3.9", "recommendation_system_app.py", "--config_file", "app/config/backend-providers-recommender-prod.yaml"] diff --git a/README.md b/README.md index d332635..38bfa25 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Build and run: 1. Make sure that you have added the `.env` file in the project root 2. Run `docker-compose -f docker-compose-autocompletion.yml up` +3. `http://localhost:4559/v1/health` should return 200 (4559 is our default port, changes in the docker-compose file) The image can be deployed using `docker-compose` if the `.env` variables are set correctly. @@ -41,7 +42,4 @@ SENTRY_SDN=https://12345... # Cronitor is used to monitor the offline updating of our RS data structures # stored in redis CRONITOR_API_KEY=123aababdas... - -# Monitoring access token is used to obtain reliability status of services in the portal before recommending them -MONITORING_API_ACCESS_TOKEN=daad2dasd... ``` diff --git a/api/databases/registry/registry_selector.py b/api/databases/registry/registry_selector.py deleted file mode 100644 index 27f655c..0000000 --- a/api/databases/registry/registry_selector.py +++ /dev/null @@ -1,15 +0,0 @@ -from api.databases.registry.catalog_api import CatalogueAPI -from api.databases.registry.rs_mongo import RSMongoDB -from api.databases.registry.catalog_dump import CatalogueDump -from api.settings import APP_SETTINGS - - -def get_registry(): - if APP_SETTINGS['BACKEND']['MODE'] == 'RS': - return RSMongoDB() - elif APP_SETTINGS['BACKEND']['MODE'] == 'AUTO-COMPLETION': - return CatalogueAPI() - elif APP_SETTINGS['BACKEND']['MODE'] == "SIMILAR_SERVICES_EVALUATION": - return CatalogueDump() - else: - pass # TODO raise error diff --git a/api/recommender/similar_services/field_suggestion/evaluation/results/tf_idf_evaluations_results.json b/api/recommender/similar_services/field_suggestion/evaluation/results/tf_idf_evaluations_results.json deleted file mode 100644 index 8aa1f96..0000000 --- a/api/recommender/similar_services/field_suggestion/evaluation/results/tf_idf_evaluations_results.json +++ /dev/null @@ -1 +0,0 @@ -{"(0.1, 5, 0, 1)": {"precision": 0.45431404958677685, "recall": 0.35225928633655895, "f1_score": 0.3722470596925143, "per_field": {"categories": {"precision": 0.21694214876033058, "recall": 0.1971992653810836, "f1_score": 0.20206611570247937}, "scientific_domains": {"precision": 0.504, "recall": 0.46828174603174605, "f1_score": 0.4763858858858858}, "target_users": {"precision": 0.642, "recall": 0.3912968475968471, "f1_score": 0.43828917748917784}}}, "(0.1, 5, 0, 3)": {"precision": 0.3769972451790637, "recall": 0.5585603164053577, "f1_score": 0.4098331911601965, "per_field": {"categories": {"precision": 0.19765840220385653, "recall": 0.3972944468295707, "f1_score": 0.24872687918142478}, "scientific_domains": {"precision": 0.3916666666666672, "recall": 0.6478666666666668, "f1_score": 0.4573228052992755}, "target_users": {"precision": 0.5416666666666672, "recall": 0.6305198357198358, "f1_score": 0.5234498889998893}}}, "(0.1, 5, 0, 5)": {"precision": 0.3585592286501374, "recall": 0.6348052238548106, "f1_score": 0.4161734836397333, "per_field": {"categories": {"precision": 0.20034435261708006, "recall": 0.4786157770145373, "f1_score": 0.2622655619556447}, "scientific_domains": {"precision": 0.3843333333333332, "recall": 0.6982325396825398, "f1_score": 0.45775431731298244}, "target_users": {"precision": 0.49099999999999894, "recall": 0.7275673548673547, "f1_score": 0.5285005716505725}}}, "(0.1, 5, 0.1, 1)": {"precision": 0.45431404958677685, "recall": 0.35225928633655895, "f1_score": 0.3722470596925143, "per_field": {"categories": {"precision": 0.21694214876033058, "recall": 0.1971992653810836, "f1_score": 0.20206611570247937}, "scientific_domains": {"precision": 0.504, "recall": 0.46828174603174605, "f1_score": 0.4763858858858858}, "target_users": {"precision": 0.642, "recall": 0.3912968475968471, "f1_score": 0.43828917748917784}}}, "(0.1, 5, 0.1, 3)": {"precision": 0.3769972451790637, "recall": 0.5585603164053577, "f1_score": 0.4098331911601965, "per_field": {"categories": {"precision": 0.19765840220385653, "recall": 0.3972944468295707, "f1_score": 0.24872687918142478}, "scientific_domains": {"precision": 0.3916666666666672, "recall": 0.6478666666666668, "f1_score": 0.4573228052992755}, "target_users": {"precision": 0.5416666666666672, "recall": 0.6305198357198358, "f1_score": 0.5234498889998893}}}, "(0.1, 5, 0.1, 5)": {"precision": 0.3585592286501374, "recall": 0.6348052238548106, "f1_score": 0.4161734836397333, "per_field": {"categories": {"precision": 0.20034435261708006, "recall": 0.4786157770145373, "f1_score": 0.2622655619556447}, "scientific_domains": {"precision": 0.3843333333333332, "recall": 0.6982325396825398, "f1_score": 0.45775431731298244}, "target_users": {"precision": 0.49099999999999894, "recall": 0.7275673548673547, "f1_score": 0.5285005716505725}}}, "(0.1, 5, 0.2, 1)": {"precision": 0.45431404958677685, "recall": 0.35225928633655895, "f1_score": 0.3722470596925143, "per_field": {"categories": {"precision": 0.21694214876033058, "recall": 0.1971992653810836, "f1_score": 0.20206611570247937}, "scientific_domains": {"precision": 0.504, "recall": 0.46828174603174605, "f1_score": 0.4763858858858858}, "target_users": {"precision": 0.642, "recall": 0.3912968475968471, "f1_score": 0.43828917748917784}}}, "(0.1, 5, 0.2, 3)": {"precision": 0.3769972451790637, "recall": 0.5585603164053577, "f1_score": 0.4098331911601965, "per_field": {"categories": {"precision": 0.19765840220385653, "recall": 0.3972944468295707, "f1_score": 0.24872687918142478}, "scientific_domains": {"precision": 0.3916666666666672, "recall": 0.6478666666666668, "f1_score": 0.4573228052992755}, "target_users": {"precision": 0.5416666666666672, "recall": 0.6305198357198358, "f1_score": 0.5234498889998893}}}, "(0.1, 5, 0.2, 5)": {"precision": 0.3585592286501374, "recall": 0.6348052238548106, "f1_score": 0.4161734836397333, "per_field": {"categories": {"precision": 0.20034435261708006, "recall": 0.4786157770145373, "f1_score": 0.2622655619556447}, "scientific_domains": {"precision": 0.3843333333333332, "recall": 0.6982325396825398, "f1_score": 0.45775431731298244}, "target_users": {"precision": 0.49099999999999894, "recall": 0.7275673548673547, "f1_score": 0.5285005716505725}}}, "(0.1, 5, 0.3, 1)": {"precision": 0.45431404958677685, "recall": 0.35225928633655895, "f1_score": 0.3722470596925143, "per_field": {"categories": {"precision": 0.21694214876033058, "recall": 0.1971992653810836, "f1_score": 0.20206611570247937}, "scientific_domains": {"precision": 0.504, "recall": 0.46828174603174605, "f1_score": 0.4763858858858858}, "target_users": {"precision": 0.642, "recall": 0.3912968475968471, "f1_score": 0.43828917748917784}}}, "(0.1, 5, 0.3, 3)": {"precision": 0.3769972451790637, "recall": 0.5585603164053577, "f1_score": 0.4098331911601965, "per_field": {"categories": {"precision": 0.19765840220385653, "recall": 0.3972944468295707, "f1_score": 0.24872687918142478}, "scientific_domains": {"precision": 0.3916666666666672, "recall": 0.6478666666666668, "f1_score": 0.4573228052992755}, "target_users": {"precision": 0.5416666666666672, "recall": 0.6305198357198358, "f1_score": 0.5234498889998893}}}, "(0.1, 5, 0.3, 5)": {"precision": 0.3585592286501374, "recall": 0.6348052238548106, "f1_score": 0.4161734836397333, "per_field": {"categories": {"precision": 0.20034435261708006, "recall": 0.4786157770145373, "f1_score": 0.2622655619556447}, "scientific_domains": {"precision": 0.3843333333333332, "recall": 0.6982325396825398, "f1_score": 0.45775431731298244}, "target_users": {"precision": 0.49099999999999894, "recall": 0.7275673548673547, "f1_score": 0.5285005716505725}}}, "(0.1, 5, 0.4, 1)": {"precision": 0.45431404958677685, "recall": 0.35225928633655895, "f1_score": 0.3722470596925143, "per_field": {"categories": {"precision": 0.21694214876033058, "recall": 0.1971992653810836, "f1_score": 0.20206611570247937}, "scientific_domains": {"precision": 0.504, "recall": 0.46828174603174605, "f1_score": 0.4763858858858858}, "target_users": {"precision": 0.642, "recall": 0.3912968475968471, "f1_score": 0.43828917748917784}}}, "(0.1, 5, 0.4, 3)": {"precision": 0.3769972451790637, "recall": 0.5585603164053577, "f1_score": 0.4098331911601965, "per_field": {"categories": {"precision": 0.19765840220385653, "recall": 0.3972944468295707, "f1_score": 0.24872687918142478}, "scientific_domains": {"precision": 0.3916666666666672, "recall": 0.6478666666666668, "f1_score": 0.4573228052992755}, "target_users": {"precision": 0.5416666666666672, "recall": 0.6305198357198358, "f1_score": 0.5234498889998893}}}, "(0.1, 5, 0.4, 5)": {"precision": 0.3585592286501374, "recall": 0.6348052238548106, "f1_score": 0.4161734836397333, "per_field": {"categories": {"precision": 0.20034435261708006, "recall": 0.4786157770145373, "f1_score": 0.2622655619556447}, "scientific_domains": {"precision": 0.3843333333333332, "recall": 0.6982325396825398, "f1_score": 0.45775431731298244}, "target_users": {"precision": 0.49099999999999894, "recall": 0.7275673548673547, "f1_score": 0.5285005716505725}}}, "(0.1, 5, 0.5, 1)": {"precision": 0.45431404958677685, "recall": 0.35225928633655895, "f1_score": 0.3722470596925143, "per_field": {"categories": {"precision": 0.21694214876033058, "recall": 0.1971992653810836, "f1_score": 0.20206611570247937}, "scientific_domains": {"precision": 0.504, "recall": 0.46828174603174605, "f1_score": 0.4763858858858858}, "target_users": {"precision": 0.642, "recall": 0.3912968475968471, "f1_score": 0.43828917748917784}}}, "(0.1, 5, 0.5, 3)": {"precision": 0.3769972451790637, "recall": 0.5585603164053577, "f1_score": 0.4098331911601965, "per_field": {"categories": {"precision": 0.19765840220385653, "recall": 0.3972944468295707, "f1_score": 0.24872687918142478}, "scientific_domains": {"precision": 0.3916666666666672, "recall": 0.6478666666666668, "f1_score": 0.4573228052992755}, "target_users": {"precision": 0.5416666666666672, "recall": 0.6305198357198358, "f1_score": 0.5234498889998893}}}, "(0.1, 5, 0.5, 5)": {"precision": 0.3585592286501374, "recall": 0.6348052238548106, "f1_score": 0.4161734836397333, "per_field": {"categories": {"precision": 0.20034435261708006, "recall": 0.4786157770145373, "f1_score": 0.2622655619556447}, "scientific_domains": {"precision": 0.3843333333333332, "recall": 0.6982325396825398, "f1_score": 0.45775431731298244}, "target_users": {"precision": 0.49099999999999894, "recall": 0.7275673548673547, "f1_score": 0.5285005716505725}}}, "(0.1, 5, 0.6, 1)": {"precision": 0.45431404958677685, "recall": 0.35225928633655895, "f1_score": 0.3722470596925143, "per_field": {"categories": {"precision": 0.21694214876033058, "recall": 0.1971992653810836, "f1_score": 0.20206611570247937}, "scientific_domains": {"precision": 0.504, "recall": 0.46828174603174605, "f1_score": 0.4763858858858858}, "target_users": {"precision": 0.642, "recall": 0.3912968475968471, "f1_score": 0.43828917748917784}}}, "(0.1, 5, 0.6, 3)": {"precision": 0.3769972451790637, "recall": 0.5585603164053577, "f1_score": 0.4098331911601965, "per_field": {"categories": {"precision": 0.19765840220385653, "recall": 0.3972944468295707, "f1_score": 0.24872687918142478}, "scientific_domains": {"precision": 0.3916666666666672, "recall": 0.6478666666666668, "f1_score": 0.4573228052992755}, "target_users": {"precision": 0.5416666666666672, "recall": 0.6305198357198358, "f1_score": 0.5234498889998893}}}, "(0.1, 5, 0.6, 5)": {"precision": 0.3585592286501374, "recall": 0.6348052238548106, "f1_score": 0.4161734836397333, "per_field": {"categories": {"precision": 0.20034435261708006, "recall": 0.4786157770145373, "f1_score": 0.2622655619556447}, "scientific_domains": {"precision": 0.3843333333333332, "recall": 0.6982325396825398, "f1_score": 0.45775431731298244}, "target_users": {"precision": 0.49099999999999894, "recall": 0.7275673548673547, "f1_score": 0.5285005716505725}}}, "(0.1, 5, 0.7, 1)": {"precision": 0.45431404958677685, "recall": 0.35225928633655895, "f1_score": 0.3722470596925143, "per_field": {"categories": {"precision": 0.21694214876033058, "recall": 0.1971992653810836, "f1_score": 0.20206611570247937}, "scientific_domains": {"precision": 0.504, "recall": 0.46828174603174605, "f1_score": 0.4763858858858858}, "target_users": {"precision": 0.642, "recall": 0.3912968475968471, "f1_score": 0.43828917748917784}}}, "(0.1, 5, 0.7, 3)": {"precision": 0.3769972451790637, "recall": 0.5585603164053577, "f1_score": 0.4098331911601965, "per_field": {"categories": {"precision": 0.19765840220385653, "recall": 0.3972944468295707, "f1_score": 0.24872687918142478}, "scientific_domains": {"precision": 0.3916666666666672, "recall": 0.6478666666666668, "f1_score": 0.4573228052992755}, "target_users": {"precision": 0.5416666666666672, "recall": 0.6305198357198358, "f1_score": 0.5234498889998893}}}, "(0.1, 5, 0.7, 5)": {"precision": 0.3585592286501374, "recall": 0.6348052238548106, "f1_score": 0.4161734836397333, "per_field": {"categories": {"precision": 0.20034435261708006, "recall": 0.4786157770145373, "f1_score": 0.2622655619556447}, "scientific_domains": {"precision": 0.3843333333333332, "recall": 0.6982325396825398, "f1_score": 0.45775431731298244}, "target_users": {"precision": 0.49099999999999894, "recall": 0.7275673548673547, "f1_score": 0.5285005716505725}}}, "(0.1, 5, 0.8, 1)": {"precision": 0.45431404958677685, "recall": 0.35225928633655895, "f1_score": 0.3722470596925143, "per_field": {"categories": {"precision": 0.21694214876033058, "recall": 0.1971992653810836, "f1_score": 0.20206611570247937}, "scientific_domains": {"precision": 0.504, "recall": 0.46828174603174605, "f1_score": 0.4763858858858858}, "target_users": {"precision": 0.642, "recall": 0.3912968475968471, "f1_score": 0.43828917748917784}}}, "(0.1, 5, 0.8, 3)": {"precision": 0.3769972451790637, "recall": 0.5585603164053577, "f1_score": 0.4098331911601965, "per_field": {"categories": {"precision": 0.19765840220385653, "recall": 0.3972944468295707, "f1_score": 0.24872687918142478}, "scientific_domains": {"precision": 0.3916666666666672, "recall": 0.6478666666666668, "f1_score": 0.4573228052992755}, "target_users": {"precision": 0.5416666666666672, "recall": 0.6305198357198358, "f1_score": 0.5234498889998893}}}, "(0.1, 5, 0.8, 5)": {"precision": 0.3585592286501374, "recall": 0.6348052238548106, "f1_score": 0.4161734836397333, "per_field": {"categories": {"precision": 0.20034435261708006, "recall": 0.4786157770145373, "f1_score": 0.2622655619556447}, "scientific_domains": {"precision": 0.3843333333333332, "recall": 0.6982325396825398, "f1_score": 0.45775431731298244}, "target_users": {"precision": 0.49099999999999894, "recall": 0.7275673548673547, "f1_score": 0.5285005716505725}}}, "(0.1, 5, 0.9, 1)": {"precision": 0.45431404958677685, "recall": 0.35225928633655895, "f1_score": 0.3722470596925143, "per_field": {"categories": {"precision": 0.21694214876033058, "recall": 0.1971992653810836, "f1_score": 0.20206611570247937}, "scientific_domains": {"precision": 0.504, "recall": 0.46828174603174605, "f1_score": 0.4763858858858858}, "target_users": {"precision": 0.642, "recall": 0.3912968475968471, "f1_score": 0.43828917748917784}}}, "(0.1, 5, 0.9, 3)": {"precision": 0.3769972451790637, "recall": 0.5585603164053577, "f1_score": 0.4098331911601965, "per_field": {"categories": {"precision": 0.19765840220385653, "recall": 0.3972944468295707, "f1_score": 0.24872687918142478}, "scientific_domains": {"precision": 0.3916666666666672, "recall": 0.6478666666666668, "f1_score": 0.4573228052992755}, "target_users": {"precision": 0.5416666666666672, "recall": 0.6305198357198358, "f1_score": 0.5234498889998893}}}, "(0.1, 5, 0.9, 5)": {"precision": 0.3585592286501374, "recall": 0.6348052238548106, "f1_score": 0.4161734836397333, "per_field": {"categories": {"precision": 0.20034435261708006, "recall": 0.4786157770145373, "f1_score": 0.2622655619556447}, "scientific_domains": {"precision": 0.3843333333333332, "recall": 0.6982325396825398, "f1_score": 0.45775431731298244}, "target_users": {"precision": 0.49099999999999894, "recall": 0.7275673548673547, "f1_score": 0.5285005716505725}}}, "(0.1, 10, 0, 1)": {"precision": 0.42167493112947657, "recall": 0.32135365505324165, "f1_score": 0.34088841394651687, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.14366808581684615, "f1_score": 0.1460204814947898}, "scientific_domains": {"precision": 0.466, "recall": 0.44190793650793647, "f1_score": 0.44807477477477475}, "target_users": {"precision": 0.642, "recall": 0.37848494283494233, "f1_score": 0.42856998556998593}}}, "(0.1, 10, 0, 3)": {"precision": 0.2940550964187329, "recall": 0.527222549519657, "f1_score": 0.3410892868767202, "per_field": {"categories": {"precision": 0.13016528925619858, "recall": 0.3234796310209534, "f1_score": 0.17802439984258167}, "scientific_domains": {"precision": 0.27999999999999947, "recall": 0.6363714285714287, "f1_score": 0.37014829549535416}, "target_users": {"precision": 0.4720000000000006, "recall": 0.6218165889665889, "f1_score": 0.4750951652922249}}}, "(0.1, 10, 0, 5)": {"precision": 0.25785224977043164, "recall": 0.621636278001154, "f1_score": 0.32734672876008597, "per_field": {"categories": {"precision": 0.12052341597796175, "recall": 0.43190814580277387, "f1_score": 0.17625884599025085}, "scientific_domains": {"precision": 0.24660000000000037, "recall": 0.6990206349206349, "f1_score": 0.3392699397022894}, "target_users": {"precision": 0.40643333333333276, "recall": 0.7339800532800531, "f1_score": 0.4665114005877177}}}, "(0.1, 10, 0.1, 1)": {"precision": 0.42167493112947657, "recall": 0.32135365505324165, "f1_score": 0.34088841394651687, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.14366808581684615, "f1_score": 0.1460204814947898}, "scientific_domains": {"precision": 0.466, "recall": 0.44190793650793647, "f1_score": 0.44807477477477475}, "target_users": {"precision": 0.642, "recall": 0.37848494283494233, "f1_score": 0.42856998556998593}}}, "(0.1, 10, 0.1, 3)": {"precision": 0.2940550964187329, "recall": 0.527222549519657, "f1_score": 0.3410892868767202, "per_field": {"categories": {"precision": 0.13016528925619858, "recall": 0.3234796310209534, "f1_score": 0.17802439984258167}, "scientific_domains": {"precision": 0.27999999999999947, "recall": 0.6363714285714287, "f1_score": 0.37014829549535416}, "target_users": {"precision": 0.4720000000000006, "recall": 0.6218165889665889, "f1_score": 0.4750951652922249}}}, "(0.1, 10, 0.1, 5)": {"precision": 0.25785224977043164, "recall": 0.621636278001154, "f1_score": 0.32734672876008597, "per_field": {"categories": {"precision": 0.12052341597796175, "recall": 0.43190814580277387, "f1_score": 0.17625884599025085}, "scientific_domains": {"precision": 0.24660000000000037, "recall": 0.6990206349206349, "f1_score": 0.3392699397022894}, "target_users": {"precision": 0.40643333333333276, "recall": 0.7339800532800531, "f1_score": 0.4665114005877177}}}, "(0.1, 10, 0.2, 1)": {"precision": 0.42167493112947657, "recall": 0.32135365505324165, "f1_score": 0.34088841394651687, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.14366808581684615, "f1_score": 0.1460204814947898}, "scientific_domains": {"precision": 0.466, "recall": 0.44190793650793647, "f1_score": 0.44807477477477475}, "target_users": {"precision": 0.642, "recall": 0.37848494283494233, "f1_score": 0.42856998556998593}}}, "(0.1, 10, 0.2, 3)": {"precision": 0.2940550964187329, "recall": 0.527222549519657, "f1_score": 0.3410892868767202, "per_field": {"categories": {"precision": 0.13016528925619858, "recall": 0.3234796310209534, "f1_score": 0.17802439984258167}, "scientific_domains": {"precision": 0.27999999999999947, "recall": 0.6363714285714287, "f1_score": 0.37014829549535416}, "target_users": {"precision": 0.4720000000000006, "recall": 0.6218165889665889, "f1_score": 0.4750951652922249}}}, "(0.1, 10, 0.2, 5)": {"precision": 0.25785224977043164, "recall": 0.621636278001154, "f1_score": 0.32734672876008597, "per_field": {"categories": {"precision": 0.12052341597796175, "recall": 0.43190814580277387, "f1_score": 0.17625884599025085}, "scientific_domains": {"precision": 0.24660000000000037, "recall": 0.6990206349206349, "f1_score": 0.3392699397022894}, "target_users": {"precision": 0.40643333333333276, "recall": 0.7339800532800531, "f1_score": 0.4665114005877177}}}, "(0.1, 10, 0.3, 1)": {"precision": 0.42167493112947657, "recall": 0.32135365505324165, "f1_score": 0.34088841394651687, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.14366808581684615, "f1_score": 0.1460204814947898}, "scientific_domains": {"precision": 0.466, "recall": 0.44190793650793647, "f1_score": 0.44807477477477475}, "target_users": {"precision": 0.642, "recall": 0.37848494283494233, "f1_score": 0.42856998556998593}}}, "(0.1, 10, 0.3, 3)": {"precision": 0.2940550964187329, "recall": 0.527222549519657, "f1_score": 0.3410892868767202, "per_field": {"categories": {"precision": 0.13016528925619858, "recall": 0.3234796310209534, "f1_score": 0.17802439984258167}, "scientific_domains": {"precision": 0.27999999999999947, "recall": 0.6363714285714287, "f1_score": 0.37014829549535416}, "target_users": {"precision": 0.4720000000000006, "recall": 0.6218165889665889, "f1_score": 0.4750951652922249}}}, "(0.1, 10, 0.3, 5)": {"precision": 0.25785224977043164, "recall": 0.621636278001154, "f1_score": 0.32734672876008597, "per_field": {"categories": {"precision": 0.12052341597796175, "recall": 0.43190814580277387, "f1_score": 0.17625884599025085}, "scientific_domains": {"precision": 0.24660000000000037, "recall": 0.6990206349206349, "f1_score": 0.3392699397022894}, "target_users": {"precision": 0.40643333333333276, "recall": 0.7339800532800531, "f1_score": 0.4665114005877177}}}, "(0.1, 10, 0.4, 1)": {"precision": 0.42167493112947657, "recall": 0.32135365505324165, "f1_score": 0.34088841394651687, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.14366808581684615, "f1_score": 0.1460204814947898}, "scientific_domains": {"precision": 0.466, "recall": 0.44190793650793647, "f1_score": 0.44807477477477475}, "target_users": {"precision": 0.642, "recall": 0.37848494283494233, "f1_score": 0.42856998556998593}}}, "(0.1, 10, 0.4, 3)": {"precision": 0.2940550964187329, "recall": 0.527222549519657, "f1_score": 0.3410892868767202, "per_field": {"categories": {"precision": 0.13016528925619858, "recall": 0.3234796310209534, "f1_score": 0.17802439984258167}, "scientific_domains": {"precision": 0.27999999999999947, "recall": 0.6363714285714287, "f1_score": 0.37014829549535416}, "target_users": {"precision": 0.4720000000000006, "recall": 0.6218165889665889, "f1_score": 0.4750951652922249}}}, "(0.1, 10, 0.4, 5)": {"precision": 0.25785224977043164, "recall": 0.621636278001154, "f1_score": 0.32734672876008597, "per_field": {"categories": {"precision": 0.12052341597796175, "recall": 0.43190814580277387, "f1_score": 0.17625884599025085}, "scientific_domains": {"precision": 0.24660000000000037, "recall": 0.6990206349206349, "f1_score": 0.3392699397022894}, "target_users": {"precision": 0.40643333333333276, "recall": 0.7339800532800531, "f1_score": 0.4665114005877177}}}, "(0.1, 10, 0.5, 1)": {"precision": 0.42167493112947657, "recall": 0.32135365505324165, "f1_score": 0.34088841394651687, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.14366808581684615, "f1_score": 0.1460204814947898}, "scientific_domains": {"precision": 0.466, "recall": 0.44190793650793647, "f1_score": 0.44807477477477475}, "target_users": {"precision": 0.642, "recall": 0.37848494283494233, "f1_score": 0.42856998556998593}}}, "(0.1, 10, 0.5, 3)": {"precision": 0.2940550964187329, "recall": 0.527222549519657, "f1_score": 0.3410892868767202, "per_field": {"categories": {"precision": 0.13016528925619858, "recall": 0.3234796310209534, "f1_score": 0.17802439984258167}, "scientific_domains": {"precision": 0.27999999999999947, "recall": 0.6363714285714287, "f1_score": 0.37014829549535416}, "target_users": {"precision": 0.4720000000000006, "recall": 0.6218165889665889, "f1_score": 0.4750951652922249}}}, "(0.1, 10, 0.5, 5)": {"precision": 0.25785224977043164, "recall": 0.621636278001154, "f1_score": 0.32734672876008597, "per_field": {"categories": {"precision": 0.12052341597796175, "recall": 0.43190814580277387, "f1_score": 0.17625884599025085}, "scientific_domains": {"precision": 0.24660000000000037, "recall": 0.6990206349206349, "f1_score": 0.3392699397022894}, "target_users": {"precision": 0.40643333333333276, "recall": 0.7339800532800531, "f1_score": 0.4665114005877177}}}, "(0.1, 10, 0.6, 1)": {"precision": 0.42167493112947657, "recall": 0.32135365505324165, "f1_score": 0.34088841394651687, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.14366808581684615, "f1_score": 0.1460204814947898}, "scientific_domains": {"precision": 0.466, "recall": 0.44190793650793647, "f1_score": 0.44807477477477475}, "target_users": {"precision": 0.642, "recall": 0.37848494283494233, "f1_score": 0.42856998556998593}}}, "(0.1, 10, 0.6, 3)": {"precision": 0.2940550964187329, "recall": 0.527222549519657, "f1_score": 0.3410892868767202, "per_field": {"categories": {"precision": 0.13016528925619858, "recall": 0.3234796310209534, "f1_score": 0.17802439984258167}, "scientific_domains": {"precision": 0.27999999999999947, "recall": 0.6363714285714287, "f1_score": 0.37014829549535416}, "target_users": {"precision": 0.4720000000000006, "recall": 0.6218165889665889, "f1_score": 0.4750951652922249}}}, "(0.1, 10, 0.6, 5)": {"precision": 0.25785224977043164, "recall": 0.621636278001154, "f1_score": 0.32734672876008597, "per_field": {"categories": {"precision": 0.12052341597796175, "recall": 0.43190814580277387, "f1_score": 0.17625884599025085}, "scientific_domains": {"precision": 0.24660000000000037, "recall": 0.6990206349206349, "f1_score": 0.3392699397022894}, "target_users": {"precision": 0.40643333333333276, "recall": 0.7339800532800531, "f1_score": 0.4665114005877177}}}, "(0.1, 10, 0.7, 1)": {"precision": 0.42167493112947657, "recall": 0.32135365505324165, "f1_score": 0.34088841394651687, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.14366808581684615, "f1_score": 0.1460204814947898}, "scientific_domains": {"precision": 0.466, "recall": 0.44190793650793647, "f1_score": 0.44807477477477475}, "target_users": {"precision": 0.642, "recall": 0.37848494283494233, "f1_score": 0.42856998556998593}}}, "(0.1, 10, 0.7, 3)": {"precision": 0.2940550964187329, "recall": 0.527222549519657, "f1_score": 0.3410892868767202, "per_field": {"categories": {"precision": 0.13016528925619858, "recall": 0.3234796310209534, "f1_score": 0.17802439984258167}, "scientific_domains": {"precision": 0.27999999999999947, "recall": 0.6363714285714287, "f1_score": 0.37014829549535416}, "target_users": {"precision": 0.4720000000000006, "recall": 0.6218165889665889, "f1_score": 0.4750951652922249}}}, "(0.1, 10, 0.7, 5)": {"precision": 0.25785224977043164, "recall": 0.621636278001154, "f1_score": 0.32734672876008597, "per_field": {"categories": {"precision": 0.12052341597796175, "recall": 0.43190814580277387, "f1_score": 0.17625884599025085}, "scientific_domains": {"precision": 0.24660000000000037, "recall": 0.6990206349206349, "f1_score": 0.3392699397022894}, "target_users": {"precision": 0.40643333333333276, "recall": 0.7339800532800531, "f1_score": 0.4665114005877177}}}, "(0.1, 10, 0.8, 1)": {"precision": 0.42167493112947657, "recall": 0.32135365505324165, "f1_score": 0.34088841394651687, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.14366808581684615, "f1_score": 0.1460204814947898}, "scientific_domains": {"precision": 0.466, "recall": 0.44190793650793647, "f1_score": 0.44807477477477475}, "target_users": {"precision": 0.642, "recall": 0.37848494283494233, "f1_score": 0.42856998556998593}}}, "(0.1, 10, 0.8, 3)": {"precision": 0.2940550964187329, "recall": 0.527222549519657, "f1_score": 0.3410892868767202, "per_field": {"categories": {"precision": 0.13016528925619858, "recall": 0.3234796310209534, "f1_score": 0.17802439984258167}, "scientific_domains": {"precision": 0.27999999999999947, "recall": 0.6363714285714287, "f1_score": 0.37014829549535416}, "target_users": {"precision": 0.4720000000000006, "recall": 0.6218165889665889, "f1_score": 0.4750951652922249}}}, "(0.1, 10, 0.8, 5)": {"precision": 0.25785224977043164, "recall": 0.621636278001154, "f1_score": 0.32734672876008597, "per_field": {"categories": {"precision": 0.12052341597796175, "recall": 0.43190814580277387, "f1_score": 0.17625884599025085}, "scientific_domains": {"precision": 0.24660000000000037, "recall": 0.6990206349206349, "f1_score": 0.3392699397022894}, "target_users": {"precision": 0.40643333333333276, "recall": 0.7339800532800531, "f1_score": 0.4665114005877177}}}, "(0.1, 10, 0.9, 1)": {"precision": 0.42167493112947657, "recall": 0.32135365505324165, "f1_score": 0.34088841394651687, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.14366808581684615, "f1_score": 0.1460204814947898}, "scientific_domains": {"precision": 0.466, "recall": 0.44190793650793647, "f1_score": 0.44807477477477475}, "target_users": {"precision": 0.642, "recall": 0.37848494283494233, "f1_score": 0.42856998556998593}}}, "(0.1, 10, 0.9, 3)": {"precision": 0.2940550964187329, "recall": 0.527222549519657, "f1_score": 0.3410892868767202, "per_field": {"categories": {"precision": 0.13016528925619858, "recall": 0.3234796310209534, "f1_score": 0.17802439984258167}, "scientific_domains": {"precision": 0.27999999999999947, "recall": 0.6363714285714287, "f1_score": 0.37014829549535416}, "target_users": {"precision": 0.4720000000000006, "recall": 0.6218165889665889, "f1_score": 0.4750951652922249}}}, "(0.1, 10, 0.9, 5)": {"precision": 0.25785224977043164, "recall": 0.621636278001154, "f1_score": 0.32734672876008597, "per_field": {"categories": {"precision": 0.12052341597796175, "recall": 0.43190814580277387, "f1_score": 0.17625884599025085}, "scientific_domains": {"precision": 0.24660000000000037, "recall": 0.6990206349206349, "f1_score": 0.3392699397022894}, "target_users": {"precision": 0.40643333333333276, "recall": 0.7339800532800531, "f1_score": 0.4665114005877177}}}, "(0.1, 15, 0, 1)": {"precision": 0.4300936639118458, "recall": 0.329035927999151, "f1_score": 0.34839732120087863, "per_field": {"categories": {"precision": 0.1962809917355372, "recall": 0.18178792052758996, "f1_score": 0.1844502335609055}, "scientific_domains": {"precision": 0.468, "recall": 0.44269365079365075, "f1_score": 0.4491081081081081}, "target_users": {"precision": 0.626, "recall": 0.36262621267621226, "f1_score": 0.41163362193362224}}}, "(0.1, 15, 0, 3)": {"precision": 0.3631928374655651, "recall": 0.5727037515438341, "f1_score": 0.405467547232472, "per_field": {"categories": {"precision": 0.20557851239669397, "recall": 0.4018129196331675, "f1_score": 0.25662695431290494}, "scientific_domains": {"precision": 0.37866666666666715, "recall": 0.6698373015873018, "f1_score": 0.4574556136673777}, "target_users": {"precision": 0.505333333333334, "recall": 0.6464610334110333, "f1_score": 0.5023200737171335}}}, "(0.1, 15, 0, 5)": {"precision": 0.3411283746556472, "recall": 0.6501481708374269, "f1_score": 0.40829411435899815, "per_field": {"categories": {"precision": 0.20578512396694223, "recall": 0.44766279978056833, "f1_score": 0.2618240154397182}, "scientific_domains": {"precision": 0.37979999999999986, "recall": 0.7443452380952381, "f1_score": 0.469281886534518}, "target_users": {"precision": 0.4377999999999995, "recall": 0.7584364746364745, "f1_score": 0.4937764411027582}}}, "(0.1, 15, 0.1, 1)": {"precision": 0.4300936639118458, "recall": 0.329035927999151, "f1_score": 0.34839732120087863, "per_field": {"categories": {"precision": 0.1962809917355372, "recall": 0.18178792052758996, "f1_score": 0.1844502335609055}, "scientific_domains": {"precision": 0.468, "recall": 0.44269365079365075, "f1_score": 0.4491081081081081}, "target_users": {"precision": 0.626, "recall": 0.36262621267621226, "f1_score": 0.41163362193362224}}}, "(0.1, 15, 0.1, 3)": {"precision": 0.3631928374655651, "recall": 0.5727037515438341, "f1_score": 0.405467547232472, "per_field": {"categories": {"precision": 0.20557851239669397, "recall": 0.4018129196331675, "f1_score": 0.25662695431290494}, "scientific_domains": {"precision": 0.37866666666666715, "recall": 0.6698373015873018, "f1_score": 0.4574556136673777}, "target_users": {"precision": 0.505333333333334, "recall": 0.6464610334110333, "f1_score": 0.5023200737171335}}}, "(0.1, 15, 0.1, 5)": {"precision": 0.3411283746556472, "recall": 0.6501481708374269, "f1_score": 0.40829411435899815, "per_field": {"categories": {"precision": 0.20578512396694223, "recall": 0.44766279978056833, "f1_score": 0.2618240154397182}, "scientific_domains": {"precision": 0.37979999999999986, "recall": 0.7443452380952381, "f1_score": 0.469281886534518}, "target_users": {"precision": 0.4377999999999995, "recall": 0.7584364746364745, "f1_score": 0.4937764411027582}}}, "(0.1, 15, 0.2, 1)": {"precision": 0.4300936639118458, "recall": 0.329035927999151, "f1_score": 0.34839732120087863, "per_field": {"categories": {"precision": 0.1962809917355372, "recall": 0.18178792052758996, "f1_score": 0.1844502335609055}, "scientific_domains": {"precision": 0.468, "recall": 0.44269365079365075, "f1_score": 0.4491081081081081}, "target_users": {"precision": 0.626, "recall": 0.36262621267621226, "f1_score": 0.41163362193362224}}}, "(0.1, 15, 0.2, 3)": {"precision": 0.3631928374655651, "recall": 0.5727037515438341, "f1_score": 0.405467547232472, "per_field": {"categories": {"precision": 0.20557851239669397, "recall": 0.4018129196331675, "f1_score": 0.25662695431290494}, "scientific_domains": {"precision": 0.37866666666666715, "recall": 0.6698373015873018, "f1_score": 0.4574556136673777}, "target_users": {"precision": 0.505333333333334, "recall": 0.6464610334110333, "f1_score": 0.5023200737171335}}}, "(0.1, 15, 0.2, 5)": {"precision": 0.3411283746556472, "recall": 0.6501481708374269, "f1_score": 0.40829411435899815, "per_field": {"categories": {"precision": 0.20578512396694223, "recall": 0.44766279978056833, "f1_score": 0.2618240154397182}, "scientific_domains": {"precision": 0.37979999999999986, "recall": 0.7443452380952381, "f1_score": 0.469281886534518}, "target_users": {"precision": 0.4377999999999995, "recall": 0.7584364746364745, "f1_score": 0.4937764411027582}}}, "(0.1, 15, 0.3, 1)": {"precision": 0.4300936639118458, "recall": 0.329035927999151, "f1_score": 0.34839732120087863, "per_field": {"categories": {"precision": 0.1962809917355372, "recall": 0.18178792052758996, "f1_score": 0.1844502335609055}, "scientific_domains": {"precision": 0.468, "recall": 0.44269365079365075, "f1_score": 0.4491081081081081}, "target_users": {"precision": 0.626, "recall": 0.36262621267621226, "f1_score": 0.41163362193362224}}}, "(0.1, 15, 0.3, 3)": {"precision": 0.3631928374655651, "recall": 0.5727037515438341, "f1_score": 0.405467547232472, "per_field": {"categories": {"precision": 0.20557851239669397, "recall": 0.4018129196331675, "f1_score": 0.25662695431290494}, "scientific_domains": {"precision": 0.37866666666666715, "recall": 0.6698373015873018, "f1_score": 0.4574556136673777}, "target_users": {"precision": 0.505333333333334, "recall": 0.6464610334110333, "f1_score": 0.5023200737171335}}}, "(0.1, 15, 0.3, 5)": {"precision": 0.3411283746556472, "recall": 0.6501481708374269, "f1_score": 0.40829411435899815, "per_field": {"categories": {"precision": 0.20578512396694223, "recall": 0.44766279978056833, "f1_score": 0.2618240154397182}, "scientific_domains": {"precision": 0.37979999999999986, "recall": 0.7443452380952381, "f1_score": 0.469281886534518}, "target_users": {"precision": 0.4377999999999995, "recall": 0.7584364746364745, "f1_score": 0.4937764411027582}}}, "(0.1, 15, 0.4, 1)": {"precision": 0.4300936639118458, "recall": 0.329035927999151, "f1_score": 0.34839732120087863, "per_field": {"categories": {"precision": 0.1962809917355372, "recall": 0.18178792052758996, "f1_score": 0.1844502335609055}, "scientific_domains": {"precision": 0.468, "recall": 0.44269365079365075, "f1_score": 0.4491081081081081}, "target_users": {"precision": 0.626, "recall": 0.36262621267621226, "f1_score": 0.41163362193362224}}}, "(0.1, 15, 0.4, 3)": {"precision": 0.3631928374655651, "recall": 0.5727037515438341, "f1_score": 0.405467547232472, "per_field": {"categories": {"precision": 0.20557851239669397, "recall": 0.4018129196331675, "f1_score": 0.25662695431290494}, "scientific_domains": {"precision": 0.37866666666666715, "recall": 0.6698373015873018, "f1_score": 0.4574556136673777}, "target_users": {"precision": 0.505333333333334, "recall": 0.6464610334110333, "f1_score": 0.5023200737171335}}}, "(0.1, 15, 0.4, 5)": {"precision": 0.3411283746556472, "recall": 0.6501481708374269, "f1_score": 0.40829411435899815, "per_field": {"categories": {"precision": 0.20578512396694223, "recall": 0.44766279978056833, "f1_score": 0.2618240154397182}, "scientific_domains": {"precision": 0.37979999999999986, "recall": 0.7443452380952381, "f1_score": 0.469281886534518}, "target_users": {"precision": 0.4377999999999995, "recall": 0.7584364746364745, "f1_score": 0.4937764411027582}}}, "(0.1, 15, 0.5, 1)": {"precision": 0.4300936639118458, "recall": 0.329035927999151, "f1_score": 0.34839732120087863, "per_field": {"categories": {"precision": 0.1962809917355372, "recall": 0.18178792052758996, "f1_score": 0.1844502335609055}, "scientific_domains": {"precision": 0.468, "recall": 0.44269365079365075, "f1_score": 0.4491081081081081}, "target_users": {"precision": 0.626, "recall": 0.36262621267621226, "f1_score": 0.41163362193362224}}}, "(0.1, 15, 0.5, 3)": {"precision": 0.3631928374655651, "recall": 0.5727037515438341, "f1_score": 0.405467547232472, "per_field": {"categories": {"precision": 0.20557851239669397, "recall": 0.4018129196331675, "f1_score": 0.25662695431290494}, "scientific_domains": {"precision": 0.37866666666666715, "recall": 0.6698373015873018, "f1_score": 0.4574556136673777}, "target_users": {"precision": 0.505333333333334, "recall": 0.6464610334110333, "f1_score": 0.5023200737171335}}}, "(0.1, 15, 0.5, 5)": {"precision": 0.3411283746556472, "recall": 0.6501481708374269, "f1_score": 0.40829411435899815, "per_field": {"categories": {"precision": 0.20578512396694223, "recall": 0.44766279978056833, "f1_score": 0.2618240154397182}, "scientific_domains": {"precision": 0.37979999999999986, "recall": 0.7443452380952381, "f1_score": 0.469281886534518}, "target_users": {"precision": 0.4377999999999995, "recall": 0.7584364746364745, "f1_score": 0.4937764411027582}}}, "(0.1, 15, 0.6, 1)": {"precision": 0.4300936639118458, "recall": 0.329035927999151, "f1_score": 0.34839732120087863, "per_field": {"categories": {"precision": 0.1962809917355372, "recall": 0.18178792052758996, "f1_score": 0.1844502335609055}, "scientific_domains": {"precision": 0.468, "recall": 0.44269365079365075, "f1_score": 0.4491081081081081}, "target_users": {"precision": 0.626, "recall": 0.36262621267621226, "f1_score": 0.41163362193362224}}}, "(0.1, 15, 0.6, 3)": {"precision": 0.3631928374655651, "recall": 0.5727037515438341, "f1_score": 0.405467547232472, "per_field": {"categories": {"precision": 0.20557851239669397, "recall": 0.4018129196331675, "f1_score": 0.25662695431290494}, "scientific_domains": {"precision": 0.37866666666666715, "recall": 0.6698373015873018, "f1_score": 0.4574556136673777}, "target_users": {"precision": 0.505333333333334, "recall": 0.6464610334110333, "f1_score": 0.5023200737171335}}}, "(0.1, 15, 0.6, 5)": {"precision": 0.3411283746556472, "recall": 0.6501481708374269, "f1_score": 0.40829411435899815, "per_field": {"categories": {"precision": 0.20578512396694223, "recall": 0.44766279978056833, "f1_score": 0.2618240154397182}, "scientific_domains": {"precision": 0.37979999999999986, "recall": 0.7443452380952381, "f1_score": 0.469281886534518}, "target_users": {"precision": 0.4377999999999995, "recall": 0.7584364746364745, "f1_score": 0.4937764411027582}}}, "(0.1, 15, 0.7, 1)": {"precision": 0.4300936639118458, "recall": 0.329035927999151, "f1_score": 0.34839732120087863, "per_field": {"categories": {"precision": 0.1962809917355372, "recall": 0.18178792052758996, "f1_score": 0.1844502335609055}, "scientific_domains": {"precision": 0.468, "recall": 0.44269365079365075, "f1_score": 0.4491081081081081}, "target_users": {"precision": 0.626, "recall": 0.36262621267621226, "f1_score": 0.41163362193362224}}}, "(0.1, 15, 0.7, 3)": {"precision": 0.3631928374655651, "recall": 0.5727037515438341, "f1_score": 0.405467547232472, "per_field": {"categories": {"precision": 0.20557851239669397, "recall": 0.4018129196331675, "f1_score": 0.25662695431290494}, "scientific_domains": {"precision": 0.37866666666666715, "recall": 0.6698373015873018, "f1_score": 0.4574556136673777}, "target_users": {"precision": 0.505333333333334, "recall": 0.6464610334110333, "f1_score": 0.5023200737171335}}}, "(0.1, 15, 0.7, 5)": {"precision": 0.3411283746556472, "recall": 0.6501481708374269, "f1_score": 0.40829411435899815, "per_field": {"categories": {"precision": 0.20578512396694223, "recall": 0.44766279978056833, "f1_score": 0.2618240154397182}, "scientific_domains": {"precision": 0.37979999999999986, "recall": 0.7443452380952381, "f1_score": 0.469281886534518}, "target_users": {"precision": 0.4377999999999995, "recall": 0.7584364746364745, "f1_score": 0.4937764411027582}}}, "(0.1, 15, 0.8, 1)": {"precision": 0.4300936639118458, "recall": 0.329035927999151, "f1_score": 0.34839732120087863, "per_field": {"categories": {"precision": 0.1962809917355372, "recall": 0.18178792052758996, "f1_score": 0.1844502335609055}, "scientific_domains": {"precision": 0.468, "recall": 0.44269365079365075, "f1_score": 0.4491081081081081}, "target_users": {"precision": 0.626, "recall": 0.36262621267621226, "f1_score": 0.41163362193362224}}}, "(0.1, 15, 0.8, 3)": {"precision": 0.3631928374655651, "recall": 0.5727037515438341, "f1_score": 0.405467547232472, "per_field": {"categories": {"precision": 0.20557851239669397, "recall": 0.4018129196331675, "f1_score": 0.25662695431290494}, "scientific_domains": {"precision": 0.37866666666666715, "recall": 0.6698373015873018, "f1_score": 0.4574556136673777}, "target_users": {"precision": 0.505333333333334, "recall": 0.6464610334110333, "f1_score": 0.5023200737171335}}}, "(0.1, 15, 0.8, 5)": {"precision": 0.3411283746556472, "recall": 0.6501481708374269, "f1_score": 0.40829411435899815, "per_field": {"categories": {"precision": 0.20578512396694223, "recall": 0.44766279978056833, "f1_score": 0.2618240154397182}, "scientific_domains": {"precision": 0.37979999999999986, "recall": 0.7443452380952381, "f1_score": 0.469281886534518}, "target_users": {"precision": 0.4377999999999995, "recall": 0.7584364746364745, "f1_score": 0.4937764411027582}}}, "(0.1, 15, 0.9, 1)": {"precision": 0.4300936639118458, "recall": 0.329035927999151, "f1_score": 0.34839732120087863, "per_field": {"categories": {"precision": 0.1962809917355372, "recall": 0.18178792052758996, "f1_score": 0.1844502335609055}, "scientific_domains": {"precision": 0.468, "recall": 0.44269365079365075, "f1_score": 0.4491081081081081}, "target_users": {"precision": 0.626, "recall": 0.36262621267621226, "f1_score": 0.41163362193362224}}}, "(0.1, 15, 0.9, 3)": {"precision": 0.3631928374655651, "recall": 0.5727037515438341, "f1_score": 0.405467547232472, "per_field": {"categories": {"precision": 0.20557851239669397, "recall": 0.4018129196331675, "f1_score": 0.25662695431290494}, "scientific_domains": {"precision": 0.37866666666666715, "recall": 0.6698373015873018, "f1_score": 0.4574556136673777}, "target_users": {"precision": 0.505333333333334, "recall": 0.6464610334110333, "f1_score": 0.5023200737171335}}}, "(0.1, 15, 0.9, 5)": {"precision": 0.3411283746556472, "recall": 0.6501481708374269, "f1_score": 0.40829411435899815, "per_field": {"categories": {"precision": 0.20578512396694223, "recall": 0.44766279978056833, "f1_score": 0.2618240154397182}, "scientific_domains": {"precision": 0.37979999999999986, "recall": 0.7443452380952381, "f1_score": 0.469281886534518}, "target_users": {"precision": 0.4377999999999995, "recall": 0.7584364746364745, "f1_score": 0.4937764411027582}}}, "(0.2, 5, 0, 1)": {"precision": 0.4511349862258953, "recall": 0.35291196900742344, "f1_score": 0.37233408579772237, "per_field": {"categories": {"precision": 0.23140495867768596, "recall": 0.21166207529843897, "f1_score": 0.21652892561983475}, "scientific_domains": {"precision": 0.516, "recall": 0.4796150793650794, "f1_score": 0.4880525525525526}, "target_users": {"precision": 0.606, "recall": 0.36745875235875197, "f1_score": 0.4124207792207796}}}, "(0.2, 5, 0, 3)": {"precision": 0.38733700642791574, "recall": 0.5413184321952919, "f1_score": 0.4134574493935457, "per_field": {"categories": {"precision": 0.20867768595041303, "recall": 0.3857537148342933, "f1_score": 0.25482192050373886}, "scientific_domains": {"precision": 0.4170000000000005, "recall": 0.6349333333333335, "f1_score": 0.47324844632491647}, "target_users": {"precision": 0.5363333333333338, "recall": 0.6032682484182484, "f1_score": 0.5123019813519817}}}, "(0.2, 5, 0, 5)": {"precision": 0.37362277318640924, "recall": 0.6030441083709679, "f1_score": 0.4211618763306051, "per_field": {"categories": {"precision": 0.2104683195592288, "recall": 0.4512840178645963, "f1_score": 0.26594968774720856}, "scientific_domains": {"precision": 0.4114999999999999, "recall": 0.6644992063492065, "f1_score": 0.47337812683679176}, "target_users": {"precision": 0.49889999999999907, "recall": 0.6933491008991008, "f1_score": 0.524157814407815}}}, "(0.2, 5, 0.1, 1)": {"precision": 0.4511349862258953, "recall": 0.35291196900742344, "f1_score": 0.37233408579772237, "per_field": {"categories": {"precision": 0.23140495867768596, "recall": 0.21166207529843897, "f1_score": 0.21652892561983475}, "scientific_domains": {"precision": 0.516, "recall": 0.4796150793650794, "f1_score": 0.4880525525525526}, "target_users": {"precision": 0.606, "recall": 0.36745875235875197, "f1_score": 0.4124207792207796}}}, "(0.2, 5, 0.1, 3)": {"precision": 0.38733700642791574, "recall": 0.5413184321952919, "f1_score": 0.4134574493935457, "per_field": {"categories": {"precision": 0.20867768595041303, "recall": 0.3857537148342933, "f1_score": 0.25482192050373886}, "scientific_domains": {"precision": 0.4170000000000005, "recall": 0.6349333333333335, "f1_score": 0.47324844632491647}, "target_users": {"precision": 0.5363333333333338, "recall": 0.6032682484182484, "f1_score": 0.5123019813519817}}}, "(0.2, 5, 0.1, 5)": {"precision": 0.37362277318640924, "recall": 0.6030441083709679, "f1_score": 0.4211618763306051, "per_field": {"categories": {"precision": 0.2104683195592288, "recall": 0.4512840178645963, "f1_score": 0.26594968774720856}, "scientific_domains": {"precision": 0.4114999999999999, "recall": 0.6644992063492065, "f1_score": 0.47337812683679176}, "target_users": {"precision": 0.49889999999999907, "recall": 0.6933491008991008, "f1_score": 0.524157814407815}}}, "(0.2, 5, 0.2, 1)": {"precision": 0.4511349862258953, "recall": 0.35291196900742344, "f1_score": 0.37233408579772237, "per_field": {"categories": {"precision": 0.23140495867768596, "recall": 0.21166207529843897, "f1_score": 0.21652892561983475}, "scientific_domains": {"precision": 0.516, "recall": 0.4796150793650794, "f1_score": 0.4880525525525526}, "target_users": {"precision": 0.606, "recall": 0.36745875235875197, "f1_score": 0.4124207792207796}}}, "(0.2, 5, 0.2, 3)": {"precision": 0.38733700642791574, "recall": 0.5413184321952919, "f1_score": 0.4134574493935457, "per_field": {"categories": {"precision": 0.20867768595041303, "recall": 0.3857537148342933, "f1_score": 0.25482192050373886}, "scientific_domains": {"precision": 0.4170000000000005, "recall": 0.6349333333333335, "f1_score": 0.47324844632491647}, "target_users": {"precision": 0.5363333333333338, "recall": 0.6032682484182484, "f1_score": 0.5123019813519817}}}, "(0.2, 5, 0.2, 5)": {"precision": 0.37362277318640924, "recall": 0.6030441083709679, "f1_score": 0.4211618763306051, "per_field": {"categories": {"precision": 0.2104683195592288, "recall": 0.4512840178645963, "f1_score": 0.26594968774720856}, "scientific_domains": {"precision": 0.4114999999999999, "recall": 0.6644992063492065, "f1_score": 0.47337812683679176}, "target_users": {"precision": 0.49889999999999907, "recall": 0.6933491008991008, "f1_score": 0.524157814407815}}}, "(0.2, 5, 0.3, 1)": {"precision": 0.4511349862258953, "recall": 0.35291196900742344, "f1_score": 0.37233408579772237, "per_field": {"categories": {"precision": 0.23140495867768596, "recall": 0.21166207529843897, "f1_score": 0.21652892561983475}, "scientific_domains": {"precision": 0.516, "recall": 0.4796150793650794, "f1_score": 0.4880525525525526}, "target_users": {"precision": 0.606, "recall": 0.36745875235875197, "f1_score": 0.4124207792207796}}}, "(0.2, 5, 0.3, 3)": {"precision": 0.38733700642791574, "recall": 0.5413184321952919, "f1_score": 0.4134574493935457, "per_field": {"categories": {"precision": 0.20867768595041303, "recall": 0.3857537148342933, "f1_score": 0.25482192050373886}, "scientific_domains": {"precision": 0.4170000000000005, "recall": 0.6349333333333335, "f1_score": 0.47324844632491647}, "target_users": {"precision": 0.5363333333333338, "recall": 0.6032682484182484, "f1_score": 0.5123019813519817}}}, "(0.2, 5, 0.3, 5)": {"precision": 0.37362277318640924, "recall": 0.6030441083709679, "f1_score": 0.4211618763306051, "per_field": {"categories": {"precision": 0.2104683195592288, "recall": 0.4512840178645963, "f1_score": 0.26594968774720856}, "scientific_domains": {"precision": 0.4114999999999999, "recall": 0.6644992063492065, "f1_score": 0.47337812683679176}, "target_users": {"precision": 0.49889999999999907, "recall": 0.6933491008991008, "f1_score": 0.524157814407815}}}, "(0.2, 5, 0.4, 1)": {"precision": 0.4511349862258953, "recall": 0.35291196900742344, "f1_score": 0.37233408579772237, "per_field": {"categories": {"precision": 0.23140495867768596, "recall": 0.21166207529843897, "f1_score": 0.21652892561983475}, "scientific_domains": {"precision": 0.516, "recall": 0.4796150793650794, "f1_score": 0.4880525525525526}, "target_users": {"precision": 0.606, "recall": 0.36745875235875197, "f1_score": 0.4124207792207796}}}, "(0.2, 5, 0.4, 3)": {"precision": 0.38733700642791574, "recall": 0.5413184321952919, "f1_score": 0.4134574493935457, "per_field": {"categories": {"precision": 0.20867768595041303, "recall": 0.3857537148342933, "f1_score": 0.25482192050373886}, "scientific_domains": {"precision": 0.4170000000000005, "recall": 0.6349333333333335, "f1_score": 0.47324844632491647}, "target_users": {"precision": 0.5363333333333338, "recall": 0.6032682484182484, "f1_score": 0.5123019813519817}}}, "(0.2, 5, 0.4, 5)": {"precision": 0.37362277318640924, "recall": 0.6030441083709679, "f1_score": 0.4211618763306051, "per_field": {"categories": {"precision": 0.2104683195592288, "recall": 0.4512840178645963, "f1_score": 0.26594968774720856}, "scientific_domains": {"precision": 0.4114999999999999, "recall": 0.6644992063492065, "f1_score": 0.47337812683679176}, "target_users": {"precision": 0.49889999999999907, "recall": 0.6933491008991008, "f1_score": 0.524157814407815}}}, "(0.2, 5, 0.5, 1)": {"precision": 0.4511349862258953, "recall": 0.35291196900742344, "f1_score": 0.37233408579772237, "per_field": {"categories": {"precision": 0.23140495867768596, "recall": 0.21166207529843897, "f1_score": 0.21652892561983475}, "scientific_domains": {"precision": 0.516, "recall": 0.4796150793650794, "f1_score": 0.4880525525525526}, "target_users": {"precision": 0.606, "recall": 0.36745875235875197, "f1_score": 0.4124207792207796}}}, "(0.2, 5, 0.5, 3)": {"precision": 0.38733700642791574, "recall": 0.5413184321952919, "f1_score": 0.4134574493935457, "per_field": {"categories": {"precision": 0.20867768595041303, "recall": 0.3857537148342933, "f1_score": 0.25482192050373886}, "scientific_domains": {"precision": 0.4170000000000005, "recall": 0.6349333333333335, "f1_score": 0.47324844632491647}, "target_users": {"precision": 0.5363333333333338, "recall": 0.6032682484182484, "f1_score": 0.5123019813519817}}}, "(0.2, 5, 0.5, 5)": {"precision": 0.37362277318640924, "recall": 0.6030441083709679, "f1_score": 0.4211618763306051, "per_field": {"categories": {"precision": 0.2104683195592288, "recall": 0.4512840178645963, "f1_score": 0.26594968774720856}, "scientific_domains": {"precision": 0.4114999999999999, "recall": 0.6644992063492065, "f1_score": 0.47337812683679176}, "target_users": {"precision": 0.49889999999999907, "recall": 0.6933491008991008, "f1_score": 0.524157814407815}}}, "(0.2, 5, 0.6, 1)": {"precision": 0.4511349862258953, "recall": 0.35291196900742344, "f1_score": 0.37233408579772237, "per_field": {"categories": {"precision": 0.23140495867768596, "recall": 0.21166207529843897, "f1_score": 0.21652892561983475}, "scientific_domains": {"precision": 0.516, "recall": 0.4796150793650794, "f1_score": 0.4880525525525526}, "target_users": {"precision": 0.606, "recall": 0.36745875235875197, "f1_score": 0.4124207792207796}}}, "(0.2, 5, 0.6, 3)": {"precision": 0.38733700642791574, "recall": 0.5413184321952919, "f1_score": 0.4134574493935457, "per_field": {"categories": {"precision": 0.20867768595041303, "recall": 0.3857537148342933, "f1_score": 0.25482192050373886}, "scientific_domains": {"precision": 0.4170000000000005, "recall": 0.6349333333333335, "f1_score": 0.47324844632491647}, "target_users": {"precision": 0.5363333333333338, "recall": 0.6032682484182484, "f1_score": 0.5123019813519817}}}, "(0.2, 5, 0.6, 5)": {"precision": 0.37362277318640924, "recall": 0.6030441083709679, "f1_score": 0.4211618763306051, "per_field": {"categories": {"precision": 0.2104683195592288, "recall": 0.4512840178645963, "f1_score": 0.26594968774720856}, "scientific_domains": {"precision": 0.4114999999999999, "recall": 0.6644992063492065, "f1_score": 0.47337812683679176}, "target_users": {"precision": 0.49889999999999907, "recall": 0.6933491008991008, "f1_score": 0.524157814407815}}}, "(0.2, 5, 0.7, 1)": {"precision": 0.4511349862258953, "recall": 0.35291196900742344, "f1_score": 0.37233408579772237, "per_field": {"categories": {"precision": 0.23140495867768596, "recall": 0.21166207529843897, "f1_score": 0.21652892561983475}, "scientific_domains": {"precision": 0.516, "recall": 0.4796150793650794, "f1_score": 0.4880525525525526}, "target_users": {"precision": 0.606, "recall": 0.36745875235875197, "f1_score": 0.4124207792207796}}}, "(0.2, 5, 0.7, 3)": {"precision": 0.38733700642791574, "recall": 0.5413184321952919, "f1_score": 0.4134574493935457, "per_field": {"categories": {"precision": 0.20867768595041303, "recall": 0.3857537148342933, "f1_score": 0.25482192050373886}, "scientific_domains": {"precision": 0.4170000000000005, "recall": 0.6349333333333335, "f1_score": 0.47324844632491647}, "target_users": {"precision": 0.5363333333333338, "recall": 0.6032682484182484, "f1_score": 0.5123019813519817}}}, "(0.2, 5, 0.7, 5)": {"precision": 0.37362277318640924, "recall": 0.6030441083709679, "f1_score": 0.4211618763306051, "per_field": {"categories": {"precision": 0.2104683195592288, "recall": 0.4512840178645963, "f1_score": 0.26594968774720856}, "scientific_domains": {"precision": 0.4114999999999999, "recall": 0.6644992063492065, "f1_score": 0.47337812683679176}, "target_users": {"precision": 0.49889999999999907, "recall": 0.6933491008991008, "f1_score": 0.524157814407815}}}, "(0.2, 5, 0.8, 1)": {"precision": 0.4511349862258953, "recall": 0.35291196900742344, "f1_score": 0.37233408579772237, "per_field": {"categories": {"precision": 0.23140495867768596, "recall": 0.21166207529843897, "f1_score": 0.21652892561983475}, "scientific_domains": {"precision": 0.516, "recall": 0.4796150793650794, "f1_score": 0.4880525525525526}, "target_users": {"precision": 0.606, "recall": 0.36745875235875197, "f1_score": 0.4124207792207796}}}, "(0.2, 5, 0.8, 3)": {"precision": 0.38733700642791574, "recall": 0.5413184321952919, "f1_score": 0.4134574493935457, "per_field": {"categories": {"precision": 0.20867768595041303, "recall": 0.3857537148342933, "f1_score": 0.25482192050373886}, "scientific_domains": {"precision": 0.4170000000000005, "recall": 0.6349333333333335, "f1_score": 0.47324844632491647}, "target_users": {"precision": 0.5363333333333338, "recall": 0.6032682484182484, "f1_score": 0.5123019813519817}}}, "(0.2, 5, 0.8, 5)": {"precision": 0.37362277318640924, "recall": 0.6030441083709679, "f1_score": 0.4211618763306051, "per_field": {"categories": {"precision": 0.2104683195592288, "recall": 0.4512840178645963, "f1_score": 0.26594968774720856}, "scientific_domains": {"precision": 0.4114999999999999, "recall": 0.6644992063492065, "f1_score": 0.47337812683679176}, "target_users": {"precision": 0.49889999999999907, "recall": 0.6933491008991008, "f1_score": 0.524157814407815}}}, "(0.2, 5, 0.9, 1)": {"precision": 0.4511349862258953, "recall": 0.35291196900742344, "f1_score": 0.37233408579772237, "per_field": {"categories": {"precision": 0.23140495867768596, "recall": 0.21166207529843897, "f1_score": 0.21652892561983475}, "scientific_domains": {"precision": 0.516, "recall": 0.4796150793650794, "f1_score": 0.4880525525525526}, "target_users": {"precision": 0.606, "recall": 0.36745875235875197, "f1_score": 0.4124207792207796}}}, "(0.2, 5, 0.9, 3)": {"precision": 0.38733700642791574, "recall": 0.5413184321952919, "f1_score": 0.4134574493935457, "per_field": {"categories": {"precision": 0.20867768595041303, "recall": 0.3857537148342933, "f1_score": 0.25482192050373886}, "scientific_domains": {"precision": 0.4170000000000005, "recall": 0.6349333333333335, "f1_score": 0.47324844632491647}, "target_users": {"precision": 0.5363333333333338, "recall": 0.6032682484182484, "f1_score": 0.5123019813519817}}}, "(0.2, 5, 0.9, 5)": {"precision": 0.37362277318640924, "recall": 0.6030441083709679, "f1_score": 0.4211618763306051, "per_field": {"categories": {"precision": 0.2104683195592288, "recall": 0.4512840178645963, "f1_score": 0.26594968774720856}, "scientific_domains": {"precision": 0.4114999999999999, "recall": 0.6644992063492065, "f1_score": 0.47337812683679176}, "target_users": {"precision": 0.49889999999999907, "recall": 0.6933491008991008, "f1_score": 0.524157814407815}}}, "(0.2, 10, 0, 1)": {"precision": 0.42260606060606065, "recall": 0.3285154373786604, "f1_score": 0.3467846883791548, "per_field": {"categories": {"precision": 0.18181818181818182, "recall": 0.16587882961849904, "f1_score": 0.1688854952688945}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.596, "recall": 0.3545166888666885, "f1_score": 0.3999271284271287}}}, "(0.2, 10, 0, 3)": {"precision": 0.3251120293847569, "recall": 0.5306325455841986, "f1_score": 0.36487773924431705, "per_field": {"categories": {"precision": 0.15633608815426991, "recall": 0.3479286668336255, "f1_score": 0.20434277843368753}, "scientific_domains": {"precision": 0.33633333333333365, "recall": 0.6373428571428574, "f1_score": 0.41340166761343217}, "target_users": {"precision": 0.48266666666666724, "recall": 0.6066261127761129, "f1_score": 0.4768887716858313}}}, "(0.2, 10, 0, 5)": {"precision": 0.2988595041322313, "recall": 0.6051641323017356, "f1_score": 0.35784555449575367, "per_field": {"categories": {"precision": 0.14724517906336124, "recall": 0.43283297854578845, "f1_score": 0.20167370890511382}, "scientific_domains": {"precision": 0.31599999999999984, "recall": 0.6796301587301588, "f1_score": 0.39566901431189005}, "target_users": {"precision": 0.4333333333333328, "recall": 0.7030292596292596, "f1_score": 0.4761939402702571}}}, "(0.2, 10, 0.1, 1)": {"precision": 0.42260606060606065, "recall": 0.3285154373786604, "f1_score": 0.3467846883791548, "per_field": {"categories": {"precision": 0.18181818181818182, "recall": 0.16587882961849904, "f1_score": 0.1688854952688945}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.596, "recall": 0.3545166888666885, "f1_score": 0.3999271284271287}}}, "(0.2, 10, 0.1, 3)": {"precision": 0.3251120293847569, "recall": 0.5306325455841986, "f1_score": 0.36487773924431705, "per_field": {"categories": {"precision": 0.15633608815426991, "recall": 0.3479286668336255, "f1_score": 0.20434277843368753}, "scientific_domains": {"precision": 0.33633333333333365, "recall": 0.6373428571428574, "f1_score": 0.41340166761343217}, "target_users": {"precision": 0.48266666666666724, "recall": 0.6066261127761129, "f1_score": 0.4768887716858313}}}, "(0.2, 10, 0.1, 5)": {"precision": 0.2988595041322313, "recall": 0.6051641323017356, "f1_score": 0.35784555449575367, "per_field": {"categories": {"precision": 0.14724517906336124, "recall": 0.43283297854578845, "f1_score": 0.20167370890511382}, "scientific_domains": {"precision": 0.31599999999999984, "recall": 0.6796301587301588, "f1_score": 0.39566901431189005}, "target_users": {"precision": 0.4333333333333328, "recall": 0.7030292596292596, "f1_score": 0.4761939402702571}}}, "(0.2, 10, 0.2, 1)": {"precision": 0.42260606060606065, "recall": 0.3285154373786604, "f1_score": 0.3467846883791548, "per_field": {"categories": {"precision": 0.18181818181818182, "recall": 0.16587882961849904, "f1_score": 0.1688854952688945}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.596, "recall": 0.3545166888666885, "f1_score": 0.3999271284271287}}}, "(0.2, 10, 0.2, 3)": {"precision": 0.3251120293847569, "recall": 0.5306325455841986, "f1_score": 0.36487773924431705, "per_field": {"categories": {"precision": 0.15633608815426991, "recall": 0.3479286668336255, "f1_score": 0.20434277843368753}, "scientific_domains": {"precision": 0.33633333333333365, "recall": 0.6373428571428574, "f1_score": 0.41340166761343217}, "target_users": {"precision": 0.48266666666666724, "recall": 0.6066261127761129, "f1_score": 0.4768887716858313}}}, "(0.2, 10, 0.2, 5)": {"precision": 0.2988595041322313, "recall": 0.6051641323017356, "f1_score": 0.35784555449575367, "per_field": {"categories": {"precision": 0.14724517906336124, "recall": 0.43283297854578845, "f1_score": 0.20167370890511382}, "scientific_domains": {"precision": 0.31599999999999984, "recall": 0.6796301587301588, "f1_score": 0.39566901431189005}, "target_users": {"precision": 0.4333333333333328, "recall": 0.7030292596292596, "f1_score": 0.4761939402702571}}}, "(0.2, 10, 0.3, 1)": {"precision": 0.42260606060606065, "recall": 0.3285154373786604, "f1_score": 0.3467846883791548, "per_field": {"categories": {"precision": 0.18181818181818182, "recall": 0.16587882961849904, "f1_score": 0.1688854952688945}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.596, "recall": 0.3545166888666885, "f1_score": 0.3999271284271287}}}, "(0.2, 10, 0.3, 3)": {"precision": 0.3251120293847569, "recall": 0.5306325455841986, "f1_score": 0.36487773924431705, "per_field": {"categories": {"precision": 0.15633608815426991, "recall": 0.3479286668336255, "f1_score": 0.20434277843368753}, "scientific_domains": {"precision": 0.33633333333333365, "recall": 0.6373428571428574, "f1_score": 0.41340166761343217}, "target_users": {"precision": 0.48266666666666724, "recall": 0.6066261127761129, "f1_score": 0.4768887716858313}}}, "(0.2, 10, 0.3, 5)": {"precision": 0.2988595041322313, "recall": 0.6051641323017356, "f1_score": 0.35784555449575367, "per_field": {"categories": {"precision": 0.14724517906336124, "recall": 0.43283297854578845, "f1_score": 0.20167370890511382}, "scientific_domains": {"precision": 0.31599999999999984, "recall": 0.6796301587301588, "f1_score": 0.39566901431189005}, "target_users": {"precision": 0.4333333333333328, "recall": 0.7030292596292596, "f1_score": 0.4761939402702571}}}, "(0.2, 10, 0.4, 1)": {"precision": 0.42260606060606065, "recall": 0.3285154373786604, "f1_score": 0.3467846883791548, "per_field": {"categories": {"precision": 0.18181818181818182, "recall": 0.16587882961849904, "f1_score": 0.1688854952688945}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.596, "recall": 0.3545166888666885, "f1_score": 0.3999271284271287}}}, "(0.2, 10, 0.4, 3)": {"precision": 0.3251120293847569, "recall": 0.5306325455841986, "f1_score": 0.36487773924431705, "per_field": {"categories": {"precision": 0.15633608815426991, "recall": 0.3479286668336255, "f1_score": 0.20434277843368753}, "scientific_domains": {"precision": 0.33633333333333365, "recall": 0.6373428571428574, "f1_score": 0.41340166761343217}, "target_users": {"precision": 0.48266666666666724, "recall": 0.6066261127761129, "f1_score": 0.4768887716858313}}}, "(0.2, 10, 0.4, 5)": {"precision": 0.2988595041322313, "recall": 0.6051641323017356, "f1_score": 0.35784555449575367, "per_field": {"categories": {"precision": 0.14724517906336124, "recall": 0.43283297854578845, "f1_score": 0.20167370890511382}, "scientific_domains": {"precision": 0.31599999999999984, "recall": 0.6796301587301588, "f1_score": 0.39566901431189005}, "target_users": {"precision": 0.4333333333333328, "recall": 0.7030292596292596, "f1_score": 0.4761939402702571}}}, "(0.2, 10, 0.5, 1)": {"precision": 0.42260606060606065, "recall": 0.3285154373786604, "f1_score": 0.3467846883791548, "per_field": {"categories": {"precision": 0.18181818181818182, "recall": 0.16587882961849904, "f1_score": 0.1688854952688945}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.596, "recall": 0.3545166888666885, "f1_score": 0.3999271284271287}}}, "(0.2, 10, 0.5, 3)": {"precision": 0.3251120293847569, "recall": 0.5306325455841986, "f1_score": 0.36487773924431705, "per_field": {"categories": {"precision": 0.15633608815426991, "recall": 0.3479286668336255, "f1_score": 0.20434277843368753}, "scientific_domains": {"precision": 0.33633333333333365, "recall": 0.6373428571428574, "f1_score": 0.41340166761343217}, "target_users": {"precision": 0.48266666666666724, "recall": 0.6066261127761129, "f1_score": 0.4768887716858313}}}, "(0.2, 10, 0.5, 5)": {"precision": 0.2988595041322313, "recall": 0.6051641323017356, "f1_score": 0.35784555449575367, "per_field": {"categories": {"precision": 0.14724517906336124, "recall": 0.43283297854578845, "f1_score": 0.20167370890511382}, "scientific_domains": {"precision": 0.31599999999999984, "recall": 0.6796301587301588, "f1_score": 0.39566901431189005}, "target_users": {"precision": 0.4333333333333328, "recall": 0.7030292596292596, "f1_score": 0.4761939402702571}}}, "(0.2, 10, 0.6, 1)": {"precision": 0.42260606060606065, "recall": 0.3285154373786604, "f1_score": 0.3467846883791548, "per_field": {"categories": {"precision": 0.18181818181818182, "recall": 0.16587882961849904, "f1_score": 0.1688854952688945}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.596, "recall": 0.3545166888666885, "f1_score": 0.3999271284271287}}}, "(0.2, 10, 0.6, 3)": {"precision": 0.3251120293847569, "recall": 0.5306325455841986, "f1_score": 0.36487773924431705, "per_field": {"categories": {"precision": 0.15633608815426991, "recall": 0.3479286668336255, "f1_score": 0.20434277843368753}, "scientific_domains": {"precision": 0.33633333333333365, "recall": 0.6373428571428574, "f1_score": 0.41340166761343217}, "target_users": {"precision": 0.48266666666666724, "recall": 0.6066261127761129, "f1_score": 0.4768887716858313}}}, "(0.2, 10, 0.6, 5)": {"precision": 0.2988595041322313, "recall": 0.6051641323017356, "f1_score": 0.35784555449575367, "per_field": {"categories": {"precision": 0.14724517906336124, "recall": 0.43283297854578845, "f1_score": 0.20167370890511382}, "scientific_domains": {"precision": 0.31599999999999984, "recall": 0.6796301587301588, "f1_score": 0.39566901431189005}, "target_users": {"precision": 0.4333333333333328, "recall": 0.7030292596292596, "f1_score": 0.4761939402702571}}}, "(0.2, 10, 0.7, 1)": {"precision": 0.42260606060606065, "recall": 0.3285154373786604, "f1_score": 0.3467846883791548, "per_field": {"categories": {"precision": 0.18181818181818182, "recall": 0.16587882961849904, "f1_score": 0.1688854952688945}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.596, "recall": 0.3545166888666885, "f1_score": 0.3999271284271287}}}, "(0.2, 10, 0.7, 3)": {"precision": 0.3251120293847569, "recall": 0.5306325455841986, "f1_score": 0.36487773924431705, "per_field": {"categories": {"precision": 0.15633608815426991, "recall": 0.3479286668336255, "f1_score": 0.20434277843368753}, "scientific_domains": {"precision": 0.33633333333333365, "recall": 0.6373428571428574, "f1_score": 0.41340166761343217}, "target_users": {"precision": 0.48266666666666724, "recall": 0.6066261127761129, "f1_score": 0.4768887716858313}}}, "(0.2, 10, 0.7, 5)": {"precision": 0.2988595041322313, "recall": 0.6051641323017356, "f1_score": 0.35784555449575367, "per_field": {"categories": {"precision": 0.14724517906336124, "recall": 0.43283297854578845, "f1_score": 0.20167370890511382}, "scientific_domains": {"precision": 0.31599999999999984, "recall": 0.6796301587301588, "f1_score": 0.39566901431189005}, "target_users": {"precision": 0.4333333333333328, "recall": 0.7030292596292596, "f1_score": 0.4761939402702571}}}, "(0.2, 10, 0.8, 1)": {"precision": 0.42260606060606065, "recall": 0.3285154373786604, "f1_score": 0.3467846883791548, "per_field": {"categories": {"precision": 0.18181818181818182, "recall": 0.16587882961849904, "f1_score": 0.1688854952688945}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.596, "recall": 0.3545166888666885, "f1_score": 0.3999271284271287}}}, "(0.2, 10, 0.8, 3)": {"precision": 0.3251120293847569, "recall": 0.5306325455841986, "f1_score": 0.36487773924431705, "per_field": {"categories": {"precision": 0.15633608815426991, "recall": 0.3479286668336255, "f1_score": 0.20434277843368753}, "scientific_domains": {"precision": 0.33633333333333365, "recall": 0.6373428571428574, "f1_score": 0.41340166761343217}, "target_users": {"precision": 0.48266666666666724, "recall": 0.6066261127761129, "f1_score": 0.4768887716858313}}}, "(0.2, 10, 0.8, 5)": {"precision": 0.2988595041322313, "recall": 0.6051641323017356, "f1_score": 0.35784555449575367, "per_field": {"categories": {"precision": 0.14724517906336124, "recall": 0.43283297854578845, "f1_score": 0.20167370890511382}, "scientific_domains": {"precision": 0.31599999999999984, "recall": 0.6796301587301588, "f1_score": 0.39566901431189005}, "target_users": {"precision": 0.4333333333333328, "recall": 0.7030292596292596, "f1_score": 0.4761939402702571}}}, "(0.2, 10, 0.9, 1)": {"precision": 0.42260606060606065, "recall": 0.3285154373786604, "f1_score": 0.3467846883791548, "per_field": {"categories": {"precision": 0.18181818181818182, "recall": 0.16587882961849904, "f1_score": 0.1688854952688945}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.596, "recall": 0.3545166888666885, "f1_score": 0.3999271284271287}}}, "(0.2, 10, 0.9, 3)": {"precision": 0.3251120293847569, "recall": 0.5306325455841986, "f1_score": 0.36487773924431705, "per_field": {"categories": {"precision": 0.15633608815426991, "recall": 0.3479286668336255, "f1_score": 0.20434277843368753}, "scientific_domains": {"precision": 0.33633333333333365, "recall": 0.6373428571428574, "f1_score": 0.41340166761343217}, "target_users": {"precision": 0.48266666666666724, "recall": 0.6066261127761129, "f1_score": 0.4768887716858313}}}, "(0.2, 10, 0.9, 5)": {"precision": 0.2988595041322313, "recall": 0.6051641323017356, "f1_score": 0.35784555449575367, "per_field": {"categories": {"precision": 0.14724517906336124, "recall": 0.43283297854578845, "f1_score": 0.20167370890511382}, "scientific_domains": {"precision": 0.31599999999999984, "recall": 0.6796301587301588, "f1_score": 0.39566901431189005}, "target_users": {"precision": 0.4333333333333328, "recall": 0.7030292596292596, "f1_score": 0.4761939402702571}}}, "(0.2, 15, 0, 1)": {"precision": 0.4241157024793389, "recall": 0.3297917459186053, "f1_score": 0.3480276635857665, "per_field": {"categories": {"precision": 0.19834710743801653, "recall": 0.18240775523833375, "f1_score": 0.1854144208887292}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.584, "recall": 0.34181668886668853, "f1_score": 0.38712712842712876}}}, "(0.2, 15, 0, 3)": {"precision": 0.35971992653810875, "recall": 0.5411570499748186, "f1_score": 0.3941138844761354, "per_field": {"categories": {"precision": 0.19249311294765822, "recall": 0.37504106889437466, "f1_score": 0.2399839003971236}, "scientific_domains": {"precision": 0.38466666666666716, "recall": 0.6352952380952382, "f1_score": 0.45176114473761486}, "target_users": {"precision": 0.5020000000000008, "recall": 0.6131348429348429, "f1_score": 0.4905966082936678}}}, "(0.2, 15, 0, 5)": {"precision": 0.3401312213039485, "recall": 0.6095720968349068, "f1_score": 0.39573447602106776, "per_field": {"categories": {"precision": 0.19042699724517925, "recall": 0.43282433246276214, "f1_score": 0.2440024010271944}, "scientific_domains": {"precision": 0.38053333333333333, "recall": 0.6883920634920636, "f1_score": 0.45471685477551976}, "target_users": {"precision": 0.44943333333333296, "recall": 0.7074998945498945, "f1_score": 0.4884841722604892}}}, "(0.2, 15, 0.1, 1)": {"precision": 0.4241157024793389, "recall": 0.3297917459186053, "f1_score": 0.3480276635857665, "per_field": {"categories": {"precision": 0.19834710743801653, "recall": 0.18240775523833375, "f1_score": 0.1854144208887292}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.584, "recall": 0.34181668886668853, "f1_score": 0.38712712842712876}}}, "(0.2, 15, 0.1, 3)": {"precision": 0.35971992653810875, "recall": 0.5411570499748186, "f1_score": 0.3941138844761354, "per_field": {"categories": {"precision": 0.19249311294765822, "recall": 0.37504106889437466, "f1_score": 0.2399839003971236}, "scientific_domains": {"precision": 0.38466666666666716, "recall": 0.6352952380952382, "f1_score": 0.45176114473761486}, "target_users": {"precision": 0.5020000000000008, "recall": 0.6131348429348429, "f1_score": 0.4905966082936678}}}, "(0.2, 15, 0.1, 5)": {"precision": 0.3401312213039485, "recall": 0.6095720968349068, "f1_score": 0.39573447602106776, "per_field": {"categories": {"precision": 0.19042699724517925, "recall": 0.43282433246276214, "f1_score": 0.2440024010271944}, "scientific_domains": {"precision": 0.38053333333333333, "recall": 0.6883920634920636, "f1_score": 0.45471685477551976}, "target_users": {"precision": 0.44943333333333296, "recall": 0.7074998945498945, "f1_score": 0.4884841722604892}}}, "(0.2, 15, 0.2, 1)": {"precision": 0.4241157024793389, "recall": 0.3297917459186053, "f1_score": 0.3480276635857665, "per_field": {"categories": {"precision": 0.19834710743801653, "recall": 0.18240775523833375, "f1_score": 0.1854144208887292}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.584, "recall": 0.34181668886668853, "f1_score": 0.38712712842712876}}}, "(0.2, 15, 0.2, 3)": {"precision": 0.35971992653810875, "recall": 0.5411570499748186, "f1_score": 0.3941138844761354, "per_field": {"categories": {"precision": 0.19249311294765822, "recall": 0.37504106889437466, "f1_score": 0.2399839003971236}, "scientific_domains": {"precision": 0.38466666666666716, "recall": 0.6352952380952382, "f1_score": 0.45176114473761486}, "target_users": {"precision": 0.5020000000000008, "recall": 0.6131348429348429, "f1_score": 0.4905966082936678}}}, "(0.2, 15, 0.2, 5)": {"precision": 0.3401312213039485, "recall": 0.6095720968349068, "f1_score": 0.39573447602106776, "per_field": {"categories": {"precision": 0.19042699724517925, "recall": 0.43282433246276214, "f1_score": 0.2440024010271944}, "scientific_domains": {"precision": 0.38053333333333333, "recall": 0.6883920634920636, "f1_score": 0.45471685477551976}, "target_users": {"precision": 0.44943333333333296, "recall": 0.7074998945498945, "f1_score": 0.4884841722604892}}}, "(0.2, 15, 0.3, 1)": {"precision": 0.4241157024793389, "recall": 0.3297917459186053, "f1_score": 0.3480276635857665, "per_field": {"categories": {"precision": 0.19834710743801653, "recall": 0.18240775523833375, "f1_score": 0.1854144208887292}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.584, "recall": 0.34181668886668853, "f1_score": 0.38712712842712876}}}, "(0.2, 15, 0.3, 3)": {"precision": 0.35971992653810875, "recall": 0.5411570499748186, "f1_score": 0.3941138844761354, "per_field": {"categories": {"precision": 0.19249311294765822, "recall": 0.37504106889437466, "f1_score": 0.2399839003971236}, "scientific_domains": {"precision": 0.38466666666666716, "recall": 0.6352952380952382, "f1_score": 0.45176114473761486}, "target_users": {"precision": 0.5020000000000008, "recall": 0.6131348429348429, "f1_score": 0.4905966082936678}}}, "(0.2, 15, 0.3, 5)": {"precision": 0.3401312213039485, "recall": 0.6095720968349068, "f1_score": 0.39573447602106776, "per_field": {"categories": {"precision": 0.19042699724517925, "recall": 0.43282433246276214, "f1_score": 0.2440024010271944}, "scientific_domains": {"precision": 0.38053333333333333, "recall": 0.6883920634920636, "f1_score": 0.45471685477551976}, "target_users": {"precision": 0.44943333333333296, "recall": 0.7074998945498945, "f1_score": 0.4884841722604892}}}, "(0.2, 15, 0.4, 1)": {"precision": 0.4241157024793389, "recall": 0.3297917459186053, "f1_score": 0.3480276635857665, "per_field": {"categories": {"precision": 0.19834710743801653, "recall": 0.18240775523833375, "f1_score": 0.1854144208887292}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.584, "recall": 0.34181668886668853, "f1_score": 0.38712712842712876}}}, "(0.2, 15, 0.4, 3)": {"precision": 0.35971992653810875, "recall": 0.5411570499748186, "f1_score": 0.3941138844761354, "per_field": {"categories": {"precision": 0.19249311294765822, "recall": 0.37504106889437466, "f1_score": 0.2399839003971236}, "scientific_domains": {"precision": 0.38466666666666716, "recall": 0.6352952380952382, "f1_score": 0.45176114473761486}, "target_users": {"precision": 0.5020000000000008, "recall": 0.6131348429348429, "f1_score": 0.4905966082936678}}}, "(0.2, 15, 0.4, 5)": {"precision": 0.3401312213039485, "recall": 0.6095720968349068, "f1_score": 0.39573447602106776, "per_field": {"categories": {"precision": 0.19042699724517925, "recall": 0.43282433246276214, "f1_score": 0.2440024010271944}, "scientific_domains": {"precision": 0.38053333333333333, "recall": 0.6883920634920636, "f1_score": 0.45471685477551976}, "target_users": {"precision": 0.44943333333333296, "recall": 0.7074998945498945, "f1_score": 0.4884841722604892}}}, "(0.2, 15, 0.5, 1)": {"precision": 0.4241157024793389, "recall": 0.3297917459186053, "f1_score": 0.3480276635857665, "per_field": {"categories": {"precision": 0.19834710743801653, "recall": 0.18240775523833375, "f1_score": 0.1854144208887292}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.584, "recall": 0.34181668886668853, "f1_score": 0.38712712842712876}}}, "(0.2, 15, 0.5, 3)": {"precision": 0.35971992653810875, "recall": 0.5411570499748186, "f1_score": 0.3941138844761354, "per_field": {"categories": {"precision": 0.19249311294765822, "recall": 0.37504106889437466, "f1_score": 0.2399839003971236}, "scientific_domains": {"precision": 0.38466666666666716, "recall": 0.6352952380952382, "f1_score": 0.45176114473761486}, "target_users": {"precision": 0.5020000000000008, "recall": 0.6131348429348429, "f1_score": 0.4905966082936678}}}, "(0.2, 15, 0.5, 5)": {"precision": 0.3401312213039485, "recall": 0.6095720968349068, "f1_score": 0.39573447602106776, "per_field": {"categories": {"precision": 0.19042699724517925, "recall": 0.43282433246276214, "f1_score": 0.2440024010271944}, "scientific_domains": {"precision": 0.38053333333333333, "recall": 0.6883920634920636, "f1_score": 0.45471685477551976}, "target_users": {"precision": 0.44943333333333296, "recall": 0.7074998945498945, "f1_score": 0.4884841722604892}}}, "(0.2, 15, 0.6, 1)": {"precision": 0.4241157024793389, "recall": 0.3297917459186053, "f1_score": 0.3480276635857665, "per_field": {"categories": {"precision": 0.19834710743801653, "recall": 0.18240775523833375, "f1_score": 0.1854144208887292}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.584, "recall": 0.34181668886668853, "f1_score": 0.38712712842712876}}}, "(0.2, 15, 0.6, 3)": {"precision": 0.35971992653810875, "recall": 0.5411570499748186, "f1_score": 0.3941138844761354, "per_field": {"categories": {"precision": 0.19249311294765822, "recall": 0.37504106889437466, "f1_score": 0.2399839003971236}, "scientific_domains": {"precision": 0.38466666666666716, "recall": 0.6352952380952382, "f1_score": 0.45176114473761486}, "target_users": {"precision": 0.5020000000000008, "recall": 0.6131348429348429, "f1_score": 0.4905966082936678}}}, "(0.2, 15, 0.6, 5)": {"precision": 0.3401312213039485, "recall": 0.6095720968349068, "f1_score": 0.39573447602106776, "per_field": {"categories": {"precision": 0.19042699724517925, "recall": 0.43282433246276214, "f1_score": 0.2440024010271944}, "scientific_domains": {"precision": 0.38053333333333333, "recall": 0.6883920634920636, "f1_score": 0.45471685477551976}, "target_users": {"precision": 0.44943333333333296, "recall": 0.7074998945498945, "f1_score": 0.4884841722604892}}}, "(0.2, 15, 0.7, 1)": {"precision": 0.4241157024793389, "recall": 0.3297917459186053, "f1_score": 0.3480276635857665, "per_field": {"categories": {"precision": 0.19834710743801653, "recall": 0.18240775523833375, "f1_score": 0.1854144208887292}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.584, "recall": 0.34181668886668853, "f1_score": 0.38712712842712876}}}, "(0.2, 15, 0.7, 3)": {"precision": 0.35971992653810875, "recall": 0.5411570499748186, "f1_score": 0.3941138844761354, "per_field": {"categories": {"precision": 0.19249311294765822, "recall": 0.37504106889437466, "f1_score": 0.2399839003971236}, "scientific_domains": {"precision": 0.38466666666666716, "recall": 0.6352952380952382, "f1_score": 0.45176114473761486}, "target_users": {"precision": 0.5020000000000008, "recall": 0.6131348429348429, "f1_score": 0.4905966082936678}}}, "(0.2, 15, 0.7, 5)": {"precision": 0.3401312213039485, "recall": 0.6095720968349068, "f1_score": 0.39573447602106776, "per_field": {"categories": {"precision": 0.19042699724517925, "recall": 0.43282433246276214, "f1_score": 0.2440024010271944}, "scientific_domains": {"precision": 0.38053333333333333, "recall": 0.6883920634920636, "f1_score": 0.45471685477551976}, "target_users": {"precision": 0.44943333333333296, "recall": 0.7074998945498945, "f1_score": 0.4884841722604892}}}, "(0.2, 15, 0.8, 1)": {"precision": 0.4241157024793389, "recall": 0.3297917459186053, "f1_score": 0.3480276635857665, "per_field": {"categories": {"precision": 0.19834710743801653, "recall": 0.18240775523833375, "f1_score": 0.1854144208887292}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.584, "recall": 0.34181668886668853, "f1_score": 0.38712712842712876}}}, "(0.2, 15, 0.8, 3)": {"precision": 0.35971992653810875, "recall": 0.5411570499748186, "f1_score": 0.3941138844761354, "per_field": {"categories": {"precision": 0.19249311294765822, "recall": 0.37504106889437466, "f1_score": 0.2399839003971236}, "scientific_domains": {"precision": 0.38466666666666716, "recall": 0.6352952380952382, "f1_score": 0.45176114473761486}, "target_users": {"precision": 0.5020000000000008, "recall": 0.6131348429348429, "f1_score": 0.4905966082936678}}}, "(0.2, 15, 0.8, 5)": {"precision": 0.3401312213039485, "recall": 0.6095720968349068, "f1_score": 0.39573447602106776, "per_field": {"categories": {"precision": 0.19042699724517925, "recall": 0.43282433246276214, "f1_score": 0.2440024010271944}, "scientific_domains": {"precision": 0.38053333333333333, "recall": 0.6883920634920636, "f1_score": 0.45471685477551976}, "target_users": {"precision": 0.44943333333333296, "recall": 0.7074998945498945, "f1_score": 0.4884841722604892}}}, "(0.2, 15, 0.9, 1)": {"precision": 0.4241157024793389, "recall": 0.3297917459186053, "f1_score": 0.3480276635857665, "per_field": {"categories": {"precision": 0.19834710743801653, "recall": 0.18240775523833375, "f1_score": 0.1854144208887292}, "scientific_domains": {"precision": 0.49, "recall": 0.46515079365079365, "f1_score": 0.47154144144144144}, "target_users": {"precision": 0.584, "recall": 0.34181668886668853, "f1_score": 0.38712712842712876}}}, "(0.2, 15, 0.9, 3)": {"precision": 0.35971992653810875, "recall": 0.5411570499748186, "f1_score": 0.3941138844761354, "per_field": {"categories": {"precision": 0.19249311294765822, "recall": 0.37504106889437466, "f1_score": 0.2399839003971236}, "scientific_domains": {"precision": 0.38466666666666716, "recall": 0.6352952380952382, "f1_score": 0.45176114473761486}, "target_users": {"precision": 0.5020000000000008, "recall": 0.6131348429348429, "f1_score": 0.4905966082936678}}}, "(0.2, 15, 0.9, 5)": {"precision": 0.3401312213039485, "recall": 0.6095720968349068, "f1_score": 0.39573447602106776, "per_field": {"categories": {"precision": 0.19042699724517925, "recall": 0.43282433246276214, "f1_score": 0.2440024010271944}, "scientific_domains": {"precision": 0.38053333333333333, "recall": 0.6883920634920636, "f1_score": 0.45471685477551976}, "target_users": {"precision": 0.44943333333333296, "recall": 0.7074998945498945, "f1_score": 0.4884841722604892}}}, "(0.3, 5, 0, 1)": {"precision": 0.4172451790633609, "recall": 0.3310435641340599, "f1_score": 0.34781140139677696, "per_field": {"categories": {"precision": 0.24173553719008264, "recall": 0.2216639535854412, "f1_score": 0.2260480297041562}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.538, "recall": 0.3369849927849924, "f1_score": 0.3746366522366525}}}, "(0.3, 5, 0, 3)": {"precision": 0.3887979797979799, "recall": 0.46459189534065554, "f1_score": 0.3931318936361718, "per_field": {"categories": {"precision": 0.23106060606060594, "recall": 0.34243102512730617, "f1_score": 0.2562652499016137}, "scientific_domains": {"precision": 0.43633333333333346, "recall": 0.5502357142857144, "f1_score": 0.4621443948208652}, "target_users": {"precision": 0.4990000000000002, "recall": 0.501108946608946, "f1_score": 0.4609860361860365}}}, "(0.3, 5, 0, 5)": {"precision": 0.38527621671258033, "recall": 0.5040024745685902, "f1_score": 0.4056368856086446, "per_field": {"categories": {"precision": 0.23422865013774113, "recall": 0.3818562693046164, "f1_score": 0.2662377414443532}, "scientific_domains": {"precision": 0.4365666666666667, "recall": 0.5613968253968254, "f1_score": 0.4658209839796489}, "target_users": {"precision": 0.4850333333333331, "recall": 0.5687543290043287, "f1_score": 0.4848519314019318}}}, "(0.3, 5, 0.1, 1)": {"precision": 0.4172451790633609, "recall": 0.3310435641340599, "f1_score": 0.34781140139677696, "per_field": {"categories": {"precision": 0.24173553719008264, "recall": 0.2216639535854412, "f1_score": 0.2260480297041562}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.538, "recall": 0.3369849927849924, "f1_score": 0.3746366522366525}}}, "(0.3, 5, 0.1, 3)": {"precision": 0.3887979797979799, "recall": 0.46459189534065554, "f1_score": 0.3931318936361718, "per_field": {"categories": {"precision": 0.23106060606060594, "recall": 0.34243102512730617, "f1_score": 0.2562652499016137}, "scientific_domains": {"precision": 0.43633333333333346, "recall": 0.5502357142857144, "f1_score": 0.4621443948208652}, "target_users": {"precision": 0.4990000000000002, "recall": 0.501108946608946, "f1_score": 0.4609860361860365}}}, "(0.3, 5, 0.1, 5)": {"precision": 0.38527621671258033, "recall": 0.5040024745685902, "f1_score": 0.4056368856086446, "per_field": {"categories": {"precision": 0.23422865013774113, "recall": 0.3818562693046164, "f1_score": 0.2662377414443532}, "scientific_domains": {"precision": 0.4365666666666667, "recall": 0.5613968253968254, "f1_score": 0.4658209839796489}, "target_users": {"precision": 0.4850333333333331, "recall": 0.5687543290043287, "f1_score": 0.4848519314019318}}}, "(0.3, 5, 0.2, 1)": {"precision": 0.4172451790633609, "recall": 0.3310435641340599, "f1_score": 0.34781140139677696, "per_field": {"categories": {"precision": 0.24173553719008264, "recall": 0.2216639535854412, "f1_score": 0.2260480297041562}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.538, "recall": 0.3369849927849924, "f1_score": 0.3746366522366525}}}, "(0.3, 5, 0.2, 3)": {"precision": 0.3887979797979799, "recall": 0.46459189534065554, "f1_score": 0.3931318936361718, "per_field": {"categories": {"precision": 0.23106060606060594, "recall": 0.34243102512730617, "f1_score": 0.2562652499016137}, "scientific_domains": {"precision": 0.43633333333333346, "recall": 0.5502357142857144, "f1_score": 0.4621443948208652}, "target_users": {"precision": 0.4990000000000002, "recall": 0.501108946608946, "f1_score": 0.4609860361860365}}}, "(0.3, 5, 0.2, 5)": {"precision": 0.38527621671258033, "recall": 0.5040024745685902, "f1_score": 0.4056368856086446, "per_field": {"categories": {"precision": 0.23422865013774113, "recall": 0.3818562693046164, "f1_score": 0.2662377414443532}, "scientific_domains": {"precision": 0.4365666666666667, "recall": 0.5613968253968254, "f1_score": 0.4658209839796489}, "target_users": {"precision": 0.4850333333333331, "recall": 0.5687543290043287, "f1_score": 0.4848519314019318}}}, "(0.3, 5, 0.3, 1)": {"precision": 0.4172451790633609, "recall": 0.3310435641340599, "f1_score": 0.34781140139677696, "per_field": {"categories": {"precision": 0.24173553719008264, "recall": 0.2216639535854412, "f1_score": 0.2260480297041562}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.538, "recall": 0.3369849927849924, "f1_score": 0.3746366522366525}}}, "(0.3, 5, 0.3, 3)": {"precision": 0.3887979797979799, "recall": 0.46459189534065554, "f1_score": 0.3931318936361718, "per_field": {"categories": {"precision": 0.23106060606060594, "recall": 0.34243102512730617, "f1_score": 0.2562652499016137}, "scientific_domains": {"precision": 0.43633333333333346, "recall": 0.5502357142857144, "f1_score": 0.4621443948208652}, "target_users": {"precision": 0.4990000000000002, "recall": 0.501108946608946, "f1_score": 0.4609860361860365}}}, "(0.3, 5, 0.3, 5)": {"precision": 0.38527621671258033, "recall": 0.5040024745685902, "f1_score": 0.4056368856086446, "per_field": {"categories": {"precision": 0.23422865013774113, "recall": 0.3818562693046164, "f1_score": 0.2662377414443532}, "scientific_domains": {"precision": 0.4365666666666667, "recall": 0.5613968253968254, "f1_score": 0.4658209839796489}, "target_users": {"precision": 0.4850333333333331, "recall": 0.5687543290043287, "f1_score": 0.4848519314019318}}}, "(0.3, 5, 0.4, 1)": {"precision": 0.4172451790633609, "recall": 0.3310435641340599, "f1_score": 0.34781140139677696, "per_field": {"categories": {"precision": 0.24173553719008264, "recall": 0.2216639535854412, "f1_score": 0.2260480297041562}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.538, "recall": 0.3369849927849924, "f1_score": 0.3746366522366525}}}, "(0.3, 5, 0.4, 3)": {"precision": 0.3887979797979799, "recall": 0.46459189534065554, "f1_score": 0.3931318936361718, "per_field": {"categories": {"precision": 0.23106060606060594, "recall": 0.34243102512730617, "f1_score": 0.2562652499016137}, "scientific_domains": {"precision": 0.43633333333333346, "recall": 0.5502357142857144, "f1_score": 0.4621443948208652}, "target_users": {"precision": 0.4990000000000002, "recall": 0.501108946608946, "f1_score": 0.4609860361860365}}}, "(0.3, 5, 0.4, 5)": {"precision": 0.38527621671258033, "recall": 0.5040024745685902, "f1_score": 0.4056368856086446, "per_field": {"categories": {"precision": 0.23422865013774113, "recall": 0.3818562693046164, "f1_score": 0.2662377414443532}, "scientific_domains": {"precision": 0.4365666666666667, "recall": 0.5613968253968254, "f1_score": 0.4658209839796489}, "target_users": {"precision": 0.4850333333333331, "recall": 0.5687543290043287, "f1_score": 0.4848519314019318}}}, "(0.3, 5, 0.5, 1)": {"precision": 0.4172451790633609, "recall": 0.3310435641340599, "f1_score": 0.34781140139677696, "per_field": {"categories": {"precision": 0.24173553719008264, "recall": 0.2216639535854412, "f1_score": 0.2260480297041562}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.538, "recall": 0.3369849927849924, "f1_score": 0.3746366522366525}}}, "(0.3, 5, 0.5, 3)": {"precision": 0.3887979797979799, "recall": 0.46459189534065554, "f1_score": 0.3931318936361718, "per_field": {"categories": {"precision": 0.23106060606060594, "recall": 0.34243102512730617, "f1_score": 0.2562652499016137}, "scientific_domains": {"precision": 0.43633333333333346, "recall": 0.5502357142857144, "f1_score": 0.4621443948208652}, "target_users": {"precision": 0.4990000000000002, "recall": 0.501108946608946, "f1_score": 0.4609860361860365}}}, "(0.3, 5, 0.5, 5)": {"precision": 0.38527621671258033, "recall": 0.5040024745685902, "f1_score": 0.4056368856086446, "per_field": {"categories": {"precision": 0.23422865013774113, "recall": 0.3818562693046164, "f1_score": 0.2662377414443532}, "scientific_domains": {"precision": 0.4365666666666667, "recall": 0.5613968253968254, "f1_score": 0.4658209839796489}, "target_users": {"precision": 0.4850333333333331, "recall": 0.5687543290043287, "f1_score": 0.4848519314019318}}}, "(0.3, 5, 0.6, 1)": {"precision": 0.4172451790633609, "recall": 0.3310435641340599, "f1_score": 0.34781140139677696, "per_field": {"categories": {"precision": 0.24173553719008264, "recall": 0.2216639535854412, "f1_score": 0.2260480297041562}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.538, "recall": 0.3369849927849924, "f1_score": 0.3746366522366525}}}, "(0.3, 5, 0.6, 3)": {"precision": 0.3887979797979799, "recall": 0.46459189534065554, "f1_score": 0.3931318936361718, "per_field": {"categories": {"precision": 0.23106060606060594, "recall": 0.34243102512730617, "f1_score": 0.2562652499016137}, "scientific_domains": {"precision": 0.43633333333333346, "recall": 0.5502357142857144, "f1_score": 0.4621443948208652}, "target_users": {"precision": 0.4990000000000002, "recall": 0.501108946608946, "f1_score": 0.4609860361860365}}}, "(0.3, 5, 0.6, 5)": {"precision": 0.38527621671258033, "recall": 0.5040024745685902, "f1_score": 0.4056368856086446, "per_field": {"categories": {"precision": 0.23422865013774113, "recall": 0.3818562693046164, "f1_score": 0.2662377414443532}, "scientific_domains": {"precision": 0.4365666666666667, "recall": 0.5613968253968254, "f1_score": 0.4658209839796489}, "target_users": {"precision": 0.4850333333333331, "recall": 0.5687543290043287, "f1_score": 0.4848519314019318}}}, "(0.3, 5, 0.7, 1)": {"precision": 0.4172451790633609, "recall": 0.3310435641340599, "f1_score": 0.34781140139677696, "per_field": {"categories": {"precision": 0.24173553719008264, "recall": 0.2216639535854412, "f1_score": 0.2260480297041562}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.538, "recall": 0.3369849927849924, "f1_score": 0.3746366522366525}}}, "(0.3, 5, 0.7, 3)": {"precision": 0.3887979797979799, "recall": 0.46459189534065554, "f1_score": 0.3931318936361718, "per_field": {"categories": {"precision": 0.23106060606060594, "recall": 0.34243102512730617, "f1_score": 0.2562652499016137}, "scientific_domains": {"precision": 0.43633333333333346, "recall": 0.5502357142857144, "f1_score": 0.4621443948208652}, "target_users": {"precision": 0.4990000000000002, "recall": 0.501108946608946, "f1_score": 0.4609860361860365}}}, "(0.3, 5, 0.7, 5)": {"precision": 0.38527621671258033, "recall": 0.5040024745685902, "f1_score": 0.4056368856086446, "per_field": {"categories": {"precision": 0.23422865013774113, "recall": 0.3818562693046164, "f1_score": 0.2662377414443532}, "scientific_domains": {"precision": 0.4365666666666667, "recall": 0.5613968253968254, "f1_score": 0.4658209839796489}, "target_users": {"precision": 0.4850333333333331, "recall": 0.5687543290043287, "f1_score": 0.4848519314019318}}}, "(0.3, 5, 0.8, 1)": {"precision": 0.4172451790633609, "recall": 0.3310435641340599, "f1_score": 0.34781140139677696, "per_field": {"categories": {"precision": 0.24173553719008264, "recall": 0.2216639535854412, "f1_score": 0.2260480297041562}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.538, "recall": 0.3369849927849924, "f1_score": 0.3746366522366525}}}, "(0.3, 5, 0.8, 3)": {"precision": 0.3887979797979799, "recall": 0.46459189534065554, "f1_score": 0.3931318936361718, "per_field": {"categories": {"precision": 0.23106060606060594, "recall": 0.34243102512730617, "f1_score": 0.2562652499016137}, "scientific_domains": {"precision": 0.43633333333333346, "recall": 0.5502357142857144, "f1_score": 0.4621443948208652}, "target_users": {"precision": 0.4990000000000002, "recall": 0.501108946608946, "f1_score": 0.4609860361860365}}}, "(0.3, 5, 0.8, 5)": {"precision": 0.38527621671258033, "recall": 0.5040024745685902, "f1_score": 0.4056368856086446, "per_field": {"categories": {"precision": 0.23422865013774113, "recall": 0.3818562693046164, "f1_score": 0.2662377414443532}, "scientific_domains": {"precision": 0.4365666666666667, "recall": 0.5613968253968254, "f1_score": 0.4658209839796489}, "target_users": {"precision": 0.4850333333333331, "recall": 0.5687543290043287, "f1_score": 0.4848519314019318}}}, "(0.3, 5, 0.9, 1)": {"precision": 0.4172451790633609, "recall": 0.3310435641340599, "f1_score": 0.34781140139677696, "per_field": {"categories": {"precision": 0.24173553719008264, "recall": 0.2216639535854412, "f1_score": 0.2260480297041562}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.538, "recall": 0.3369849927849924, "f1_score": 0.3746366522366525}}}, "(0.3, 5, 0.9, 3)": {"precision": 0.3887979797979799, "recall": 0.46459189534065554, "f1_score": 0.3931318936361718, "per_field": {"categories": {"precision": 0.23106060606060594, "recall": 0.34243102512730617, "f1_score": 0.2562652499016137}, "scientific_domains": {"precision": 0.43633333333333346, "recall": 0.5502357142857144, "f1_score": 0.4621443948208652}, "target_users": {"precision": 0.4990000000000002, "recall": 0.501108946608946, "f1_score": 0.4609860361860365}}}, "(0.3, 5, 0.9, 5)": {"precision": 0.38527621671258033, "recall": 0.5040024745685902, "f1_score": 0.4056368856086446, "per_field": {"categories": {"precision": 0.23422865013774113, "recall": 0.3818562693046164, "f1_score": 0.2662377414443532}, "scientific_domains": {"precision": 0.4365666666666667, "recall": 0.5613968253968254, "f1_score": 0.4658209839796489}, "target_users": {"precision": 0.4850333333333331, "recall": 0.5687543290043287, "f1_score": 0.4848519314019318}}}, "(0.3, 10, 0, 1)": {"precision": 0.41169146005509644, "recall": 0.3248787340146843, "f1_score": 0.34188942842025855, "per_field": {"categories": {"precision": 0.22107438016528927, "recall": 0.2010027965606478, "f1_score": 0.20538687267936281}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.542, "recall": 0.33915165945165904, "f1_score": 0.37753189033189055}}}, "(0.3, 10, 0, 3)": {"precision": 0.35517906336088156, "recall": 0.459420269996303, "f1_score": 0.36653928960720417, "per_field": {"categories": {"precision": 0.19387052341597785, "recall": 0.314882815760915, "f1_score": 0.2197638724911453}, "scientific_domains": {"precision": 0.40033333333333354, "recall": 0.5658357142857144, "f1_score": 0.43931106148753185}, "target_users": {"precision": 0.4713333333333334, "recall": 0.49754227994227945, "f1_score": 0.44054293484293533}}}, "(0.3, 10, 0, 5)": {"precision": 0.34936528925619825, "recall": 0.5086187148939215, "f1_score": 0.3771824430814749, "per_field": {"categories": {"precision": 0.19466253443526182, "recall": 0.37772403789965775, "f1_score": 0.22934887273730262}, "scientific_domains": {"precision": 0.3979666666666667, "recall": 0.5773968253968255, "f1_score": 0.43942098397964896}, "target_users": {"precision": 0.4554666666666663, "recall": 0.5707352813852813, "f1_score": 0.46277747252747314}}}, "(0.3, 10, 0.1, 1)": {"precision": 0.41169146005509644, "recall": 0.3248787340146843, "f1_score": 0.34188942842025855, "per_field": {"categories": {"precision": 0.22107438016528927, "recall": 0.2010027965606478, "f1_score": 0.20538687267936281}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.542, "recall": 0.33915165945165904, "f1_score": 0.37753189033189055}}}, "(0.3, 10, 0.1, 3)": {"precision": 0.35517906336088156, "recall": 0.459420269996303, "f1_score": 0.36653928960720417, "per_field": {"categories": {"precision": 0.19387052341597785, "recall": 0.314882815760915, "f1_score": 0.2197638724911453}, "scientific_domains": {"precision": 0.40033333333333354, "recall": 0.5658357142857144, "f1_score": 0.43931106148753185}, "target_users": {"precision": 0.4713333333333334, "recall": 0.49754227994227945, "f1_score": 0.44054293484293533}}}, "(0.3, 10, 0.1, 5)": {"precision": 0.34936528925619825, "recall": 0.5086187148939215, "f1_score": 0.3771824430814749, "per_field": {"categories": {"precision": 0.19466253443526182, "recall": 0.37772403789965775, "f1_score": 0.22934887273730262}, "scientific_domains": {"precision": 0.3979666666666667, "recall": 0.5773968253968255, "f1_score": 0.43942098397964896}, "target_users": {"precision": 0.4554666666666663, "recall": 0.5707352813852813, "f1_score": 0.46277747252747314}}}, "(0.3, 10, 0.2, 1)": {"precision": 0.41169146005509644, "recall": 0.3248787340146843, "f1_score": 0.34188942842025855, "per_field": {"categories": {"precision": 0.22107438016528927, "recall": 0.2010027965606478, "f1_score": 0.20538687267936281}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.542, "recall": 0.33915165945165904, "f1_score": 0.37753189033189055}}}, "(0.3, 10, 0.2, 3)": {"precision": 0.35517906336088156, "recall": 0.459420269996303, "f1_score": 0.36653928960720417, "per_field": {"categories": {"precision": 0.19387052341597785, "recall": 0.314882815760915, "f1_score": 0.2197638724911453}, "scientific_domains": {"precision": 0.40033333333333354, "recall": 0.5658357142857144, "f1_score": 0.43931106148753185}, "target_users": {"precision": 0.4713333333333334, "recall": 0.49754227994227945, "f1_score": 0.44054293484293533}}}, "(0.3, 10, 0.2, 5)": {"precision": 0.34936528925619825, "recall": 0.5086187148939215, "f1_score": 0.3771824430814749, "per_field": {"categories": {"precision": 0.19466253443526182, "recall": 0.37772403789965775, "f1_score": 0.22934887273730262}, "scientific_domains": {"precision": 0.3979666666666667, "recall": 0.5773968253968255, "f1_score": 0.43942098397964896}, "target_users": {"precision": 0.4554666666666663, "recall": 0.5707352813852813, "f1_score": 0.46277747252747314}}}, "(0.3, 10, 0.3, 1)": {"precision": 0.41169146005509644, "recall": 0.3248787340146843, "f1_score": 0.34188942842025855, "per_field": {"categories": {"precision": 0.22107438016528927, "recall": 0.2010027965606478, "f1_score": 0.20538687267936281}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.542, "recall": 0.33915165945165904, "f1_score": 0.37753189033189055}}}, "(0.3, 10, 0.3, 3)": {"precision": 0.35517906336088156, "recall": 0.459420269996303, "f1_score": 0.36653928960720417, "per_field": {"categories": {"precision": 0.19387052341597785, "recall": 0.314882815760915, "f1_score": 0.2197638724911453}, "scientific_domains": {"precision": 0.40033333333333354, "recall": 0.5658357142857144, "f1_score": 0.43931106148753185}, "target_users": {"precision": 0.4713333333333334, "recall": 0.49754227994227945, "f1_score": 0.44054293484293533}}}, "(0.3, 10, 0.3, 5)": {"precision": 0.34936528925619825, "recall": 0.5086187148939215, "f1_score": 0.3771824430814749, "per_field": {"categories": {"precision": 0.19466253443526182, "recall": 0.37772403789965775, "f1_score": 0.22934887273730262}, "scientific_domains": {"precision": 0.3979666666666667, "recall": 0.5773968253968255, "f1_score": 0.43942098397964896}, "target_users": {"precision": 0.4554666666666663, "recall": 0.5707352813852813, "f1_score": 0.46277747252747314}}}, "(0.3, 10, 0.4, 1)": {"precision": 0.41169146005509644, "recall": 0.3248787340146843, "f1_score": 0.34188942842025855, "per_field": {"categories": {"precision": 0.22107438016528927, "recall": 0.2010027965606478, "f1_score": 0.20538687267936281}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.542, "recall": 0.33915165945165904, "f1_score": 0.37753189033189055}}}, "(0.3, 10, 0.4, 3)": {"precision": 0.35517906336088156, "recall": 0.459420269996303, "f1_score": 0.36653928960720417, "per_field": {"categories": {"precision": 0.19387052341597785, "recall": 0.314882815760915, "f1_score": 0.2197638724911453}, "scientific_domains": {"precision": 0.40033333333333354, "recall": 0.5658357142857144, "f1_score": 0.43931106148753185}, "target_users": {"precision": 0.4713333333333334, "recall": 0.49754227994227945, "f1_score": 0.44054293484293533}}}, "(0.3, 10, 0.4, 5)": {"precision": 0.34936528925619825, "recall": 0.5086187148939215, "f1_score": 0.3771824430814749, "per_field": {"categories": {"precision": 0.19466253443526182, "recall": 0.37772403789965775, "f1_score": 0.22934887273730262}, "scientific_domains": {"precision": 0.3979666666666667, "recall": 0.5773968253968255, "f1_score": 0.43942098397964896}, "target_users": {"precision": 0.4554666666666663, "recall": 0.5707352813852813, "f1_score": 0.46277747252747314}}}, "(0.3, 10, 0.5, 1)": {"precision": 0.41169146005509644, "recall": 0.3248787340146843, "f1_score": 0.34188942842025855, "per_field": {"categories": {"precision": 0.22107438016528927, "recall": 0.2010027965606478, "f1_score": 0.20538687267936281}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.542, "recall": 0.33915165945165904, "f1_score": 0.37753189033189055}}}, "(0.3, 10, 0.5, 3)": {"precision": 0.35517906336088156, "recall": 0.459420269996303, "f1_score": 0.36653928960720417, "per_field": {"categories": {"precision": 0.19387052341597785, "recall": 0.314882815760915, "f1_score": 0.2197638724911453}, "scientific_domains": {"precision": 0.40033333333333354, "recall": 0.5658357142857144, "f1_score": 0.43931106148753185}, "target_users": {"precision": 0.4713333333333334, "recall": 0.49754227994227945, "f1_score": 0.44054293484293533}}}, "(0.3, 10, 0.5, 5)": {"precision": 0.34936528925619825, "recall": 0.5086187148939215, "f1_score": 0.3771824430814749, "per_field": {"categories": {"precision": 0.19466253443526182, "recall": 0.37772403789965775, "f1_score": 0.22934887273730262}, "scientific_domains": {"precision": 0.3979666666666667, "recall": 0.5773968253968255, "f1_score": 0.43942098397964896}, "target_users": {"precision": 0.4554666666666663, "recall": 0.5707352813852813, "f1_score": 0.46277747252747314}}}, "(0.3, 10, 0.6, 1)": {"precision": 0.41169146005509644, "recall": 0.3248787340146843, "f1_score": 0.34188942842025855, "per_field": {"categories": {"precision": 0.22107438016528927, "recall": 0.2010027965606478, "f1_score": 0.20538687267936281}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.542, "recall": 0.33915165945165904, "f1_score": 0.37753189033189055}}}, "(0.3, 10, 0.6, 3)": {"precision": 0.35517906336088156, "recall": 0.459420269996303, "f1_score": 0.36653928960720417, "per_field": {"categories": {"precision": 0.19387052341597785, "recall": 0.314882815760915, "f1_score": 0.2197638724911453}, "scientific_domains": {"precision": 0.40033333333333354, "recall": 0.5658357142857144, "f1_score": 0.43931106148753185}, "target_users": {"precision": 0.4713333333333334, "recall": 0.49754227994227945, "f1_score": 0.44054293484293533}}}, "(0.3, 10, 0.6, 5)": {"precision": 0.34936528925619825, "recall": 0.5086187148939215, "f1_score": 0.3771824430814749, "per_field": {"categories": {"precision": 0.19466253443526182, "recall": 0.37772403789965775, "f1_score": 0.22934887273730262}, "scientific_domains": {"precision": 0.3979666666666667, "recall": 0.5773968253968255, "f1_score": 0.43942098397964896}, "target_users": {"precision": 0.4554666666666663, "recall": 0.5707352813852813, "f1_score": 0.46277747252747314}}}, "(0.3, 10, 0.7, 1)": {"precision": 0.41169146005509644, "recall": 0.3248787340146843, "f1_score": 0.34188942842025855, "per_field": {"categories": {"precision": 0.22107438016528927, "recall": 0.2010027965606478, "f1_score": 0.20538687267936281}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.542, "recall": 0.33915165945165904, "f1_score": 0.37753189033189055}}}, "(0.3, 10, 0.7, 3)": {"precision": 0.35517906336088156, "recall": 0.459420269996303, "f1_score": 0.36653928960720417, "per_field": {"categories": {"precision": 0.19387052341597785, "recall": 0.314882815760915, "f1_score": 0.2197638724911453}, "scientific_domains": {"precision": 0.40033333333333354, "recall": 0.5658357142857144, "f1_score": 0.43931106148753185}, "target_users": {"precision": 0.4713333333333334, "recall": 0.49754227994227945, "f1_score": 0.44054293484293533}}}, "(0.3, 10, 0.7, 5)": {"precision": 0.34936528925619825, "recall": 0.5086187148939215, "f1_score": 0.3771824430814749, "per_field": {"categories": {"precision": 0.19466253443526182, "recall": 0.37772403789965775, "f1_score": 0.22934887273730262}, "scientific_domains": {"precision": 0.3979666666666667, "recall": 0.5773968253968255, "f1_score": 0.43942098397964896}, "target_users": {"precision": 0.4554666666666663, "recall": 0.5707352813852813, "f1_score": 0.46277747252747314}}}, "(0.3, 10, 0.8, 1)": {"precision": 0.41169146005509644, "recall": 0.3248787340146843, "f1_score": 0.34188942842025855, "per_field": {"categories": {"precision": 0.22107438016528927, "recall": 0.2010027965606478, "f1_score": 0.20538687267936281}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.542, "recall": 0.33915165945165904, "f1_score": 0.37753189033189055}}}, "(0.3, 10, 0.8, 3)": {"precision": 0.35517906336088156, "recall": 0.459420269996303, "f1_score": 0.36653928960720417, "per_field": {"categories": {"precision": 0.19387052341597785, "recall": 0.314882815760915, "f1_score": 0.2197638724911453}, "scientific_domains": {"precision": 0.40033333333333354, "recall": 0.5658357142857144, "f1_score": 0.43931106148753185}, "target_users": {"precision": 0.4713333333333334, "recall": 0.49754227994227945, "f1_score": 0.44054293484293533}}}, "(0.3, 10, 0.8, 5)": {"precision": 0.34936528925619825, "recall": 0.5086187148939215, "f1_score": 0.3771824430814749, "per_field": {"categories": {"precision": 0.19466253443526182, "recall": 0.37772403789965775, "f1_score": 0.22934887273730262}, "scientific_domains": {"precision": 0.3979666666666667, "recall": 0.5773968253968255, "f1_score": 0.43942098397964896}, "target_users": {"precision": 0.4554666666666663, "recall": 0.5707352813852813, "f1_score": 0.46277747252747314}}}, "(0.3, 10, 0.9, 1)": {"precision": 0.41169146005509644, "recall": 0.3248787340146843, "f1_score": 0.34188942842025855, "per_field": {"categories": {"precision": 0.22107438016528927, "recall": 0.2010027965606478, "f1_score": 0.20538687267936281}, "scientific_domains": {"precision": 0.472, "recall": 0.43448174603174605, "f1_score": 0.44274952224952224}, "target_users": {"precision": 0.542, "recall": 0.33915165945165904, "f1_score": 0.37753189033189055}}}, "(0.3, 10, 0.9, 3)": {"precision": 0.35517906336088156, "recall": 0.459420269996303, "f1_score": 0.36653928960720417, "per_field": {"categories": {"precision": 0.19387052341597785, "recall": 0.314882815760915, "f1_score": 0.2197638724911453}, "scientific_domains": {"precision": 0.40033333333333354, "recall": 0.5658357142857144, "f1_score": 0.43931106148753185}, "target_users": {"precision": 0.4713333333333334, "recall": 0.49754227994227945, "f1_score": 0.44054293484293533}}}, "(0.3, 10, 0.9, 5)": {"precision": 0.34936528925619825, "recall": 0.5086187148939215, "f1_score": 0.3771824430814749, "per_field": {"categories": {"precision": 0.19466253443526182, "recall": 0.37772403789965775, "f1_score": 0.22934887273730262}, "scientific_domains": {"precision": 0.3979666666666667, "recall": 0.5773968253968255, "f1_score": 0.43942098397964896}, "target_users": {"precision": 0.4554666666666663, "recall": 0.5707352813852813, "f1_score": 0.46277747252747314}}}, "(0.3, 15, 0, 1)": {"precision": 0.41162534435261705, "recall": 0.3242570627566494, "f1_score": 0.3413471222415887, "per_field": {"categories": {"precision": 0.21487603305785125, "recall": 0.1948044494532098, "f1_score": 0.1991885255719248}, "scientific_domains": {"precision": 0.476, "recall": 0.43848174603174606, "f1_score": 0.4467495222495222}, "target_users": {"precision": 0.544, "recall": 0.3394849927849924, "f1_score": 0.3781033189033191}}}, "(0.3, 15, 0, 3)": {"precision": 0.3651450872359964, "recall": 0.45852045365103206, "f1_score": 0.3744890031932813, "per_field": {"categories": {"precision": 0.20110192837465554, "recall": 0.31281670005843565, "f1_score": 0.2259622195985833}, "scientific_domains": {"precision": 0.41133333333333355, "recall": 0.5638357142857144, "f1_score": 0.4489777281541985}, "target_users": {"precision": 0.483, "recall": 0.49890894660894614, "f1_score": 0.4485270618270623}}}, "(0.3, 15, 0, 5)": {"precision": 0.36073691460055096, "recall": 0.5055746377589353, "f1_score": 0.38661528229613223, "per_field": {"categories": {"precision": 0.20554407713498632, "recall": 0.3735918064946991, "f1_score": 0.239669612603497}, "scientific_domains": {"precision": 0.4100666666666668, "recall": 0.5733968253968255, "f1_score": 0.4502876506463156}, "target_users": {"precision": 0.4665999999999997, "recall": 0.5697352813852813, "f1_score": 0.4698885836385842}}}, "(0.3, 15, 0.1, 1)": {"precision": 0.41162534435261705, "recall": 0.3242570627566494, "f1_score": 0.3413471222415887, "per_field": {"categories": {"precision": 0.21487603305785125, "recall": 0.1948044494532098, "f1_score": 0.1991885255719248}, "scientific_domains": {"precision": 0.476, "recall": 0.43848174603174606, "f1_score": 0.4467495222495222}, "target_users": {"precision": 0.544, "recall": 0.3394849927849924, "f1_score": 0.3781033189033191}}}, "(0.3, 15, 0.1, 3)": {"precision": 0.3651450872359964, "recall": 0.45852045365103206, "f1_score": 0.3744890031932813, "per_field": {"categories": {"precision": 0.20110192837465554, "recall": 0.31281670005843565, "f1_score": 0.2259622195985833}, "scientific_domains": {"precision": 0.41133333333333355, "recall": 0.5638357142857144, "f1_score": 0.4489777281541985}, "target_users": {"precision": 0.483, "recall": 0.49890894660894614, "f1_score": 0.4485270618270623}}}, "(0.3, 15, 0.1, 5)": {"precision": 0.36073691460055096, "recall": 0.5055746377589353, "f1_score": 0.38661528229613223, "per_field": {"categories": {"precision": 0.20554407713498632, "recall": 0.3735918064946991, "f1_score": 0.239669612603497}, "scientific_domains": {"precision": 0.4100666666666668, "recall": 0.5733968253968255, "f1_score": 0.4502876506463156}, "target_users": {"precision": 0.4665999999999997, "recall": 0.5697352813852813, "f1_score": 0.4698885836385842}}}, "(0.3, 15, 0.2, 1)": {"precision": 0.41162534435261705, "recall": 0.3242570627566494, "f1_score": 0.3413471222415887, "per_field": {"categories": {"precision": 0.21487603305785125, "recall": 0.1948044494532098, "f1_score": 0.1991885255719248}, "scientific_domains": {"precision": 0.476, "recall": 0.43848174603174606, "f1_score": 0.4467495222495222}, "target_users": {"precision": 0.544, "recall": 0.3394849927849924, "f1_score": 0.3781033189033191}}}, "(0.3, 15, 0.2, 3)": {"precision": 0.3651450872359964, "recall": 0.45852045365103206, "f1_score": 0.3744890031932813, "per_field": {"categories": {"precision": 0.20110192837465554, "recall": 0.31281670005843565, "f1_score": 0.2259622195985833}, "scientific_domains": {"precision": 0.41133333333333355, "recall": 0.5638357142857144, "f1_score": 0.4489777281541985}, "target_users": {"precision": 0.483, "recall": 0.49890894660894614, "f1_score": 0.4485270618270623}}}, "(0.3, 15, 0.2, 5)": {"precision": 0.36073691460055096, "recall": 0.5055746377589353, "f1_score": 0.38661528229613223, "per_field": {"categories": {"precision": 0.20554407713498632, "recall": 0.3735918064946991, "f1_score": 0.239669612603497}, "scientific_domains": {"precision": 0.4100666666666668, "recall": 0.5733968253968255, "f1_score": 0.4502876506463156}, "target_users": {"precision": 0.4665999999999997, "recall": 0.5697352813852813, "f1_score": 0.4698885836385842}}}, "(0.3, 15, 0.3, 1)": {"precision": 0.41162534435261705, "recall": 0.3242570627566494, "f1_score": 0.3413471222415887, "per_field": {"categories": {"precision": 0.21487603305785125, "recall": 0.1948044494532098, "f1_score": 0.1991885255719248}, "scientific_domains": {"precision": 0.476, "recall": 0.43848174603174606, "f1_score": 0.4467495222495222}, "target_users": {"precision": 0.544, "recall": 0.3394849927849924, "f1_score": 0.3781033189033191}}}, "(0.3, 15, 0.3, 3)": {"precision": 0.3651450872359964, "recall": 0.45852045365103206, "f1_score": 0.3744890031932813, "per_field": {"categories": {"precision": 0.20110192837465554, "recall": 0.31281670005843565, "f1_score": 0.2259622195985833}, "scientific_domains": {"precision": 0.41133333333333355, "recall": 0.5638357142857144, "f1_score": 0.4489777281541985}, "target_users": {"precision": 0.483, "recall": 0.49890894660894614, "f1_score": 0.4485270618270623}}}, "(0.3, 15, 0.3, 5)": {"precision": 0.36073691460055096, "recall": 0.5055746377589353, "f1_score": 0.38661528229613223, "per_field": {"categories": {"precision": 0.20554407713498632, "recall": 0.3735918064946991, "f1_score": 0.239669612603497}, "scientific_domains": {"precision": 0.4100666666666668, "recall": 0.5733968253968255, "f1_score": 0.4502876506463156}, "target_users": {"precision": 0.4665999999999997, "recall": 0.5697352813852813, "f1_score": 0.4698885836385842}}}, "(0.3, 15, 0.4, 1)": {"precision": 0.41162534435261705, "recall": 0.3242570627566494, "f1_score": 0.3413471222415887, "per_field": {"categories": {"precision": 0.21487603305785125, "recall": 0.1948044494532098, "f1_score": 0.1991885255719248}, "scientific_domains": {"precision": 0.476, "recall": 0.43848174603174606, "f1_score": 0.4467495222495222}, "target_users": {"precision": 0.544, "recall": 0.3394849927849924, "f1_score": 0.3781033189033191}}}, "(0.3, 15, 0.4, 3)": {"precision": 0.3651450872359964, "recall": 0.45852045365103206, "f1_score": 0.3744890031932813, "per_field": {"categories": {"precision": 0.20110192837465554, "recall": 0.31281670005843565, "f1_score": 0.2259622195985833}, "scientific_domains": {"precision": 0.41133333333333355, "recall": 0.5638357142857144, "f1_score": 0.4489777281541985}, "target_users": {"precision": 0.483, "recall": 0.49890894660894614, "f1_score": 0.4485270618270623}}}, "(0.3, 15, 0.4, 5)": {"precision": 0.36073691460055096, "recall": 0.5055746377589353, "f1_score": 0.38661528229613223, "per_field": {"categories": {"precision": 0.20554407713498632, "recall": 0.3735918064946991, "f1_score": 0.239669612603497}, "scientific_domains": {"precision": 0.4100666666666668, "recall": 0.5733968253968255, "f1_score": 0.4502876506463156}, "target_users": {"precision": 0.4665999999999997, "recall": 0.5697352813852813, "f1_score": 0.4698885836385842}}}, "(0.3, 15, 0.5, 1)": {"precision": 0.41162534435261705, "recall": 0.3242570627566494, "f1_score": 0.3413471222415887, "per_field": {"categories": {"precision": 0.21487603305785125, "recall": 0.1948044494532098, "f1_score": 0.1991885255719248}, "scientific_domains": {"precision": 0.476, "recall": 0.43848174603174606, "f1_score": 0.4467495222495222}, "target_users": {"precision": 0.544, "recall": 0.3394849927849924, "f1_score": 0.3781033189033191}}}, "(0.3, 15, 0.5, 3)": {"precision": 0.3651450872359964, "recall": 0.45852045365103206, "f1_score": 0.3744890031932813, "per_field": {"categories": {"precision": 0.20110192837465554, "recall": 0.31281670005843565, "f1_score": 0.2259622195985833}, "scientific_domains": {"precision": 0.41133333333333355, "recall": 0.5638357142857144, "f1_score": 0.4489777281541985}, "target_users": {"precision": 0.483, "recall": 0.49890894660894614, "f1_score": 0.4485270618270623}}}, "(0.3, 15, 0.5, 5)": {"precision": 0.36073691460055096, "recall": 0.5055746377589353, "f1_score": 0.38661528229613223, "per_field": {"categories": {"precision": 0.20554407713498632, "recall": 0.3735918064946991, "f1_score": 0.239669612603497}, "scientific_domains": {"precision": 0.4100666666666668, "recall": 0.5733968253968255, "f1_score": 0.4502876506463156}, "target_users": {"precision": 0.4665999999999997, "recall": 0.5697352813852813, "f1_score": 0.4698885836385842}}}, "(0.3, 15, 0.6, 1)": {"precision": 0.41162534435261705, "recall": 0.3242570627566494, "f1_score": 0.3413471222415887, "per_field": {"categories": {"precision": 0.21487603305785125, "recall": 0.1948044494532098, "f1_score": 0.1991885255719248}, "scientific_domains": {"precision": 0.476, "recall": 0.43848174603174606, "f1_score": 0.4467495222495222}, "target_users": {"precision": 0.544, "recall": 0.3394849927849924, "f1_score": 0.3781033189033191}}}, "(0.3, 15, 0.6, 3)": {"precision": 0.3651450872359964, "recall": 0.45852045365103206, "f1_score": 0.3744890031932813, "per_field": {"categories": {"precision": 0.20110192837465554, "recall": 0.31281670005843565, "f1_score": 0.2259622195985833}, "scientific_domains": {"precision": 0.41133333333333355, "recall": 0.5638357142857144, "f1_score": 0.4489777281541985}, "target_users": {"precision": 0.483, "recall": 0.49890894660894614, "f1_score": 0.4485270618270623}}}, "(0.3, 15, 0.6, 5)": {"precision": 0.36073691460055096, "recall": 0.5055746377589353, "f1_score": 0.38661528229613223, "per_field": {"categories": {"precision": 0.20554407713498632, "recall": 0.3735918064946991, "f1_score": 0.239669612603497}, "scientific_domains": {"precision": 0.4100666666666668, "recall": 0.5733968253968255, "f1_score": 0.4502876506463156}, "target_users": {"precision": 0.4665999999999997, "recall": 0.5697352813852813, "f1_score": 0.4698885836385842}}}, "(0.3, 15, 0.7, 1)": {"precision": 0.41162534435261705, "recall": 0.3242570627566494, "f1_score": 0.3413471222415887, "per_field": {"categories": {"precision": 0.21487603305785125, "recall": 0.1948044494532098, "f1_score": 0.1991885255719248}, "scientific_domains": {"precision": 0.476, "recall": 0.43848174603174606, "f1_score": 0.4467495222495222}, "target_users": {"precision": 0.544, "recall": 0.3394849927849924, "f1_score": 0.3781033189033191}}}, "(0.3, 15, 0.7, 3)": {"precision": 0.3651450872359964, "recall": 0.45852045365103206, "f1_score": 0.3744890031932813, "per_field": {"categories": {"precision": 0.20110192837465554, "recall": 0.31281670005843565, "f1_score": 0.2259622195985833}, "scientific_domains": {"precision": 0.41133333333333355, "recall": 0.5638357142857144, "f1_score": 0.4489777281541985}, "target_users": {"precision": 0.483, "recall": 0.49890894660894614, "f1_score": 0.4485270618270623}}}, "(0.3, 15, 0.7, 5)": {"precision": 0.36073691460055096, "recall": 0.5055746377589353, "f1_score": 0.38661528229613223, "per_field": {"categories": {"precision": 0.20554407713498632, "recall": 0.3735918064946991, "f1_score": 0.239669612603497}, "scientific_domains": {"precision": 0.4100666666666668, "recall": 0.5733968253968255, "f1_score": 0.4502876506463156}, "target_users": {"precision": 0.4665999999999997, "recall": 0.5697352813852813, "f1_score": 0.4698885836385842}}}, "(0.3, 15, 0.8, 1)": {"precision": 0.41162534435261705, "recall": 0.3242570627566494, "f1_score": 0.3413471222415887, "per_field": {"categories": {"precision": 0.21487603305785125, "recall": 0.1948044494532098, "f1_score": 0.1991885255719248}, "scientific_domains": {"precision": 0.476, "recall": 0.43848174603174606, "f1_score": 0.4467495222495222}, "target_users": {"precision": 0.544, "recall": 0.3394849927849924, "f1_score": 0.3781033189033191}}}, "(0.3, 15, 0.8, 3)": {"precision": 0.3651450872359964, "recall": 0.45852045365103206, "f1_score": 0.3744890031932813, "per_field": {"categories": {"precision": 0.20110192837465554, "recall": 0.31281670005843565, "f1_score": 0.2259622195985833}, "scientific_domains": {"precision": 0.41133333333333355, "recall": 0.5638357142857144, "f1_score": 0.4489777281541985}, "target_users": {"precision": 0.483, "recall": 0.49890894660894614, "f1_score": 0.4485270618270623}}}, "(0.3, 15, 0.8, 5)": {"precision": 0.36073691460055096, "recall": 0.5055746377589353, "f1_score": 0.38661528229613223, "per_field": {"categories": {"precision": 0.20554407713498632, "recall": 0.3735918064946991, "f1_score": 0.239669612603497}, "scientific_domains": {"precision": 0.4100666666666668, "recall": 0.5733968253968255, "f1_score": 0.4502876506463156}, "target_users": {"precision": 0.4665999999999997, "recall": 0.5697352813852813, "f1_score": 0.4698885836385842}}}, "(0.3, 15, 0.9, 1)": {"precision": 0.41162534435261705, "recall": 0.3242570627566494, "f1_score": 0.3413471222415887, "per_field": {"categories": {"precision": 0.21487603305785125, "recall": 0.1948044494532098, "f1_score": 0.1991885255719248}, "scientific_domains": {"precision": 0.476, "recall": 0.43848174603174606, "f1_score": 0.4467495222495222}, "target_users": {"precision": 0.544, "recall": 0.3394849927849924, "f1_score": 0.3781033189033191}}}, "(0.3, 15, 0.9, 3)": {"precision": 0.3651450872359964, "recall": 0.45852045365103206, "f1_score": 0.3744890031932813, "per_field": {"categories": {"precision": 0.20110192837465554, "recall": 0.31281670005843565, "f1_score": 0.2259622195985833}, "scientific_domains": {"precision": 0.41133333333333355, "recall": 0.5638357142857144, "f1_score": 0.4489777281541985}, "target_users": {"precision": 0.483, "recall": 0.49890894660894614, "f1_score": 0.4485270618270623}}}, "(0.3, 15, 0.9, 5)": {"precision": 0.36073691460055096, "recall": 0.5055746377589353, "f1_score": 0.38661528229613223, "per_field": {"categories": {"precision": 0.20554407713498632, "recall": 0.3735918064946991, "f1_score": 0.239669612603497}, "scientific_domains": {"precision": 0.4100666666666668, "recall": 0.5733968253968255, "f1_score": 0.4502876506463156}, "target_users": {"precision": 0.4665999999999997, "recall": 0.5697352813852813, "f1_score": 0.4698885836385842}}}, "(0.4, 5, 0, 1)": {"precision": 0.36644628099173554, "recall": 0.300514308372986, "f1_score": 0.31402339544513463, "per_field": {"categories": {"precision": 0.22933884297520662, "recall": 0.21497777360380665, "f1_score": 0.21810496267018006}, "scientific_domains": {"precision": 0.414, "recall": 0.3843571428571429, "f1_score": 0.39159999999999995}, "target_users": {"precision": 0.456, "recall": 0.3022080086580083, "f1_score": 0.33236522366522386}}}, "(0.4, 5, 0, 3)": {"precision": 0.34526170798898065, "recall": 0.38795329942478696, "f1_score": 0.3457364466193343, "per_field": {"categories": {"precision": 0.2224517906336088, "recall": 0.2996509516654145, "f1_score": 0.240018365472911}, "scientific_domains": {"precision": 0.3843333333333333, "recall": 0.44821428571428573, "f1_score": 0.3998768440709617}, "target_users": {"precision": 0.42899999999999994, "recall": 0.41599466089466064, "f1_score": 0.3973141303141303}}}, "(0.4, 5, 0, 5)": {"precision": 0.34345555555555557, "recall": 0.40535204166815736, "f1_score": 0.353173030212475, "per_field": {"categories": {"precision": 0.22499999999999998, "recall": 0.3115279864763336, "f1_score": 0.245271586387289}, "scientific_domains": {"precision": 0.3844, "recall": 0.4522857142857143, "f1_score": 0.4017989139515455}, "target_users": {"precision": 0.4209666666666667, "recall": 0.4522424242424242, "f1_score": 0.4124485902985905}}}, "(0.4, 5, 0.1, 1)": {"precision": 0.36644628099173554, "recall": 0.300514308372986, "f1_score": 0.31402339544513463, "per_field": {"categories": {"precision": 0.22933884297520662, "recall": 0.21497777360380665, "f1_score": 0.21810496267018006}, "scientific_domains": {"precision": 0.414, "recall": 0.3843571428571429, "f1_score": 0.39159999999999995}, "target_users": {"precision": 0.456, "recall": 0.3022080086580083, "f1_score": 0.33236522366522386}}}, "(0.4, 5, 0.1, 3)": {"precision": 0.34526170798898065, "recall": 0.38795329942478696, "f1_score": 0.3457364466193343, "per_field": {"categories": {"precision": 0.2224517906336088, "recall": 0.2996509516654145, "f1_score": 0.240018365472911}, "scientific_domains": {"precision": 0.3843333333333333, "recall": 0.44821428571428573, "f1_score": 0.3998768440709617}, "target_users": {"precision": 0.42899999999999994, "recall": 0.41599466089466064, "f1_score": 0.3973141303141303}}}, "(0.4, 5, 0.1, 5)": {"precision": 0.34345555555555557, "recall": 0.40535204166815736, "f1_score": 0.353173030212475, "per_field": {"categories": {"precision": 0.22499999999999998, "recall": 0.3115279864763336, "f1_score": 0.245271586387289}, "scientific_domains": {"precision": 0.3844, "recall": 0.4522857142857143, "f1_score": 0.4017989139515455}, "target_users": {"precision": 0.4209666666666667, "recall": 0.4522424242424242, "f1_score": 0.4124485902985905}}}, "(0.4, 5, 0.2, 1)": {"precision": 0.36644628099173554, "recall": 0.300514308372986, "f1_score": 0.31402339544513463, "per_field": {"categories": {"precision": 0.22933884297520662, "recall": 0.21497777360380665, "f1_score": 0.21810496267018006}, "scientific_domains": {"precision": 0.414, "recall": 0.3843571428571429, "f1_score": 0.39159999999999995}, "target_users": {"precision": 0.456, "recall": 0.3022080086580083, "f1_score": 0.33236522366522386}}}, "(0.4, 5, 0.2, 3)": {"precision": 0.34526170798898065, "recall": 0.38795329942478696, "f1_score": 0.3457364466193343, "per_field": {"categories": {"precision": 0.2224517906336088, "recall": 0.2996509516654145, "f1_score": 0.240018365472911}, "scientific_domains": {"precision": 0.3843333333333333, "recall": 0.44821428571428573, "f1_score": 0.3998768440709617}, "target_users": {"precision": 0.42899999999999994, "recall": 0.41599466089466064, "f1_score": 0.3973141303141303}}}, "(0.4, 5, 0.2, 5)": {"precision": 0.34345555555555557, "recall": 0.40535204166815736, "f1_score": 0.353173030212475, "per_field": {"categories": {"precision": 0.22499999999999998, "recall": 0.3115279864763336, "f1_score": 0.245271586387289}, "scientific_domains": {"precision": 0.3844, "recall": 0.4522857142857143, "f1_score": 0.4017989139515455}, "target_users": {"precision": 0.4209666666666667, "recall": 0.4522424242424242, "f1_score": 0.4124485902985905}}}, "(0.4, 5, 0.3, 1)": {"precision": 0.36644628099173554, "recall": 0.300514308372986, "f1_score": 0.31402339544513463, "per_field": {"categories": {"precision": 0.22933884297520662, "recall": 0.21497777360380665, "f1_score": 0.21810496267018006}, "scientific_domains": {"precision": 0.414, "recall": 0.3843571428571429, "f1_score": 0.39159999999999995}, "target_users": {"precision": 0.456, "recall": 0.3022080086580083, "f1_score": 0.33236522366522386}}}, "(0.4, 5, 0.3, 3)": {"precision": 0.34526170798898065, "recall": 0.38795329942478696, "f1_score": 0.3457364466193343, "per_field": {"categories": {"precision": 0.2224517906336088, "recall": 0.2996509516654145, "f1_score": 0.240018365472911}, "scientific_domains": {"precision": 0.3843333333333333, "recall": 0.44821428571428573, "f1_score": 0.3998768440709617}, "target_users": {"precision": 0.42899999999999994, "recall": 0.41599466089466064, "f1_score": 0.3973141303141303}}}, "(0.4, 5, 0.3, 5)": {"precision": 0.34345555555555557, "recall": 0.40535204166815736, "f1_score": 0.353173030212475, "per_field": {"categories": {"precision": 0.22499999999999998, "recall": 0.3115279864763336, "f1_score": 0.245271586387289}, "scientific_domains": {"precision": 0.3844, "recall": 0.4522857142857143, "f1_score": 0.4017989139515455}, "target_users": {"precision": 0.4209666666666667, "recall": 0.4522424242424242, "f1_score": 0.4124485902985905}}}, "(0.4, 5, 0.4, 1)": {"precision": 0.36644628099173554, "recall": 0.300514308372986, "f1_score": 0.31402339544513463, "per_field": {"categories": {"precision": 0.22933884297520662, "recall": 0.21497777360380665, "f1_score": 0.21810496267018006}, "scientific_domains": {"precision": 0.414, "recall": 0.3843571428571429, "f1_score": 0.39159999999999995}, "target_users": {"precision": 0.456, "recall": 0.3022080086580083, "f1_score": 0.33236522366522386}}}, "(0.4, 5, 0.4, 3)": {"precision": 0.34526170798898065, "recall": 0.38795329942478696, "f1_score": 0.3457364466193343, "per_field": {"categories": {"precision": 0.2224517906336088, "recall": 0.2996509516654145, "f1_score": 0.240018365472911}, "scientific_domains": {"precision": 0.3843333333333333, "recall": 0.44821428571428573, "f1_score": 0.3998768440709617}, "target_users": {"precision": 0.42899999999999994, "recall": 0.41599466089466064, "f1_score": 0.3973141303141303}}}, "(0.4, 5, 0.4, 5)": {"precision": 0.34345555555555557, "recall": 0.40535204166815736, "f1_score": 0.353173030212475, "per_field": {"categories": {"precision": 0.22499999999999998, "recall": 0.3115279864763336, "f1_score": 0.245271586387289}, "scientific_domains": {"precision": 0.3844, "recall": 0.4522857142857143, "f1_score": 0.4017989139515455}, "target_users": {"precision": 0.4209666666666667, "recall": 0.4522424242424242, "f1_score": 0.4124485902985905}}}, "(0.4, 5, 0.5, 1)": {"precision": 0.36644628099173554, "recall": 0.300514308372986, "f1_score": 0.31402339544513463, "per_field": {"categories": {"precision": 0.22933884297520662, "recall": 0.21497777360380665, "f1_score": 0.21810496267018006}, "scientific_domains": {"precision": 0.414, "recall": 0.3843571428571429, "f1_score": 0.39159999999999995}, "target_users": {"precision": 0.456, "recall": 0.3022080086580083, "f1_score": 0.33236522366522386}}}, "(0.4, 5, 0.5, 3)": {"precision": 0.34526170798898065, "recall": 0.38795329942478696, "f1_score": 0.3457364466193343, "per_field": {"categories": {"precision": 0.2224517906336088, "recall": 0.2996509516654145, "f1_score": 0.240018365472911}, "scientific_domains": {"precision": 0.3843333333333333, "recall": 0.44821428571428573, "f1_score": 0.3998768440709617}, "target_users": {"precision": 0.42899999999999994, "recall": 0.41599466089466064, "f1_score": 0.3973141303141303}}}, "(0.4, 5, 0.5, 5)": {"precision": 0.34345555555555557, "recall": 0.40535204166815736, "f1_score": 0.353173030212475, "per_field": {"categories": {"precision": 0.22499999999999998, "recall": 0.3115279864763336, "f1_score": 0.245271586387289}, "scientific_domains": {"precision": 0.3844, "recall": 0.4522857142857143, "f1_score": 0.4017989139515455}, "target_users": {"precision": 0.4209666666666667, "recall": 0.4522424242424242, "f1_score": 0.4124485902985905}}}, "(0.4, 5, 0.6, 1)": {"precision": 0.36644628099173554, "recall": 0.300514308372986, "f1_score": 0.31402339544513463, "per_field": {"categories": {"precision": 0.22933884297520662, "recall": 0.21497777360380665, "f1_score": 0.21810496267018006}, "scientific_domains": {"precision": 0.414, "recall": 0.3843571428571429, "f1_score": 0.39159999999999995}, "target_users": {"precision": 0.456, "recall": 0.3022080086580083, "f1_score": 0.33236522366522386}}}, "(0.4, 5, 0.6, 3)": {"precision": 0.34526170798898065, "recall": 0.38795329942478696, "f1_score": 0.3457364466193343, "per_field": {"categories": {"precision": 0.2224517906336088, "recall": 0.2996509516654145, "f1_score": 0.240018365472911}, "scientific_domains": {"precision": 0.3843333333333333, "recall": 0.44821428571428573, "f1_score": 0.3998768440709617}, "target_users": {"precision": 0.42899999999999994, "recall": 0.41599466089466064, "f1_score": 0.3973141303141303}}}, "(0.4, 5, 0.6, 5)": {"precision": 0.34345555555555557, "recall": 0.40535204166815736, "f1_score": 0.353173030212475, "per_field": {"categories": {"precision": 0.22499999999999998, "recall": 0.3115279864763336, "f1_score": 0.245271586387289}, "scientific_domains": {"precision": 0.3844, "recall": 0.4522857142857143, "f1_score": 0.4017989139515455}, "target_users": {"precision": 0.4209666666666667, "recall": 0.4522424242424242, "f1_score": 0.4124485902985905}}}, "(0.4, 5, 0.7, 1)": {"precision": 0.36644628099173554, "recall": 0.300514308372986, "f1_score": 0.31402339544513463, "per_field": {"categories": {"precision": 0.22933884297520662, "recall": 0.21497777360380665, "f1_score": 0.21810496267018006}, "scientific_domains": {"precision": 0.414, "recall": 0.3843571428571429, "f1_score": 0.39159999999999995}, "target_users": {"precision": 0.456, "recall": 0.3022080086580083, "f1_score": 0.33236522366522386}}}, "(0.4, 5, 0.7, 3)": {"precision": 0.34526170798898065, "recall": 0.38795329942478696, "f1_score": 0.3457364466193343, "per_field": {"categories": {"precision": 0.2224517906336088, "recall": 0.2996509516654145, "f1_score": 0.240018365472911}, "scientific_domains": {"precision": 0.3843333333333333, "recall": 0.44821428571428573, "f1_score": 0.3998768440709617}, "target_users": {"precision": 0.42899999999999994, "recall": 0.41599466089466064, "f1_score": 0.3973141303141303}}}, "(0.4, 5, 0.7, 5)": {"precision": 0.34345555555555557, "recall": 0.40535204166815736, "f1_score": 0.353173030212475, "per_field": {"categories": {"precision": 0.22499999999999998, "recall": 0.3115279864763336, "f1_score": 0.245271586387289}, "scientific_domains": {"precision": 0.3844, "recall": 0.4522857142857143, "f1_score": 0.4017989139515455}, "target_users": {"precision": 0.4209666666666667, "recall": 0.4522424242424242, "f1_score": 0.4124485902985905}}}, "(0.4, 5, 0.8, 1)": {"precision": 0.36644628099173554, "recall": 0.300514308372986, "f1_score": 0.31402339544513463, "per_field": {"categories": {"precision": 0.22933884297520662, "recall": 0.21497777360380665, "f1_score": 0.21810496267018006}, "scientific_domains": {"precision": 0.414, "recall": 0.3843571428571429, "f1_score": 0.39159999999999995}, "target_users": {"precision": 0.456, "recall": 0.3022080086580083, "f1_score": 0.33236522366522386}}}, "(0.4, 5, 0.8, 3)": {"precision": 0.34526170798898065, "recall": 0.38795329942478696, "f1_score": 0.3457364466193343, "per_field": {"categories": {"precision": 0.2224517906336088, "recall": 0.2996509516654145, "f1_score": 0.240018365472911}, "scientific_domains": {"precision": 0.3843333333333333, "recall": 0.44821428571428573, "f1_score": 0.3998768440709617}, "target_users": {"precision": 0.42899999999999994, "recall": 0.41599466089466064, "f1_score": 0.3973141303141303}}}, "(0.4, 5, 0.8, 5)": {"precision": 0.34345555555555557, "recall": 0.40535204166815736, "f1_score": 0.353173030212475, "per_field": {"categories": {"precision": 0.22499999999999998, "recall": 0.3115279864763336, "f1_score": 0.245271586387289}, "scientific_domains": {"precision": 0.3844, "recall": 0.4522857142857143, "f1_score": 0.4017989139515455}, "target_users": {"precision": 0.4209666666666667, "recall": 0.4522424242424242, "f1_score": 0.4124485902985905}}}, "(0.4, 5, 0.9, 1)": {"precision": 0.36644628099173554, "recall": 0.300514308372986, "f1_score": 0.31402339544513463, "per_field": {"categories": {"precision": 0.22933884297520662, "recall": 0.21497777360380665, "f1_score": 0.21810496267018006}, "scientific_domains": {"precision": 0.414, "recall": 0.3843571428571429, "f1_score": 0.39159999999999995}, "target_users": {"precision": 0.456, "recall": 0.3022080086580083, "f1_score": 0.33236522366522386}}}, "(0.4, 5, 0.9, 3)": {"precision": 0.34526170798898065, "recall": 0.38795329942478696, "f1_score": 0.3457364466193343, "per_field": {"categories": {"precision": 0.2224517906336088, "recall": 0.2996509516654145, "f1_score": 0.240018365472911}, "scientific_domains": {"precision": 0.3843333333333333, "recall": 0.44821428571428573, "f1_score": 0.3998768440709617}, "target_users": {"precision": 0.42899999999999994, "recall": 0.41599466089466064, "f1_score": 0.3973141303141303}}}, "(0.4, 5, 0.9, 5)": {"precision": 0.34345555555555557, "recall": 0.40535204166815736, "f1_score": 0.353173030212475, "per_field": {"categories": {"precision": 0.22499999999999998, "recall": 0.3115279864763336, "f1_score": 0.245271586387289}, "scientific_domains": {"precision": 0.3844, "recall": 0.4522857142857143, "f1_score": 0.4017989139515455}, "target_users": {"precision": 0.4209666666666667, "recall": 0.4522424242424242, "f1_score": 0.4124485902985905}}}, "(0.4, 10, 0, 1)": {"precision": 0.3615371900826447, "recall": 0.29443855079722836, "f1_score": 0.3082698600915993, "per_field": {"categories": {"precision": 0.2066115702479339, "recall": 0.19225050087653392, "f1_score": 0.19537768994290733}, "scientific_domains": {"precision": 0.416, "recall": 0.3863571428571429, "f1_score": 0.39359999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 10, 0, 3)": {"precision": 0.3271864095500459, "recall": 0.38646569611900183, "f1_score": 0.33188685984247485, "per_field": {"categories": {"precision": 0.19455922865013778, "recall": 0.2851881417480591, "f1_score": 0.21453627180899906}, "scientific_domains": {"precision": 0.37333333333333324, "recall": 0.45821428571428574, "f1_score": 0.3938768440709617}, "target_users": {"precision": 0.4136666666666666, "recall": 0.41599466089466064, "f1_score": 0.38724746364746376}}}, "(0.4, 10, 0, 5)": {"precision": 0.3245961432506887, "recall": 0.40801870833482407, "f1_score": 0.3390027298058109, "per_field": {"categories": {"precision": 0.19755509641873284, "recall": 0.3115279864763336, "f1_score": 0.2218654470720587}, "scientific_domains": {"precision": 0.3724666666666667, "recall": 0.4602857142857143, "f1_score": 0.39446558061821213}, "target_users": {"precision": 0.4037666666666666, "recall": 0.4522424242424242, "f1_score": 0.400677161727162}}}, "(0.4, 10, 0.1, 1)": {"precision": 0.3615371900826447, "recall": 0.29443855079722836, "f1_score": 0.3082698600915993, "per_field": {"categories": {"precision": 0.2066115702479339, "recall": 0.19225050087653392, "f1_score": 0.19537768994290733}, "scientific_domains": {"precision": 0.416, "recall": 0.3863571428571429, "f1_score": 0.39359999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 10, 0.1, 3)": {"precision": 0.3271864095500459, "recall": 0.38646569611900183, "f1_score": 0.33188685984247485, "per_field": {"categories": {"precision": 0.19455922865013778, "recall": 0.2851881417480591, "f1_score": 0.21453627180899906}, "scientific_domains": {"precision": 0.37333333333333324, "recall": 0.45821428571428574, "f1_score": 0.3938768440709617}, "target_users": {"precision": 0.4136666666666666, "recall": 0.41599466089466064, "f1_score": 0.38724746364746376}}}, "(0.4, 10, 0.1, 5)": {"precision": 0.3245961432506887, "recall": 0.40801870833482407, "f1_score": 0.3390027298058109, "per_field": {"categories": {"precision": 0.19755509641873284, "recall": 0.3115279864763336, "f1_score": 0.2218654470720587}, "scientific_domains": {"precision": 0.3724666666666667, "recall": 0.4602857142857143, "f1_score": 0.39446558061821213}, "target_users": {"precision": 0.4037666666666666, "recall": 0.4522424242424242, "f1_score": 0.400677161727162}}}, "(0.4, 10, 0.2, 1)": {"precision": 0.3615371900826447, "recall": 0.29443855079722836, "f1_score": 0.3082698600915993, "per_field": {"categories": {"precision": 0.2066115702479339, "recall": 0.19225050087653392, "f1_score": 0.19537768994290733}, "scientific_domains": {"precision": 0.416, "recall": 0.3863571428571429, "f1_score": 0.39359999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 10, 0.2, 3)": {"precision": 0.3271864095500459, "recall": 0.38646569611900183, "f1_score": 0.33188685984247485, "per_field": {"categories": {"precision": 0.19455922865013778, "recall": 0.2851881417480591, "f1_score": 0.21453627180899906}, "scientific_domains": {"precision": 0.37333333333333324, "recall": 0.45821428571428574, "f1_score": 0.3938768440709617}, "target_users": {"precision": 0.4136666666666666, "recall": 0.41599466089466064, "f1_score": 0.38724746364746376}}}, "(0.4, 10, 0.2, 5)": {"precision": 0.3245961432506887, "recall": 0.40801870833482407, "f1_score": 0.3390027298058109, "per_field": {"categories": {"precision": 0.19755509641873284, "recall": 0.3115279864763336, "f1_score": 0.2218654470720587}, "scientific_domains": {"precision": 0.3724666666666667, "recall": 0.4602857142857143, "f1_score": 0.39446558061821213}, "target_users": {"precision": 0.4037666666666666, "recall": 0.4522424242424242, "f1_score": 0.400677161727162}}}, "(0.4, 10, 0.3, 1)": {"precision": 0.3615371900826447, "recall": 0.29443855079722836, "f1_score": 0.3082698600915993, "per_field": {"categories": {"precision": 0.2066115702479339, "recall": 0.19225050087653392, "f1_score": 0.19537768994290733}, "scientific_domains": {"precision": 0.416, "recall": 0.3863571428571429, "f1_score": 0.39359999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 10, 0.3, 3)": {"precision": 0.3271864095500459, "recall": 0.38646569611900183, "f1_score": 0.33188685984247485, "per_field": {"categories": {"precision": 0.19455922865013778, "recall": 0.2851881417480591, "f1_score": 0.21453627180899906}, "scientific_domains": {"precision": 0.37333333333333324, "recall": 0.45821428571428574, "f1_score": 0.3938768440709617}, "target_users": {"precision": 0.4136666666666666, "recall": 0.41599466089466064, "f1_score": 0.38724746364746376}}}, "(0.4, 10, 0.3, 5)": {"precision": 0.3245961432506887, "recall": 0.40801870833482407, "f1_score": 0.3390027298058109, "per_field": {"categories": {"precision": 0.19755509641873284, "recall": 0.3115279864763336, "f1_score": 0.2218654470720587}, "scientific_domains": {"precision": 0.3724666666666667, "recall": 0.4602857142857143, "f1_score": 0.39446558061821213}, "target_users": {"precision": 0.4037666666666666, "recall": 0.4522424242424242, "f1_score": 0.400677161727162}}}, "(0.4, 10, 0.4, 1)": {"precision": 0.3615371900826447, "recall": 0.29443855079722836, "f1_score": 0.3082698600915993, "per_field": {"categories": {"precision": 0.2066115702479339, "recall": 0.19225050087653392, "f1_score": 0.19537768994290733}, "scientific_domains": {"precision": 0.416, "recall": 0.3863571428571429, "f1_score": 0.39359999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 10, 0.4, 3)": {"precision": 0.3271864095500459, "recall": 0.38646569611900183, "f1_score": 0.33188685984247485, "per_field": {"categories": {"precision": 0.19455922865013778, "recall": 0.2851881417480591, "f1_score": 0.21453627180899906}, "scientific_domains": {"precision": 0.37333333333333324, "recall": 0.45821428571428574, "f1_score": 0.3938768440709617}, "target_users": {"precision": 0.4136666666666666, "recall": 0.41599466089466064, "f1_score": 0.38724746364746376}}}, "(0.4, 10, 0.4, 5)": {"precision": 0.3245961432506887, "recall": 0.40801870833482407, "f1_score": 0.3390027298058109, "per_field": {"categories": {"precision": 0.19755509641873284, "recall": 0.3115279864763336, "f1_score": 0.2218654470720587}, "scientific_domains": {"precision": 0.3724666666666667, "recall": 0.4602857142857143, "f1_score": 0.39446558061821213}, "target_users": {"precision": 0.4037666666666666, "recall": 0.4522424242424242, "f1_score": 0.400677161727162}}}, "(0.4, 10, 0.5, 1)": {"precision": 0.3615371900826447, "recall": 0.29443855079722836, "f1_score": 0.3082698600915993, "per_field": {"categories": {"precision": 0.2066115702479339, "recall": 0.19225050087653392, "f1_score": 0.19537768994290733}, "scientific_domains": {"precision": 0.416, "recall": 0.3863571428571429, "f1_score": 0.39359999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 10, 0.5, 3)": {"precision": 0.3271864095500459, "recall": 0.38646569611900183, "f1_score": 0.33188685984247485, "per_field": {"categories": {"precision": 0.19455922865013778, "recall": 0.2851881417480591, "f1_score": 0.21453627180899906}, "scientific_domains": {"precision": 0.37333333333333324, "recall": 0.45821428571428574, "f1_score": 0.3938768440709617}, "target_users": {"precision": 0.4136666666666666, "recall": 0.41599466089466064, "f1_score": 0.38724746364746376}}}, "(0.4, 10, 0.5, 5)": {"precision": 0.3245961432506887, "recall": 0.40801870833482407, "f1_score": 0.3390027298058109, "per_field": {"categories": {"precision": 0.19755509641873284, "recall": 0.3115279864763336, "f1_score": 0.2218654470720587}, "scientific_domains": {"precision": 0.3724666666666667, "recall": 0.4602857142857143, "f1_score": 0.39446558061821213}, "target_users": {"precision": 0.4037666666666666, "recall": 0.4522424242424242, "f1_score": 0.400677161727162}}}, "(0.4, 10, 0.6, 1)": {"precision": 0.3615371900826447, "recall": 0.29443855079722836, "f1_score": 0.3082698600915993, "per_field": {"categories": {"precision": 0.2066115702479339, "recall": 0.19225050087653392, "f1_score": 0.19537768994290733}, "scientific_domains": {"precision": 0.416, "recall": 0.3863571428571429, "f1_score": 0.39359999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 10, 0.6, 3)": {"precision": 0.3271864095500459, "recall": 0.38646569611900183, "f1_score": 0.33188685984247485, "per_field": {"categories": {"precision": 0.19455922865013778, "recall": 0.2851881417480591, "f1_score": 0.21453627180899906}, "scientific_domains": {"precision": 0.37333333333333324, "recall": 0.45821428571428574, "f1_score": 0.3938768440709617}, "target_users": {"precision": 0.4136666666666666, "recall": 0.41599466089466064, "f1_score": 0.38724746364746376}}}, "(0.4, 10, 0.6, 5)": {"precision": 0.3245961432506887, "recall": 0.40801870833482407, "f1_score": 0.3390027298058109, "per_field": {"categories": {"precision": 0.19755509641873284, "recall": 0.3115279864763336, "f1_score": 0.2218654470720587}, "scientific_domains": {"precision": 0.3724666666666667, "recall": 0.4602857142857143, "f1_score": 0.39446558061821213}, "target_users": {"precision": 0.4037666666666666, "recall": 0.4522424242424242, "f1_score": 0.400677161727162}}}, "(0.4, 10, 0.7, 1)": {"precision": 0.3615371900826447, "recall": 0.29443855079722836, "f1_score": 0.3082698600915993, "per_field": {"categories": {"precision": 0.2066115702479339, "recall": 0.19225050087653392, "f1_score": 0.19537768994290733}, "scientific_domains": {"precision": 0.416, "recall": 0.3863571428571429, "f1_score": 0.39359999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 10, 0.7, 3)": {"precision": 0.3271864095500459, "recall": 0.38646569611900183, "f1_score": 0.33188685984247485, "per_field": {"categories": {"precision": 0.19455922865013778, "recall": 0.2851881417480591, "f1_score": 0.21453627180899906}, "scientific_domains": {"precision": 0.37333333333333324, "recall": 0.45821428571428574, "f1_score": 0.3938768440709617}, "target_users": {"precision": 0.4136666666666666, "recall": 0.41599466089466064, "f1_score": 0.38724746364746376}}}, "(0.4, 10, 0.7, 5)": {"precision": 0.3245961432506887, "recall": 0.40801870833482407, "f1_score": 0.3390027298058109, "per_field": {"categories": {"precision": 0.19755509641873284, "recall": 0.3115279864763336, "f1_score": 0.2218654470720587}, "scientific_domains": {"precision": 0.3724666666666667, "recall": 0.4602857142857143, "f1_score": 0.39446558061821213}, "target_users": {"precision": 0.4037666666666666, "recall": 0.4522424242424242, "f1_score": 0.400677161727162}}}, "(0.4, 10, 0.8, 1)": {"precision": 0.3615371900826447, "recall": 0.29443855079722836, "f1_score": 0.3082698600915993, "per_field": {"categories": {"precision": 0.2066115702479339, "recall": 0.19225050087653392, "f1_score": 0.19537768994290733}, "scientific_domains": {"precision": 0.416, "recall": 0.3863571428571429, "f1_score": 0.39359999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 10, 0.8, 3)": {"precision": 0.3271864095500459, "recall": 0.38646569611900183, "f1_score": 0.33188685984247485, "per_field": {"categories": {"precision": 0.19455922865013778, "recall": 0.2851881417480591, "f1_score": 0.21453627180899906}, "scientific_domains": {"precision": 0.37333333333333324, "recall": 0.45821428571428574, "f1_score": 0.3938768440709617}, "target_users": {"precision": 0.4136666666666666, "recall": 0.41599466089466064, "f1_score": 0.38724746364746376}}}, "(0.4, 10, 0.8, 5)": {"precision": 0.3245961432506887, "recall": 0.40801870833482407, "f1_score": 0.3390027298058109, "per_field": {"categories": {"precision": 0.19755509641873284, "recall": 0.3115279864763336, "f1_score": 0.2218654470720587}, "scientific_domains": {"precision": 0.3724666666666667, "recall": 0.4602857142857143, "f1_score": 0.39446558061821213}, "target_users": {"precision": 0.4037666666666666, "recall": 0.4522424242424242, "f1_score": 0.400677161727162}}}, "(0.4, 10, 0.9, 1)": {"precision": 0.3615371900826447, "recall": 0.29443855079722836, "f1_score": 0.3082698600915993, "per_field": {"categories": {"precision": 0.2066115702479339, "recall": 0.19225050087653392, "f1_score": 0.19537768994290733}, "scientific_domains": {"precision": 0.416, "recall": 0.3863571428571429, "f1_score": 0.39359999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 10, 0.9, 3)": {"precision": 0.3271864095500459, "recall": 0.38646569611900183, "f1_score": 0.33188685984247485, "per_field": {"categories": {"precision": 0.19455922865013778, "recall": 0.2851881417480591, "f1_score": 0.21453627180899906}, "scientific_domains": {"precision": 0.37333333333333324, "recall": 0.45821428571428574, "f1_score": 0.3938768440709617}, "target_users": {"precision": 0.4136666666666666, "recall": 0.41599466089466064, "f1_score": 0.38724746364746376}}}, "(0.4, 10, 0.9, 5)": {"precision": 0.3245961432506887, "recall": 0.40801870833482407, "f1_score": 0.3390027298058109, "per_field": {"categories": {"precision": 0.19755509641873284, "recall": 0.3115279864763336, "f1_score": 0.2218654470720587}, "scientific_domains": {"precision": 0.3724666666666667, "recall": 0.4602857142857143, "f1_score": 0.39446558061821213}, "target_users": {"precision": 0.4037666666666666, "recall": 0.4522424242424242, "f1_score": 0.400677161727162}}}, "(0.4, 15, 0, 1)": {"precision": 0.36082644628099175, "recall": 0.2937278069955755, "f1_score": 0.30755911628994637, "per_field": {"categories": {"precision": 0.2024793388429752, "recall": 0.18811826947157526, "f1_score": 0.19124545853794866}, "scientific_domains": {"precision": 0.418, "recall": 0.3883571428571429, "f1_score": 0.39559999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 15, 0, 3)": {"precision": 0.3347006427915518, "recall": 0.3851323627856685, "f1_score": 0.33772708022814973, "per_field": {"categories": {"precision": 0.20110192837465568, "recall": 0.2851881417480591, "f1_score": 0.2203902662993572}, "scientific_domains": {"precision": 0.3773333333333333, "recall": 0.45421428571428574, "f1_score": 0.39754351073762834}, "target_users": {"precision": 0.4256666666666666, "recall": 0.41599466089466064, "f1_score": 0.3952474636474637}}}, "(0.4, 15, 0, 5)": {"precision": 0.332648025711662, "recall": 0.40599666976733095, "f1_score": 0.3454376242043417, "per_field": {"categories": {"precision": 0.20554407713498624, "recall": 0.3094618707738543, "f1_score": 0.22930346360098436}, "scientific_domains": {"precision": 0.3766333333333333, "recall": 0.4562857142857143, "f1_score": 0.39833224728487887}, "target_users": {"precision": 0.4157666666666666, "recall": 0.4522424242424242, "f1_score": 0.40867716172716195}}}, "(0.4, 15, 0.1, 1)": {"precision": 0.36082644628099175, "recall": 0.2937278069955755, "f1_score": 0.30755911628994637, "per_field": {"categories": {"precision": 0.2024793388429752, "recall": 0.18811826947157526, "f1_score": 0.19124545853794866}, "scientific_domains": {"precision": 0.418, "recall": 0.3883571428571429, "f1_score": 0.39559999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 15, 0.1, 3)": {"precision": 0.3347006427915518, "recall": 0.3851323627856685, "f1_score": 0.33772708022814973, "per_field": {"categories": {"precision": 0.20110192837465568, "recall": 0.2851881417480591, "f1_score": 0.2203902662993572}, "scientific_domains": {"precision": 0.3773333333333333, "recall": 0.45421428571428574, "f1_score": 0.39754351073762834}, "target_users": {"precision": 0.4256666666666666, "recall": 0.41599466089466064, "f1_score": 0.3952474636474637}}}, "(0.4, 15, 0.1, 5)": {"precision": 0.332648025711662, "recall": 0.40599666976733095, "f1_score": 0.3454376242043417, "per_field": {"categories": {"precision": 0.20554407713498624, "recall": 0.3094618707738543, "f1_score": 0.22930346360098436}, "scientific_domains": {"precision": 0.3766333333333333, "recall": 0.4562857142857143, "f1_score": 0.39833224728487887}, "target_users": {"precision": 0.4157666666666666, "recall": 0.4522424242424242, "f1_score": 0.40867716172716195}}}, "(0.4, 15, 0.2, 1)": {"precision": 0.36082644628099175, "recall": 0.2937278069955755, "f1_score": 0.30755911628994637, "per_field": {"categories": {"precision": 0.2024793388429752, "recall": 0.18811826947157526, "f1_score": 0.19124545853794866}, "scientific_domains": {"precision": 0.418, "recall": 0.3883571428571429, "f1_score": 0.39559999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 15, 0.2, 3)": {"precision": 0.3347006427915518, "recall": 0.3851323627856685, "f1_score": 0.33772708022814973, "per_field": {"categories": {"precision": 0.20110192837465568, "recall": 0.2851881417480591, "f1_score": 0.2203902662993572}, "scientific_domains": {"precision": 0.3773333333333333, "recall": 0.45421428571428574, "f1_score": 0.39754351073762834}, "target_users": {"precision": 0.4256666666666666, "recall": 0.41599466089466064, "f1_score": 0.3952474636474637}}}, "(0.4, 15, 0.2, 5)": {"precision": 0.332648025711662, "recall": 0.40599666976733095, "f1_score": 0.3454376242043417, "per_field": {"categories": {"precision": 0.20554407713498624, "recall": 0.3094618707738543, "f1_score": 0.22930346360098436}, "scientific_domains": {"precision": 0.3766333333333333, "recall": 0.4562857142857143, "f1_score": 0.39833224728487887}, "target_users": {"precision": 0.4157666666666666, "recall": 0.4522424242424242, "f1_score": 0.40867716172716195}}}, "(0.4, 15, 0.3, 1)": {"precision": 0.36082644628099175, "recall": 0.2937278069955755, "f1_score": 0.30755911628994637, "per_field": {"categories": {"precision": 0.2024793388429752, "recall": 0.18811826947157526, "f1_score": 0.19124545853794866}, "scientific_domains": {"precision": 0.418, "recall": 0.3883571428571429, "f1_score": 0.39559999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 15, 0.3, 3)": {"precision": 0.3347006427915518, "recall": 0.3851323627856685, "f1_score": 0.33772708022814973, "per_field": {"categories": {"precision": 0.20110192837465568, "recall": 0.2851881417480591, "f1_score": 0.2203902662993572}, "scientific_domains": {"precision": 0.3773333333333333, "recall": 0.45421428571428574, "f1_score": 0.39754351073762834}, "target_users": {"precision": 0.4256666666666666, "recall": 0.41599466089466064, "f1_score": 0.3952474636474637}}}, "(0.4, 15, 0.3, 5)": {"precision": 0.332648025711662, "recall": 0.40599666976733095, "f1_score": 0.3454376242043417, "per_field": {"categories": {"precision": 0.20554407713498624, "recall": 0.3094618707738543, "f1_score": 0.22930346360098436}, "scientific_domains": {"precision": 0.3766333333333333, "recall": 0.4562857142857143, "f1_score": 0.39833224728487887}, "target_users": {"precision": 0.4157666666666666, "recall": 0.4522424242424242, "f1_score": 0.40867716172716195}}}, "(0.4, 15, 0.4, 1)": {"precision": 0.36082644628099175, "recall": 0.2937278069955755, "f1_score": 0.30755911628994637, "per_field": {"categories": {"precision": 0.2024793388429752, "recall": 0.18811826947157526, "f1_score": 0.19124545853794866}, "scientific_domains": {"precision": 0.418, "recall": 0.3883571428571429, "f1_score": 0.39559999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 15, 0.4, 3)": {"precision": 0.3347006427915518, "recall": 0.3851323627856685, "f1_score": 0.33772708022814973, "per_field": {"categories": {"precision": 0.20110192837465568, "recall": 0.2851881417480591, "f1_score": 0.2203902662993572}, "scientific_domains": {"precision": 0.3773333333333333, "recall": 0.45421428571428574, "f1_score": 0.39754351073762834}, "target_users": {"precision": 0.4256666666666666, "recall": 0.41599466089466064, "f1_score": 0.3952474636474637}}}, "(0.4, 15, 0.4, 5)": {"precision": 0.332648025711662, "recall": 0.40599666976733095, "f1_score": 0.3454376242043417, "per_field": {"categories": {"precision": 0.20554407713498624, "recall": 0.3094618707738543, "f1_score": 0.22930346360098436}, "scientific_domains": {"precision": 0.3766333333333333, "recall": 0.4562857142857143, "f1_score": 0.39833224728487887}, "target_users": {"precision": 0.4157666666666666, "recall": 0.4522424242424242, "f1_score": 0.40867716172716195}}}, "(0.4, 15, 0.5, 1)": {"precision": 0.36082644628099175, "recall": 0.2937278069955755, "f1_score": 0.30755911628994637, "per_field": {"categories": {"precision": 0.2024793388429752, "recall": 0.18811826947157526, "f1_score": 0.19124545853794866}, "scientific_domains": {"precision": 0.418, "recall": 0.3883571428571429, "f1_score": 0.39559999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 15, 0.5, 3)": {"precision": 0.3347006427915518, "recall": 0.3851323627856685, "f1_score": 0.33772708022814973, "per_field": {"categories": {"precision": 0.20110192837465568, "recall": 0.2851881417480591, "f1_score": 0.2203902662993572}, "scientific_domains": {"precision": 0.3773333333333333, "recall": 0.45421428571428574, "f1_score": 0.39754351073762834}, "target_users": {"precision": 0.4256666666666666, "recall": 0.41599466089466064, "f1_score": 0.3952474636474637}}}, "(0.4, 15, 0.5, 5)": {"precision": 0.332648025711662, "recall": 0.40599666976733095, "f1_score": 0.3454376242043417, "per_field": {"categories": {"precision": 0.20554407713498624, "recall": 0.3094618707738543, "f1_score": 0.22930346360098436}, "scientific_domains": {"precision": 0.3766333333333333, "recall": 0.4562857142857143, "f1_score": 0.39833224728487887}, "target_users": {"precision": 0.4157666666666666, "recall": 0.4522424242424242, "f1_score": 0.40867716172716195}}}, "(0.4, 15, 0.6, 1)": {"precision": 0.36082644628099175, "recall": 0.2937278069955755, "f1_score": 0.30755911628994637, "per_field": {"categories": {"precision": 0.2024793388429752, "recall": 0.18811826947157526, "f1_score": 0.19124545853794866}, "scientific_domains": {"precision": 0.418, "recall": 0.3883571428571429, "f1_score": 0.39559999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 15, 0.6, 3)": {"precision": 0.3347006427915518, "recall": 0.3851323627856685, "f1_score": 0.33772708022814973, "per_field": {"categories": {"precision": 0.20110192837465568, "recall": 0.2851881417480591, "f1_score": 0.2203902662993572}, "scientific_domains": {"precision": 0.3773333333333333, "recall": 0.45421428571428574, "f1_score": 0.39754351073762834}, "target_users": {"precision": 0.4256666666666666, "recall": 0.41599466089466064, "f1_score": 0.3952474636474637}}}, "(0.4, 15, 0.6, 5)": {"precision": 0.332648025711662, "recall": 0.40599666976733095, "f1_score": 0.3454376242043417, "per_field": {"categories": {"precision": 0.20554407713498624, "recall": 0.3094618707738543, "f1_score": 0.22930346360098436}, "scientific_domains": {"precision": 0.3766333333333333, "recall": 0.4562857142857143, "f1_score": 0.39833224728487887}, "target_users": {"precision": 0.4157666666666666, "recall": 0.4522424242424242, "f1_score": 0.40867716172716195}}}, "(0.4, 15, 0.7, 1)": {"precision": 0.36082644628099175, "recall": 0.2937278069955755, "f1_score": 0.30755911628994637, "per_field": {"categories": {"precision": 0.2024793388429752, "recall": 0.18811826947157526, "f1_score": 0.19124545853794866}, "scientific_domains": {"precision": 0.418, "recall": 0.3883571428571429, "f1_score": 0.39559999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 15, 0.7, 3)": {"precision": 0.3347006427915518, "recall": 0.3851323627856685, "f1_score": 0.33772708022814973, "per_field": {"categories": {"precision": 0.20110192837465568, "recall": 0.2851881417480591, "f1_score": 0.2203902662993572}, "scientific_domains": {"precision": 0.3773333333333333, "recall": 0.45421428571428574, "f1_score": 0.39754351073762834}, "target_users": {"precision": 0.4256666666666666, "recall": 0.41599466089466064, "f1_score": 0.3952474636474637}}}, "(0.4, 15, 0.7, 5)": {"precision": 0.332648025711662, "recall": 0.40599666976733095, "f1_score": 0.3454376242043417, "per_field": {"categories": {"precision": 0.20554407713498624, "recall": 0.3094618707738543, "f1_score": 0.22930346360098436}, "scientific_domains": {"precision": 0.3766333333333333, "recall": 0.4562857142857143, "f1_score": 0.39833224728487887}, "target_users": {"precision": 0.4157666666666666, "recall": 0.4522424242424242, "f1_score": 0.40867716172716195}}}, "(0.4, 15, 0.8, 1)": {"precision": 0.36082644628099175, "recall": 0.2937278069955755, "f1_score": 0.30755911628994637, "per_field": {"categories": {"precision": 0.2024793388429752, "recall": 0.18811826947157526, "f1_score": 0.19124545853794866}, "scientific_domains": {"precision": 0.418, "recall": 0.3883571428571429, "f1_score": 0.39559999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 15, 0.8, 3)": {"precision": 0.3347006427915518, "recall": 0.3851323627856685, "f1_score": 0.33772708022814973, "per_field": {"categories": {"precision": 0.20110192837465568, "recall": 0.2851881417480591, "f1_score": 0.2203902662993572}, "scientific_domains": {"precision": 0.3773333333333333, "recall": 0.45421428571428574, "f1_score": 0.39754351073762834}, "target_users": {"precision": 0.4256666666666666, "recall": 0.41599466089466064, "f1_score": 0.3952474636474637}}}, "(0.4, 15, 0.8, 5)": {"precision": 0.332648025711662, "recall": 0.40599666976733095, "f1_score": 0.3454376242043417, "per_field": {"categories": {"precision": 0.20554407713498624, "recall": 0.3094618707738543, "f1_score": 0.22930346360098436}, "scientific_domains": {"precision": 0.3766333333333333, "recall": 0.4562857142857143, "f1_score": 0.39833224728487887}, "target_users": {"precision": 0.4157666666666666, "recall": 0.4522424242424242, "f1_score": 0.40867716172716195}}}, "(0.4, 15, 0.9, 1)": {"precision": 0.36082644628099175, "recall": 0.2937278069955755, "f1_score": 0.30755911628994637, "per_field": {"categories": {"precision": 0.2024793388429752, "recall": 0.18811826947157526, "f1_score": 0.19124545853794866}, "scientific_domains": {"precision": 0.418, "recall": 0.3883571428571429, "f1_score": 0.39559999999999995}, "target_users": {"precision": 0.462, "recall": 0.3047080086580083, "f1_score": 0.33583189033189054}}}, "(0.4, 15, 0.9, 3)": {"precision": 0.3347006427915518, "recall": 0.3851323627856685, "f1_score": 0.33772708022814973, "per_field": {"categories": {"precision": 0.20110192837465568, "recall": 0.2851881417480591, "f1_score": 0.2203902662993572}, "scientific_domains": {"precision": 0.3773333333333333, "recall": 0.45421428571428574, "f1_score": 0.39754351073762834}, "target_users": {"precision": 0.4256666666666666, "recall": 0.41599466089466064, "f1_score": 0.3952474636474637}}}, "(0.4, 15, 0.9, 5)": {"precision": 0.332648025711662, "recall": 0.40599666976733095, "f1_score": 0.3454376242043417, "per_field": {"categories": {"precision": 0.20554407713498624, "recall": 0.3094618707738543, "f1_score": 0.22930346360098436}, "scientific_domains": {"precision": 0.3766333333333333, "recall": 0.4562857142857143, "f1_score": 0.39833224728487887}, "target_users": {"precision": 0.4157666666666666, "recall": 0.4522424242424242, "f1_score": 0.40867716172716195}}}, "(0.5, 5, 0, 1)": {"precision": 0.3230468319559229, "recall": 0.26740680039275083, "f1_score": 0.2790778228359256, "per_field": {"categories": {"precision": 0.2231404958677686, "recall": 0.2101568369646882, "f1_score": 0.2129396734139817}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.388, "recall": 0.26199213564213564, "f1_score": 0.28719379509379517}}}, "(0.5, 5, 0, 3)": {"precision": 0.30504040404040406, "recall": 0.34110315731100854, "f1_score": 0.3066616204354172, "per_field": {"categories": {"precision": 0.21212121212121207, "recall": 0.27446401738757115, "f1_score": 0.22596877869605145}, "scientific_domains": {"precision": 0.3323333333333333, "recall": 0.38492857142857145, "f1_score": 0.3447768440709617}, "target_users": {"precision": 0.3706666666666668, "recall": 0.36391688311688297, "f1_score": 0.34923923853923855}}}, "(0.5, 5, 0, 5)": {"precision": 0.30464692378328745, "recall": 0.3561937750189817, "f1_score": 0.3142131412798587, "per_field": {"categories": {"precision": 0.2137741046831956, "recall": 0.2825039801796001, "f1_score": 0.22984458914210984}, "scientific_domains": {"precision": 0.33190000000000003, "recall": 0.387, "f1_score": 0.34589891395154554}, "target_users": {"precision": 0.36826666666666663, "recall": 0.399077344877345, "f1_score": 0.3668959207459208}}}, "(0.5, 5, 0.1, 1)": {"precision": 0.3230468319559229, "recall": 0.26740680039275083, "f1_score": 0.2790778228359256, "per_field": {"categories": {"precision": 0.2231404958677686, "recall": 0.2101568369646882, "f1_score": 0.2129396734139817}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.388, "recall": 0.26199213564213564, "f1_score": 0.28719379509379517}}}, "(0.5, 5, 0.1, 3)": {"precision": 0.30504040404040406, "recall": 0.34110315731100854, "f1_score": 0.3066616204354172, "per_field": {"categories": {"precision": 0.21212121212121207, "recall": 0.27446401738757115, "f1_score": 0.22596877869605145}, "scientific_domains": {"precision": 0.3323333333333333, "recall": 0.38492857142857145, "f1_score": 0.3447768440709617}, "target_users": {"precision": 0.3706666666666668, "recall": 0.36391688311688297, "f1_score": 0.34923923853923855}}}, "(0.5, 5, 0.1, 5)": {"precision": 0.30464692378328745, "recall": 0.3561937750189817, "f1_score": 0.3142131412798587, "per_field": {"categories": {"precision": 0.2137741046831956, "recall": 0.2825039801796001, "f1_score": 0.22984458914210984}, "scientific_domains": {"precision": 0.33190000000000003, "recall": 0.387, "f1_score": 0.34589891395154554}, "target_users": {"precision": 0.36826666666666663, "recall": 0.399077344877345, "f1_score": 0.3668959207459208}}}, "(0.5, 5, 0.2, 1)": {"precision": 0.3230468319559229, "recall": 0.26740680039275083, "f1_score": 0.2790778228359256, "per_field": {"categories": {"precision": 0.2231404958677686, "recall": 0.2101568369646882, "f1_score": 0.2129396734139817}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.388, "recall": 0.26199213564213564, "f1_score": 0.28719379509379517}}}, "(0.5, 5, 0.2, 3)": {"precision": 0.30504040404040406, "recall": 0.34110315731100854, "f1_score": 0.3066616204354172, "per_field": {"categories": {"precision": 0.21212121212121207, "recall": 0.27446401738757115, "f1_score": 0.22596877869605145}, "scientific_domains": {"precision": 0.3323333333333333, "recall": 0.38492857142857145, "f1_score": 0.3447768440709617}, "target_users": {"precision": 0.3706666666666668, "recall": 0.36391688311688297, "f1_score": 0.34923923853923855}}}, "(0.5, 5, 0.2, 5)": {"precision": 0.30464692378328745, "recall": 0.3561937750189817, "f1_score": 0.3142131412798587, "per_field": {"categories": {"precision": 0.2137741046831956, "recall": 0.2825039801796001, "f1_score": 0.22984458914210984}, "scientific_domains": {"precision": 0.33190000000000003, "recall": 0.387, "f1_score": 0.34589891395154554}, "target_users": {"precision": 0.36826666666666663, "recall": 0.399077344877345, "f1_score": 0.3668959207459208}}}, "(0.5, 5, 0.3, 1)": {"precision": 0.3230468319559229, "recall": 0.26740680039275083, "f1_score": 0.2790778228359256, "per_field": {"categories": {"precision": 0.2231404958677686, "recall": 0.2101568369646882, "f1_score": 0.2129396734139817}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.388, "recall": 0.26199213564213564, "f1_score": 0.28719379509379517}}}, "(0.5, 5, 0.3, 3)": {"precision": 0.30504040404040406, "recall": 0.34110315731100854, "f1_score": 0.3066616204354172, "per_field": {"categories": {"precision": 0.21212121212121207, "recall": 0.27446401738757115, "f1_score": 0.22596877869605145}, "scientific_domains": {"precision": 0.3323333333333333, "recall": 0.38492857142857145, "f1_score": 0.3447768440709617}, "target_users": {"precision": 0.3706666666666668, "recall": 0.36391688311688297, "f1_score": 0.34923923853923855}}}, "(0.5, 5, 0.3, 5)": {"precision": 0.30464692378328745, "recall": 0.3561937750189817, "f1_score": 0.3142131412798587, "per_field": {"categories": {"precision": 0.2137741046831956, "recall": 0.2825039801796001, "f1_score": 0.22984458914210984}, "scientific_domains": {"precision": 0.33190000000000003, "recall": 0.387, "f1_score": 0.34589891395154554}, "target_users": {"precision": 0.36826666666666663, "recall": 0.399077344877345, "f1_score": 0.3668959207459208}}}, "(0.5, 5, 0.4, 1)": {"precision": 0.3230468319559229, "recall": 0.26740680039275083, "f1_score": 0.2790778228359256, "per_field": {"categories": {"precision": 0.2231404958677686, "recall": 0.2101568369646882, "f1_score": 0.2129396734139817}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.388, "recall": 0.26199213564213564, "f1_score": 0.28719379509379517}}}, "(0.5, 5, 0.4, 3)": {"precision": 0.30504040404040406, "recall": 0.34110315731100854, "f1_score": 0.3066616204354172, "per_field": {"categories": {"precision": 0.21212121212121207, "recall": 0.27446401738757115, "f1_score": 0.22596877869605145}, "scientific_domains": {"precision": 0.3323333333333333, "recall": 0.38492857142857145, "f1_score": 0.3447768440709617}, "target_users": {"precision": 0.3706666666666668, "recall": 0.36391688311688297, "f1_score": 0.34923923853923855}}}, "(0.5, 5, 0.4, 5)": {"precision": 0.30464692378328745, "recall": 0.3561937750189817, "f1_score": 0.3142131412798587, "per_field": {"categories": {"precision": 0.2137741046831956, "recall": 0.2825039801796001, "f1_score": 0.22984458914210984}, "scientific_domains": {"precision": 0.33190000000000003, "recall": 0.387, "f1_score": 0.34589891395154554}, "target_users": {"precision": 0.36826666666666663, "recall": 0.399077344877345, "f1_score": 0.3668959207459208}}}, "(0.5, 5, 0.5, 1)": {"precision": 0.3230468319559229, "recall": 0.26740680039275083, "f1_score": 0.2790778228359256, "per_field": {"categories": {"precision": 0.2231404958677686, "recall": 0.2101568369646882, "f1_score": 0.2129396734139817}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.388, "recall": 0.26199213564213564, "f1_score": 0.28719379509379517}}}, "(0.5, 5, 0.5, 3)": {"precision": 0.30504040404040406, "recall": 0.34110315731100854, "f1_score": 0.3066616204354172, "per_field": {"categories": {"precision": 0.21212121212121207, "recall": 0.27446401738757115, "f1_score": 0.22596877869605145}, "scientific_domains": {"precision": 0.3323333333333333, "recall": 0.38492857142857145, "f1_score": 0.3447768440709617}, "target_users": {"precision": 0.3706666666666668, "recall": 0.36391688311688297, "f1_score": 0.34923923853923855}}}, "(0.5, 5, 0.5, 5)": {"precision": 0.30464692378328745, "recall": 0.3561937750189817, "f1_score": 0.3142131412798587, "per_field": {"categories": {"precision": 0.2137741046831956, "recall": 0.2825039801796001, "f1_score": 0.22984458914210984}, "scientific_domains": {"precision": 0.33190000000000003, "recall": 0.387, "f1_score": 0.34589891395154554}, "target_users": {"precision": 0.36826666666666663, "recall": 0.399077344877345, "f1_score": 0.3668959207459208}}}, "(0.5, 5, 0.6, 1)": {"precision": 0.3230468319559229, "recall": 0.26740680039275083, "f1_score": 0.2790778228359256, "per_field": {"categories": {"precision": 0.2231404958677686, "recall": 0.2101568369646882, "f1_score": 0.2129396734139817}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.388, "recall": 0.26199213564213564, "f1_score": 0.28719379509379517}}}, "(0.5, 5, 0.6, 3)": {"precision": 0.30504040404040406, "recall": 0.34110315731100854, "f1_score": 0.3066616204354172, "per_field": {"categories": {"precision": 0.21212121212121207, "recall": 0.27446401738757115, "f1_score": 0.22596877869605145}, "scientific_domains": {"precision": 0.3323333333333333, "recall": 0.38492857142857145, "f1_score": 0.3447768440709617}, "target_users": {"precision": 0.3706666666666668, "recall": 0.36391688311688297, "f1_score": 0.34923923853923855}}}, "(0.5, 5, 0.6, 5)": {"precision": 0.30464692378328745, "recall": 0.3561937750189817, "f1_score": 0.3142131412798587, "per_field": {"categories": {"precision": 0.2137741046831956, "recall": 0.2825039801796001, "f1_score": 0.22984458914210984}, "scientific_domains": {"precision": 0.33190000000000003, "recall": 0.387, "f1_score": 0.34589891395154554}, "target_users": {"precision": 0.36826666666666663, "recall": 0.399077344877345, "f1_score": 0.3668959207459208}}}, "(0.5, 5, 0.7, 1)": {"precision": 0.3230468319559229, "recall": 0.26740680039275083, "f1_score": 0.2790778228359256, "per_field": {"categories": {"precision": 0.2231404958677686, "recall": 0.2101568369646882, "f1_score": 0.2129396734139817}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.388, "recall": 0.26199213564213564, "f1_score": 0.28719379509379517}}}, "(0.5, 5, 0.7, 3)": {"precision": 0.30504040404040406, "recall": 0.34110315731100854, "f1_score": 0.3066616204354172, "per_field": {"categories": {"precision": 0.21212121212121207, "recall": 0.27446401738757115, "f1_score": 0.22596877869605145}, "scientific_domains": {"precision": 0.3323333333333333, "recall": 0.38492857142857145, "f1_score": 0.3447768440709617}, "target_users": {"precision": 0.3706666666666668, "recall": 0.36391688311688297, "f1_score": 0.34923923853923855}}}, "(0.5, 5, 0.7, 5)": {"precision": 0.30464692378328745, "recall": 0.3561937750189817, "f1_score": 0.3142131412798587, "per_field": {"categories": {"precision": 0.2137741046831956, "recall": 0.2825039801796001, "f1_score": 0.22984458914210984}, "scientific_domains": {"precision": 0.33190000000000003, "recall": 0.387, "f1_score": 0.34589891395154554}, "target_users": {"precision": 0.36826666666666663, "recall": 0.399077344877345, "f1_score": 0.3668959207459208}}}, "(0.5, 5, 0.8, 1)": {"precision": 0.3230468319559229, "recall": 0.26740680039275083, "f1_score": 0.2790778228359256, "per_field": {"categories": {"precision": 0.2231404958677686, "recall": 0.2101568369646882, "f1_score": 0.2129396734139817}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.388, "recall": 0.26199213564213564, "f1_score": 0.28719379509379517}}}, "(0.5, 5, 0.8, 3)": {"precision": 0.30504040404040406, "recall": 0.34110315731100854, "f1_score": 0.3066616204354172, "per_field": {"categories": {"precision": 0.21212121212121207, "recall": 0.27446401738757115, "f1_score": 0.22596877869605145}, "scientific_domains": {"precision": 0.3323333333333333, "recall": 0.38492857142857145, "f1_score": 0.3447768440709617}, "target_users": {"precision": 0.3706666666666668, "recall": 0.36391688311688297, "f1_score": 0.34923923853923855}}}, "(0.5, 5, 0.8, 5)": {"precision": 0.30464692378328745, "recall": 0.3561937750189817, "f1_score": 0.3142131412798587, "per_field": {"categories": {"precision": 0.2137741046831956, "recall": 0.2825039801796001, "f1_score": 0.22984458914210984}, "scientific_domains": {"precision": 0.33190000000000003, "recall": 0.387, "f1_score": 0.34589891395154554}, "target_users": {"precision": 0.36826666666666663, "recall": 0.399077344877345, "f1_score": 0.3668959207459208}}}, "(0.5, 5, 0.9, 1)": {"precision": 0.3230468319559229, "recall": 0.26740680039275083, "f1_score": 0.2790778228359256, "per_field": {"categories": {"precision": 0.2231404958677686, "recall": 0.2101568369646882, "f1_score": 0.2129396734139817}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.388, "recall": 0.26199213564213564, "f1_score": 0.28719379509379517}}}, "(0.5, 5, 0.9, 3)": {"precision": 0.30504040404040406, "recall": 0.34110315731100854, "f1_score": 0.3066616204354172, "per_field": {"categories": {"precision": 0.21212121212121207, "recall": 0.27446401738757115, "f1_score": 0.22596877869605145}, "scientific_domains": {"precision": 0.3323333333333333, "recall": 0.38492857142857145, "f1_score": 0.3447768440709617}, "target_users": {"precision": 0.3706666666666668, "recall": 0.36391688311688297, "f1_score": 0.34923923853923855}}}, "(0.5, 5, 0.9, 5)": {"precision": 0.30464692378328745, "recall": 0.3561937750189817, "f1_score": 0.3142131412798587, "per_field": {"categories": {"precision": 0.2137741046831956, "recall": 0.2825039801796001, "f1_score": 0.22984458914210984}, "scientific_domains": {"precision": 0.33190000000000003, "recall": 0.387, "f1_score": 0.34589891395154554}, "target_users": {"precision": 0.36826666666666663, "recall": 0.399077344877345, "f1_score": 0.3668959207459208}}}, "(0.5, 10, 0, 1)": {"precision": 0.3188484848484849, "recall": 0.26204178661864613, "f1_score": 0.27403503128404316, "per_field": {"categories": {"precision": 0.20454545454545456, "recall": 0.19156179564237416, "f1_score": 0.19434463209166766}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 10, 0, 3)": {"precision": 0.2918126721763085, "recall": 0.3410150030410361, "f1_score": 0.2968188288835348, "per_field": {"categories": {"precision": 0.1907713498622589, "recall": 0.26619955457765376, "f1_score": 0.20737373737373738}, "scientific_domains": {"precision": 0.3256666666666666, "recall": 0.39292857142857146, "f1_score": 0.34144351073762835}, "target_users": {"precision": 0.35900000000000004, "recall": 0.36391688311688297, "f1_score": 0.3416392385392386}}}, "(0.5, 10, 0, 5)": {"precision": 0.2910769513314968, "recall": 0.3588604416856484, "f1_score": 0.3043432737736276, "per_field": {"categories": {"precision": 0.19263085399449037, "recall": 0.2825039801796001, "f1_score": 0.21228260567103546}, "scientific_domains": {"precision": 0.32506666666666667, "recall": 0.395, "f1_score": 0.3423655806182121}, "target_users": {"precision": 0.35553333333333337, "recall": 0.399077344877345, "f1_score": 0.35838163503163506}}}, "(0.5, 10, 0.1, 1)": {"precision": 0.3188484848484849, "recall": 0.26204178661864613, "f1_score": 0.27403503128404316, "per_field": {"categories": {"precision": 0.20454545454545456, "recall": 0.19156179564237416, "f1_score": 0.19434463209166766}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 10, 0.1, 3)": {"precision": 0.2918126721763085, "recall": 0.3410150030410361, "f1_score": 0.2968188288835348, "per_field": {"categories": {"precision": 0.1907713498622589, "recall": 0.26619955457765376, "f1_score": 0.20737373737373738}, "scientific_domains": {"precision": 0.3256666666666666, "recall": 0.39292857142857146, "f1_score": 0.34144351073762835}, "target_users": {"precision": 0.35900000000000004, "recall": 0.36391688311688297, "f1_score": 0.3416392385392386}}}, "(0.5, 10, 0.1, 5)": {"precision": 0.2910769513314968, "recall": 0.3588604416856484, "f1_score": 0.3043432737736276, "per_field": {"categories": {"precision": 0.19263085399449037, "recall": 0.2825039801796001, "f1_score": 0.21228260567103546}, "scientific_domains": {"precision": 0.32506666666666667, "recall": 0.395, "f1_score": 0.3423655806182121}, "target_users": {"precision": 0.35553333333333337, "recall": 0.399077344877345, "f1_score": 0.35838163503163506}}}, "(0.5, 10, 0.2, 1)": {"precision": 0.3188484848484849, "recall": 0.26204178661864613, "f1_score": 0.27403503128404316, "per_field": {"categories": {"precision": 0.20454545454545456, "recall": 0.19156179564237416, "f1_score": 0.19434463209166766}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 10, 0.2, 3)": {"precision": 0.2918126721763085, "recall": 0.3410150030410361, "f1_score": 0.2968188288835348, "per_field": {"categories": {"precision": 0.1907713498622589, "recall": 0.26619955457765376, "f1_score": 0.20737373737373738}, "scientific_domains": {"precision": 0.3256666666666666, "recall": 0.39292857142857146, "f1_score": 0.34144351073762835}, "target_users": {"precision": 0.35900000000000004, "recall": 0.36391688311688297, "f1_score": 0.3416392385392386}}}, "(0.5, 10, 0.2, 5)": {"precision": 0.2910769513314968, "recall": 0.3588604416856484, "f1_score": 0.3043432737736276, "per_field": {"categories": {"precision": 0.19263085399449037, "recall": 0.2825039801796001, "f1_score": 0.21228260567103546}, "scientific_domains": {"precision": 0.32506666666666667, "recall": 0.395, "f1_score": 0.3423655806182121}, "target_users": {"precision": 0.35553333333333337, "recall": 0.399077344877345, "f1_score": 0.35838163503163506}}}, "(0.5, 10, 0.3, 1)": {"precision": 0.3188484848484849, "recall": 0.26204178661864613, "f1_score": 0.27403503128404316, "per_field": {"categories": {"precision": 0.20454545454545456, "recall": 0.19156179564237416, "f1_score": 0.19434463209166766}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 10, 0.3, 3)": {"precision": 0.2918126721763085, "recall": 0.3410150030410361, "f1_score": 0.2968188288835348, "per_field": {"categories": {"precision": 0.1907713498622589, "recall": 0.26619955457765376, "f1_score": 0.20737373737373738}, "scientific_domains": {"precision": 0.3256666666666666, "recall": 0.39292857142857146, "f1_score": 0.34144351073762835}, "target_users": {"precision": 0.35900000000000004, "recall": 0.36391688311688297, "f1_score": 0.3416392385392386}}}, "(0.5, 10, 0.3, 5)": {"precision": 0.2910769513314968, "recall": 0.3588604416856484, "f1_score": 0.3043432737736276, "per_field": {"categories": {"precision": 0.19263085399449037, "recall": 0.2825039801796001, "f1_score": 0.21228260567103546}, "scientific_domains": {"precision": 0.32506666666666667, "recall": 0.395, "f1_score": 0.3423655806182121}, "target_users": {"precision": 0.35553333333333337, "recall": 0.399077344877345, "f1_score": 0.35838163503163506}}}, "(0.5, 10, 0.4, 1)": {"precision": 0.3188484848484849, "recall": 0.26204178661864613, "f1_score": 0.27403503128404316, "per_field": {"categories": {"precision": 0.20454545454545456, "recall": 0.19156179564237416, "f1_score": 0.19434463209166766}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 10, 0.4, 3)": {"precision": 0.2918126721763085, "recall": 0.3410150030410361, "f1_score": 0.2968188288835348, "per_field": {"categories": {"precision": 0.1907713498622589, "recall": 0.26619955457765376, "f1_score": 0.20737373737373738}, "scientific_domains": {"precision": 0.3256666666666666, "recall": 0.39292857142857146, "f1_score": 0.34144351073762835}, "target_users": {"precision": 0.35900000000000004, "recall": 0.36391688311688297, "f1_score": 0.3416392385392386}}}, "(0.5, 10, 0.4, 5)": {"precision": 0.2910769513314968, "recall": 0.3588604416856484, "f1_score": 0.3043432737736276, "per_field": {"categories": {"precision": 0.19263085399449037, "recall": 0.2825039801796001, "f1_score": 0.21228260567103546}, "scientific_domains": {"precision": 0.32506666666666667, "recall": 0.395, "f1_score": 0.3423655806182121}, "target_users": {"precision": 0.35553333333333337, "recall": 0.399077344877345, "f1_score": 0.35838163503163506}}}, "(0.5, 10, 0.5, 1)": {"precision": 0.3188484848484849, "recall": 0.26204178661864613, "f1_score": 0.27403503128404316, "per_field": {"categories": {"precision": 0.20454545454545456, "recall": 0.19156179564237416, "f1_score": 0.19434463209166766}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 10, 0.5, 3)": {"precision": 0.2918126721763085, "recall": 0.3410150030410361, "f1_score": 0.2968188288835348, "per_field": {"categories": {"precision": 0.1907713498622589, "recall": 0.26619955457765376, "f1_score": 0.20737373737373738}, "scientific_domains": {"precision": 0.3256666666666666, "recall": 0.39292857142857146, "f1_score": 0.34144351073762835}, "target_users": {"precision": 0.35900000000000004, "recall": 0.36391688311688297, "f1_score": 0.3416392385392386}}}, "(0.5, 10, 0.5, 5)": {"precision": 0.2910769513314968, "recall": 0.3588604416856484, "f1_score": 0.3043432737736276, "per_field": {"categories": {"precision": 0.19263085399449037, "recall": 0.2825039801796001, "f1_score": 0.21228260567103546}, "scientific_domains": {"precision": 0.32506666666666667, "recall": 0.395, "f1_score": 0.3423655806182121}, "target_users": {"precision": 0.35553333333333337, "recall": 0.399077344877345, "f1_score": 0.35838163503163506}}}, "(0.5, 10, 0.6, 1)": {"precision": 0.3188484848484849, "recall": 0.26204178661864613, "f1_score": 0.27403503128404316, "per_field": {"categories": {"precision": 0.20454545454545456, "recall": 0.19156179564237416, "f1_score": 0.19434463209166766}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 10, 0.6, 3)": {"precision": 0.2918126721763085, "recall": 0.3410150030410361, "f1_score": 0.2968188288835348, "per_field": {"categories": {"precision": 0.1907713498622589, "recall": 0.26619955457765376, "f1_score": 0.20737373737373738}, "scientific_domains": {"precision": 0.3256666666666666, "recall": 0.39292857142857146, "f1_score": 0.34144351073762835}, "target_users": {"precision": 0.35900000000000004, "recall": 0.36391688311688297, "f1_score": 0.3416392385392386}}}, "(0.5, 10, 0.6, 5)": {"precision": 0.2910769513314968, "recall": 0.3588604416856484, "f1_score": 0.3043432737736276, "per_field": {"categories": {"precision": 0.19263085399449037, "recall": 0.2825039801796001, "f1_score": 0.21228260567103546}, "scientific_domains": {"precision": 0.32506666666666667, "recall": 0.395, "f1_score": 0.3423655806182121}, "target_users": {"precision": 0.35553333333333337, "recall": 0.399077344877345, "f1_score": 0.35838163503163506}}}, "(0.5, 10, 0.7, 1)": {"precision": 0.3188484848484849, "recall": 0.26204178661864613, "f1_score": 0.27403503128404316, "per_field": {"categories": {"precision": 0.20454545454545456, "recall": 0.19156179564237416, "f1_score": 0.19434463209166766}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 10, 0.7, 3)": {"precision": 0.2918126721763085, "recall": 0.3410150030410361, "f1_score": 0.2968188288835348, "per_field": {"categories": {"precision": 0.1907713498622589, "recall": 0.26619955457765376, "f1_score": 0.20737373737373738}, "scientific_domains": {"precision": 0.3256666666666666, "recall": 0.39292857142857146, "f1_score": 0.34144351073762835}, "target_users": {"precision": 0.35900000000000004, "recall": 0.36391688311688297, "f1_score": 0.3416392385392386}}}, "(0.5, 10, 0.7, 5)": {"precision": 0.2910769513314968, "recall": 0.3588604416856484, "f1_score": 0.3043432737736276, "per_field": {"categories": {"precision": 0.19263085399449037, "recall": 0.2825039801796001, "f1_score": 0.21228260567103546}, "scientific_domains": {"precision": 0.32506666666666667, "recall": 0.395, "f1_score": 0.3423655806182121}, "target_users": {"precision": 0.35553333333333337, "recall": 0.399077344877345, "f1_score": 0.35838163503163506}}}, "(0.5, 10, 0.8, 1)": {"precision": 0.3188484848484849, "recall": 0.26204178661864613, "f1_score": 0.27403503128404316, "per_field": {"categories": {"precision": 0.20454545454545456, "recall": 0.19156179564237416, "f1_score": 0.19434463209166766}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 10, 0.8, 3)": {"precision": 0.2918126721763085, "recall": 0.3410150030410361, "f1_score": 0.2968188288835348, "per_field": {"categories": {"precision": 0.1907713498622589, "recall": 0.26619955457765376, "f1_score": 0.20737373737373738}, "scientific_domains": {"precision": 0.3256666666666666, "recall": 0.39292857142857146, "f1_score": 0.34144351073762835}, "target_users": {"precision": 0.35900000000000004, "recall": 0.36391688311688297, "f1_score": 0.3416392385392386}}}, "(0.5, 10, 0.8, 5)": {"precision": 0.2910769513314968, "recall": 0.3588604416856484, "f1_score": 0.3043432737736276, "per_field": {"categories": {"precision": 0.19263085399449037, "recall": 0.2825039801796001, "f1_score": 0.21228260567103546}, "scientific_domains": {"precision": 0.32506666666666667, "recall": 0.395, "f1_score": 0.3423655806182121}, "target_users": {"precision": 0.35553333333333337, "recall": 0.399077344877345, "f1_score": 0.35838163503163506}}}, "(0.5, 10, 0.9, 1)": {"precision": 0.3188484848484849, "recall": 0.26204178661864613, "f1_score": 0.27403503128404316, "per_field": {"categories": {"precision": 0.20454545454545456, "recall": 0.19156179564237416, "f1_score": 0.19434463209166766}, "scientific_domains": {"precision": 0.358, "recall": 0.33007142857142857, "f1_score": 0.33709999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 10, 0.9, 3)": {"precision": 0.2918126721763085, "recall": 0.3410150030410361, "f1_score": 0.2968188288835348, "per_field": {"categories": {"precision": 0.1907713498622589, "recall": 0.26619955457765376, "f1_score": 0.20737373737373738}, "scientific_domains": {"precision": 0.3256666666666666, "recall": 0.39292857142857146, "f1_score": 0.34144351073762835}, "target_users": {"precision": 0.35900000000000004, "recall": 0.36391688311688297, "f1_score": 0.3416392385392386}}}, "(0.5, 10, 0.9, 5)": {"precision": 0.2910769513314968, "recall": 0.3588604416856484, "f1_score": 0.3043432737736276, "per_field": {"categories": {"precision": 0.19263085399449037, "recall": 0.2825039801796001, "f1_score": 0.21228260567103546}, "scientific_domains": {"precision": 0.32506666666666667, "recall": 0.395, "f1_score": 0.3423655806182121}, "target_users": {"precision": 0.35553333333333337, "recall": 0.399077344877345, "f1_score": 0.35838163503163506}}}, "(0.5, 15, 0, 1)": {"precision": 0.3160716253443526, "recall": 0.25926492711451393, "f1_score": 0.2712581717799109, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18123121712997747, "f1_score": 0.18401405357927098}, "scientific_domains": {"precision": 0.36, "recall": 0.3320714285714286, "f1_score": 0.33909999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 15, 0, 3)": {"precision": 0.29807162534435266, "recall": 0.338992964473543, "f1_score": 0.3012926580846366, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.26413343887517443, "f1_score": 0.21012855831037647}, "scientific_domains": {"precision": 0.329, "recall": 0.38892857142857146, "f1_score": 0.34411017740429495}, "target_users": {"precision": 0.37100000000000005, "recall": 0.36391688311688297, "f1_score": 0.3496392385392385}}}, "(0.5, 15, 0, 5)": {"precision": 0.2974832874196511, "recall": 0.3568384031181553, "f1_score": 0.30899855384708946, "per_field": {"categories": {"precision": 0.19634986225895315, "recall": 0.28043786447712077, "f1_score": 0.21538177922475443}, "scientific_domains": {"precision": 0.3285666666666667, "recall": 0.391, "f1_score": 0.34523224728487883}, "target_users": {"precision": 0.3675333333333333, "recall": 0.399077344877345, "f1_score": 0.36638163503163507}}}, "(0.5, 15, 0.1, 1)": {"precision": 0.3160716253443526, "recall": 0.25926492711451393, "f1_score": 0.2712581717799109, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18123121712997747, "f1_score": 0.18401405357927098}, "scientific_domains": {"precision": 0.36, "recall": 0.3320714285714286, "f1_score": 0.33909999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 15, 0.1, 3)": {"precision": 0.29807162534435266, "recall": 0.338992964473543, "f1_score": 0.3012926580846366, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.26413343887517443, "f1_score": 0.21012855831037647}, "scientific_domains": {"precision": 0.329, "recall": 0.38892857142857146, "f1_score": 0.34411017740429495}, "target_users": {"precision": 0.37100000000000005, "recall": 0.36391688311688297, "f1_score": 0.3496392385392385}}}, "(0.5, 15, 0.1, 5)": {"precision": 0.2974832874196511, "recall": 0.3568384031181553, "f1_score": 0.30899855384708946, "per_field": {"categories": {"precision": 0.19634986225895315, "recall": 0.28043786447712077, "f1_score": 0.21538177922475443}, "scientific_domains": {"precision": 0.3285666666666667, "recall": 0.391, "f1_score": 0.34523224728487883}, "target_users": {"precision": 0.3675333333333333, "recall": 0.399077344877345, "f1_score": 0.36638163503163507}}}, "(0.5, 15, 0.2, 1)": {"precision": 0.3160716253443526, "recall": 0.25926492711451393, "f1_score": 0.2712581717799109, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18123121712997747, "f1_score": 0.18401405357927098}, "scientific_domains": {"precision": 0.36, "recall": 0.3320714285714286, "f1_score": 0.33909999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 15, 0.2, 3)": {"precision": 0.29807162534435266, "recall": 0.338992964473543, "f1_score": 0.3012926580846366, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.26413343887517443, "f1_score": 0.21012855831037647}, "scientific_domains": {"precision": 0.329, "recall": 0.38892857142857146, "f1_score": 0.34411017740429495}, "target_users": {"precision": 0.37100000000000005, "recall": 0.36391688311688297, "f1_score": 0.3496392385392385}}}, "(0.5, 15, 0.2, 5)": {"precision": 0.2974832874196511, "recall": 0.3568384031181553, "f1_score": 0.30899855384708946, "per_field": {"categories": {"precision": 0.19634986225895315, "recall": 0.28043786447712077, "f1_score": 0.21538177922475443}, "scientific_domains": {"precision": 0.3285666666666667, "recall": 0.391, "f1_score": 0.34523224728487883}, "target_users": {"precision": 0.3675333333333333, "recall": 0.399077344877345, "f1_score": 0.36638163503163507}}}, "(0.5, 15, 0.3, 1)": {"precision": 0.3160716253443526, "recall": 0.25926492711451393, "f1_score": 0.2712581717799109, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18123121712997747, "f1_score": 0.18401405357927098}, "scientific_domains": {"precision": 0.36, "recall": 0.3320714285714286, "f1_score": 0.33909999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 15, 0.3, 3)": {"precision": 0.29807162534435266, "recall": 0.338992964473543, "f1_score": 0.3012926580846366, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.26413343887517443, "f1_score": 0.21012855831037647}, "scientific_domains": {"precision": 0.329, "recall": 0.38892857142857146, "f1_score": 0.34411017740429495}, "target_users": {"precision": 0.37100000000000005, "recall": 0.36391688311688297, "f1_score": 0.3496392385392385}}}, "(0.5, 15, 0.3, 5)": {"precision": 0.2974832874196511, "recall": 0.3568384031181553, "f1_score": 0.30899855384708946, "per_field": {"categories": {"precision": 0.19634986225895315, "recall": 0.28043786447712077, "f1_score": 0.21538177922475443}, "scientific_domains": {"precision": 0.3285666666666667, "recall": 0.391, "f1_score": 0.34523224728487883}, "target_users": {"precision": 0.3675333333333333, "recall": 0.399077344877345, "f1_score": 0.36638163503163507}}}, "(0.5, 15, 0.4, 1)": {"precision": 0.3160716253443526, "recall": 0.25926492711451393, "f1_score": 0.2712581717799109, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18123121712997747, "f1_score": 0.18401405357927098}, "scientific_domains": {"precision": 0.36, "recall": 0.3320714285714286, "f1_score": 0.33909999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 15, 0.4, 3)": {"precision": 0.29807162534435266, "recall": 0.338992964473543, "f1_score": 0.3012926580846366, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.26413343887517443, "f1_score": 0.21012855831037647}, "scientific_domains": {"precision": 0.329, "recall": 0.38892857142857146, "f1_score": 0.34411017740429495}, "target_users": {"precision": 0.37100000000000005, "recall": 0.36391688311688297, "f1_score": 0.3496392385392385}}}, "(0.5, 15, 0.4, 5)": {"precision": 0.2974832874196511, "recall": 0.3568384031181553, "f1_score": 0.30899855384708946, "per_field": {"categories": {"precision": 0.19634986225895315, "recall": 0.28043786447712077, "f1_score": 0.21538177922475443}, "scientific_domains": {"precision": 0.3285666666666667, "recall": 0.391, "f1_score": 0.34523224728487883}, "target_users": {"precision": 0.3675333333333333, "recall": 0.399077344877345, "f1_score": 0.36638163503163507}}}, "(0.5, 15, 0.5, 1)": {"precision": 0.3160716253443526, "recall": 0.25926492711451393, "f1_score": 0.2712581717799109, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18123121712997747, "f1_score": 0.18401405357927098}, "scientific_domains": {"precision": 0.36, "recall": 0.3320714285714286, "f1_score": 0.33909999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 15, 0.5, 3)": {"precision": 0.29807162534435266, "recall": 0.338992964473543, "f1_score": 0.3012926580846366, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.26413343887517443, "f1_score": 0.21012855831037647}, "scientific_domains": {"precision": 0.329, "recall": 0.38892857142857146, "f1_score": 0.34411017740429495}, "target_users": {"precision": 0.37100000000000005, "recall": 0.36391688311688297, "f1_score": 0.3496392385392385}}}, "(0.5, 15, 0.5, 5)": {"precision": 0.2974832874196511, "recall": 0.3568384031181553, "f1_score": 0.30899855384708946, "per_field": {"categories": {"precision": 0.19634986225895315, "recall": 0.28043786447712077, "f1_score": 0.21538177922475443}, "scientific_domains": {"precision": 0.3285666666666667, "recall": 0.391, "f1_score": 0.34523224728487883}, "target_users": {"precision": 0.3675333333333333, "recall": 0.399077344877345, "f1_score": 0.36638163503163507}}}, "(0.5, 15, 0.6, 1)": {"precision": 0.3160716253443526, "recall": 0.25926492711451393, "f1_score": 0.2712581717799109, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18123121712997747, "f1_score": 0.18401405357927098}, "scientific_domains": {"precision": 0.36, "recall": 0.3320714285714286, "f1_score": 0.33909999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 15, 0.6, 3)": {"precision": 0.29807162534435266, "recall": 0.338992964473543, "f1_score": 0.3012926580846366, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.26413343887517443, "f1_score": 0.21012855831037647}, "scientific_domains": {"precision": 0.329, "recall": 0.38892857142857146, "f1_score": 0.34411017740429495}, "target_users": {"precision": 0.37100000000000005, "recall": 0.36391688311688297, "f1_score": 0.3496392385392385}}}, "(0.5, 15, 0.6, 5)": {"precision": 0.2974832874196511, "recall": 0.3568384031181553, "f1_score": 0.30899855384708946, "per_field": {"categories": {"precision": 0.19634986225895315, "recall": 0.28043786447712077, "f1_score": 0.21538177922475443}, "scientific_domains": {"precision": 0.3285666666666667, "recall": 0.391, "f1_score": 0.34523224728487883}, "target_users": {"precision": 0.3675333333333333, "recall": 0.399077344877345, "f1_score": 0.36638163503163507}}}, "(0.5, 15, 0.7, 1)": {"precision": 0.3160716253443526, "recall": 0.25926492711451393, "f1_score": 0.2712581717799109, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18123121712997747, "f1_score": 0.18401405357927098}, "scientific_domains": {"precision": 0.36, "recall": 0.3320714285714286, "f1_score": 0.33909999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 15, 0.7, 3)": {"precision": 0.29807162534435266, "recall": 0.338992964473543, "f1_score": 0.3012926580846366, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.26413343887517443, "f1_score": 0.21012855831037647}, "scientific_domains": {"precision": 0.329, "recall": 0.38892857142857146, "f1_score": 0.34411017740429495}, "target_users": {"precision": 0.37100000000000005, "recall": 0.36391688311688297, "f1_score": 0.3496392385392385}}}, "(0.5, 15, 0.7, 5)": {"precision": 0.2974832874196511, "recall": 0.3568384031181553, "f1_score": 0.30899855384708946, "per_field": {"categories": {"precision": 0.19634986225895315, "recall": 0.28043786447712077, "f1_score": 0.21538177922475443}, "scientific_domains": {"precision": 0.3285666666666667, "recall": 0.391, "f1_score": 0.34523224728487883}, "target_users": {"precision": 0.3675333333333333, "recall": 0.399077344877345, "f1_score": 0.36638163503163507}}}, "(0.5, 15, 0.8, 1)": {"precision": 0.3160716253443526, "recall": 0.25926492711451393, "f1_score": 0.2712581717799109, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18123121712997747, "f1_score": 0.18401405357927098}, "scientific_domains": {"precision": 0.36, "recall": 0.3320714285714286, "f1_score": 0.33909999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 15, 0.8, 3)": {"precision": 0.29807162534435266, "recall": 0.338992964473543, "f1_score": 0.3012926580846366, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.26413343887517443, "f1_score": 0.21012855831037647}, "scientific_domains": {"precision": 0.329, "recall": 0.38892857142857146, "f1_score": 0.34411017740429495}, "target_users": {"precision": 0.37100000000000005, "recall": 0.36391688311688297, "f1_score": 0.3496392385392385}}}, "(0.5, 15, 0.8, 5)": {"precision": 0.2974832874196511, "recall": 0.3568384031181553, "f1_score": 0.30899855384708946, "per_field": {"categories": {"precision": 0.19634986225895315, "recall": 0.28043786447712077, "f1_score": 0.21538177922475443}, "scientific_domains": {"precision": 0.3285666666666667, "recall": 0.391, "f1_score": 0.34523224728487883}, "target_users": {"precision": 0.3675333333333333, "recall": 0.399077344877345, "f1_score": 0.36638163503163507}}}, "(0.5, 15, 0.9, 1)": {"precision": 0.3160716253443526, "recall": 0.25926492711451393, "f1_score": 0.2712581717799109, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18123121712997747, "f1_score": 0.18401405357927098}, "scientific_domains": {"precision": 0.36, "recall": 0.3320714285714286, "f1_score": 0.33909999999999996}, "target_users": {"precision": 0.394, "recall": 0.26449213564213564, "f1_score": 0.29066046176046184}}}, "(0.5, 15, 0.9, 3)": {"precision": 0.29807162534435266, "recall": 0.338992964473543, "f1_score": 0.3012926580846366, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.26413343887517443, "f1_score": 0.21012855831037647}, "scientific_domains": {"precision": 0.329, "recall": 0.38892857142857146, "f1_score": 0.34411017740429495}, "target_users": {"precision": 0.37100000000000005, "recall": 0.36391688311688297, "f1_score": 0.3496392385392385}}}, "(0.5, 15, 0.9, 5)": {"precision": 0.2974832874196511, "recall": 0.3568384031181553, "f1_score": 0.30899855384708946, "per_field": {"categories": {"precision": 0.19634986225895315, "recall": 0.28043786447712077, "f1_score": 0.21538177922475443}, "scientific_domains": {"precision": 0.3285666666666667, "recall": 0.391, "f1_score": 0.34523224728487883}, "target_users": {"precision": 0.3675333333333333, "recall": 0.399077344877345, "f1_score": 0.36638163503163507}}}, "(0.6, 5, 0, 1)": {"precision": 0.27540495867768594, "recall": 0.23455512782289642, "f1_score": 0.2436548135220072, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18433039068369647, "f1_score": 0.1860801692817503}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.326, "recall": 0.23754927849927854, "f1_score": 0.25631760461760467}}}, "(0.6, 5, 0, 3)": {"precision": 0.26435353535353534, "recall": 0.29455198323256176, "f1_score": 0.26662401571492483, "per_field": {"categories": {"precision": 0.18939393939393934, "recall": 0.23945811419984975, "f1_score": 0.2001423324150597}, "scientific_domains": {"precision": 0.285, "recall": 0.32807142857142857, "f1_score": 0.29603174603174615}, "target_users": {"precision": 0.3186666666666667, "recall": 0.31612640692640687, "f1_score": 0.3036979686979687}}}, "(0.6, 5, 0, 5)": {"precision": 0.26492938475665745, "recall": 0.3041398461195982, "f1_score": 0.2718135235695842, "per_field": {"categories": {"precision": 0.1902548209366391, "recall": 0.24396600300525922, "f1_score": 0.20212733394551574}, "scientific_domains": {"precision": 0.28473333333333334, "recall": 0.32957142857142857, "f1_score": 0.29666031746031757}, "target_users": {"precision": 0.31980000000000003, "recall": 0.33888210678210684, "f1_score": 0.3166529193029194}}}, "(0.6, 5, 0.1, 1)": {"precision": 0.27540495867768594, "recall": 0.23455512782289642, "f1_score": 0.2436548135220072, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18433039068369647, "f1_score": 0.1860801692817503}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.326, "recall": 0.23754927849927854, "f1_score": 0.25631760461760467}}}, "(0.6, 5, 0.1, 3)": {"precision": 0.26435353535353534, "recall": 0.29455198323256176, "f1_score": 0.26662401571492483, "per_field": {"categories": {"precision": 0.18939393939393934, "recall": 0.23945811419984975, "f1_score": 0.2001423324150597}, "scientific_domains": {"precision": 0.285, "recall": 0.32807142857142857, "f1_score": 0.29603174603174615}, "target_users": {"precision": 0.3186666666666667, "recall": 0.31612640692640687, "f1_score": 0.3036979686979687}}}, "(0.6, 5, 0.1, 5)": {"precision": 0.26492938475665745, "recall": 0.3041398461195982, "f1_score": 0.2718135235695842, "per_field": {"categories": {"precision": 0.1902548209366391, "recall": 0.24396600300525922, "f1_score": 0.20212733394551574}, "scientific_domains": {"precision": 0.28473333333333334, "recall": 0.32957142857142857, "f1_score": 0.29666031746031757}, "target_users": {"precision": 0.31980000000000003, "recall": 0.33888210678210684, "f1_score": 0.3166529193029194}}}, "(0.6, 5, 0.2, 1)": {"precision": 0.27540495867768594, "recall": 0.23455512782289642, "f1_score": 0.2436548135220072, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18433039068369647, "f1_score": 0.1860801692817503}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.326, "recall": 0.23754927849927854, "f1_score": 0.25631760461760467}}}, "(0.6, 5, 0.2, 3)": {"precision": 0.26435353535353534, "recall": 0.29455198323256176, "f1_score": 0.26662401571492483, "per_field": {"categories": {"precision": 0.18939393939393934, "recall": 0.23945811419984975, "f1_score": 0.2001423324150597}, "scientific_domains": {"precision": 0.285, "recall": 0.32807142857142857, "f1_score": 0.29603174603174615}, "target_users": {"precision": 0.3186666666666667, "recall": 0.31612640692640687, "f1_score": 0.3036979686979687}}}, "(0.6, 5, 0.2, 5)": {"precision": 0.26492938475665745, "recall": 0.3041398461195982, "f1_score": 0.2718135235695842, "per_field": {"categories": {"precision": 0.1902548209366391, "recall": 0.24396600300525922, "f1_score": 0.20212733394551574}, "scientific_domains": {"precision": 0.28473333333333334, "recall": 0.32957142857142857, "f1_score": 0.29666031746031757}, "target_users": {"precision": 0.31980000000000003, "recall": 0.33888210678210684, "f1_score": 0.3166529193029194}}}, "(0.6, 5, 0.3, 1)": {"precision": 0.27540495867768594, "recall": 0.23455512782289642, "f1_score": 0.2436548135220072, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18433039068369647, "f1_score": 0.1860801692817503}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.326, "recall": 0.23754927849927854, "f1_score": 0.25631760461760467}}}, "(0.6, 5, 0.3, 3)": {"precision": 0.26435353535353534, "recall": 0.29455198323256176, "f1_score": 0.26662401571492483, "per_field": {"categories": {"precision": 0.18939393939393934, "recall": 0.23945811419984975, "f1_score": 0.2001423324150597}, "scientific_domains": {"precision": 0.285, "recall": 0.32807142857142857, "f1_score": 0.29603174603174615}, "target_users": {"precision": 0.3186666666666667, "recall": 0.31612640692640687, "f1_score": 0.3036979686979687}}}, "(0.6, 5, 0.3, 5)": {"precision": 0.26492938475665745, "recall": 0.3041398461195982, "f1_score": 0.2718135235695842, "per_field": {"categories": {"precision": 0.1902548209366391, "recall": 0.24396600300525922, "f1_score": 0.20212733394551574}, "scientific_domains": {"precision": 0.28473333333333334, "recall": 0.32957142857142857, "f1_score": 0.29666031746031757}, "target_users": {"precision": 0.31980000000000003, "recall": 0.33888210678210684, "f1_score": 0.3166529193029194}}}, "(0.6, 5, 0.4, 1)": {"precision": 0.27540495867768594, "recall": 0.23455512782289642, "f1_score": 0.2436548135220072, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18433039068369647, "f1_score": 0.1860801692817503}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.326, "recall": 0.23754927849927854, "f1_score": 0.25631760461760467}}}, "(0.6, 5, 0.4, 3)": {"precision": 0.26435353535353534, "recall": 0.29455198323256176, "f1_score": 0.26662401571492483, "per_field": {"categories": {"precision": 0.18939393939393934, "recall": 0.23945811419984975, "f1_score": 0.2001423324150597}, "scientific_domains": {"precision": 0.285, "recall": 0.32807142857142857, "f1_score": 0.29603174603174615}, "target_users": {"precision": 0.3186666666666667, "recall": 0.31612640692640687, "f1_score": 0.3036979686979687}}}, "(0.6, 5, 0.4, 5)": {"precision": 0.26492938475665745, "recall": 0.3041398461195982, "f1_score": 0.2718135235695842, "per_field": {"categories": {"precision": 0.1902548209366391, "recall": 0.24396600300525922, "f1_score": 0.20212733394551574}, "scientific_domains": {"precision": 0.28473333333333334, "recall": 0.32957142857142857, "f1_score": 0.29666031746031757}, "target_users": {"precision": 0.31980000000000003, "recall": 0.33888210678210684, "f1_score": 0.3166529193029194}}}, "(0.6, 5, 0.5, 1)": {"precision": 0.27540495867768594, "recall": 0.23455512782289642, "f1_score": 0.2436548135220072, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18433039068369647, "f1_score": 0.1860801692817503}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.326, "recall": 0.23754927849927854, "f1_score": 0.25631760461760467}}}, "(0.6, 5, 0.5, 3)": {"precision": 0.26435353535353534, "recall": 0.29455198323256176, "f1_score": 0.26662401571492483, "per_field": {"categories": {"precision": 0.18939393939393934, "recall": 0.23945811419984975, "f1_score": 0.2001423324150597}, "scientific_domains": {"precision": 0.285, "recall": 0.32807142857142857, "f1_score": 0.29603174603174615}, "target_users": {"precision": 0.3186666666666667, "recall": 0.31612640692640687, "f1_score": 0.3036979686979687}}}, "(0.6, 5, 0.5, 5)": {"precision": 0.26492938475665745, "recall": 0.3041398461195982, "f1_score": 0.2718135235695842, "per_field": {"categories": {"precision": 0.1902548209366391, "recall": 0.24396600300525922, "f1_score": 0.20212733394551574}, "scientific_domains": {"precision": 0.28473333333333334, "recall": 0.32957142857142857, "f1_score": 0.29666031746031757}, "target_users": {"precision": 0.31980000000000003, "recall": 0.33888210678210684, "f1_score": 0.3166529193029194}}}, "(0.6, 5, 0.6, 1)": {"precision": 0.27540495867768594, "recall": 0.23455512782289642, "f1_score": 0.2436548135220072, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18433039068369647, "f1_score": 0.1860801692817503}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.326, "recall": 0.23754927849927854, "f1_score": 0.25631760461760467}}}, "(0.6, 5, 0.6, 3)": {"precision": 0.26435353535353534, "recall": 0.29455198323256176, "f1_score": 0.26662401571492483, "per_field": {"categories": {"precision": 0.18939393939393934, "recall": 0.23945811419984975, "f1_score": 0.2001423324150597}, "scientific_domains": {"precision": 0.285, "recall": 0.32807142857142857, "f1_score": 0.29603174603174615}, "target_users": {"precision": 0.3186666666666667, "recall": 0.31612640692640687, "f1_score": 0.3036979686979687}}}, "(0.6, 5, 0.6, 5)": {"precision": 0.26492938475665745, "recall": 0.3041398461195982, "f1_score": 0.2718135235695842, "per_field": {"categories": {"precision": 0.1902548209366391, "recall": 0.24396600300525922, "f1_score": 0.20212733394551574}, "scientific_domains": {"precision": 0.28473333333333334, "recall": 0.32957142857142857, "f1_score": 0.29666031746031757}, "target_users": {"precision": 0.31980000000000003, "recall": 0.33888210678210684, "f1_score": 0.3166529193029194}}}, "(0.6, 5, 0.7, 1)": {"precision": 0.27540495867768594, "recall": 0.23455512782289642, "f1_score": 0.2436548135220072, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18433039068369647, "f1_score": 0.1860801692817503}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.326, "recall": 0.23754927849927854, "f1_score": 0.25631760461760467}}}, "(0.6, 5, 0.7, 3)": {"precision": 0.26435353535353534, "recall": 0.29455198323256176, "f1_score": 0.26662401571492483, "per_field": {"categories": {"precision": 0.18939393939393934, "recall": 0.23945811419984975, "f1_score": 0.2001423324150597}, "scientific_domains": {"precision": 0.285, "recall": 0.32807142857142857, "f1_score": 0.29603174603174615}, "target_users": {"precision": 0.3186666666666667, "recall": 0.31612640692640687, "f1_score": 0.3036979686979687}}}, "(0.6, 5, 0.7, 5)": {"precision": 0.26492938475665745, "recall": 0.3041398461195982, "f1_score": 0.2718135235695842, "per_field": {"categories": {"precision": 0.1902548209366391, "recall": 0.24396600300525922, "f1_score": 0.20212733394551574}, "scientific_domains": {"precision": 0.28473333333333334, "recall": 0.32957142857142857, "f1_score": 0.29666031746031757}, "target_users": {"precision": 0.31980000000000003, "recall": 0.33888210678210684, "f1_score": 0.3166529193029194}}}, "(0.6, 5, 0.8, 1)": {"precision": 0.27540495867768594, "recall": 0.23455512782289642, "f1_score": 0.2436548135220072, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18433039068369647, "f1_score": 0.1860801692817503}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.326, "recall": 0.23754927849927854, "f1_score": 0.25631760461760467}}}, "(0.6, 5, 0.8, 3)": {"precision": 0.26435353535353534, "recall": 0.29455198323256176, "f1_score": 0.26662401571492483, "per_field": {"categories": {"precision": 0.18939393939393934, "recall": 0.23945811419984975, "f1_score": 0.2001423324150597}, "scientific_domains": {"precision": 0.285, "recall": 0.32807142857142857, "f1_score": 0.29603174603174615}, "target_users": {"precision": 0.3186666666666667, "recall": 0.31612640692640687, "f1_score": 0.3036979686979687}}}, "(0.6, 5, 0.8, 5)": {"precision": 0.26492938475665745, "recall": 0.3041398461195982, "f1_score": 0.2718135235695842, "per_field": {"categories": {"precision": 0.1902548209366391, "recall": 0.24396600300525922, "f1_score": 0.20212733394551574}, "scientific_domains": {"precision": 0.28473333333333334, "recall": 0.32957142857142857, "f1_score": 0.29666031746031757}, "target_users": {"precision": 0.31980000000000003, "recall": 0.33888210678210684, "f1_score": 0.3166529193029194}}}, "(0.6, 5, 0.9, 1)": {"precision": 0.27540495867768594, "recall": 0.23455512782289642, "f1_score": 0.2436548135220072, "per_field": {"categories": {"precision": 0.19421487603305784, "recall": 0.18433039068369647, "f1_score": 0.1860801692817503}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.326, "recall": 0.23754927849927854, "f1_score": 0.25631760461760467}}}, "(0.6, 5, 0.9, 3)": {"precision": 0.26435353535353534, "recall": 0.29455198323256176, "f1_score": 0.26662401571492483, "per_field": {"categories": {"precision": 0.18939393939393934, "recall": 0.23945811419984975, "f1_score": 0.2001423324150597}, "scientific_domains": {"precision": 0.285, "recall": 0.32807142857142857, "f1_score": 0.29603174603174615}, "target_users": {"precision": 0.3186666666666667, "recall": 0.31612640692640687, "f1_score": 0.3036979686979687}}}, "(0.6, 5, 0.9, 5)": {"precision": 0.26492938475665745, "recall": 0.3041398461195982, "f1_score": 0.2718135235695842, "per_field": {"categories": {"precision": 0.1902548209366391, "recall": 0.24396600300525922, "f1_score": 0.20212733394551574}, "scientific_domains": {"precision": 0.28473333333333334, "recall": 0.32957142857142857, "f1_score": 0.29666031746031757}, "target_users": {"precision": 0.31980000000000003, "recall": 0.33888210678210684, "f1_score": 0.3166529193029194}}}, "(0.6, 10, 0, 1)": {"precision": 0.27122865013774106, "recall": 0.22971215261628486, "f1_score": 0.2390340605376179, "per_field": {"categories": {"precision": 0.17768595041322313, "recall": 0.16780146506386176, "f1_score": 0.16955124366191562}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 10, 0, 3)": {"precision": 0.2554214876033058, "recall": 0.2951525341967491, "f1_score": 0.25999554923191287, "per_field": {"categories": {"precision": 0.174931129476584, "recall": 0.23325976709241172, "f1_score": 0.18705693296602383}, "scientific_domains": {"precision": 0.281, "recall": 0.3360714285714286, "f1_score": 0.2946984126984128}, "target_users": {"precision": 0.3103333333333333, "recall": 0.31612640692640687, "f1_score": 0.2982313020313021}}}, "(0.6, 10, 0, 5)": {"precision": 0.25633204775022955, "recall": 0.3074952180204247, "f1_score": 0.26585300933634276, "per_field": {"categories": {"precision": 0.17696280991735538, "recall": 0.24603211870773856, "f1_score": 0.19124579124579127}, "scientific_domains": {"precision": 0.2805666666666667, "recall": 0.3375714285714286, "f1_score": 0.2951269841269843}, "target_users": {"precision": 0.31146666666666656, "recall": 0.33888210678210684, "f1_score": 0.31118625263625266}}}, "(0.6, 10, 0.1, 1)": {"precision": 0.27122865013774106, "recall": 0.22971215261628486, "f1_score": 0.2390340605376179, "per_field": {"categories": {"precision": 0.17768595041322313, "recall": 0.16780146506386176, "f1_score": 0.16955124366191562}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 10, 0.1, 3)": {"precision": 0.2554214876033058, "recall": 0.2951525341967491, "f1_score": 0.25999554923191287, "per_field": {"categories": {"precision": 0.174931129476584, "recall": 0.23325976709241172, "f1_score": 0.18705693296602383}, "scientific_domains": {"precision": 0.281, "recall": 0.3360714285714286, "f1_score": 0.2946984126984128}, "target_users": {"precision": 0.3103333333333333, "recall": 0.31612640692640687, "f1_score": 0.2982313020313021}}}, "(0.6, 10, 0.1, 5)": {"precision": 0.25633204775022955, "recall": 0.3074952180204247, "f1_score": 0.26585300933634276, "per_field": {"categories": {"precision": 0.17696280991735538, "recall": 0.24603211870773856, "f1_score": 0.19124579124579127}, "scientific_domains": {"precision": 0.2805666666666667, "recall": 0.3375714285714286, "f1_score": 0.2951269841269843}, "target_users": {"precision": 0.31146666666666656, "recall": 0.33888210678210684, "f1_score": 0.31118625263625266}}}, "(0.6, 10, 0.2, 1)": {"precision": 0.27122865013774106, "recall": 0.22971215261628486, "f1_score": 0.2390340605376179, "per_field": {"categories": {"precision": 0.17768595041322313, "recall": 0.16780146506386176, "f1_score": 0.16955124366191562}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 10, 0.2, 3)": {"precision": 0.2554214876033058, "recall": 0.2951525341967491, "f1_score": 0.25999554923191287, "per_field": {"categories": {"precision": 0.174931129476584, "recall": 0.23325976709241172, "f1_score": 0.18705693296602383}, "scientific_domains": {"precision": 0.281, "recall": 0.3360714285714286, "f1_score": 0.2946984126984128}, "target_users": {"precision": 0.3103333333333333, "recall": 0.31612640692640687, "f1_score": 0.2982313020313021}}}, "(0.6, 10, 0.2, 5)": {"precision": 0.25633204775022955, "recall": 0.3074952180204247, "f1_score": 0.26585300933634276, "per_field": {"categories": {"precision": 0.17696280991735538, "recall": 0.24603211870773856, "f1_score": 0.19124579124579127}, "scientific_domains": {"precision": 0.2805666666666667, "recall": 0.3375714285714286, "f1_score": 0.2951269841269843}, "target_users": {"precision": 0.31146666666666656, "recall": 0.33888210678210684, "f1_score": 0.31118625263625266}}}, "(0.6, 10, 0.3, 1)": {"precision": 0.27122865013774106, "recall": 0.22971215261628486, "f1_score": 0.2390340605376179, "per_field": {"categories": {"precision": 0.17768595041322313, "recall": 0.16780146506386176, "f1_score": 0.16955124366191562}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 10, 0.3, 3)": {"precision": 0.2554214876033058, "recall": 0.2951525341967491, "f1_score": 0.25999554923191287, "per_field": {"categories": {"precision": 0.174931129476584, "recall": 0.23325976709241172, "f1_score": 0.18705693296602383}, "scientific_domains": {"precision": 0.281, "recall": 0.3360714285714286, "f1_score": 0.2946984126984128}, "target_users": {"precision": 0.3103333333333333, "recall": 0.31612640692640687, "f1_score": 0.2982313020313021}}}, "(0.6, 10, 0.3, 5)": {"precision": 0.25633204775022955, "recall": 0.3074952180204247, "f1_score": 0.26585300933634276, "per_field": {"categories": {"precision": 0.17696280991735538, "recall": 0.24603211870773856, "f1_score": 0.19124579124579127}, "scientific_domains": {"precision": 0.2805666666666667, "recall": 0.3375714285714286, "f1_score": 0.2951269841269843}, "target_users": {"precision": 0.31146666666666656, "recall": 0.33888210678210684, "f1_score": 0.31118625263625266}}}, "(0.6, 10, 0.4, 1)": {"precision": 0.27122865013774106, "recall": 0.22971215261628486, "f1_score": 0.2390340605376179, "per_field": {"categories": {"precision": 0.17768595041322313, "recall": 0.16780146506386176, "f1_score": 0.16955124366191562}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 10, 0.4, 3)": {"precision": 0.2554214876033058, "recall": 0.2951525341967491, "f1_score": 0.25999554923191287, "per_field": {"categories": {"precision": 0.174931129476584, "recall": 0.23325976709241172, "f1_score": 0.18705693296602383}, "scientific_domains": {"precision": 0.281, "recall": 0.3360714285714286, "f1_score": 0.2946984126984128}, "target_users": {"precision": 0.3103333333333333, "recall": 0.31612640692640687, "f1_score": 0.2982313020313021}}}, "(0.6, 10, 0.4, 5)": {"precision": 0.25633204775022955, "recall": 0.3074952180204247, "f1_score": 0.26585300933634276, "per_field": {"categories": {"precision": 0.17696280991735538, "recall": 0.24603211870773856, "f1_score": 0.19124579124579127}, "scientific_domains": {"precision": 0.2805666666666667, "recall": 0.3375714285714286, "f1_score": 0.2951269841269843}, "target_users": {"precision": 0.31146666666666656, "recall": 0.33888210678210684, "f1_score": 0.31118625263625266}}}, "(0.6, 10, 0.5, 1)": {"precision": 0.27122865013774106, "recall": 0.22971215261628486, "f1_score": 0.2390340605376179, "per_field": {"categories": {"precision": 0.17768595041322313, "recall": 0.16780146506386176, "f1_score": 0.16955124366191562}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 10, 0.5, 3)": {"precision": 0.2554214876033058, "recall": 0.2951525341967491, "f1_score": 0.25999554923191287, "per_field": {"categories": {"precision": 0.174931129476584, "recall": 0.23325976709241172, "f1_score": 0.18705693296602383}, "scientific_domains": {"precision": 0.281, "recall": 0.3360714285714286, "f1_score": 0.2946984126984128}, "target_users": {"precision": 0.3103333333333333, "recall": 0.31612640692640687, "f1_score": 0.2982313020313021}}}, "(0.6, 10, 0.5, 5)": {"precision": 0.25633204775022955, "recall": 0.3074952180204247, "f1_score": 0.26585300933634276, "per_field": {"categories": {"precision": 0.17696280991735538, "recall": 0.24603211870773856, "f1_score": 0.19124579124579127}, "scientific_domains": {"precision": 0.2805666666666667, "recall": 0.3375714285714286, "f1_score": 0.2951269841269843}, "target_users": {"precision": 0.31146666666666656, "recall": 0.33888210678210684, "f1_score": 0.31118625263625266}}}, "(0.6, 10, 0.6, 1)": {"precision": 0.27122865013774106, "recall": 0.22971215261628486, "f1_score": 0.2390340605376179, "per_field": {"categories": {"precision": 0.17768595041322313, "recall": 0.16780146506386176, "f1_score": 0.16955124366191562}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 10, 0.6, 3)": {"precision": 0.2554214876033058, "recall": 0.2951525341967491, "f1_score": 0.25999554923191287, "per_field": {"categories": {"precision": 0.174931129476584, "recall": 0.23325976709241172, "f1_score": 0.18705693296602383}, "scientific_domains": {"precision": 0.281, "recall": 0.3360714285714286, "f1_score": 0.2946984126984128}, "target_users": {"precision": 0.3103333333333333, "recall": 0.31612640692640687, "f1_score": 0.2982313020313021}}}, "(0.6, 10, 0.6, 5)": {"precision": 0.25633204775022955, "recall": 0.3074952180204247, "f1_score": 0.26585300933634276, "per_field": {"categories": {"precision": 0.17696280991735538, "recall": 0.24603211870773856, "f1_score": 0.19124579124579127}, "scientific_domains": {"precision": 0.2805666666666667, "recall": 0.3375714285714286, "f1_score": 0.2951269841269843}, "target_users": {"precision": 0.31146666666666656, "recall": 0.33888210678210684, "f1_score": 0.31118625263625266}}}, "(0.6, 10, 0.7, 1)": {"precision": 0.27122865013774106, "recall": 0.22971215261628486, "f1_score": 0.2390340605376179, "per_field": {"categories": {"precision": 0.17768595041322313, "recall": 0.16780146506386176, "f1_score": 0.16955124366191562}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 10, 0.7, 3)": {"precision": 0.2554214876033058, "recall": 0.2951525341967491, "f1_score": 0.25999554923191287, "per_field": {"categories": {"precision": 0.174931129476584, "recall": 0.23325976709241172, "f1_score": 0.18705693296602383}, "scientific_domains": {"precision": 0.281, "recall": 0.3360714285714286, "f1_score": 0.2946984126984128}, "target_users": {"precision": 0.3103333333333333, "recall": 0.31612640692640687, "f1_score": 0.2982313020313021}}}, "(0.6, 10, 0.7, 5)": {"precision": 0.25633204775022955, "recall": 0.3074952180204247, "f1_score": 0.26585300933634276, "per_field": {"categories": {"precision": 0.17696280991735538, "recall": 0.24603211870773856, "f1_score": 0.19124579124579127}, "scientific_domains": {"precision": 0.2805666666666667, "recall": 0.3375714285714286, "f1_score": 0.2951269841269843}, "target_users": {"precision": 0.31146666666666656, "recall": 0.33888210678210684, "f1_score": 0.31118625263625266}}}, "(0.6, 10, 0.8, 1)": {"precision": 0.27122865013774106, "recall": 0.22971215261628486, "f1_score": 0.2390340605376179, "per_field": {"categories": {"precision": 0.17768595041322313, "recall": 0.16780146506386176, "f1_score": 0.16955124366191562}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 10, 0.8, 3)": {"precision": 0.2554214876033058, "recall": 0.2951525341967491, "f1_score": 0.25999554923191287, "per_field": {"categories": {"precision": 0.174931129476584, "recall": 0.23325976709241172, "f1_score": 0.18705693296602383}, "scientific_domains": {"precision": 0.281, "recall": 0.3360714285714286, "f1_score": 0.2946984126984128}, "target_users": {"precision": 0.3103333333333333, "recall": 0.31612640692640687, "f1_score": 0.2982313020313021}}}, "(0.6, 10, 0.8, 5)": {"precision": 0.25633204775022955, "recall": 0.3074952180204247, "f1_score": 0.26585300933634276, "per_field": {"categories": {"precision": 0.17696280991735538, "recall": 0.24603211870773856, "f1_score": 0.19124579124579127}, "scientific_domains": {"precision": 0.2805666666666667, "recall": 0.3375714285714286, "f1_score": 0.2951269841269843}, "target_users": {"precision": 0.31146666666666656, "recall": 0.33888210678210684, "f1_score": 0.31118625263625266}}}, "(0.6, 10, 0.9, 1)": {"precision": 0.27122865013774106, "recall": 0.22971215261628486, "f1_score": 0.2390340605376179, "per_field": {"categories": {"precision": 0.17768595041322313, "recall": 0.16780146506386176, "f1_score": 0.16955124366191562}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 10, 0.9, 3)": {"precision": 0.2554214876033058, "recall": 0.2951525341967491, "f1_score": 0.25999554923191287, "per_field": {"categories": {"precision": 0.174931129476584, "recall": 0.23325976709241172, "f1_score": 0.18705693296602383}, "scientific_domains": {"precision": 0.281, "recall": 0.3360714285714286, "f1_score": 0.2946984126984128}, "target_users": {"precision": 0.3103333333333333, "recall": 0.31612640692640687, "f1_score": 0.2982313020313021}}}, "(0.6, 10, 0.9, 5)": {"precision": 0.25633204775022955, "recall": 0.3074952180204247, "f1_score": 0.26585300933634276, "per_field": {"categories": {"precision": 0.17696280991735538, "recall": 0.24603211870773856, "f1_score": 0.19124579124579127}, "scientific_domains": {"precision": 0.2805666666666667, "recall": 0.3375714285714286, "f1_score": 0.2951269841269843}, "target_users": {"precision": 0.31146666666666656, "recall": 0.33888210678210684, "f1_score": 0.31118625263625266}}}, "(0.6, 15, 0, 1)": {"precision": 0.2670964187327824, "recall": 0.22557992121132617, "f1_score": 0.23490182913265922, "per_field": {"categories": {"precision": 0.1652892561983471, "recall": 0.15540477084898574, "f1_score": 0.1571545494470396}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 15, 0, 3)": {"precision": 0.2606657483930211, "recall": 0.29244179039509616, "f1_score": 0.2633399018489928, "per_field": {"categories": {"precision": 0.17699724517906334, "recall": 0.22912753568745306, "f1_score": 0.18808999081726352}, "scientific_domains": {"precision": 0.2826666666666667, "recall": 0.3320714285714286, "f1_score": 0.29569841269841285}, "target_users": {"precision": 0.3223333333333333, "recall": 0.31612640692640687, "f1_score": 0.30623130203130206}}}, "(0.6, 15, 0, 5)": {"precision": 0.26163186409550043, "recall": 0.30478447421877175, "f1_score": 0.2692640286200893, "per_field": {"categories": {"precision": 0.1790289256198347, "recall": 0.2418998873027799, "f1_score": 0.19227884909703094}, "scientific_domains": {"precision": 0.28240000000000004, "recall": 0.3335714285714286, "f1_score": 0.29632698412698427}, "target_users": {"precision": 0.32346666666666657, "recall": 0.33888210678210684, "f1_score": 0.31918625263625267}}}, "(0.6, 15, 0.1, 1)": {"precision": 0.2670964187327824, "recall": 0.22557992121132617, "f1_score": 0.23490182913265922, "per_field": {"categories": {"precision": 0.1652892561983471, "recall": 0.15540477084898574, "f1_score": 0.1571545494470396}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 15, 0.1, 3)": {"precision": 0.2606657483930211, "recall": 0.29244179039509616, "f1_score": 0.2633399018489928, "per_field": {"categories": {"precision": 0.17699724517906334, "recall": 0.22912753568745306, "f1_score": 0.18808999081726352}, "scientific_domains": {"precision": 0.2826666666666667, "recall": 0.3320714285714286, "f1_score": 0.29569841269841285}, "target_users": {"precision": 0.3223333333333333, "recall": 0.31612640692640687, "f1_score": 0.30623130203130206}}}, "(0.6, 15, 0.1, 5)": {"precision": 0.26163186409550043, "recall": 0.30478447421877175, "f1_score": 0.2692640286200893, "per_field": {"categories": {"precision": 0.1790289256198347, "recall": 0.2418998873027799, "f1_score": 0.19227884909703094}, "scientific_domains": {"precision": 0.28240000000000004, "recall": 0.3335714285714286, "f1_score": 0.29632698412698427}, "target_users": {"precision": 0.32346666666666657, "recall": 0.33888210678210684, "f1_score": 0.31918625263625267}}}, "(0.6, 15, 0.2, 1)": {"precision": 0.2670964187327824, "recall": 0.22557992121132617, "f1_score": 0.23490182913265922, "per_field": {"categories": {"precision": 0.1652892561983471, "recall": 0.15540477084898574, "f1_score": 0.1571545494470396}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 15, 0.2, 3)": {"precision": 0.2606657483930211, "recall": 0.29244179039509616, "f1_score": 0.2633399018489928, "per_field": {"categories": {"precision": 0.17699724517906334, "recall": 0.22912753568745306, "f1_score": 0.18808999081726352}, "scientific_domains": {"precision": 0.2826666666666667, "recall": 0.3320714285714286, "f1_score": 0.29569841269841285}, "target_users": {"precision": 0.3223333333333333, "recall": 0.31612640692640687, "f1_score": 0.30623130203130206}}}, "(0.6, 15, 0.2, 5)": {"precision": 0.26163186409550043, "recall": 0.30478447421877175, "f1_score": 0.2692640286200893, "per_field": {"categories": {"precision": 0.1790289256198347, "recall": 0.2418998873027799, "f1_score": 0.19227884909703094}, "scientific_domains": {"precision": 0.28240000000000004, "recall": 0.3335714285714286, "f1_score": 0.29632698412698427}, "target_users": {"precision": 0.32346666666666657, "recall": 0.33888210678210684, "f1_score": 0.31918625263625267}}}, "(0.6, 15, 0.3, 1)": {"precision": 0.2670964187327824, "recall": 0.22557992121132617, "f1_score": 0.23490182913265922, "per_field": {"categories": {"precision": 0.1652892561983471, "recall": 0.15540477084898574, "f1_score": 0.1571545494470396}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 15, 0.3, 3)": {"precision": 0.2606657483930211, "recall": 0.29244179039509616, "f1_score": 0.2633399018489928, "per_field": {"categories": {"precision": 0.17699724517906334, "recall": 0.22912753568745306, "f1_score": 0.18808999081726352}, "scientific_domains": {"precision": 0.2826666666666667, "recall": 0.3320714285714286, "f1_score": 0.29569841269841285}, "target_users": {"precision": 0.3223333333333333, "recall": 0.31612640692640687, "f1_score": 0.30623130203130206}}}, "(0.6, 15, 0.3, 5)": {"precision": 0.26163186409550043, "recall": 0.30478447421877175, "f1_score": 0.2692640286200893, "per_field": {"categories": {"precision": 0.1790289256198347, "recall": 0.2418998873027799, "f1_score": 0.19227884909703094}, "scientific_domains": {"precision": 0.28240000000000004, "recall": 0.3335714285714286, "f1_score": 0.29632698412698427}, "target_users": {"precision": 0.32346666666666657, "recall": 0.33888210678210684, "f1_score": 0.31918625263625267}}}, "(0.6, 15, 0.4, 1)": {"precision": 0.2670964187327824, "recall": 0.22557992121132617, "f1_score": 0.23490182913265922, "per_field": {"categories": {"precision": 0.1652892561983471, "recall": 0.15540477084898574, "f1_score": 0.1571545494470396}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 15, 0.4, 3)": {"precision": 0.2606657483930211, "recall": 0.29244179039509616, "f1_score": 0.2633399018489928, "per_field": {"categories": {"precision": 0.17699724517906334, "recall": 0.22912753568745306, "f1_score": 0.18808999081726352}, "scientific_domains": {"precision": 0.2826666666666667, "recall": 0.3320714285714286, "f1_score": 0.29569841269841285}, "target_users": {"precision": 0.3223333333333333, "recall": 0.31612640692640687, "f1_score": 0.30623130203130206}}}, "(0.6, 15, 0.4, 5)": {"precision": 0.26163186409550043, "recall": 0.30478447421877175, "f1_score": 0.2692640286200893, "per_field": {"categories": {"precision": 0.1790289256198347, "recall": 0.2418998873027799, "f1_score": 0.19227884909703094}, "scientific_domains": {"precision": 0.28240000000000004, "recall": 0.3335714285714286, "f1_score": 0.29632698412698427}, "target_users": {"precision": 0.32346666666666657, "recall": 0.33888210678210684, "f1_score": 0.31918625263625267}}}, "(0.6, 15, 0.5, 1)": {"precision": 0.2670964187327824, "recall": 0.22557992121132617, "f1_score": 0.23490182913265922, "per_field": {"categories": {"precision": 0.1652892561983471, "recall": 0.15540477084898574, "f1_score": 0.1571545494470396}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 15, 0.5, 3)": {"precision": 0.2606657483930211, "recall": 0.29244179039509616, "f1_score": 0.2633399018489928, "per_field": {"categories": {"precision": 0.17699724517906334, "recall": 0.22912753568745306, "f1_score": 0.18808999081726352}, "scientific_domains": {"precision": 0.2826666666666667, "recall": 0.3320714285714286, "f1_score": 0.29569841269841285}, "target_users": {"precision": 0.3223333333333333, "recall": 0.31612640692640687, "f1_score": 0.30623130203130206}}}, "(0.6, 15, 0.5, 5)": {"precision": 0.26163186409550043, "recall": 0.30478447421877175, "f1_score": 0.2692640286200893, "per_field": {"categories": {"precision": 0.1790289256198347, "recall": 0.2418998873027799, "f1_score": 0.19227884909703094}, "scientific_domains": {"precision": 0.28240000000000004, "recall": 0.3335714285714286, "f1_score": 0.29632698412698427}, "target_users": {"precision": 0.32346666666666657, "recall": 0.33888210678210684, "f1_score": 0.31918625263625267}}}, "(0.6, 15, 0.6, 1)": {"precision": 0.2670964187327824, "recall": 0.22557992121132617, "f1_score": 0.23490182913265922, "per_field": {"categories": {"precision": 0.1652892561983471, "recall": 0.15540477084898574, "f1_score": 0.1571545494470396}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 15, 0.6, 3)": {"precision": 0.2606657483930211, "recall": 0.29244179039509616, "f1_score": 0.2633399018489928, "per_field": {"categories": {"precision": 0.17699724517906334, "recall": 0.22912753568745306, "f1_score": 0.18808999081726352}, "scientific_domains": {"precision": 0.2826666666666667, "recall": 0.3320714285714286, "f1_score": 0.29569841269841285}, "target_users": {"precision": 0.3223333333333333, "recall": 0.31612640692640687, "f1_score": 0.30623130203130206}}}, "(0.6, 15, 0.6, 5)": {"precision": 0.26163186409550043, "recall": 0.30478447421877175, "f1_score": 0.2692640286200893, "per_field": {"categories": {"precision": 0.1790289256198347, "recall": 0.2418998873027799, "f1_score": 0.19227884909703094}, "scientific_domains": {"precision": 0.28240000000000004, "recall": 0.3335714285714286, "f1_score": 0.29632698412698427}, "target_users": {"precision": 0.32346666666666657, "recall": 0.33888210678210684, "f1_score": 0.31918625263625267}}}, "(0.6, 15, 0.7, 1)": {"precision": 0.2670964187327824, "recall": 0.22557992121132617, "f1_score": 0.23490182913265922, "per_field": {"categories": {"precision": 0.1652892561983471, "recall": 0.15540477084898574, "f1_score": 0.1571545494470396}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 15, 0.7, 3)": {"precision": 0.2606657483930211, "recall": 0.29244179039509616, "f1_score": 0.2633399018489928, "per_field": {"categories": {"precision": 0.17699724517906334, "recall": 0.22912753568745306, "f1_score": 0.18808999081726352}, "scientific_domains": {"precision": 0.2826666666666667, "recall": 0.3320714285714286, "f1_score": 0.29569841269841285}, "target_users": {"precision": 0.3223333333333333, "recall": 0.31612640692640687, "f1_score": 0.30623130203130206}}}, "(0.6, 15, 0.7, 5)": {"precision": 0.26163186409550043, "recall": 0.30478447421877175, "f1_score": 0.2692640286200893, "per_field": {"categories": {"precision": 0.1790289256198347, "recall": 0.2418998873027799, "f1_score": 0.19227884909703094}, "scientific_domains": {"precision": 0.28240000000000004, "recall": 0.3335714285714286, "f1_score": 0.29632698412698427}, "target_users": {"precision": 0.32346666666666657, "recall": 0.33888210678210684, "f1_score": 0.31918625263625267}}}, "(0.6, 15, 0.8, 1)": {"precision": 0.2670964187327824, "recall": 0.22557992121132617, "f1_score": 0.23490182913265922, "per_field": {"categories": {"precision": 0.1652892561983471, "recall": 0.15540477084898574, "f1_score": 0.1571545494470396}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 15, 0.8, 3)": {"precision": 0.2606657483930211, "recall": 0.29244179039509616, "f1_score": 0.2633399018489928, "per_field": {"categories": {"precision": 0.17699724517906334, "recall": 0.22912753568745306, "f1_score": 0.18808999081726352}, "scientific_domains": {"precision": 0.2826666666666667, "recall": 0.3320714285714286, "f1_score": 0.29569841269841285}, "target_users": {"precision": 0.3223333333333333, "recall": 0.31612640692640687, "f1_score": 0.30623130203130206}}}, "(0.6, 15, 0.8, 5)": {"precision": 0.26163186409550043, "recall": 0.30478447421877175, "f1_score": 0.2692640286200893, "per_field": {"categories": {"precision": 0.1790289256198347, "recall": 0.2418998873027799, "f1_score": 0.19227884909703094}, "scientific_domains": {"precision": 0.28240000000000004, "recall": 0.3335714285714286, "f1_score": 0.29632698412698427}, "target_users": {"precision": 0.32346666666666657, "recall": 0.33888210678210684, "f1_score": 0.31918625263625267}}}, "(0.6, 15, 0.9, 1)": {"precision": 0.2670964187327824, "recall": 0.22557992121132617, "f1_score": 0.23490182913265922, "per_field": {"categories": {"precision": 0.1652892561983471, "recall": 0.15540477084898574, "f1_score": 0.1571545494470396}, "scientific_domains": {"precision": 0.306, "recall": 0.28178571428571425, "f1_score": 0.28856666666666664}, "target_users": {"precision": 0.33, "recall": 0.23954927849927854, "f1_score": 0.2589842712842714}}}, "(0.6, 15, 0.9, 3)": {"precision": 0.2606657483930211, "recall": 0.29244179039509616, "f1_score": 0.2633399018489928, "per_field": {"categories": {"precision": 0.17699724517906334, "recall": 0.22912753568745306, "f1_score": 0.18808999081726352}, "scientific_domains": {"precision": 0.2826666666666667, "recall": 0.3320714285714286, "f1_score": 0.29569841269841285}, "target_users": {"precision": 0.3223333333333333, "recall": 0.31612640692640687, "f1_score": 0.30623130203130206}}}, "(0.6, 15, 0.9, 5)": {"precision": 0.26163186409550043, "recall": 0.30478447421877175, "f1_score": 0.2692640286200893, "per_field": {"categories": {"precision": 0.1790289256198347, "recall": 0.2418998873027799, "f1_score": 0.19227884909703094}, "scientific_domains": {"precision": 0.28240000000000004, "recall": 0.3335714285714286, "f1_score": 0.29632698412698427}, "target_users": {"precision": 0.32346666666666657, "recall": 0.33888210678210684, "f1_score": 0.31918625263625267}}}, "(0.7, 5, 0, 1)": {"precision": 0.24111845730027548, "recall": 0.20250466868607364, "f1_score": 0.21111884780422333, "per_field": {"categories": {"precision": 0.16735537190082644, "recall": 0.15824567993989483, "f1_score": 0.16013893879506527}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.288, "recall": 0.20414927849927852, "f1_score": 0.22165093795093802}}}, "(0.7, 5, 0, 3)": {"precision": 0.23130027548209364, "recall": 0.2553233527721131, "f1_score": 0.23206769055859966, "per_field": {"categories": {"precision": 0.16356749311294763, "recall": 0.1999436513899324, "f1_score": 0.1711019283746556}, "scientific_domains": {"precision": 0.24733333333333332, "recall": 0.28807142857142853, "f1_score": 0.2576984126984128}, "target_users": {"precision": 0.2829999999999999, "recall": 0.27795497835497834, "f1_score": 0.2674027306027306}}}, "(0.7, 5, 0, 5)": {"precision": 0.23143168044077134, "recall": 0.2633778823258162, "f1_score": 0.23636672222278285, "per_field": {"categories": {"precision": 0.16442837465564739, "recall": 0.20445154019534187, "f1_score": 0.17308692990511168}, "scientific_domains": {"precision": 0.24706666666666666, "recall": 0.28957142857142854, "f1_score": 0.25832698412698424}, "target_users": {"precision": 0.28279999999999994, "recall": 0.29611067821067827, "f1_score": 0.27768625263625263}}}, "(0.7, 5, 0.1, 1)": {"precision": 0.24111845730027548, "recall": 0.20250466868607364, "f1_score": 0.21111884780422333, "per_field": {"categories": {"precision": 0.16735537190082644, "recall": 0.15824567993989483, "f1_score": 0.16013893879506527}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.288, "recall": 0.20414927849927852, "f1_score": 0.22165093795093802}}}, "(0.7, 5, 0.1, 3)": {"precision": 0.23130027548209364, "recall": 0.2553233527721131, "f1_score": 0.23206769055859966, "per_field": {"categories": {"precision": 0.16356749311294763, "recall": 0.1999436513899324, "f1_score": 0.1711019283746556}, "scientific_domains": {"precision": 0.24733333333333332, "recall": 0.28807142857142853, "f1_score": 0.2576984126984128}, "target_users": {"precision": 0.2829999999999999, "recall": 0.27795497835497834, "f1_score": 0.2674027306027306}}}, "(0.7, 5, 0.1, 5)": {"precision": 0.23143168044077134, "recall": 0.2633778823258162, "f1_score": 0.23636672222278285, "per_field": {"categories": {"precision": 0.16442837465564739, "recall": 0.20445154019534187, "f1_score": 0.17308692990511168}, "scientific_domains": {"precision": 0.24706666666666666, "recall": 0.28957142857142854, "f1_score": 0.25832698412698424}, "target_users": {"precision": 0.28279999999999994, "recall": 0.29611067821067827, "f1_score": 0.27768625263625263}}}, "(0.7, 5, 0.2, 1)": {"precision": 0.24111845730027548, "recall": 0.20250466868607364, "f1_score": 0.21111884780422333, "per_field": {"categories": {"precision": 0.16735537190082644, "recall": 0.15824567993989483, "f1_score": 0.16013893879506527}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.288, "recall": 0.20414927849927852, "f1_score": 0.22165093795093802}}}, "(0.7, 5, 0.2, 3)": {"precision": 0.23130027548209364, "recall": 0.2553233527721131, "f1_score": 0.23206769055859966, "per_field": {"categories": {"precision": 0.16356749311294763, "recall": 0.1999436513899324, "f1_score": 0.1711019283746556}, "scientific_domains": {"precision": 0.24733333333333332, "recall": 0.28807142857142853, "f1_score": 0.2576984126984128}, "target_users": {"precision": 0.2829999999999999, "recall": 0.27795497835497834, "f1_score": 0.2674027306027306}}}, "(0.7, 5, 0.2, 5)": {"precision": 0.23143168044077134, "recall": 0.2633778823258162, "f1_score": 0.23636672222278285, "per_field": {"categories": {"precision": 0.16442837465564739, "recall": 0.20445154019534187, "f1_score": 0.17308692990511168}, "scientific_domains": {"precision": 0.24706666666666666, "recall": 0.28957142857142854, "f1_score": 0.25832698412698424}, "target_users": {"precision": 0.28279999999999994, "recall": 0.29611067821067827, "f1_score": 0.27768625263625263}}}, "(0.7, 5, 0.3, 1)": {"precision": 0.24111845730027548, "recall": 0.20250466868607364, "f1_score": 0.21111884780422333, "per_field": {"categories": {"precision": 0.16735537190082644, "recall": 0.15824567993989483, "f1_score": 0.16013893879506527}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.288, "recall": 0.20414927849927852, "f1_score": 0.22165093795093802}}}, "(0.7, 5, 0.3, 3)": {"precision": 0.23130027548209364, "recall": 0.2553233527721131, "f1_score": 0.23206769055859966, "per_field": {"categories": {"precision": 0.16356749311294763, "recall": 0.1999436513899324, "f1_score": 0.1711019283746556}, "scientific_domains": {"precision": 0.24733333333333332, "recall": 0.28807142857142853, "f1_score": 0.2576984126984128}, "target_users": {"precision": 0.2829999999999999, "recall": 0.27795497835497834, "f1_score": 0.2674027306027306}}}, "(0.7, 5, 0.3, 5)": {"precision": 0.23143168044077134, "recall": 0.2633778823258162, "f1_score": 0.23636672222278285, "per_field": {"categories": {"precision": 0.16442837465564739, "recall": 0.20445154019534187, "f1_score": 0.17308692990511168}, "scientific_domains": {"precision": 0.24706666666666666, "recall": 0.28957142857142854, "f1_score": 0.25832698412698424}, "target_users": {"precision": 0.28279999999999994, "recall": 0.29611067821067827, "f1_score": 0.27768625263625263}}}, "(0.7, 5, 0.4, 1)": {"precision": 0.24111845730027548, "recall": 0.20250466868607364, "f1_score": 0.21111884780422333, "per_field": {"categories": {"precision": 0.16735537190082644, "recall": 0.15824567993989483, "f1_score": 0.16013893879506527}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.288, "recall": 0.20414927849927852, "f1_score": 0.22165093795093802}}}, "(0.7, 5, 0.4, 3)": {"precision": 0.23130027548209364, "recall": 0.2553233527721131, "f1_score": 0.23206769055859966, "per_field": {"categories": {"precision": 0.16356749311294763, "recall": 0.1999436513899324, "f1_score": 0.1711019283746556}, "scientific_domains": {"precision": 0.24733333333333332, "recall": 0.28807142857142853, "f1_score": 0.2576984126984128}, "target_users": {"precision": 0.2829999999999999, "recall": 0.27795497835497834, "f1_score": 0.2674027306027306}}}, "(0.7, 5, 0.4, 5)": {"precision": 0.23143168044077134, "recall": 0.2633778823258162, "f1_score": 0.23636672222278285, "per_field": {"categories": {"precision": 0.16442837465564739, "recall": 0.20445154019534187, "f1_score": 0.17308692990511168}, "scientific_domains": {"precision": 0.24706666666666666, "recall": 0.28957142857142854, "f1_score": 0.25832698412698424}, "target_users": {"precision": 0.28279999999999994, "recall": 0.29611067821067827, "f1_score": 0.27768625263625263}}}, "(0.7, 5, 0.5, 1)": {"precision": 0.24111845730027548, "recall": 0.20250466868607364, "f1_score": 0.21111884780422333, "per_field": {"categories": {"precision": 0.16735537190082644, "recall": 0.15824567993989483, "f1_score": 0.16013893879506527}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.288, "recall": 0.20414927849927852, "f1_score": 0.22165093795093802}}}, "(0.7, 5, 0.5, 3)": {"precision": 0.23130027548209364, "recall": 0.2553233527721131, "f1_score": 0.23206769055859966, "per_field": {"categories": {"precision": 0.16356749311294763, "recall": 0.1999436513899324, "f1_score": 0.1711019283746556}, "scientific_domains": {"precision": 0.24733333333333332, "recall": 0.28807142857142853, "f1_score": 0.2576984126984128}, "target_users": {"precision": 0.2829999999999999, "recall": 0.27795497835497834, "f1_score": 0.2674027306027306}}}, "(0.7, 5, 0.5, 5)": {"precision": 0.23143168044077134, "recall": 0.2633778823258162, "f1_score": 0.23636672222278285, "per_field": {"categories": {"precision": 0.16442837465564739, "recall": 0.20445154019534187, "f1_score": 0.17308692990511168}, "scientific_domains": {"precision": 0.24706666666666666, "recall": 0.28957142857142854, "f1_score": 0.25832698412698424}, "target_users": {"precision": 0.28279999999999994, "recall": 0.29611067821067827, "f1_score": 0.27768625263625263}}}, "(0.7, 5, 0.6, 1)": {"precision": 0.24111845730027548, "recall": 0.20250466868607364, "f1_score": 0.21111884780422333, "per_field": {"categories": {"precision": 0.16735537190082644, "recall": 0.15824567993989483, "f1_score": 0.16013893879506527}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.288, "recall": 0.20414927849927852, "f1_score": 0.22165093795093802}}}, "(0.7, 5, 0.6, 3)": {"precision": 0.23130027548209364, "recall": 0.2553233527721131, "f1_score": 0.23206769055859966, "per_field": {"categories": {"precision": 0.16356749311294763, "recall": 0.1999436513899324, "f1_score": 0.1711019283746556}, "scientific_domains": {"precision": 0.24733333333333332, "recall": 0.28807142857142853, "f1_score": 0.2576984126984128}, "target_users": {"precision": 0.2829999999999999, "recall": 0.27795497835497834, "f1_score": 0.2674027306027306}}}, "(0.7, 5, 0.6, 5)": {"precision": 0.23143168044077134, "recall": 0.2633778823258162, "f1_score": 0.23636672222278285, "per_field": {"categories": {"precision": 0.16442837465564739, "recall": 0.20445154019534187, "f1_score": 0.17308692990511168}, "scientific_domains": {"precision": 0.24706666666666666, "recall": 0.28957142857142854, "f1_score": 0.25832698412698424}, "target_users": {"precision": 0.28279999999999994, "recall": 0.29611067821067827, "f1_score": 0.27768625263625263}}}, "(0.7, 5, 0.7, 1)": {"precision": 0.24111845730027548, "recall": 0.20250466868607364, "f1_score": 0.21111884780422333, "per_field": {"categories": {"precision": 0.16735537190082644, "recall": 0.15824567993989483, "f1_score": 0.16013893879506527}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.288, "recall": 0.20414927849927852, "f1_score": 0.22165093795093802}}}, "(0.7, 5, 0.7, 3)": {"precision": 0.23130027548209364, "recall": 0.2553233527721131, "f1_score": 0.23206769055859966, "per_field": {"categories": {"precision": 0.16356749311294763, "recall": 0.1999436513899324, "f1_score": 0.1711019283746556}, "scientific_domains": {"precision": 0.24733333333333332, "recall": 0.28807142857142853, "f1_score": 0.2576984126984128}, "target_users": {"precision": 0.2829999999999999, "recall": 0.27795497835497834, "f1_score": 0.2674027306027306}}}, "(0.7, 5, 0.7, 5)": {"precision": 0.23143168044077134, "recall": 0.2633778823258162, "f1_score": 0.23636672222278285, "per_field": {"categories": {"precision": 0.16442837465564739, "recall": 0.20445154019534187, "f1_score": 0.17308692990511168}, "scientific_domains": {"precision": 0.24706666666666666, "recall": 0.28957142857142854, "f1_score": 0.25832698412698424}, "target_users": {"precision": 0.28279999999999994, "recall": 0.29611067821067827, "f1_score": 0.27768625263625263}}}, "(0.7, 5, 0.8, 1)": {"precision": 0.24111845730027548, "recall": 0.20250466868607364, "f1_score": 0.21111884780422333, "per_field": {"categories": {"precision": 0.16735537190082644, "recall": 0.15824567993989483, "f1_score": 0.16013893879506527}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.288, "recall": 0.20414927849927852, "f1_score": 0.22165093795093802}}}, "(0.7, 5, 0.8, 3)": {"precision": 0.23130027548209364, "recall": 0.2553233527721131, "f1_score": 0.23206769055859966, "per_field": {"categories": {"precision": 0.16356749311294763, "recall": 0.1999436513899324, "f1_score": 0.1711019283746556}, "scientific_domains": {"precision": 0.24733333333333332, "recall": 0.28807142857142853, "f1_score": 0.2576984126984128}, "target_users": {"precision": 0.2829999999999999, "recall": 0.27795497835497834, "f1_score": 0.2674027306027306}}}, "(0.7, 5, 0.8, 5)": {"precision": 0.23143168044077134, "recall": 0.2633778823258162, "f1_score": 0.23636672222278285, "per_field": {"categories": {"precision": 0.16442837465564739, "recall": 0.20445154019534187, "f1_score": 0.17308692990511168}, "scientific_domains": {"precision": 0.24706666666666666, "recall": 0.28957142857142854, "f1_score": 0.25832698412698424}, "target_users": {"precision": 0.28279999999999994, "recall": 0.29611067821067827, "f1_score": 0.27768625263625263}}}, "(0.7, 5, 0.9, 1)": {"precision": 0.24111845730027548, "recall": 0.20250466868607364, "f1_score": 0.21111884780422333, "per_field": {"categories": {"precision": 0.16735537190082644, "recall": 0.15824567993989483, "f1_score": 0.16013893879506527}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.288, "recall": 0.20414927849927852, "f1_score": 0.22165093795093802}}}, "(0.7, 5, 0.9, 3)": {"precision": 0.23130027548209364, "recall": 0.2553233527721131, "f1_score": 0.23206769055859966, "per_field": {"categories": {"precision": 0.16356749311294763, "recall": 0.1999436513899324, "f1_score": 0.1711019283746556}, "scientific_domains": {"precision": 0.24733333333333332, "recall": 0.28807142857142853, "f1_score": 0.2576984126984128}, "target_users": {"precision": 0.2829999999999999, "recall": 0.27795497835497834, "f1_score": 0.2674027306027306}}}, "(0.7, 5, 0.9, 5)": {"precision": 0.23143168044077134, "recall": 0.2633778823258162, "f1_score": 0.23636672222278285, "per_field": {"categories": {"precision": 0.16442837465564739, "recall": 0.20445154019534187, "f1_score": 0.17308692990511168}, "scientific_domains": {"precision": 0.24706666666666666, "recall": 0.28957142857142854, "f1_score": 0.25832698412698424}, "target_users": {"precision": 0.28279999999999994, "recall": 0.29611067821067827, "f1_score": 0.27768625263625263}}}, "(0.7, 10, 0, 1)": {"precision": 0.2376528925619835, "recall": 0.1987057706144483, "f1_score": 0.2074310608437091, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 10, 0, 3)": {"precision": 0.2255270890725436, "recall": 0.25730131420461994, "f1_score": 0.22817953628862728, "per_field": {"categories": {"precision": 0.15358126721763085, "recall": 0.19787753568745306, "f1_score": 0.1628374655647383}, "scientific_domains": {"precision": 0.24666666666666665, "recall": 0.29607142857142854, "f1_score": 0.2586984126984128}, "target_users": {"precision": 0.2763333333333333, "recall": 0.27795497835497834, "f1_score": 0.26300273060273066}}}, "(0.7, 10, 0, 5)": {"precision": 0.22596841138659318, "recall": 0.2667332542266427, "f1_score": 0.23298361845786095, "per_field": {"categories": {"precision": 0.15537190082644628, "recall": 0.2065176558978212, "f1_score": 0.1663376186103459}, "scientific_domains": {"precision": 0.2464, "recall": 0.29757142857142854, "f1_score": 0.25932698412698424}, "target_users": {"precision": 0.2761333333333333, "recall": 0.29611067821067827, "f1_score": 0.2732862526362526}}}, "(0.7, 10, 0.1, 1)": {"precision": 0.2376528925619835, "recall": 0.1987057706144483, "f1_score": 0.2074310608437091, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 10, 0.1, 3)": {"precision": 0.2255270890725436, "recall": 0.25730131420461994, "f1_score": 0.22817953628862728, "per_field": {"categories": {"precision": 0.15358126721763085, "recall": 0.19787753568745306, "f1_score": 0.1628374655647383}, "scientific_domains": {"precision": 0.24666666666666665, "recall": 0.29607142857142854, "f1_score": 0.2586984126984128}, "target_users": {"precision": 0.2763333333333333, "recall": 0.27795497835497834, "f1_score": 0.26300273060273066}}}, "(0.7, 10, 0.1, 5)": {"precision": 0.22596841138659318, "recall": 0.2667332542266427, "f1_score": 0.23298361845786095, "per_field": {"categories": {"precision": 0.15537190082644628, "recall": 0.2065176558978212, "f1_score": 0.1663376186103459}, "scientific_domains": {"precision": 0.2464, "recall": 0.29757142857142854, "f1_score": 0.25932698412698424}, "target_users": {"precision": 0.2761333333333333, "recall": 0.29611067821067827, "f1_score": 0.2732862526362526}}}, "(0.7, 10, 0.2, 1)": {"precision": 0.2376528925619835, "recall": 0.1987057706144483, "f1_score": 0.2074310608437091, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 10, 0.2, 3)": {"precision": 0.2255270890725436, "recall": 0.25730131420461994, "f1_score": 0.22817953628862728, "per_field": {"categories": {"precision": 0.15358126721763085, "recall": 0.19787753568745306, "f1_score": 0.1628374655647383}, "scientific_domains": {"precision": 0.24666666666666665, "recall": 0.29607142857142854, "f1_score": 0.2586984126984128}, "target_users": {"precision": 0.2763333333333333, "recall": 0.27795497835497834, "f1_score": 0.26300273060273066}}}, "(0.7, 10, 0.2, 5)": {"precision": 0.22596841138659318, "recall": 0.2667332542266427, "f1_score": 0.23298361845786095, "per_field": {"categories": {"precision": 0.15537190082644628, "recall": 0.2065176558978212, "f1_score": 0.1663376186103459}, "scientific_domains": {"precision": 0.2464, "recall": 0.29757142857142854, "f1_score": 0.25932698412698424}, "target_users": {"precision": 0.2761333333333333, "recall": 0.29611067821067827, "f1_score": 0.2732862526362526}}}, "(0.7, 10, 0.3, 1)": {"precision": 0.2376528925619835, "recall": 0.1987057706144483, "f1_score": 0.2074310608437091, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 10, 0.3, 3)": {"precision": 0.2255270890725436, "recall": 0.25730131420461994, "f1_score": 0.22817953628862728, "per_field": {"categories": {"precision": 0.15358126721763085, "recall": 0.19787753568745306, "f1_score": 0.1628374655647383}, "scientific_domains": {"precision": 0.24666666666666665, "recall": 0.29607142857142854, "f1_score": 0.2586984126984128}, "target_users": {"precision": 0.2763333333333333, "recall": 0.27795497835497834, "f1_score": 0.26300273060273066}}}, "(0.7, 10, 0.3, 5)": {"precision": 0.22596841138659318, "recall": 0.2667332542266427, "f1_score": 0.23298361845786095, "per_field": {"categories": {"precision": 0.15537190082644628, "recall": 0.2065176558978212, "f1_score": 0.1663376186103459}, "scientific_domains": {"precision": 0.2464, "recall": 0.29757142857142854, "f1_score": 0.25932698412698424}, "target_users": {"precision": 0.2761333333333333, "recall": 0.29611067821067827, "f1_score": 0.2732862526362526}}}, "(0.7, 10, 0.4, 1)": {"precision": 0.2376528925619835, "recall": 0.1987057706144483, "f1_score": 0.2074310608437091, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 10, 0.4, 3)": {"precision": 0.2255270890725436, "recall": 0.25730131420461994, "f1_score": 0.22817953628862728, "per_field": {"categories": {"precision": 0.15358126721763085, "recall": 0.19787753568745306, "f1_score": 0.1628374655647383}, "scientific_domains": {"precision": 0.24666666666666665, "recall": 0.29607142857142854, "f1_score": 0.2586984126984128}, "target_users": {"precision": 0.2763333333333333, "recall": 0.27795497835497834, "f1_score": 0.26300273060273066}}}, "(0.7, 10, 0.4, 5)": {"precision": 0.22596841138659318, "recall": 0.2667332542266427, "f1_score": 0.23298361845786095, "per_field": {"categories": {"precision": 0.15537190082644628, "recall": 0.2065176558978212, "f1_score": 0.1663376186103459}, "scientific_domains": {"precision": 0.2464, "recall": 0.29757142857142854, "f1_score": 0.25932698412698424}, "target_users": {"precision": 0.2761333333333333, "recall": 0.29611067821067827, "f1_score": 0.2732862526362526}}}, "(0.7, 10, 0.5, 1)": {"precision": 0.2376528925619835, "recall": 0.1987057706144483, "f1_score": 0.2074310608437091, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 10, 0.5, 3)": {"precision": 0.2255270890725436, "recall": 0.25730131420461994, "f1_score": 0.22817953628862728, "per_field": {"categories": {"precision": 0.15358126721763085, "recall": 0.19787753568745306, "f1_score": 0.1628374655647383}, "scientific_domains": {"precision": 0.24666666666666665, "recall": 0.29607142857142854, "f1_score": 0.2586984126984128}, "target_users": {"precision": 0.2763333333333333, "recall": 0.27795497835497834, "f1_score": 0.26300273060273066}}}, "(0.7, 10, 0.5, 5)": {"precision": 0.22596841138659318, "recall": 0.2667332542266427, "f1_score": 0.23298361845786095, "per_field": {"categories": {"precision": 0.15537190082644628, "recall": 0.2065176558978212, "f1_score": 0.1663376186103459}, "scientific_domains": {"precision": 0.2464, "recall": 0.29757142857142854, "f1_score": 0.25932698412698424}, "target_users": {"precision": 0.2761333333333333, "recall": 0.29611067821067827, "f1_score": 0.2732862526362526}}}, "(0.7, 10, 0.6, 1)": {"precision": 0.2376528925619835, "recall": 0.1987057706144483, "f1_score": 0.2074310608437091, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 10, 0.6, 3)": {"precision": 0.2255270890725436, "recall": 0.25730131420461994, "f1_score": 0.22817953628862728, "per_field": {"categories": {"precision": 0.15358126721763085, "recall": 0.19787753568745306, "f1_score": 0.1628374655647383}, "scientific_domains": {"precision": 0.24666666666666665, "recall": 0.29607142857142854, "f1_score": 0.2586984126984128}, "target_users": {"precision": 0.2763333333333333, "recall": 0.27795497835497834, "f1_score": 0.26300273060273066}}}, "(0.7, 10, 0.6, 5)": {"precision": 0.22596841138659318, "recall": 0.2667332542266427, "f1_score": 0.23298361845786095, "per_field": {"categories": {"precision": 0.15537190082644628, "recall": 0.2065176558978212, "f1_score": 0.1663376186103459}, "scientific_domains": {"precision": 0.2464, "recall": 0.29757142857142854, "f1_score": 0.25932698412698424}, "target_users": {"precision": 0.2761333333333333, "recall": 0.29611067821067827, "f1_score": 0.2732862526362526}}}, "(0.7, 10, 0.7, 1)": {"precision": 0.2376528925619835, "recall": 0.1987057706144483, "f1_score": 0.2074310608437091, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 10, 0.7, 3)": {"precision": 0.2255270890725436, "recall": 0.25730131420461994, "f1_score": 0.22817953628862728, "per_field": {"categories": {"precision": 0.15358126721763085, "recall": 0.19787753568745306, "f1_score": 0.1628374655647383}, "scientific_domains": {"precision": 0.24666666666666665, "recall": 0.29607142857142854, "f1_score": 0.2586984126984128}, "target_users": {"precision": 0.2763333333333333, "recall": 0.27795497835497834, "f1_score": 0.26300273060273066}}}, "(0.7, 10, 0.7, 5)": {"precision": 0.22596841138659318, "recall": 0.2667332542266427, "f1_score": 0.23298361845786095, "per_field": {"categories": {"precision": 0.15537190082644628, "recall": 0.2065176558978212, "f1_score": 0.1663376186103459}, "scientific_domains": {"precision": 0.2464, "recall": 0.29757142857142854, "f1_score": 0.25932698412698424}, "target_users": {"precision": 0.2761333333333333, "recall": 0.29611067821067827, "f1_score": 0.2732862526362526}}}, "(0.7, 10, 0.8, 1)": {"precision": 0.2376528925619835, "recall": 0.1987057706144483, "f1_score": 0.2074310608437091, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 10, 0.8, 3)": {"precision": 0.2255270890725436, "recall": 0.25730131420461994, "f1_score": 0.22817953628862728, "per_field": {"categories": {"precision": 0.15358126721763085, "recall": 0.19787753568745306, "f1_score": 0.1628374655647383}, "scientific_domains": {"precision": 0.24666666666666665, "recall": 0.29607142857142854, "f1_score": 0.2586984126984128}, "target_users": {"precision": 0.2763333333333333, "recall": 0.27795497835497834, "f1_score": 0.26300273060273066}}}, "(0.7, 10, 0.8, 5)": {"precision": 0.22596841138659318, "recall": 0.2667332542266427, "f1_score": 0.23298361845786095, "per_field": {"categories": {"precision": 0.15537190082644628, "recall": 0.2065176558978212, "f1_score": 0.1663376186103459}, "scientific_domains": {"precision": 0.2464, "recall": 0.29757142857142854, "f1_score": 0.25932698412698424}, "target_users": {"precision": 0.2761333333333333, "recall": 0.29611067821067827, "f1_score": 0.2732862526362526}}}, "(0.7, 10, 0.9, 1)": {"precision": 0.2376528925619835, "recall": 0.1987057706144483, "f1_score": 0.2074310608437091, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 10, 0.9, 3)": {"precision": 0.2255270890725436, "recall": 0.25730131420461994, "f1_score": 0.22817953628862728, "per_field": {"categories": {"precision": 0.15358126721763085, "recall": 0.19787753568745306, "f1_score": 0.1628374655647383}, "scientific_domains": {"precision": 0.24666666666666665, "recall": 0.29607142857142854, "f1_score": 0.2586984126984128}, "target_users": {"precision": 0.2763333333333333, "recall": 0.27795497835497834, "f1_score": 0.26300273060273066}}}, "(0.7, 10, 0.9, 5)": {"precision": 0.22596841138659318, "recall": 0.2667332542266427, "f1_score": 0.23298361845786095, "per_field": {"categories": {"precision": 0.15537190082644628, "recall": 0.2065176558978212, "f1_score": 0.1663376186103459}, "scientific_domains": {"precision": 0.2464, "recall": 0.29757142857142854, "f1_score": 0.25932698412698424}, "target_users": {"precision": 0.2761333333333333, "recall": 0.29611067821067827, "f1_score": 0.2732862526362526}}}, "(0.7, 15, 0, 1)": {"precision": 0.23558677685950413, "recall": 0.19663965491196897, "f1_score": 0.20536494514122974, "per_field": {"categories": {"precision": 0.1487603305785124, "recall": 0.13965063861758076, "f1_score": 0.14154389747275123}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 15, 0, 3)": {"precision": 0.2308971533516988, "recall": 0.2552792756371269, "f1_score": 0.23176080350625808, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.19581141998497373, "f1_score": 0.1652479338842975}, "scientific_domains": {"precision": 0.24833333333333332, "recall": 0.29207142857142854, "f1_score": 0.2596984126984128}, "target_users": {"precision": 0.2873333333333333, "recall": 0.27795497835497834, "f1_score": 0.27033606393606396}}}, "(0.7, 15, 0, 5)": {"precision": 0.2313384756657484, "recall": 0.26471121565914957, "f1_score": 0.2365648856754918, "per_field": {"categories": {"precision": 0.15881542699724518, "recall": 0.20445154019534187, "f1_score": 0.16874808692990512}, "scientific_domains": {"precision": 0.24806666666666666, "recall": 0.29357142857142854, "f1_score": 0.26032698412698424}, "target_users": {"precision": 0.2871333333333333, "recall": 0.29611067821067827, "f1_score": 0.280619585969586}}}, "(0.7, 15, 0.1, 1)": {"precision": 0.23558677685950413, "recall": 0.19663965491196897, "f1_score": 0.20536494514122974, "per_field": {"categories": {"precision": 0.1487603305785124, "recall": 0.13965063861758076, "f1_score": 0.14154389747275123}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 15, 0.1, 3)": {"precision": 0.2308971533516988, "recall": 0.2552792756371269, "f1_score": 0.23176080350625808, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.19581141998497373, "f1_score": 0.1652479338842975}, "scientific_domains": {"precision": 0.24833333333333332, "recall": 0.29207142857142854, "f1_score": 0.2596984126984128}, "target_users": {"precision": 0.2873333333333333, "recall": 0.27795497835497834, "f1_score": 0.27033606393606396}}}, "(0.7, 15, 0.1, 5)": {"precision": 0.2313384756657484, "recall": 0.26471121565914957, "f1_score": 0.2365648856754918, "per_field": {"categories": {"precision": 0.15881542699724518, "recall": 0.20445154019534187, "f1_score": 0.16874808692990512}, "scientific_domains": {"precision": 0.24806666666666666, "recall": 0.29357142857142854, "f1_score": 0.26032698412698424}, "target_users": {"precision": 0.2871333333333333, "recall": 0.29611067821067827, "f1_score": 0.280619585969586}}}, "(0.7, 15, 0.2, 1)": {"precision": 0.23558677685950413, "recall": 0.19663965491196897, "f1_score": 0.20536494514122974, "per_field": {"categories": {"precision": 0.1487603305785124, "recall": 0.13965063861758076, "f1_score": 0.14154389747275123}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 15, 0.2, 3)": {"precision": 0.2308971533516988, "recall": 0.2552792756371269, "f1_score": 0.23176080350625808, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.19581141998497373, "f1_score": 0.1652479338842975}, "scientific_domains": {"precision": 0.24833333333333332, "recall": 0.29207142857142854, "f1_score": 0.2596984126984128}, "target_users": {"precision": 0.2873333333333333, "recall": 0.27795497835497834, "f1_score": 0.27033606393606396}}}, "(0.7, 15, 0.2, 5)": {"precision": 0.2313384756657484, "recall": 0.26471121565914957, "f1_score": 0.2365648856754918, "per_field": {"categories": {"precision": 0.15881542699724518, "recall": 0.20445154019534187, "f1_score": 0.16874808692990512}, "scientific_domains": {"precision": 0.24806666666666666, "recall": 0.29357142857142854, "f1_score": 0.26032698412698424}, "target_users": {"precision": 0.2871333333333333, "recall": 0.29611067821067827, "f1_score": 0.280619585969586}}}, "(0.7, 15, 0.3, 1)": {"precision": 0.23558677685950413, "recall": 0.19663965491196897, "f1_score": 0.20536494514122974, "per_field": {"categories": {"precision": 0.1487603305785124, "recall": 0.13965063861758076, "f1_score": 0.14154389747275123}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 15, 0.3, 3)": {"precision": 0.2308971533516988, "recall": 0.2552792756371269, "f1_score": 0.23176080350625808, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.19581141998497373, "f1_score": 0.1652479338842975}, "scientific_domains": {"precision": 0.24833333333333332, "recall": 0.29207142857142854, "f1_score": 0.2596984126984128}, "target_users": {"precision": 0.2873333333333333, "recall": 0.27795497835497834, "f1_score": 0.27033606393606396}}}, "(0.7, 15, 0.3, 5)": {"precision": 0.2313384756657484, "recall": 0.26471121565914957, "f1_score": 0.2365648856754918, "per_field": {"categories": {"precision": 0.15881542699724518, "recall": 0.20445154019534187, "f1_score": 0.16874808692990512}, "scientific_domains": {"precision": 0.24806666666666666, "recall": 0.29357142857142854, "f1_score": 0.26032698412698424}, "target_users": {"precision": 0.2871333333333333, "recall": 0.29611067821067827, "f1_score": 0.280619585969586}}}, "(0.7, 15, 0.4, 1)": {"precision": 0.23558677685950413, "recall": 0.19663965491196897, "f1_score": 0.20536494514122974, "per_field": {"categories": {"precision": 0.1487603305785124, "recall": 0.13965063861758076, "f1_score": 0.14154389747275123}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 15, 0.4, 3)": {"precision": 0.2308971533516988, "recall": 0.2552792756371269, "f1_score": 0.23176080350625808, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.19581141998497373, "f1_score": 0.1652479338842975}, "scientific_domains": {"precision": 0.24833333333333332, "recall": 0.29207142857142854, "f1_score": 0.2596984126984128}, "target_users": {"precision": 0.2873333333333333, "recall": 0.27795497835497834, "f1_score": 0.27033606393606396}}}, "(0.7, 15, 0.4, 5)": {"precision": 0.2313384756657484, "recall": 0.26471121565914957, "f1_score": 0.2365648856754918, "per_field": {"categories": {"precision": 0.15881542699724518, "recall": 0.20445154019534187, "f1_score": 0.16874808692990512}, "scientific_domains": {"precision": 0.24806666666666666, "recall": 0.29357142857142854, "f1_score": 0.26032698412698424}, "target_users": {"precision": 0.2871333333333333, "recall": 0.29611067821067827, "f1_score": 0.280619585969586}}}, "(0.7, 15, 0.5, 1)": {"precision": 0.23558677685950413, "recall": 0.19663965491196897, "f1_score": 0.20536494514122974, "per_field": {"categories": {"precision": 0.1487603305785124, "recall": 0.13965063861758076, "f1_score": 0.14154389747275123}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 15, 0.5, 3)": {"precision": 0.2308971533516988, "recall": 0.2552792756371269, "f1_score": 0.23176080350625808, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.19581141998497373, "f1_score": 0.1652479338842975}, "scientific_domains": {"precision": 0.24833333333333332, "recall": 0.29207142857142854, "f1_score": 0.2596984126984128}, "target_users": {"precision": 0.2873333333333333, "recall": 0.27795497835497834, "f1_score": 0.27033606393606396}}}, "(0.7, 15, 0.5, 5)": {"precision": 0.2313384756657484, "recall": 0.26471121565914957, "f1_score": 0.2365648856754918, "per_field": {"categories": {"precision": 0.15881542699724518, "recall": 0.20445154019534187, "f1_score": 0.16874808692990512}, "scientific_domains": {"precision": 0.24806666666666666, "recall": 0.29357142857142854, "f1_score": 0.26032698412698424}, "target_users": {"precision": 0.2871333333333333, "recall": 0.29611067821067827, "f1_score": 0.280619585969586}}}, "(0.7, 15, 0.6, 1)": {"precision": 0.23558677685950413, "recall": 0.19663965491196897, "f1_score": 0.20536494514122974, "per_field": {"categories": {"precision": 0.1487603305785124, "recall": 0.13965063861758076, "f1_score": 0.14154389747275123}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 15, 0.6, 3)": {"precision": 0.2308971533516988, "recall": 0.2552792756371269, "f1_score": 0.23176080350625808, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.19581141998497373, "f1_score": 0.1652479338842975}, "scientific_domains": {"precision": 0.24833333333333332, "recall": 0.29207142857142854, "f1_score": 0.2596984126984128}, "target_users": {"precision": 0.2873333333333333, "recall": 0.27795497835497834, "f1_score": 0.27033606393606396}}}, "(0.7, 15, 0.6, 5)": {"precision": 0.2313384756657484, "recall": 0.26471121565914957, "f1_score": 0.2365648856754918, "per_field": {"categories": {"precision": 0.15881542699724518, "recall": 0.20445154019534187, "f1_score": 0.16874808692990512}, "scientific_domains": {"precision": 0.24806666666666666, "recall": 0.29357142857142854, "f1_score": 0.26032698412698424}, "target_users": {"precision": 0.2871333333333333, "recall": 0.29611067821067827, "f1_score": 0.280619585969586}}}, "(0.7, 15, 0.7, 1)": {"precision": 0.23558677685950413, "recall": 0.19663965491196897, "f1_score": 0.20536494514122974, "per_field": {"categories": {"precision": 0.1487603305785124, "recall": 0.13965063861758076, "f1_score": 0.14154389747275123}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 15, 0.7, 3)": {"precision": 0.2308971533516988, "recall": 0.2552792756371269, "f1_score": 0.23176080350625808, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.19581141998497373, "f1_score": 0.1652479338842975}, "scientific_domains": {"precision": 0.24833333333333332, "recall": 0.29207142857142854, "f1_score": 0.2596984126984128}, "target_users": {"precision": 0.2873333333333333, "recall": 0.27795497835497834, "f1_score": 0.27033606393606396}}}, "(0.7, 15, 0.7, 5)": {"precision": 0.2313384756657484, "recall": 0.26471121565914957, "f1_score": 0.2365648856754918, "per_field": {"categories": {"precision": 0.15881542699724518, "recall": 0.20445154019534187, "f1_score": 0.16874808692990512}, "scientific_domains": {"precision": 0.24806666666666666, "recall": 0.29357142857142854, "f1_score": 0.26032698412698424}, "target_users": {"precision": 0.2871333333333333, "recall": 0.29611067821067827, "f1_score": 0.280619585969586}}}, "(0.7, 15, 0.8, 1)": {"precision": 0.23558677685950413, "recall": 0.19663965491196897, "f1_score": 0.20536494514122974, "per_field": {"categories": {"precision": 0.1487603305785124, "recall": 0.13965063861758076, "f1_score": 0.14154389747275123}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 15, 0.8, 3)": {"precision": 0.2308971533516988, "recall": 0.2552792756371269, "f1_score": 0.23176080350625808, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.19581141998497373, "f1_score": 0.1652479338842975}, "scientific_domains": {"precision": 0.24833333333333332, "recall": 0.29207142857142854, "f1_score": 0.2596984126984128}, "target_users": {"precision": 0.2873333333333333, "recall": 0.27795497835497834, "f1_score": 0.27033606393606396}}}, "(0.7, 15, 0.8, 5)": {"precision": 0.2313384756657484, "recall": 0.26471121565914957, "f1_score": 0.2365648856754918, "per_field": {"categories": {"precision": 0.15881542699724518, "recall": 0.20445154019534187, "f1_score": 0.16874808692990512}, "scientific_domains": {"precision": 0.24806666666666666, "recall": 0.29357142857142854, "f1_score": 0.26032698412698424}, "target_users": {"precision": 0.2871333333333333, "recall": 0.29611067821067827, "f1_score": 0.280619585969586}}}, "(0.7, 15, 0.9, 1)": {"precision": 0.23558677685950413, "recall": 0.19663965491196897, "f1_score": 0.20536494514122974, "per_field": {"categories": {"precision": 0.1487603305785124, "recall": 0.13965063861758076, "f1_score": 0.14154389747275123}, "scientific_domains": {"precision": 0.268, "recall": 0.2451190476190476, "f1_score": 0.2515666666666667}, "target_users": {"precision": 0.29, "recall": 0.20514927849927853, "f1_score": 0.22298427128427137}}}, "(0.7, 15, 0.9, 3)": {"precision": 0.2308971533516988, "recall": 0.2552792756371269, "f1_score": 0.23176080350625808, "per_field": {"categories": {"precision": 0.15702479338842976, "recall": 0.19581141998497373, "f1_score": 0.1652479338842975}, "scientific_domains": {"precision": 0.24833333333333332, "recall": 0.29207142857142854, "f1_score": 0.2596984126984128}, "target_users": {"precision": 0.2873333333333333, "recall": 0.27795497835497834, "f1_score": 0.27033606393606396}}}, "(0.7, 15, 0.9, 5)": {"precision": 0.2313384756657484, "recall": 0.26471121565914957, "f1_score": 0.2365648856754918, "per_field": {"categories": {"precision": 0.15881542699724518, "recall": 0.20445154019534187, "f1_score": 0.16874808692990512}, "scientific_domains": {"precision": 0.24806666666666666, "recall": 0.29357142857142854, "f1_score": 0.26032698412698424}, "target_users": {"precision": 0.2871333333333333, "recall": 0.29611067821067827, "f1_score": 0.280619585969586}}}, "(0.8, 5, 0, 1)": {"precision": 0.19965289256198346, "recall": 0.16797719918587686, "f1_score": 0.17528344179609004, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.208, "recall": 0.18811904761904763, "f1_score": 0.19356666666666666}, "target_users": {"precision": 0.236, "recall": 0.1699635642135642, "f1_score": 0.1845414141414142}}}, "(0.8, 5, 0, 3)": {"precision": 0.1902607897153352, "recall": 0.20720650900981477, "f1_score": 0.1899882992428447, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1751502629601803, "f1_score": 0.15353994490358125}, "scientific_domains": {"precision": 0.1946666666666667, "recall": 0.22107142857142859, "f1_score": 0.20069841269841274}, "target_users": {"precision": 0.22666666666666666, "recall": 0.22539783549783546, "f1_score": 0.21572654012654008}}}, "(0.8, 5, 0, 5)": {"precision": 0.19031634527089072, "recall": 0.21170426301583326, "f1_score": 0.19246901790689666, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1755259203606311, "f1_score": 0.15407866544230178}, "scientific_domains": {"precision": 0.19440000000000004, "recall": 0.2225714285714286, "f1_score": 0.2013269841269842}, "target_users": {"precision": 0.22709999999999997, "recall": 0.23701544011544007, "f1_score": 0.22200140415140404}}}, "(0.8, 5, 0.1, 1)": {"precision": 0.19965289256198346, "recall": 0.16797719918587686, "f1_score": 0.17528344179609004, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.208, "recall": 0.18811904761904763, "f1_score": 0.19356666666666666}, "target_users": {"precision": 0.236, "recall": 0.1699635642135642, "f1_score": 0.1845414141414142}}}, "(0.8, 5, 0.1, 3)": {"precision": 0.1902607897153352, "recall": 0.20720650900981477, "f1_score": 0.1899882992428447, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1751502629601803, "f1_score": 0.15353994490358125}, "scientific_domains": {"precision": 0.1946666666666667, "recall": 0.22107142857142859, "f1_score": 0.20069841269841274}, "target_users": {"precision": 0.22666666666666666, "recall": 0.22539783549783546, "f1_score": 0.21572654012654008}}}, "(0.8, 5, 0.1, 5)": {"precision": 0.19031634527089072, "recall": 0.21170426301583326, "f1_score": 0.19246901790689666, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1755259203606311, "f1_score": 0.15407866544230178}, "scientific_domains": {"precision": 0.19440000000000004, "recall": 0.2225714285714286, "f1_score": 0.2013269841269842}, "target_users": {"precision": 0.22709999999999997, "recall": 0.23701544011544007, "f1_score": 0.22200140415140404}}}, "(0.8, 5, 0.2, 1)": {"precision": 0.19965289256198346, "recall": 0.16797719918587686, "f1_score": 0.17528344179609004, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.208, "recall": 0.18811904761904763, "f1_score": 0.19356666666666666}, "target_users": {"precision": 0.236, "recall": 0.1699635642135642, "f1_score": 0.1845414141414142}}}, "(0.8, 5, 0.2, 3)": {"precision": 0.1902607897153352, "recall": 0.20720650900981477, "f1_score": 0.1899882992428447, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1751502629601803, "f1_score": 0.15353994490358125}, "scientific_domains": {"precision": 0.1946666666666667, "recall": 0.22107142857142859, "f1_score": 0.20069841269841274}, "target_users": {"precision": 0.22666666666666666, "recall": 0.22539783549783546, "f1_score": 0.21572654012654008}}}, "(0.8, 5, 0.2, 5)": {"precision": 0.19031634527089072, "recall": 0.21170426301583326, "f1_score": 0.19246901790689666, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1755259203606311, "f1_score": 0.15407866544230178}, "scientific_domains": {"precision": 0.19440000000000004, "recall": 0.2225714285714286, "f1_score": 0.2013269841269842}, "target_users": {"precision": 0.22709999999999997, "recall": 0.23701544011544007, "f1_score": 0.22200140415140404}}}, "(0.8, 5, 0.3, 1)": {"precision": 0.19965289256198346, "recall": 0.16797719918587686, "f1_score": 0.17528344179609004, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.208, "recall": 0.18811904761904763, "f1_score": 0.19356666666666666}, "target_users": {"precision": 0.236, "recall": 0.1699635642135642, "f1_score": 0.1845414141414142}}}, "(0.8, 5, 0.3, 3)": {"precision": 0.1902607897153352, "recall": 0.20720650900981477, "f1_score": 0.1899882992428447, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1751502629601803, "f1_score": 0.15353994490358125}, "scientific_domains": {"precision": 0.1946666666666667, "recall": 0.22107142857142859, "f1_score": 0.20069841269841274}, "target_users": {"precision": 0.22666666666666666, "recall": 0.22539783549783546, "f1_score": 0.21572654012654008}}}, "(0.8, 5, 0.3, 5)": {"precision": 0.19031634527089072, "recall": 0.21170426301583326, "f1_score": 0.19246901790689666, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1755259203606311, "f1_score": 0.15407866544230178}, "scientific_domains": {"precision": 0.19440000000000004, "recall": 0.2225714285714286, "f1_score": 0.2013269841269842}, "target_users": {"precision": 0.22709999999999997, "recall": 0.23701544011544007, "f1_score": 0.22200140415140404}}}, "(0.8, 5, 0.4, 1)": {"precision": 0.19965289256198346, "recall": 0.16797719918587686, "f1_score": 0.17528344179609004, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.208, "recall": 0.18811904761904763, "f1_score": 0.19356666666666666}, "target_users": {"precision": 0.236, "recall": 0.1699635642135642, "f1_score": 0.1845414141414142}}}, "(0.8, 5, 0.4, 3)": {"precision": 0.1902607897153352, "recall": 0.20720650900981477, "f1_score": 0.1899882992428447, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1751502629601803, "f1_score": 0.15353994490358125}, "scientific_domains": {"precision": 0.1946666666666667, "recall": 0.22107142857142859, "f1_score": 0.20069841269841274}, "target_users": {"precision": 0.22666666666666666, "recall": 0.22539783549783546, "f1_score": 0.21572654012654008}}}, "(0.8, 5, 0.4, 5)": {"precision": 0.19031634527089072, "recall": 0.21170426301583326, "f1_score": 0.19246901790689666, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1755259203606311, "f1_score": 0.15407866544230178}, "scientific_domains": {"precision": 0.19440000000000004, "recall": 0.2225714285714286, "f1_score": 0.2013269841269842}, "target_users": {"precision": 0.22709999999999997, "recall": 0.23701544011544007, "f1_score": 0.22200140415140404}}}, "(0.8, 5, 0.5, 1)": {"precision": 0.19965289256198346, "recall": 0.16797719918587686, "f1_score": 0.17528344179609004, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.208, "recall": 0.18811904761904763, "f1_score": 0.19356666666666666}, "target_users": {"precision": 0.236, "recall": 0.1699635642135642, "f1_score": 0.1845414141414142}}}, "(0.8, 5, 0.5, 3)": {"precision": 0.1902607897153352, "recall": 0.20720650900981477, "f1_score": 0.1899882992428447, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1751502629601803, "f1_score": 0.15353994490358125}, "scientific_domains": {"precision": 0.1946666666666667, "recall": 0.22107142857142859, "f1_score": 0.20069841269841274}, "target_users": {"precision": 0.22666666666666666, "recall": 0.22539783549783546, "f1_score": 0.21572654012654008}}}, "(0.8, 5, 0.5, 5)": {"precision": 0.19031634527089072, "recall": 0.21170426301583326, "f1_score": 0.19246901790689666, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1755259203606311, "f1_score": 0.15407866544230178}, "scientific_domains": {"precision": 0.19440000000000004, "recall": 0.2225714285714286, "f1_score": 0.2013269841269842}, "target_users": {"precision": 0.22709999999999997, "recall": 0.23701544011544007, "f1_score": 0.22200140415140404}}}, "(0.8, 5, 0.6, 1)": {"precision": 0.19965289256198346, "recall": 0.16797719918587686, "f1_score": 0.17528344179609004, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.208, "recall": 0.18811904761904763, "f1_score": 0.19356666666666666}, "target_users": {"precision": 0.236, "recall": 0.1699635642135642, "f1_score": 0.1845414141414142}}}, "(0.8, 5, 0.6, 3)": {"precision": 0.1902607897153352, "recall": 0.20720650900981477, "f1_score": 0.1899882992428447, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1751502629601803, "f1_score": 0.15353994490358125}, "scientific_domains": {"precision": 0.1946666666666667, "recall": 0.22107142857142859, "f1_score": 0.20069841269841274}, "target_users": {"precision": 0.22666666666666666, "recall": 0.22539783549783546, "f1_score": 0.21572654012654008}}}, "(0.8, 5, 0.6, 5)": {"precision": 0.19031634527089072, "recall": 0.21170426301583326, "f1_score": 0.19246901790689666, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1755259203606311, "f1_score": 0.15407866544230178}, "scientific_domains": {"precision": 0.19440000000000004, "recall": 0.2225714285714286, "f1_score": 0.2013269841269842}, "target_users": {"precision": 0.22709999999999997, "recall": 0.23701544011544007, "f1_score": 0.22200140415140404}}}, "(0.8, 5, 0.7, 1)": {"precision": 0.19965289256198346, "recall": 0.16797719918587686, "f1_score": 0.17528344179609004, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.208, "recall": 0.18811904761904763, "f1_score": 0.19356666666666666}, "target_users": {"precision": 0.236, "recall": 0.1699635642135642, "f1_score": 0.1845414141414142}}}, "(0.8, 5, 0.7, 3)": {"precision": 0.1902607897153352, "recall": 0.20720650900981477, "f1_score": 0.1899882992428447, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1751502629601803, "f1_score": 0.15353994490358125}, "scientific_domains": {"precision": 0.1946666666666667, "recall": 0.22107142857142859, "f1_score": 0.20069841269841274}, "target_users": {"precision": 0.22666666666666666, "recall": 0.22539783549783546, "f1_score": 0.21572654012654008}}}, "(0.8, 5, 0.7, 5)": {"precision": 0.19031634527089072, "recall": 0.21170426301583326, "f1_score": 0.19246901790689666, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1755259203606311, "f1_score": 0.15407866544230178}, "scientific_domains": {"precision": 0.19440000000000004, "recall": 0.2225714285714286, "f1_score": 0.2013269841269842}, "target_users": {"precision": 0.22709999999999997, "recall": 0.23701544011544007, "f1_score": 0.22200140415140404}}}, "(0.8, 5, 0.8, 1)": {"precision": 0.19965289256198346, "recall": 0.16797719918587686, "f1_score": 0.17528344179609004, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.208, "recall": 0.18811904761904763, "f1_score": 0.19356666666666666}, "target_users": {"precision": 0.236, "recall": 0.1699635642135642, "f1_score": 0.1845414141414142}}}, "(0.8, 5, 0.8, 3)": {"precision": 0.1902607897153352, "recall": 0.20720650900981477, "f1_score": 0.1899882992428447, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1751502629601803, "f1_score": 0.15353994490358125}, "scientific_domains": {"precision": 0.1946666666666667, "recall": 0.22107142857142859, "f1_score": 0.20069841269841274}, "target_users": {"precision": 0.22666666666666666, "recall": 0.22539783549783546, "f1_score": 0.21572654012654008}}}, "(0.8, 5, 0.8, 5)": {"precision": 0.19031634527089072, "recall": 0.21170426301583326, "f1_score": 0.19246901790689666, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1755259203606311, "f1_score": 0.15407866544230178}, "scientific_domains": {"precision": 0.19440000000000004, "recall": 0.2225714285714286, "f1_score": 0.2013269841269842}, "target_users": {"precision": 0.22709999999999997, "recall": 0.23701544011544007, "f1_score": 0.22200140415140404}}}, "(0.8, 5, 0.9, 1)": {"precision": 0.19965289256198346, "recall": 0.16797719918587686, "f1_score": 0.17528344179609004, "per_field": {"categories": {"precision": 0.15495867768595042, "recall": 0.14584898572501878, "f1_score": 0.14774224458018925}, "scientific_domains": {"precision": 0.208, "recall": 0.18811904761904763, "f1_score": 0.19356666666666666}, "target_users": {"precision": 0.236, "recall": 0.1699635642135642, "f1_score": 0.1845414141414142}}}, "(0.8, 5, 0.9, 3)": {"precision": 0.1902607897153352, "recall": 0.20720650900981477, "f1_score": 0.1899882992428447, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1751502629601803, "f1_score": 0.15353994490358125}, "scientific_domains": {"precision": 0.1946666666666667, "recall": 0.22107142857142859, "f1_score": 0.20069841269841274}, "target_users": {"precision": 0.22666666666666666, "recall": 0.22539783549783546, "f1_score": 0.21572654012654008}}}, "(0.8, 5, 0.9, 5)": {"precision": 0.19031634527089072, "recall": 0.21170426301583326, "f1_score": 0.19246901790689666, "per_field": {"categories": {"precision": 0.14944903581267216, "recall": 0.1755259203606311, "f1_score": 0.15407866544230178}, "scientific_domains": {"precision": 0.19440000000000004, "recall": 0.2225714285714286, "f1_score": 0.2013269841269842}, "target_users": {"precision": 0.22709999999999997, "recall": 0.23701544011544007, "f1_score": 0.22200140415140404}}}, "(0.8, 10, 0, 1)": {"precision": 0.19887603305785126, "recall": 0.16653367301507796, "f1_score": 0.17406213784751337, "per_field": {"categories": {"precision": 0.1446280991735537, "recall": 0.1355184072126221, "f1_score": 0.13741166606779254}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 10, 0, 3)": {"precision": 0.18573186409550047, "recall": 0.208517803775655, "f1_score": 0.18691851044578314, "per_field": {"categories": {"precision": 0.1415289256198347, "recall": 0.17308414725770097, "f1_score": 0.14699724517906332}, "scientific_domains": {"precision": 0.19500000000000003, "recall": 0.2270714285714286, "f1_score": 0.20203174603174612}, "target_users": {"precision": 0.22066666666666665, "recall": 0.22539783549783546, "f1_score": 0.21172654012654005}}}, "(0.8, 10, 0, 5)": {"precision": 0.1861317722681359, "recall": 0.21439296824999302, "f1_score": 0.18995019329716298, "per_field": {"categories": {"precision": 0.14256198347107438, "recall": 0.17759203606311044, "f1_score": 0.14918885827976736}, "scientific_domains": {"precision": 0.19473333333333337, "recall": 0.2285714285714286, "f1_score": 0.2026603174603175}, "target_users": {"precision": 0.22109999999999996, "recall": 0.23701544011544007, "f1_score": 0.21800140415140404}}}, "(0.8, 10, 0.1, 1)": {"precision": 0.19887603305785126, "recall": 0.16653367301507796, "f1_score": 0.17406213784751337, "per_field": {"categories": {"precision": 0.1446280991735537, "recall": 0.1355184072126221, "f1_score": 0.13741166606779254}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 10, 0.1, 3)": {"precision": 0.18573186409550047, "recall": 0.208517803775655, "f1_score": 0.18691851044578314, "per_field": {"categories": {"precision": 0.1415289256198347, "recall": 0.17308414725770097, "f1_score": 0.14699724517906332}, "scientific_domains": {"precision": 0.19500000000000003, "recall": 0.2270714285714286, "f1_score": 0.20203174603174612}, "target_users": {"precision": 0.22066666666666665, "recall": 0.22539783549783546, "f1_score": 0.21172654012654005}}}, "(0.8, 10, 0.1, 5)": {"precision": 0.1861317722681359, "recall": 0.21439296824999302, "f1_score": 0.18995019329716298, "per_field": {"categories": {"precision": 0.14256198347107438, "recall": 0.17759203606311044, "f1_score": 0.14918885827976736}, "scientific_domains": {"precision": 0.19473333333333337, "recall": 0.2285714285714286, "f1_score": 0.2026603174603175}, "target_users": {"precision": 0.22109999999999996, "recall": 0.23701544011544007, "f1_score": 0.21800140415140404}}}, "(0.8, 10, 0.2, 1)": {"precision": 0.19887603305785126, "recall": 0.16653367301507796, "f1_score": 0.17406213784751337, "per_field": {"categories": {"precision": 0.1446280991735537, "recall": 0.1355184072126221, "f1_score": 0.13741166606779254}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 10, 0.2, 3)": {"precision": 0.18573186409550047, "recall": 0.208517803775655, "f1_score": 0.18691851044578314, "per_field": {"categories": {"precision": 0.1415289256198347, "recall": 0.17308414725770097, "f1_score": 0.14699724517906332}, "scientific_domains": {"precision": 0.19500000000000003, "recall": 0.2270714285714286, "f1_score": 0.20203174603174612}, "target_users": {"precision": 0.22066666666666665, "recall": 0.22539783549783546, "f1_score": 0.21172654012654005}}}, "(0.8, 10, 0.2, 5)": {"precision": 0.1861317722681359, "recall": 0.21439296824999302, "f1_score": 0.18995019329716298, "per_field": {"categories": {"precision": 0.14256198347107438, "recall": 0.17759203606311044, "f1_score": 0.14918885827976736}, "scientific_domains": {"precision": 0.19473333333333337, "recall": 0.2285714285714286, "f1_score": 0.2026603174603175}, "target_users": {"precision": 0.22109999999999996, "recall": 0.23701544011544007, "f1_score": 0.21800140415140404}}}, "(0.8, 10, 0.3, 1)": {"precision": 0.19887603305785126, "recall": 0.16653367301507796, "f1_score": 0.17406213784751337, "per_field": {"categories": {"precision": 0.1446280991735537, "recall": 0.1355184072126221, "f1_score": 0.13741166606779254}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 10, 0.3, 3)": {"precision": 0.18573186409550047, "recall": 0.208517803775655, "f1_score": 0.18691851044578314, "per_field": {"categories": {"precision": 0.1415289256198347, "recall": 0.17308414725770097, "f1_score": 0.14699724517906332}, "scientific_domains": {"precision": 0.19500000000000003, "recall": 0.2270714285714286, "f1_score": 0.20203174603174612}, "target_users": {"precision": 0.22066666666666665, "recall": 0.22539783549783546, "f1_score": 0.21172654012654005}}}, "(0.8, 10, 0.3, 5)": {"precision": 0.1861317722681359, "recall": 0.21439296824999302, "f1_score": 0.18995019329716298, "per_field": {"categories": {"precision": 0.14256198347107438, "recall": 0.17759203606311044, "f1_score": 0.14918885827976736}, "scientific_domains": {"precision": 0.19473333333333337, "recall": 0.2285714285714286, "f1_score": 0.2026603174603175}, "target_users": {"precision": 0.22109999999999996, "recall": 0.23701544011544007, "f1_score": 0.21800140415140404}}}, "(0.8, 10, 0.4, 1)": {"precision": 0.19887603305785126, "recall": 0.16653367301507796, "f1_score": 0.17406213784751337, "per_field": {"categories": {"precision": 0.1446280991735537, "recall": 0.1355184072126221, "f1_score": 0.13741166606779254}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 10, 0.4, 3)": {"precision": 0.18573186409550047, "recall": 0.208517803775655, "f1_score": 0.18691851044578314, "per_field": {"categories": {"precision": 0.1415289256198347, "recall": 0.17308414725770097, "f1_score": 0.14699724517906332}, "scientific_domains": {"precision": 0.19500000000000003, "recall": 0.2270714285714286, "f1_score": 0.20203174603174612}, "target_users": {"precision": 0.22066666666666665, "recall": 0.22539783549783546, "f1_score": 0.21172654012654005}}}, "(0.8, 10, 0.4, 5)": {"precision": 0.1861317722681359, "recall": 0.21439296824999302, "f1_score": 0.18995019329716298, "per_field": {"categories": {"precision": 0.14256198347107438, "recall": 0.17759203606311044, "f1_score": 0.14918885827976736}, "scientific_domains": {"precision": 0.19473333333333337, "recall": 0.2285714285714286, "f1_score": 0.2026603174603175}, "target_users": {"precision": 0.22109999999999996, "recall": 0.23701544011544007, "f1_score": 0.21800140415140404}}}, "(0.8, 10, 0.5, 1)": {"precision": 0.19887603305785126, "recall": 0.16653367301507796, "f1_score": 0.17406213784751337, "per_field": {"categories": {"precision": 0.1446280991735537, "recall": 0.1355184072126221, "f1_score": 0.13741166606779254}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 10, 0.5, 3)": {"precision": 0.18573186409550047, "recall": 0.208517803775655, "f1_score": 0.18691851044578314, "per_field": {"categories": {"precision": 0.1415289256198347, "recall": 0.17308414725770097, "f1_score": 0.14699724517906332}, "scientific_domains": {"precision": 0.19500000000000003, "recall": 0.2270714285714286, "f1_score": 0.20203174603174612}, "target_users": {"precision": 0.22066666666666665, "recall": 0.22539783549783546, "f1_score": 0.21172654012654005}}}, "(0.8, 10, 0.5, 5)": {"precision": 0.1861317722681359, "recall": 0.21439296824999302, "f1_score": 0.18995019329716298, "per_field": {"categories": {"precision": 0.14256198347107438, "recall": 0.17759203606311044, "f1_score": 0.14918885827976736}, "scientific_domains": {"precision": 0.19473333333333337, "recall": 0.2285714285714286, "f1_score": 0.2026603174603175}, "target_users": {"precision": 0.22109999999999996, "recall": 0.23701544011544007, "f1_score": 0.21800140415140404}}}, "(0.8, 10, 0.6, 1)": {"precision": 0.19887603305785126, "recall": 0.16653367301507796, "f1_score": 0.17406213784751337, "per_field": {"categories": {"precision": 0.1446280991735537, "recall": 0.1355184072126221, "f1_score": 0.13741166606779254}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 10, 0.6, 3)": {"precision": 0.18573186409550047, "recall": 0.208517803775655, "f1_score": 0.18691851044578314, "per_field": {"categories": {"precision": 0.1415289256198347, "recall": 0.17308414725770097, "f1_score": 0.14699724517906332}, "scientific_domains": {"precision": 0.19500000000000003, "recall": 0.2270714285714286, "f1_score": 0.20203174603174612}, "target_users": {"precision": 0.22066666666666665, "recall": 0.22539783549783546, "f1_score": 0.21172654012654005}}}, "(0.8, 10, 0.6, 5)": {"precision": 0.1861317722681359, "recall": 0.21439296824999302, "f1_score": 0.18995019329716298, "per_field": {"categories": {"precision": 0.14256198347107438, "recall": 0.17759203606311044, "f1_score": 0.14918885827976736}, "scientific_domains": {"precision": 0.19473333333333337, "recall": 0.2285714285714286, "f1_score": 0.2026603174603175}, "target_users": {"precision": 0.22109999999999996, "recall": 0.23701544011544007, "f1_score": 0.21800140415140404}}}, "(0.8, 10, 0.7, 1)": {"precision": 0.19887603305785126, "recall": 0.16653367301507796, "f1_score": 0.17406213784751337, "per_field": {"categories": {"precision": 0.1446280991735537, "recall": 0.1355184072126221, "f1_score": 0.13741166606779254}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 10, 0.7, 3)": {"precision": 0.18573186409550047, "recall": 0.208517803775655, "f1_score": 0.18691851044578314, "per_field": {"categories": {"precision": 0.1415289256198347, "recall": 0.17308414725770097, "f1_score": 0.14699724517906332}, "scientific_domains": {"precision": 0.19500000000000003, "recall": 0.2270714285714286, "f1_score": 0.20203174603174612}, "target_users": {"precision": 0.22066666666666665, "recall": 0.22539783549783546, "f1_score": 0.21172654012654005}}}, "(0.8, 10, 0.7, 5)": {"precision": 0.1861317722681359, "recall": 0.21439296824999302, "f1_score": 0.18995019329716298, "per_field": {"categories": {"precision": 0.14256198347107438, "recall": 0.17759203606311044, "f1_score": 0.14918885827976736}, "scientific_domains": {"precision": 0.19473333333333337, "recall": 0.2285714285714286, "f1_score": 0.2026603174603175}, "target_users": {"precision": 0.22109999999999996, "recall": 0.23701544011544007, "f1_score": 0.21800140415140404}}}, "(0.8, 10, 0.8, 1)": {"precision": 0.19887603305785126, "recall": 0.16653367301507796, "f1_score": 0.17406213784751337, "per_field": {"categories": {"precision": 0.1446280991735537, "recall": 0.1355184072126221, "f1_score": 0.13741166606779254}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 10, 0.8, 3)": {"precision": 0.18573186409550047, "recall": 0.208517803775655, "f1_score": 0.18691851044578314, "per_field": {"categories": {"precision": 0.1415289256198347, "recall": 0.17308414725770097, "f1_score": 0.14699724517906332}, "scientific_domains": {"precision": 0.19500000000000003, "recall": 0.2270714285714286, "f1_score": 0.20203174603174612}, "target_users": {"precision": 0.22066666666666665, "recall": 0.22539783549783546, "f1_score": 0.21172654012654005}}}, "(0.8, 10, 0.8, 5)": {"precision": 0.1861317722681359, "recall": 0.21439296824999302, "f1_score": 0.18995019329716298, "per_field": {"categories": {"precision": 0.14256198347107438, "recall": 0.17759203606311044, "f1_score": 0.14918885827976736}, "scientific_domains": {"precision": 0.19473333333333337, "recall": 0.2285714285714286, "f1_score": 0.2026603174603175}, "target_users": {"precision": 0.22109999999999996, "recall": 0.23701544011544007, "f1_score": 0.21800140415140404}}}, "(0.8, 10, 0.9, 1)": {"precision": 0.19887603305785126, "recall": 0.16653367301507796, "f1_score": 0.17406213784751337, "per_field": {"categories": {"precision": 0.1446280991735537, "recall": 0.1355184072126221, "f1_score": 0.13741166606779254}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 10, 0.9, 3)": {"precision": 0.18573186409550047, "recall": 0.208517803775655, "f1_score": 0.18691851044578314, "per_field": {"categories": {"precision": 0.1415289256198347, "recall": 0.17308414725770097, "f1_score": 0.14699724517906332}, "scientific_domains": {"precision": 0.19500000000000003, "recall": 0.2270714285714286, "f1_score": 0.20203174603174612}, "target_users": {"precision": 0.22066666666666665, "recall": 0.22539783549783546, "f1_score": 0.21172654012654005}}}, "(0.8, 10, 0.9, 5)": {"precision": 0.1861317722681359, "recall": 0.21439296824999302, "f1_score": 0.18995019329716298, "per_field": {"categories": {"precision": 0.14256198347107438, "recall": 0.17759203606311044, "f1_score": 0.14918885827976736}, "scientific_domains": {"precision": 0.19473333333333337, "recall": 0.2285714285714286, "f1_score": 0.2026603174603175}, "target_users": {"precision": 0.22109999999999996, "recall": 0.23701544011544007, "f1_score": 0.21800140415140404}}}, "(0.8, 15, 0, 1)": {"precision": 0.20163085399449035, "recall": 0.16928849395171708, "f1_score": 0.1768169587841525, "per_field": {"categories": {"precision": 0.15289256198347106, "recall": 0.14378287002253945, "f1_score": 0.1456761288777099}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 15, 0, 3)": {"precision": 0.1935123966942149, "recall": 0.2064957652081619, "f1_score": 0.19210675654312018, "per_field": {"categories": {"precision": 0.1522038567493113, "recall": 0.17101803155522163, "f1_score": 0.154228650137741}, "scientific_domains": {"precision": 0.1966666666666667, "recall": 0.2230714285714286, "f1_score": 0.2030317460317461}, "target_users": {"precision": 0.23166666666666666, "recall": 0.22539783549783546, "f1_score": 0.21905987345987343}}}, "(0.8, 15, 0, 5)": {"precision": 0.19391230486685032, "recall": 0.21237092968249993, "f1_score": 0.1951384393945, "per_field": {"categories": {"precision": 0.15323691460055097, "recall": 0.1755259203606311, "f1_score": 0.15642026323844505}, "scientific_domains": {"precision": 0.19640000000000005, "recall": 0.2245714285714286, "f1_score": 0.2036603174603175}, "target_users": {"precision": 0.23209999999999997, "recall": 0.23701544011544007, "f1_score": 0.22533473748473742}}}, "(0.8, 15, 0.1, 1)": {"precision": 0.20163085399449035, "recall": 0.16928849395171708, "f1_score": 0.1768169587841525, "per_field": {"categories": {"precision": 0.15289256198347106, "recall": 0.14378287002253945, "f1_score": 0.1456761288777099}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 15, 0.1, 3)": {"precision": 0.1935123966942149, "recall": 0.2064957652081619, "f1_score": 0.19210675654312018, "per_field": {"categories": {"precision": 0.1522038567493113, "recall": 0.17101803155522163, "f1_score": 0.154228650137741}, "scientific_domains": {"precision": 0.1966666666666667, "recall": 0.2230714285714286, "f1_score": 0.2030317460317461}, "target_users": {"precision": 0.23166666666666666, "recall": 0.22539783549783546, "f1_score": 0.21905987345987343}}}, "(0.8, 15, 0.1, 5)": {"precision": 0.19391230486685032, "recall": 0.21237092968249993, "f1_score": 0.1951384393945, "per_field": {"categories": {"precision": 0.15323691460055097, "recall": 0.1755259203606311, "f1_score": 0.15642026323844505}, "scientific_domains": {"precision": 0.19640000000000005, "recall": 0.2245714285714286, "f1_score": 0.2036603174603175}, "target_users": {"precision": 0.23209999999999997, "recall": 0.23701544011544007, "f1_score": 0.22533473748473742}}}, "(0.8, 15, 0.2, 1)": {"precision": 0.20163085399449035, "recall": 0.16928849395171708, "f1_score": 0.1768169587841525, "per_field": {"categories": {"precision": 0.15289256198347106, "recall": 0.14378287002253945, "f1_score": 0.1456761288777099}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 15, 0.2, 3)": {"precision": 0.1935123966942149, "recall": 0.2064957652081619, "f1_score": 0.19210675654312018, "per_field": {"categories": {"precision": 0.1522038567493113, "recall": 0.17101803155522163, "f1_score": 0.154228650137741}, "scientific_domains": {"precision": 0.1966666666666667, "recall": 0.2230714285714286, "f1_score": 0.2030317460317461}, "target_users": {"precision": 0.23166666666666666, "recall": 0.22539783549783546, "f1_score": 0.21905987345987343}}}, "(0.8, 15, 0.2, 5)": {"precision": 0.19391230486685032, "recall": 0.21237092968249993, "f1_score": 0.1951384393945, "per_field": {"categories": {"precision": 0.15323691460055097, "recall": 0.1755259203606311, "f1_score": 0.15642026323844505}, "scientific_domains": {"precision": 0.19640000000000005, "recall": 0.2245714285714286, "f1_score": 0.2036603174603175}, "target_users": {"precision": 0.23209999999999997, "recall": 0.23701544011544007, "f1_score": 0.22533473748473742}}}, "(0.8, 15, 0.3, 1)": {"precision": 0.20163085399449035, "recall": 0.16928849395171708, "f1_score": 0.1768169587841525, "per_field": {"categories": {"precision": 0.15289256198347106, "recall": 0.14378287002253945, "f1_score": 0.1456761288777099}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 15, 0.3, 3)": {"precision": 0.1935123966942149, "recall": 0.2064957652081619, "f1_score": 0.19210675654312018, "per_field": {"categories": {"precision": 0.1522038567493113, "recall": 0.17101803155522163, "f1_score": 0.154228650137741}, "scientific_domains": {"precision": 0.1966666666666667, "recall": 0.2230714285714286, "f1_score": 0.2030317460317461}, "target_users": {"precision": 0.23166666666666666, "recall": 0.22539783549783546, "f1_score": 0.21905987345987343}}}, "(0.8, 15, 0.3, 5)": {"precision": 0.19391230486685032, "recall": 0.21237092968249993, "f1_score": 0.1951384393945, "per_field": {"categories": {"precision": 0.15323691460055097, "recall": 0.1755259203606311, "f1_score": 0.15642026323844505}, "scientific_domains": {"precision": 0.19640000000000005, "recall": 0.2245714285714286, "f1_score": 0.2036603174603175}, "target_users": {"precision": 0.23209999999999997, "recall": 0.23701544011544007, "f1_score": 0.22533473748473742}}}, "(0.8, 15, 0.4, 1)": {"precision": 0.20163085399449035, "recall": 0.16928849395171708, "f1_score": 0.1768169587841525, "per_field": {"categories": {"precision": 0.15289256198347106, "recall": 0.14378287002253945, "f1_score": 0.1456761288777099}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 15, 0.4, 3)": {"precision": 0.1935123966942149, "recall": 0.2064957652081619, "f1_score": 0.19210675654312018, "per_field": {"categories": {"precision": 0.1522038567493113, "recall": 0.17101803155522163, "f1_score": 0.154228650137741}, "scientific_domains": {"precision": 0.1966666666666667, "recall": 0.2230714285714286, "f1_score": 0.2030317460317461}, "target_users": {"precision": 0.23166666666666666, "recall": 0.22539783549783546, "f1_score": 0.21905987345987343}}}, "(0.8, 15, 0.4, 5)": {"precision": 0.19391230486685032, "recall": 0.21237092968249993, "f1_score": 0.1951384393945, "per_field": {"categories": {"precision": 0.15323691460055097, "recall": 0.1755259203606311, "f1_score": 0.15642026323844505}, "scientific_domains": {"precision": 0.19640000000000005, "recall": 0.2245714285714286, "f1_score": 0.2036603174603175}, "target_users": {"precision": 0.23209999999999997, "recall": 0.23701544011544007, "f1_score": 0.22533473748473742}}}, "(0.8, 15, 0.5, 1)": {"precision": 0.20163085399449035, "recall": 0.16928849395171708, "f1_score": 0.1768169587841525, "per_field": {"categories": {"precision": 0.15289256198347106, "recall": 0.14378287002253945, "f1_score": 0.1456761288777099}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 15, 0.5, 3)": {"precision": 0.1935123966942149, "recall": 0.2064957652081619, "f1_score": 0.19210675654312018, "per_field": {"categories": {"precision": 0.1522038567493113, "recall": 0.17101803155522163, "f1_score": 0.154228650137741}, "scientific_domains": {"precision": 0.1966666666666667, "recall": 0.2230714285714286, "f1_score": 0.2030317460317461}, "target_users": {"precision": 0.23166666666666666, "recall": 0.22539783549783546, "f1_score": 0.21905987345987343}}}, "(0.8, 15, 0.5, 5)": {"precision": 0.19391230486685032, "recall": 0.21237092968249993, "f1_score": 0.1951384393945, "per_field": {"categories": {"precision": 0.15323691460055097, "recall": 0.1755259203606311, "f1_score": 0.15642026323844505}, "scientific_domains": {"precision": 0.19640000000000005, "recall": 0.2245714285714286, "f1_score": 0.2036603174603175}, "target_users": {"precision": 0.23209999999999997, "recall": 0.23701544011544007, "f1_score": 0.22533473748473742}}}, "(0.8, 15, 0.6, 1)": {"precision": 0.20163085399449035, "recall": 0.16928849395171708, "f1_score": 0.1768169587841525, "per_field": {"categories": {"precision": 0.15289256198347106, "recall": 0.14378287002253945, "f1_score": 0.1456761288777099}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 15, 0.6, 3)": {"precision": 0.1935123966942149, "recall": 0.2064957652081619, "f1_score": 0.19210675654312018, "per_field": {"categories": {"precision": 0.1522038567493113, "recall": 0.17101803155522163, "f1_score": 0.154228650137741}, "scientific_domains": {"precision": 0.1966666666666667, "recall": 0.2230714285714286, "f1_score": 0.2030317460317461}, "target_users": {"precision": 0.23166666666666666, "recall": 0.22539783549783546, "f1_score": 0.21905987345987343}}}, "(0.8, 15, 0.6, 5)": {"precision": 0.19391230486685032, "recall": 0.21237092968249993, "f1_score": 0.1951384393945, "per_field": {"categories": {"precision": 0.15323691460055097, "recall": 0.1755259203606311, "f1_score": 0.15642026323844505}, "scientific_domains": {"precision": 0.19640000000000005, "recall": 0.2245714285714286, "f1_score": 0.2036603174603175}, "target_users": {"precision": 0.23209999999999997, "recall": 0.23701544011544007, "f1_score": 0.22533473748473742}}}, "(0.8, 15, 0.7, 1)": {"precision": 0.20163085399449035, "recall": 0.16928849395171708, "f1_score": 0.1768169587841525, "per_field": {"categories": {"precision": 0.15289256198347106, "recall": 0.14378287002253945, "f1_score": 0.1456761288777099}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 15, 0.7, 3)": {"precision": 0.1935123966942149, "recall": 0.2064957652081619, "f1_score": 0.19210675654312018, "per_field": {"categories": {"precision": 0.1522038567493113, "recall": 0.17101803155522163, "f1_score": 0.154228650137741}, "scientific_domains": {"precision": 0.1966666666666667, "recall": 0.2230714285714286, "f1_score": 0.2030317460317461}, "target_users": {"precision": 0.23166666666666666, "recall": 0.22539783549783546, "f1_score": 0.21905987345987343}}}, "(0.8, 15, 0.7, 5)": {"precision": 0.19391230486685032, "recall": 0.21237092968249993, "f1_score": 0.1951384393945, "per_field": {"categories": {"precision": 0.15323691460055097, "recall": 0.1755259203606311, "f1_score": 0.15642026323844505}, "scientific_domains": {"precision": 0.19640000000000005, "recall": 0.2245714285714286, "f1_score": 0.2036603174603175}, "target_users": {"precision": 0.23209999999999997, "recall": 0.23701544011544007, "f1_score": 0.22533473748473742}}}, "(0.8, 15, 0.8, 1)": {"precision": 0.20163085399449035, "recall": 0.16928849395171708, "f1_score": 0.1768169587841525, "per_field": {"categories": {"precision": 0.15289256198347106, "recall": 0.14378287002253945, "f1_score": 0.1456761288777099}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 15, 0.8, 3)": {"precision": 0.1935123966942149, "recall": 0.2064957652081619, "f1_score": 0.19210675654312018, "per_field": {"categories": {"precision": 0.1522038567493113, "recall": 0.17101803155522163, "f1_score": 0.154228650137741}, "scientific_domains": {"precision": 0.1966666666666667, "recall": 0.2230714285714286, "f1_score": 0.2030317460317461}, "target_users": {"precision": 0.23166666666666666, "recall": 0.22539783549783546, "f1_score": 0.21905987345987343}}}, "(0.8, 15, 0.8, 5)": {"precision": 0.19391230486685032, "recall": 0.21237092968249993, "f1_score": 0.1951384393945, "per_field": {"categories": {"precision": 0.15323691460055097, "recall": 0.1755259203606311, "f1_score": 0.15642026323844505}, "scientific_domains": {"precision": 0.19640000000000005, "recall": 0.2245714285714286, "f1_score": 0.2036603174603175}, "target_users": {"precision": 0.23209999999999997, "recall": 0.23701544011544007, "f1_score": 0.22533473748473742}}}, "(0.8, 15, 0.9, 1)": {"precision": 0.20163085399449035, "recall": 0.16928849395171708, "f1_score": 0.1768169587841525, "per_field": {"categories": {"precision": 0.15289256198347106, "recall": 0.14378287002253945, "f1_score": 0.1456761288777099}, "scientific_domains": {"precision": 0.212, "recall": 0.19211904761904763, "f1_score": 0.19756666666666667}, "target_users": {"precision": 0.24, "recall": 0.1719635642135642, "f1_score": 0.1872080808080809}}}, "(0.8, 15, 0.9, 3)": {"precision": 0.1935123966942149, "recall": 0.2064957652081619, "f1_score": 0.19210675654312018, "per_field": {"categories": {"precision": 0.1522038567493113, "recall": 0.17101803155522163, "f1_score": 0.154228650137741}, "scientific_domains": {"precision": 0.1966666666666667, "recall": 0.2230714285714286, "f1_score": 0.2030317460317461}, "target_users": {"precision": 0.23166666666666666, "recall": 0.22539783549783546, "f1_score": 0.21905987345987343}}}, "(0.8, 15, 0.9, 5)": {"precision": 0.19391230486685032, "recall": 0.21237092968249993, "f1_score": 0.1951384393945, "per_field": {"categories": {"precision": 0.15323691460055097, "recall": 0.1755259203606311, "f1_score": 0.15642026323844505}, "scientific_domains": {"precision": 0.19640000000000005, "recall": 0.2245714285714286, "f1_score": 0.2036603174603175}, "target_users": {"precision": 0.23209999999999997, "recall": 0.23701544011544007, "f1_score": 0.22533473748473742}}}, "(0.9, 5, 0, 1)": {"precision": 0.1506115702479339, "recall": 0.12515109397795351, "f1_score": 0.13139473656193024, "per_field": {"categories": {"precision": 0.11983471074380166, "recall": 0.1148572501878287, "f1_score": 0.1153730985746796}, "scientific_domains": {"precision": 0.15, "recall": 0.1335, "f1_score": 0.1384}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 5, 0, 3)": {"precision": 0.1413011937557392, "recall": 0.14985449536689205, "f1_score": 0.14045873015873014, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12556348610067616, "f1_score": 0.11393939393939392}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.1565, "f1_score": 0.14514285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 5, 0, 5)": {"precision": 0.14135674931129474, "recall": 0.15256437058503172, "f1_score": 0.14211724695058028, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12593914350112698, "f1_score": 0.11447811447811447}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.158, "f1_score": 0.146}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 5, 0.1, 1)": {"precision": 0.1506115702479339, "recall": 0.12515109397795351, "f1_score": 0.13139473656193024, "per_field": {"categories": {"precision": 0.11983471074380166, "recall": 0.1148572501878287, "f1_score": 0.1153730985746796}, "scientific_domains": {"precision": 0.15, "recall": 0.1335, "f1_score": 0.1384}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 5, 0.1, 3)": {"precision": 0.1413011937557392, "recall": 0.14985449536689205, "f1_score": 0.14045873015873014, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12556348610067616, "f1_score": 0.11393939393939392}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.1565, "f1_score": 0.14514285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 5, 0.1, 5)": {"precision": 0.14135674931129474, "recall": 0.15256437058503172, "f1_score": 0.14211724695058028, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12593914350112698, "f1_score": 0.11447811447811447}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.158, "f1_score": 0.146}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 5, 0.2, 1)": {"precision": 0.1506115702479339, "recall": 0.12515109397795351, "f1_score": 0.13139473656193024, "per_field": {"categories": {"precision": 0.11983471074380166, "recall": 0.1148572501878287, "f1_score": 0.1153730985746796}, "scientific_domains": {"precision": 0.15, "recall": 0.1335, "f1_score": 0.1384}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 5, 0.2, 3)": {"precision": 0.1413011937557392, "recall": 0.14985449536689205, "f1_score": 0.14045873015873014, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12556348610067616, "f1_score": 0.11393939393939392}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.1565, "f1_score": 0.14514285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 5, 0.2, 5)": {"precision": 0.14135674931129474, "recall": 0.15256437058503172, "f1_score": 0.14211724695058028, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12593914350112698, "f1_score": 0.11447811447811447}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.158, "f1_score": 0.146}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 5, 0.3, 1)": {"precision": 0.1506115702479339, "recall": 0.12515109397795351, "f1_score": 0.13139473656193024, "per_field": {"categories": {"precision": 0.11983471074380166, "recall": 0.1148572501878287, "f1_score": 0.1153730985746796}, "scientific_domains": {"precision": 0.15, "recall": 0.1335, "f1_score": 0.1384}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 5, 0.3, 3)": {"precision": 0.1413011937557392, "recall": 0.14985449536689205, "f1_score": 0.14045873015873014, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12556348610067616, "f1_score": 0.11393939393939392}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.1565, "f1_score": 0.14514285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 5, 0.3, 5)": {"precision": 0.14135674931129474, "recall": 0.15256437058503172, "f1_score": 0.14211724695058028, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12593914350112698, "f1_score": 0.11447811447811447}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.158, "f1_score": 0.146}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 5, 0.4, 1)": {"precision": 0.1506115702479339, "recall": 0.12515109397795351, "f1_score": 0.13139473656193024, "per_field": {"categories": {"precision": 0.11983471074380166, "recall": 0.1148572501878287, "f1_score": 0.1153730985746796}, "scientific_domains": {"precision": 0.15, "recall": 0.1335, "f1_score": 0.1384}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 5, 0.4, 3)": {"precision": 0.1413011937557392, "recall": 0.14985449536689205, "f1_score": 0.14045873015873014, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12556348610067616, "f1_score": 0.11393939393939392}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.1565, "f1_score": 0.14514285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 5, 0.4, 5)": {"precision": 0.14135674931129474, "recall": 0.15256437058503172, "f1_score": 0.14211724695058028, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12593914350112698, "f1_score": 0.11447811447811447}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.158, "f1_score": 0.146}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 5, 0.5, 1)": {"precision": 0.1506115702479339, "recall": 0.12515109397795351, "f1_score": 0.13139473656193024, "per_field": {"categories": {"precision": 0.11983471074380166, "recall": 0.1148572501878287, "f1_score": 0.1153730985746796}, "scientific_domains": {"precision": 0.15, "recall": 0.1335, "f1_score": 0.1384}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 5, 0.5, 3)": {"precision": 0.1413011937557392, "recall": 0.14985449536689205, "f1_score": 0.14045873015873014, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12556348610067616, "f1_score": 0.11393939393939392}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.1565, "f1_score": 0.14514285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 5, 0.5, 5)": {"precision": 0.14135674931129474, "recall": 0.15256437058503172, "f1_score": 0.14211724695058028, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12593914350112698, "f1_score": 0.11447811447811447}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.158, "f1_score": 0.146}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 5, 0.6, 1)": {"precision": 0.1506115702479339, "recall": 0.12515109397795351, "f1_score": 0.13139473656193024, "per_field": {"categories": {"precision": 0.11983471074380166, "recall": 0.1148572501878287, "f1_score": 0.1153730985746796}, "scientific_domains": {"precision": 0.15, "recall": 0.1335, "f1_score": 0.1384}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 5, 0.6, 3)": {"precision": 0.1413011937557392, "recall": 0.14985449536689205, "f1_score": 0.14045873015873014, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12556348610067616, "f1_score": 0.11393939393939392}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.1565, "f1_score": 0.14514285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 5, 0.6, 5)": {"precision": 0.14135674931129474, "recall": 0.15256437058503172, "f1_score": 0.14211724695058028, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12593914350112698, "f1_score": 0.11447811447811447}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.158, "f1_score": 0.146}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 5, 0.7, 1)": {"precision": 0.1506115702479339, "recall": 0.12515109397795351, "f1_score": 0.13139473656193024, "per_field": {"categories": {"precision": 0.11983471074380166, "recall": 0.1148572501878287, "f1_score": 0.1153730985746796}, "scientific_domains": {"precision": 0.15, "recall": 0.1335, "f1_score": 0.1384}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 5, 0.7, 3)": {"precision": 0.1413011937557392, "recall": 0.14985449536689205, "f1_score": 0.14045873015873014, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12556348610067616, "f1_score": 0.11393939393939392}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.1565, "f1_score": 0.14514285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 5, 0.7, 5)": {"precision": 0.14135674931129474, "recall": 0.15256437058503172, "f1_score": 0.14211724695058028, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12593914350112698, "f1_score": 0.11447811447811447}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.158, "f1_score": 0.146}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 5, 0.8, 1)": {"precision": 0.1506115702479339, "recall": 0.12515109397795351, "f1_score": 0.13139473656193024, "per_field": {"categories": {"precision": 0.11983471074380166, "recall": 0.1148572501878287, "f1_score": 0.1153730985746796}, "scientific_domains": {"precision": 0.15, "recall": 0.1335, "f1_score": 0.1384}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 5, 0.8, 3)": {"precision": 0.1413011937557392, "recall": 0.14985449536689205, "f1_score": 0.14045873015873014, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12556348610067616, "f1_score": 0.11393939393939392}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.1565, "f1_score": 0.14514285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 5, 0.8, 5)": {"precision": 0.14135674931129474, "recall": 0.15256437058503172, "f1_score": 0.14211724695058028, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12593914350112698, "f1_score": 0.11447811447811447}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.158, "f1_score": 0.146}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 5, 0.9, 1)": {"precision": 0.1506115702479339, "recall": 0.12515109397795351, "f1_score": 0.13139473656193024, "per_field": {"categories": {"precision": 0.11983471074380166, "recall": 0.1148572501878287, "f1_score": 0.1153730985746796}, "scientific_domains": {"precision": 0.15, "recall": 0.1335, "f1_score": 0.1384}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 5, 0.9, 3)": {"precision": 0.1413011937557392, "recall": 0.14985449536689205, "f1_score": 0.14045873015873014, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12556348610067616, "f1_score": 0.11393939393939392}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.1565, "f1_score": 0.14514285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 5, 0.9, 5)": {"precision": 0.14135674931129474, "recall": 0.15256437058503172, "f1_score": 0.14211724695058028, "per_field": {"categories": {"precision": 0.1115702479338843, "recall": 0.12593914350112698, "f1_score": 0.11447811447811447}, "scientific_domains": {"precision": 0.14233333333333334, "recall": 0.158, "f1_score": 0.146}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 10, 0, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 10, 0, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 10, 0, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 10, 0.1, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 10, 0.1, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 10, 0.1, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 10, 0.2, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 10, 0.2, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 10, 0.2, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 10, 0.3, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 10, 0.3, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 10, 0.3, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 10, 0.4, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 10, 0.4, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 10, 0.4, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 10, 0.5, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 10, 0.5, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 10, 0.5, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 10, 0.6, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 10, 0.6, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 10, 0.6, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 10, 0.7, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 10, 0.7, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 10, 0.7, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 10, 0.8, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 10, 0.8, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 10, 0.8, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 10, 0.9, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 10, 0.9, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 10, 0.9, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 15, 0, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 15, 0, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 15, 0, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 15, 0.1, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 15, 0.1, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 15, 0.1, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 15, 0.2, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 15, 0.2, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 15, 0.2, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 15, 0.3, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 15, 0.3, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 15, 0.3, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 15, 0.4, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 15, 0.4, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 15, 0.4, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 15, 0.5, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 15, 0.5, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 15, 0.5, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 15, 0.6, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 15, 0.6, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 15, 0.6, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 15, 0.7, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 15, 0.7, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 15, 0.7, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 15, 0.8, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 15, 0.8, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 15, 0.8, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}, "(0.9, 15, 0.9, 1)": {"precision": 0.15058953168044079, "recall": 0.1251290554104604, "f1_score": 0.13137269799443715, "per_field": {"categories": {"precision": 0.11776859504132231, "recall": 0.11279113448534937, "f1_score": 0.11330698287220027}, "scientific_domains": {"precision": 0.152, "recall": 0.1355, "f1_score": 0.1404}, "target_users": {"precision": 0.182, "recall": 0.12709603174603182, "f1_score": 0.14041111111111113}}}, "(0.9, 15, 0.9, 3)": {"precision": 0.14117906336088157, "recall": 0.15052116203355873, "f1_score": 0.1405624950806769, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12556348610067616, "f1_score": 0.11325068870523416}, "scientific_domains": {"precision": 0.143, "recall": 0.1585, "f1_score": 0.14614285714285716}, "target_users": {"precision": 0.17, "recall": 0.16749999999999998, "f1_score": 0.16229393939393935}}}, "(0.9, 15, 0.9, 5)": {"precision": 0.1412346189164371, "recall": 0.1532310372516984, "f1_score": 0.142221011872527, "per_field": {"categories": {"precision": 0.11053719008264463, "recall": 0.12593914350112698, "f1_score": 0.1137894092439547}, "scientific_domains": {"precision": 0.143, "recall": 0.16, "f1_score": 0.147}, "target_users": {"precision": 0.17016666666666666, "recall": 0.17375396825396824, "f1_score": 0.16587362637362635}}}} \ No newline at end of file diff --git a/api/recommender/similar_services/field_suggestion/suggestion_candidates.py b/api/recommender/similar_services/field_suggestion/suggestion_candidates.py deleted file mode 100644 index 237ec44..0000000 --- a/api/recommender/similar_services/field_suggestion/suggestion_candidates.py +++ /dev/null @@ -1,25 +0,0 @@ -import math -from collections import Counter -from itertools import chain - - -def get_candidates(field_values, frequency_threshold=None): - """ - Returns a list of values based on frequency across values sets - @param field_values: list>, a list with the values of a field for several services - @return list - """ - if len(field_values) == 0: - return [] - - if frequency_threshold is None: - frequency_threshold = 0 - - # Calculate the appearances of each value - values_count = dict(Counter(chain(*field_values))) - - # Filter values - counts_threshold = math.ceil(frequency_threshold * len(field_values)) - filtered_values = dict(filter(lambda elem: elem[1] >= counts_threshold, values_count.items())).keys() - - return list(filtered_values) diff --git a/api/recommender/similar_services/field_suggestion/suggestion_generation.py b/api/recommender/similar_services/field_suggestion/suggestion_generation.py deleted file mode 100644 index c3cab7f..0000000 --- a/api/recommender/similar_services/field_suggestion/suggestion_generation.py +++ /dev/null @@ -1,51 +0,0 @@ -from api.databases.registry.registry_selector import get_registry -from api.recommender.similar_services.field_suggestion.similar_services import get_similar_services -from api.recommender.similar_services.preprocessor.embeddings.text_embeddings import \ - create_text_embedding -from api.recommender.similar_services.field_suggestion.suggestion_candidates import \ - get_candidates -from api.settings import APP_SETTINGS - - -def get_auto_complete_suggestions(new_service, requested_fields, maximum_suggestions=5, evaluation_mode=False, - similarity_threshold=None, - considered_services_threshold=None, frequency_threshold=None): - """ - @param new_service: dict with the name and value for each filled field of a service - @param requested_fields: list, the names of the fields for which auto-completion will be done - @param maximum_suggestions: int, the maximum number of suggestions per field - @return: dict with the names and suggested values for all requested fields - """ - - # Calculate the text embedding of the new service - text_embedding = create_text_embedding(new_service) - - # Get similar services ids - similar_services_ids_per_field = get_similar_services(requested_fields, text_embedding, similarity_threshold, - considered_services_threshold) - - if evaluation_mode: - # Remove evaluated service from similar services of every field - for _, similar_services in similar_services_ids_per_field.items(): - if new_service["service_id"] in similar_services: - similar_services.remove(new_service["service_id"]) - - # Get the requested fields for all similar services - all_similar_services_ids = list( - set().union(*[similar_services for _, similar_services in similar_services_ids_per_field.items()])) - - db = get_registry() - similar_services = db.get_services_by_ids(ids=all_similar_services_ids, attributes=requested_fields) - - # Find the most used values for every requested field - suggestions = {} - for requested_field in requested_fields: - field_suggestions = get_candidates(similar_services[similar_services["service_id"] - .isin(similar_services_ids_per_field[requested_field])][requested_field] - .values.tolist(), APP_SETTINGS["BACKEND"]["AUTO_COMPLETION"] - [requested_field]["FREQUENCY_THRESHOLD"]) - - suggestions[requested_field] = field_suggestions \ - if len(field_suggestions) <= maximum_suggestions else field_suggestions[:maximum_suggestions] - - return suggestions diff --git a/api/recommender/similar_services/service_recommendation/evaluation/services_for_manual_evaluation.py b/api/recommender/similar_services/service_recommendation/evaluation/services_for_manual_evaluation.py deleted file mode 100644 index de596cf..0000000 --- a/api/recommender/similar_services/service_recommendation/evaluation/services_for_manual_evaluation.py +++ /dev/null @@ -1,52 +0,0 @@ -import pandas as pd -import numpy as np -import json - - -def select_services_with_poor_recommendations(recommendations_ratings, num=10): - # Get recommendations sets with only irrelevant services - poor_recommendations = recommendations_ratings[recommendations_ratings.sum(axis=1) == 0] - return list(poor_recommendations.head(num).index) - - -def select_services_with_good_recommendations(recommendations_ratings, num=10): - # Get recommendations sets with only relevant services - rec_set_len = len(recommendations_ratings.columns) - good_recommendations = recommendations_ratings[recommendations_ratings.sum(axis=1) == rec_set_len] - return list(good_recommendations.head(num).index) - - -if __name__ == '__main__': - # Read csv with the manual evaluation results - results = pd.read_csv( - "./api/recommender/similar_services/service_recommendation/evaluation/results/manual_evaluation_250.csv", - header=[0, 1, 2]) - - # Add ids of the services as index - with open("./api/recommender/similar_services/service_recommendation/evaluation/results/evaluated_services.json") as f: - ids = np.array(json.load(f)) - - # Keep the relevance result of each recommended service - results = results.iloc[:, [3, 5, 7, 9, 11, 13, 15]] - # Rename columns - results.columns = [1, 2, 3, 4, 5, 6, "better_choices"] - - # Remove services with None id - none_indices = np.where(ids == "")[0] - ids = np.delete(ids, none_indices) - results = results.drop(none_indices) - - results = results.set_index(ids) - # Convert all values to int. Missing values(-) => Nan - for column in [1, 2, 3, 4, 5, 6]: - results[column] = pd.to_numeric(results[column], errors='coerce') - - tricky_services = list(results[results["better_choices"].notnull()].index) - - challenging_services = select_services_with_poor_recommendations(results[[1, 2, 3, 4, 5, 6]]) - non_challenging_services = select_services_with_good_recommendations(results[[1, 2, 3, 4, 5, 6]]) - - selected_services = set(tricky_services + challenging_services + non_challenging_services) - print(f"challenging services: {challenging_services}\nnon challenging services: {non_challenging_services}\n" - f"tricky services: {tricky_services}") - diff --git a/api/recommender/similar_services/service_recommendation/recommendation_generation.py b/api/recommender/similar_services/service_recommendation/recommendation_generation.py deleted file mode 100644 index 5cec3b1..0000000 --- a/api/recommender/similar_services/service_recommendation/recommendation_generation.py +++ /dev/null @@ -1,71 +0,0 @@ -import logging - -from api.databases.content_based_rec_db import ContentBasedRecsMongoDB -from api.databases.registry.registry_selector import get_registry -from api.exceptions import IdNotExists -from api.recommender.similar_services.service_recommendation.components.filtering import \ - filtering -from api.recommender.similar_services.service_recommendation.components.ordering import \ - ordering -from api.recommender.similar_services.service_recommendation.components.recommendation_candidates import \ - get_recommendation_candidates -from api.recommender.similar_services.service_recommendation.components.reranking import \ - re_ranking -from api.settings import APP_SETTINGS - -logger = logging.getLogger(__name__) - - -def service_exists(db, viewed_service_id): - """ - Checks if the given service id exists - """ - if not db.is_valid_service(viewed_service_id): - raise IdNotExists("Service id does not exist.") - - -def valid_user(db, user_id): - """ - Return the user_id if user_id is valid else None - """ - if user_id is not None and not db.is_valid_user(user_id): - return None - return user_id - - -def create_recommendation(viewed_resource_id, recommendations_num=5, user_id=None, - viewed_weight=None, metadata_weight=None): - viewed_weight = APP_SETTINGS["BACKEND"]["SIMILAR_SERVICES"]["VIEWED_WEIGHT"] \ - if viewed_weight is None else viewed_weight - metadata_weight = APP_SETTINGS["BACKEND"]["SIMILAR_SERVICES"]["METADATA_WEIGHT"] \ - if metadata_weight is None else metadata_weight - - db = get_registry() - - service_exists(db, viewed_resource_id) - - user_id = valid_user(db, user_id) - - logger.debug("Get user purchases...") - purchases = list(db.get_user_services(user_id)) if user_id is not None else [] - - candidates = get_recommendation_candidates(viewed_resource_id, - purchased_resources=purchases, - view_weight=viewed_weight, - metadata_weight=metadata_weight) - - candidates = filtering(db, candidates, viewed_resource_id, purchases) - - candidates = ordering(candidates) - - candidates = re_ranking(target_service=viewed_resource_id, candidates=candidates, - recommendations_num=recommendations_num) - - recommendation = [{"service_id": service_id, "score": score} for service_id, score in - candidates[:recommendations_num].items()] - - content_based_recs_db = ContentBasedRecsMongoDB() - content_based_recs_db.save_recommendation(recommendation=recommendation, service_id=viewed_resource_id, - user_id=user_id, history_service_ids=purchases) - - return recommendation diff --git a/api/routes/update.py b/api/routes/update.py deleted file mode 100644 index 79d9d93..0000000 --- a/api/routes/update.py +++ /dev/null @@ -1,83 +0,0 @@ -import logging - -from api.exceptions import IdNotExists, NoneProjects, NoneServices -from api.recommender.project_completion.initialization import association_rules -from api.recommender.similar_services.preprocessor.embeddings import ( - metadata_embeddings, text_embeddings) -from api.recommender.similar_services.preprocessor.similarities import ( - metadata_similarities, text_similarities) -from api.recommender.similar_services.preprocessor.reports import monitoring_reports -from api.settings import APP_SETTINGS -from fastapi import APIRouter, HTTPException - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix='/v1') - - -@router.get( - "/update", - summary="Update all data structures", - description="The data structures created (such as embeddings) need updating every x hours.", - tags=["update"] -) -def update(): - try: - if APP_SETTINGS["BACKEND"]["MODE"] == "AUTO-COMPLETION": - text_embeddings.create_text_embeddings() - elif APP_SETTINGS["BACKEND"]["MODE"] == "RS": - # Update similar services - metadata_embeddings.create_metadata_embeddings() - metadata_similarities.create_metadata_similarities() - text_embeddings.create_text_embeddings() - text_similarities.create_text_similarities() - monitoring_reports.update_status_report() - monitoring_reports.update_ar_report() - - # Update project completion - association_rules.create_association_rules() - elif APP_SETTINGS["BACKEND"]["MODE"] == "SIMILAR_SERVICES_EVALUATION": - # Update similar services - metadata_embeddings.create_metadata_embeddings() - metadata_similarities.create_metadata_similarities() - text_embeddings.create_text_embeddings() - text_similarities.create_text_similarities() - monitoring_reports.update_status_report() - monitoring_reports.update_ar_report() - else: - pass # TODO raise error - - except (NoneServices, NoneProjects) as e: - # Delete all structures that have been initialized - metadata_similarities.delete_metadata_similarities() - text_similarities.delete_text_similarities() - association_rules.delete_association_rules() - monitoring_reports.delete_status_report() - monitoring_reports.delete_ar_report() - - logger.error("Failed to update recommenders: " + str(e)) - raise HTTPException(status_code=500, detail="Failed to update recommenders: " + str(e)) - - -@router.get( - "/update_for_new_service", - summary="Updates data structures for similar services", - tags=["update"] -) -def update_for_new_service(service_id: int): - try: - if APP_SETTINGS["BACKEND"]["MODE"] == "AUTO-COMPLETION": - text_embeddings.update_text_embedding(new_service_id=service_id) - elif APP_SETTINGS["BACKEND"]["MODE"] == "RS": - metadata_embeddings.update_metadata_embedding(new_service_id=service_id) - metadata_similarities.update_metadata_similarities(new_service_id=service_id) - text_embeddings.update_text_embedding(new_service_id=service_id) - text_similarities.update_text_similarities(new_service_id=service_id) - monitoring_reports.update_status_report() - monitoring_reports.update_ar_report() - else: - pass # TODO raise Exception - - except IdNotExists as e: - logger.error("Failed to update similar services recommender: " + str(e)) - raise HTTPException(status_code=500, detail="Failed to update similar services recommender: " + str(e)) diff --git a/api/scheduling/initialization.py b/api/scheduling/initialization.py deleted file mode 100644 index d1e8652..0000000 --- a/api/scheduling/initialization.py +++ /dev/null @@ -1,72 +0,0 @@ -import logging -from multiprocessing import Process - -import cronitor -from api.recommender.project_completion.initialization import association_rules -from api.recommender.similar_services.preprocessor.embeddings.metadata_embeddings import initialize_metadata_embeddings -from api.recommender.similar_services.preprocessor.embeddings.text_embeddings import initialize_text_embeddings -from api.recommender.similar_services.preprocessor.reports.monitoring_reports import initialise_ar_report, \ - initialise_status_report -from api.recommender.similar_services.preprocessor.similarities.metadata_similarities import \ - initialize_metadata_similarities -from api.recommender.similar_services.preprocessor.similarities.text_similarities import initialize_text_similarities -from api.routes.update import update -from api.settings import APP_SETTINGS -from apscheduler.schedulers.blocking import BlockingScheduler - -cronitor.api_key = APP_SETTINGS['CREDENTIALS']['CRONITOR_API_KEY'] -cronitor.Monitor.put( - key='update-rs', - type='job', - schedule=f'0 */{APP_SETTINGS["BACKEND"]["SCHEDULING"]["EVERY_N_HOURS"]} * * *' -) - - -def init_scheduler(): - scheduler = BlockingScheduler() - scheduler.add_job( - scheduled_update, 'cron', - hour=f'*/{APP_SETTINGS["BACKEND"]["SCHEDULING"]["EVERY_N_HOURS"]}' - # minute="*/5" - ) - try: - scheduler.start() - except (KeyboardInterrupt, SystemExit): - pass - - -@cronitor.job('update-rs') -def scheduled_update(): - logging.info("Running scheduled update...") - update() - - -def initialize_structures_if_not_exist(): - if APP_SETTINGS["BACKEND"]["MODE"] == "AUTO-COMPLETION": - initialize_text_embeddings() - elif APP_SETTINGS["BACKEND"]["MODE"] == "RS": - # Create metadata structures if they do not exist - initialize_metadata_embeddings() - initialize_metadata_similarities() - - # Create report structures if they do not exist - initialise_status_report() - initialise_ar_report() - - # Create text structures if they do not exist - initialize_text_embeddings() - initialize_text_similarities() - - if not association_rules.existence_association_rules(): - logging.info("Association rules do not exist. Creating...") - association_rules.create_association_rules() - else: - pass # TODO raise exception - - -def start_updating_process(): - initialize_structures_if_not_exist() - - p = Process(target=init_scheduler) - logging.info("Starting process...") - p.start() diff --git a/api/__init__.py b/app/__init__.py similarity index 100% rename from api/__init__.py rename to app/__init__.py diff --git a/api/config/backend-rs-dev.yaml b/app/config/backend-portal-recommender-dev.yaml similarity index 94% rename from api/config/backend-rs-dev.yaml rename to app/config/backend-portal-recommender-dev.yaml index 7de4433..cc1bc74 100644 --- a/api/config/backend-rs-dev.yaml +++ b/app/config/backend-portal-recommender-dev.yaml @@ -1,5 +1,5 @@ VERSION_NAME: "v1" -MODE: "RS" +MODE: "PORTAL-RECOMMENDER" FASTAPI: WORKERS: 1 @@ -21,6 +21,8 @@ SIMILAR_SERVICES: METADATA_WEIGHT: 0.5 VIEWED_WEIGHT: 0.5 + DIVERSITY_WEIGHT: 0.5 + METHOD: "SBERT" # tf-idf or SBERT SBERT: MODEL_NAME: 'paraphrase-MiniLM-L6-v2' diff --git a/api/config/backend-rs-prod.yaml b/app/config/backend-portal-recommender-prod.yaml similarity index 82% rename from api/config/backend-rs-prod.yaml rename to app/config/backend-portal-recommender-prod.yaml index acca307..1a5c8e5 100644 --- a/api/config/backend-rs-prod.yaml +++ b/app/config/backend-portal-recommender-prod.yaml @@ -1,5 +1,5 @@ VERSION_NAME: "v1" -MODE: "RS" +MODE: "PORTAL-RECOMMENDER" FASTAPI: WORKERS: 4 @@ -14,12 +14,15 @@ SCHEDULING: CREDENTIALS: "credentials.yml" SIMILAR_SERVICES: - METADATA: ["categories", "scientific_domains", "target_users"] - TEXT_ATTRIBUTES: ["name", "description"] - METADATA_WEIGHT: 0.5 + METADATA: ["categories", "scientific_domains"] + TEXT_ATTRIBUTES: ["tagline", "description"] + + METADATA_WEIGHT: 0.25 VIEWED_WEIGHT: 0.5 + DIVERSITY_WEIGHT: 0 + METHOD: "SBERT" # tf-idf or SBERT SBERT: MODEL_NAME: 'paraphrase-MiniLM-L6-v2' diff --git a/api/config/backend-auto-dev.yaml b/app/config/backend-providers-recommender-dev.yaml similarity index 85% rename from api/config/backend-auto-dev.yaml rename to app/config/backend-providers-recommender-dev.yaml index 3f4f1b9..725fb8b 100644 --- a/api/config/backend-auto-dev.yaml +++ b/app/config/backend-providers-recommender-dev.yaml @@ -1,5 +1,5 @@ VERSION_NAME: "v1" -MODE: "AUTO-COMPLETION" +MODE: "PROVIDERS-RECOMMENDER" FASTAPI: WORKERS: 1 @@ -15,8 +15,10 @@ SCHEDULING: CREDENTIALS: "credentials.yml" SIMILAR_SERVICES: - METADATA: ["categories", "scientific_domains", "target_users"] - TEXT_ATTRIBUTES: ["name", "description"] + + METADATA: ["categories", "scientific_domains"] + TEXT_ATTRIBUTES: ["description", "tagline"] + METADATA_WEIGHT: 0.5 VIEWED_WEIGHT: 0.5 diff --git a/api/config/backend-auto-prod.yaml b/app/config/backend-providers-recommender-prod.yaml similarity index 96% rename from api/config/backend-auto-prod.yaml rename to app/config/backend-providers-recommender-prod.yaml index fa1446d..e142919 100644 --- a/api/config/backend-auto-prod.yaml +++ b/app/config/backend-providers-recommender-prod.yaml @@ -1,5 +1,5 @@ VERSION_NAME: "v1" -MODE: "AUTO-COMPLETION" +MODE: "PROVIDERS-RECOMMENDER" FASTAPI: WORKERS: 4 diff --git a/api/databases/__init__.py b/app/databases/__init__.py similarity index 100% rename from api/databases/__init__.py rename to app/databases/__init__.py diff --git a/api/databases/argo_monitoring_api.py b/app/databases/argo_monitoring_api.py similarity index 88% rename from api/databases/argo_monitoring_api.py rename to app/databases/argo_monitoring_api.py index 89c6273..9876069 100644 --- a/api/databases/argo_monitoring_api.py +++ b/app/databases/argo_monitoring_api.py @@ -1,9 +1,9 @@ from datetime import datetime, timedelta + import pandas as pd import requests - -from api.databases.redis_db import store_object -from api.settings import APP_SETTINGS +from app.databases.redis_db import store_object +from app.settings import APP_SETTINGS class ArgoMonitoringApi: @@ -12,12 +12,9 @@ def __init__(self, n_days_ago=7): 'x-api-key': APP_SETTINGS['CREDENTIALS']['MONITORING_API_ACCESS_TOKEN']} self.N_DAYS_AGO = n_days_ago - def get_status_report(self, store_in_redis=True): + def get_status_report(self): """Get the latest status report from monitoring services - Args: - store_in_redis (bool): Default TRUE - Returns: status_report_df (datframe): response from api """ @@ -34,12 +31,9 @@ def get_status_report(self, store_in_redis=True): tmp_statuses.append(status['value']) status_report_df['statuses'][i] = tmp_statuses - if store_in_redis is True: - store_object(status_report_df, "STATUS_REPORT") - return status_report_df - def get_ar_report(self, store_in_redis=True): + def get_ar_report(self): """Get availability and reliability reports from monitoring services Notes: @@ -104,7 +98,5 @@ def get_ar_report(self, store_in_redis=True): results_dict[key].append(value) ar_report_df = pd.DataFrame.from_dict(results_dict) - if store_in_redis is True: - store_object(ar_report_df, "AR_REPORT") return ar_report_df diff --git a/api/databases/content_based_rec_db.py b/app/databases/content_based_rec_db.py similarity index 96% rename from api/databases/content_based_rec_db.py rename to app/databases/content_based_rec_db.py index f2bdeab..0a38f02 100644 --- a/api/databases/content_based_rec_db.py +++ b/app/databases/content_based_rec_db.py @@ -2,9 +2,9 @@ import logging from typing import Optional -from api.databases.utils.mongo_connector import (MongoDbConnector, +from app.databases.utils.mongo_connector import (MongoDbConnector, form_mongo_url) -from api.settings import APP_SETTINGS +from app.settings import APP_SETTINGS from pymongo.errors import ServerSelectionTimeoutError logger = logging.getLogger(__name__) diff --git a/api/databases/redis_db.py b/app/databases/redis_db.py similarity index 96% rename from api/databases/redis_db.py rename to app/databases/redis_db.py index dfbd851..7a21f7f 100644 --- a/api/databases/redis_db.py +++ b/app/databases/redis_db.py @@ -4,7 +4,7 @@ from typing import Optional import redis -from api.settings import APP_SETTINGS +from app.settings import APP_SETTINGS logger = logging.getLogger(__name__) diff --git a/api/databases/registry/__init__.py b/app/databases/registry/__init__.py similarity index 100% rename from api/databases/registry/__init__.py rename to app/databases/registry/__init__.py diff --git a/api/databases/registry/catalog_api.py b/app/databases/registry/catalog_api.py similarity index 74% rename from api/databases/registry/catalog_api.py rename to app/databases/registry/catalog_api.py index 86b97d3..2317bef 100644 --- a/api/databases/registry/catalog_api.py +++ b/app/databases/registry/catalog_api.py @@ -1,21 +1,28 @@ +from typing import Optional import pandas as pd import requests -from api.databases.registry.registry_abc import Registry -from api.exceptions import APIResponseFormatException + +from app.databases.registry.registry_abc import Registry +from app.exceptions import APIResponseFormatException, IdNotExists, APIResponseError class CatalogueAPI(Registry): - # TODO implement - def check_health(self) -> bool: - ... + def check_health(self) -> Optional[str]: + try: + self._get_request("https://api.eosc-portal.eu/vocabulary/byType") + except APIResponseError as e: + return "Cannot connect with catalogue API" + return None @staticmethod def _get_request(request): response = requests.get(request) - if response.status_code != 200: - raise APIResponseFormatException("Problem with catalogue API!") + if response.status_code == 404: + return None + elif response.status_code != 200: + raise APIResponseError("Error at request in catalogue API!") return response.json() @@ -32,10 +39,13 @@ def _reformat_service(service): return service # TODO change to one call - def get_services_by_ids(self, ids, attributes=None): + def get_services_by_ids(self, ids, attributes=None, remove_generic_attributes=False): services = [] for id in ids: - services.append(self._reformat_service(self._get_request(f"https://api.eosc-portal.eu/resource/{id}?catalogue_id=eosc"))) + service = self._get_request(f"https://api.eosc-portal.eu/resource/{id}?catalogue_id=eosc") + if service is None: + raise IdNotExists(f"Service id {id} does not exist!") + services.append(self._reformat_service(service)) if len(services): services_df = pd.DataFrame(services) @@ -44,6 +54,9 @@ def get_services_by_ids(self, ids, attributes=None): else: services_df = pd.DataFrame(columns=["service_id"] + attributes) + if remove_generic_attributes: + self._remove_general_attributes_from_services(services_df) + return services_df def get_services(self, attributes=None, reformat=True): @@ -77,11 +90,17 @@ def get_services(self, attributes=None, reformat=True): return services_df - def get_service(self, service_id, reformat=True): + def get_service(self, service_id, reformat=True, remove_generic_attributes=True): service = self._get_request(f"https://api.eosc-portal.eu/resource/{service_id}?catalogue_id=eosc") - if reformat and service is not None: + + if service is None: + raise IdNotExists(f"Service id {service_id} does not exist!") + + if reformat: service = self._reformat_service(service) - self._remove_general_attributes_from_single_service(service) + + if remove_generic_attributes: + self._remove_general_attributes_from_single_service(service) return service def get_scientific_domains(self): diff --git a/api/databases/registry/catalog_dump.py b/app/databases/registry/catalog_dump.py similarity index 89% rename from api/databases/registry/catalog_dump.py rename to app/databases/registry/catalog_dump.py index f999e63..efc8199 100644 --- a/api/databases/registry/catalog_dump.py +++ b/app/databases/registry/catalog_dump.py @@ -1,10 +1,10 @@ import pandas as pd import requests -from api.databases.registry.registry_abc import Registry -from api.databases.utils.mongo_connector import (MongoDbConnector, +from app.databases.registry.registry_abc import Registry +from app.databases.utils.mongo_connector import (MongoDbConnector, form_mongo_url) -from api.exceptions import APIResponseFormatException -from api.settings import APP_SETTINGS +from app.exceptions import APIResponseFormatException +from app.settings import APP_SETTINGS from pymongo import MongoClient @@ -34,10 +34,10 @@ def _reformat_service(service): return service - def get_services_by_ids(self, ids, attributes=None, conditions=None): - return self.get_services(attributes=attributes, conditions={'id': {'$in': ids}}) + def get_services_by_ids(self, ids, attributes=None, conditions=None, remove_generic_attributes=False): + return self.get_services(attributes=attributes, conditions={'id': {'$in': ids}}, remove_generic_attributes=remove_generic_attributes) - def get_services(self, attributes=None, conditions=None, reformat=True): + def get_services(self, attributes=None, conditions=None, reformat=True, remove_generic_attributes=True): """ Args: attributes: list, the requested attributes for the services @@ -58,15 +58,17 @@ def get_services(self, attributes=None, conditions=None, reformat=True): else: # If there are no services services_df = pd.DataFrame(columns=list(set(["service_id"] + attributes))) - self._remove_general_attributes_from_services(services_df) + if remove_generic_attributes: + self._remove_general_attributes_from_services(services_df) return services_df - def get_service(self, service_id, reformat=True): + def get_service(self, service_id, reformat=True, remove_generic_attributes=True): service = self.mongo_connector.get_db()["service"].find_one({'id': service_id}) if reformat and service is not None: service = self._reformat_service(service) - self._remove_general_attributes_from_single_service(service) + if remove_generic_attributes: + self._remove_general_attributes_from_single_service(service) return service def get_scientific_domains(self): diff --git a/api/databases/registry/registry_abc.py b/app/databases/registry/registry_abc.py similarity index 100% rename from api/databases/registry/registry_abc.py rename to app/databases/registry/registry_abc.py diff --git a/app/databases/registry/registry_selector.py b/app/databases/registry/registry_selector.py new file mode 100644 index 0000000..08d0b4d --- /dev/null +++ b/app/databases/registry/registry_selector.py @@ -0,0 +1,16 @@ +from app.databases.registry.catalog_api import CatalogueAPI +from app.databases.registry.catalog_dump import CatalogueDump +from app.databases.registry.rs_mongo import RSMongoDB +from app.exceptions import ModeDoesNotExist +from app.settings import APP_SETTINGS + + +def get_registry(): + if APP_SETTINGS['BACKEND']['MODE'] == 'PORTAL-RECOMMENDER': + return RSMongoDB() + elif APP_SETTINGS['BACKEND']['MODE'] == 'PROVIDERS-RECOMMENDER': + return CatalogueAPI() + elif APP_SETTINGS['BACKEND']['MODE'] == "SIMILAR_SERVICES_EVALUATION": + return CatalogueDump() + else: + raise ModeDoesNotExist(f"Mode {APP_SETTINGS['BACKEND']['MODE']} is not recognised.") diff --git a/api/databases/registry/rs_mongo.py b/app/databases/registry/rs_mongo.py similarity index 89% rename from api/databases/registry/rs_mongo.py rename to app/databases/registry/rs_mongo.py index 82616b9..f79d3ea 100644 --- a/api/databases/registry/rs_mongo.py +++ b/app/databases/registry/rs_mongo.py @@ -2,10 +2,10 @@ from typing import Optional import pandas as pd -from api.databases.registry.registry_abc import Registry -from api.databases.utils.mongo_connector import (MongoDbConnector, +from app.databases.registry.registry_abc import Registry +from app.databases.utils.mongo_connector import (MongoDbConnector, form_mongo_url) -from api.settings import APP_SETTINGS +from app.settings import APP_SETTINGS from pymongo.errors import ServerSelectionTimeoutError logger = logging.getLogger(__name__) @@ -43,11 +43,11 @@ def check_health(self) -> Optional[str]: return f"Collections {' '.join(missing_collections)} are missing" if len(missing_collections) != 0 else None - def get_services_by_ids(self, ids, attributes=None): - return self.get_services(attributes=attributes, conditions={'_id': {'$in': ids}}) + def get_services_by_ids(self, ids, attributes=None, remove_generic_attributes=False): + return self.get_services(attributes=attributes, conditions={'_id': {'$in': ids}}, remove_generic_attributes=remove_generic_attributes) # TODO get only attributes? - def get_services(self, attributes=None, conditions=None): + def get_services(self, attributes=None, conditions=None, remove_generic_attributes=True): if conditions is None: conditions = {} if attributes is None: @@ -61,7 +61,8 @@ def get_services(self, attributes=None, conditions=None): else: # If there are no services services_df = pd.DataFrame(columns=list(set(["service_id"] + attributes))) - self._remove_general_attributes_from_services(services_df) + if remove_generic_attributes: + self._remove_general_attributes_from_services(services_df) return services_df @@ -71,9 +72,10 @@ def get_non_published_services(self, considered_services=None): conditions["_id"] = {'$in': considered_services} return list(self.get_services(conditions=conditions)["service_id"].to_list()) - def get_service(self, service_id): + def get_service(self, service_id, remove_generic_attributes=True): service = self.mongo_connector.get_db()["service"].find_one({'_id': int(service_id)}) - self._remove_general_attributes_from_single_service(service) + if remove_generic_attributes: + self._remove_general_attributes_from_single_service(service) return service def get_project(self, project_id): diff --git a/api/databases/utils/__init__.py b/app/databases/utils/__init__.py similarity index 100% rename from api/databases/utils/__init__.py rename to app/databases/utils/__init__.py diff --git a/api/databases/utils/mongo_connector.py b/app/databases/utils/mongo_connector.py similarity index 99% rename from api/databases/utils/mongo_connector.py rename to app/databases/utils/mongo_connector.py index 19b8f8d..342109f 100644 --- a/api/databases/utils/mongo_connector.py +++ b/app/databases/utils/mongo_connector.py @@ -1,4 +1,5 @@ import logging + from pymongo import MongoClient logger = logging.getLogger(__name__) diff --git a/api/exceptions.py b/app/exceptions.py similarity index 75% rename from api/exceptions.py rename to app/exceptions.py index 2c42df5..6a21c55 100644 --- a/api/exceptions.py +++ b/app/exceptions.py @@ -31,5 +31,16 @@ class APIResponseFormatException(Exception): class APIResponseError(Exception): - """ Will be thrown when an api request has a status different than 200""" - pass \ No newline at end of file + """ Will be thrown when an api request has a status different from 200 or 404""" + pass + + +class ModeDoesNotExist(Exception): + """ + Will be thrown when the mode parameter passed in is not recognised. + Current modes allowed: + * PORTAL-RECOMMENDER + * PROVIDERS-RECOMMENDER + * SIMILAR_SERVICES_EVALUATION + """ + pass diff --git a/api/health/__init__.py b/app/health/__init__.py similarity index 100% rename from api/health/__init__.py rename to app/health/__init__.py diff --git a/api/health/monitor_health.py b/app/health/monitor_health.py similarity index 56% rename from api/health/monitor_health.py rename to app/health/monitor_health.py index b93237c..aec3478 100644 --- a/api/health/monitor_health.py +++ b/app/health/monitor_health.py @@ -1,6 +1,9 @@ -from api.databases import redis_db -from api.databases.content_based_rec_db import ContentBasedRecsMongoDB -from api.databases.registry.rs_mongo import RSMongoDB +from app.databases import redis_db +from app.databases.content_based_rec_db import ContentBasedRecsMongoDB +from app.databases.registry.catalog_api import CatalogueAPI +from app.databases.registry.rs_mongo import RSMongoDB +from app.exceptions import ModeDoesNotExist +from app.settings import APP_SETTINGS def test_rs_mongo(): @@ -25,6 +28,28 @@ def test_rs_mongo(): } +def test_catalogue_api(): + db = CatalogueAPI() + + health_check_error = db.check_health() + + if health_check_error is None: + return { + "catalog_api": { + "status": "UP", + "database_type": "API" + } + } + else: + return { + "catalog_api": { + "status": "DOWN", + "error": health_check_error, + "database_type": "API" + } + } + + def test_content_based_rs_mongo(): db = ContentBasedRecsMongoDB() health_check_error = db.check_health() @@ -66,12 +91,20 @@ def test_redis(): } +def get_mode_tests(): + if APP_SETTINGS['BACKEND']['MODE'] == 'PORTAL-RECOMMENDER': + return [test_rs_mongo(), test_content_based_rs_mongo(), test_redis()] + elif APP_SETTINGS['BACKEND']['MODE'] == 'PROVIDERS-RECOMMENDER': + return [test_catalogue_api(), test_redis()] + elif APP_SETTINGS['BACKEND']['MODE'] == "SIMILAR_SERVICES_EVALUATION": + return [] + else: + raise ModeDoesNotExist(f"Mode {APP_SETTINGS['BACKEND']['MODE']} is not recognised.") + + def service_health_test(): - tests = [ - test_rs_mongo(), - test_content_based_rs_mongo(), - test_redis() - ] + tests = get_mode_tests() + response = { "status": "UP" } diff --git a/api/main.py b/app/main.py similarity index 75% rename from api/main.py rename to app/main.py index 667b553..88be753 100644 --- a/api/main.py +++ b/app/main.py @@ -1,16 +1,16 @@ import logging -import api.scheduling.initialization import sentry_sdk import uvicorn -from api.databases.content_based_rec_db import ContentBasedRecsMongoDB -from api.settings import APP_SETTINGS +from app.databases.content_based_rec_db import ContentBasedRecsMongoDB +from app.scheduler import start_scheduler_process +from app.settings import APP_SETTINGS, mode_setting_validation logging.basicConfig(level=logging.INFO if APP_SETTINGS['BACKEND']['PROD'] else logging.DEBUG, format='%(levelname)s | %(asctime)s | %(message)s', datefmt='%m/%d/%Y %I:%M:%S') -from api.routes.add_routes import initialize_routes +from app.routes.add_routes import initialize_routes from fastapi import FastAPI sentry_sdk.init( @@ -27,23 +27,25 @@ @app.on_event("startup") async def startup_event(): # Keep track of the RS version we are running (specified in config file) - if APP_SETTINGS["BACKEND"]["MODE"] == "RS": + if APP_SETTINGS["BACKEND"]["MODE"] == "PORTAL-RECOMMENDER": db = ContentBasedRecsMongoDB() db.update_version() def start_app(): + mode_setting_validation() + # The update scheduler starts before uvicorn creates many workers # The following call will also create necessary structures if they do not exist in redis - api.scheduling.initialization.start_updating_process() + start_scheduler_process() - uvicorn.run("api.main:app", + uvicorn.run("app.main:app", host=APP_SETTINGS['BACKEND']['FASTAPI']['HOST'], port=APP_SETTINGS['BACKEND']['FASTAPI']['PORT'], reload=APP_SETTINGS['BACKEND']['FASTAPI']['RELOAD'], debug=APP_SETTINGS['BACKEND']['FASTAPI']['DEBUG'], workers=APP_SETTINGS['BACKEND']['FASTAPI']['WORKERS'], - reload_dirs=["recommendation_system/api"], + reload_dirs=["recommendation_system/app"], log_level="info" if APP_SETTINGS['BACKEND']['PROD'] else "debug") diff --git a/api/recommender/__init__.py b/app/recommender/__init__.py similarity index 100% rename from api/recommender/__init__.py rename to app/recommender/__init__.py diff --git a/api/recommender/monitoring_services/__init__.py b/app/recommender/project_completion/__init__.py similarity index 100% rename from api/recommender/monitoring_services/__init__.py rename to app/recommender/project_completion/__init__.py diff --git a/api/recommender/project_completion/components/filtering.py b/app/recommender/project_completion/components/filtering.py similarity index 92% rename from api/recommender/project_completion/components/filtering.py rename to app/recommender/project_completion/components/filtering.py index 1412b84..7580a5e 100644 --- a/api/recommender/project_completion/components/filtering.py +++ b/app/recommender/project_completion/components/filtering.py @@ -1,6 +1,6 @@ import logging -from api.databases.registry.registry_selector import get_registry +from app.databases.registry.registry_selector import get_registry logger = logging.getLogger(__name__) diff --git a/api/recommender/project_completion/components/recommendation_cadidates.py b/app/recommender/project_completion/components/recommendation_cadidates.py similarity index 82% rename from api/recommender/project_completion/components/recommendation_cadidates.py rename to app/recommender/project_completion/components/recommendation_cadidates.py index d92ff98..9729b60 100644 --- a/api/recommender/project_completion/components/recommendation_cadidates.py +++ b/app/recommender/project_completion/components/recommendation_cadidates.py @@ -1,5 +1,5 @@ -from api.recommender.project_completion.components.filtering import filtering -from api.recommender.project_completion.initialization.association_rules import \ +from app.recommender.project_completion.components.filtering import filtering +from app.recommender.project_completion.initialization.association_rules import \ get_association_rules diff --git a/api/recommender/project_completion/__init__.py b/app/recommender/project_completion/initialization/__init__.py similarity index 100% rename from api/recommender/project_completion/__init__.py rename to app/recommender/project_completion/initialization/__init__.py diff --git a/api/recommender/project_completion/initialization/association_rules.py b/app/recommender/project_completion/initialization/association_rules.py similarity index 82% rename from api/recommender/project_completion/initialization/association_rules.py rename to app/recommender/project_completion/initialization/association_rules.py index 5a975bd..d0e62a0 100644 --- a/api/recommender/project_completion/initialization/association_rules.py +++ b/app/recommender/project_completion/initialization/association_rules.py @@ -1,14 +1,14 @@ +import logging + import pandas as pd +from app.databases.redis_db import (check_key_existence, delete_object, + get_object, store_object) +from app.databases.registry.registry_selector import get_registry +from app.exceptions import NoneProjects +from app.settings import APP_SETTINGS from mlxtend.frequent_patterns import association_rules, fpgrowth from mlxtend.preprocessing import TransactionEncoder -from api.databases.registry.registry_selector import get_registry - -from api.databases.redis_db import (check_key_existence, delete_object, - get_object, store_object) -from api.settings import APP_SETTINGS -from api.exceptions import NoneProjects - def get_projects_services(db): # Get all project ids @@ -61,3 +61,9 @@ def delete_association_rules(): def existence_association_rules(): return check_key_existence("ASSOCIATION_RULES") + + +def initialize_association_rules(): + if not existence_association_rules(): + logging.debug("Association rules do not exist. Creating...") + create_association_rules() diff --git a/api/recommender/project_completion/recommendation_generation.py b/app/recommender/project_completion/recommendation_generation.py similarity index 77% rename from api/recommender/project_completion/recommendation_generation.py rename to app/recommender/project_completion/recommendation_generation.py index 5bcc99a..0bb9b71 100644 --- a/api/recommender/project_completion/recommendation_generation.py +++ b/app/recommender/project_completion/recommendation_generation.py @@ -1,9 +1,9 @@ import logging -from api.databases.registry.registry_selector import get_registry -from api.recommender.project_completion.components.recommendation_cadidates import \ +from app.databases.registry.registry_selector import get_registry +from app.exceptions import IdNotExists +from app.recommender.project_completion.components.recommendation_cadidates import \ get_recommendation_candidates -from api.exceptions import IdNotExists logger = logging.getLogger(__name__) diff --git a/app/recommender/project_completion/update.py b/app/recommender/project_completion/update.py new file mode 100644 index 0000000..c36e8f9 --- /dev/null +++ b/app/recommender/project_completion/update.py @@ -0,0 +1,16 @@ +from app.recommender.project_completion.initialization import association_rules +from app.recommender.update.update import Update + + +class ProjectCompletionUpdate(Update): + def initialize(self): + association_rules.initialize_association_rules() + + def update(self): + association_rules.create_association_rules() + + def update_for_new_service(self, service_id: int): + pass + + def revert(self): + association_rules.delete_association_rules() diff --git a/api/recommender/project_completion/initialization/__init__.py b/app/recommender/similar_services/__init__.py similarity index 100% rename from api/recommender/project_completion/initialization/__init__.py rename to app/recommender/similar_services/__init__.py diff --git a/api/recommender/similar_services/__init__.py b/app/recommender/similar_services/field_suggestion/__init__.py similarity index 100% rename from api/recommender/similar_services/__init__.py rename to app/recommender/similar_services/field_suggestion/__init__.py diff --git a/api/recommender/similar_services/field_suggestion/__init__.py b/app/recommender/similar_services/field_suggestion/evaluation/__init__.py similarity index 100% rename from api/recommender/similar_services/field_suggestion/__init__.py rename to app/recommender/similar_services/field_suggestion/evaluation/__init__.py diff --git a/api/recommender/similar_services/field_suggestion/evaluation/baselines.py b/app/recommender/similar_services/field_suggestion/evaluation/baselines.py similarity index 94% rename from api/recommender/similar_services/field_suggestion/evaluation/baselines.py rename to app/recommender/similar_services/field_suggestion/evaluation/baselines.py index 1529089..3774d04 100644 --- a/api/recommender/similar_services/field_suggestion/evaluation/baselines.py +++ b/app/recommender/similar_services/field_suggestion/evaluation/baselines.py @@ -1,9 +1,9 @@ import random import pandas as pd - -from api.databases.registry.registry_selector import get_registry -from api.recommender.similar_services.field_suggestion.evaluation.evaluation import evaluate_one +from app.databases.registry.registry_selector import get_registry +from app.recommender.similar_services.field_suggestion.evaluation.evaluation import \ + evaluate_one def create_random_suggestions(fields_values, max_suggestions_per_field): diff --git a/api/recommender/similar_services/field_suggestion/evaluation/evaluation.py b/app/recommender/similar_services/field_suggestion/evaluation/evaluation.py similarity index 94% rename from api/recommender/similar_services/field_suggestion/evaluation/evaluation.py rename to app/recommender/similar_services/field_suggestion/evaluation/evaluation.py index 59ad061..af3ed73 100644 --- a/api/recommender/similar_services/field_suggestion/evaluation/evaluation.py +++ b/app/recommender/similar_services/field_suggestion/evaluation/evaluation.py @@ -1,7 +1,7 @@ -from api.databases.registry.registry_selector import get_registry -from api.recommender.similar_services.field_suggestion.suggestion_generation import \ +from app.databases.registry.registry_selector import get_registry +from app.recommender.similar_services.field_suggestion.suggestion_generation import \ get_auto_complete_suggestions -from api.settings import APP_SETTINGS +from app.settings import APP_SETTINGS def evaluate_one(gold_values, suggestions): diff --git a/api/recommender/similar_services/field_suggestion/evaluation/optimal_configuration.py b/app/recommender/similar_services/field_suggestion/evaluation/optimal_configuration.py similarity index 98% rename from api/recommender/similar_services/field_suggestion/evaluation/optimal_configuration.py rename to app/recommender/similar_services/field_suggestion/evaluation/optimal_configuration.py index 00c5fe4..9d95fed 100644 --- a/api/recommender/similar_services/field_suggestion/evaluation/optimal_configuration.py +++ b/app/recommender/similar_services/field_suggestion/evaluation/optimal_configuration.py @@ -1,7 +1,7 @@ import itertools import json -from api.recommender.similar_services.field_suggestion.evaluation.evaluation import \ +from app.recommender.similar_services.field_suggestion.evaluation.evaluation import \ evaluation from tqdm import tqdm diff --git a/api/recommender/similar_services/field_suggestion/similar_services.py b/app/recommender/similar_services/field_suggestion/similar_services.py similarity index 93% rename from api/recommender/similar_services/field_suggestion/similar_services.py rename to app/recommender/similar_services/field_suggestion/similar_services.py index 4bdc5c7..0d76ed9 100644 --- a/api/recommender/similar_services/field_suggestion/similar_services.py +++ b/app/recommender/similar_services/field_suggestion/similar_services.py @@ -3,10 +3,10 @@ import numpy as np import pandas as pd -from api.exceptions import MissingStructure -from api.recommender.similar_services.preprocessor.embeddings.text_embeddings import ( +from app.exceptions import MissingStructure +from app.recommender.similar_services.preprocessor.embeddings.text_embeddings import ( existence_text_embeddings, get_text_embeddings) -from api.settings import APP_SETTINGS +from app.settings import APP_SETTINGS from sklearn.metrics.pairwise import cosine_similarity logger = logging.getLogger(__name__) diff --git a/app/recommender/similar_services/field_suggestion/suggestion_candidates.py b/app/recommender/similar_services/field_suggestion/suggestion_candidates.py new file mode 100644 index 0000000..022e3fc --- /dev/null +++ b/app/recommender/similar_services/field_suggestion/suggestion_candidates.py @@ -0,0 +1,34 @@ +import math +from collections import Counter +from itertools import chain + + +def get_candidates(field_values, frequency_threshold=None, existing_values=None): + """ + Returns a list of values based on frequency across values sets + Args: + field_values: list, a list with all values of a field of the most similar services + frequency_threshold: float, the required frequency threshold for each candidate value + existing_values: list, the existing values for the field + + Returns: list of candidate values + + """ + if len(field_values) == 0: + return [] + + if frequency_threshold is None: + frequency_threshold = 0 + + # Calculate the appearances of each value + values_count = dict(Counter(chain(*field_values))) + + # Filter values based on + counts_threshold = math.ceil(frequency_threshold * len(field_values)) + filtered_values = list(dict(filter(lambda elem: elem[1] >= counts_threshold, values_count.items())).keys()) + + # If there are already some values in the field, filter them + if existing_values is not None: + filtered_values = set(filtered_values).difference(set(existing_values)) + + return filtered_values diff --git a/app/recommender/similar_services/field_suggestion/suggestion_generation.py b/app/recommender/similar_services/field_suggestion/suggestion_generation.py new file mode 100644 index 0000000..af1c5d3 --- /dev/null +++ b/app/recommender/similar_services/field_suggestion/suggestion_generation.py @@ -0,0 +1,70 @@ +from app.databases.registry.registry_selector import get_registry +from app.recommender.similar_services.field_suggestion.similar_services import \ + get_similar_services +from app.recommender.similar_services.field_suggestion.suggestion_candidates import \ + get_candidates +from app.recommender.similar_services.preprocessor.embeddings.text_embeddings import \ + create_text_embedding +from app.settings import APP_SETTINGS + + +def get_auto_complete_suggestions(new_service, requested_fields, maximum_suggestions=5, existing_fields_values=None, + evaluation_mode=False, similarity_threshold=None, + considered_services_threshold=None, frequency_threshold=None): + """ + Args: + new_service: dict with the name and value for each filled field of a service + requested_fields: list, the names of the fields for which auto-completion will be implemented + maximum_suggestions: int, the maximum number of suggestions per field + existing_fields_values: dict with the name and the current values of each field + evaluation_mode: boolean + similarity_threshold: float, the similarity threshold to be considered for all the fields + considered_services_threshold: int, the number of services to be considered for the selection + of the fields values + frequency_threshold: float, the required frequency threshold of the values in the considered services + for all fields + + Returns: dict with the names and suggested values for all requested fields + """ + + # Calculate the text embedding of the new service + text_embedding = create_text_embedding(new_service) + + # Get similar services ids + similar_services_ids_per_field = get_similar_services(requested_fields, text_embedding, similarity_threshold, + considered_services_threshold) + + if evaluation_mode: + # Remove evaluated service from similar services of every field + for _, similar_services in similar_services_ids_per_field.items(): + if new_service["service_id"] in similar_services: + similar_services.remove(new_service["service_id"]) + + # Get the requested fields for all similar services + all_similar_services_ids = list( + set().union(*[similar_services for _, similar_services in similar_services_ids_per_field.items()])) + + db = get_registry() + similar_services = db.get_services_by_ids(ids=all_similar_services_ids, + attributes=requested_fields, + remove_generic_attributes=True) + + # Find the most used values for every requested field + suggestions = {} + for requested_field in requested_fields: + if existing_fields_values is not None and requested_field in existing_fields_values: + existing_values = existing_fields_values[requested_field] + else: + existing_values = None + + field_suggestions = get_candidates(field_values=similar_services[similar_services["service_id"] + .isin(similar_services_ids_per_field[requested_field])][requested_field] + .values.tolist(), + frequency_threshold=APP_SETTINGS["BACKEND"]["AUTO_COMPLETION"] + [requested_field]["FREQUENCY_THRESHOLD"], + existing_values=existing_values) + + suggestions[requested_field] = field_suggestions \ + if len(field_suggestions) <= maximum_suggestions else field_suggestions[:maximum_suggestions] + + return suggestions diff --git a/app/recommender/similar_services/field_suggestion/update.py b/app/recommender/similar_services/field_suggestion/update.py new file mode 100644 index 0000000..c48e5e2 --- /dev/null +++ b/app/recommender/similar_services/field_suggestion/update.py @@ -0,0 +1,17 @@ +from app.recommender.similar_services.preprocessor.embeddings import \ + text_embeddings +from app.recommender.update.update import Update + + +class FieldSuggestionUpdate(Update): + def initialize(self): + text_embeddings.initialize_text_embeddings() + + def update(self): + text_embeddings.create_text_embeddings() + + def update_for_new_service(self, service_id: int): + text_embeddings.update_text_embedding(new_service_id=service_id) + + def revert(self): + text_embeddings.delete_text_embeddings() diff --git a/api/recommender/similar_services/field_suggestion/evaluation/__init__.py b/app/recommender/similar_services/preprocessor/__init__.py similarity index 100% rename from api/recommender/similar_services/field_suggestion/evaluation/__init__.py rename to app/recommender/similar_services/preprocessor/__init__.py diff --git a/api/recommender/similar_services/preprocessor/__init__.py b/app/recommender/similar_services/preprocessor/embeddings/__init__.py similarity index 100% rename from api/recommender/similar_services/preprocessor/__init__.py rename to app/recommender/similar_services/preprocessor/embeddings/__init__.py diff --git a/api/recommender/similar_services/preprocessor/embeddings/metadata_embeddings.py b/app/recommender/similar_services/preprocessor/embeddings/metadata_embeddings.py similarity index 93% rename from api/recommender/similar_services/preprocessor/embeddings/metadata_embeddings.py rename to app/recommender/similar_services/preprocessor/embeddings/metadata_embeddings.py index a4545ca..90d8eb0 100644 --- a/api/recommender/similar_services/preprocessor/embeddings/metadata_embeddings.py +++ b/app/recommender/similar_services/preprocessor/embeddings/metadata_embeddings.py @@ -2,12 +2,11 @@ import numpy as np import pandas as pd - -from api.databases.redis_db import (check_key_existence, delete_object, +from app.databases.redis_db import (check_key_existence, delete_object, get_object, store_object) -from api.databases.registry.registry_selector import get_registry -from api.exceptions import MissingStructure, NoneServices, IdNotExists -from api.settings import APP_SETTINGS +from app.databases.registry.registry_selector import get_registry +from app.exceptions import IdNotExists, MissingStructure, NoneServices +from app.settings import APP_SETTINGS from sklearn.preprocessing import MultiLabelBinarizer logger = logging.getLogger(__name__) diff --git a/api/recommender/similar_services/preprocessor/embeddings/text_embeddings.py b/app/recommender/similar_services/preprocessor/embeddings/text_embeddings.py similarity index 94% rename from api/recommender/similar_services/preprocessor/embeddings/text_embeddings.py rename to app/recommender/similar_services/preprocessor/embeddings/text_embeddings.py index 50c9cd3..b984356 100644 --- a/api/recommender/similar_services/preprocessor/embeddings/text_embeddings.py +++ b/app/recommender/similar_services/preprocessor/embeddings/text_embeddings.py @@ -1,12 +1,12 @@ import logging import pandas as pd -from api.databases.redis_db import (check_key_existence, delete_object, +from app.databases.redis_db import (check_key_existence, delete_object, get_object, store_object) -from api.databases.registry.registry_selector import get_registry -from api.exceptions import (IdNotExists, MissingAttribute, MissingStructure, +from app.databases.registry.registry_selector import get_registry +from app.exceptions import (IdNotExists, MissingAttribute, MissingStructure, NoneServices) -from api.settings import APP_SETTINGS +from app.settings import APP_SETTINGS from sklearn.decomposition import TruncatedSVD from sklearn.feature_extraction.text import TfidfVectorizer diff --git a/api/recommender/similar_services/preprocessor/embeddings/__init__.py b/app/recommender/similar_services/preprocessor/reports/__init__.py similarity index 100% rename from api/recommender/similar_services/preprocessor/embeddings/__init__.py rename to app/recommender/similar_services/preprocessor/reports/__init__.py diff --git a/api/recommender/similar_services/preprocessor/reports/monitoring_reports.py b/app/recommender/similar_services/preprocessor/reports/monitoring_reports.py similarity index 73% rename from api/recommender/similar_services/preprocessor/reports/monitoring_reports.py rename to app/recommender/similar_services/preprocessor/reports/monitoring_reports.py index 90b0ab9..5cad710 100644 --- a/api/recommender/similar_services/preprocessor/reports/monitoring_reports.py +++ b/app/recommender/similar_services/preprocessor/reports/monitoring_reports.py @@ -1,30 +1,30 @@ import logging -from api.databases.redis_db import (check_key_existence, delete_object, get_object) -from api.exceptions import MissingStructure -from api.databases.argo_monitoring_api import ArgoMonitoringApi + +from app.databases.argo_monitoring_api import ArgoMonitoringApi +from app.databases.redis_db import (check_key_existence, delete_object, + get_object, store_object) +from app.exceptions import MissingStructure def create_status_report(): mon = ArgoMonitoringApi() status_report = mon.get_status_report() - return status_report + + store_object(status_report, "STATUS_REPORT") def create_ar_report(): mon = ArgoMonitoringApi() ar_report = mon.get_ar_report() - return ar_report + + store_object(ar_report, "AR_REPORT") def update_status_report(): - if existence_status_report(): - delete_status_report() create_status_report() def update_ar_report(): - if existence_ar_report(): - delete_ar_report() create_ar_report() @@ -57,12 +57,14 @@ def delete_status_report(): def initialise_status_report(): + # TODO: Is existence check needed on this use case if not existence_status_report(): logging.info("Status report does not exist. Creating...") create_status_report() def initialise_ar_report(): + # TODO: Is existence check needed on this use case if not existence_ar_report(): logging.info("Status report does not exist. Creating...") create_ar_report() diff --git a/api/recommender/similar_services/preprocessor/reports/__init__.py b/app/recommender/similar_services/preprocessor/similarities/__init__.py similarity index 100% rename from api/recommender/similar_services/preprocessor/reports/__init__.py rename to app/recommender/similar_services/preprocessor/similarities/__init__.py diff --git a/api/recommender/similar_services/preprocessor/similarities/metadata_similarities.py b/app/recommender/similar_services/preprocessor/similarities/metadata_similarities.py similarity index 79% rename from api/recommender/similar_services/preprocessor/similarities/metadata_similarities.py rename to app/recommender/similar_services/preprocessor/similarities/metadata_similarities.py index ba85c0e..6d3f18c 100644 --- a/api/recommender/similar_services/preprocessor/similarities/metadata_similarities.py +++ b/app/recommender/similar_services/preprocessor/similarities/metadata_similarities.py @@ -1,11 +1,12 @@ import logging -from api.databases.redis_db import (check_key_existence, delete_object, +from app.databases.redis_db import (check_key_existence, delete_object, get_object, store_object) -from api.exceptions import MissingStructure -from api.recommender.similar_services.preprocessor.embeddings.metadata_embeddings import get_metadata_embeddings -from api.recommender.similar_services.preprocessor.similarities.similarities import ( - create_similarities, update_similarities, initialize_similarities) +from app.exceptions import MissingStructure +from app.recommender.similar_services.preprocessor.embeddings.metadata_embeddings import \ + get_metadata_embeddings +from app.recommender.similar_services.preprocessor.similarities.similarities import ( + create_similarities, initialize_similarities, update_similarities) logger = logging.getLogger(__name__) diff --git a/api/recommender/similar_services/preprocessor/similarities/similarities.py b/app/recommender/similar_services/preprocessor/similarities/similarities.py similarity index 99% rename from api/recommender/similar_services/preprocessor/similarities/similarities.py rename to app/recommender/similar_services/preprocessor/similarities/similarities.py index 466b0d9..4f45e61 100644 --- a/api/recommender/similar_services/preprocessor/similarities/similarities.py +++ b/app/recommender/similar_services/preprocessor/similarities/similarities.py @@ -1,4 +1,5 @@ import logging + import pandas as pd from sklearn.metrics.pairwise import cosine_similarity diff --git a/api/recommender/similar_services/preprocessor/similarities/text_similarities.py b/app/recommender/similar_services/preprocessor/similarities/text_similarities.py similarity index 79% rename from api/recommender/similar_services/preprocessor/similarities/text_similarities.py rename to app/recommender/similar_services/preprocessor/similarities/text_similarities.py index 1257a1c..c394c3d 100644 --- a/api/recommender/similar_services/preprocessor/similarities/text_similarities.py +++ b/app/recommender/similar_services/preprocessor/similarities/text_similarities.py @@ -1,12 +1,12 @@ import logging -from api.databases.redis_db import (check_key_existence, delete_object, +from app.databases.redis_db import (check_key_existence, delete_object, get_object, store_object) -from api.exceptions import MissingStructure -from api.recommender.similar_services.preprocessor.embeddings.text_embeddings import ( - get_text_embeddings) -from api.recommender.similar_services.preprocessor.similarities.similarities import ( - create_similarities, update_similarities, initialize_similarities) +from app.exceptions import MissingStructure +from app.recommender.similar_services.preprocessor.embeddings.text_embeddings import \ + get_text_embeddings +from app.recommender.similar_services.preprocessor.similarities.similarities import ( + create_similarities, initialize_similarities, update_similarities) logger = logging.getLogger(__name__) diff --git a/api/recommender/similar_services/preprocessor/similarities/__init__.py b/app/recommender/similar_services/project_assistant/__init__.py similarity index 100% rename from api/recommender/similar_services/preprocessor/similarities/__init__.py rename to app/recommender/similar_services/project_assistant/__init__.py diff --git a/api/recommender/similar_services/project_assistant/recommendation_generation.py b/app/recommender/similar_services/project_assistant/recommendation_generation.py similarity index 86% rename from api/recommender/similar_services/project_assistant/recommendation_generation.py rename to app/recommender/similar_services/project_assistant/recommendation_generation.py index 39aa663..1495a4e 100644 --- a/api/recommender/similar_services/project_assistant/recommendation_generation.py +++ b/app/recommender/similar_services/project_assistant/recommendation_generation.py @@ -1,11 +1,11 @@ -import pandas as pd import numpy as np +import pandas as pd +from app.databases.registry.registry_selector import get_registry +from app.recommender.similar_services.preprocessor.embeddings.text_embeddings import ( + generate_sbert_embedding, get_text_embeddings) +from app.settings import APP_SETTINGS from sklearn.metrics.pairwise import cosine_similarity -from api.databases.registry.registry_selector import get_registry -from api.settings import APP_SETTINGS -from api.recommender.similar_services.preprocessor.embeddings.text_embeddings import \ - get_text_embeddings, generate_sbert_embedding def filter_by_status(db, services): # Get non-published resources @@ -53,4 +53,4 @@ def project_assistant_recommendation(description, max_num): similar_services = similar_services.sort_values(by=["similarity"], ascending=False) return [{"service_id": service_id, "score": score["similarity"]} for service_id, score in - similar_services[:max_num].iterrows()] \ No newline at end of file + similar_services[:max_num].iterrows()] diff --git a/app/recommender/similar_services/project_assistant/update.py b/app/recommender/similar_services/project_assistant/update.py new file mode 100644 index 0000000..5346787 --- /dev/null +++ b/app/recommender/similar_services/project_assistant/update.py @@ -0,0 +1,17 @@ +from app.recommender.similar_services.preprocessor.embeddings import \ + text_embeddings +from app.recommender.update.update import Update + + +class ProjectAssistantUpdate(Update): + def initialize(self): + text_embeddings.initialize_text_embeddings() + + def update(self): + text_embeddings.create_text_embeddings() + + def update_for_new_service(self, service_id: int): + text_embeddings.update_text_embedding(new_service_id=service_id) + + def revert(self): + text_embeddings.delete_text_embeddings() diff --git a/api/recommender/similar_services/project_assistant/__init__.py b/app/recommender/similar_services/service_recommendation/__init__.py similarity index 100% rename from api/recommender/similar_services/project_assistant/__init__.py rename to app/recommender/similar_services/service_recommendation/__init__.py diff --git a/api/recommender/similar_services/service_recommendation/__init__.py b/app/recommender/similar_services/service_recommendation/components/__init__.py similarity index 100% rename from api/recommender/similar_services/service_recommendation/__init__.py rename to app/recommender/similar_services/service_recommendation/components/__init__.py diff --git a/api/recommender/similar_services/service_recommendation/components/filtering.py b/app/recommender/similar_services/service_recommendation/components/filtering.py similarity index 71% rename from api/recommender/similar_services/service_recommendation/components/filtering.py rename to app/recommender/similar_services/service_recommendation/components/filtering.py index b27d1a6..06e8533 100644 --- a/api/recommender/similar_services/service_recommendation/components/filtering.py +++ b/app/recommender/similar_services/service_recommendation/components/filtering.py @@ -1,6 +1,6 @@ import logging -from api.databases.redis_db import get_object +from app.databases.redis_db import get_object logger = logging.getLogger(__name__) @@ -23,10 +23,8 @@ def filtering(db, resources, viewing_resource, purchased_resources): """ logger.debug(f"Filter resources...") - # Get non-published resources + # Get non-published resources ids non_published_resources = db.get_non_published_services() - # Get the indexes of viewing, purchased and non-published resources - indexes_to_drop = list(set(non_published_resources + [viewing_resource] + purchased_resources)) - - return resources.drop(labels=indexes_to_drop) + # Remove every non-published resource existing in the resources and return them + return resources.drop(labels=non_published_resources, errors='ignore') diff --git a/api/recommender/similar_services/service_recommendation/components/ordering.py b/app/recommender/similar_services/service_recommendation/components/ordering.py similarity index 100% rename from api/recommender/similar_services/service_recommendation/components/ordering.py rename to app/recommender/similar_services/service_recommendation/components/ordering.py diff --git a/api/recommender/similar_services/service_recommendation/components/reranking.py b/app/recommender/similar_services/service_recommendation/components/reranking.py similarity index 52% rename from api/recommender/similar_services/service_recommendation/components/reranking.py rename to app/recommender/similar_services/service_recommendation/components/reranking.py index 693d58e..cbb791a 100644 --- a/api/recommender/similar_services/service_recommendation/components/reranking.py +++ b/app/recommender/similar_services/service_recommendation/components/reranking.py @@ -1,20 +1,23 @@ import pandas as pd -from api.databases.redis_db import get_object +from app.recommender.similar_services.service_recommendation.components.resources_similarity import \ + resources_similarity +from app.settings import APP_SETTINGS -def re_ranking(target_service, candidates, recommendations_num): +def re_ranking(target_service, purchases, candidates, recommendations_num, viewed_weight, metadata_weight): """Re-rank recommendations w.r.t similarity and diversity of a service Args: target_service (id): target service id + purchases (list): list of ids of purchased services candidates (dict): service ids related to target service and their similarity recommendations_num (int): number of recommendations to generate - + viewed_weight: float [0,1], the weight of the viewed resource similarity in the score calculation + metadata_weight: float [0,1], the weight of the metadata similarity in the score calculation Returns: recommended_services (dict): recommended services ids and their quality score """ - _metadata_similarities = get_object('METADATA_SIMILARITY') - _text_similarities = get_object('TEXT_SIMILARITY') + diversity_weight = APP_SETTINGS["BACKEND"]["SIMILAR_SERVICES"]["DIVERSITY_WEIGHT"] # R' recommended_services = {} @@ -32,9 +35,11 @@ def re_ranking(target_service, candidates, recommendations_num): quality = quality_metric( target_service=target_service, current_service=service_id, + purchases=purchases, relative_services=recommended_services, - metadata_similarities=_metadata_similarities, - text_similarities=_text_similarities + metadata_weight=metadata_weight, + viewed_weight=viewed_weight, + diversity_weight=diversity_weight ) top_k_similar_services.loc[service_id] = quality # Get first service in C' w.r.t Quality metric @@ -47,15 +52,18 @@ def re_ranking(target_service, candidates, recommendations_num): top_k_similar_services.drop(first, inplace=True) recommended_services = pd.Series(recommended_services) + # Re-order recommendations based on similarity to target service + recommended_services = candidates.loc[recommended_services.index].sort_values(ascending=False) return recommended_services -def quality_metric(target_service, current_service, relative_services, metadata_similarities, text_similarities): +def quality_metric(target_service, current_service, purchases, relative_services, + viewed_weight, metadata_weight, diversity_weight): """Quality metric that combines diversity and similarity - quality = 1/2 * similarity(target_service, current_service) + - 1/2 * relative_diversity(current_service, relative_services) + quality = (1 - diversity_weight) * similarity(target_service, current_service) + + diversity_weight * relative_diversity(current_service, relative_services) References: Bradley and Smyth. "Improving Recommendation Diversity", 2001 @@ -63,9 +71,11 @@ def quality_metric(target_service, current_service, relative_services, metadata_ Args: target_service (int): target service id current_service (int): current service id + purchases (list): list of ids of purchased services relative_services (dict): relative services ids - metadata_similarities (dataframe): metadata services - text_similarities (dataframe): + viewed_weight: float [0,1], the weight of the viewed resource similarity in the score calculation + metadata_weight: float [0,1], the weight of the metadata similarity in the score calculation + diversity_weight: float [0,1], the weight of the diversity vs. the similarity Returns: quality (float): quality metric """ @@ -76,18 +86,23 @@ def quality_metric(target_service, current_service, relative_services, metadata_ relative_diversity = 1 else: diversities = [] - for rel_service in relative_services: - sim = 1 / 2 * metadata_similarities.loc[current_service][rel_service] + \ - 1 / 2 * text_similarities.loc[current_service][rel_service] - sim_df = pd.DataFrame([sim]) - diversities.append(diversity_metric(sim_df)) + similarity = resources_similarity(resource=current_service, + compared_resources=list(relative_services.keys()), + view_weight=viewed_weight, + metadata_weight=metadata_weight).values + + similarity_df = pd.DataFrame([similarity]) + diversities.append(diversity_metric(similarity_df)) relative_diversity = sum(diversities) / m # calculate similarity of target service and current service - target_current_similarity = 1 / 2 * metadata_similarities.loc[target_service][current_service] + \ - 1 / 2 * text_similarities.loc[target_service][current_service] + target_current_similarity = resources_similarity(resource=target_service, + purchased_resources=purchases, + compared_resources=[current_service], + view_weight=viewed_weight, + metadata_weight=metadata_weight).values - quality = 1 / 2 * target_current_similarity + 1 / 2 * relative_diversity + quality = (1 - diversity_weight) * target_current_similarity + diversity_weight * relative_diversity return quality diff --git a/api/recommender/similar_services/service_recommendation/components/recommendation_candidates.py b/app/recommender/similar_services/service_recommendation/components/resources_similarity.py similarity index 53% rename from api/recommender/similar_services/service_recommendation/components/recommendation_candidates.py rename to app/recommender/similar_services/service_recommendation/components/resources_similarity.py index b73b6b8..3960541 100644 --- a/api/recommender/similar_services/service_recommendation/components/recommendation_candidates.py +++ b/app/recommender/similar_services/service_recommendation/components/resources_similarity.py @@ -2,32 +2,46 @@ import numpy as np import pandas as pd -from api.recommender.similar_services.preprocessor.similarities.metadata_similarities import \ +from app.recommender.similar_services.preprocessor.similarities.metadata_similarities import \ get_metadata_similarities -from api.recommender.similar_services.preprocessor.similarities.text_similarities import \ +from app.recommender.similar_services.preprocessor.similarities.text_similarities import \ get_text_similarities logger = logging.getLogger(__name__) # TODO: use of purchase time? -def get_recommendation_candidates(view_resource, purchased_resources, view_weight=0.5, metadata_weight=0.5): +def resources_similarity(resource, compared_resources=None, purchased_resources=None, + view_weight=0.5, metadata_weight=0.5): """ - Creates a structure with the score of each resource based on the viewing and purchased resources + Creates a structure with the similarity of each resource with the viewing and purchased resources Score of resource i S_i = metadata_weight * metadata_similarity(i, viewed_resource, purchased_resources) + (1-metadata_weight) * text_similarity(i, viewed_resource, purchased_resources) , where <>_similarity(i, view_resource, purchased_resources) = view_weight * sim(i, view_resource) + (1-view_weight) * avg(sim(i, purchased_resource_j)) - @param view_resource: str, the id of the currently viewing resource - @param purchased_resources: list, the ids of the user purchased resources - @param view_weight: float [0,1], the weight of the viewed resource similarity in the score calculation - @param metadata_weight: float [0,1], the weight of the metadata similarity in the score calculation - @return: + Args: + resource: the id of the resource to be compared + purchased_resources: list, the ids of the user purchased resources that need to be considered + compared_resources: list, the list of ids of the resources for which we want to calculate the similarity. If None, all resources are considered + view_weight: float [0,1], the weight of the viewed resource similarity in the score calculation + metadata_weight: float [0,1], the weight of the metadata similarity in the score calculation + + Returns: + """ - logger.debug("Calculating similarities...") + logger.debug("Calculating resources similarities...") metadata_similarities = get_metadata_similarities() text_similarities = get_text_similarities() + if purchased_resources is None: + purchased_resources = [] + + # # Filter resources based on compared_resources + if compared_resources is not None: + considered_resources = [resource]+purchased_resources+compared_resources + metadata_similarities = metadata_similarities.loc[considered_resources, considered_resources] + text_similarities = text_similarities.loc[considered_resources, considered_resources] + # Initialize weights indexing = metadata_similarities.index.to_list() weights = pd.Series(np.zeros(metadata_similarities.shape[0]), index=indexing) @@ -35,9 +49,9 @@ def get_recommendation_candidates(view_resource, purchased_resources, view_weigh # Add the weights of view_resource and purchased resources if len(purchased_resources) > 0: weights.loc[purchased_resources] = (1-view_weight)*(1/len(purchased_resources)) - weights.loc[view_resource] = view_weight + weights.loc[resource] = view_weight else: - weights.loc[view_resource] = 1.0 + weights.loc[resource] = 1.0 # Calculate the metadata and text similarity of each resource metadata_similarity = pd.Series(np.average(metadata_similarities, weights=weights, axis=1), @@ -49,4 +63,4 @@ def get_recommendation_candidates(view_resource, purchased_resources, view_weigh candidates = pd.Series(np.average(pd.concat([metadata_similarity, text_similarity], axis=1), weights=[metadata_weight, 1-metadata_weight], axis=1), index=indexing) - return candidates + return candidates.drop([resource]+purchased_resources) diff --git a/api/recommender/similar_services/service_recommendation/components/__init__.py b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/__init__.py similarity index 100% rename from api/recommender/similar_services/service_recommendation/components/__init__.py rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/__init__.py diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/evaluation.py b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/calculate_metrics.py similarity index 71% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/evaluation.py rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/calculate_metrics.py index 6431af6..6f6261f 100644 --- a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/evaluation.py +++ b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/calculate_metrics.py @@ -1,8 +1,23 @@ +import glob + import numpy as np import pandas as pd -import glob +EVALUATION_RESULTS_DIR = "app/recommender/similar_services/service_recommendation/evaluation/" \ + "manual_evaluation/storage/metrics" + +EXPERIMENTS_DIRS = [ + "app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/" + "storage/manual_evaluation_final_files/phase1_metadata", + "app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/" + "storage/manual_evaluation_final_files/phase2_text", + "app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/" + "storage/manual_evaluation_final_files/phase3_metadata_weight", + "app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/" + "storage/manual_evaluation_final_files/phase4_diversity_weight", +] + def dcg(recommendation): """ Returns the Discounted cumulative gain DCG = Σ(relevance_pos/ln(pos+1)) for pos=1,...,N @@ -85,19 +100,9 @@ def get_evaluations_results(annotation_files): if __name__ == '__main__': - # Evaluation results of phase 1 - metadata_annotation_files_path = "api/recommender/similar_services/service_recommendation/evaluation/" \ - "manual_evaluation/storage/manual_evaluation_results/phase1_metadata/" - metadata_annotation_files = glob.glob(metadata_annotation_files_path + '*') - - metadata_results = get_evaluations_results(metadata_annotation_files) - - # Evaluation results of phase 2 - text_annotation_files_path = "api/recommender/similar_services/service_recommendation/evaluation/" \ - "manual_evaluation/storage/manual_evaluation_results/phase2_text/" - text_annotation_files = glob.glob(text_annotation_files_path + '*') - - text_results = get_evaluations_results(text_annotation_files) - - print("End") - + for experiment_dir in EXPERIMENTS_DIRS: + annotation_files = glob.glob(experiment_dir + '/*') + results = get_evaluations_results(annotation_files) + # Save evaluation results + results.to_excel(f"{EVALUATION_RESULTS_DIR}/{experiment_dir.split('/')[-1]}_evaluation_results.xlsx", + index_label="Version") diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/__init__.py b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/conflict_resolution/__init__.py similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/__init__.py rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/conflict_resolution/__init__.py diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/mark_annotation_conflicts.py b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/conflict_resolution/mark_annotation_conflicts.py similarity index 72% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/mark_annotation_conflicts.py rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/conflict_resolution/mark_annotation_conflicts.py index 9b1e0cb..846dcb8 100644 --- a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/mark_annotation_conflicts.py +++ b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/conflict_resolution/mark_annotation_conflicts.py @@ -1,8 +1,16 @@ import glob import pandas as pd +from app.recommender.similar_services.service_recommendation.evaluation.manual_evaluation.manual_annotations.create_manual_annotations import \ + ANNOTATORS_POOL -ANNOTATORS_POOL = ["Anna", "Katerina", "Mike"] +ANNOTATION_DIR = \ + 'app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/' \ + 'conflict_resolution/storage/annotated_files/' + +MARKED_CONFLICTS_DIR = \ + 'app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/' \ + 'conflict_resolution/storage/marked_conflicts/' def mark_annotation_conflicts(annotation_file): @@ -38,7 +46,8 @@ def mark_annotation_conflicts(annotation_file): annotation_df = annotation_df.style.highlight_null() # Save file - writer = pd.ExcelWriter(f"{annotation_file[:-5]}_marked_conflicts.xlsx", engine='xlsxwriter') + writer = pd.ExcelWriter(f"{MARKED_CONFLICTS_DIR}" + f"{annotation_file.split('/')[-1][:-5]}_marked_conflicts.xlsx", engine='xlsxwriter') annotation_df.to_excel(writer) writer.save() @@ -47,9 +56,6 @@ def mark_annotation_conflicts(annotation_file): if __name__ == '__main__': # --config_file parameter needs to be filled # You can point it to any valid config file (it will not be taken into account for creating the evaluation files) - annotation_files_path = "api/recommender/similar_services/service_recommendation/evaluation/" \ - "manual_evaluation/storage/manual_evaluation_results/" - - annotation_files = glob.glob(annotation_files_path + '*') + annotation_files = glob.glob(ANNOTATION_DIR + '*') for annotation_file in annotation_files: mark_annotation_conflicts(annotation_file=annotation_file) diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/empty_manual_evaluations/.gitkeep b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/conflict_resolution/storage/annotated_files/.gitkeep similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/empty_manual_evaluations/.gitkeep rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/conflict_resolution/storage/annotated_files/.gitkeep diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/manual_evaluation_results/.gitkeep b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/conflict_resolution/storage/marked_conflicts/.gitkeep similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/manual_evaluation_results/.gitkeep rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/conflict_resolution/storage/marked_conflicts/.gitkeep diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/ground_truth.py b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/ground_truth.py similarity index 77% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/ground_truth.py rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/ground_truth.py index 8f3954e..da26226 100644 --- a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/ground_truth.py +++ b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/ground_truth.py @@ -1,7 +1,6 @@ import glob import pandas as pd -import csv def _get_relatability(annotation, relatability_columns): @@ -15,6 +14,7 @@ def _get_relatability(annotation, relatability_columns): dissimilar_services_ids.append(annotation[f"Id{relatability_column[len('Relatability'):]}"]) return similar_services_ids, dissimilar_services_ids + def update_ground_truth(ground_truth, annotation_file): """ Updates the structure with the known similar services of each service @@ -34,17 +34,21 @@ def update_ground_truth(ground_truth, annotation_file): similar_services_ids, dissimilar_services_ids = _get_relatability(row, relatability_columns) # Update the similar services of the service with id if viewed_service_id in ground_truth: - ground_truth[viewed_service_id]["similar_services"] = list(set(similar_services_ids + ground_truth[viewed_service_id]["similar_services"])) - ground_truth[viewed_service_id]["dissimilar_services"] = list(set(dissimilar_services_ids + ground_truth[viewed_service_id]["dissimilar_services"])) + ground_truth[viewed_service_id]["similar_services"] = \ + list(set(similar_services_ids + ground_truth[viewed_service_id]["similar_services"])) + ground_truth[viewed_service_id]["dissimilar_services"] = \ + list(set(dissimilar_services_ids + ground_truth[viewed_service_id]["dissimilar_services"])) else: ground_truth[viewed_service_id] = {"similar_services": similar_services_ids, "dissimilar_services": dissimilar_services_ids} return ground_truth + if __name__ == '__main__': - annotation_files = glob.glob("api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/manual_evaluation_results/**/*.xlsx") + annotation_files = glob.glob("app/recommender/similar_services/service_recommendation/evaluation/" + "manual_evaluation/storage/manual_evaluation_final_files/**/*.xlsx") ground_truth = {} for annotation_file in annotation_files: @@ -54,5 +58,6 @@ def update_ground_truth(ground_truth, annotation_file): ground_truth_df = pd.DataFrame(columns=["similar_services", "dissimilar_services"], index=list(ground_truth.keys()), data=list(ground_truth.values())) - ground_truth_df.to_excel("api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/ground_truth.xlsx", - index_label="service_id") \ No newline at end of file + ground_truth_df.to_excel("app/recommender/similar_services/service_recommendation/evaluation/" + "manual_evaluation/storage/ground_truth.xlsx", + index_label="service_id") diff --git a/api/routes/__init__.py b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/__init__.py similarity index 100% rename from api/routes/__init__.py rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/__init__.py diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_C+SD+TU.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_C+SD+TU.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_C+SD+TU.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_C+SD+TU.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_C+SD.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_C+SD.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_C+SD.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_C+SD.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_C.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_C.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_C.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_C.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_D+TL.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_D+TL.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_D+TL.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_D+TL.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_D.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_D.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_D.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_D.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+D+TL.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_N+D+TL.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+D+TL.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_N+D+TL.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+D.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_N+D.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+D.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_N+D.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_N.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_N.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_SD.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_SD.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_SD.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_SD.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_TL.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_TL.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_TL.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/version_TL.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/create_manual_annotations.py b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/create_manual_annotations.py similarity index 78% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/create_manual_annotations.py rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/create_manual_annotations.py index dfe2f36..ba72240 100644 --- a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/create_manual_annotations.py +++ b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/create_manual_annotations.py @@ -1,38 +1,41 @@ import glob import random -from tqdm import tqdm from collections import defaultdict -import pandas as pd -from api.databases.registry.registry_selector import get_registry -from api.routes.update import update as structures_update -from api.settings import APP_SETTINGS, update_backend_settings -from api.recommender.similar_services.service_recommendation.recommendation_generation import create_recommendation +import pandas as pd +from app.databases.registry.registry_selector import get_registry +from app.recommender.similar_services.service_recommendation.recommendation_generation import \ + create_recommendation +from app.routes.update import update as structures_update +from app.settings import APP_SETTINGS, update_backend_settings +from tqdm import tqdm +CONFIG_DIR = \ + 'app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/configs/' GOLD_SERVICES_PATH = \ - 'api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/gold_services.csv' + 'app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/gold_services.csv' OUTPUT_DIR = \ - 'api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/empty_manual_evaluations/' + 'app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/storage/' GROUND_TRUTH_PATH = \ - 'api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/ground_truth.xlsx' + 'app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/ground_truth.xlsx' ANNOTATORS_POOL = ['Anna', 'Katerina', 'Mike'] ANNOTATOR_OVERLAP = 2 def get_recommendations(viewed_service_id): - # Get the produced recommendations for the viewed service recommendations = create_recommendation(viewed_service_id, recommendations_num=6) # Get the names of the viewed a recommended services db = get_registry() - services_with_names = db.get_services_by_ids(ids=[viewed_service_id]+ + services_with_names = db.get_services_by_ids(ids=[viewed_service_id] + [rec_item["service_id"] for rec_item in recommendations], attributes=["service_id", "name"]) return { 'viewed_service_id': viewed_service_id, - 'viewed_service_name': services_with_names[services_with_names["service_id"] == viewed_service_id].iloc[0]["name"], + 'viewed_service_name': services_with_names[services_with_names["service_id"] == viewed_service_id].iloc[0][ + "name"], 'recommendations': [ [rec_item["service_id"], rec_item["name"]] for _, rec_item in services_with_names[services_with_names["service_id"] != viewed_service_id].iterrows() @@ -50,8 +53,10 @@ def generate_column_names(recommendations_numb=6): def get_ground_truth_relatability(viewed_service, recommended_service, relatability_ground_truth_df): try: - similar_services = relatability_ground_truth_df[relatability_ground_truth_df["service_id"]==viewed_service]["similar_services"].values[0] - dissimilar_services = relatability_ground_truth_df[relatability_ground_truth_df["service_id"]==viewed_service]["dissimilar_services"].values[0] + similar_services = relatability_ground_truth_df[relatability_ground_truth_df["service_id"] + == viewed_service]["similar_services"].values[0] + dissimilar_services = relatability_ground_truth_df[relatability_ground_truth_df["service_id"] + == viewed_service]["dissimilar_services"].values[0] relatability = None if recommended_service in similar_services: relatability = 1 @@ -66,8 +71,8 @@ def flatten_recommendations_to_row(recommendation_result, relatability_ground_tr row = [recommendation_result['viewed_service_id'], recommendation_result['viewed_service_name']] for rec in recommendation_result['recommendations']: relatability = get_ground_truth_relatability(viewed_service=recommendation_result['viewed_service_id'], - recommended_service=rec[0], - relatability_ground_truth_df=relatability_ground_truth_df) + recommended_service=rec[0], + relatability_ground_truth_df=relatability_ground_truth_df) row += rec + [relatability] return row @@ -134,10 +139,10 @@ def iterate_over_configs(config_dir): print(config_files) for path in tqdm(config_files): - generate_excel(path, gold_service_ids, OUTPUT_DIR + path.split('/')[-1][:-5] + '.xls') + generate_excel(path, gold_service_ids, OUTPUT_DIR + path.split('/')[-1][:-5] + '.xlsx') if __name__ == '__main__': # --config_file parameter needs to be filled # You can point it to any valid config file (it will not be taken into account for creating the evaluation files) - iterate_over_configs('api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/') + iterate_over_configs(CONFIG_DIR) diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/recommendations_for_distance/.gitkeep b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/storage/.gitkeep similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/recommendations_for_distance/.gitkeep rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/manual_annotations/storage/.gitkeep diff --git a/api/scheduling/__init__.py b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/__init__.py similarity index 100% rename from api/scheduling/__init__.py rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/__init__.py diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_C+SD+TU.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_C+SD+TU.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_C+SD+TU.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_C+SD+TU.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_C+SD.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_C+SD.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_C+SD.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_C+SD.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_C+TU.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_C+TU.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_C+TU.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_C+TU.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_C.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_C.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_C.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_C.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_SD+TU.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_SD+TU.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_SD+TU.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_SD+TU.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_SD.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_SD.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_SD.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_SD.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_TU.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_TU.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/version_TU.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/version_TU.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_A+D+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_A+D+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_A+D+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_A+D+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_A+D+TL+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_A+D+TL+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_A+D+TL+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_A+D+TL+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_A+D+TL.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_A+D+TL.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_A+D+TL.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_A+D+TL.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_A+TL+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_A+TL+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_A+TL+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_A+TL+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_D+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_D+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_D+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_D+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_D+TL+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_D+TL+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_D+TL+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_D+TL+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_D+TL.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_D+TL.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_D+TL.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_D+TL.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_D.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_D.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_D.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_D.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+D+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+D+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+D+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+D+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+D+TL+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+D+TL+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+D+TL+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+D+TL+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+D+TL.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+D+TL.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+D+TL.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+D+TL.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+D.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+D.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+D.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+D.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+TL+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+TL+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+TL+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+TL+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+TL.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+TL.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A+TL.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A+TL.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+A.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+A.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+D+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+D+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+D+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+D+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+D+TL+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+D+TL+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+D+TL+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+D+TL+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_N+D+TL.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+D+TL.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_N+D+TL.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+D+TL.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_N+D.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+D.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_N+D.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+D.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+TL+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+TL+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+TL+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+TL+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+TL.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+TL.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_N+TL.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N+TL.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_N.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_N.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_N.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_TL+T.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_TL+T.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_text_attributes/version_TL+T.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_TL+T.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_TL.yaml b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_TL.yaml similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/manual_evaluation_configs/version_TL.yaml rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/configs/best_text_attributes/version_TL.yaml diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommendation_sets_distance.py b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/recommendation_sets_distance.py similarity index 77% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommendation_sets_distance.py rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/recommendation_sets_distance.py index 40c0457..ff750ad 100644 --- a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommendation_sets_distance.py +++ b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/recommendation_sets_distance.py @@ -1,14 +1,28 @@ import glob import matplotlib.pyplot as plt +import numpy as np import pandas as pd import seaborn as sns -from api.routes.update import update as structures_update -from api.settings import update_backend_settings +from app.recommender.similar_services.service_recommendation.recommendation_generation import \ + create_recommendation +from app.routes.update import update as structures_update +from app.settings import update_backend_settings from tqdm import tqdm +GOLD_SERVICES_PATH = \ + 'app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/gold_services.csv' +CONFIG_DIR = \ + "app/recommender/similar_services/service_recommendation/evaluation/" \ + "manual_evaluation/recommender_version_distances/configs/best_metadata_attributes/" -from api.recommender.similar_services.service_recommendation.recommendation_generation import create_recommendation +RECOMMENDATIONS_STORE_DIR = \ + 'app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/' \ + 'recommender_version_distances/storage/produced_recommendations/' + +HEATMAP_STORAGE_FILE = \ + 'app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/' \ + 'recommender_version_distances/storage/results/heatmap.png' def difference_of_sets(rec_list_1, rec_list_2): @@ -88,11 +102,16 @@ def pairwise_dissimilarities(recommendation_lists, distance_metric): def plot_pairwise_similarities(dissimilarities_matrix, variations_names=None): + mask = np.triu(np.ones_like(dissimilarities_matrix, dtype=bool)) + axes_labels = variations_names if variations_names is not None else False sns.heatmap(dissimilarities_matrix, xticklabels=axes_labels, yticklabels=axes_labels, - vmin=0, vmax=1, annot=True) + vmin=0, vmax=1, annot=True, mask=mask) + + plt.tight_layout() + plt.savefig(HEATMAP_STORAGE_FILE, dpi=900) plt.show() @@ -129,16 +148,9 @@ def find_dissimilar_variations(dir_path, distance_metric): def main(): - gold_services_path = \ - 'api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/gold_services.csv' - config_dir = \ - "api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/configs/distance_configs/best_metadata_attributes/" - recommendations_store_dir = \ - 'api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/recommendations_for_distance/' - - gold_service_ids = pd.read_csv(gold_services_path)['id'] + gold_service_ids = pd.read_csv(GOLD_SERVICES_PATH)['id'] - config_files = glob.glob(config_dir + '*') + config_files = glob.glob(CONFIG_DIR + '*') print("Config files given:") print(config_files) @@ -147,10 +159,10 @@ def main(): update_backend_settings(config_file) structures_update() - store_recommendations(gold_service_ids, recommendations_store_dir + config_file.split('/')[-1][:-5] + '.csv') + store_recommendations(gold_service_ids, RECOMMENDATIONS_STORE_DIR + config_file.split('/')[-1][:-5] + '.csv') # Then we compare their distance - find_dissimilar_variations(recommendations_store_dir, difference_of_sets) + find_dissimilar_variations(RECOMMENDATIONS_STORE_DIR, difference_of_sets) if __name__ == '__main__': diff --git a/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/.gitkeep b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+SD+TU.csv b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+SD+TU.csv new file mode 100644 index 0000000..f46243c --- /dev/null +++ b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+SD+TU.csv @@ -0,0 +1,50 @@ +geant.ocreexoscale,geant.ocreaws,geant.ocreoracle,geant.ocreorangebusiness,geant.ocre_cloud services by vancis,geant.ocreposita +prace.ptc,prace.patc,prace.seasonal_schools_and_international_summer_school,egi-fed.iso_27001_training,egi-fed.fitsm_training,openaire.open_science_training +capsh.dissemin,doabf.prism,vamdc.species_database,elixir-uk.cyverse_uk,ugr-es.glacier_lagoons_of_sierra_nevada,gbif-es.species_portal +seadatanet.european_directory_of_marine_environmental_research_projects,seadatanet.seadatanet_cdi_ogc_wms,seadatanet.seadatanet_cdi,eodc.data_catalogue_service,seadatanet.seadatanet_cdi_sparql,seadatanet.european_directory_of_marine_environmental_data_edmed +geant.mdvpn,geant.l3vpn,blue-cloud.plankton_interact,csic.csic_cloud_infrastructure,asgc.icomcot_tsunami_wave_propagation_simulation_portal,infn.paas_orchestrator +seadatanet.european_directory_of_the_cruise_summary_reports_csr,seadatanet.vocabulary_services_-_underpinned_by_the_nerc_vocabulary_server_nvs,blue-cloud.oceanregimes,seadatanet.seadatanet_cdi,kit.o3as_ozone_assessment,blue-cloud.storm_ssi +egi-fed.cloud_container_compute,collabwith.collabwith_marketplace,egi-fed.high-throughput_compute,arkivum.arkivum_digital_archiving_and_preservation_solution,csic.csic_cloud_infrastructure,openaire.argos +cnb-csic.scipioncloud,wenmr.amber-based_portal_server_for_nmr_structures_amps-nmr,openaire.neuroinformatics_openaire_dashboard,inaf.space-ml_caesar_service,wenmr.fanten_finding_anisotropy_tensor,centerdata.surveycodingsorg +t-systems.open_telekom_cloud,prace.code_vault,capsh.dissemin,cloudferro.data_collections_catalog,cloudferro.data_related_services_-_eo_browser,unitartu.ut.rocket +eurac.edp-portal_-_metadata_catalogue_of_eurac_research,sciences_po.ethnic_and_migrant_minorities_survey_question_data_bank,icos_eric.stilt_worker,icos_eric.data_discovery_and_access_portal,lnec.worsica_-_water_monitoring_sentinel_cloud_platform,ill.visa_-_virtual_infrastructure_for_scientific_analysis +jelastic.platform-as-a-service,seadatanet.european_directory_of_marine_environmental_research_projects,seadatanet.seadatanet_cdi_ogc_wms,seadatanet.seadatanet_cdi,eodc.data_catalogue_service,seadatanet.seadatanet_cdi_ogc_wfs +expertai.document_enrichment_api,expertai.recommender_api,expertai.search_api,ifca-csic.deepaas_training_facility,egi-fed.check-in,cines.etdr +seadatanet.european_directory_of_the_cruise_summary_reports_csr,seadatanet.seadatanet_cdi,jelastic.platform-as-a-service,blue-cloud.storm_ssi,nilu.actris_data_portal,dkrz.enes_climate_analytics_service +egi-fed.check-in,geant.clouds_service_infrastructure_as_a_service,geant.edugain,openaire.openaire_login,infn.indigo_identity_and_access_management,eudat.b2handle +lifewatch-eric.ecoportal,lifewatch-eric.wrims_taxon_match,icos_eric.data_discovery_and_access_portal,lifewatch-eric.oceanographic_buoy_vida,lifewatch-eric.rvlab_vre,bsc-es.bdrc_-_barcelona_dust_regional_center +doabf.prism,capsh.dissemin,ifca-csic.remote_monitoring_and_smart_sensing,embl-ebi.identifiersorg,ugr-es.glacier_lagoons_of_sierra_nevada,gbif-es.images_portal +instruct-eric.aria_access_to_research_infrastructure_management,geant.clouds_service_infrastructure_as_a_service,geant.geantargus,geant.ocreionos,geant.ocreequinix,geant.ocre100 +t-systems.open_telekom_cloud,upv-es.lemonade,cloudferro.data_related_services_-_eo_finder,cloudferro.data_collections_catalog,cloudferro.data_related_services_-_eo_browser,jelastic.platform-as-a-service +prace.ptc,prace.training_portal,egi-fed.iso_27001_training,prace.seasonal_schools_and_international_summer_school,egi-fed.fitsm_training,egi-fed.online_storage +grycap.elastic_cloud_compute_cluster,blue-cloud.rstudio,100percentit.100_percent_it_trusted_cloud,tib.open_research_knowledge_graph_orkg,egi-fed.fitsm_training,openknowledgemaps.open_knowledge_maps +sinergise.sentinel_hub,smartsmear.smartsmear,athena.uw-map,egi-fed.high-throughput_compute,100percentit.100_percent_it_trusted_cloud,cesnet.metacentrum_cloud +lifewatch-eric.gbif_nis_verifier,lifewatch-eric.data-driven_classifier,lifewatch-eric.biotope_griss_extractor,lifewatch-eric.metabarcoding_occurrence_intersector,lifewatch-eric.biotope_gbif_extractor,lifewatch-eric.cimpal_calculator_cumulative_impacts_of_invasive_alien_species_calculator +unibo.opencitations,psnc.symbiote,kit.re3data_-_registry_of_research_data_repositories,sks.digital_production_for_conferences_workshops_roundtables_and_other_academic_and_professional_events,openaire.open_science_observatory,blue-cloud.carbon_notebooks +clarin-eric.virtual_collection_registry,100percentit.100_percent_it_trusted_cloud,openknowledgemaps.open_knowledge_maps,egi-fed.check-in,cines.etdr,cnr-iia.geo_dab +csc-fi.cpouta,desy.pan_notebook,unitartu.ut.rocket,sixsq.nuvla_multi-cloud_application_management_platform,switch.switchengines,desy.pan_gitlab +dcc-uk.dmponline,dariah_eric.ssh_open_marketplace,sciences_po.wpss_for_ess,sciences_po.web_panel_sample_service,sciences_po.ethnic_and_migrant_minority_survey_registry,openedition.operas_research_for_society +gbif-es.spatial_portal,unifl.snap4city,gbif-es.images_portal,prace.prace_massive_open_online_courses_mooc,ugr-es.glacier_lagoons_of_sierra_nevada,ifca-csic.remote_monitoring_and_smart_sensing +switch.switchengines,psi.remote_desktop_service,desy.pan_notebook,exoscale.european_cloud_hosting,sixsq.nuvla_multi-cloud_application_management_platform,unitartu.ut.rocket +cloudferro.infrastructure,unitartu.ut.rocket,switch.switchengines,exoscale.european_cloud_hosting,csc-fi.cpouta,psi.remote_desktop_service +prace.ptc,prace.training_portal,prace.patc,egi-fed.iso_27001_training,openaire.open_science_training,egi-fed.fitsm_training +ibergrid.fair_eva,csic.csic_cloud_infrastructure,ifca-csic.ifca-csic_cloud_infrastructure,blue-cloud.phytoplankton_eovs,blue-cloud.zooplankton_eovs,blue-cloud.plankton_genomics_vlab +expertai.search_api,expertai.recommender_api,d4science.alien_and_invasive_species_vre,iisas.secret_management_service,scipedia.topos_for_individuals,ifremer.argo_marine_floats_metadata_api +embl-ebi.embassy_cloud,100percentit.100_percent_it_trusted_cloud,egi-fed.high-throughput_compute,csc-fi.rahti_container_cloud,athena.rolect,desy.pan_gitlab +geant.l3vpn,geant.lambda,geant.mdvpn,t-systems.open_telekom_cloud,exoscale.european_cloud_hosting,desy.pan_gitlab +desy.pan_gitlab,psi.remote_desktop_service,switch.switchengines,desy.pan_notebook,sixsq.nuvla_multi-cloud_application_management_platform,unitartu.ut.rocket +sobigdata.tagme,blue-cloud.jupyter_hub,iisas.secret_management_service,ifin-hh.cloudifin,cesga.finisterrae,ukaea.prominence +desy.pan_gitlab,unitartu.ut.rocket,exoscale.european_cloud_hosting,csc-fi.cpouta,psi.remote_desktop_service,cloudferro.infrastructure +unimib.latent_space_explorer,lindatclariah-cz.udpipe_tool_for_lemmatization_morphological_analysis_pos_tagging_and_dependency_parsing_in_multiple_languages,athena.uw-map,lindatclariah-cz.ker_-_keyword_extractor,grycap.saps_surface_energy_balance_automated_processing_service,inaf.space-ml_caesar_service +crg.rd-connect_gpap,blue-cloud.discovery_access,blue-cloud.phytoplankton_eovs,blue-cloud.plankton_genomics_vlab,blue-cloud.zooplankton_eovs,infn.paas_orchestrator +icos_eric.data_discovery_and_access_portal,bsc-es.bdrc_-_barcelona_dust_regional_center,blue-cloud.fisheries_atlas,dariah_eric.dariah-campus,ibergrid.fair_eva,sciences_po.ethnic_and_migrant_minorities_survey_question_data_bank +forschungsdaten.forschungsdateninfo,csic.csic_cloud_infrastructure,lifewatch-eric.oceanographic_buoy_vida,lifewatch-eric.rvlab_vre,openaire.topos_observatory_for_organisations,geant.eduroam +iisas.secret_management_service,t-systems.open_telekom_cloud,openaire.open_science_training,geant.ip,desy.pan_gitlab,cloudferro.data_related_services_-_eo_finder +lifewatch-eric.cimpal_calculator_cumulative_impacts_of_invasive_alien_species_calculator,lifewatch-eric.data-driven_classifier,lifewatch-eric.gbif_nis_verifier,lifewatch-eric.occurrences_datacube_builder,lifewatch-eric.pema_sequences_retriever,lifewatch-eric.extractor_resampler_and_masking +openaire.topos_observatory_for_organisations,hostkey.gpu_servers_grant_program,osmooc.open_science_mooc,egi-fed.online_storage,egi-fed.notebook,egi-fed.data_transfer +elixir-italy.laniakea_recas,gbif-es.collections_registry,uni-freiburg.european_galaxy_server,embl-ebi.embassy_cloud,tib.open_research_knowledge_graph_orkg,bsc-es.openebench +seadatanet.webodv_-_online_extraction_analysis_and_visualization_of_seadatanet_and_argo_data,lifewatch-eric.extractor_resampler_and_masking,lifewatch-eric.pema_runner,lifewatch-eric.biotope_gbif_extractor,lifewatch-eric.occurrences_datacube_analyst,lifewatch-eric.data-driven_classifier +d4science.rprototypinglab_virtual_research_environment,geant.eduteams,forschungsdaten.forschungsdateninfo,athena.lct,openaire.topos_observatory_for_organisations,unige.astronomical_online_data_analysis_astrooda +t-systems.open_telekom_cloud,cloudferro.data_collections_catalog,cloudferro.data_related_services_-_eo_browser,upv-es.lemonade,seadatanet.seadatanet_cdi_ogc_wfs,seadatanet.seadatanet_cdi_sparql +openaire.argos,egi-fed.cloud_container_compute,egi-fed.cloud_compute,northern_data_cloud_services.northern_data_cloud_services,dcc-uk.dmponline,cs_group.ai4geo_engine +enhancer.swiss_escience_grid_certificates,scipedia.topos_for_individuals,geant.wifimon,infn.indigo_identity_and_access_management,gesis.doi_registration_service,egi-fed.check-in diff --git a/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+SD+TU_2.csv b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+SD+TU_2.csv new file mode 100644 index 0000000..f46243c --- /dev/null +++ b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+SD+TU_2.csv @@ -0,0 +1,50 @@ +geant.ocreexoscale,geant.ocreaws,geant.ocreoracle,geant.ocreorangebusiness,geant.ocre_cloud services by vancis,geant.ocreposita +prace.ptc,prace.patc,prace.seasonal_schools_and_international_summer_school,egi-fed.iso_27001_training,egi-fed.fitsm_training,openaire.open_science_training +capsh.dissemin,doabf.prism,vamdc.species_database,elixir-uk.cyverse_uk,ugr-es.glacier_lagoons_of_sierra_nevada,gbif-es.species_portal +seadatanet.european_directory_of_marine_environmental_research_projects,seadatanet.seadatanet_cdi_ogc_wms,seadatanet.seadatanet_cdi,eodc.data_catalogue_service,seadatanet.seadatanet_cdi_sparql,seadatanet.european_directory_of_marine_environmental_data_edmed +geant.mdvpn,geant.l3vpn,blue-cloud.plankton_interact,csic.csic_cloud_infrastructure,asgc.icomcot_tsunami_wave_propagation_simulation_portal,infn.paas_orchestrator +seadatanet.european_directory_of_the_cruise_summary_reports_csr,seadatanet.vocabulary_services_-_underpinned_by_the_nerc_vocabulary_server_nvs,blue-cloud.oceanregimes,seadatanet.seadatanet_cdi,kit.o3as_ozone_assessment,blue-cloud.storm_ssi +egi-fed.cloud_container_compute,collabwith.collabwith_marketplace,egi-fed.high-throughput_compute,arkivum.arkivum_digital_archiving_and_preservation_solution,csic.csic_cloud_infrastructure,openaire.argos +cnb-csic.scipioncloud,wenmr.amber-based_portal_server_for_nmr_structures_amps-nmr,openaire.neuroinformatics_openaire_dashboard,inaf.space-ml_caesar_service,wenmr.fanten_finding_anisotropy_tensor,centerdata.surveycodingsorg +t-systems.open_telekom_cloud,prace.code_vault,capsh.dissemin,cloudferro.data_collections_catalog,cloudferro.data_related_services_-_eo_browser,unitartu.ut.rocket +eurac.edp-portal_-_metadata_catalogue_of_eurac_research,sciences_po.ethnic_and_migrant_minorities_survey_question_data_bank,icos_eric.stilt_worker,icos_eric.data_discovery_and_access_portal,lnec.worsica_-_water_monitoring_sentinel_cloud_platform,ill.visa_-_virtual_infrastructure_for_scientific_analysis +jelastic.platform-as-a-service,seadatanet.european_directory_of_marine_environmental_research_projects,seadatanet.seadatanet_cdi_ogc_wms,seadatanet.seadatanet_cdi,eodc.data_catalogue_service,seadatanet.seadatanet_cdi_ogc_wfs +expertai.document_enrichment_api,expertai.recommender_api,expertai.search_api,ifca-csic.deepaas_training_facility,egi-fed.check-in,cines.etdr +seadatanet.european_directory_of_the_cruise_summary_reports_csr,seadatanet.seadatanet_cdi,jelastic.platform-as-a-service,blue-cloud.storm_ssi,nilu.actris_data_portal,dkrz.enes_climate_analytics_service +egi-fed.check-in,geant.clouds_service_infrastructure_as_a_service,geant.edugain,openaire.openaire_login,infn.indigo_identity_and_access_management,eudat.b2handle +lifewatch-eric.ecoportal,lifewatch-eric.wrims_taxon_match,icos_eric.data_discovery_and_access_portal,lifewatch-eric.oceanographic_buoy_vida,lifewatch-eric.rvlab_vre,bsc-es.bdrc_-_barcelona_dust_regional_center +doabf.prism,capsh.dissemin,ifca-csic.remote_monitoring_and_smart_sensing,embl-ebi.identifiersorg,ugr-es.glacier_lagoons_of_sierra_nevada,gbif-es.images_portal +instruct-eric.aria_access_to_research_infrastructure_management,geant.clouds_service_infrastructure_as_a_service,geant.geantargus,geant.ocreionos,geant.ocreequinix,geant.ocre100 +t-systems.open_telekom_cloud,upv-es.lemonade,cloudferro.data_related_services_-_eo_finder,cloudferro.data_collections_catalog,cloudferro.data_related_services_-_eo_browser,jelastic.platform-as-a-service +prace.ptc,prace.training_portal,egi-fed.iso_27001_training,prace.seasonal_schools_and_international_summer_school,egi-fed.fitsm_training,egi-fed.online_storage +grycap.elastic_cloud_compute_cluster,blue-cloud.rstudio,100percentit.100_percent_it_trusted_cloud,tib.open_research_knowledge_graph_orkg,egi-fed.fitsm_training,openknowledgemaps.open_knowledge_maps +sinergise.sentinel_hub,smartsmear.smartsmear,athena.uw-map,egi-fed.high-throughput_compute,100percentit.100_percent_it_trusted_cloud,cesnet.metacentrum_cloud +lifewatch-eric.gbif_nis_verifier,lifewatch-eric.data-driven_classifier,lifewatch-eric.biotope_griss_extractor,lifewatch-eric.metabarcoding_occurrence_intersector,lifewatch-eric.biotope_gbif_extractor,lifewatch-eric.cimpal_calculator_cumulative_impacts_of_invasive_alien_species_calculator +unibo.opencitations,psnc.symbiote,kit.re3data_-_registry_of_research_data_repositories,sks.digital_production_for_conferences_workshops_roundtables_and_other_academic_and_professional_events,openaire.open_science_observatory,blue-cloud.carbon_notebooks +clarin-eric.virtual_collection_registry,100percentit.100_percent_it_trusted_cloud,openknowledgemaps.open_knowledge_maps,egi-fed.check-in,cines.etdr,cnr-iia.geo_dab +csc-fi.cpouta,desy.pan_notebook,unitartu.ut.rocket,sixsq.nuvla_multi-cloud_application_management_platform,switch.switchengines,desy.pan_gitlab +dcc-uk.dmponline,dariah_eric.ssh_open_marketplace,sciences_po.wpss_for_ess,sciences_po.web_panel_sample_service,sciences_po.ethnic_and_migrant_minority_survey_registry,openedition.operas_research_for_society +gbif-es.spatial_portal,unifl.snap4city,gbif-es.images_portal,prace.prace_massive_open_online_courses_mooc,ugr-es.glacier_lagoons_of_sierra_nevada,ifca-csic.remote_monitoring_and_smart_sensing +switch.switchengines,psi.remote_desktop_service,desy.pan_notebook,exoscale.european_cloud_hosting,sixsq.nuvla_multi-cloud_application_management_platform,unitartu.ut.rocket +cloudferro.infrastructure,unitartu.ut.rocket,switch.switchengines,exoscale.european_cloud_hosting,csc-fi.cpouta,psi.remote_desktop_service +prace.ptc,prace.training_portal,prace.patc,egi-fed.iso_27001_training,openaire.open_science_training,egi-fed.fitsm_training +ibergrid.fair_eva,csic.csic_cloud_infrastructure,ifca-csic.ifca-csic_cloud_infrastructure,blue-cloud.phytoplankton_eovs,blue-cloud.zooplankton_eovs,blue-cloud.plankton_genomics_vlab +expertai.search_api,expertai.recommender_api,d4science.alien_and_invasive_species_vre,iisas.secret_management_service,scipedia.topos_for_individuals,ifremer.argo_marine_floats_metadata_api +embl-ebi.embassy_cloud,100percentit.100_percent_it_trusted_cloud,egi-fed.high-throughput_compute,csc-fi.rahti_container_cloud,athena.rolect,desy.pan_gitlab +geant.l3vpn,geant.lambda,geant.mdvpn,t-systems.open_telekom_cloud,exoscale.european_cloud_hosting,desy.pan_gitlab +desy.pan_gitlab,psi.remote_desktop_service,switch.switchengines,desy.pan_notebook,sixsq.nuvla_multi-cloud_application_management_platform,unitartu.ut.rocket +sobigdata.tagme,blue-cloud.jupyter_hub,iisas.secret_management_service,ifin-hh.cloudifin,cesga.finisterrae,ukaea.prominence +desy.pan_gitlab,unitartu.ut.rocket,exoscale.european_cloud_hosting,csc-fi.cpouta,psi.remote_desktop_service,cloudferro.infrastructure +unimib.latent_space_explorer,lindatclariah-cz.udpipe_tool_for_lemmatization_morphological_analysis_pos_tagging_and_dependency_parsing_in_multiple_languages,athena.uw-map,lindatclariah-cz.ker_-_keyword_extractor,grycap.saps_surface_energy_balance_automated_processing_service,inaf.space-ml_caesar_service +crg.rd-connect_gpap,blue-cloud.discovery_access,blue-cloud.phytoplankton_eovs,blue-cloud.plankton_genomics_vlab,blue-cloud.zooplankton_eovs,infn.paas_orchestrator +icos_eric.data_discovery_and_access_portal,bsc-es.bdrc_-_barcelona_dust_regional_center,blue-cloud.fisheries_atlas,dariah_eric.dariah-campus,ibergrid.fair_eva,sciences_po.ethnic_and_migrant_minorities_survey_question_data_bank +forschungsdaten.forschungsdateninfo,csic.csic_cloud_infrastructure,lifewatch-eric.oceanographic_buoy_vida,lifewatch-eric.rvlab_vre,openaire.topos_observatory_for_organisations,geant.eduroam +iisas.secret_management_service,t-systems.open_telekom_cloud,openaire.open_science_training,geant.ip,desy.pan_gitlab,cloudferro.data_related_services_-_eo_finder +lifewatch-eric.cimpal_calculator_cumulative_impacts_of_invasive_alien_species_calculator,lifewatch-eric.data-driven_classifier,lifewatch-eric.gbif_nis_verifier,lifewatch-eric.occurrences_datacube_builder,lifewatch-eric.pema_sequences_retriever,lifewatch-eric.extractor_resampler_and_masking +openaire.topos_observatory_for_organisations,hostkey.gpu_servers_grant_program,osmooc.open_science_mooc,egi-fed.online_storage,egi-fed.notebook,egi-fed.data_transfer +elixir-italy.laniakea_recas,gbif-es.collections_registry,uni-freiburg.european_galaxy_server,embl-ebi.embassy_cloud,tib.open_research_knowledge_graph_orkg,bsc-es.openebench +seadatanet.webodv_-_online_extraction_analysis_and_visualization_of_seadatanet_and_argo_data,lifewatch-eric.extractor_resampler_and_masking,lifewatch-eric.pema_runner,lifewatch-eric.biotope_gbif_extractor,lifewatch-eric.occurrences_datacube_analyst,lifewatch-eric.data-driven_classifier +d4science.rprototypinglab_virtual_research_environment,geant.eduteams,forschungsdaten.forschungsdateninfo,athena.lct,openaire.topos_observatory_for_organisations,unige.astronomical_online_data_analysis_astrooda +t-systems.open_telekom_cloud,cloudferro.data_collections_catalog,cloudferro.data_related_services_-_eo_browser,upv-es.lemonade,seadatanet.seadatanet_cdi_ogc_wfs,seadatanet.seadatanet_cdi_sparql +openaire.argos,egi-fed.cloud_container_compute,egi-fed.cloud_compute,northern_data_cloud_services.northern_data_cloud_services,dcc-uk.dmponline,cs_group.ai4geo_engine +enhancer.swiss_escience_grid_certificates,scipedia.topos_for_individuals,geant.wifimon,infn.indigo_identity_and_access_management,gesis.doi_registration_service,egi-fed.check-in diff --git a/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+SD.csv b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+SD.csv new file mode 100644 index 0000000..aacd3ed --- /dev/null +++ b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+SD.csv @@ -0,0 +1,50 @@ +egi-fed.egi_datahub,blue-cloud.discovery_access,infn.infn-cloud_object_storage_dice,readcoop.transkribus,blue-cloud.carbon_notebooks,dsmz.bacdive__the_bacterial_diversity_metadatabase +prace.seasonal_schools_and_international_summer_school,prace.ptc,egi-fed.iso_27001_training,prace.patc,egi-fed.fitsm_training,geant.transits_training +capsh.dissemin,doabf.prism,icos_eric.open_sparl_endpoint,gbif-es.collections_registry,gbif-es.images_portal,vamdc.species_database +seadatanet.seadatanet_cdi_sparql,seadatanet.seadatanet_cdi_ogc_wfs,seadatanet.european_directory_of_marine_environmental_data_edmed,seadatanet.european_directory_of_marine_environmental_research_projects,eodc.data_catalogue_service,seadatanet.seadatanet_cdi_ogc_wms +geant.ip,geant.mdvpn,geant.l3vpn,csi_piemonte.nivola2,seadatanet.european_directory_of_the_cruise_summary_reports_csr,northern_data_cloud_services.northern_data_cloud_services +seadatanet.european_directory_of_the_cruise_summary_reports_csr,seadatanet.vocabulary_services_-_underpinned_by_the_nerc_vocabulary_server_nvs,cines.etdr,emso_eric.eosc_future_environment_dashboard,icos_eric.stilt_viewer,seadatanet.doi_minting_service +geant.ocreequinix,geant.ocre_cloud services by t-systems,geant.ocregoogle,geant.ocrecsipiemonte,geant.ocre100,geant.ocrecomtrade +ibergrid.fair_eva,openknowledgemaps.open_knowledge_maps,earthwatch.mics_measuring_the_impact_of_citizen_science,desy.desy_visa,ibergrid.sqaaas,cs_group.ai4geo_engine +enhancer.openrdmeu,openbiomaps.openbiomaps,seadatanet.european_directory_of_the_cruise_summary_reports_csr,nilu.actris_data_portal,ifca-csic.plant_classification,ifremer.argo_floats_observations__discover_and_use_in_situ_data_from_the_global_network_of_ocean_profiling_floats +cessda-eric.elsst__european_language_social_science_thesaurus,prace.prace_massive_open_online_courses_mooc,lnec.worsica_-_water_monitoring_sentinel_cloud_platform,cesnet.metacentrum_cloud,cscs.object_storage,kit.o3as_ozone_assessment +seadatanet.seadatanet_cdi_sparql,seadatanet.seadatanet_cdi_ogc_wfs,seadatanet.european_directory_of_marine_environmental_research_projects,eodc.data_catalogue_service,seadatanet.seadatanet_cdi_ogc_wms,jelastic.platform-as-a-service +plantnet.plntnet_identification_service,expertai.search_api,sobigdata.tagme,expertai.recommender_api,ifca-csic.deepaas_training_facility,expertai.document_enrichment_api +emso_eric.eosc_future_environment_dashboard,datacite.datacite_doi_registration_service,seadatanet.european_directory_of_the_cruise_summary_reports_csr,digitalglobe.earthwatch,lifewatch-eric.gbif_nis_verifier,eiscat.eiscat_data_access_portal +openaire.openaire_login,geant.edugain,geant.inacademia,egi-fed.check-in,geant.eduteams,authenix.authenix +icos_eric.stilt_viewer,icos_eric.open_sparql_endpoint_gui,seadatanet.doi_minting_service,athena.atmo-flud,lifewatch-eric.ecoportal,lifewatch-eric.trophic_positions_modeler +icos_eric.open_sparl_endpoint,capsh.dissemin,doabf.prism,infn.dynamic_on_demand_analysis_service,astron.,gbif-es.collections_registry +upf.multilingual_corpus_of_survey_questionnaires,ess_eric.european_social_survey_ess_as_a_service,obp.thoth,clarin-eric.language_resource_switchboard,fzj-inm7.datalad,lago.onedatasim +t-systems.open_telekom_cloud,fairdi.nomad_repository,upv-es.lemonade,openaire.graph,dcc-uk.dmponline,cloudferro.data_related_services_-_eo_finder +egi-fed.iso_27001_training,prace.seasonal_schools_and_international_summer_school,prace.ptc,prace.training_portal,egi-fed.fitsm_training,northern_data_cloud_services.northern_data_cloud_services +grycap.elastic_cloud_compute_cluster,blue-cloud.rstudio,embl-ebi.identifiersorg,dkrz.enes_climate_analytics_service,tib.open_research_knowledge_graph_orkg,csc-fi.chipster +icos_eric.data_discovery_and_access_portal,smartsmear.smartsmear,blue-cloud.fisheries_atlas,cmcc.enes_data_space,emso_eric.eosc_future_environment_dashboard,openaire.broker +emso_eric.eosc_future_environment_dashboard,seadatanet.european_directory_of_the_cruise_summary_reports_csr,icos_eric.open_sparql_endpoint_gui,seadatanet.doi_minting_service,lifewatch-eric.metabarcoding_occurrence_intersector,lifewatch-eric.oceanographic_buoy_vida +grycap.infrastructure_manager,sztaki.deep_learning_by_horovod,europeana.europeana_apis,icos_eric.open_sparql_endpoint_gui,psnc.symbiote,libnova.libnova_labdrive_the_ultimate_research_data_management_and_digital_preservation_platform +clarin-eric.virtual_collection_registry,obp.thoth,unibi-ub.bielefeld_academic_search_engine_base,kit.re3data_-_registry_of_research_data_repositories,dariah_eric.ssh_open_marketplace,openedition.operas_research_for_society +egi-fed.high-throughput_compute,unitartu.ut.rocket,sixsq.nuvla_multi-cloud_application_management_platform,csc-fi.cpouta,exoscale.european_cloud_hosting,csc-fi.csc_epouta +openedition.operas_research_for_society,dariah_eric.ssh_open_marketplace,operas.gotriple_discovery_platform,gesis.doi_registration_service,cessda-eric.elsst__european_language_social_science_thesaurus,sciences_po.web_panel_sample_service +osmooc.open_science_mooc,embl-ebi.identifiersorg_resolution_services,uni-freiburg.european_galaxy_server,embl-ebi.embassy_cloud,dsmz.bacdive__the_bacterial_diversity_metadatabase,gbif-es.spatial_portal +egi-fed.high-throughput_compute,cloudferro.infrastructure,sixsq.nuvla_multi-cloud_application_management_platform,csc-fi.cpouta,exoscale.european_cloud_hosting,csc-fi.csc_epouta +egi-fed.high-throughput_compute,unitartu.ut.rocket,csc-fi.cpouta,sixsq.nuvla_multi-cloud_application_management_platform,exoscale.european_cloud_hosting,csc-fi.csc_epouta +prace.ptc,prace.training_portal,egi-fed.iso_27001_training,prace.patc,geant.transits_training,egi-fed.fitsm_training +egi-fed.training_infrastructure,psnc.learneosc-synergy,openaire.digital_humanities_and_cultural_heritage_openaire_community_gateway,d4science.visual_media_service_vre,obp.thoth,clarin-eric.language_resource_switchboard +expertai.search_api,expertai.recommender_api,d4science.alien_and_invasive_species_vre,lindatclariah-cz.udpipe_tool_for_lemmatization_morphological_analysis_pos_tagging_and_dependency_parsing_in_multiple_languages,unimib.latent_space_explorer,athena.uw-map +egi-fed.high-throughput_compute,cloudferro.infrastructure,sixsq.nuvla_multi-cloud_application_management_platform,csc-fi.cpouta,exoscale.european_cloud_hosting,desy.pan_gitlab +geant.lambda,geant.mdvpn,geant.l3vpn,csi_piemonte.nivola2,seadatanet.european_directory_of_the_cruise_summary_reports_csr,nilu.actris_data_portal +egi-fed.high-throughput_compute,cloudferro.infrastructure,sixsq.nuvla_multi-cloud_application_management_platform,csc-fi.cpouta,csc-fi.csc_epouta,desy.pan_gitlab +sobigdata.tagme,cesga.finisterrae,ukaea.prominence,northern_data_cloud_services.northern_data_cloud_services,unimib.latent_space_explorer,blue-cloud.jupyter_hub +cesnet.metacentrum_cloud,egi-fed.high-throughput_compute,unitartu.ut.rocket,psi.remote_desktop_service,csc-fi.cpouta,sixsq.nuvla_multi-cloud_application_management_platform +unimib.latent_space_explorer,d4science.alien_and_invasive_species_vre,cnb-csic.scipioncloud,lindatclariah-cz.udpipe_tool_for_lemmatization_morphological_analysis_pos_tagging_and_dependency_parsing_in_multiple_languages,athena.uw-map,d4science.rprototypinglab_virtual_research_environment +openaire.open_science_helpdesk,csi_piemonte.nivola2,seadatanet.european_directory_of_the_cruise_summary_reports_csr,nilu.actris_data_portal,ifremer.argo_floats_observations__discover_and_use_in_situ_data_from_the_global_network_of_ocean_profiling_floats,rolos.machine_intelligence_platfrom_for_research +centerdata.surveycodingsorg,upf.multilingual_corpus_of_survey_questionnaires,fssda.kuha2_metadata_server,clarin-eric.language_resource_switchboard,bsc-es.bdrc_-_barcelona_dust_regional_center,obp.thoth +prace.prace_massive_open_online_courses_mooc,unifl.snap4city,gbif-es.e-Learning_platform,ubora.ubora_e-platform,uni-freiburg.european_galaxy_server,gcc_umcg.molgenis +iisas.secret_management_service,seadatanet.european_directory_of_the_cruise_summary_reports_csr,desy.pan_gitlab,ifremer.argo_floats_observations__discover_and_use_in_situ_data_from_the_global_network_of_ocean_profiling_floats,rolos.machine_intelligence_platfrom_for_research,geant.ocre_cloud services by cloud and heat +inaf.space-ml_caesar_service,esrf.jupyter_notebook_is_an_open_source_web_application_that_can_used_to_create_and_share_documents_that_contain_live_code_equations_visualizations_and_text,cs_group.ai4geo_engine,desy.desy_visa,openknowledgemaps.open_knowledge_maps,unimib.latent_space_explorer +seadatanet.european_directory_of_the_cruise_summary_reports_csr,nilu.actris_data_portal,ifca-csic.plant_classification,ifremer.argo_floats_observations__discover_and_use_in_situ_data_from_the_global_network_of_ocean_profiling_floats,rolos.machine_intelligence_platfrom_for_research,geant.ocre_cloud services by cloud and heat +csc-fi.chipster,tib.open_research_knowledge_graph_orkg,gbif-es.e-Learning_platform,bsc-es.openebench,cnio.pandrugs,ubora.ubora_e-platform +diamond_light_source.diamond_remote_desktop,cs_group.ai4geo_engine,openknowledgemaps.open_knowledge_maps,desy.desy_visa,ukri_-_stfc.idaaas,uni-freiburg.european_galaxy_server +d4science.rprototypinglab_virtual_research_environment,d4science.alien_and_invasive_species_vre,icos_eric.icos_jupyter_hub,lindatclariah-cz.udpipe_tool_for_lemmatization_morphological_analysis_pos_tagging_and_dependency_parsing_in_multiple_languages,unimib.latent_space_explorer,athena.uw-map +t-systems.open_telekom_cloud,dcc-uk.dmponline,cloudferro.data_collections_catalog,openaire.graph,cloudferro.data_related_services_-_eo_browser,upv-es.lemonade +seadatanet.european_directory_of_the_cruise_summary_reports_csr,desy.pan_gitlab,ifremer.argo_floats_observations__discover_and_use_in_situ_data_from_the_global_network_of_ocean_profiling_floats,rolos.machine_intelligence_platfrom_for_research,geant.ocre_cloud services by cloud and heat,geant.ocre_cloud services by scc +upf.multilingual_corpus_of_survey_questionnaires,obp.thoth,creaf.nimmbus_geospatial_user_feedback,inoe_2000.infra-art_spectral_library,blue-cloud.plankton_eov_vlab,csi_piemonte.nivola2 diff --git a/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+SD_2.csv b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+SD_2.csv new file mode 100644 index 0000000..aacd3ed --- /dev/null +++ b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+SD_2.csv @@ -0,0 +1,50 @@ +egi-fed.egi_datahub,blue-cloud.discovery_access,infn.infn-cloud_object_storage_dice,readcoop.transkribus,blue-cloud.carbon_notebooks,dsmz.bacdive__the_bacterial_diversity_metadatabase +prace.seasonal_schools_and_international_summer_school,prace.ptc,egi-fed.iso_27001_training,prace.patc,egi-fed.fitsm_training,geant.transits_training +capsh.dissemin,doabf.prism,icos_eric.open_sparl_endpoint,gbif-es.collections_registry,gbif-es.images_portal,vamdc.species_database +seadatanet.seadatanet_cdi_sparql,seadatanet.seadatanet_cdi_ogc_wfs,seadatanet.european_directory_of_marine_environmental_data_edmed,seadatanet.european_directory_of_marine_environmental_research_projects,eodc.data_catalogue_service,seadatanet.seadatanet_cdi_ogc_wms +geant.ip,geant.mdvpn,geant.l3vpn,csi_piemonte.nivola2,seadatanet.european_directory_of_the_cruise_summary_reports_csr,northern_data_cloud_services.northern_data_cloud_services +seadatanet.european_directory_of_the_cruise_summary_reports_csr,seadatanet.vocabulary_services_-_underpinned_by_the_nerc_vocabulary_server_nvs,cines.etdr,emso_eric.eosc_future_environment_dashboard,icos_eric.stilt_viewer,seadatanet.doi_minting_service +geant.ocreequinix,geant.ocre_cloud services by t-systems,geant.ocregoogle,geant.ocrecsipiemonte,geant.ocre100,geant.ocrecomtrade +ibergrid.fair_eva,openknowledgemaps.open_knowledge_maps,earthwatch.mics_measuring_the_impact_of_citizen_science,desy.desy_visa,ibergrid.sqaaas,cs_group.ai4geo_engine +enhancer.openrdmeu,openbiomaps.openbiomaps,seadatanet.european_directory_of_the_cruise_summary_reports_csr,nilu.actris_data_portal,ifca-csic.plant_classification,ifremer.argo_floats_observations__discover_and_use_in_situ_data_from_the_global_network_of_ocean_profiling_floats +cessda-eric.elsst__european_language_social_science_thesaurus,prace.prace_massive_open_online_courses_mooc,lnec.worsica_-_water_monitoring_sentinel_cloud_platform,cesnet.metacentrum_cloud,cscs.object_storage,kit.o3as_ozone_assessment +seadatanet.seadatanet_cdi_sparql,seadatanet.seadatanet_cdi_ogc_wfs,seadatanet.european_directory_of_marine_environmental_research_projects,eodc.data_catalogue_service,seadatanet.seadatanet_cdi_ogc_wms,jelastic.platform-as-a-service +plantnet.plntnet_identification_service,expertai.search_api,sobigdata.tagme,expertai.recommender_api,ifca-csic.deepaas_training_facility,expertai.document_enrichment_api +emso_eric.eosc_future_environment_dashboard,datacite.datacite_doi_registration_service,seadatanet.european_directory_of_the_cruise_summary_reports_csr,digitalglobe.earthwatch,lifewatch-eric.gbif_nis_verifier,eiscat.eiscat_data_access_portal +openaire.openaire_login,geant.edugain,geant.inacademia,egi-fed.check-in,geant.eduteams,authenix.authenix +icos_eric.stilt_viewer,icos_eric.open_sparql_endpoint_gui,seadatanet.doi_minting_service,athena.atmo-flud,lifewatch-eric.ecoportal,lifewatch-eric.trophic_positions_modeler +icos_eric.open_sparl_endpoint,capsh.dissemin,doabf.prism,infn.dynamic_on_demand_analysis_service,astron.,gbif-es.collections_registry +upf.multilingual_corpus_of_survey_questionnaires,ess_eric.european_social_survey_ess_as_a_service,obp.thoth,clarin-eric.language_resource_switchboard,fzj-inm7.datalad,lago.onedatasim +t-systems.open_telekom_cloud,fairdi.nomad_repository,upv-es.lemonade,openaire.graph,dcc-uk.dmponline,cloudferro.data_related_services_-_eo_finder +egi-fed.iso_27001_training,prace.seasonal_schools_and_international_summer_school,prace.ptc,prace.training_portal,egi-fed.fitsm_training,northern_data_cloud_services.northern_data_cloud_services +grycap.elastic_cloud_compute_cluster,blue-cloud.rstudio,embl-ebi.identifiersorg,dkrz.enes_climate_analytics_service,tib.open_research_knowledge_graph_orkg,csc-fi.chipster +icos_eric.data_discovery_and_access_portal,smartsmear.smartsmear,blue-cloud.fisheries_atlas,cmcc.enes_data_space,emso_eric.eosc_future_environment_dashboard,openaire.broker +emso_eric.eosc_future_environment_dashboard,seadatanet.european_directory_of_the_cruise_summary_reports_csr,icos_eric.open_sparql_endpoint_gui,seadatanet.doi_minting_service,lifewatch-eric.metabarcoding_occurrence_intersector,lifewatch-eric.oceanographic_buoy_vida +grycap.infrastructure_manager,sztaki.deep_learning_by_horovod,europeana.europeana_apis,icos_eric.open_sparql_endpoint_gui,psnc.symbiote,libnova.libnova_labdrive_the_ultimate_research_data_management_and_digital_preservation_platform +clarin-eric.virtual_collection_registry,obp.thoth,unibi-ub.bielefeld_academic_search_engine_base,kit.re3data_-_registry_of_research_data_repositories,dariah_eric.ssh_open_marketplace,openedition.operas_research_for_society +egi-fed.high-throughput_compute,unitartu.ut.rocket,sixsq.nuvla_multi-cloud_application_management_platform,csc-fi.cpouta,exoscale.european_cloud_hosting,csc-fi.csc_epouta +openedition.operas_research_for_society,dariah_eric.ssh_open_marketplace,operas.gotriple_discovery_platform,gesis.doi_registration_service,cessda-eric.elsst__european_language_social_science_thesaurus,sciences_po.web_panel_sample_service +osmooc.open_science_mooc,embl-ebi.identifiersorg_resolution_services,uni-freiburg.european_galaxy_server,embl-ebi.embassy_cloud,dsmz.bacdive__the_bacterial_diversity_metadatabase,gbif-es.spatial_portal +egi-fed.high-throughput_compute,cloudferro.infrastructure,sixsq.nuvla_multi-cloud_application_management_platform,csc-fi.cpouta,exoscale.european_cloud_hosting,csc-fi.csc_epouta +egi-fed.high-throughput_compute,unitartu.ut.rocket,csc-fi.cpouta,sixsq.nuvla_multi-cloud_application_management_platform,exoscale.european_cloud_hosting,csc-fi.csc_epouta +prace.ptc,prace.training_portal,egi-fed.iso_27001_training,prace.patc,geant.transits_training,egi-fed.fitsm_training +egi-fed.training_infrastructure,psnc.learneosc-synergy,openaire.digital_humanities_and_cultural_heritage_openaire_community_gateway,d4science.visual_media_service_vre,obp.thoth,clarin-eric.language_resource_switchboard +expertai.search_api,expertai.recommender_api,d4science.alien_and_invasive_species_vre,lindatclariah-cz.udpipe_tool_for_lemmatization_morphological_analysis_pos_tagging_and_dependency_parsing_in_multiple_languages,unimib.latent_space_explorer,athena.uw-map +egi-fed.high-throughput_compute,cloudferro.infrastructure,sixsq.nuvla_multi-cloud_application_management_platform,csc-fi.cpouta,exoscale.european_cloud_hosting,desy.pan_gitlab +geant.lambda,geant.mdvpn,geant.l3vpn,csi_piemonte.nivola2,seadatanet.european_directory_of_the_cruise_summary_reports_csr,nilu.actris_data_portal +egi-fed.high-throughput_compute,cloudferro.infrastructure,sixsq.nuvla_multi-cloud_application_management_platform,csc-fi.cpouta,csc-fi.csc_epouta,desy.pan_gitlab +sobigdata.tagme,cesga.finisterrae,ukaea.prominence,northern_data_cloud_services.northern_data_cloud_services,unimib.latent_space_explorer,blue-cloud.jupyter_hub +cesnet.metacentrum_cloud,egi-fed.high-throughput_compute,unitartu.ut.rocket,psi.remote_desktop_service,csc-fi.cpouta,sixsq.nuvla_multi-cloud_application_management_platform +unimib.latent_space_explorer,d4science.alien_and_invasive_species_vre,cnb-csic.scipioncloud,lindatclariah-cz.udpipe_tool_for_lemmatization_morphological_analysis_pos_tagging_and_dependency_parsing_in_multiple_languages,athena.uw-map,d4science.rprototypinglab_virtual_research_environment +openaire.open_science_helpdesk,csi_piemonte.nivola2,seadatanet.european_directory_of_the_cruise_summary_reports_csr,nilu.actris_data_portal,ifremer.argo_floats_observations__discover_and_use_in_situ_data_from_the_global_network_of_ocean_profiling_floats,rolos.machine_intelligence_platfrom_for_research +centerdata.surveycodingsorg,upf.multilingual_corpus_of_survey_questionnaires,fssda.kuha2_metadata_server,clarin-eric.language_resource_switchboard,bsc-es.bdrc_-_barcelona_dust_regional_center,obp.thoth +prace.prace_massive_open_online_courses_mooc,unifl.snap4city,gbif-es.e-Learning_platform,ubora.ubora_e-platform,uni-freiburg.european_galaxy_server,gcc_umcg.molgenis +iisas.secret_management_service,seadatanet.european_directory_of_the_cruise_summary_reports_csr,desy.pan_gitlab,ifremer.argo_floats_observations__discover_and_use_in_situ_data_from_the_global_network_of_ocean_profiling_floats,rolos.machine_intelligence_platfrom_for_research,geant.ocre_cloud services by cloud and heat +inaf.space-ml_caesar_service,esrf.jupyter_notebook_is_an_open_source_web_application_that_can_used_to_create_and_share_documents_that_contain_live_code_equations_visualizations_and_text,cs_group.ai4geo_engine,desy.desy_visa,openknowledgemaps.open_knowledge_maps,unimib.latent_space_explorer +seadatanet.european_directory_of_the_cruise_summary_reports_csr,nilu.actris_data_portal,ifca-csic.plant_classification,ifremer.argo_floats_observations__discover_and_use_in_situ_data_from_the_global_network_of_ocean_profiling_floats,rolos.machine_intelligence_platfrom_for_research,geant.ocre_cloud services by cloud and heat +csc-fi.chipster,tib.open_research_knowledge_graph_orkg,gbif-es.e-Learning_platform,bsc-es.openebench,cnio.pandrugs,ubora.ubora_e-platform +diamond_light_source.diamond_remote_desktop,cs_group.ai4geo_engine,openknowledgemaps.open_knowledge_maps,desy.desy_visa,ukri_-_stfc.idaaas,uni-freiburg.european_galaxy_server +d4science.rprototypinglab_virtual_research_environment,d4science.alien_and_invasive_species_vre,icos_eric.icos_jupyter_hub,lindatclariah-cz.udpipe_tool_for_lemmatization_morphological_analysis_pos_tagging_and_dependency_parsing_in_multiple_languages,unimib.latent_space_explorer,athena.uw-map +t-systems.open_telekom_cloud,dcc-uk.dmponline,cloudferro.data_collections_catalog,openaire.graph,cloudferro.data_related_services_-_eo_browser,upv-es.lemonade +seadatanet.european_directory_of_the_cruise_summary_reports_csr,desy.pan_gitlab,ifremer.argo_floats_observations__discover_and_use_in_situ_data_from_the_global_network_of_ocean_profiling_floats,rolos.machine_intelligence_platfrom_for_research,geant.ocre_cloud services by cloud and heat,geant.ocre_cloud services by scc +upf.multilingual_corpus_of_survey_questionnaires,obp.thoth,creaf.nimmbus_geospatial_user_feedback,inoe_2000.infra-art_spectral_library,blue-cloud.plankton_eov_vlab,csi_piemonte.nivola2 diff --git a/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+TU.csv b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+TU.csv new file mode 100644 index 0000000..45a0031 --- /dev/null +++ b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+TU.csv @@ -0,0 +1,50 @@ +geant.ocreaws,geant.ocreoracle,geant.ocreorangebusiness,geant.ocreexoscale,bineo.cos4bio,geant.ocrecsipiemonte +prace.ptc,prace.patc,prace.seasonal_schools_and_international_summer_school,egi-fed.iso_27001_training,egi-fed.fitsm_training,openaire.open_science_training +ifca-csic.remote_monitoring_and_smart_sensing,ugr-es.glacier_lagoons_of_sierra_nevada,vamdc.query_store,openedition.operas_research_for_society,gbif-es.species_portal,vamdc.species_database +t-systems.open_telekom_cloud,seadatanet.seadatanet_cdi_ogc_wfs,seadatanet.european_directory_of_marine_environmental_data_edmed,europeana.europeana_apis,seadatanet.seadatanet_cdi_ogc_wms,cloudferro.data_related_services_-_eo_finder +geant.mdvpn,geant.l3vpn,blue-cloud.plankton_interact,asgc.icomcot_tsunami_wave_propagation_simulation_portal,blue-cloud.plankton_genomics_vlab,csic.csic_cloud_infrastructure +seadatanet.european_directory_of_the_cruise_summary_reports_csr,seadatanet.vocabulary_services_-_underpinned_by_the_nerc_vocabulary_server_nvs,cines.etdr,blue-cloud.oceanregimes,d4science.visual_media_service_vre,cesnet.umsa_-_untargeted_mass_spectrometry_data_analysis +egi-fed.cloud_container_compute,collabwith.collabwith_marketplace,egi-fed.high-throughput_compute,arkivum.arkivum_digital_archiving_and_preservation_solution,csic.csic_cloud_infrastructure,ubiwhere.atmo-4cast +cnb-csic.scipioncloud,wenmr.amber-based_portal_server_for_nmr_structures_amps-nmr,centerdata.surveycodingsorg,openaire.neuroinformatics_openaire_dashboard,wenmr.pdb-tools_web,wenmr.haddock24_web_portal +seadatanet.european_directory_of_the_cruise_summary_reports_csr,openaire.open_science_training,nilu.actris_data_portal,desy.pan_gitlab,ifca-csic.remote_monitoring_and_smart_sensing,dkrz.enes_climate_analytics_service +eurac.edp-portal_-_metadata_catalogue_of_eurac_research,sciences_po.ethnic_and_migrant_minorities_survey_question_data_bank,icos_eric.stilt_worker,icos_eric.data_discovery_and_access_portal,lnec.worsica_-_water_monitoring_sentinel_cloud_platform,sciences_po.ethnic_and_migrant_minority_survey_registry +t-systems.open_telekom_cloud,upv-es.lemonade,seadatanet.seadatanet_cdi_ogc_wms,europeana.europeana_apis,seadatanet.european_directory_of_marine_environmental_research_projects,seadatanet.seadatanet_cdi_ogc_wfs +expertai.recommender_api,expertai.search_api,expertai.document_enrichment_api,d4science.rprototypinglab_virtual_research_environment,gbif-es.collections_registry,icos_eric.open_sparql_endpoint_gui +seadatanet.european_directory_of_the_cruise_summary_reports_csr,openaire.open_science_training,nilu.actris_data_portal,desy.pan_gitlab,ifca-csic.remote_monitoring_and_smart_sensing,dkrz.enes_climate_analytics_service +egi-fed.check-in,geant.clouds_service_infrastructure_as_a_service,geant.edugain,openaire.openaire_login,gbif-es.collections_registry,infn.indigo_identity_and_access_management +egi-fed.notebook,osmooc.open_science_mooc,meeo.adam_space,fssda.data_service_portal_aila,lifewatch-eric.ecoportal,forschungsdaten.forschungsdateninfo +embl-ebi.identifiersorg,openedition.operas_research_for_society,doabf.prism,capsh.dissemin,ifca-csic.remote_monitoring_and_smart_sensing,ugr-es.glacier_lagoons_of_sierra_nevada +geant.clouds_service_infrastructure_as_a_service,instruct-eric.aria_access_to_research_infrastructure_management,geant.geantargus,geant.ocreionos,geant.ocresafespring,geant.ocre_cloud services by x-ion +t-systems.open_telekom_cloud,jelastic.platform-as-a-service,seadatanet.european_directory_of_marine_environmental_data_edmed,seadatanet.seadatanet_cdi_ogc_wms,seadatanet.european_directory_of_marine_environmental_research_projects,cloudferro.data_related_services_-_eo_finder +prace.ptc,prace.training_portal,egi-fed.iso_27001_training,prace.seasonal_schools_and_international_summer_school,egi-fed.fitsm_training,egi-fed.data_transfer +eodc.jupyterhub_for_global_copernicus_data,blue-cloud.oceanregimes,digitalglobe.earthwatch,d4science.visual_media_service_vre,csc-fi.chipster,ehri.ehri_entity_matching_tool +clarin-eric.language_resource_switchboard,clarin-eric.virtual_collection_registry,cesnet.metacentrum_cloud,psnc.symbiote,100percentit.100_percent_it_trusted_cloud,egi-fed.high-throughput_compute +lindatclariah-cz.elixirfm,lindatclariah-cz.morphodita,lindatclariah-cz.nametag,lindatclariah-cz.machine_translation,coronis_computing_sl.uw-mos,coronis_computing_sl.vd-maps +unibo.opencitations,sciences_po.ethnic_and_migrant_minority_survey_registry,kit.re3data_-_registry_of_research_data_repositories,sks.digital_production_for_conferences_workshops_roundtables_and_other_academic_and_professional_events,emso_eric.eosc_future_environment_dashboard,athena.verbal_aggression_analyser_va_analyser +clarin-eric.virtual_collection_registry,100percentit.100_percent_it_trusted_cloud,cines.etdr,egi-fed.check-in,cnr-iia.geo_dab,openknowledgemaps.open_knowledge_maps +embl-ebi.embassy_cloud,cyberbotics.robotbenchmark,unitartu.ut.rocket,desy.pan_notebook,psi.aiiDA_lab,ifca-csic.plant_classification +dcc-uk.dmponline,blue-cloud.plankton_genomics_vlab,uit.dataverseno,csic.csic_cloud_infrastructure,sciences_po.ethnic_and_migrant_minorities_survey_question_data_bank,osmooc.open_science_mooc +prace.prace_massive_open_online_courses_mooc,unifl.snap4city,ubora.ubora_e-platform,osmooc.open_science_mooc,seadatanet.european_directory_of_the_cruise_summary_reports_csr,cesnet.umsa_-_untargeted_mass_spectrometry_data_analysis +grycap.infrastructure_manager,cloudferro.infrastructure,ifca-csic.plant_classification,exoscale.european_cloud_hosting,elixir-uk.cyverse_uk,csc-fi.cpouta +grycap.infrastructure_manager,cloudferro.infrastructure,desy.pan_gitlab,exoscale.european_cloud_hosting,elixir-uk.cyverse_uk,csc-fi.cpouta +prace.ptc,prace.training_portal,prace.patc,egi-fed.iso_27001_training,openaire.open_science_training,egi-fed.fitsm_training +ibergrid.fair_eva,csic.csic_cloud_infrastructure,blue-cloud.phytoplankton_eovs,blue-cloud.plankton_genomics_vlab,blue-cloud.zooplankton_eovs,uit.dataverseno +expertai.recommender_api,expertai.search_api,d4science.alien_and_invasive_species_vre,ifremer.argo_marine_floats_metadata_api,ifremer.argo_marine_floats_vocabularies_from_nvs_vocabulary_server,ifremer.argo_marine_floats_opensearch_api +embl-ebi.embassy_cloud,egi-fed.high-throughput_compute,100percentit.100_percent_it_trusted_cloud,csc-fi.rahti_container_cloud,cesnet.metacentrum_cloud,athena.rolect +geant.l3vpn,geant.mdvpn,geant.lambda,cesnet.umsa_-_untargeted_mass_spectrometry_data_analysis,psi.panet_ontology_service,infn.dynamic_on_demand_analysis_service +grycap.infrastructure_manager,cloudferro.infrastructure,desy.pan_gitlab,ifca-csic.plant_classification,elixir-uk.cyverse_uk,csc-fi.cpouta +blue-cloud.jupyter_hub,sobigdata.tagme,ifin-hh.cloudifin,blue-cloud.analytics_engine,iisas.secret_management_service,uni-freiburg.european_galaxy_server +elixir-uk.cyverse_uk,cloudferro.infrastructure,desy.pan_gitlab,ifca-csic.plant_classification,exoscale.european_cloud_hosting,grycap.infrastructure_manager +lindatclariah-cz.udpipe_tool_for_lemmatization_morphological_analysis_pos_tagging_and_dependency_parsing_in_multiple_languages,unimib.latent_space_explorer,athena.uw-map,lindatclariah-cz.ker_-_keyword_extractor,inaf.space-ml_caesar_service,coronis_computing_sl.vd-maps +crg.rd-connect_gpap,blue-cloud.phytoplankton_eovs,blue-cloud.zooplankton_eovs,blue-cloud.discovery_access,blue-cloud.plankton_genomics_vlab,dariah_eric.dariah-campus +icos_eric.data_discovery_and_access_portal,bsc-es.bdrc_-_barcelona_dust_regional_center,uit.dataverseno,blue-cloud.fisheries_atlas,sciences_po.ethnic_and_migrant_minorities_survey_question_data_bank,dariah_eric.dariah-campus +forschungsdaten.forschungsdateninfo,lifewatch-eric.rvlab_vre,lifewatch-eric.oceanographic_buoy_vida,uit.dataverseno,csic.csic_cloud_infrastructure,meeo.adam_platform +iisas.secret_management_service,seadatanet.european_directory_of_the_cruise_summary_reports_csr,blue-cloud.oceanregimes,openaire.open_science_training,d4science.visual_media_service_vre,cesnet.umsa_-_untargeted_mass_spectrometry_data_analysis +lifewatch-eric.environmental_data_extractor,lifewatch-eric.biotope_griss_extractor,lifewatch-eric.trophic_positions_modeler,lifewatch-eric.data-driven_classifier,lifewatch-eric.extractor_resampler_and_masking,lifewatch-eric.metabarcoding_occurrence_intersector +openaire.topos_observatory_for_organisations,hostkey.gpu_servers_grant_program,egi-fed.data_transfer,ill.panosc_software_catalogue,cessda-eric.data_management_expert_guide_dmeg,egi-fed.online_storage +gesis.doi_registration_service,elixir-italy.laniakea_recas,tib.open_research_knowledge_graph_orkg,gbif-es.collections_registry,fairdi.nomad_repository,eiscat.eiscat_data_access_portal +seadatanet.webodv_-_online_extraction_analysis_and_visualization_of_seadatanet_and_argo_data,lifewatch-eric.gbif_nis_verifier,lifewatch-eric.environmental_data_extractor,astron.,lifewatch-eric.trophic_positions_modeler,lifewatch-eric.biotope_gbif_extractor +d4science.rprototypinglab_virtual_research_environment,geant.eduteams,unige.astronomical_online_data_analysis_astrooda,openaire.neuroinformatics_openaire_dashboard,openaire.european_marine_science_openaire_dashboard,forschungsdaten.forschungsdateninfo +t-systems.open_telekom_cloud,upv-es.lemonade,cloudferro.data_collections_catalog,seadatanet.seadatanet_cdi_sparql,cloudferro.data_related_services_-_eo_browser,europeana.europeana_apis +openaire.argos,egi-fed.cloud_compute,egi-fed.cloud_container_compute,northern_data_cloud_services.northern_data_cloud_services,ubiwhere.atmo-4cast,sciences_po.ethnic_and_migrant_minorities_survey_question_data_bank +enhancer.swiss_escience_grid_certificates,scipedia.topos_for_individuals,ehri.international_research_portal_for_records_related_to_nazi-era_cultural_property,gesis.doi_registration_service,infn.indigo_identity_and_access_management,icos_eric.open_sparql_endpoint_gui diff --git a/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+TU_2.csv b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+TU_2.csv new file mode 100644 index 0000000..45a0031 --- /dev/null +++ b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/produced_recommendations/version_C+TU_2.csv @@ -0,0 +1,50 @@ +geant.ocreaws,geant.ocreoracle,geant.ocreorangebusiness,geant.ocreexoscale,bineo.cos4bio,geant.ocrecsipiemonte +prace.ptc,prace.patc,prace.seasonal_schools_and_international_summer_school,egi-fed.iso_27001_training,egi-fed.fitsm_training,openaire.open_science_training +ifca-csic.remote_monitoring_and_smart_sensing,ugr-es.glacier_lagoons_of_sierra_nevada,vamdc.query_store,openedition.operas_research_for_society,gbif-es.species_portal,vamdc.species_database +t-systems.open_telekom_cloud,seadatanet.seadatanet_cdi_ogc_wfs,seadatanet.european_directory_of_marine_environmental_data_edmed,europeana.europeana_apis,seadatanet.seadatanet_cdi_ogc_wms,cloudferro.data_related_services_-_eo_finder +geant.mdvpn,geant.l3vpn,blue-cloud.plankton_interact,asgc.icomcot_tsunami_wave_propagation_simulation_portal,blue-cloud.plankton_genomics_vlab,csic.csic_cloud_infrastructure +seadatanet.european_directory_of_the_cruise_summary_reports_csr,seadatanet.vocabulary_services_-_underpinned_by_the_nerc_vocabulary_server_nvs,cines.etdr,blue-cloud.oceanregimes,d4science.visual_media_service_vre,cesnet.umsa_-_untargeted_mass_spectrometry_data_analysis +egi-fed.cloud_container_compute,collabwith.collabwith_marketplace,egi-fed.high-throughput_compute,arkivum.arkivum_digital_archiving_and_preservation_solution,csic.csic_cloud_infrastructure,ubiwhere.atmo-4cast +cnb-csic.scipioncloud,wenmr.amber-based_portal_server_for_nmr_structures_amps-nmr,centerdata.surveycodingsorg,openaire.neuroinformatics_openaire_dashboard,wenmr.pdb-tools_web,wenmr.haddock24_web_portal +seadatanet.european_directory_of_the_cruise_summary_reports_csr,openaire.open_science_training,nilu.actris_data_portal,desy.pan_gitlab,ifca-csic.remote_monitoring_and_smart_sensing,dkrz.enes_climate_analytics_service +eurac.edp-portal_-_metadata_catalogue_of_eurac_research,sciences_po.ethnic_and_migrant_minorities_survey_question_data_bank,icos_eric.stilt_worker,icos_eric.data_discovery_and_access_portal,lnec.worsica_-_water_monitoring_sentinel_cloud_platform,sciences_po.ethnic_and_migrant_minority_survey_registry +t-systems.open_telekom_cloud,upv-es.lemonade,seadatanet.seadatanet_cdi_ogc_wms,europeana.europeana_apis,seadatanet.european_directory_of_marine_environmental_research_projects,seadatanet.seadatanet_cdi_ogc_wfs +expertai.recommender_api,expertai.search_api,expertai.document_enrichment_api,d4science.rprototypinglab_virtual_research_environment,gbif-es.collections_registry,icos_eric.open_sparql_endpoint_gui +seadatanet.european_directory_of_the_cruise_summary_reports_csr,openaire.open_science_training,nilu.actris_data_portal,desy.pan_gitlab,ifca-csic.remote_monitoring_and_smart_sensing,dkrz.enes_climate_analytics_service +egi-fed.check-in,geant.clouds_service_infrastructure_as_a_service,geant.edugain,openaire.openaire_login,gbif-es.collections_registry,infn.indigo_identity_and_access_management +egi-fed.notebook,osmooc.open_science_mooc,meeo.adam_space,fssda.data_service_portal_aila,lifewatch-eric.ecoportal,forschungsdaten.forschungsdateninfo +embl-ebi.identifiersorg,openedition.operas_research_for_society,doabf.prism,capsh.dissemin,ifca-csic.remote_monitoring_and_smart_sensing,ugr-es.glacier_lagoons_of_sierra_nevada +geant.clouds_service_infrastructure_as_a_service,instruct-eric.aria_access_to_research_infrastructure_management,geant.geantargus,geant.ocreionos,geant.ocresafespring,geant.ocre_cloud services by x-ion +t-systems.open_telekom_cloud,jelastic.platform-as-a-service,seadatanet.european_directory_of_marine_environmental_data_edmed,seadatanet.seadatanet_cdi_ogc_wms,seadatanet.european_directory_of_marine_environmental_research_projects,cloudferro.data_related_services_-_eo_finder +prace.ptc,prace.training_portal,egi-fed.iso_27001_training,prace.seasonal_schools_and_international_summer_school,egi-fed.fitsm_training,egi-fed.data_transfer +eodc.jupyterhub_for_global_copernicus_data,blue-cloud.oceanregimes,digitalglobe.earthwatch,d4science.visual_media_service_vre,csc-fi.chipster,ehri.ehri_entity_matching_tool +clarin-eric.language_resource_switchboard,clarin-eric.virtual_collection_registry,cesnet.metacentrum_cloud,psnc.symbiote,100percentit.100_percent_it_trusted_cloud,egi-fed.high-throughput_compute +lindatclariah-cz.elixirfm,lindatclariah-cz.morphodita,lindatclariah-cz.nametag,lindatclariah-cz.machine_translation,coronis_computing_sl.uw-mos,coronis_computing_sl.vd-maps +unibo.opencitations,sciences_po.ethnic_and_migrant_minority_survey_registry,kit.re3data_-_registry_of_research_data_repositories,sks.digital_production_for_conferences_workshops_roundtables_and_other_academic_and_professional_events,emso_eric.eosc_future_environment_dashboard,athena.verbal_aggression_analyser_va_analyser +clarin-eric.virtual_collection_registry,100percentit.100_percent_it_trusted_cloud,cines.etdr,egi-fed.check-in,cnr-iia.geo_dab,openknowledgemaps.open_knowledge_maps +embl-ebi.embassy_cloud,cyberbotics.robotbenchmark,unitartu.ut.rocket,desy.pan_notebook,psi.aiiDA_lab,ifca-csic.plant_classification +dcc-uk.dmponline,blue-cloud.plankton_genomics_vlab,uit.dataverseno,csic.csic_cloud_infrastructure,sciences_po.ethnic_and_migrant_minorities_survey_question_data_bank,osmooc.open_science_mooc +prace.prace_massive_open_online_courses_mooc,unifl.snap4city,ubora.ubora_e-platform,osmooc.open_science_mooc,seadatanet.european_directory_of_the_cruise_summary_reports_csr,cesnet.umsa_-_untargeted_mass_spectrometry_data_analysis +grycap.infrastructure_manager,cloudferro.infrastructure,ifca-csic.plant_classification,exoscale.european_cloud_hosting,elixir-uk.cyverse_uk,csc-fi.cpouta +grycap.infrastructure_manager,cloudferro.infrastructure,desy.pan_gitlab,exoscale.european_cloud_hosting,elixir-uk.cyverse_uk,csc-fi.cpouta +prace.ptc,prace.training_portal,prace.patc,egi-fed.iso_27001_training,openaire.open_science_training,egi-fed.fitsm_training +ibergrid.fair_eva,csic.csic_cloud_infrastructure,blue-cloud.phytoplankton_eovs,blue-cloud.plankton_genomics_vlab,blue-cloud.zooplankton_eovs,uit.dataverseno +expertai.recommender_api,expertai.search_api,d4science.alien_and_invasive_species_vre,ifremer.argo_marine_floats_metadata_api,ifremer.argo_marine_floats_vocabularies_from_nvs_vocabulary_server,ifremer.argo_marine_floats_opensearch_api +embl-ebi.embassy_cloud,egi-fed.high-throughput_compute,100percentit.100_percent_it_trusted_cloud,csc-fi.rahti_container_cloud,cesnet.metacentrum_cloud,athena.rolect +geant.l3vpn,geant.mdvpn,geant.lambda,cesnet.umsa_-_untargeted_mass_spectrometry_data_analysis,psi.panet_ontology_service,infn.dynamic_on_demand_analysis_service +grycap.infrastructure_manager,cloudferro.infrastructure,desy.pan_gitlab,ifca-csic.plant_classification,elixir-uk.cyverse_uk,csc-fi.cpouta +blue-cloud.jupyter_hub,sobigdata.tagme,ifin-hh.cloudifin,blue-cloud.analytics_engine,iisas.secret_management_service,uni-freiburg.european_galaxy_server +elixir-uk.cyverse_uk,cloudferro.infrastructure,desy.pan_gitlab,ifca-csic.plant_classification,exoscale.european_cloud_hosting,grycap.infrastructure_manager +lindatclariah-cz.udpipe_tool_for_lemmatization_morphological_analysis_pos_tagging_and_dependency_parsing_in_multiple_languages,unimib.latent_space_explorer,athena.uw-map,lindatclariah-cz.ker_-_keyword_extractor,inaf.space-ml_caesar_service,coronis_computing_sl.vd-maps +crg.rd-connect_gpap,blue-cloud.phytoplankton_eovs,blue-cloud.zooplankton_eovs,blue-cloud.discovery_access,blue-cloud.plankton_genomics_vlab,dariah_eric.dariah-campus +icos_eric.data_discovery_and_access_portal,bsc-es.bdrc_-_barcelona_dust_regional_center,uit.dataverseno,blue-cloud.fisheries_atlas,sciences_po.ethnic_and_migrant_minorities_survey_question_data_bank,dariah_eric.dariah-campus +forschungsdaten.forschungsdateninfo,lifewatch-eric.rvlab_vre,lifewatch-eric.oceanographic_buoy_vida,uit.dataverseno,csic.csic_cloud_infrastructure,meeo.adam_platform +iisas.secret_management_service,seadatanet.european_directory_of_the_cruise_summary_reports_csr,blue-cloud.oceanregimes,openaire.open_science_training,d4science.visual_media_service_vre,cesnet.umsa_-_untargeted_mass_spectrometry_data_analysis +lifewatch-eric.environmental_data_extractor,lifewatch-eric.biotope_griss_extractor,lifewatch-eric.trophic_positions_modeler,lifewatch-eric.data-driven_classifier,lifewatch-eric.extractor_resampler_and_masking,lifewatch-eric.metabarcoding_occurrence_intersector +openaire.topos_observatory_for_organisations,hostkey.gpu_servers_grant_program,egi-fed.data_transfer,ill.panosc_software_catalogue,cessda-eric.data_management_expert_guide_dmeg,egi-fed.online_storage +gesis.doi_registration_service,elixir-italy.laniakea_recas,tib.open_research_knowledge_graph_orkg,gbif-es.collections_registry,fairdi.nomad_repository,eiscat.eiscat_data_access_portal +seadatanet.webodv_-_online_extraction_analysis_and_visualization_of_seadatanet_and_argo_data,lifewatch-eric.gbif_nis_verifier,lifewatch-eric.environmental_data_extractor,astron.,lifewatch-eric.trophic_positions_modeler,lifewatch-eric.biotope_gbif_extractor +d4science.rprototypinglab_virtual_research_environment,geant.eduteams,unige.astronomical_online_data_analysis_astrooda,openaire.neuroinformatics_openaire_dashboard,openaire.european_marine_science_openaire_dashboard,forschungsdaten.forschungsdateninfo +t-systems.open_telekom_cloud,upv-es.lemonade,cloudferro.data_collections_catalog,seadatanet.seadatanet_cdi_sparql,cloudferro.data_related_services_-_eo_browser,europeana.europeana_apis +openaire.argos,egi-fed.cloud_compute,egi-fed.cloud_container_compute,northern_data_cloud_services.northern_data_cloud_services,ubiwhere.atmo-4cast,sciences_po.ethnic_and_migrant_minorities_survey_question_data_bank +enhancer.swiss_escience_grid_certificates,scipedia.topos_for_individuals,ehri.international_research_portal_for_records_related_to_nazi-era_cultural_property,gesis.doi_registration_service,infn.indigo_identity_and_access_management,icos_eric.open_sparql_endpoint_gui diff --git a/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/results/.gitkeep b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/results/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/results/heatmap.png b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/results/heatmap.png new file mode 100644 index 0000000..c2423a8 Binary files /dev/null and b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/recommender_version_distances/storage/results/heatmap.png differ diff --git a/api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/gold_services.csv b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/gold_services.csv similarity index 100% rename from api/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/gold_services.csv rename to app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/gold_services.csv diff --git a/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/manual_evaluation_final_files/.gitkeep b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/manual_evaluation_final_files/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/metrics/.gitkeep b/app/recommender/similar_services/service_recommendation/evaluation/manual_evaluation/storage/metrics/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/app/recommender/similar_services/service_recommendation/recommendation_generation.py b/app/recommender/similar_services/service_recommendation/recommendation_generation.py new file mode 100644 index 0000000..8800a84 --- /dev/null +++ b/app/recommender/similar_services/service_recommendation/recommendation_generation.py @@ -0,0 +1,90 @@ +import logging + +from app.databases.content_based_rec_db import ContentBasedRecsMongoDB +from app.databases.registry.registry_selector import get_registry +from app.exceptions import IdNotExists +from app.recommender.similar_services.service_recommendation.components.filtering import \ + filtering +from app.recommender.similar_services.service_recommendation.components.ordering import \ + ordering +from app.recommender.similar_services.service_recommendation.components.reranking import \ + re_ranking +from app.recommender.similar_services.service_recommendation.components.resources_similarity import \ + resources_similarity +from app.settings import APP_SETTINGS + +logger = logging.getLogger(__name__) + + +class User: + def __init__(self, user_id): + # TODO: Currently if a user_id does not exist we consider the user as anonymous + # TODO: Required from front since a new user will not exist in the RS Mongo yet + self.user_id = user_id if self.is_valid_user(user_id) else None + self.registered_user = True if self.user_id is not None else False + + @staticmethod + def is_valid_user(user_id): + """ + Return True if the user_id exists or is None (anonymous), otherwise return False + """ + if user_id is not None and not get_registry().is_valid_user(user_id): + return False + + return True + + def get_purchases(self): + logger.debug("Get user purchases...") + if self.registered_user: + return list(get_registry().get_user_services(self.user_id)) + else: + return [] + + +def service_exists(db, viewed_service_id): + """ + Checks if the given service id exists + """ + if not db.is_valid_service(viewed_service_id): + raise IdNotExists("Service id does not exist.") + + +def create_recommendation(viewed_resource_id, recommendations_num=5, user_id=None, + viewed_weight=None, metadata_weight=None): + viewed_weight = APP_SETTINGS["BACKEND"]["SIMILAR_SERVICES"]["VIEWED_WEIGHT"] \ + if viewed_weight is None else viewed_weight + metadata_weight = APP_SETTINGS["BACKEND"]["SIMILAR_SERVICES"]["METADATA_WEIGHT"] \ + if metadata_weight is None else metadata_weight + + db = get_registry() + + service_exists(db, viewed_resource_id) + + user = User(user_id) + purchases = user.get_purchases() + + candidates = resources_similarity(viewed_resource_id, + purchased_resources=purchases, + view_weight=viewed_weight, + metadata_weight=metadata_weight) + + candidates = filtering(db, candidates, viewed_resource_id, purchases) + + candidates = ordering(candidates) + + candidates = re_ranking(target_service=viewed_resource_id, + purchases=purchases, + candidates=candidates, + recommendations_num=recommendations_num, + viewed_weight=viewed_weight, + metadata_weight=metadata_weight + ) + + recommendation = [{"service_id": service_id, "score": score} for service_id, score in + candidates[:recommendations_num].items()] + + content_based_recs_db = ContentBasedRecsMongoDB() + content_based_recs_db.save_recommendation(recommendation=recommendation, service_id=viewed_resource_id, + user_id=user_id, history_service_ids=purchases) + + return recommendation diff --git a/app/recommender/similar_services/service_recommendation/update.py b/app/recommender/similar_services/service_recommendation/update.py new file mode 100644 index 0000000..37dd765 --- /dev/null +++ b/app/recommender/similar_services/service_recommendation/update.py @@ -0,0 +1,39 @@ +from app.recommender.similar_services.preprocessor.embeddings import ( + metadata_embeddings, text_embeddings) +from app.recommender.similar_services.preprocessor.reports import \ + monitoring_reports +from app.recommender.similar_services.preprocessor.similarities import ( + metadata_similarities, text_similarities) +from app.recommender.update.update import Update + + +class ServicesRecommendationUpdate(Update): + def initialize(self): + metadata_embeddings.initialize_metadata_embeddings() + metadata_similarities.initialize_metadata_similarities() + text_embeddings.initialize_text_embeddings() + text_similarities.initialize_text_similarities() + monitoring_reports.initialise_ar_report() + monitoring_reports.initialise_status_report() + + def update(self): + metadata_embeddings.create_metadata_embeddings() + metadata_similarities.create_metadata_similarities() + text_embeddings.create_text_embeddings() + text_similarities.create_text_similarities() + monitoring_reports.update_status_report() + monitoring_reports.update_ar_report() + + def update_for_new_service(self, service_id: int): + metadata_embeddings.update_metadata_embedding(new_service_id=service_id) + metadata_similarities.update_metadata_similarities(new_service_id=service_id) + text_embeddings.update_text_embedding(new_service_id=service_id) + text_similarities.update_text_similarities(new_service_id=service_id) + + def revert(self): + metadata_embeddings.delete_metadata_embeddings() + metadata_similarities.delete_metadata_similarities() + text_embeddings.delete_text_embeddings() + text_similarities.delete_text_similarities() + monitoring_reports.delete_status_report() + monitoring_reports.delete_ar_report() diff --git a/app/recommender/update/__init__.py b/app/recommender/update/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/recommender/update/update.py b/app/recommender/update/update.py new file mode 100644 index 0000000..bc4e901 --- /dev/null +++ b/app/recommender/update/update.py @@ -0,0 +1,53 @@ +from abc import ABC, abstractmethod + +from app.exceptions import NoneProjects, NoneServices + + +class Update(ABC): + @abstractmethod + def initialize(self): + pass + + @abstractmethod + def update(self): + pass + + @abstractmethod + def update_for_new_service(self, service_id: int): + pass + + @abstractmethod + def revert(self): + pass + + +class AggregatedUpdate: + def __init__(self, updaters: list[Update]): + """ + + Args: + updaters: List of objects that perform use case updating + """ + self.updaters = updaters + + def initialize(self): + try: + for updater in self.updaters: + updater.initialize() + except (NoneServices, NoneProjects) as e: + for updater in self.updaters: + updater.revert() + raise e + + def update(self): + try: + for updater in self.updaters: + updater.update() + except (NoneServices, NoneProjects) as e: + for updater in self.updaters: + updater.revert() + raise e + + def update_for_new_service(self, service_id: int): + for updater in self.updaters: + updater.update_for_new_service(service_id) diff --git a/app/recommender/update/updater_selector.py b/app/recommender/update/updater_selector.py new file mode 100644 index 0000000..7d287ed --- /dev/null +++ b/app/recommender/update/updater_selector.py @@ -0,0 +1,27 @@ +from app.exceptions import ModeDoesNotExist +from app.recommender.project_completion.update import ProjectCompletionUpdate +from app.recommender.similar_services.field_suggestion.update import \ + FieldSuggestionUpdate +from app.recommender.similar_services.project_assistant.update import \ + ProjectAssistantUpdate +from app.recommender.similar_services.service_recommendation.update import \ + ServicesRecommendationUpdate +from app.recommender.update.update import AggregatedUpdate +from app.settings import APP_SETTINGS + + +def get_updater(): + if APP_SETTINGS["BACKEND"]["MODE"] == "PROVIDERS-RECOMMENDER": + return AggregatedUpdate([FieldSuggestionUpdate()]) + elif APP_SETTINGS["BACKEND"]["MODE"] == "PORTAL-RECOMMENDER": + return AggregatedUpdate([ + ServicesRecommendationUpdate(), + ProjectAssistantUpdate(), + ProjectCompletionUpdate() + ]) + elif APP_SETTINGS["BACKEND"]["MODE"] == "SIMILAR_SERVICES_EVALUATION": + return AggregatedUpdate([ + ServicesRecommendationUpdate() + ]) + else: + raise ModeDoesNotExist(f"Mode {APP_SETTINGS['BACKEND']['MODE']} is not recognised.") diff --git a/app/routes/__init__.py b/app/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/routes/add_routes.py b/app/routes/add_routes.py similarity index 54% rename from api/routes/add_routes.py rename to app/routes/add_routes.py index c337d13..aa6afea 100644 --- a/api/routes/add_routes.py +++ b/app/routes/add_routes.py @@ -1,16 +1,18 @@ -from api.routes import (auto_completion, health, project_assistant, +from app.routes import (auto_completion, health, project_assistant, project_completion, similar_services, update) -from api.settings import APP_SETTINGS +from app.settings import APP_SETTINGS def initialize_routes(app): - if APP_SETTINGS['BACKEND']['MODE'] == 'RS': - app.include_router(health.router) # Only implemented health checks for the RS + + app.include_router(health.router) + + if APP_SETTINGS['BACKEND']['MODE'] == 'PORTAL-RECOMMENDER': app.include_router(similar_services.router) app.include_router(project_assistant.router) app.include_router(project_completion.router) - if APP_SETTINGS['BACKEND']['MODE'] == 'AUTO-COMPLETION': + if APP_SETTINGS['BACKEND']['MODE'] == 'PROVIDERS-RECOMMENDER': app.include_router(auto_completion.router) app.include_router(update.router) diff --git a/api/routes/auto_completion.py b/app/routes/auto_completion.py similarity index 67% rename from api/routes/auto_completion.py rename to app/routes/auto_completion.py index e4a6cd3..d588db8 100644 --- a/api/routes/auto_completion.py +++ b/app/routes/auto_completion.py @@ -1,10 +1,10 @@ import logging -from typing import List +from typing import Dict, List, Optional -from api.exceptions import MissingAttribute, MissingStructure -from api.recommender.similar_services.field_suggestion.evaluation.evaluation import \ +from app.exceptions import MissingAttribute, MissingStructure +from app.recommender.similar_services.field_suggestion.evaluation.evaluation import \ evaluation -from api.recommender.similar_services.field_suggestion.suggestion_generation import \ +from app.recommender.similar_services.field_suggestion.suggestion_generation import \ get_auto_complete_suggestions from fastapi import APIRouter, HTTPException from pydantic import BaseModel @@ -15,7 +15,6 @@ class Request(BaseModel): - new_service: dict # Fields to suggest options @@ -24,15 +23,23 @@ class Request(BaseModel): # The maximum suggestions per field maximum_suggestions: int + existing_fields_values: Optional[Dict[str, List[str]]] = None + class Config: schema_extra = { "example": { "new_service": { "name": "Name of the service...", + "tagline": "Tagline of the new service...", "description": "Description of the added service..." }, - "fields_to_suggest": ["categories", "target_users", "scientific_domains"], - "maximum_suggestions": 3 + "fields_to_suggest": ["categories", "scientific_domains"], + "maximum_suggestions": 3, + "existing_fields_values": { + "categories": ["subcategory-access_physical_and_eInfrastructures-compute-job_execution", + "subcategory-access_physical_and_eInfrastructures-compute-other"], + "scientific_domains": ["scientific_subdomain-agricultural_sciences-other_agricultural_sciences"] + } } } @@ -57,13 +64,15 @@ def auto_completion_suggestions(request: Request): - **new_service**: the filled fields of the new partial created service - **fields_to_suggest**: the fields for which suggestion will be generated - **maximum_suggestions**: the maximum number of suggestions per field + - **existing_fields_values**: the existing values for each suggested field **Returns** a list with the name and the suggestions for every requested field """ try: return [FieldSuggestions(field_name=field, suggestions=suggestions) for field, suggestions in get_auto_complete_suggestions(request.new_service, request.fields_to_suggest, - request.maximum_suggestions).items()] + request.maximum_suggestions, + request.existing_fields_values).items()] except (MissingStructure, MissingAttribute) as e: logger.error((str(e))) raise HTTPException(status_code=404, detail=str(e)) diff --git a/api/routes/health.py b/app/routes/health.py similarity index 82% rename from api/routes/health.py rename to app/routes/health.py index f691c9b..25445ad 100644 --- a/api/routes/health.py +++ b/app/routes/health.py @@ -1,6 +1,6 @@ import logging -from api.health.monitor_health import service_health_test +from app.health.monitor_health import service_health_test from fastapi import APIRouter logger = logging.getLogger(__name__) diff --git a/api/routes/project_assistant.py b/app/routes/project_assistant.py similarity index 94% rename from api/routes/project_assistant.py rename to app/routes/project_assistant.py index bd5b645..c1875dd 100644 --- a/api/routes/project_assistant.py +++ b/app/routes/project_assistant.py @@ -1,9 +1,9 @@ import logging from typing import List -from api.recommender.similar_services.project_assistant.recommendation_generation import \ +from app.recommender.similar_services.project_assistant.recommendation_generation import \ project_assistant_recommendation -from api.settings import APP_SETTINGS +from app.settings import APP_SETTINGS from fastapi import APIRouter from pydantic import BaseModel, validator diff --git a/api/routes/project_completion.py b/app/routes/project_completion.py similarity index 96% rename from api/routes/project_completion.py rename to app/routes/project_completion.py index ce9f252..cda6462 100644 --- a/api/routes/project_completion.py +++ b/app/routes/project_completion.py @@ -1,9 +1,9 @@ import logging from typing import List -from api.recommender.project_completion.recommendation_generation import \ +from app.recommender.project_completion.recommendation_generation import \ create_recommendation as project_completion_recommendation -from api.settings import APP_SETTINGS +from app.settings import APP_SETTINGS from fastapi import APIRouter, HTTPException from pydantic import BaseModel, validator diff --git a/api/routes/similar_services.py b/app/routes/similar_services.py similarity index 94% rename from api/routes/similar_services.py rename to app/routes/similar_services.py index 15271db..a8824b4 100644 --- a/api/routes/similar_services.py +++ b/app/routes/similar_services.py @@ -1,11 +1,11 @@ import logging -from typing import List, Any +from typing import Any, List -from api.recommender.similar_services.service_recommendation.recommendation_generation import \ +from app.recommender.similar_services.service_recommendation.recommendation_generation import \ IdNotExists -from api.recommender.similar_services.service_recommendation.recommendation_generation import \ +from app.recommender.similar_services.service_recommendation.recommendation_generation import \ create_recommendation as similar_services_recommendation -from api.settings import APP_SETTINGS +from app.settings import APP_SETTINGS from fastapi import APIRouter, HTTPException from pydantic import BaseModel, validator diff --git a/app/routes/update.py b/app/routes/update.py new file mode 100644 index 0000000..fb15e5c --- /dev/null +++ b/app/routes/update.py @@ -0,0 +1,40 @@ +import logging + +from app.exceptions import IdNotExists, NoneProjects, NoneServices +from app.recommender.update.updater_selector import get_updater +from fastapi import APIRouter, HTTPException + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix='/v1') + + +@router.get( + "/update", + summary="Update all data structures", + description="The data structures created (such as embeddings) need updating every x hours.", + tags=["update"] +) +def update(): + updater = get_updater() + + try: + updater.update() + except (NoneServices, NoneProjects) as e: + logger.error("Failed to update recommenders: " + str(e)) + raise HTTPException(status_code=500, detail="Failed to update recommenders: " + str(e)) + + +@router.get( + "/update_for_new_service", + summary="Updates data structures for similar services", + tags=["update"] +) +def update_for_new_service(service_id: int): + updater = get_updater() + + try: + updater.update_for_new_service(service_id) + except IdNotExists as e: + logger.error("Failed to update similar services recommender: " + str(e)) + raise HTTPException(status_code=500, detail="Failed to update similar services recommender: " + str(e)) diff --git a/app/scheduler.py b/app/scheduler.py new file mode 100644 index 0000000..05a6c2b --- /dev/null +++ b/app/scheduler.py @@ -0,0 +1,43 @@ +import logging +from multiprocessing import Process + +import cronitor +from app.recommender.update.updater_selector import get_updater +from app.routes.update import update +from app.settings import APP_SETTINGS +from apscheduler.schedulers.blocking import BlockingScheduler + +cronitor.api_key = APP_SETTINGS['CREDENTIALS']['CRONITOR_API_KEY'] +cronitor.Monitor.put( + key='update-rs', + type='job', + schedule=f'0 */{APP_SETTINGS["BACKEND"]["SCHEDULING"]["EVERY_N_HOURS"]} * * *' +) + + +def init_scheduler(): + scheduler = BlockingScheduler() + scheduler.add_job( + scheduled_update, 'cron', + hour=f'*/{APP_SETTINGS["BACKEND"]["SCHEDULING"]["EVERY_N_HOURS"]}' + # minute="*/5" + ) + try: + scheduler.start() + except (KeyboardInterrupt, SystemExit): + pass + + +@cronitor.job('update-rs') +def scheduled_update(): + logging.info("Running scheduled update...") + update() + + +def start_scheduler_process(): + updater = get_updater() + updater.initialize() + + p = Process(target=init_scheduler) + logging.info("Starting process...") + p.start() diff --git a/api/settings.py b/app/settings.py similarity index 79% rename from api/settings.py rename to app/settings.py index b01f866..5a6d5c7 100644 --- a/api/settings.py +++ b/app/settings.py @@ -1,9 +1,16 @@ import argparse import yaml +from app.exceptions import ModeDoesNotExist from dotenv import dotenv_values from sentence_transformers import SentenceTransformer +VALID_MODES = [ + 'PORTAL-RECOMMENDER', + 'PROVIDERS-RECOMMENDER', + 'SIMILAR_SERVICES_EVALUATION' +] + def load_sbert_model(sbert_settings): model = SentenceTransformer(sbert_settings["MODEL_NAME"], device=sbert_settings["DEVICE"]) @@ -52,4 +59,10 @@ def update_backend_settings(config_path): load_sbert_model(backend_settings["SIMILAR_SERVICES"]["SBERT"]) +def mode_setting_validation(): + if APP_SETTINGS['BACKEND']['MODE'] not in VALID_MODES: + raise ModeDoesNotExist(f"FATAL: Mode {APP_SETTINGS['BACKEND']['MODE']} is not valid. Check your config file. " + f"Available modes: {VALID_MODES}") + + APP_SETTINGS = read_settings() diff --git a/docker-compose-autocompletion.yml b/docker-compose-autocompletion.yml index daaea3b..813b135 100644 --- a/docker-compose-autocompletion.yml +++ b/docker-compose-autocompletion.yml @@ -17,7 +17,7 @@ services: dockerfile: ./Dockerfile-autocompletion container_name: autocompletion-system-app ports: - - "0.0.0.0:4560:4559" + - "0.0.0.0:4559:4559" networks: - EOSC_RECOMMENDATION_SYSTEM_APP depends_on: diff --git a/notebooks/evaluate_rs.ipynb b/notebooks/evaluate_rs.ipynb deleted file mode 100644 index fad33c1..0000000 --- a/notebooks/evaluate_rs.ipynb +++ /dev/null @@ -1,886 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4498ab49-2ee2-4780-838d-5cf82fcf85d2", - "metadata": {}, - "source": [ - "# Evaluate RS" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "723b1b59-b741-4542-baff-c88bb665b8aa", - "metadata": {}, - "outputs": [], - "source": [ - "import psycopg2\n", - "import pandas as pd\n", - "import requests" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "3a007982-f7e6-4733-9f12-45bf7d11364b", - "metadata": {}, - "outputs": [], - "source": [ - "def connect_and_query(query: str, params):\n", - " conn = psycopg2.connect(\n", - " host=\"localhost\",\n", - " port=5432,\n", - " database=\"mp_dump\",\n", - " user=\"postgres\",\n", - " password=\"changeme\"\n", - " )\n", - "\n", - " cur = conn.cursor()\n", - "\n", - " cur.execute(query, params)\n", - " res = cur.fetchall()\n", - "\n", - " cur.close()\n", - "\n", - " return res" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "27b8eff1-0ad6-4621-98ee-453f17040ffc", - "metadata": {}, - "outputs": [], - "source": [ - "def request_recommendations(viewed_service, purchase_history, metadata_weight, view_weight):\n", - " data = {\n", - " \"service_id\": viewed_service,\n", - " \"purchase_ids\": purchase_history,\n", - " \"num\": 5,\n", - " \"view_weight\": view_weight,\n", - " \"metadata_weight\": metadata_weight \n", - " }\n", - "\n", - " r = requests.post('http://0.0.0.0:4559/rs_evaluation/recommendation', json=data)\n", - " \n", - " return r.json()['service_ids']" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "9b0f0de9-90b8-423f-9048-f4a59a1816e8", - "metadata": {}, - "outputs": [], - "source": [ - "EOSC_SERVICES_URL = \"https://marketplace.eosc-portal.eu/services/\"\n", - "\n", - "def get_service_info(service_id):\n", - " query = f\"\"\"\n", - " SELECT id, name, slug\n", - " FROM services\n", - " WHERE id={service_id}\n", - " \"\"\"\n", - " \n", - " res = pd.DataFrame(connect_and_query(query, ()), columns=[\"id\", \"name\", \"slug\"])\n", - " res['slug'] = res['slug'].apply(lambda x: EOSC_SERVICES_URL + x)\n", - " \n", - " return res.to_dict()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "8bc360f4-1d93-424f-8da1-0f6e95f74b23", - "metadata": {}, - "outputs": [], - "source": [ - "def present_recommendations(viewed_service, purchase_history, metadata_weight=0.5, view_weight=0.5):\n", - " viewed_df = get_service_info(viewed_service)\n", - " purchase_history_df = [get_service_info(service_id) for service_id in purchase_history]\n", - " \n", - " recommended_ids = request_recommendations(viewed_service, purchase_history, metadata_weight, view_weight)\n", - " recommended_df = [get_service_info(service_id) for service_id in recommended_ids]\n", - " \n", - " \n", - " print(\"#\"*180)\n", - " print(\"> Currently viewing:\")\n", - " print(f\"\\t- {viewed_df['id'][0]} | {viewed_df['name'][0]} | {viewed_df['slug'][0]}\")\n", - " print()\n", - " \n", - " print(\"> Purchase history:\")\n", - " for service in purchase_history_df:\n", - " print(f\"\\t- {service['id'][0]} | {service['name'][0]} | {service['slug'][0]}\")\n", - " print()\n", - " \n", - " print(\"> Recommendations:\")\n", - " for service in recommended_df:\n", - " print(f\"\\t- {service['id'][0]} | {service['name'][0]} | {service['slug'][0]}\")" - ] - }, - { - "cell_type": "markdown", - "id": "623b9fbe-c915-4400-aa63-b0e1d5f2b449", - "metadata": {}, - "source": [ - "## Experiments" - ] - }, - { - "cell_type": "markdown", - "id": "d20e346a-d8a0-47f9-8f75-dfa9420e809c", - "metadata": {}, - "source": [ - "**100 Percent IT Cloud**" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "8e04c794-d130-4a1b-b35b-3981a4fd48b7", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 76 | 100 Percent IT Trusted Cloud | https://marketplace.eosc-portal.eu/services/100-percent-it-trusted-cloud\n", - "\n", - "> Purchase history:\n", - "\n", - "> Recommendations:\n", - "\t- 55 | ePouta Virtual Private Cloud | https://marketplace.eosc-portal.eu/services/csc-epouta\n", - "\t- 177 | European Cloud Hosting | https://marketplace.eosc-portal.eu/services/european-cloud-hosting\n", - "\t- 2 | EGI High-Throughput Compute | https://marketplace.eosc-portal.eu/services/egi-high-throughput-compute\n", - "\t- 1 | EGI Cloud Compute | https://marketplace.eosc-portal.eu/services/egi-cloud-compute\n", - "\t- 69 | Embassy Cloud | https://marketplace.eosc-portal.eu/services/embassy-cloud\n" - ] - } - ], - "source": [ - "present_recommendations(76, [], metadata_weight=0.5, view_weight=0.5)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "16316054-0f33-40e1-a9ab-beb07d466286", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 76 | 100 Percent IT Trusted Cloud | https://marketplace.eosc-portal.eu/services/100-percent-it-trusted-cloud\n", - "\n", - "> Purchase history:\n", - "\n", - "> Recommendations:\n", - "\t- 55 | ePouta Virtual Private Cloud | https://marketplace.eosc-portal.eu/services/csc-epouta\n", - "\t- 2 | EGI High-Throughput Compute | https://marketplace.eosc-portal.eu/services/egi-high-throughput-compute\n", - "\t- 111 | Rahti Container Cloud | https://marketplace.eosc-portal.eu/services/rahti-container-cloud\n", - "\t- 45 | EGI Check-In | https://marketplace.eosc-portal.eu/services/egi-check-in\n", - "\t- 362 | Open Knowledge Maps | https://marketplace.eosc-portal.eu/services/open-knowledge-maps\n" - ] - } - ], - "source": [ - "present_recommendations(76, [], metadata_weight=1)\n", - "\n", - "# We can see services that do not relate with the service we currently view EGI Check-In, Open Knowledge Maps" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "cca2d38c-e115-40db-bce4-3b7e37d74bef", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 76 | 100 Percent IT Trusted Cloud | https://marketplace.eosc-portal.eu/services/100-percent-it-trusted-cloud\n", - "\n", - "> Purchase history:\n", - "\n", - "> Recommendations:\n", - "\t- 511 | CLOUDIFIN | https://marketplace.eosc-portal.eu/services/cloudifin\n", - "\t- 1 | EGI Cloud Compute | https://marketplace.eosc-portal.eu/services/egi-cloud-compute\n", - "\t- 79 | MetaCentrum Cloud | https://marketplace.eosc-portal.eu/services/metacentrum-cloud\n", - "\t- 227 | Open Telekom Cloud | https://marketplace.eosc-portal.eu/services/open-telekom-cloud\n", - "\t- 622 | SCIGNE Cloud Compute | https://marketplace.eosc-portal.eu/services/scigne-cloud-compute\n" - ] - } - ], - "source": [ - "present_recommendations(76, [], metadata_weight=0)\n", - "\n", - "# Similar to metada_weight=0.5" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "d09b10a5-78df-41e8-af04-69d00d5cd1d0", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 76 | 100 Percent IT Trusted Cloud | https://marketplace.eosc-portal.eu/services/100-percent-it-trusted-cloud\n", - "\n", - "> Purchase history:\n", - "\t- 419 | DisVis web portal | https://marketplace.eosc-portal.eu/services/disvis-web-portal-6eab178c-9bc5-4c62-b7ce-aeeb18d5cba9\n", - "\n", - "> Recommendations:\n", - "\t- 69 | Embassy Cloud | https://marketplace.eosc-portal.eu/services/embassy-cloud\n", - "\t- 177 | European Cloud Hosting | https://marketplace.eosc-portal.eu/services/european-cloud-hosting\n", - "\t- 363 | de.NBI Cloud: Cloud Computing for Life Sciences | https://marketplace.eosc-portal.eu/services/cloud-computing-for-life-sciences\n", - "\t- 425 | PDB-Tools web | https://marketplace.eosc-portal.eu/services/pdb-tools-web\n", - "\t- 42 | CloudFerro Infrastructure | https://marketplace.eosc-portal.eu/services/cloudferro-infrastructure\n" - ] - } - ], - "source": [ - "present_recommendations(76, [419], metadata_weight=0.5)\n", - "\n", - "# Interesting recommendation: de.NBI Cloud: Cloud Computing for Life Sciences which combines both viewing and history" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "4f5bfc28-52ba-484e-913d-b92ee52d43f0", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 76 | 100 Percent IT Trusted Cloud | https://marketplace.eosc-portal.eu/services/100-percent-it-trusted-cloud\n", - "\n", - "> Purchase history:\n", - "\t- 419 | DisVis web portal | https://marketplace.eosc-portal.eu/services/disvis-web-portal-6eab178c-9bc5-4c62-b7ce-aeeb18d5cba9\n", - "\n", - "> Recommendations:\n", - "\t- 420 | HADDOCK2.4 web portal | https://marketplace.eosc-portal.eu/services/haddock2-4-web-portal\n", - "\t- 421 | PowerFit web portal | https://marketplace.eosc-portal.eu/services/powerfit-web-portal-b8ddee6c-78f5-43d8-a5a2-9e3b7f1cb24e\n", - "\t- 425 | PDB-Tools web | https://marketplace.eosc-portal.eu/services/pdb-tools-web\n", - "\t- 422 | SpotOn web portal | https://marketplace.eosc-portal.eu/services/spoton-c5db8fd5-a546-4342-8bae-2b2b4777b67e\n", - "\t- 69 | Embassy Cloud | https://marketplace.eosc-portal.eu/services/embassy-cloud\n" - ] - } - ], - "source": [ - "present_recommendations(76, [419], metadata_weight=1)\n", - "\n", - "# Focuses on web portals and offers 1 cloud solution" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "4a21b7cd-c3ac-4be9-81a9-64ebba9688a7", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 76 | 100 Percent IT Trusted Cloud | https://marketplace.eosc-portal.eu/services/100-percent-it-trusted-cloud\n", - "\n", - "> Purchase history:\n", - "\t- 419 | DisVis web portal | https://marketplace.eosc-portal.eu/services/disvis-web-portal-6eab178c-9bc5-4c62-b7ce-aeeb18d5cba9\n", - "\n", - "> Recommendations:\n", - "\t- 177 | European Cloud Hosting | https://marketplace.eosc-portal.eu/services/european-cloud-hosting\n", - "\t- 69 | Embassy Cloud | https://marketplace.eosc-portal.eu/services/embassy-cloud\n", - "\t- 55 | ePouta Virtual Private Cloud | https://marketplace.eosc-portal.eu/services/csc-epouta\n", - "\t- 2 | EGI High-Throughput Compute | https://marketplace.eosc-portal.eu/services/egi-high-throughput-compute\n", - "\t- 1 | EGI Cloud Compute | https://marketplace.eosc-portal.eu/services/egi-cloud-compute\n" - ] - } - ], - "source": [ - "present_recommendations(76, [419], metadata_weight=0.5, view_weight=0.8)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "37c9b1cc-3d23-49f7-b0ae-daa9cffb8cc9", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 76 | 100 Percent IT Trusted Cloud | https://marketplace.eosc-portal.eu/services/100-percent-it-trusted-cloud\n", - "\n", - "> Purchase history:\n", - "\t- 419 | DisVis web portal | https://marketplace.eosc-portal.eu/services/disvis-web-portal-6eab178c-9bc5-4c62-b7ce-aeeb18d5cba9\n", - "\t- 352 | E-Learning Platform of GBIF Spain | https://marketplace.eosc-portal.eu/services/e-learning-platform-of-gbif-spain\n", - "\n", - "> Recommendations:\n", - "\t- 69 | Embassy Cloud | https://marketplace.eosc-portal.eu/services/embassy-cloud\n", - "\t- 177 | European Cloud Hosting | https://marketplace.eosc-portal.eu/services/european-cloud-hosting\n", - "\t- 363 | de.NBI Cloud: Cloud Computing for Life Sciences | https://marketplace.eosc-portal.eu/services/cloud-computing-for-life-sciences\n", - "\t- 2 | EGI High-Throughput Compute | https://marketplace.eosc-portal.eu/services/egi-high-throughput-compute\n", - "\t- 42 | CloudFerro Infrastructure | https://marketplace.eosc-portal.eu/services/cloudferro-infrastructure\n" - ] - } - ], - "source": [ - "present_recommendations(76, [419, 352], metadata_weight=0.5, view_weight=0.5)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "ba30d90b-29c9-47ad-a82e-8545f93425ed", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 352 | E-Learning Platform of GBIF Spain | https://marketplace.eosc-portal.eu/services/e-learning-platform-of-gbif-spain\n", - "\n", - "> Purchase history:\n", - "\t- 419 | DisVis web portal | https://marketplace.eosc-portal.eu/services/disvis-web-portal-6eab178c-9bc5-4c62-b7ce-aeeb18d5cba9\n", - "\t- 76 | 100 Percent IT Trusted Cloud | https://marketplace.eosc-portal.eu/services/100-percent-it-trusted-cloud\n", - "\n", - "> Recommendations:\n", - "\t- 363 | de.NBI Cloud: Cloud Computing for Life Sciences | https://marketplace.eosc-portal.eu/services/cloud-computing-for-life-sciences\n", - "\t- 210 | Software Integration Support | https://marketplace.eosc-portal.eu/services/software-integration-support\n", - "\t- 51 | PaN faas | https://marketplace.eosc-portal.eu/services/pan-faas\n", - "\t- 511 | CLOUDIFIN | https://marketplace.eosc-portal.eu/services/cloudifin\n", - "\t- 174 | Scientific Training Environment | https://marketplace.eosc-portal.eu/services/scientific-training-environment\n" - ] - } - ], - "source": [ - "present_recommendations(352, [419, 76], metadata_weight=0.5, view_weight=0.5)" - ] - }, - { - "cell_type": "markdown", - "id": "24b5db7d-3a71-4cc0-bb5f-42f16674479d", - "metadata": {}, - "source": [ - "**KER - Keyword Extractor**" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "1eb16862-83d8-409d-89e2-cedee660c14e", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 496 | KER - Keyword Extractor | https://marketplace.eosc-portal.eu/services/ker-keyword-extractor\n", - "\n", - "> Purchase history:\n", - "\n", - "> Recommendations:\n", - "\t- 493 | MorphoDiTa | https://marketplace.eosc-portal.eu/services/morphodita\n", - "\t- 495 | Machine Translation | https://marketplace.eosc-portal.eu/services/machine-translation\n", - "\t- 492 | ElixirFM | https://marketplace.eosc-portal.eu/services/elixirfm\n", - "\t- 494 | NameTag | https://marketplace.eosc-portal.eu/services/nametag\n", - "\t- 434 | UDPipe: tool for lemmatization, morphological analysis, POS tagging and dependency parsing in multiple languages | https://marketplace.eosc-portal.eu/services/udpipe-tool-for-lemmatization-morphological-analysis-pos-tagging-and-dependency-parsing-in-multiple-languages\n" - ] - } - ], - "source": [ - "present_recommendations(496, [], metadata_weight=0.5, view_weight=0.5)\n", - "# Proposes similar solutions from the same team" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "d572393c-18c2-47c4-89de-0fb972babec2", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 496 | KER - Keyword Extractor | https://marketplace.eosc-portal.eu/services/ker-keyword-extractor\n", - "\n", - "> Purchase history:\n", - "\n", - "> Recommendations:\n", - "\t- 626 | The Tromsø Repository of Language and Linguistics (TROLLing) | https://marketplace.eosc-portal.eu/services/the-tromso-repository-of-language-and-linguistics-trolling\n", - "\t- 434 | UDPipe: tool for lemmatization, morphological analysis, POS tagging and dependency parsing in multiple languages | https://marketplace.eosc-portal.eu/services/udpipe-tool-for-lemmatization-morphological-analysis-pos-tagging-and-dependency-parsing-in-multiple-languages\n", - "\t- 145 | V-SEEM CLOWDER | https://marketplace.eosc-portal.eu/services/v-seem-clowder\n", - "\t- 424 | AMBER-based Portal Server for NMR structures (AMPS-NMR) | https://marketplace.eosc-portal.eu/services/amber-based-portal-server-for-nmr-structures-amps-nmr\n", - "\t- 62 | Identifiers.org | https://marketplace.eosc-portal.eu/services/identifiers-org\n" - ] - } - ], - "source": [ - "present_recommendations(496, [], metadata_weight=0, view_weight=0.5)\n", - "# While the first two are related (with the first one being from a diff organisation) the rest of them are not" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "6b6e9868-cc03-4d60-9ec5-ae33f5bd50de", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 496 | KER - Keyword Extractor | https://marketplace.eosc-portal.eu/services/ker-keyword-extractor\n", - "\n", - "> Purchase history:\n", - "\t- 17 | Language Resource Switchboard | https://marketplace.eosc-portal.eu/services/language-resource-switchboard\n", - "\n", - "> Recommendations:\n", - "\t- 495 | Machine Translation | https://marketplace.eosc-portal.eu/services/machine-translation\n", - "\t- 493 | MorphoDiTa | https://marketplace.eosc-portal.eu/services/morphodita\n", - "\t- 18 | Virtual Collection Registry | https://marketplace.eosc-portal.eu/services/virtual-collection-registry\n", - "\t- 492 | ElixirFM | https://marketplace.eosc-portal.eu/services/elixirfm\n", - "\t- 16 | Virtual Language Observatory | https://marketplace.eosc-portal.eu/services/virtual-language-observatory\n" - ] - } - ], - "source": [ - "present_recommendations(496, [17], metadata_weight=0.5, view_weight=0.5)\n", - "# Virtual Language Observatory relates with the purchase history" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Evaluate with use cases**" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "1) **User description**: Graduate of the History and Archeology department currently studying for her Msc in linguistics\n", - "\n", - " **Use case**: Using the platform to search tools related to enity recognision in natural language texts.\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 494 | NameTag | https://marketplace.eosc-portal.eu/services/nametag\n", - "\n", - "> Purchase history:\n", - "\n", - "> Recommendations:\n", - "\t- 493 | MorphoDiTa | https://marketplace.eosc-portal.eu/services/morphodita\n", - "\t- 492 | ElixirFM | https://marketplace.eosc-portal.eu/services/elixirfm\n", - "\t- 495 | Machine Translation | https://marketplace.eosc-portal.eu/services/machine-translation\n", - "\t- 496 | KER - Keyword Extractor | https://marketplace.eosc-portal.eu/services/ker-keyword-extractor\n", - "\t- 626 | The Tromsø Repository of Language and Linguistics (TROLLing) | https://marketplace.eosc-portal.eu/services/the-tromso-repository-of-language-and-linguistics-trolling\n" - ] - } - ], - "source": [ - "present_recommendations(494, [])" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 494 | NameTag | https://marketplace.eosc-portal.eu/services/nametag\n", - "\n", - "> Purchase history:\n", - "\t- 137 | Digital Humanities and Cultural Heritage OpenAIRE Community Gateway | https://marketplace.eosc-portal.eu/services/digital-humanities-and-cultural-heritage-openaire-community-gateway\n", - "\t- 101 | Europeana APIs | https://marketplace.eosc-portal.eu/services/europeana-apis\n", - "\n", - "> Recommendations:\n", - "\t- 493 | MorphoDiTa | https://marketplace.eosc-portal.eu/services/morphodita\n", - "\t- 495 | Machine Translation | https://marketplace.eosc-portal.eu/services/machine-translation\n", - "\t- 492 | ElixirFM | https://marketplace.eosc-portal.eu/services/elixirfm\n", - "\t- 496 | KER - Keyword Extractor | https://marketplace.eosc-portal.eu/services/ker-keyword-extractor\n", - "\t- 142 | BiOnym | https://marketplace.eosc-portal.eu/services/bionym\n" - ] - } - ], - "source": [ - "present_recommendations(494, [137, 101])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "2) **User description**: A full-stack developer, member of a startup related to bioinformatics\n", - "\n", - " **Use case**: He is searching for anonymization tool for their collected datasets" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 11 | B2ACCESS | https://marketplace.eosc-portal.eu/services/b2access\n", - "\n", - "> Purchase history:\n", - "\n", - "> Recommendations:\n", - "\t- 6 | B2HANDLE | https://marketplace.eosc-portal.eu/services/b2handle\n", - "\t- 9 | B2SHARE | https://marketplace.eosc-portal.eu/services/b2share\n", - "\t- 8 | B2SAFE | https://marketplace.eosc-portal.eu/services/b2safe\n", - "\t- 45 | EGI Check-In | https://marketplace.eosc-portal.eu/services/egi-check-in\n", - "\t- 515 | OpenAIRE Login | https://marketplace.eosc-portal.eu/services/openaire-login\n" - ] - } - ], - "source": [ - "present_recommendations(11, [])" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 11 | B2ACCESS | https://marketplace.eosc-portal.eu/services/b2access\n", - "\n", - "> Purchase history:\n", - "\t- 429 | MetalPDB: a database of metal-binding sites in 3D structures of biological macromolecules | https://marketplace.eosc-portal.eu/services/metalpdb-21ea9621-ff08-4e08-8c38-366d7aa07c88\n", - "\t- 419 | DisVis web portal | https://marketplace.eosc-portal.eu/services/disvis-web-portal-6eab178c-9bc5-4c62-b7ce-aeeb18d5cba9\n", - "\n", - "> Recommendations:\n", - "\t- 420 | HADDOCK2.4 web portal | https://marketplace.eosc-portal.eu/services/haddock2-4-web-portal\n", - "\t- 422 | SpotOn web portal | https://marketplace.eosc-portal.eu/services/spoton-c5db8fd5-a546-4342-8bae-2b2b4777b67e\n", - "\t- 9 | B2SHARE | https://marketplace.eosc-portal.eu/services/b2share\n", - "\t- 6 | B2HANDLE | https://marketplace.eosc-portal.eu/services/b2handle\n", - "\t- 425 | PDB-Tools web | https://marketplace.eosc-portal.eu/services/pdb-tools-web\n" - ] - } - ], - "source": [ - "present_recommendations(11, [429, 419])" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 11 | B2ACCESS | https://marketplace.eosc-portal.eu/services/b2access\n", - "\n", - "> Purchase history:\n", - "\t- 429 | MetalPDB: a database of metal-binding sites in 3D structures of biological macromolecules | https://marketplace.eosc-portal.eu/services/metalpdb-21ea9621-ff08-4e08-8c38-366d7aa07c88\n", - "\t- 419 | DisVis web portal | https://marketplace.eosc-portal.eu/services/disvis-web-portal-6eab178c-9bc5-4c62-b7ce-aeeb18d5cba9\n", - "\t- 247 | LEMONADE - Live Exploration and Mining Of a Non-trivial Amount of Data from Everywhere | https://marketplace.eosc-portal.eu/services/lemonade-live-exploration-and-mining-of-a-non-trivial-amount-of-data-from-everywhere\n", - "\t- 109 | NOMAD repository | https://marketplace.eosc-portal.eu/services/nomad-repository\n", - "\n", - "> Recommendations:\n", - "\t- 9 | B2SHARE | https://marketplace.eosc-portal.eu/services/b2share\n", - "\t- 6 | B2HANDLE | https://marketplace.eosc-portal.eu/services/b2handle\n", - "\t- 8 | B2SAFE | https://marketplace.eosc-portal.eu/services/b2safe\n", - "\t- 5 | B2FIND | https://marketplace.eosc-portal.eu/services/b2find\n", - "\t- 372 | OpenAIRE Research Community Dashboard | https://marketplace.eosc-portal.eu/services/openaire-research-community-dashboard-e347a58e-d556-4610-985d-8d74e96b3172\n" - ] - } - ], - "source": [ - "present_recommendations(11, [429, 419, 247, 109])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "3) **User description**: A practising phycologist\n", - "\n", - " **Use case**: Search for a registry with surveys about migrants and refugees" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 416 | ISIDORE | https://marketplace.eosc-portal.eu/services/isidore\n", - "\n", - "> Purchase history:\n", - "\n", - "> Recommendations:\n", - "\t- 17 | Language Resource Switchboard | https://marketplace.eosc-portal.eu/services/language-resource-switchboard\n", - "\t- 92 | OpenAIRE Mining Service | https://marketplace.eosc-portal.eu/services/openaire-mining-service\n", - "\t- 106 | Data Service Portal Aila | https://marketplace.eosc-portal.eu/services/data-service-portal-aila\n", - "\t- 137 | Digital Humanities and Cultural Heritage OpenAIRE Community Gateway | https://marketplace.eosc-portal.eu/services/digital-humanities-and-cultural-heritage-openaire-community-gateway\n", - "\t- 660 | OpenBioMaps | https://marketplace.eosc-portal.eu/services/openbiomaps\n" - ] - } - ], - "source": [ - "present_recommendations(416, [])" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 416 | ISIDORE | https://marketplace.eosc-portal.eu/services/isidore\n", - "\n", - "> Purchase history:\n", - "\t- 408 | ELSST – European Language Social Science Thesaurus | https://marketplace.eosc-portal.eu/services/elsst-european-language-social-science-thesaurus\n", - "\t- 668 | European Social Survey (ESS) as a service | https://marketplace.eosc-portal.eu/services/european-social-survey-ess-as-a-service\n", - "\n", - "> Recommendations:\n", - "\t- 106 | Data Service Portal Aila | https://marketplace.eosc-portal.eu/services/data-service-portal-aila\n", - "\t- 17 | Language Resource Switchboard | https://marketplace.eosc-portal.eu/services/language-resource-switchboard\n", - "\t- 92 | OpenAIRE Mining Service | https://marketplace.eosc-portal.eu/services/openaire-mining-service\n", - "\t- 387 | OPERAS Research for Society (Hypotheses) | https://marketplace.eosc-portal.eu/services/operas-research-for-society-hypotheses\n", - "\t- 16 | Virtual Language Observatory | https://marketplace.eosc-portal.eu/services/virtual-language-observatory\n" - ] - } - ], - "source": [ - "present_recommendations(416, [408, 668])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "4) **User description**: An employee at the national observatory of Athens\n", - "\n", - " **Use case**: Searching for a tool for terrain mapping" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 521 | UW-Mos | https://marketplace.eosc-portal.eu/services/uw-mos\n", - "\n", - "> Purchase history:\n", - "\n", - "> Recommendations:\n", - "\t- 560 | VD-Maps | https://marketplace.eosc-portal.eu/services/vd-maps\n", - "\t- 507 | UW-MAP | https://marketplace.eosc-portal.eu/services/uw-map\n", - "\t- 554 | ADAM Platform | https://marketplace.eosc-portal.eu/services/adam-platform\n", - "\t- 661 | LOFAR Science Processing | https://marketplace.eosc-portal.eu/services/lofar-science-processing\n", - "\t- 663 | Climadjust | https://marketplace.eosc-portal.eu/services/climadjust\n" - ] - } - ], - "source": [ - "present_recommendations(521, [])" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 521 | UW-Mos | https://marketplace.eosc-portal.eu/services/uw-mos\n", - "\n", - "> Purchase history:\n", - "\t- 37 | EODC Data Catalogue Service | https://marketplace.eosc-portal.eu/services/eodc-data-catalogue-service\n", - "\t- 15 | GEO Discovery and Access Broker | https://marketplace.eosc-portal.eu/services/geo-dab\n", - "\t- 26 | OPENCoastS Portal | https://marketplace.eosc-portal.eu/services/opencoasts-portal\n", - "\n", - "> Recommendations:\n", - "\t- 507 | UW-MAP | https://marketplace.eosc-portal.eu/services/uw-map\n", - "\t- 560 | VD-Maps | https://marketplace.eosc-portal.eu/services/vd-maps\n", - "\t- 178 | Indian Ocean Tuna Commission Spatial Data Catalog | https://marketplace.eosc-portal.eu/services/indian-ocean-tuna-commission-spatial-data-catalog\n", - "\t- 225 | Global Tuna Atlas Spatial Data Catalog | https://marketplace.eosc-portal.eu/services/global-tuna-atlas-spatial-data-catalog\n", - "\t- 230 | Western Central Atlantic Fishery Commission Spatial Data Catalog | https://marketplace.eosc-portal.eu/services/western-central-atlantic-fishery-commission-spatial-data-catalog\n" - ] - } - ], - "source": [ - "present_recommendations(521, [37, 15, 26])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "5) **User description**: Phd vet\n", - "\n", - " **Use case**: General search for new tools related to his profession" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 348 | Ubora | https://marketplace.eosc-portal.eu/services/ubora\n", - "\n", - "> Purchase history:\n", - "\n", - "> Recommendations:\n", - "\t- 184 | UBORA e-platform | https://marketplace.eosc-portal.eu/services/ubora-e-platform\n", - "\t- 174 | Scientific Training Environment | https://marketplace.eosc-portal.eu/services/scientific-training-environment\n", - "\t- 159 | OpenMinTeD Support and Training | https://marketplace.eosc-portal.eu/services/openminted-support-and-training\n", - "\t- 102 | Snap4City | https://marketplace.eosc-portal.eu/services/snap4city\n", - "\t- 210 | Software Integration Support | https://marketplace.eosc-portal.eu/services/software-integration-support\n" - ] - } - ], - "source": [ - "present_recommendations(348, [])" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "####################################################################################################################################################################################\n", - "> Currently viewing:\n", - "\t- 348 | Ubora | https://marketplace.eosc-portal.eu/services/ubora\n", - "\n", - "> Purchase history:\n", - "\t- 514 | GBIF Species Occurrence Data | https://marketplace.eosc-portal.eu/services/gbif-species-occurrence-data\n", - "\t- 138 | PhenoMeNal | https://marketplace.eosc-portal.eu/services/phenomenal\n", - "\n", - "> Recommendations:\n", - "\t- 184 | UBORA e-platform | https://marketplace.eosc-portal.eu/services/ubora-e-platform\n", - "\t- 174 | Scientific Training Environment | https://marketplace.eosc-portal.eu/services/scientific-training-environment\n", - "\t- 352 | E-Learning Platform of GBIF Spain | https://marketplace.eosc-portal.eu/services/e-learning-platform-of-gbif-spain\n", - "\t- 156 | D4Science Spatial Data Catalog | https://marketplace.eosc-portal.eu/services/d4science-spatial-data-catalog\n", - "\t- 40 | Datacube | https://marketplace.eosc-portal.eu/services/rasdaman-eo-datacube\n" - ] - } - ], - "source": [ - "present_recommendations(348, [514, 138])" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/projects_analysis.ipynb b/notebooks/projects_analysis.ipynb deleted file mode 100644 index c988b88..0000000 --- a/notebooks/projects_analysis.ipynb +++ /dev/null @@ -1,180 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "027059ed-3fd9-4ed3-98da-02ceddee38ac", - "metadata": {}, - "source": [ - "# Project Analysis" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "e6d4e6d3-ba73-4357-94b8-9e40241cc9fc", - "metadata": {}, - "outputs": [], - "source": [ - "import psycopg2\n", - "import pandas as pd\n", - "import requests\n", - "from matplotlib import pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "cb8aa8e5-63bb-4d67-bf55-47666281897b", - "metadata": {}, - "outputs": [], - "source": [ - "def connect_and_query(query: str, params):\n", - " conn = psycopg2.connect(\n", - " host=\"localhost\",\n", - " port=5432,\n", - " database=\"mp_dump\",\n", - " user=\"postgres\",\n", - " password=\"changeme\"\n", - " )\n", - "\n", - " cur = conn.cursor()\n", - "\n", - " cur.execute(query, params)\n", - " res = cur.fetchall()\n", - "\n", - " cur.close()\n", - "\n", - " return res" - ] - }, - { - "cell_type": "code", - "execution_count": 88, - "id": "cb707d7e-e31a-4b63-92df-8a9185f725a1", - "metadata": {}, - "outputs": [], - "source": [ - "def get_number_of_services_per_project():\n", - " query = \"\"\"\n", - " SELECT COUNT(*)\n", - " FROM projects, offers, services, project_items\n", - " WHERE project_items.offer_id = offers.id AND offers.service_id = services.id AND project_items.project_id = projects.id\n", - " GROUP BY projects.id\n", - " \"\"\"\n", - " res = pd.DataFrame(connect_and_query(query, ()), columns=[\"services_numb\"])\n", - " \n", - " return res\n", - "\n", - "\n", - "def get_number_of_projects_per_user():\n", - " query = \"\"\"\n", - " SELECT COUNT(*)\n", - " FROM projects\n", - " GROUP BY projects.user_id\n", - " \"\"\"\n", - " res = pd.DataFrame(connect_and_query(query, ()), columns=[\"projects_numb\"])\n", - " \n", - " return res\n", - "\n", - "\n", - "def get_number_of_services_added_to_projects_per_user():\n", - " query = \"\"\"\n", - " SELECT COUNT(*)\n", - " FROM projects, offers, services, project_items\n", - " WHERE project_items.offer_id = offers.id AND offers.service_id = services.id AND project_items.project_id = projects.id\n", - " GROUP BY projects.user_id;\n", - " \"\"\"\n", - " res = pd.DataFrame(connect_and_query(query, ()), columns=[\"services_numb\"])\n", - " \n", - " return res\n", - "\n", - "services_per_proj = get_number_of_services_per_project()['services_numb'].tolist()\n", - "services_per_proj += [0] * (457 - len(services_per_proj))\n", - "\n", - "projects_per_user = get_number_of_projects_per_user()['projects_numb'].tolist()\n", - "\n", - "number_of_services_added_to_projects_per_user = get_number_of_services_added_to_projects_per_user()['services_numb'].tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "id": "613dc855-682f-44e2-a799-8d40ba35132f", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABDQAAAVpCAYAAACXiVznAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAEAAElEQVR4nOzdd5hcVfnA8e9LQoAAgdBCldBrEJEuQhCUXkSaFImIWFEERUDQgJSg1B8oVQSkBASkCkhLAOmhSO+ht0AChEACyfn9ce4mk8ns7kyyu7M3fD/PM8/snHvuve+9c3dn7zunREoJSZIkSZKkMpml2QFIkiRJkiQ1yoSGJEmSJEkqHRMakiRJkiSpdExoSJIkSZKk0jGhIUmSJEmSSseEhiRJkiRJKh0TGpKkDhURqXh8FBELtFJnUFHnnK6Orx4RMayIr3+zY+loEdEvIi6MiLcjYmJxnNvXsV6fiPhDRDxUvLfjI+K1iLgnIo6PiA27IPyGVVxrg5sdy8wqIvpX/N63PCZFxPsRMbx4D6IL4kgRMbKz9yNJ6j56NjsASdJMay7gt8Bvmh2IpvI3YCvgUeA/wETglbZWiIgvAcOB/sDHwH3A28B8wJrAusCqwB2dFbRK4/ziuQewDPB1YENgU2CPZgXVmSJiGLARsFRKaWRzo5GkLxYTGpKkzpCA8cDPIuKElNJbzQ5IEBG9gM2BkcAaKaVJda56GjmZcROwW0rp/YptzgIMBFbryFg70L+Ae4FRzQ7kiyClNKjydUR8E/g3sHtEXJJSur4Td78S8Fknbl+S1M3Y5USS1BkmAWcAcwCHNDkWTbEw+Zvzl+tNZkTEHMAWxcufVyYzAFJKk1JKt6WUTu7QSDtISumDlNLTKSUTGk2QUroZ+EfxcvtO3tfTKaUXOnMfkqTuxYSGJKmzDAHGAT+KiMXrWSEiziv6wQ9sZfnIiEhVZQOLdc6LiPkj4syIeCMixkXE/RGxeUXdHYoxH8ZGxHsR8beImLedmPaIiBHF9t6NiAvaOp6I2Dwiri/qjo+IFyPixIiYv63jjYjNIuL2iBhTlLUZV1V8d0bEBxHxSUQ8FhG/KxIRlfUS8HLxcqOKsQ6GtbOLvkxp0flOPTFV7XediPhnRLwZEROKcTfOKbqxVNcdXMQ0KCLWjojrivcpRcRaETGq2MY057JYf+mi7gsVZa2OoRHZdyPi5mI/nxbX2GURsUmN+ktExGkR8UJR9/0ixvVbiWfdiLii2OanEfFO5DFIToqIfnWev8prZIvivf4oIkZHxFURsXIb63bUuV+9nljb8HDxPHm/xXZHRkSviPh9RDxd/L5cVVFniYg4IyJeLpa9ExH/ioh1WjneVsfQiIiVinP5arGttyNiaESs0lrQxfkbGhGvF+u8GRG3RsQPi+X9i9+rjYpVXqr4vUqtbVeS1HFMaEiSOkVK6W1yV4XZgN91wS77krsWbAXcCTxAHt/h2ojYOCL2By4DPgVuJHeJ2Ru4KqLVAQt/DVwAjAWuBj4C9gTua+WmcAhwA3m8gGeAa4DPgV8V67R2E7tbsd6cxfMD5G47bYqIM8nffq8J/Be4jtwK4yhgWETMWVH9fOCK4ue3i9fnk89FW0aRzxnAj9uLqSq+nwJ3AzuQx+m4CngP+AHwYESs1MqqGwJ3kbu5/Ic8Nsd48vs3K7BTK+vtVjxfVEdsPYBLgYuL/T1K7p7yGvka2q+q/npFnZ+RuzVcDzwObAbcERG7VNXfmvye7EBOBF1Ffl/nAPYHlmsvxio7Ffucjfw+vw5sB9wbEV+pcXwdee7r7ZrUmrmL5/FV5bMUcR0EvED+HXuziH8A8BDwI+AT4ErgOXIrj/9GxK717jzyoLcPA3uRz8E1wEvAzsD9UWNA24j4Jfn87QK8Vez/cfJYMX8uqo0l/w69Xby+gim/V+cjSep8KSUfPnz48OGjwx7kG/HPi5/nBz4EJgBLVtQZVNQ7p2rd84ryga1se2T+6JqqbGCxTgIuAXpVLPthUf4sMBpYr2LZvMBTtfYHDCvKPwO2rCjvWRHjdVXr7FSUPwEsV1EewBHFsqGtHG8CdmnwPH+nWO+Nqv3NQ07oJODkqnX6F+XDGtzXGRVx3g/8AdgSWLCNddYlJ3PeANaqWvaDYlv3VpUPrtjPQTW2uX6x7M5W9tnyfi5f41obXFX3sKL8SWDpqmXzABtVvO5THMdEYM+qumsC75OTXQtWlLdcQzvViHMVoF+d577yGvlh1XV1VFH+PyA689y3E2PLdZVqLAtyYiABR1WUt+zrOWCxGuv8r1h+XNWxfad4H8bWWC8BI2vENrZ4fKtq2ebkv02vMPXfjQ3JSZwPa6zTk4q/CVXvdf9GzpsPHz58+Jjxhy00JEmdJqX0HnAK+Vv1wzt5dx+Sx3iYUFF2LrmFwXLAX1JK91TENoZ8ow5TmoxXuyyl9O+KdVpaW4wFtoyIpSrqtrRC2S2l9FzFOol8s/gIsGPUnsr2+pTSpe0dYJVfFM+Dq/b3AfATihvgiOjd4HZr+RXw92Kba5GP53rg7Yi4r7p1QuFg8ngdP04pPVC5IKX0N/K35OvUal0APMaUb8Er17sbeBH4WlRNqRsRawArAg+mlJ5t62AiD456IFMSDi9W7eeDlNLwiqK9gUWAU1JK/6iq+yDwR/KsPpWzeCxYPP+nxnE8kXILpkbcnVI6u2IbLdfVy8AApr6GO/zcNyoiekTEcuTfwfXIrTP+XqPqISml16vKBpKP6RXgsOJYW+K/gtxaYk5gnzpC2b+oe2hKaar3IqV0I3A6sAS5VU6Lg8lJlYNrrPN55d8ESVJzmdCQJHW2E4AxwF4RsUwn7mdEkUCZLKU0kSnjRkxzY0lu5g6waCvbHFpdkFIaTe6mEcAGABGxEPBl4PmU0qM11knk7gc9gK/W2M81rey/poiYlfwtPNToXpFSepzcxL43sEYj264lpfRJSmlvYHnyzd515Gb2AawNDI2IUyrimwXYhDyGyg2tbPbO4nntGsuuq7yJrXJRsd/dqsp3L54vbPtogNyqYl5y8uOJOup/q3i+opXltY5lRPF8QUSs2Ua3pnrVuhY/B/5ZvPw6dPq5b1fF+BGfk1tGDSK3XvlumnbAzgRcW2MzXy+eL0sp1Zq15IKqem1p6L2LiJ7khMpEpgxmKknqppy2VZLUqVJKYyLiBPK32H8AvtdJu3qtlfKxbSxvWTZbK+u+3Er5yOK5JRHSv3heto7BAGu10HilnXWqzQ/0AkallD5upc5L5GTGYg1uu1UppefJXQCOg8mtIgYD2wC/iIjLUkr/JR/jXMVqE9q5l2/0fFxIbu2zO3BMEccswK7km9BpbvxrWKJ4brMlR4X+xfNdDRzLIeSuJdsWjzERcQ+5Zcv5KaWxNdZvS73XYmee+3q0jB3R0mXjMeDKIhFY7Z2UUvW4GjDlWEa2so+Xiud6ru3+xfNrdZ6L+cnjnLyeUvqoju1LkprIhIYkqSucQm76vVtEHDMD22mrZWF7AxfO6MCGbWmJ6y3gpnbq1rox/bRGWbeXUnqoGHDxPnKrh63ILVFazsdYWv9mvEWtFhKtno+U0rMR8QCwVkR8JaX0MLAx+Sb4punoylGPluO5HGgtgQTwdEWcr0fE2kVsW5K7hGxGngL3kIjYIKU0shNj7fBzX4+U0qAGqnfFdd9yPtobpPO+zg5EktTxTGhIkjpdSumjiPgT+Zv9wbQ+s0bL+BdzVS8omoIv0ikBtm5J8uCEtcohD7oIU1p/jGrwhm56vUc+VwtExJyttNLoXzxXj0/QoVJKkyJiODmh0fItd8vMKJOA789IF4ZWXEgey2N3cteaRrqbALxaPNc708hrwArAkJTSiPYqtyi6PN1SPIiIhckz/3wHOJopcddjyXbKW67Fzj73XaHlWPq3srylvJ5r+zVgGeDA6i5prRhFnlVl4YiYazpa0kiSupBjaEiSuspp5HEXdiZPfVjLm8XzCjWWbUzXJ+J3ri6IiHnJsyNAbo1ASuk18rfzK0fE8p0dVDGuwL3Fy2luiiNiFXJ3k3HkqS8727LF8+tFfJ+TZ37oQx7PoaMNJY/R8N1i0NPvkFtO/KvO9UeQx3VZKyJWrqP+zcXztxuMcyoppbeAI4uXrf0OtKbWtdgD2LF4eVexj84+912hZVyLnYvxYqrtWVWvLQ29d0USahh5vJs92q49WUsi1i8KJamLmdCQJHWJlNI4YAh5QMeftFJtWPH802KgTQAiYlng1E4NsLZdImKzijh6AicCcwM3VM2O8Ufy5+oVtWaPiIj5I+KHHRhby/n4Q3F+WvYzN/AX8nk+uzjv0y0i5o2I+yNix2J2kMpls0TEPuQxIiYxdULh6KLs7xGxcY3tzhURe0fEHI3GlFJ6h3yjuihwPPnm/eo2xhOpXn88cBL5HF1WNVsNETFPRFTOGnIm8A5wUET8qEgkVNbvGRGbRcSqFWUHREStFkVbFs+v1ljWlg0iYu+qst8DS5Gnnh1WUd5p576LDCOPvfEl4I+VA6pGxLfJSZxxwDl1bOsEcouL4yPiO9ULI2K24tpevKL4OIopYyNik6r6PSNiS6bW0qKkViJWktSJzCRLkrrSGcBvaH1WkWHAbcA3gCci4r/APMA65JlAZqf1pved4Szghoi4g9x6ZG1gafJYGT+trJhSurhoGXEo8GBEPEKeRSXITd5XI49rcDYdIKV0eUScBewLPBYRt5Jv8gaSpwy9nylTyc6otcizaYyNiBHkG7i5ya0M+pNv/n6XUprcPSeldFdE/IzcMue2iHiCPAjnhGKd1cmDsV5JvuFs1IXk8Sh+UvG6EccAXwG2B56OiLvILYiWILduuRkYXhzLmIjYjjwjxxnAYRHxOPA+sHBRf15yK4DHi+3/HvhzRDxGPu5JwMrk6UjHMaWlRr1OB86JiB+Rr6sB5PM/FtiramrTzj73nSqllCJid/Lfgt8C20XEw+QEx9fIg7/uU2O611rbej4ivgtcDFweEc8DT5HP2+Lk925O8rXwWrHO8Ig4CPgTcEtxzT9L7lL1ZfK5m7diN9cAewEXR8R/gA+K7dQzrawkaQbYQkOS1GVSSp+Svz1ubXki32CeBnxGvmFdjHzz18h4Ax3leGBvclJl++L5ImCdWgM6ppR+Rx788Qryje725K4yPcg3pNt2ZHAppR+Rm9+PKPa7LfAueRaQjettsdCOD4D1yGOfPEhOKH0b2JQpU1tukFIaUiO+M8hja5xPHhdlK/LAmHORz+PWxfanx1VMmaWmpcVG3YquGd8hTyt6L3k63R3IN7nXASdX1b+XnET4UxHzhuT3d0ly4mMQxVgZhf2AS8g3v5uRj3VW8nWwWkrp/kbiBS4jv78Ti+clyDfS66aUHqxxfJ157jtdSukxcrLhTHLCYUdyC4irydfbJa2tWmNbV5MTin8tln+TfPwLkZNUO5NbuVSuczz5d+pf5HO9IzmB9BhwYFXdK4FfkRMi2wA/KB6SpE4W5RwrSpIkaeYXEeeRv/3fOKU0rLnRdF9F95lxwJMppVWaHY8kqWvYQkOSJEllt2bx/HxTo5AkdSkTGpIkSSqliNggIq4gdxOCxsdSkSSVmAkNSZIkldWywHbksWP2Tyn9s8nxSJK6kGNoSJIkSZKk0rGFhiRJkiRJKp2ezQ6gO1hggQVS//79mx2GJEmSJEmqMGLEiFEppQVrLTOhAfTv358HH5xmCndJkiRJktREEfFya8vsciJJkiRJkkrHhIYkSZIkSSodExqSJEmSJKl0TGhIkiRJkqTSMaEhSZIkSZJKx4SGJEmSJEkqHRMakiRJkiSpdExoSJIkSZKk0jGhIUmSJEmSSqdpCY2I2CkiromI1yNibESMiIjvVtUZFhGpxmP2qnqLRcS/IuKjiBgVEadFRO+uPSJJkiRJktRVejZx3wcALwG/AkYBWwIXR8QCKaVTK+rdDhxate74lh8iYlbgJmACsCswL3Bi8bxHJ8UuSZIkSZKaqJkJjW1SSqMqXt8WEYuSEx2VCY33U0r3trGdHYGVgGVTSi8BRMRnwNCIOCKl9FxHBy5JkiRJkpqraV1OqpIZLR4GFm1wU1sAD7QkMwpXkVtsbD590UmSJEmSpO6suw0Kuh7wbFXZtyJiXPG4KSJWq1q+IvB0ZUFKaQLwQrFMkiRJkiTNZLpNQiMiNgG2B06oKB4O/BLYDNgX+BJwZ0T0r6jTFxhTY5Oji2WSJEmSJGkm08wxNCYrEhQXA1enlM5rKU8p/aGi2p0RcQu5Ncb+xWNG9rkvOUlCv379GDZs2IxsTpIkSZIkdaGmJzQiYj7gBuBlYPe26qaU3oqI/wJrVBSPBuapUb0v8Ggb2zoLOAtgzTXXTAMHDmwscE23/gdf3+wQpjJyyFbNDkGSJEmS1KCmdjmJiN7AdUAvYOuU0rg6VkvFo8XTVI2VERG9gKWpGltDkiRJkiTNHJqW0IiInsA/geWAzVNK79SxzsLABsCIiuIbgLUiYsmKsm2B2YAbOy5iSZIkSZLUXTSzy8lfgS3Jg37OHxHzVyx7GFgBOJac9HiZPCDoIcAk4OSKupcDvwOujIjDyd1PTgIuTik918nHIEmSJEmSmqCZCY1vFc+n1Fi2FPAeEOSkxvzAR8AwYPuU0istFVNKn0XE5sBpwGXAeGAo8JtOi1ySJEmSJDVV0xIaKaX+dVTbss5tvUae8lWSJEmSJH0BNHVQUEmSJEmSpOlhQkOSJEmSJJWOCQ1JkiRJklQ6JjQkSZIkSVLpmNCQJEmSJEmlY0JDkiRJkiSVjgkNSZIkSZJUOiY0JEmSJElS6ZjQkCRJkiRJpWNCQ5IkSZIklY4JDUmSJEmSVDomNCRJkiRJUumY0JAkSZIkSaVjQkOSJEmSJJWOCQ1JkiRJklQ6JjQkSZIkSVLpmNCQJEmSJEmlY0JDkiRJkiSVjgkNSZIkSZJUOiY0JEmSJElS6ZjQkCRJkiRJpWNCQ5IkSZIklY4JDUmSJEmSVDomNCRJkiRJUumY0JAkSZIkSaVjQkOSJEmSJJWOCQ1JkiRJklQ6JjQkSZIkSVLpmNCQJEmSJEmlY0JDkiRJkiSVjgkNSZIkSZJUOiY0JEmSJElS6ZjQkCRJkiRJpWNCQ5IkSZIklY4JDUmSJEmSVDomNCRJkiRJUumY0JAkSZIkSaVjQkOSJEmSJJWOCQ1JkiRJklQ6JjQkSZIkSVLpmNCQJEmSJEmlY0JDkiRJkiSVjgkNSZIkSZJUOiY0JEmSJElS6ZjQkCRJkiRJpWNCQ5IkSZIklY4JDUmSJEmSVDomNCRJkiRJUumY0JAkSZIkSaVjQkOSJEmSJJWOCQ1JkiRJklQ6JjQkSZIkSVLpmNCQJEmSJEmlY0JDkiRJkiSVjgkNSZIkSZJUOiY0JEmSJElS6ZjQkCRJkiRJpWNCQ5IkSZIklY4JDUmSJEmSVDomNCRJkiRJUuk0LaERETtFxDUR8XpEjI2IERHx3Rr1fhgRz0XEp0WdTWrUWSwi/hURH0XEqIg4LSJ6d82RSJIkSZKkrtbMFhoHAGOBXwHbArcDF0fEfi0VigTHGcAFwBbAE8B1EbFqRZ1ZgZuAJYFdgV8COwFndc1hSJIkSZKkrtazifveJqU0quL1bRGxKDnRcWpRNhg4P6X0R4CIGA58BTgY2KOosyOwErBsSumlot5nwNCIOCKl9FynH4kkSZIkSepSTWuhUZXMaPEwsChARCwNLA9cVrHOJOCf5NYaLbYAHmhJZhSuAiYAm3ds1JIkSZIkqTvoboOCrgc8W/y8YvH8dFWdp4D5ImLBinpT1UkpTQBeqNiGJEmSJEmaiXSbhEYx2Of2wAlFUd/ieUxV1dFVy/vWqNNSr2+NckmSJEmSVHLNHENjsojoD1wMXJ1SOq+L9rkvsC9Av379GDZsWFfsVsCBAz5vdghT8b2XJEmSpPJpekIjIuYDbgBeBnavWNTSEmMepm6B0bdq+eiiTrW+wKOt7TeldBbFTChrrrlmGjhwYIORa3oNOvj6ZocwlZG7D2x2CJIkSZKkBjW1y0lE9AauA3oBW6eUxlUsbhkXo3ocjBWB91NK71bUm6pORPQClmba8TckSZIkSdJMoGkJjYjoSZ6xZDlg85TSO5XLU0ovkgcI3alinVmK1zdUVL0BWCsilqwo2xaYDbixc6KXJEmSJEnN1MwuJ38FtgR+CcwfEfNXLHs4pTQeGAxcGBEjgf8Ce5ETILtV1L0c+B1wZUQcTu5+chJwcUrpuc4+CEmSJEmS1PWamdD4VvF8So1lSwEjU0qXRMRcwG+Bw4EnyF1THm+pmFL6LCI2B04DLgPGA0OB33Rm8JIkSZIkqXmaltBIKfWvs97ZwNnt1HmNPOWrJEmSJEn6AmjqoKCSJEmSJEnTw4SGJEmSJEkqHRMakiRJkiSpdExoSJIkSZKk0jGhIUmSJEmSSseEhiRJkiRJKh0TGpIkSZIkqXRMaEiSJEmSpNIxoSFJkiRJkkrHhIYkSZIkSSodExqSJEmSJKl0TGhIkiRJkqTSMaEhSZIkSZJKx4SGJEmSJEkqHRMakiRJkiSpdExoSJIkSZKk0qk7oRERG0bEL6vKdo+IFyLivYg4NSJMkEiSJEmSpE7XSALiCOBrLS8iYnngXGACcD/wU2C/Do1OkiRJkiSphkYSGqsA91W83g34FFg3pbQFcDHw/Q6MTZIkSZIkqaZGEhp9gPcrXm8O3JJS+qB4PRxYuqMCkyRJkiRJak0jCY23KBIWETE/8FXg1orlcwGTOi40SZIkSZKk2no2UPcm4OcRMQYYSE5eXFOxfGXglQ6LTJIkSZIkqRWNJDQOA1YE/kweCPSAlNJrABExO7AjcEGHRyhJkiRJklSl7oRGSuldYKOI6AN8klL6rGJxABtjCw1JkiRJktQF6h5DIyJ+HxGrppQ+rEpmkFL6BPgcp22VJEmSJEldoJFBQQcDq7WxfFXgDzMUjSRJkiRJUh0aSWi0py95bA1JkiRJkqRO1eYYGhGxAbBhRdF2EdG/RtV5gd2AJzosMkmSJEmSpFa0NyjoJkzpRpKAnYpHLa8AB3RQXJIkSZIkSa1qL6FxMnAeeRaTF4H9gaur6iTg45TSex0cmyRJkiRJUk1tJjRSSh8AHwBExMbAk8X0rZIkSZIkSU3TyKCgzwIrtrYwIr4eEQvPeEiSJEmSJElta6/LSaXjgf7A11pZfjTwErDXDMYkSZIkSZLUpkZaaGwIXN/G8huAgTMUjSRJkiRJUh0aSWgsCLQ1fsZ7QL8ZC0eSJEmSJKl9jSQ03gZWa2P5l4FRMxaOJEmSJElS+xpJaFwH/DAiNqxeEBEDgX2KOpIkSZIkSZ2qkUFBjwC2Am6PiJuBx4ryAcA3gdeBP3RseJIkSZIkSdOqO6GRUnonItYGhgDfBr5VLPoQOA84NKX0dodHKEmSJEmSVKWRFhqklN4B9o6IH5AHCQV4N6WUOjwySZIkSZKkVjQyhkal2YBZgTEmMyRJkiRJUldrKKERERtExJ3AR8ArwAZF+QIRcWtEfKvNDUiSJEmSJHWAuhMaEbEBcCuwMHAOEC3LUkqjitd7d3SAkiRJkiRJ1RppoXEU8CSwKnB4jeXDgbU7IihJkiRJkqS2NJLQWBM4L6U0Hqg1bsbr5NYbkiRJkiRJnaqRhMYkaicyWiwKjJuxcCRJkiRJktrXSELjAWDbWgsiohewO3B3RwQlSZIkSZLUlkYSGscAAyPiAnL3E4AlImJr4A5gqaKOJEmSJElSp+pZb8WU0q0RsRvwV3JrDIBzybObjAF2Synd2+ERSpIkSZIkVak7oQGQUrosIq4DvgksT27h8TxwU0ppbCfEJ0mSJEmSNI2GEhoAKaVxwNWdEIskSZIkSVJdGhlDQ5IkSZIkqVtoNaEREZMi4vNiBpOW1xPreEyIiDci4oKIWLDrDkWSJEmSJH1RtNXl5EggAZ9XvW5PD2AxYEdgTuA7MxKgJEmSJElStVYTGimlwW29bk9EPA0cMl1RSZIkSZIktaHhQUEbcA0woRO3L0mSJEmSvqAaGhQ0IuaNiCER8XhEfFw8Ho+IYyJi3sq6KaVnUkqndGi0kiRJkiRJNJDQiIjFgIeBg4qi64tHAg4GHoqIRTs8QkmSJEmSpCqNdDk5FugHbJ1S+nflgojYAri8qLNXx4UnSZIkSZI0rUa6nGwOnFKdzABIKd0AnAps0VGBSZIkSZIktaaRhMbcwOttLH+tqCNJkiRJktSpGkloPAPsGBHTrBMRPYAdizp1i4hlI+LMiPhfREyMiGE16oyMiFT1eKtGvZUj4taIGBcRb0TEkUVckiRJkiRpJtPIGBr/B5wD3BoRJzMlebEi8Evg68A+De5/FWBL4F5g1jbqXUzu0tJiqulgI6IvcAvwJLAdsAxwAjlhc1iDMUmSJEmSpG6u7oRGSunciFgI+ANwZcWiAMYDh6aU/t7g/q9NKV0NEBGXAwu0Uu/NlNK9bWznx8AcwA4ppQ+BmyOiDzA4Iv5UlEmSJEmSpJlEIy00SCkNiYizgU2B/kXxSODmlNL7je48pTSp0XVasQVwU1XiYihwHLARcG0H7UeSJEmSJHUDdY2hERG9I+K9iPhNSum9lNKlKaXjisel05PMaNAPImJCRHwQEZdHxJJVy1cEnq4sSCm9AowrlkmSJEmSpJlIXS00UkrjImISMLaT46nlavIYG68BK5G7vNwZEQNSSh8UdfoCY2qsO7pYJkmSJEmSZiKNdDm5FtgGOL2TYqkppfTLipd3RsTdwCPA94GTp3e7EbEvsC9Av379GDZs2PQHqYYcOODzZocwFd97SZIkSSqfRhIaJwKXRMSVwBnAC8An1ZVSSm90UGw1pZQej4hngDUqikcD89So3rdYVms7ZwFnAay55ppp4MCBHRypWjPo4OubHcJURu4+sNkhSJIkSZIa1EhC43/F8yrkqVFb02P6w6lbKh4tnqZqrIyIWALoTdXYGpIkSZIkqfwaSWgcydRJhKaIiFXJyYuzKopvAH4TEXOnlD4qynYhtyAZ3sUhSpIkSZKkTlZ3QiOlNLijdx4RvYEti5eLAX0iYsfi9b+BjYE9gOuAN8iJjMOAV4DzKjZ1BvAL4MqIOA5YGhgMnFg1laskSZIkSZoJtJvQiIh+wCBgKeBd4PKU0qMdtP+FgH9WlbW8Xgp4tahzMjAv8B5wI3BoZaIipTQ6IjYBTiMPXjoGOImc1JAkSZIkSTOZNhMaEbEc8F9gfiCK4oMjYo+U0qUzuvOU0siK7bZmkzq39STwjRmNSZIkSZIkdX+ztLP8SKAP8CtgVeDbwOvASRHRXiJCkiRJkiSpU7TX5WQgcE5K6f+K109GxCTgKmAl4MnOC02SJEmSJKm29lpoLAiMqCp7kNxNZMFOiUiSJEmSJKkd7SU0ZgHGV5VNKJ57dHw4kiRJkiRJ7atn2tblImL9itfzFM+rRMSn1ZVTSnd3SGSSJEmSJEmtqCeh8fviUe0UIFW8juK1LTckSZIkSVKnai+h8f0uiUKSJEmSJKkBbSY0Ukrnd1UgkiRJkiRJ9WpvUFBJkiRJkqRux4SGJEmSJEkqHRMakiRJkiSpdExoSJIkSZKk0jGhIUmSJEmSSqfVhEZEvB8RO1a8/n1ErNo1YUmSJEmSJLWurRYacwFzVLweDKzWqdFIkiRJkiTVoa2Exkhgy4iYu6IsdW44kiRJkiRJ7WsroXEqsAswJiImkpMZF0bExDYen3dJ1JIkSZIk6QutZ2sLUkqnRsT/gE2BfsAPgOHAC10UmyRJkiRJUk2tJjQAUkrDyUkMImIf4JyU0sVdEZgkSZIkSVJr2kxoVEopOcWrJEmSJEnqFupOaLSIiM2BrYGliqKXgGtSSv/pyMAkSZIkSZJaU3dCIyJ6Af8kJzMCGFUs2gL4SURcC+ycUprQ4VFKkiRJkiRVaKQbyR+AbYATgIVSSgullBYCFgSOB7YFDu/4ECVJkiRJkqbWSEJjN+DClNJBKaWW1hmklN5LKf0WuBDYo6MDlCRJkiRJqtZIQmNR4O42lt8DLDJj4UiSJEmSJLWvkYTGG8C6bSxfB3hzxsKRJEmSJElqXyMJjUuAPSPiqIjo21IYEX0j4o/AnsDFHR2gJEmSJElStUambT0CWB04FDgkIt4tyhckz3pyI3Bkh0YnSZIkSZJUQ90JjZTSeGDLiNga2AroXywaCVybUvp3h0cnSZIkSZJUQyMtNABIKV0HXNcJsUiSJEmSJNWlkTE0JEmSJEmSugUTGpIkSZIkqXRMaEiSJEmSpNIxoSFJkiRJkkrHhIYkSZIkSSqduhIaEdE7It6LiN90dkCSJEmSJEntqSuhkVIaB0wCxnZuOJIkSZIkSe1rpMvJtcA2nRWIJEmSJElSvXo2UPdE4JKIuBI4A3gB+KS6UkrpjQ6KTZIkSZIkqaZGEhr/K55XAbZro16P6Q9HkiRJkiSpfY0kNI4EUmcFIkmSJEmSVK+6ExoppcGdGIckSZIkSVLdGhkUdLKImD0iFouIXh0dkCRJkiRJUnsaSmhExAYRcSfwEfAKsEFRvkBE3BoR3+qEGCVJkiRJkqZSd0IjIjYAbgUWBs4BomVZSmlU8Xrvjg5QkiRJkiSpWiMtNI4CngRWBQ6vsXw4sHZHBCVJkiRJktSWRhIaawLnpZTGU3u2k9fJrTckSZIkSZI6VSMJjUm0PW3rosC4GQtHkiRJkiSpfY0kNB4Atq21oJjtZHfg7o4ISpIkSZIkqS2NJDSOAQZGxAXk7icAS0TE1sAdwFJFHUmSJEmSpE7Vs96KKaVbI2I34K/k1hgA55JnNxkD7JZSurfDI5QkSZIkSapSd0IDIKV0WURcB3wLWI7cwuN54KaU0thOiE+SJEmSJGkaDSU0AFJK44CrOj4USZIkSZKk+jSc0IiIbwJbk8fMAHgJuD6l9J+ODEySJEmSJKk1dSc0ImJO4FJgC/K4GaOLRVsDP4+Im4CdUkofd3iUkiRJkiRJFRqZ5eTPwJbAscBCKaX5U0rzAwsBQ4DNizqSJEmSJEmdqpGExs7AuSmlw1JKo1oKU0qjUkq/A84r6kiSJEmSJHWqRhIavYAH21j+ADDrjIUjSZIkSZLUvkYSGrcDm7SxfJOijiRJkiRJUqdqJKHxU2CFiDg3Ir4cEbMXjy9HxN+B5Yo6kiRJkiRJnarVhEZEfBYRE1oe5OlZVwYGAQ8BHxePh4DvAasUdeoWEctGxJkR8b+ImBgRw2rUiYg4NCJejYhPIuKOiFi9Rr2VI+LWiBgXEW9ExJER0aOReCRJkiRJUjm0NW3rRUDq5P2vQp455V5aH3/jYOBw4DfA08ABwC0RsWpK6S2AiOgL3AI8CWwHLAOcQE7YHNaZByBJkiRJkrpeqwmNlNKgLtj/tSmlqwEi4nJggcqFETE7OaFxbErptKLsHmAk8HOmJCt+DMwB7JBS+hC4OSL6AIMj4k9FmSRJkiRJmkk0MoZGh0spTWqnyvpAH+CyinU+Bq4FtqiotwVwU1XiYig5ybFRx0QrSZIkSZK6i7a6nEwjImYBNgWWBvoCUVUlpZSO7aDYAFYEJgLPVZU/BexSVe+2qkBeiYhxxbJrOzAmSZIkSZLUZHUnNCJiTeCfwJeYNpHRIgEdmdDoC4xNKU2sKh8N9I6IXimlCUW9MTXWH10skyRJkiRJM5FGWmicSU4O/Ay4h9oJhNKIiH2BfQH69evHsGHDmhvQF8iBAz5vdghT8b2XJEmSpPJpJKGxMjA4pXRGZwVTw2hgrojoUdVKoy8wrmid0VJvnhrr9y2WTSOldBZwFsCaa66ZBg4c2GFBq22DDr6+2SFMZeTuA5sdgiRJkiSpQY0MCvoS0NVfrT8N9ACWrSpfsVhWWW/FygoRsQTQu6qeJEmSJEmaCTSS0DgW+FFEzNtJsdRyN/AhsFNLQUT0BrYBbqiodwOwWUTMXVG2C/AJMLwL4pQkSZIkSV2o7i4nKaV/RMSswHMR8S/gVfIMJFXV6p/lpEhObFm8XAzoExE7Fq//nVIaFxFDgMMjYjS5tcUB5ETMqRWbOgP4BXBlRBxHnoVlMHBi1VSukiRJkiRpJtDILCcDgCOA+YF9WqnW6CwnC5FnTqnU8nopYCQwhJzAOKTY94PAN1NKb0/eaUqjI2IT4DTyFK1jgJPISQ1JkiRJkjSTaWRQ0DOAuYGf0kGznKSURtL6FLAtdRJwdPFoq96TwDdmNCZJkiRJktT9NZLQ+ApwRBfPciJJkiRJkjSNRgYFfYWun+VEkiRJkiRpGo0kNIaQZznp21nBSJIkSZIk1aORLieLAh8BL0TE5XTALCeSJEmSJEnTo5GExlEVP3fULCeSJEmSJEkNayShsVSnRSFJkiRJktSAuhMaKaWXOzMQSZIkSZKkejUyKKgkSZIkSVK3UHcLjYh4iTxGRltSSmmZGQtJkiRJkiSpbY2MoTGcaRMaPYD+wPrAY8AjHRKVJEmSJElSGxoZQ2NQa8siYg3gBuCADohJkiRJkiSpTR0yhkZK6SHgbOBPHbE9SZIkSZKktnTkoKCvAat04PYkSZIkSZJq6pCERkTMCuwKvN0R25MkSZIkSWpLI7OcnNvKonmB9YB+wH4dEJMkSZIkSVKbGpnl5BtMO8tJAkYDdwBnppRu66jAJEmSJEmSWtPILCf9OzEOSZIkSZKkunXkoKCSJEmSJEldwoSGJEmSJEkqnTa7nETEJKYdN6MtKaXUyLgckiRJkiRJDWsv+XAu9SU01gC+MuPhSJIkSZIkta/NhEZKaZ+2lkfEl4EjyMmMD4GTOywySZIkSZKkVkxX95CIWAUYDHwbGAccAxyfUhrTYZFJkiRJkiS1oqGERkSsSE5k7Ah8AhwP/Cml9H7HhyZJkiRJklRbXQmNiFgO+AOwCzABOImcyHi3E2OTJEmSJEmqqb1ZTpYGfg/sBnwOnAYMSSm93QWxSZIkSZIk1dReC41ngFmAh4EhwBvAMhGxTGsrpJTu7rjwJEmSJEmSptVeQqNH8bwGcGk7dYM8xWuPdupJkiRJkiTNkPYSGt/vkijUsP4HX9/sEKYxcshWzQ5BkiRJkvQF0WZCI6V0flcFIkmSJEmSVK9Zmh2AJEmSJElSo0xoSJIkSZKk0jGhIUmSJEmSSseEhiRJkiRJKh0TGpIkSZIkqXRMaEiSJEmSpNIxoSFJkiRJkkrHhIYkSZIkSSodExqSJEmSJKl0TGhIkiRJkqTSMaEhSZIkSZJKx4SGJEmSJEkqHRMakiRJkiSpdExoSJIkSZKk0jGhIUmSJEmSSseEhiRJkiRJKh0TGpIkSZIkqXR6NjsAqSz6H3x9s0OYxsghWzU7BEmSJElqCltoSJIkSZKk0jGhIUmSJEmSSseEhiRJkiRJKh0TGpIkSZIkqXRMaEiSJEmSpNIxoSFJkiRJkkrHhIYkSZIkSSodExqSJEmSJKl0TGhIkiRJkqTSMaEhSZIkSZJKx4SGJEmSJEkqHRMakiRJkiSpdExoSJIkSZKk0un2CY2IGBQRqcbjxxV1IiIOjYhXI+KTiLgjIlZvYtiSJEmSJKkT9Wx2AA34BvBJxesXK34+GDgc+A3wNHAAcEtErJpSeqvrQpQkSZIkSV2hTAmNB1JKY6sLI2J2ckLj2JTSaUXZPcBI4OfAYV0ZpCRJkiRJ6nzdvstJHdYH+gCXtRSklD4GrgW2aFZQkiRJkiSp85QpofFCRHweEc9ExI8qylcEJgLPVdV/qlgmSZIkSZJmMmXocvImeXyM+4EewK7AGRHRO6V0EtAXGJtSmli13migd0T0SilNqN5oROwL7AvQr18/hg0b1omH0PEOHPB5s0OYRr3nsLvFXta4of7YJUmSJGlmEymlZsfQsIi4FNgUWBA4BPhNSmneqjr7AGcDs9VKaFRac80104MPPthJ0XaO/gdf3+wQpjFyyFZ11etusZc1bqg/dkmSJEkqo4gYkVJas9ayMnU5qXQ5MB/Qn9wSY66I6FFVpy8wrr1khiRJkiRJKp+yJjRSxfPT5K4oy1bVWbFYJkmSJEmSZjJlTWjsCIwCXgbuBj4EdmpZGBG9gW2AG5oSnSRJkiRJ6lTdflDQiLiCPCDo/8gtMXYpHr9IKU0CPo2IIcDhETGa3CrjAHKy5tTmRC1JkiRJkjpTt09oAM8AewNLAAE8CXwvpfSPijpDyAmMQ4D5gQeBb6aU3u7iWCVJkiRJUhfo9gmNlNKhwKHt1EnA0cVDkiRJkiTN5Mo6hoYkSZIkSfoCM6EhSZIkSZJKx4SGJEmSJEkqHRMakiRJkiSpdExoSJIkSZKk0jGhIUmSJEmSSseEhiRJkiRJKh0TGpIkSZIkqXRMaEiSJEmSpNIxoSFJkiRJkkrHhIYkSZIkSSodExqSJEmSJKl0TGhIkiRJkqTSMaEhSZIkSZJKx4SGJEmSJEkqHRMakiRJkiSpdExoSJIkSZKk0unZ7AAkda7+B1/f7BCmMXLIVs0OQZIkSVLJ2UJDkiRJkiSVjgkNSZIkSZJUOiY0JEmSJElS6ZjQkCRJkiRJpWNCQ5IkSZIklY4JDUmSJEmSVDomNCRJkiRJUumY0JAkSZIkSaVjQkOSJEmSJJWOCQ1JkiRJklQ6JjQkSZIkSVLpmNCQJEmSJEmlY0JDkiRJkiSVjgkNSZIkSZJUOiY0JEmSJElS6ZjQkCRJkiRJpWNCQ5IkSZIklY4JDUmSJEmSVDomNCRJkiRJUumY0JAkSZIkSaVjQkOSJEmSJJWOCQ1JkiRJklQ6JjQkSZIkSVLpmNCQJEmSJEmlY0JDkiRJkiSVjgkNSZIkSZJUOiY0JEmSJElS6ZjQkCRJkiRJpWNCQ5IkSZIklY4JDUmSJEmSVDomNCRJkiRJUumY0JAkSZIkSaVjQkOSJEmSJJWOCQ1JkiRJklQ6JjQkSZIkSVLpmNCQJEmSJEml07PZAUhSLf0Pvr7ZIUxj5JCt2q1T1rglSZKksrGFhiRJkiRJKh1baEiSbFkiSZKk0rGFhiRJkiRJKh0TGpIkSZIkqXRMaEiSJEmSpNKZacbQiIiVgVOB9YAxwDnAESmlic2MS5LUeRz7Q5Ik6YtrpkhoRERf4BbgSWA7YBngBHILlMOaGJokSdMwESNJkjTjZoqEBvBjYA5gh5TSh8DNEdEHGBwRfyrKJEmSJEnSTGJmSWhsAdxUlbgYChwHbARc25SoJEmayXS31iW2LJEk6YtrZklorAjcVlmQUnolIsYVy0xoSJKk0uluCSSoL4lU1rjLyvMt6YtqZpnlpC95INBqo4tlkiRJkiRpJhIppWbHMMMi4jPgNymlk6vKXwMuSCkdWmOdfYF9i5crAM90dpzd2ALAqGYHMR2Mu2sZd9cra+zG3bWMu2sZd9cy7q5X1tiNu2sZd9cqa9wdZcmU0oK1FswsXU5GA/PUKO9bLJtGSuks4KzODKosIuLBlNKazY6jUcbdtYy765U1duPuWsbdtYy7axl31ytr7MbdtYy7a5U17q4ws3Q5eZo8VsZkEbEE0LtYJkmSJEmSZiIzS0LjBmCziJi7omwX4BNgeHNCkiRJkiRJnWVmSWicAYwHroyITYvxMQYDJ1ZN5araytr1xri7lnF3vbLGbtxdy7i7lnF3LePuemWN3bi7lnF3rbLG3elmikFBASJiZeA0YD3yjCfnAINTShObGZckSZIkSep4M01CQ5IkSZIkfXHMLF1O1KCIWDkibo2IcRHxRkQcGRE9mh1XeyJi2Yg4MyL+FxETI2JYs2NqT0TsFBHXRMTrETE2IkZExHebHVc9ImLHiLg7It6LiE8j4pmIOCwiejU7tnpFxGLFeU8RMVez42lLRAwq4qx+/LjZsbUnInpGxMER8VxEjI+I1yLipGbH1ZaIGNbK+U4RsV6z42tLROwaEQ8V1/brEXFBRCza7LjaExHbF3+/x0fESxFxQLNjqlbP50xkh0bEqxHxSUTcERGrd320U8VUT9w/jYjri7/pKSIGdnmg08bUZtwRsUhE/DkiHi2u91cj4vzucL3XEXuviLgsIl4srpN3I+KGiPhqk0Juiauh/6Ui4qTiejm+i0JsLY56rvGRNf6ev9WEcCtjqut8R8SAiLguIj6IiI8i4v5mXit1XN8D2/gMvalJYdd7nSwSEX+PKf+bPxwRuzch3MqY6ol73og4NyLeL+K+ISKWbUK43crMMm2rGhARfYFbgCeB7YBlgBPICa7DmhhaPVYBtgTuBWZtciz1OgB4CfgVef7oLYGLI2KBlNKpTY2sffMDtwF/JnflWps8Ps3CwM+bFlVj/gyMBeZsdiAN+AZ5UOMWLzYrkAacR477CPLsUksAKzczoDr8FOhTVXYk8BXgga4Ppz4RsS1wCfAX4DfAIsBRwPUR8dWU0qRmxteaiPgacCVwLvBrYB3guIiYlFI6uZmxVannc+Zg4HDy+X+a/Hf+lohYNaXUrJuneuL+HpCAm4DuklhvL+6vAt8mdyW+D+hH/hy6uzjfY7sozlrai70H+XwfC7xA/nvzK+C2iPhKSqlZf9vr/l8qcpfuHwDdYUy6euO+GKj8/2pCZwZVh3bjLhKidwJXkyc2AFgLmKML4mtNe3E/RO7qX+lLwKXkCRuapc24I2IW4Bry/7gHAW8BOwIXRsQnKaUruzDWSvVc35cCqwK/BD4g37fdGhEDvtDjRqaUfHzBHsAhwGigT0XZQcC4yrLu+ABmqfj5cmBYs2OqI+YFapRdDLzU7Nim83iOJic3otmx1BHrhsD75JunBMzV7JjaiXdQGeKsEffmwGfAys2OZQaPo1dxvZze7FjaiXMoMKKqbNvi2lmp2fG1EfdNwJ1VZScU57xXs+OriKnNzxlgdvI/kr+vKJsTeBc4qrvGXVmH/A9xAgaW4HzPC/SsKlu+iH+v7hx7K+vMRR7I/oAyxA3cCvwRGAkc393Pd3eIczrjvhe4uNmxTu91UlHvN8BEYNHuGjewYvH3Y5uq8oeAS7tx3OsVcW9SUdaPfP/262ZfL8182OXki2kL4KY0dSZvKDkLvFFzQqpP6qbfPLYlpTSqRvHDQNOby06n98g3ft1a5C5Up5K/ca/1Hqjj7A3cllJ6stmBzKDNgb7k1g/d2azkG+pKY4rn6NpQGrI6cHNV2X/I57zbdPGp43NmffI37ZdVrPMxcC3587Up6vl87I6foe3FlFIak1L6vKrsWfI/8U39HJ3O8/kx8ClN/BytN+6I2JF88zekcyOqT3e8fuvRXtxFK5h1mLpVSdNN5/n+LjA8pfRGR8dTrzribmn9UOtztGmfoXXEvTr5y6NhFeu8DTwKbNVpgZWACY0vphXJTWQnSym9Qv7nYMWmRPTFsx7wbLODqFdE9IiI3hGxAfAL8jfY3X1E4R8Ds5Gb5ZfNCxHxeeQxS37U7GDqsA7wbEScFhEfRh6b58ru0Me9QbsCr5Gb/XZn5wJfj4jvRUSfiFie3OWkuyeVZmfapt8tr1fq4lhmxIrkbyCfqyp/Cj9Du0RErAb0piSfo5H1jIiFgT+Rr59unTiNiDnILagOLhJ2ZfKDiJhQjEVxeUQs2eyA2rFO8dw38lgxn0fECxHxg6ZG1aDis+grdPNrG3ic3H3tyIhYrvgcHQR8DTijqZG1bXZgYpp2Bs8JlOsztMOZ0Phi6suUb/MqjS6WqRNFxCbA9uR/FMri4+JxJzCc3KSw24qI+clNZA9IKX3W7Hga8Ca5X/6ewDbkJqhnRMSvmhpV+xYmd5dZnZwU+D653/u/IqI7txiYLCJ6k7ttXNbdk3UppevJ5/ss8jdMz5D76n+niWHV43lyn/BKaxfP83VxLDOiLzC2xj+Vo4HeUaJBk8uo6P9+CjmhdE2Tw6nXb8nfrL4J7AVsmVJ6ubkhtesQcrwXNjuQBl1NHh9pE/L/KusBd0bEPE2Nqm0LF88XABcB3wRuBM6JiC2bFlXjdiVf51c0O5C2FJ/xW5Dvg58lf46eBeydUrqtmbG143lg9ogY0FJQJB5XpVyfoR3OQUGlLhQR/cnjZ1ydUjqvudE0ZH3yt2FrA78HTiP/w9BdHQ3cm1L6d7MDaURK6SbyOAMtboiI2YHDIuKUbtzcNorHdiml9wAi4k1y8usb5D7Y3d025HEQuvs3S0TExuRvkU4hD7zWMkjivyJi0xo32t3FGeQE3Q/J/YPXJg+mCdBdr211P8eSb1I3KlHC+jzyYOyLkD87r4uIDbtri6qIWIo89tTG3T3BWy2l9MuKl3dGxN3AI+RE+8nNiKkOLYn/c1JKfyp+vj0iViInlsryv8yuwH9SSu83O5C2FEnRC8iDgu4CvEMejPNvEfFeSunGZsbXhpvIkwycGRHfJw/UOwSYB/i8rRVndiY0vphGky/+an2LZeoEETEf+ebjZaCpU0M1KqX0UPHjXRExCjg/Ik5IKb3QzLhqiYhVyGM6bBgR8xbFvYvneSJiYkrpk5ord0+XAzsD/em+s52MBl5sSWYU7iI3g1yZciQ0dgWeTyk92OxA6nACcE1K6bctBRHxCLkr4XbkmUS6o3OBLwOnk78NG0f+5vpU8ijzZTEamCsielQlj/oC41JKzZ5RYaYVET8lf+v+3ZTSfc2Op14pz3zzFkBE3AA8QZ4p53vNjKsNQ8j/rzxT8Tk6CzBb8fqDsiQ6UkqPR8QzwBrNjqUNLf97315Vfht5VpxuLyK+TO72cHSzY6nD1sVj+ZRSS9fBYRGxBLlLWLdMaKSUJkTEruQvXlqGDriLnJz5RtMC6wbscvLF9DRV/XyLX+LeVI2toY5RNGe/jjwI2NYppXFNDmlGtCQ3lmpqFK1bjjzg0z3kfxJGM2UcjdfoZoNu1SFVPXdHT1F7IK2gBN+8F02Rt6AErTMKK5K/cZwspfQMearfZZoRUD1SShNTSj8HFgRWI7csubdYfG+rK3Y/T5O7+CxbVT7N+FTqOBHxHfLf74NSSpc2O57pVQxw+hiwdLNjacMKwA5M+QwdTZ6K++fFz4s1L7Tpkuj+n6Ew7edoKT5DC7uSP4OubnYgdViRnHyuHgfpYbrxZyhASul+8mfPisCyKaWvAwtRrs/QDmdC44vpBmCziJi7omwX8h+i4c0JaeYVET2Bf5JvtDdPKb3T5JBm1NeK55eaGkXr7gI2rnocVyzbEvhzk+KaXjuSZ2npzv2trwMGRMQCFWUbkhNLjzYnpIZ8mzyAbFkSGi9T9W1j0TR5DvKUhd1aSml0SumxlNJYcvP7u1NKZUoE3E1u6rtTS0GRtN6G/PmqDhYRA8ljC5yaUjq+udHMmKIb4Rp0389QgH2Y9nP0bfLMPhuTpyguhYhYlXzzN6LZsbThbnKiqPpb9k0ox2co5ITGtcXf9e7uZfJ4RytUlX+VcnyGppTSMymlFyJiOWBT4G/NjquZ7HLyxXQGeaaKKyPiOPK3BIOBE6umcu12in8aWwZIWgzoU0wrBvDvbtry4a/kmH8JzF8MWNni4ZTS+OaE1b6IuJHc7/cJ8qjsXwMOJM/T3e26m8DkaXKHVZYVY5cA3NmdP2wj4grgfuB/5G+Adykev+jG42dA7j7wC+DaiDgGmJucRLolpXRXUyOrz67Aoymlp9qt2T2cAZwUEW8wZQyN35P/Eeu2fa0jYl1gA3Lrkj7k6f02K8q6jXo+ZyJiCHB4RIwmt8o4gPwlUdNagNUZ95rk7mtLFOUbFYnIkc3qbtVe3MCSwFXk83xpcR21eLeZn0V1xL4dufXXjcAbTBlDYxHgxK6Ndoo6rpVproWI+BR4NaU0rGuinFYd53tjYA9ykv0NciLjMOAV8jgmTVHn7+aRwJ8iYgzwAHmQ5w2Bjbo63hb1/s9d/E72p5t0j6njOvk3+Zq4qjjv75KnPd0Z+FkXhztZndfJ4eS/haOAAeSB5IemlKqnRP9iSSn5+AI+yP3abyO3yniTPCNEj2bHVUfc/ZnSdLD60b/Z8bUS88iyxVwR+x/J01uNJc+M8xCwHzBrs2Nr8DgGFed7rmbH0k6cx5BnrBhX/G6OAPZsdlx1xr4s+Z+Ej8nfNJ0H9G12XHXEvQB5VPaDmx1LAzEH8BNy4utj4HXgUmDpZsfWTtxfJf+jPpbcwuF6YECz46oRZ7ufM8V78DtyN7ZPyDNAfaUEcZ/XyvLzumvcFX+/u1Xcdcb+leI6fwsYX/w/cCmwSneOu5V1RgLHd+e4yV3ZbiXfoH5WnPfzgEW7c9wV9Q4gt9yZQO6WtENJ4j6Z/D/ibM2Mt5G4yf+z/JOc+BpLbgnzIyC6edwnkz/zx5NnPfkt0LPZ57zZjyhOjiRJkiRJUmk4hoYkSZIkSSodExqSJEmSJKl0TGhIkiRJkqTSMaEhSZIkSZJKx4SGJEmSJEkqHRMakiRJkiSpdExoSJKkukTEeRHxebPjaERE7BcRz0XEZxExptnxNCoi+kdEiohBzY5FkqTuxoSGJEndREQMKm5eP4+IZWssPyciUjNiK6OIGAj8H/AwsA+wbzv1V4yIiyLixYj4NCLeioh7ImJIRMzV+RFLkqRG9Gx2AJIkaRo9gN8D32t2ICW3afG8b0ppTFsVI2IdYBjwDnA+8AqwMLAa8HPgDGBsZwXahpeBOYDPmrBvSZK6NRMakiR1Pw8Du0XE0SmlZ5odTFeLiDlTSh93wKYWAmgvmVE4HPgUWCul9E5VPPMA4zsgHiJiViBSShPqqZ9SSkVckiSpil1OJEnqfoaQv5H/Q3sVI2JkRJxXo/y8iBhZ8bplLIbDIuL7EfFMRHwSEXdHxGpFnT0j4qmiu8VDEbFWK/tcPCKujoiPIuK9iDi9VpeMiNgkIm4r6n0cEcMj4utVdQYXcQ2IiHMjYhTwWjvHPHtEHFMc+4Ti+ZiImK2iTgJ+2PJz8RjcxmaXBZ6uTmYApJQ+SClNlVSIiGUi4uKIeDcixkfE4xHxw6o6A4v9fi8ifle8H58CXym6FR1f49jmiIgPI+IfxeuaY2hERL+I+EtEvFLs/7UinsUq6sxa7Pfpos5bEXFWRMxXta3VI+K6iHi7eO9fj4grImLxNs6XJElNZwsNSZK6nzeAM4H9IuKolNKTHbjtHYA+xfZ7AgcDN0TE74FDgLPIXV4OBi6PiGVSSpUDgQZwA/As8FtgTeDHwJLAlpMrRewMXAIMJ7d+CGAQcGtEbJpSuqMqrkvIiYw/AK2OVxERAVwJbAFcCNwDrF/EPgDYpqi6J/ADYGDxM8D/2jgvI4ENI2LNlNKDbdQjIpYv9vs+cBIwuojnrIiYP6U0pGqVg8hfIv0F+Bx4E7gN2DkiflO0wmixNTA3cHEb++8H3AcsCpwDPAosCGxFTsy8XpynK4BvAn8rjn1pYD9g7YhYN6X0aUQsCNxSHMOJwKhiu5sBi9FOckmSpGYyoSFJUvc0hDyI5WBg5w7c7tLAsimlUQCRZ/74K/AnYPmU0ntV5ZsCN1asPwvwUEppr5aCiHgTODQiNksp3RQRcwKnA0NTSrtX1DsDeAw4FvhaVVwvANtX3dzXshU5eTAkpXRIUfbXIoZfR8SWKaV/p5QuLAYFHZhSurCO8zKkONb7I+Ih4E5yMubmGt1fTiXf+K9Rsez0iLgYODwiTk8pfVBRvy+wYkrpo5aCou7fgQ2KfbX4brHtm9uI9VhyAukbKaXbK8qPKhIZALuSkzvfSilN3lZE3AbcRE7ynE1OBs0PbJFSeqBiW39sY/+SJHULdjmRJKkbSim9RU4o7BgRAzpw01e0JDMK9xTPV7UkM6rKl66xjZNbeb118fxNYD7gwohYoOUBzEluDbBuRPSu2sbpdSQzKvdR3V3jT1XLG5JSGkZOsvwLWAHYv/j53Yj4dUu9iOhLPr5/AnNUHd8NQG9g3arN/6MymVH4F7n7yXcrtt2H3Mrln1WtYqioMwu5lc3NVcmMluNoOYe7AC8CD1fF+BDwAfCNot6Y4nnbyi47kiSVgQkNSZK6rz8B48itNDrKK1Wvx7RT3rfGNqYaqDSl9C65y8JSRdHyxfO/gXerHj8i//8xf9U2X2g38qw/8G5V8qUlhlEVMTQspXRfSuk7wLzAqsAB5PPw54oxLJYjd5/5HdMe2wVFnYWqNj3NsRUtOP5NTli1tJj9NjAbbXQ3IXctmYfc0qUty5OTUdUxvlus3xLjHcBQ4DDgvYi4MSL2i4jq90eSpG7HLieSJHVTKaV3IuIvwG8iYvXWqrVS3qOV8okNlkcr5W1p+cLkB0ybKGnxbtXrT6ZjP50ipTQReAJ4IiKuAZ4jT6F7HlOO7f+Aa1vZxBNVr1s7tovJrS02IXcD+S75fP13emOvMAvwNHnMjFpGw+QWHd+NiD+Tu/NsSh4X5PcRsXFK6fEOiEWSpE5hQkOSpO7tT8BPgSOYNgkA+cZ03hrl/TsvJFYgTy0LQDGwZF/gpaLo+eJ5VErplg7e90hgs2LwzcmtNIruFAtUxNAhUkovRMT75IEyYUpri4kdcGzXAx+SEwoPkRMbJ7bT9ebdYp32uiE9D6wD3JZSmtReICmlh8jdUf4YedabEcCBwPfbPQpJkprELieSJHVjxU37/wHbAmvUqPI8sF7VlKVrkAd77Cz7t/L6+uL5JnJXjcNqjctQJECmV0uriAOqyn9TtbwhkaeYneb/oohYh9w95mmY3LXlVuAHEbFkjfp1H1sxFey/gO2BPchfNLXV3YQiOXEF8M2I2LjG/lta1AwlJ3j2r1GnR8vUrRHRt2KdFk+RW5XMW++xSJLUDLbQkCSp+zse+DnwlRrLzgR2Av4TEUPJU23uCzxOHiuho00C1oiIK8g39muSv8X/T0rpRoCU0kcRsS95KtbHIuIi8lS0iwEbFduZ5ma8Tv8mz7pyaEQsTp6+dF3yrB3XpZRumM7tngL0iYiryF1GEnkcjb3IN/dHVdT9KblbyKMRcQ55Ctv5gNXJyYnZG9jvJcU+jgCeSik9Wsc6h5IHJr2p2P+jxf63JI+FMRy4CPgOcEJEfL0o+xxYpij/PbkLzV7k6YH/RU6O9STPkDJ3EZskSd2WCQ1Jkrq5lNLoiDiZfBNavezWiPgF8Gvy2AdPkMdi2BMY2BnhkKdN/QtwHPAZcFax/8q4/hkRr5Nvvn9Jnv3jLeB+4Nzp3nlKKSJ2IJ+L3cjH+iZ5KtMjp3e75PhbxrPYC5ijiPda8hSxk8eSSCk9GxFfLWLYlTzA5ihyy4YDG9zvLcA7xTbabJ1Rsf+3ImJtchJke2Af4G1y0uK5ok6KiB3JY2gMAjYHJgAvA5cCtxWbG05OSn0HWJg8CO0T5Cl0r27wWCRJ6lJR3wxpkiRJkiRJ3YdjaEiSJEmSpNIxoSFJkiRJkkrHhIYkSZIkSSodExqSJEmSJKl0TGhIkiRJkqTSMaEhSZIkSZJKx4SGJEmSJEkqHRMakiRJkiSpdExoSJIkSZKk0jGhIUmSJEmSSseEhiRJkiRJKh0TGpIkSZIkqXRMaEiSJEmSpNIxoSFJkiRJkkrHhIYkqduIiFQ8PoqIBVqpM6ioc05Xx1ePiBhWxNe/2bF0tIjoFxEXRsTbETGxOM7t21lncMX72vKYEBGvFNv6chfE3XLNDO7sfal1Fe9//zbqtLxXw7ouMklSWZnQkCR1R3MBv212EJrG34DdgTeBi4HzgVfqXPeFov75wDXApGJb90fEVh0favNFRH9vziVJ6jw9mx2AJElVEjAe+FlEnJBSeqvZAQkiohewOTASWCOlNKnBTdyVUhpUsb1Zgb8C+wBnRcRSKaUJHRRutX8B9wKjOmn7kiSpCWyhIUnqbiYBZwBzAIc0ORZNsTDQA3h5OpIZ00gpfQbsD4wFFgXWmtFttrGvD1JKT6eUTGhIkjQTMaEhSeqOhgDjgB9FxOL1rBAR5xXN+we2snxkRKSqsoHFOudFxPwRcWZEvBER4yLi/ojYvKLuDhFxT0SMjYj3IuJvETFvOzHtEREjiu29GxEXtHU8EbF5RFxf1B0fES9GxIkRMX9bxxsRm0XE7RExpihrM66q+O6MiA8i4pOIeCwifhcRc1TVS8DLxcuNKsZCGFbPflqTUvoYeLZ4+aViX5PHu4iI5SNiaDFmx6TK8ToiYouI+E9EvB8Rn0bEsxHxp4joW+M4Wx1DI7LvRsRtETG62NZTxf5714o7ImaNiB9HxF3FOf8kIp6PiL9HxFeLOoOBl4pVKs9ZiojzKrb1pYg4LSKeiYiPi+09Xby/a9ZzHqvO2QoRcUVxjY6LiLsjYps21l2i2P8LxbG/HxHXRcT6NepW/r4sHBHnRMRrEfF5ROxfT6zTq3ifdo2I4RHxVvH78Vrx+rBW1unS3ydJUtczoSFJ6nZSSm8DpwGzAb/rgl32JXdJ2Aq4E3gAWBO4NiI2Lm7WLgM+BW4kd4nZG7gqIqKVbf4auIDcAuFq4CNgT+C+iPhSdeWIGALcAGwKPEMeZ+Jz4FfFOv1a2c9uxXpzFs8PkLvttCkizgT+URznf4HryK0wjgKGRcScFdXPB64ofn6bKWNh3Njefuowd/E8vqp8BfKxrA3cDtwMfFbEfgjwb+AbwMPAVeQWPb8hj8mxSD07johZgIvI44GsBTxSbHdO4A/A7TWSO3MCtwCnA18hXzdXk7uz7E5+jym2VeucnQ/cVWxrceAh4Gfk/8luBG4lXyt7kLv4NGIZ4L4irv+Qz9+6wDURsU+N418PeLTY/2fA9cDjwGbAHRGxSyv7WbDY9lbAPeTrblyDsTZqCHAJ+XgeJ5/bp4FlgcHVlbv690mS1CQpJR8+fPjw4aNbPMg3Dp8XP88PfAhMAJasqDOoqHdO1brnFeUDW9n2yPyxN1XZwGKdRL5Z6lWx7IdF+bPAaGC9imXzAk/V2h8wrCj/DNiyorxnRYzXVa2zU1H+BLBcRXkARxTLhrZyvAnYpcHz/J1ivTeq9jcPOaGTgJOr1ulflA9rcF+Di/XOq7FsZfJNZgKWrXp/E3Aq0KNqnbWAieSb/nUrymcjJ50ScFXVOi3bHFxV/puifDiwSEV5L+CcYtmQqnVayu8AFqpa1g9Yp95zVnFuTquxrB+wSp3nuPKcnQ/0rFi2RXEtjgOWqCjvU7z/E4E9q7a3JvB+cY4XbOX35Upg9un4/U5A/zqOZVhF2ezkZOKHwDJV9Wdh2t/BLv198uHDhw8fzXvYQkOS1C2llN4DTgFmBQ7v5N19CPw8TT0o5bnkb92XA/6SUrqnIrYx5HE+ADZqZZuXpZT+XbFOy7fDY4EtI2KpirotrVB2Syk9V7FOIt/0PgLsGLWnsr0+pXRpewdY5RfF8+Cq/X0A/IR8U/fD1rpczKiImDMiNiHfFPcAbkkpPV9V7V3gtymliVXlPyffxJ6aUrq3Ivbx5JYG44Bto51pcyOiJ3BQUX/XlNKbFduaAOwHvAXsW7TkICIWJd9wfwx8J6X0TuU2U0pvp5Tua/8MTLZg8Xxz9YJiW080sC3I19b+xbXWsp0bgEvJLVh+UFF3b2AR4JSU0j+q9v0g8EfybEN71NjPeGC/lNKnDcY3veYmJ6xeSCm9ULkgpTQppTSsqn5X/z5JkprEhIYkqTs7ARgD7BURy3TifkYUCZTJihvplnEj/lNjnZYbq0Vb2ebQ6oKU0mhyt4IANgCIiIWALwPPp5QerbFOIncJ6QF8tcZ+rmll/zVFnl1k3eLlRTX29zi5G0dvYI1Gtt2OvVrGkCDfeN9C7lbyIFO6aVS6JaVUqxvD14vnWrG/S9X5bcMawALA3ZXJjIptfQKMIHdHWq4oHkh+H/5d7GtGjSiej4mILSNithnc3n+Ka6zaJcXz1yvKvlU8X0FtdxbPa9dY9lBK6fXpiG+6FOf6FWD1iDimKhk4la7+fZIkNZfTtkqSuq2U0piIOIH8bfEfgO910q5ea6V8bBvLW5a1dhP6civlI4vnlkRI/+J52agatLSGWt8ov9LOOtXmJ3epGJXyoJy1vES+4V+swW235QWKsSPIXSDeJt8035xqz5rS2nG1nLeRrSxvGYizvdj7F8+b1nnenwGWKF4/20bdRpwPbEIet+F6YHxEjCC32Dg3pdToe1vvNQdTjv+u1oeBATrmmmtESzDV78le5CThIcAhEfEaudvPFeQuRi3XUP/iuat+nyRJTWRCQ5LU3Z1Cnt5zt4g4Zga201arxPamIZ3haUrb0BLXW8BN7dStdcPaVc3+Z9RdKaVBDdTv7ONqOe/Pk7+xb8t77SyfLkUroN0j4jhgW/Igp+sC6wMHR8ROKaVrO2PfTDn+y8ldaFrzdI2y6X1vxpFb/szZRp2WZVPFlFIaFhHLAVuSBy3diJwI2g0YHhHfTHkq4C/K75MkCRMakqRuLqX0UUT8CTiO3P+9tZk1Wsa/mKt6QTFeQl0zX3SgJYH/tVIOeUBGmNL6Y1SDN/zT6z3yuVogIuZspZVG/+K5y7oVNOANYClyjLXGmOhfPLcXe8t5f7qB8/5q8bxcm7UalFL6H/laOaoYt2R/4GjyOC2NJDSWbKf8jYqy18hdfoaklEZMu0qneA1YHlia2u8dxbKWulNJKX1EHg/kUoCIWI3camMj8tgmZ9P1v0+SpCZyDA1JUhmcRu6isDOwait1WsZBWKHGso3p+iT+ztUFETEvU6bi/C9ASuk18rfgK0fE8p0dVPEtdstgmrvXiHEVcneTceQpRbublrEdasW+APn8JqZ0b2nNA8AHwEYRMV+d+x5GnhVky1YGlKzWkmSr+9pLKY1LKR1Dnlln0YjoW++6wLeKa6zarsVz5TlpGYj02w1sf0YNL563qbWwGHx16+LlHe1trEgEtQzOu2pR1qW/T5Kk5jKhIUnq9orBIYeQ+9f/pJVqw4rnnxYDAwIQEcuSp//sartExGYVcfQETiTP2HBDSunFirp/JH8mXxERX6neUETMHxE/7MDYWs7HH4rz07KfuYG/kM/z2a0MytlsfyF3AdovItZpKYyIXuTE15zANSmlkW1tpJgV5U/k9+NftQadjYjFImLPinXeAC4gtwK6PCIWrKq/UGVM5FlyPgOWiYgeNba/Z0QMqFG+Hnkw0g/IU6fWay7gxOJaa9nWZsB3yV0pzq2oeybwDnBQRPyoOr6I6BkRm0VEawnE6XEaeZre70fEVImUIuY/kVu/vErFYKUR8aWI+H5EzFm1Tg+mDG76asWirv59kiQ1iV1OJEllcQbwG1qfVWQYcBt5HIInIuK/wDzAOuSZC2an9Sb5neEs4IaIuIPcemRtcnP6t4CfVlZMKV1ctIw4FHgwIh4hD6IZwDLAauRBSM/uiMBSSpdHxFnAvsBjEXEruUXGQPJUovczZerLbiWldH9EHAYcQx7Qchg5cfA18qCdz9N60qvaEGBF8iwrT0XEw+RBRWclt/RZmdwVpHJa018WyzYCXoqIO8mJhyXJLVtOB+4rYp0QETeSWyQ8GhEPkVtt/Del9HfgO8AFEfFSsZ+Pi2P4WrGvwyqnYK3DRcAOwMCIuJfczWoj8nX085TS5DEjigF3tyN3aTkDOCwiHgfeBxYujmVecguOxxuIoVUppf9FxH7kpNSVEfE/4Eny+V6XPJDr+8BORcKpxXzkZMxfikFTXyX/Pq9D/nvwPHBOxX669PdJktQ8ttCQJJVCSulT8rgCrS1PwPbkb4E/A7Yg3yAdSY3uCV3geGBvclJl++L5ImCdWq0HUkq/I998XkG+odye3FWmB/kmeduODC6l9CPyjfyIYr/bAu8ChwMbtzEDStOllI4FtiInsdYk38SPJ5/zdWpNw9qyatV2JqWUvgdsR56at3+xrQ3ILRr+TH4PK9f5iPy+/JI8DsQG5HO3IPn9vaBqn/uQEyLzkwew/AH5fENusXMaeWrir5ETHEsAVwMbpZROa/dkTO15YD1ycmRz8g3//cD2KaUzqyunlO4FBpBbRnwAbEi+7pYkdw8ZRJ5et8OklM4o4rqIKQmTLYr9nwisllK6r2q1F4ADgVvJv9Pbk8/hO+TE21oppTFV++nS3ydJUnNE/v9PkiRp5hQRPwH+ChyUUvpzs+PpaBExCPg7cERKaXBzo5EkqevYQkOSJM3s1iyen29qFJIkqUM5hoYkSZopRcQh5K4XG5K7J/ynuRFJkqSOZAsNSZI0s9oMWIs8HsQW3XlcEEmS1DjH0JAkSZIkSaVjCw1JkiRJklQ6jqEBLLDAAql///7NDkOSJEmSJFUYMWLEqJTSgrWWmdAA+vfvz4MPPtjsMCRJkiRJUoWIeLm1ZXY5kSRJkiRJpWNCQ5IkSZIklY4JDUmSJEmSVDomNCRJkiRJUumY0JAkSZIkSaVjQkOSJEmSJJWOCQ1JkiRJklQ6JjQkSZIkSVLpmNCQJEmSJEmlY0JDkiRJkiSVjgkNSZIkSZJUOiY0JEmSJElS6ZjQkCRJkiRJpWNCQ5IkSZIklY4JDUmSJEmSVDomNCRJkiRJUun0bHYAmj79D76+2SE01cghWzU7BEmSJElSE9lCQ5IkSZIklY4JDUmSJEmSVDomNCRJkiRJUumY0JAkSZIkSaVjQkOSJEmSJJWOCQ1JkiRJklQ6JjQkSZIkSVLpmNCQJEmSJEmlY0JDkiRJkiSVjgkNSZIkSZJUOiY0JEmSJElS6ZjQkCRJkiRJpWNCQ5IkSZIklY4JDUmSJEmSVDomNCRJkiRJUumY0JAkSZIkSaVjQkOSJEmSJJWOCQ1JkiRJklQ6JjQkSZIkSVLpmNCQJEmSJEmlY0JDkiRJkiSVjgkNSZIkSZJUOiY0JEmSJElS6ZjQkCRJkiRJpdO0hEZE7BgRd0fEexHxaUQ8ExGHRUSvijoREYdGxKsR8UlE3BERq9fY1soRcWtEjIuINyLiyIjo0aUHJEmSJEmSukzPJu57fuA24M/AGGBtYDCwMPDzos7BwOHAb4CngQOAWyJi1ZTSWwAR0Re4BXgS2A5YBjiBnKw5rGsORZIkSZIkdaWmJTRSSmdWFd0eEX2An0XEfsBs5ITGsSml0wAi4h5gJDnh0ZKs+DEwB7BDSulD4OZiO4Mj4k9FmSRJkiRJmol0tzE03gNaupysD/QBLmtZmFL6GLgW2KJinS2Am6oSF0PJSY6NOjVaSZIkSZLUFE1PaEREj4joHREbAL8ATk8pJWBFYCLwXNUqTxXLWqxI7o4yWUrpFWBcVT1JkiRJkjSTaOYYGi0+JncvAbiAPF4GQF9gbEppYlX90UDviOiVUppQ1BtTY7uji2WSJEmSJGkm0x0SGusDvcmDgv4eOA34aWfvNCL2BfYF6NevH8OGDevsXXaoAwd83uwQmqps75ckSZIkqWM1PaGRUnqo+PGuiBgFnB8RJ5BbWMwVET2qWmn0BcYVrTMo6s1TY9N9i2Wt7fcs4CyANddcMw0cOHDGDqSLDTr4+maH0FQjdx/Y7BAkSZIkSU3U9DE0qrQkN5Yij4vRA1i2qk71mBlPUzVWRkQsQW71MdXYGpIkSZIkaebQ3RIaXyueXwLuBj4EdmpZGBG9gW2AGyrWuQHYLCLmrijbBfgEGN6p0UqSJEmSpKZoWpeTiLgRuAV4gjybydeAA4FLU0ovFHWGAIdHxGhya4sDyEmYUys2dQZ5dpQrI+I4YGlgMHBi1VSukiRJkiRpJtHMMTQeAAYB/YHPgReBQ8gJihZDyAmMQ4D5gQeBb6aU3m6pkFIaHRGbkAcTvZY848lJ5KSGJEmSJEmaCTUtoZFSOhw4vJ06CTi6eLRV70ngGx0XnSRJkiRJ6s662xgakiRJkiRJ7TKhIUmSJEmSSseEhiRJkiRJKh0TGpIkSZIkqXRMaEiSJEmSpNIxoSFJkiRJkkrHhIYkSZIkSSodExqSJEmSJKl0TGhIkiRJkqTSMaEhSZIkSZJKx4SGJEmSJEkqHRMakiRJkiSpdExoSJIkSZKk0jGhIUmSJEmSSseEhiRJkiRJKh0TGpIkSZIkqXRMaEiSJEmSpNIxoSFJkiRJkkrHhIYkSZIkSSodExqSJEmSJKl0TGhIkiRJkqTSMaEhSZIkSZJKx4SGJEmSJEkqHRMakiRJkiSpdExoSJIkSZKk0jGhIUmSJEmSSseEhiRJkiRJKh0TGpIkSZIkqXR6NjsASZKkepx88u8ZM+aVZochSZpJzDvvl9h//yObHYZmgAkNSZJUCmPGvMLgwf2bHYYkaSYxePDIZoegGWSXE0mSJEmSVDomNCRJkiRJUumY0JAkSZIkSaVjQkOSJEmSJJWOCQ1JkiRJklQ6JjQkSZIkSVLpmNCQJEmSJEmlY0JDkiRJkiSVjgkNSZIkdQvjxn3GjjtexjzzDCHiCEaOHNPskKbbEUcMY++9r252GADstNM/OeGEu9ut17//yRx/fPv1ymzYsJFEHMGoUeOaHcoXgudbnc2EhiRJmun85S/3s9pqp9Onz7H06XMs6633N66//tmp6kQcUfPxs59dP7nOoEFXTbN83XXP6erD6dbOO+8R5prrmA7Z1rnnPswdd7zMXXd9nzffPJAllugzTZ1hw0ay3XZDWWSRE+jd+2hWW+10zj334anq1HrfIo5gzjmPmWo7teo8/fSoqbZ1yin3suKKpzHHHEez+OIn8rOfXc/YsRPaPI533vmYE064h8MO23By2R13vMy2217CYoudSMQRnHfeI9Osd/jht7Hiiqcx55zH0LfvcWyyyQXcfferU9U54ICbmG++41hiiZO46KL/TbXs2mufYYMNziWlNFX573+/IUcffScffPBpm3GXWUdehx1t4MDz+PnP/93sMJpi/fWX4M03D2T++efokO2NHDmGiCN48ME3OmR7Kr+ezQ5AkiSpoy2+eB+OO25TlltufiZNSpx//iNsv/2ljBixL6ut1g+AN988cKp1HnzwDbbZ5hJ23nmVqco33XRp/vGPb09+3atXj84/gAZ9/vkkevQIIqLZocyQ559/n5VWWpABA/q1Wufuu19lwICFOOig9Vlkkbm56abn2Xffa5l99p7sttsAAE45ZXOGDNl0qvW+9rVz2XDDJafZ3hNP/JT55ptys7Xggr0n/3zxxY9x0EG3cM452/D1ry/Jiy+O5gc/uIZPP/2cv/1tu1ZjPOech1h77cVYeum+k8vGjp3AqqsuxPe+92W+971/1VxvhRUW4C9/2ZKllurLJ598xkkn3cvmm1/Ic8/tR79+c3Httc9w8cWP8Z//7Mlzz73H3ntfw2abLcsCC/Tmo4/G86tf3cQ113x3mutgwIB+LL10Xy688H/87Gdrtxq3VK9JkxIpJXr0aPv78V69erDwwnN1UVT6IrKFhiRJmulst92KbLHFciy77Hwsv/z8HH30Jsw9dy/uuWfKt90LLzzXVI+rr36a5Zefn4026j/VtmabrcdU9SpvfmsZPHgYq67616nKqr89fvXVD9huu6HMN99x9O59NCuueBpDhz4+efnrr3/IrrteTt++x9G373FstdXFPPfce9Ps47zzHmGZZf6P2WY7io8//ow77niZddc9h7nmOoZ55hnC2mufzeOPv9NqrKNHf8Jee11F377HMcccR7PpphfwxBNT6tf61ruyCfmwYSP5/vev5uOPP5vcwmHw4GGt7u/KK59iwIDTmW22o1hiiZM4+ug7JrcmGDjwPE455T7uuONlIo5g4MDzam7j0EO/zlFHfYOvfe1LLL10X37yk7XYYYeVuOKKpybXmWee2ad6z1544X1efHE0P/zhGtNsb6GF5pyqbuUN2t13v8q66y7Onnt+mf795+Ub31iK731vNe677/VWjxFyImSbbZafqmzLLZfjmGM2YccdV2aWWWonnvbYYzU22WRpll66L6usshAnnrgZH300gUceeQuAp54axcCB/VlzzUX57ncH0KfPbLz00ujivNzKHnusxsorL1hz29tuuwKXXPJ4zWWVPv30c370o2vp0+dYFl/8RP785/9OtfzEE+9htdVOZ845j2GxxU5kn32uYcyY3PLjww/HM8ccR3Pttc9Mtc5//vMCs876R95552Og/eu7lrb22+h1WO3uu19lo43Oo3fvo1lssRP5yU+u48MPx09efuONz/P1r/+dvn2PY775jmOzzS7kqafenWobRx45nCWXPJnZZjuKhRc+fnLSatCgqxg+/GX+8pcHJsfWWleq8eM/Z//9b6Rfv+OZffajWHfdc7jrrlcmL2/53bv11hdZZ51z6N37aNZc8yweeujNho6nWst2r7vuWVZf/Qxmn/0ovvrVsxgxYkoriJa/Bf/+93Osuupf6dXrjzz11Kh2/4bU6nLSXnwpJU444W6WW+5UZpvtKBZf/EQOOeQWAJZa6hQA1lrr7Db/TuiLw4SGJEmaqU2cOImhQx9n7NgJrL/+EjXrjB07gaFDn6h5w3vXXa+w0EJ/ZvnlT+WHP7xm8k3ZjPjpT//NuHGfcfvte/HEEz/l5JM3Z955ZwfyOBIbb3w+s8/ek+HDB3HPPT9gkUXmYtNN/8G4cZ9N3sZLL43h4osf45//3IlHH/0xs8/ek+22G8oGG3yJRx/9Mffdtw/7778uPXq03mpj0KCrue++17j66l25//596N17Vjbf/CI++eSzVteptP76S3DyyZvRu/esvPnmgbz55oH8+tfr16w7YsQb7LTTP9lhhxV57LGfMGTIJhx77F2cdtr9AFx55S58//urs956i/Pmmwdy5ZW71Hs6+fDD8fTtO3ury88++yFWWWXBmu//mmuexSKLnMAmm1zA7be/NNWyDTb4Eo888hb33vsaAK+88gHXXPMsW265XKv7ev/9T3jyyXdZc81F646/lgkTJnLWWSPo02c2Vl99YQC+/OV+PPjgG4we/QkjRrzBJ598xrLLzse9977G7beP5NBDv97q9tZeezHuv//1dt/bk066lwED+vHQQz/it7/9GgcddMtUicBZZglOPnlznnjip1x88Q7cf//r7LffDQD06TMb22yzPBdd9NhU27zoosf45jeXZqGF5qz7+q7W1n4buQ6rPfbY23zrW/9g222X59FHf8yVV+7MI4+8PdX4Jx9/PIH991+H++/fh2HDBjHPPLOxzTaXMGHCRACuuOJJjj/+bv761y157rn9uO663Vh77cWA3FpovfUW5/vfX31ybLW6UgEcdNDNXHrpE5x77rY8/PCPGDBgITbf/ELefPOjqeodcsitDBmyCQ899CPmn783u+9+5eTEYD3H05pf//o/HHfcpjz44L4svXRftt76kqnek08//Zw//vEOzjxza5588mcsueQ8Df8NqSe+Qw+9lT/+8Q4OOWQDnnjip/zznzuxxBLzAHD//fsAcOONuzf8d0Izp6Z1OYmInYA9ga8C8wDPAMenlC6pqDMM2KjG6nOklD6tqLcYcBqwKTAeGAoclFJy9BlJkr6gHnvsbdZb7298+unnzDVXL/71r11a7cpw8cWPMWHCRPba68tTlW+++bLssMNKLLXUvIwcOYbDDrudb3zjfEaM2JfZZpv+f6NefnkM3/nOSnz5y/lGdamlpnRNGDr0cVKCv/99u8ldB848c2sWWuh4rrvu2cldYiZMmMg//vFt+vXLzbnff/8Txoz5lG22WZ5llpkPgBVXXKDVGJ577j2uueYZhg8fNLkrxj/+8W2+9KWTueiix9hnn2mTO9V69erBPPPMTgTtNis/8cR72WijJTniiI0BWH75+Xnuufc57rj/st9+6zDffHPQu/esDTdRv+66Z7n11pf473/3rrn8gw8+5bLLnuDYYzeZqnyRRebi9NO3Yq21Fi3O5f/YZJMLGD58EF//ej4fu+66Ku+9N44NN/w7KeWuPXvuuRrHHbdprV0BOemREiy66Nx1H0P18ey66+WMG/cZiywyNzffvOfk93izzZZljz1WY621zmaOOWbl/PO3Z665erHvvtdyxhlb8/e/P8zJJ99H796zcuqpW0yVwFl00bn57LNJvPHGR5Ovj1q+9a1l+PnPc7eU/fZbh//7v/u59daXWG+9vK399193ct3+/eflT3/6JtttN5Tzz9+eWWYJ9thjNXbd9XI++mg8c889G5988hn/+tdTnHHG1kD913e1tvbbyHVY7c9/vptddlmFAw+ckgA5/fSt+MpXzuSddz5moYXm5DvfWXmqdf7+9+3o02cI99//Ohts8CVefvkDFllkbr71rWWYddYefOlL80xOaM0zz+z06tWD3r1nbTO2jz+ewOmnP8g552zLVlvl1j1nnLE1t902kr/85QGOOuobk+v+8Y8bs/HGSwF5fJQNNvg7r7/+EYsv3qeu42nN4YdvyGabLTv5GBdf/EQuvnjK34KJExOnnbYFX/1qPrbp+RvSXny9e8/KSSfdy8knb87ee38FgGWXnW/y9bfggjn++efvbVcWAc0dQ+MA4CXgV8AoYEvg4ohYIKV0akW924FDq9ad3CYpImYFbgImALsC8wInFs97dFLskiSpm1thhQV45JEf88EHn3L55U+y115XMWzYIFZddaFp6p599kNst90Kk/9ZbrHrrqtO/nnAgH589auLsuSSJ3P99c+xww4rTXdsv/zlOvz4x9dz440vsMkmS/Htb684+SZhxIg3eOml0cw997FTrTNu3Ge88ML7k18vvnifyTe6APPNNweDBq3OZptdyCabLM0mmyzFjjuuzJe+NE/NGJ56ahSzzBKst97ik8vmmWd2BgxYiCeffLfmOjPiqafeZautpm7ZsMEGX+KII4bz4Yfj6dNntoa3+d//vsJuu13B//3f5pO/Ea924YX/Y9KkxJ57Tp2sWmGFBVhhhSkJn/XWW4KRI8fw5z/fPTmhMXz4SP74xzv461+3Yp11FuP559/nl7+8kT/8YRhHHrlxzf21fDM9++zT92/2xhv355FHfsyoUeM4++wR7LzzP4tWDDlBMnjwQAYPHji5/tFH38H66y/BPPPMxu9/P4xHHvkRjz32Djvv/E9efPGXk8d8mWOOnkV8n7e5/9VWm/r3Y9FF556qVdJtt73EscfexVNPvcsHH4xn4sRJTJgwkbfeGsuii87NFlssS+/es/Kvfz3N9773Za655hlSgu23XxGo//qu1t5+p9eIEW/y/PPvc+mlT0wuaxlT9YUX3mehhebkhRfe5/DDb+e++17n3Xc/ZtKkxKRJiVde+QCAnXZamVNOuY+lljqFzTZbhs03X5Ztt12hoaTnCy+M5rPPJvG1r01JQvXoMQvrrbf4NL+PLeMAwZTE2TvvfMzii/ep63ha05I0AJhrrl4MGNBvqn337DnL5NZCMH1/Q9qLr0ePWRg/fiKbbLJUq3FKlZqZ0NgmpVQ5jPRtEbEoOdFRmdB4P6V0bxvb2RFYCVg2pfQSQER8BgyNiCNSSs91dOCSJKn769WrB8sum7+J/upXF+WBB97gpJPumWYwx0ceeYsHH3yDY475Rq3NTGXRRedm8cX7tNnff5ZZgqpJJvjss4lTvf7BD9Zgs82W5d//fo5bbnmR9dc/l0MO2YDBgwcyaVJi9dUXZujQHafZduX4HXPOOes0y//+9+3Yf/91uPHG57nmmmf43e9u46qrdpn8rWu9WsaUrOdYOsL0jGV6112vsOWWF3HkkRvzk5+s1Wq9s89+iO98Z+V2xz4BWGedxRg6dMqN1mGH3c53v7vq5G+aBwzox8cff8Y++1zD73+/ET17Ttt7e4EF8qCio0d/OjkJ0Yg55+zFssvOx7LLzse66y7OcsudyjnnPMThh0/baPnZZ9/jb397mIcf/hHnn/8oG264JIssMjeLLDI348dP5JlnRk1ulfT++58AUw96Wsuss0496G1EHgAScsuirba6mB/+cA2OPHIg88/fm4ceepPvfveKyd0vZp21BzvvvAoXXfQY3/vel7noosf49rdXpHfvfL3We31Xqme/02vSpMQ++3yFX/1qvWmWLbZYfv+23voSFl+8D2eeuTWLLTY3PXvOwsor/3XyvpdYYh6eeebn3Hrri9xyy4sceOB/OOKI4dx33z7MOWevGYoPmGaQ18r3qGVZy3tUz/FMr9lm69HuIKBT4qpd3l58jz3W+pg/Ui1NG0OjKpnR4mGg0Q6HWwAPtCQzCleRW2xsPn3RSZKkmc2kSYnx46e9+TnrrBEstdS8bLrp0u1uY9Socbz++odt3qguuGBv3n577FRTZ7YM6lhp8cX7sO++X+Wyy3biyCMHctZZIwBYY41FeP7591lggd6Tb2xbHvXclH/5ywvz299uwLBhgxg4sD/nn/9ozXorrbQAkyYl7rnntcllH344nscee2fywJILLtibceM+m2rAvupj6dWrBxMnVmU9au5vQf7736mnIL3rrldYfPE+zD13Y60z7rjjZbbY4iIGDx44VVeEavff/zqPPvp2zbFRannkkbdZZJEprV7Gjftsmhu4Hj2mTfJUWmaZ+ejTZ7YOa+XS2nWbUmLffa/l+OO/xTzzzM6kSWlysiml/HPl+/L44++w2GJzT9Wqp1EPPvgGEyZM5KSTNmO99ZZg+eXn5403Ppqm3h57rMatt77Ik0++y403Ps8ee6w2edn0XN/17Lfe67DaGmsswhNPvDtNLMsuOx9zzDEr7703jqefHsWhh27AppsuzUorLchHH03g888nTbWd2WfvyVZbLc9JJ23OAw/8kCeeeHfy9Z5jm1Rr95Mts0xfevXqMdXvyMSJk7jnntdYeeXWu441ejxtaRkrBnIXmMcff4eVVmp93/X8DWk0vpVWWoDZZuvBrbe+VHP9lhZH7Z1PfXF0t0FB1wOerSr7VkSMKx43RcRqVctXBJ6uLEgpTQBeKJZJkqQvmIMPvoU773yZkSPH8Nhjb3PIIbcwbNhIdt99wFT1xo37jIsueowf/OAr03wLOnbsBH796/9wzz2vMnLkGIYNG8k221zCQgvNybe/3fq/GAMH9uf99z/hmGPu5IUX3udvf3uIyy9/aqo6v/zlDdx44/O8+OJoHnnkLW688YXJNwC7774a/frNxXbbDWX48JG89NJo7rjjZQ488KY2W4a89NJoDj74Fu6++1VefnkMt9/+Ev/739ut3lgst9z8bLfdCvzoR9dx550v89hjb7PHHlfSp89sk6c/XWedxZlzzlk55JBbeP7597niiif5618fnGo7/fvPy6effs7NN7/AqFHjWh3Y8cAD12P48JcZPHgYzz77Hhdd9D9OOOEeDjqovsEbWwwbNpIttriIH//4q+y22wDeemssb701lnffnXaw1rPOGsFyy83HwIH9p1l28sn3ctVVT/Pcc+/xxBPvcMght3DVVU9PHj8CYJttluess0YwdOjjvPTSaG6++QUOP/x2tt56+ZqtMyC3atl006Wnmp0C8vX0yCNv8cgjb03urvDII29N7rbw4YfjOeyw27jvvtd45ZUPGDHiDfbe+2pee+3DmuNK/O1v/8/encfLNd4PHP98JUJiSyQSSxH7rmjsWgmqdq3aKlrRlqqilqpQKihCraX2n8ae2vdIbYl93/c1iD0kiJCQPL8/zrkxmTtz7wxz79yJz/v1mtdknuc553znnJnJne88y+P06NF1+tCn9dZbhDvueJ177nmTM854mFln7cQyy/Sc3v7uu9/kZz9bovUT3IKmZZBPOeUBXn99PJdd9jSnnNK8I/U66yzMoot2Z6edrqJXr24zDB/4Nq/vSo5b6euw2EEHrctDD73NHnvcyOOPZ8MhbrzxJf7whxsA6NGjK716dePccx/jlVc+ZvToMeyxx40zXP9hw57gvPMe4+mn3+f118fzn/88wayzzsJSS807PbaHHnqHMWMmMG7cpOm9KQrNMUcX/vjHfhx00G3cfPPLPP/8h/zxjzfx/vsT2XPP8j2Qqn0+LfnHP+7i1ltf5dlnP+C3v72eLl06Tf8sKKWSz5Bq45trrtn485/X5OCDb+c//3mcV1/9mIceepszz3wYyFYl6tq1MyNHvsr770/kk0+yaRWvueZ5ll32dN5++9OKz5VmDvUccjKDiNgQ+DlQOKPTaOAC4BVgUeBvwN0R8cOU0pi8TQ9gQoldjs/rJEnS98x7701k552v4b33JjLPPLOx8sp9GDFiYLOhF//97zN8/vkUdt111Wb76NQpePrpD7jwwieZMCEbPjBgQF8uv3zbFnsULLfcfJx55uYcc8w9HHPMPWy55dIccsh6/O1vd0xvM21aYu+9R/DWW58w11yzseGGi3HiiRsD0K3brNx11yAGD76d7ba7gk8+mcyCC2bH7tGjfA+Nbt1m5aWXPmK77a5g3LhJ9OkzBwMHrsRBB61bdptsiMpIttpqOF9++TXrrrswt9wycPovufPO25VLLtmGAw+8lfPPf4Kf/GRRjjpqAL/+9TXT97HOOguzxx4/4le/uoqPPvqCww9ff4Y5HpqsttoCXHHFdhx++CiOOeZu+vSZk8GD150hgVCJYcOeYNKkrzjhhPs54YT7p5cvuug8jBmz7/THn302meHDn+Hvfy81v3w2qeqBB97K2LGf0rVrZ1ZYoTc33bTTDCuYHHroT4iAww67k7FjP6VXr25sueXSHH10y8OTdt99NQYNuo4TT9x4eg+PRx55hwEDLpje5vDDR3H44aPYZZcfMmzYz+nceRaeffZDzj//cT766At69uzK6qsvxF13DZphzgSA99+fyD/+cdcME6H267cgBx+8Hr/4xX+Za64uXHTRL6Zfxy+//JprrnmBkSO/2/RyK6/ch1NP3YTjjruXQw+9k3XWWZgTTtiYHXa4slnbgQNX4qij7mK//daaoZfLt3l9V3LcSl+HpfZ91127cuihd7D++sOYOjWx+OI9pictZ5kl+O9/t2WffUaw4opnsOSS83LiiRvzy19ePn0f3bvPznHH3ctf/vI/vvpqGssvPx9XX73D9Ml+//KXddhll2tZfvl/88UXX/P663+mb9/uzWJpmmx2112vY8KEL1l11fm55Zadqxq61NrzacnQoRtxwAH/48UXP2KFFebjxht/1eqQmdY+Q75NfMceuxE9enTlqKPuYuzYT+nTZ05+85vsN+3OnWfhX//alCOPHM0RR4zmxz9ehFGjBvHJJ5N58cWP+Oore25830Rqqc9cewUR0Rd4ELgvpfSLFtrNT9YbY1hKad+87GXgpqbHBW3vAd5MKe1UZl+7A7sD9OnT50fDhw//7k+kHT399if1DqGuVlqo9ARnkqSZ1/DhQxk8eP7WG0odwDbb3MzOOy/DNtt8t14RtXDhhS9w661vctFFG9c7FHVADzzwHr/61UgefXQH5p23/PLH38bo0W8zaNBtPP/8wG89UW5bGjr0PXbccXC9w1ArBgwY8GhKqV+purq/qiJiXmAE8AYwsKW2KaX3IuJeoHAg5HiyZV+L9QBKDxrN9nUOcA5Av379Uv/+/asLvM4GDb6p3iHU1ZiB/esdgiSpnY0aNYy+ffvWOwypIsOG/ZLHH3+3Q7xm55//I847rx99+1Y+F4O+P8aMye4XWWSR6ZPa1sL770/kwQefZYklerDsstVNTNxe5p8fGu17oGZU14RGRHQDbgS6AFuklCZVsFnKb01eoGiujIjoAiwOnFWjUCVJkqSKrbxyn2ZDRepl991/VO8Q9D202WaX8tlnkznrrC3qHYpmYnVLaEREZ+AKYClgnZRSq2v05ENO1gPOLygeAewUEYumlN7Iy7YCZgNuqW3UkiRJkjTz6N+/LykdXvP9Pvro7jXfp1Ssnj00zgA2A/4M9IyIngV1jwPLAMeSJT3eABYBDgamAacUtL2SbLLQqyPiMLLhJycDl6aUXm7j5yBJkiRJkuqgngmNplmJTi1RtxjwERBkSY2ewGfAKODnKaXp62CllL6KiE2A04HLgcnAcODANotckiRJkiTVVd0SGimlvhU026zCfY0lW/JVkiRJkiR9D8zSehNJkiRJkqSOxYSGJEmSJElqOCY0JEmSJElSwzGhIUmSJEmSGo4JDUmSJEmS1HBMaEiSJEmSpIZTt2VbJUmSqtG9+yIMGTKm3mFIkmYS3bsvUu8Q9B2Z0JAkSQ1h332PrHcIkiSpA3HIiSRJkiRJajgmNCRJkiRJUsOpOKERET+JiD8XlQ2MiFcj4qOIOC0iTJBIkiRJkqQ2V00C4ghg3aYHEbE0cD4wBXgI2BPYu6bRSZIkSZIklVBNQmMF4MGCxzsBXwJrpZQ2BS4Fdq1hbJIkSZIkSSVVk9CYG/i44PEmwG0ppU/yx6OBxWsVmCRJkiRJUjnVJDTeI09YRERP4EfA7QX1cwLTaheaJEmSJElSaZ2raDsS2CsiJgD9yZIX1xfULw+8WbPIJEmSJEmSyqgmoXEosCzwT7KJQPdPKY0FiIjZgW2BC2seoSRJkiRJUpGKExoppQ+B9SNibuCLlNJXBdUBDADeqnF8kiRJkiRJzVQ0h0ZEdIuIRyNij5TSp0XJDFJKX6SUnkwpfVxuH5IkSZIkSbVSUUIjpTSJbEJQJ/2UJEmSJEl1V80qJ3eSTQYqSZIkSZJUV9UkNPYBfhgRJ0fE0hFRzYSikiRJkiRJNVNNUuL1/H5ZsuQGETG1qE1KKc1Wi8AkSZIkSZLKqSahcQmQ2ioQSZIkSZKkSlWzbOugNoxDkiRJkiSpYtXMoSFJkiRJktQhVJXQiIglI+KiiHg7IqZExAZ5ea+IOD8i1mibMCVJkiRJkr5RcUIjIlYEHgE2Ax4HOjXVpZTGASsBe9Q6QEmSJEmSpGLVTAo6FPgYWJNsctAPiupvAXaoUVySJEmSJEllVTPk5MfAmSmlDym92smbwII1iUqSJEmSJKkF1SQ0OgMTW6jvBXz93cKRJEmSJElqXTUJjaeADUpVREQA25DNsSFJkiRJktSmqklonAT8MiKOBPrkZV3yyUKvBFYDTqxxfJIkSZIkSc1UPCloSumKiFgUOAb4W158U34/FTggpTSixvFJkiRJkiQ1U80qJ6SUToiI4cC2wFJkPTxeAa5KKY2pfXiSJEmSJEnNVZXQAEgpjQVOqX0okiRJkiRJlak4oRERXYG5UkofFJT1Bv4AdAcuTyk9WPMIJUmSJEmSilTTQ+McYAWyyT+JiG7AA0DfvH7viBiQUrq3phFKkiRJkiQVqWaVkx8D1xc83pEsmbEpsCDwAnBIzSKTJEmSJEkqo5qERm/gzYLHWwAPpZRGppTeA/5D3ntDkiRJkiSpLVWT0JgEzAUQEbMA/YFbC+onks2lIUmSJEmS1KaqSWg8BewcEfMCvwPmAUYU1PcFPiixnSRJkiRJUk1VMynoUcBNwIf549EppfsK6rcAHqpVYJIkSZIkSeVU3EMjpXQn2RwZ+wG7kk0GCkDea+MO4PRK9xcR20XE9RHxdkRMjIhHI+JXJdrtFhEvR8SXeZsNS7RZKCKuiYjPImJcRJyer8IiSZIkSZJmQtX00CCl9ALZaibF5R+TJTqqsT/wer7dOGAz4NKI6JVSOg0gT3CcBQwB7iFLpNwYEaunlJ7J28wKjASmkK280h04Kb/fucqYJEmSJElSA6gqoVFjW6aUxhU8viMiFiRLdJyWlw0BLkgpHQUQEaOBVYHBfJOs2BZYDlgypfR63u4rYHhEHJFSernNn4kkSZIkSWpXLQ45iYhpETG1hdvEiHg2Io7Lh51UrCiZ0eRxYMH82IsDSwOXF2wzDbiCguEu+b8fbkpm5K4l67GxSTUxSZIkSZKkxtBaD43zgdRCfTey3hEHAttGxJplEhWVWht4Kf/3svl98RCX54F5I2K+lNKHebvnChuklKZExKsF+5AkSZIkSTORFhMaKaXfV7KTiFifbB6Lw4A/f5tA8sk+fw78Ni/qkd9PKGo6vqD+w/y+uE1Tux4lyiVJkiRJUoOryRwaKaXREfF/ZEu3Vp3QiIi+wKXAdSmlYbWIqYJj7g7sDtCnTx9GjRrVHoetmQNW+rreIdRVo10vSZIkSVJt1XJS0KeA31W7UT73xgjgDWBgQVVTT4x5mLEHRo+i+vF5m2I9gCfLHTeldA5wDkC/fv1S//79q4y8vgYNvqneIdTVmIH96x2CJEmSJKmOWpwUtErzAl9Ws0FEdANuBLoAW6SUJhVUN82dUTwPxrLAx/n8GU3tZmgTEV2AxSmxxKwkSZIkSWp8NUloREQnYBuyVUoq3aYz2YolSwGbpJQ+KKxPKb1GNkHodgXbzJI/HlHQdASwekQsWlC2FTAbcEt1z0SSJEmSJDWCFoecRMSCrWzflax3xJ7AasC2VRz7DGAzsjk3ekZEz4K6x1NKk4EhwMURMQa4F9iFLAGyU0HbK4G/AVdHxGFkw09OBi5NKb1cRTySJEmSJKlBtDaHxlhaXrYVIIDJwF9TStdUceyN8/tTS9QtBoxJKV0WEXMCB5GtoPIs2dCUZ5oappS+iohNgNOBy/NYhpMtJStJkiRJkmZCrSU0jqTlhMaXwBjgtpTSR9UcOKXUt8J25wLnttJmLNmSr5IkSZIk6XugxYRGSmlIO8UhSZIkSZJUsVquciJJkiRJktQuTGhIkiRJkqSGY0JDkiRJkiQ1HBMakiRJkiSp4ZjQkCRJkiRJDadsQiMiPo6IbQse/z0iVmyfsCRJkiRJksprqYfGnEDXgsdDgJXbNBpJkiRJkqQKtJTQGANsFhFzFZSltg1HkiRJkiSpdS0lNE4DdgAmRMRUsmTGxRExtYXb1+0StSRJkiRJ+l7rXK4ipXRaRDwFbAT0AX4HjAZebafYJEmSJEmSSiqb0ABIKY0mS2IQEb8HzkspXdoegUmSJEmSJJXTYkKjUErJJV4lSZIkSVKHUHFCo0lEbAJsASyWF70OXJ9S+l8tA5MkSZIkSSqn4oRGRHQBriBLZgQwLq/aFPhjRNwAbJ9SmlLzKCVJkiRJkgpUM4zkcGBL4ESgd0qpd0qpNzAfcAKwFXBY7UOUJEmSJEmaUTUJjZ2Ai1NKf00pNfXOIKX0UUrpIOBiYOdaByhJkiRJklSsmoTGgsB9LdTfDyzw3cKRJEmSJElqXTUJjXeAtVqoXxN497uFI0mSJEmS1LpqEhqXAb+OiH9ERI+mwojoERFHAb8GLq11gJIkSZIkScWqWbb1CGAV4BDg4Ij4MC+fj2zVk1uAI2sanSRJkiRJUgkVJzRSSpOBzSJiC2BzoG9eNQa4IaV0c82jkyRJkiRJKqGaHhoApJRuBG5sg1gkSZIkSZIqUs0cGpIkSZIkSR2CCQ1JkiRJktRwTGhIkiRJkqSGY0JDkiRJkiQ1HBMakiRJkiSp4VSU0IiIbhHxUUQc2NYBSZIkSZIktaaihEZKaRIwDZjYtuFIkiRJkiS1rpohJzcAW7ZVIJIkSZIkSZXqXEXbk4DLIuJq4CzgVeCL4kYppXdqFJskSZIkSVJJ1SQ0nsrvVwC2bqFdp28fjiRJkiRJUuuqSWgcCaS2CkSSJEmSJKlSFSc0UkpD2jAOSZIkSZKkilUzKeh0ETF7RCwUEV1qHZAkSZIkSVJrqkpoRMR6EXE38BnwJrBeXt4rIm6PiI3bIEZJkiRJkqQZVJzQiIj1gNuB+YHzgGiqSymNyx//ttYBSpIkSZIkFaumh8Y/gOeAFYHDStSPBtaoRVCSJEmSJEktqSah0Q8YllKaTOnVTt4m670hSZIkSZLUpqpJaEyj5WVbFwQmfbdwJEmSJEmSWldNQuNhYKtSFflqJwOB+2oRlCRJkiRJUkuqSWgcA/SPiAvJhp8ALBwRWwB3AYvlbSoWEUtGxNkR8VRETI2IUSXajImIVHR7r0S75fOVViZFxDsRcWREdKomHkmSJEmS1Bg6V9owpXR7ROwEnEHWGwPgfLLVTSYAO6WUHqjy+CsAmwEPALO20O5S4LSCx1MKKyOiB3Ab2aSlWwNLACeSJWwOrTImSZIkSZLUwVWc0ABIKV0eETcCGwNLkSUMXgFGppQmfovj35BSug4gIq4EepVp924ryZI9gK7ANimlT4FbI2JuYEhEHJ+XSZIkSZKkmURVCQ2AlNIk4NpaHDylNK0W+wE2JUuqFCYuhgPHAesDN9ToOJIkSZIkqQOoZg4NACLipxFxakRcn99OjYiN2yK4Ar+LiCkR8UlEXBkRixbVLwu8UFiQUnqTbNWVZds4NkmSJEmS1M4q7qEREXMA/yXrDRHA+LxqC2CviBgJbJdS+rzGMV5HNsfGWGA54HDg7ohYKaX0Sd6mB9k8HsXG53WSJEmSJGkmUs2Qk3+STeB5DHBKSmkcQET0AvYDDs7b7FnLAFNKfy54eHdE3Ac8AewKnPJt9xsRuwO7A/Tp04dRo0Z9+yDr4ICVvq53CHXVaNdLkiRJklRb1SQ0tgfOTynNsGpIntj4W0QskLepaUKjWErpmYh4EVitoHg8ME+J5j34pidJ8X7OAc4B6NevX+rfv3+NI21bgwbfVO8Q6mrMwP71DkGSJEmSVEfVzKHRBXikhfqHaXnp1VpK+a3JCxTNlRERCwPdKJpbQ5IkSZIkNb5qEhp3Ahu2UL9h3qZNRcSKZMmLRwuKRwA/i4i5Csp2AL4ARrd1TJIkSZIkqX1VM+RkT2BERJwPnAq8mJcvA+wLLEU2YWjFIqIb2bwcAAsBc0fEtvnjm4EBwM7AjcA7ZImMQ4E3gWEFuzoL2Ae4OiKOAxYHhgAnFS3lKkmSJEmSZgJlExoR8RUzDuuArEfHisAuJTZJwOvAbFUcvzdwRVFZ0+PFgLfyNqcA3YGPgFuAQwoTFSml8RGxIXA6cAPZiicnkyU1JEmSJEnSTKalHhqX0DyhUVMppTFkS8C2pKVhLoX7eg7Y4LvGJEmSJEmSOr6yCY2U0qB2jEOSJEmSJKli1UwKKkmSJEmS1CFUMykoETELsBHZpJs9aD5cJKWUjq1RbJIkSZIkSSVVnNCIiH5kE3YuQvl5LxJgQkOSJEmSJLWpanponE3WK+NPwP1kK4lIkiRJkiS1u2oSGssDQ1JKZ7VVMJIkSZIkSZWoZlLQ14Gv2yoQSZIkSZKkSlWT0DgW+ENEdG+jWCRJkiRJkipS8ZCTlNJFETEr8HJEXAO8BUxt3sxVTiRJkiRJUtuqZpWTlYAjgJ7A78s0c5UTSZIkSZLU5qqZFPQsYC5gT1zlRJIkSZIk1VE1CY1VgSNc5USSJEmSJNVbNZOCvomrnEiSJEmSpA6gmoTGULJVTnq0VTCSJEmSJEmVqGbIyYLAZ8CrEXElrnIiSZIkSZLqpJqExj8K/u0qJ5IkSZIkqW6qSWgs1mZRSJIkSZIkVaHihEZK6Y22DESSJEmSJKlS1UwKKkmSJEmS1CFU3EMjIl4nmyOjJSmltMR3C0mSJEmSJKll1cyhMZrmCY1OQF9gHeBp4ImaRCVJkiRJktSCaubQGFSuLiJWA0YA+9cgJkmSJEmSpBbVZA6NlNJjwLnA8bXYnyRJkiRJUktqOSnoWGCFGu5PkiRJkiSppJokNCJiVmBH4P1a7E+SJEmSJKkl1axycn6Zqu7A2kAfYO8axCRJkiRJktSialY52YDmq5wkYDxwF3B2SumOWgUmSZIkSZJUTjWrnPRtwzgkSZIkSZIqVstJQSVJkiRJktqFCQ1JkiRJktRwWhxyEhHTaD5vRktSSqmaeTkkSZIkSZKq1lry4XwqS2isBqz63cORJEmSJElqXYsJjZTS71uqj4gfAkeQJTM+BU6pWWSSJEmSJEllfKvhIRGxAjAE+AUwCTgGOCGlNKFmkUmSJEmSJJVRVUIjIpYlS2RsC3wBnAAcn1L6uPahSZIkSZIklVZRQiMilgIOB3YApgAnkyUyPmzD2CRJkiRJkkpqbZWTxYG/AzsBXwOnA0NTSu+3Q2ySJEmSJEkltdZD40VgFuBxYCjwDrBERCxRboOU0n21C0+SJEmSJKm51hIanfL71YD/ttI2yJZ47dRKO0mSJEmSpO+ktYTGru0ShSRJkiRJUhVaTGiklC5or0AkSZIkSZIqNUu9A5AkSZIkSaqWCQ1JkiRJktRw6prQiIglI+LsiHgqIqZGxKgSbSIiDomItyLii4i4KyJWKdFu+Yi4PSImRcQ7EXFkRDhBqSRJkiRJM6F699BYAdiMbHnYl8q0GQwcBhwHbAlMBG6LiPmbGkRED+A2slVWtgaOBA4AjmizyCVJkiRJUt3UO6FxQ0pp4ZTSdsCzxZURMTtZQuPYlNLpKaXbgO3IEhd7FTTdA+gKbJNSujWldBZZMmP/iJi7zZ+FJEmSJElqV3VNaKSUprXSZB1gbuDygm0+B24ANi1otykwMqX0aUHZcLIkx/q1iVaSJEmSJHUU9e6h0ZplganAy0Xlz+d1he1eKGyQUnoTmFTUTpIkSZIkzQQ6ekKjBzAxpTS1qHw80C0iuhS0m1Bi+/F5nSRJkiRJmol0rncA9RIRuwO7A/Tp04dRo0bVN6AqHbDS1/UOoa4a7XpJkiRJkmqroyc0xgNzRkSnol4aPYBJKaUpBe3mKbF9j7yumZTSOcA5AP369Uv9+/evWdDtYdDgm+odQl2NGdi/3iFIkiRJkuqoow85eQHoBCxZVF48Z8YLFM2VERELA92K2kmSJEmSpJlAR09o3Ad8SrZUKwAR0Q3YEhhR0G4E8LOImKugbAfgC2B0O8QpSZIkSZLaUV2HnOTJic3yhwsBc0fEtvnjm1NKkyJiKHBYRIwn622xP1ki5rSCXZ0F7ANcHRHHAYsDQ4CTipZylSRJkiRJM4F6z6HRG7iiqKzp8WLAGGAoWQLjYKAn8Ajw05TS+00bpJTGR8SGwOnADWQrnpxMltSQJEmSJEkzmbomNFJKY4BopU0Cjs5vLbV7DtigZsFJkiRJkqQOq6PPoSFJkiRJktSMCQ1JkiRJktRwTGhIkiRJkqSGY0JDkiRJkiQ1HBMakiRJkiSp4ZjQkCRJkiRJDceEhiRJkiRJajgmNCRJkiRJUsMxoSFJkiRJkhqOCQ1JkiRJktRwTGhIkiRJkqSGY0JDkiRJkiQ1HBMakiRJkiSp4ZjQkCRJkiRJDceEhiRJkiRJajgmNCRJkiRJUsMxoSFJkiRJkhqOCQ1JkiRJktRwTGhIkiRJkqSGY0JDkiRJkiQ1HBMakiRJkiSp4ZjQkCRJkiRJDceEhiRJkiRJajgmNCRJkiRJUsMxoSFJkiRJkhqOCQ1JkiRJktRwTGhIkiRJkqSGY0JDkiRJkiQ1HBMakiRJkiSp4ZjQkCRJkiRJDceEhiRJkiRJajgmNCRJkiRJUsMxoSFJkiRJkhqOCQ1JkiRJktRwTGhIkiRJkqSGY0JDkiRJkiQ1HBMakiRJkiSp4ZjQkCRJkiRJDceEhiRJkiRJajgmNCRJkiRJUsMxoSFJkiRJkhqOCQ1JkiRJktRwTGhIkiRJkqSG0+ETGhExKCJSidseBW0iIg6JiLci4ouIuCsiVqlj2JIkSZIkqQ11rncAVdgA+KLg8WsF/x4MHAYcCLwA7A/cFhErppTea78QJUmSJElSe2ikhMbDKaWJxYURMTtZQuPYlNLpedn9wBhgL+DQ9gxSkiRJkiS1vQ4/5KQC6wBzA5c3FaSUPgduADatV1CSJEmSJKntNFJC49WI+DoiXoyIPxSULwtMBV4uav98XidJkiRJkmYyjTDk5F2y+TEeAjoBOwJnRUS3lNLJQA9gYkppatF244FuEdElpTSlXSOWJEmSJEltqsMnNFJKI4GRBUUj8nkzDo2IU7/tfiNid2B3gD59+jBq1KjvFGd7O2Clr+sdQl012vWSJEmSJNVWh09olHElsD3Ql6wnxpwR0amol0YPYFK53hkppXOAcwD69euX+vfv36YB19qgwTfVO4S6GjOwf71DkCRJkiTVUSPNoVEoFdy/QDYUZcmiNsvmdZIkSZIkaSbTqAmNbYFxwBvAfcCnwHZNlRHRDdgSGFGX6CRJkiRJUpvq8ENOIuIqsglBnyLribFDftsnpTQN+DIihgKHRcR4sl4Z+5Mla06rT9SSJEmSJKktdfiEBvAi8FtgYSCA54DfpJQuKmgzlCyBcTDQE3gE+GlK6f12jlWSJEmSJLWDDp/QSCkdAhzSSpsEHJ3fJEmSJEnSTK5R59CQJEmSJEnfYyY0JEmSJElSwzGhIUmSJEmSGo4JDUmSJEmS1HBMaEiSJEmSpIZjQkOSJEmSJDUcExqSJEmSJKnhmNCQJEmSJEkNx4SGJEmSJElqOCY0JEmSJElSwzGhIUmSJEmSGo4JDUmSJEmS1HBMaEiSJEmSpIZjQkOSJEmSJDUcExqSJEmSJKnhmNCQJEmSJEkNx4SGJEmSJElqOJ3rHYA0s+s7+KZ6h1A3Y4ZuXu8QJEmSJM2k7KEhSZIkSZIajgkNSZIkSZLUcExoSJIkSZKkhmNCQ5IkSZIkNRwTGpIkSZIkqeGY0JAkSZIkSQ3HhIYkSZIkSWo4JjQkSZIkSVLDMaEhSZIkSZIajgkNSZIkSZLUcExoSJIkSZKkhmNCQ5IkSZIkNRwTGpIkSZIkqeGY0JAkSZIkSQ3HhIYkSZIkSWo4JjQkSZIkSVLDMaEhSZIkSZIaTud6ByBJHUHfwTfVO4S6GjN083qHIEmSJFXFHhqSJEmSJKnhmNCQJEmSJEkNx4SGJEmSJElqOCY0JEmSJElSwzGhIUmSJEmSGo4JDUmSJEmS1HBmmoRGRCwfEbdHxKSIeCcijoyITvWOS5IkSZIk1V7negdQCxHRA7gNeA7YGlgCOJEsYXNoHUOTJEmSJEltYKZIaAB7AF2BbVJKnwK3RsTcwJCIOD4vkyRJkiRJM4mZJaGxKTCyKHExHDgOWB+4oS5RSZJqqu/gm+odQl2NGbp5vUOQJEnqMGaWhMaywB2FBSmlNyNiUl5nQkOSpHZi4snEkyRJ7WFmSWj0ACaUKB+f10mSJEkN6fucJGykBKHXSWp/kVKqdwzfWUR8BRyYUjqlqHwscGFK6ZAS2+wO7J4/XAZ4sa3jnMn0AsbVOwhVxGvVGLxOjcHr1Bi8To3B69QYvE6Nw2vVGLxO1Vs0pTRfqYqZpYfGeGCeEuU98rpmUkrnAOe0ZVAzs4h4JKXUr95xqHVeq8bgdWoMXqfG4HVqDF6nxuB1ahxeq8bgdaqtWeodQI28QDZXxnQRsTDQLa+TJEmSJEkzkZkloTEC+FlEzFVQtgPwBTC6PiFJkiRJkqS2MrMkNM4CJgNXR8RG+fwYQ4CTipZyVe04XKdxeK0ag9epMXidGoPXqTF4nRqD16lxeK0ag9ephmaKSUEBImJ54HRgbbIVT84DhqSUptYzLkmSJEmSVHszTUJDkiRJkiR9f8wsQ07UDiJiyYg4OyKeioipETGq3jGpuYjYLiKuj4i3I2JiRDwaEb+qd1yaUURsGxH3RcRHEfFlRLwYEYdGRJd6x6byImKh/H2VImLOesejTEQMyq9J8W2PesemGUVE54gYHBEvR8TkiBgbESfXOy7NKCJGlXlPpYhYu97x6RsRsWNEPJb/3/R2RFwYEQvWOy7NKCJ+nn+HmhwRr0fE/vWOaWYxsyzbqvaxArAZ8AAwa51jUXn7A68D+5Gtcb0ZcGlE9EopnVbXyFSoJ3AH8E+yYXJrkM39Mz+wV92iUmv+CUwE5qh3ICppA7IJwZu8Vq9AVNYwsut0BNlKdAsDy9czIJW0JzB3UdmRwKrAw+0fjkqJiK2Ay4B/AwcCCwD/AG6KiB+llKbVMz5lImJd4GrgfOAvwJrAcRExLaV0Sj1jmxk45EQVi4hZmj4YI+JKoFdKqX99o1KxPHExrqjsUmDtlNJidQpLFYiIo4E/AT2SH84dTkT8BLgWOIYssTFXSmliXYMSkPXQAP6D16RDi4hNgBuAH6aUnqt3PKpc3nvwPeC/KaU/1jseZSJiOLBUSulHBWVbAdcBy6eUnq9bcJouIkYC3VJKPy4oOxHYFZg/pTSlbsHNBBxyooqZ5W0MxcmM3OOA3Q87vo8Ah5x0QBHRCTiN7BfKUu8xSa37LXCHyYyGtAnQg6w3gDqOWYFPisom5PfRvqGoBasAtxaV/Y/sPeUQru/IhIb0/bA28FK9g1BzEdEpIrpFxHrAPsCZ9s7okPYAZiPr1quO69WI+Dqfk+YP9Q5GzawJvBQRp0fEpxExKSKudrx/Q9gRGAvcXe9ANIPzgR9HxG8iYu6IWJpsyImJw45ldqC4F0bT4+XaOZaZjgkNaSYXERsCPwdOrHMoKu3z/HY3MJpsDKw6kIjoCRwF7J9S+qre8aikd4HDgF8DW5LN9XRWROxX16hUbH5gENmvlTuSdbf+EXBNRPhrcgcVEd2ArYDLTbh3LCmlm8jeU+eQ9dR4EegE/LKOYam5V4DVi8rWyO/nbedYZjpOCirNxCKiL3ApcF1KaVh9o1EZ6wDdyP5j+ztwOtlkbOo4jgYeSCndXO9AVFpKaSQwsqBoRETMDhwaEac6ZLLDiPy2dUrpI4CIeJcsmbsBcHsdY1N5W5JNhOxwkw4mIgYAZwGnAiOAPmQTjF8TERullKbWMTx94yyyJPtuwJVkf/M1rXLi/0/fkQkNaSYVEfOS/ef2BjCwzuGojJTSY/k/74mIccAFEXFiSunVesalTESsQDbu/ycR0T0v7pbfzxMRU1NKX5TcWPV2JbA90BdXO+koxgOvNSUzcveQdb1eHhMaHdWOwCsppUfqHYiaORG4PqV0UFNBRDxBtoLQ1mQra6j+zgd+CJxJ1ptmEnAQ2dxc79UxrpmCQ06kmVDePfRGsgkmt0gpTapzSKpMU3LD1Wg6jqXIJl27n+zL2Hi+mUdjLNkfI+qYUtG96u95Sk9UGPgrZYcUEfMAm2LvjI5qWeCJwoKU0otky1cvUY+A1FxKaWpKaS9gPmBlsp40D+TVD5TdUBWxh4Y0k4mIzsAVZF/E1kkpfVDnkFS5dfP71+sahQrdAwwoKtuE7JeVzfCX/45sW7IVad6odyCa7kbgiKLlxX9CljR8sn5hqQW/IJsQ2YRGx/QGsFphQUQsB3QFxtQjIJWXUmr6YYSI2BO4L6X0Qn2janwmNFSx/Ff/zfKHCwFzR8S2+eOb7QXQYZxBdp3+DPTMJzRs8nhKaXJ9wlKhiLgFuA14FphKlsw4APivw006jvxL16jCsnxuGoC7U0oT2zsmNRcRVwEPAU+RTYi3Q37bx/kzOpRzyFZzuiEijgHmAo4Dbksp3VPXyFTOjsCTKaXn6x2ISjoLODki3uGbOTT+TpbMcN6nDiIi1gLWI+tNMzfwK+BneZm+IxMaqkZvsl/+CzU9XgwzwR3Fxvn9qSXqvE4dx8NkM5P3Bb4m+6X/YLI/TiRV50WyuU4WJhu+8Bzwm5TSRXWNSjNIKX0aERsA/wKGk82dcR3gajQdUET0AjYkW0FIHdO/yN5HfyRbXnwCWc/Cg1NKn9cxLs3oK7Ik+xCy4XV3A+umlJ6uZ1Azi3D1JUmSJEmS1GicFFSSJEmSJDUcExqSJEmSJKnhmNCQJEmSJEkNx4SGJEmSJElqOCY0JEmSJElSwzGhIUmSJEmSGo4JDUmSVJWIGBYRX9c7jmpExN4R8XJEfBURE+odD0BEDImIVO84JElqVCY0JEnqYCJiUESkiPg6IpYsUX+eX4QrFxH9gX8BjwO/B3ZvoW3TuW+6TY2I9yJieEQs3T4R10ZEdM+TJj+pdyySJLWFzvUOQJIkldUJ+Dvwm3oH0uA2yu93TylNqHCbo4CXgNmAHwG/AzaKiJVSSu/WKK5/AENrtK9SugOHA18Dd7XhcSRJqgt7aEiS1HE9DuwUEcvUO5B6iIg5arSr3gBVJDMA/pdSujil9H8ppT2BvwI9gUHlNqg23pTS1ymlL6vZRpIkfcOEhiRJHddQ4CuyX9lbFBFjImJYifJhETGm4HHffCjFoRGxa0S8GBFfRMR9EbFy3ubXEfF8RHwZEY9FxOpljvmDiLguIj6LiI8i4syImLNEuw0j4o683ecRMToiflzUZkge10oRcX5EjAPGtvKcZ4+IY/LnPiW/PyYiZitok4Ddmv6d34a0tN8ybsvvF6sk3oj4bUQ8mZ/DDyPiooj4QannXOJ5tXq+8nZ9IuLfEfFmREyOiLERcWlELJQPs3k9b3pU8XOPiN4RcXZEvJFv+2FEjMq3kySpITjkRJKkjusd4Gxg74j4R0rpuRruextg7nz/nYHBwIiI+DtwMHAO2ZCXwcCVEbFESqlwItAARpANyzgI6AfsASwKbDa9UcT2wGXAaOCwfLtBwO0RsVFKqXgoxGVkiYHDgWbJkYL9BnA1sClwMXA/sE4e+0rAlnnTX5MNF+mf/xvgqVbOTSlNc5mMay3eiBgMHAvcQ9azYyFgH+AnEbFqSunjFp5XRecrIvoADwILAucBTwLzAZvnsT4P7AecDFwJXFf03K8AVgH+DbwKzAusCawKjKrwnEiSVFcmNCRJ6tiGkk1iOQTYvob7XRxYMqU0DiBf+eMM4Hhg6ZTSR0XlGwG3FGw/C/BYSmmXpoKIeBc4JCJ+llIamQ/BOBMYnlIaWNDuLOBpsi/96xbF9Srw85RSa5Oebk6WzBiaUjo4Lzsjj+EvEbFZSunmlNLFea+D/imliys4L03miYheQBeyOTROBaaRJQfKxptvMwS4G9igKQkUEXcBN5IliP5a6oBVnq9jyZJHG6SU7izYzT8iIlJKKSKuJUtoPFn43CNiHuAnwF9TSv+s4pxIktShOOREkqQOLKX0HllCYduIWKmGu76qKZmRuz+/v7YpmVFUvniJfZxS5vEW+f1PyX75vzgiejXdgDnIhnCsFRHdivZxZgXJjMJjnFBUfnxR/bd1I/Ah8DZwPTA78OuU0mNF7Yrj3YhsItFTCnu0pJRuAp5rJa6KzldEzELWw+bWomRG07FaO39fAFOA/vn+JUlqSPbQkCSp4zuebDjHEOCXNdrnm0WPJ7RS3qPEPl4sfJBS+jAixpPPMwE0LXN6cwtx9AQmFTx+tYW2hfoCHxYlX5piGFcQw7e1H/AMMJUssfF8SmlqiXbF8fbN718o0ba1hEal52sKMA9Zr42qpZSmRMQBZL033ouIR8l631yaUnqx5a0lSeo4TGhIktTBpZQ+iIh/AwdGxCrlmpUp71SmvNSX85bKo0x5S5p6gv6O5omSJh8WPf7iWxynLTySUrqngna1jLfS8zXPdz1QSun0iLge2ArYENgfODgifpdSuui77l+SpPZgQkOSpMZwPLAncATNkwAA44HuJcr7tl1ILEO2tCwAETEfWU+OptU1Xsnvx6WUbqO2xgA/i4iehb008iEUvQpiaG9j8vtlyXpkFFqOluOq6HxFxBTgU7LJT1vS4tCTlNKbwOnA6RHRg2x40ZGACQ1JUkNwDg1JkhpA/qX9X2S/qK9WoskrwNpFS5auRrbyR1vZt8zjm/L7kWRDVg4tjKtJngD5tm7I7/cvKj+wqL693QpMBv4cEdN/OIqITYEVWomrovOVUpoGXAX8NCIGlGjX1Jvm8/y+e1F9t4joWliWUhpPloyZoa0kSR2ZPTQkSWocJwB7kS2tWexsYDvgfxExnGyp0N3J5oH4zkMUSpgGrBYRVwG3ky3buivwv5TSLQAppc8iYneyZUifjohLyJaiXQhYP99Psy/kFbqZbN6HQyLiB2RLmK5FtjTrjSmlEd9yv99JSumjiBhCtgrJ7RFxJd8s2/omcFwL21Zzvg4hm0R0ZEQ0Lds6L9mSuYcCo1NK4yLiTWCniHiVrBfPM2R//92Zx/YcMJFs1ZOfka2yIklSQ7CHhiRJDSL/Ff2UMnW3k31p7ks22eOmwK8oGBJS63DyY3Qm+5L+c+AcYNuiuK4g+7L8EvBn4DRgENmwmbJf7ls9eLaSxzZky9r2Jzsv65MlErb7tvuthZTSUOD3ZMNvTsj/fTWwbkrp41a2reh85avfrAEMIzv3p5ENSXoDeLlgl7sA75Gdn8vIrs9bZMNK1iYbwnQy8EPgALLXkCRJDSEqWxlNkiRJtRQRRwEHp5TsMStJ0rdgDw1JkqT6WBAYV+8gJElqVP4iIEmS1I4iYllgS7KhMfWavFSSpIZnDw1JkqT2tQnwN+AOmq/SIkmSKuQcGpIkSZIkqeHYQ0OSJEmSJDUcExqSJEmSJKnhmNCQJEmSJEkNx4SGJEmSJElqOCY0JEmSJElSwzGhIUmSJEmSGo4JDUmSJEmS1HBMaEiSJEmSpIZjQkOSJEmSJDUcExqSJEmSJKnhmNCQpFxEpPz2WUT0KtNmUN7mvPaOrxIRMSqPr2+9Y6m1iOgTERdHxPsRMTV/nj+vYLu5I+LwiHgsv7aTI2JsRNwfESdExE/aIfyqFbzWhtQ7lo4gIvrm52NUlduNiYjURmEVHqdDv/cK4iu8TYqI5/L3Qe92iGFYftz+bX0slRYR/fNrMKaVdsP8/JHUCExoSFJzcwIH1TsINfN/wEDgXeBS4ALgzZY2iIhFgCeBIcDSwEPAVcAzwFLAAcAhbRaxVEYdE1b3kr13LgBGAfOTvQ8ej4hF2zmWdmFyUJJmXp3rHYAkdTAJmAz8KSJOTCm9V++ABBHRBdgEGAOsllKaVuGmpwN9gZHATimljwv2OQvQH1i5lrHW0DXAA8C4egeimcp5KaVhTQ8iYj7gZqAfcAKwXRse+2BgKK0kIiVJqpQ9NCRpRtOAs4CuZH98q2OYH+gEvFFpMiMiugKb5g/3KkxmAKSUpqWU7kgpnVLTSGskpfRJSumFlJIJDbWZlNKHZD00ADaPiFnb8Fjv5q/pSW11DEnS94sJDUlqbigwCfhDRPygkg1aGxteahx/wVjmYRHRMyLOjoh38nHtD0XEJgVtt8nnfJgYER9FxP9FRPdWYto5Ih7N9/dhRFzY0vOJiE0i4qa87eSIeC0iToqIni0934j4WUTcGRET8rIW4yqK7+6I+CQivoiIpyPib3kiorBdAt7IH65fMP5/VCuH6ME3PRE/qCSmouOuGRFXRMS7ETEln3fjvHwYS3HbIXlMgyJijYi4Mb9OKSJWj4hx+T6anct8+8Xztq8WlJXtJh+ZX0XErflxvsxfY5dHxIYl2i8cEadHxKt524/zGNcpE89aEXFVvs8vI+KDyOYgOTki+lR4/haKiL/mr4238tfUh/lxB7Sw3cIRcVHedlL+Gt65lWN1jYijI+L1PN5XI+KIyHr2tLTdcvlruSm+9yNieESsUKZ9p4j4S0S8kB/nrYg4NSLmruScFOxnFPCf/OHhMeO8FoOK2m4aEf/Lr9mXEfFSRBwfET2qOWYrHs/vuwK98uO2+h6PiM4RsVdEPBLZZ9PEiHg4Iv4UEc16AUcLn5MR0S0iDo6Ixwv29UBE7FIu6Mg+N4+O7LPj84j4NP/38RGxQN5mFBWc6xq95gvP2aaRfb59FhHjI+LaiFi+hW1r9XmzSiWxflsRMUf+vn48f16f5++76yNi+xLtO0fEHyP7/+vTyD7rn4iIfcu8RsbkzyMiYu+IeDKyz4En2vJ5SWpcJjQkqUhK6X2yoQqzAX9rh0P2IBtasDlwN/AwWffvGyJiQETsC1wOfAncQjYk5rfAtRERZfb5F+BCYCJwHfAZ8GvgwTJ/IA8FRgAbAS8C1wNfA/vl25T7g36nfLs58vuHyYbttCgizgYuyp/nvcCNZL0w/gGMiog5CppfQDbvBcD7fDP+/5ZWDjOO7JwB7NFaTEXx7QncB2xD1j3+WuAj4HfAIxGxXJlNfwLcQzbM5X/AXWTX63JgVsp3598pv7+kgtg6Af8lm0fkJ2RzhFwDjCV7De1d1H7tvM2fgK+Am8jmEPkZcFdE7FDUfguya7INWSLoWrLr2hXYl2zukUr8AjgOWAh4No/xVWAz4PaI+E2J57YY2TwnOwOfkr12Pyd7LR9Q3D7fpgvZkKJDgHny5/c8cCBwJVDyPRLZhLKPA7uQXdvrgdeB7YGHovRksRcD/wQWJru+D+fb30H2eVGpW8jOMWTX5oKC2ysFMR5MNhxkgzzWa8muw4F5jAtUccyWzFXw78lFdSXf4/nr8DrgNLL5aW4FbgOWIfv8vDqyYV2timxC0vuBY8g+B0aTvXeWBYZFxGkltlkOeILsus9H9hq4jex6HwismTdt9VzX8DXfZDuy1+FsZJ9tbwNbAw9ExKolnkstP28qHY5Xtfya30r2vl6E7P+rG8g+e/oDuxe175rHdQbZ6+KBfPsFgJOBq1p4jZwFnEh2Pa4HXqvts5E000gpefPmzZu3lCD7Iv51/u+eZF+opgCLFrQZlLc7r2jbYXl5/zL7HpN95M5Q1j/fJgGXAV0K6nbLy18CxgNrF9R1J/vC1ux4ZJP8JbIvrpsVlHcuiPHGom22y8ufBZYqKA/giLxueJnnm4AdqjzPv8y3e6foePOQ/YGcgFOKtumbl4+q8lhnFcT5EHA42Rfq+VrYZi2yZM47wOpFdb/L9/VAUfmQguP8tcQ+18nr7i5zzKbruXSJ19qQoraH5uXPAYsX1c0DrF/weO78eUwFfl3Uth/wMVmya76C8qbX0HYl4lwB6FPhuf8hsGKJ8n7AhPw2R1HdiPzY/wd0LijfIr8mzV4DZBP4JuAxoGdB+RJkXyQTzd97fcmSfROBjYvqNiF737/JjO/JHfJ9vQH0LSifD3i64Pr3bem8tHZ9C+pXz6/bZ8BaBeWzkSXIEnBtFe+Fpus6qETdH/K6sQVlw2jhPU6WYEpkybE+BeULAC/kdfsWbdO0z+LPrZvy8tOA2QvK+/BNknSTgvLOBcf4FzBbidfpElWc66Zz811f84XnbLeC8iBL1ibgKSAK6mr+edNKjP3z7cZU+FyGlNj2YaBrUfuuha/TvOzfefsrgHkKyucquOZ7FG0zJi//EFihmufmzZu37+et7gF48+bNW0e5UZDQyB8fRVHygrZJaHxCwRexvK5T/gddAv5RYn9/zusOLypv+sP8khLb9CD7cjQNWKyg/Il8mx+W2CbIfhn+GuhV4vneWLxNBed5dL7t7iXqVszj+xzoVlDel2+X0OgKnJ/vMxXcpgEPUvqL2rV5m63K7PO6vH7VgrIhlPiyUrTdq/lx+xaVr9b0JaGovOm1NqSgrAtZgmtaJX/sk/26nICTytTvl9fvV1D2bF42Ty3fX0XHPTo/xpYFZYsXvB+aHRsYXuo1QJZgSMCAEtvs0XTNi8pPycv3KRPfqXn9L0q8bnct0X6TgtdW31L7LLFNs+tbVH9BXn9Mibr58vdIs9dTC8cbRVFCI9/Prvk5T8DfCuqG0cJ7vOC8b1yibvO87vWi8qZ99i8oWyUvewSYpcS+Vs3rryso2z4ve6Dc+63Kc12T13zB87u3RF1nvvmyXvj8r6UNPm9aiLE/3z6h0ZT8PrmC4/QmSwyOpShxmdfPT9Yb6Mmi8qZz9Jfvci28efP2/bk55ESSyjuR7FfkXSJiiTY8zqMppY8KC1JKU/lm3oj/ldimaa6FBcvsc3hxQUppPFn36wDWg+ldvX8IvJJSerLENomsK3Yn4EcljnN9meOXFNmEg2vlD5sNr0gpPUOWQOlG9kX/O0kpfZFS+i1Zl/jBZN2/3yc7B2sAwyPi1IL4ZgE2JJtDZUSZ3d6d369Rou7G/JyVckl+3J2Kygfm9xe3/GyArHdDd+CRlNKzFbTfOL+/qkx9qefyaH5/YUT0a2FYU6sioktEbBkRR0U2R8ywiBhG9qUKZuzKv15+f0tK6ZMSu7usxP4XIev6/kFK6c5KtslVdV6KXrf/LW6cUrqFLNFUSz/O70u9Tz6k6L1chf80zSFB1p3/fLKePBeQzR9UrNl7vOC8f5hSavb5lFK6iWzIRN9ofR6ipmtxTSox4W9K6XGynjSFr9GN8vsLW3i/VaNmr/lcqc/fr8l6KkB+bdv486YtPEGWRPttRPw+Wp7HpT/ZMLsRKaXPiytTtoLYy8BKUTRvUq6q/1skfX+5bKsklZFSmhARJ5L11DgcaDbmv0bGlimf2EJ9U125cftvlCkfk983JUL65vdLRtGkpSX0KlFW7fKLPcl6GYwr9Udu7nWyZMZCVe67rJTSK2Tjvo8DiIjVyH7l3BLYJyIuTyndS/Yc58w3m9LK95pqz8fFwGFkCYxj8jhmAXYkG1rQ7EtQCQvn9y9V0Ba+ub73VPFcDibrZr9VfpsQEfeTdRG/IKU0scT2zUQ2seb1ZD0vyimcu6HpNdnaa7dQi9uklD6JiAlkSaBCffP7sRWel6bX7Yep/Aodb5D1gqqVpuc2pkz96/l9te+Te/lmno4vyeIekVJ6okz7Uq/p1q4VZHH3zOMr9xkH31yLf0TEP1poN3vBv6t9H7SmJq/5ApV+/rbl5005lSZAmoKZ3j6l9HJE7A8cD5wLnBMRzwF3AhellB4q2L5vfv/7iPh9K8eal2x4WCGX9pVUERMaktSyU8m67e8UEcd8h/201COutUnc2mySN76J6z2ySfVaUuqP9C9LlHV4KaXH8kkhHyTr9bA52Re9pvMxkfK/3jcp1UOi7PlIKb0UEQ8Dq0fEqvkvzwPIvtyMTNlktLXW9HyuJBuiUM4LBXG+HRFr5LFtBqxPNoHopsDBEbFeSmlMSwfNf+G+giyZcS5wJlmvookppWkRsTtwNmUm7GwHTeflglbaPdjWgdTBeSmlYVW0b+v3eNO1uIdvep61q1q85r+lNvu8aUFTQm6OFlt9Uz/D50ZK6dSIuJJsktONyCYm3QvYKyKOTSkdkjdtem5PkE3G2pLiiWhJKTXk/y2S2p8JDUlqQUrps4g4nuyX/SGUX1ljSn4/Z3FFvjRdrVYjqNSiZOOrS5VDNgEdfPPL6biU0qC2DoqsG/oUoFdEzFGml0bf/L74F7uayr9YjyZLaDT9+tm0Mso0srkSat2d+2KyyR4Hkg2tqWa4CcBb+X2lqy6MJVtdYGhK6dHWGjfJhzzdlt+IiPnJVq74Jdn8FwPLbw1kq1MsRzacavcS9UuWKHs3v1+0RF258ha3iWw51e4lqsaSTRp6QPFwrzKaXrfzRUTXlNIXJdo0Wz3oO3oHWIzs/VDqy2zf/L5N3ydlNH1+lLtWUHl8TZ9B16aUTqzw+IXvg9sq3KZFNXjNF2rtNdx0/tr686aUpvPdKyLmTil9Wqbd4kXtp0spvU22cskZeS+zzcmGdw2OiGEppZcKtrsnpbR38T4kqVacQ0OSWnc62bwL25NNWllK0xerZUrUDaD9E8jbFxdERHeyyQshX8YwpTSW7Nf55SNi6bYOKqX0FdlEflDiC0I+TGE1sl8RH2vrePjmi/XbeXxfk02eODfZ2PZaG042weqvIqIb2Zelz8mWNK3Eo2TzuqweEctX0P7W/P4XVcY5g3y8+5H5w3LvgUJNQy/eKq7Il1ndpsQ29+T3m+SJiGI7lojrjfwYvSNi/Uq2yVV1XvLXbVNvjVLvrY3Jus1XoykJWu6zoWnuhFLvk158MxHpPcX1bS2l9CbZkID58uc+g4jYlGy4yZj8M6Yl3+Y12pTE+E2F8120dq6b+Rav+UKlXiOdgG3zh/fkx2jrz5tmUkrvks1dAdnqQc1ExKJkcytNo5XXV0ppWkrpBrK5noJs6A5kw1CmAlvkc9BIUpswoSFJrcjHzA8l+2Ptj2Wajcrv98wn2gQgIpYkW4qwve0QET8riKMzcBLZnAUjUkqvFbQ9iuz/g6siYtXiHUVEz4jYrYaxNZ2Pw/Pz03ScuciW+Qvg3BbmKqhIRHSPiIciYtv8S3Rh3Sz5uO6tyP5oL0woHJ2X/SciBpTY75wR8dsyE9m1KKX0AdkXuAWBE8i+yFzXwnwixdtPBk4mO0eXR8RiRbHNU/TF/myyiR//GhF/yL9UFbbvHBE/i4gVC8r2j4hSPYo2y++bJSlKeIXsHG4QEcsW7HtWshVGmk2ym1J6lexL0dzAiYWxRsRmZCsslHJmfn9iRExPKuTn5u9ltjkR+AI4ISJ+WVwZEbPlr5vCCS2bjnNEPilmU9tewD/LHKclTb/Sl0qCQvZemAbsHRFrFhyvC1mSdQ7g+jYaClGJpvfxiUWfefOTnV/Ihuy1KKX0INl7Yt2I+HepZFZE/DAiNikoupps/oy1gJNKvL9XiIjCuVtaPNc1es0XWi8ifltU9neyHjfP8c3/F9CGnzctOCW/Py4ilis6Xg/gP2T/J1xZmJCKiAERsXGJz5HeZD3dID9XeS+O88l66lyWvy4o2m7JUu8/SapKvZdZ8ebNm7eOcqNo2daiutnJfsVvWpqxeNnWAG7P6z4kW4rvTrKeBsNpednWYWWOOYoyy0CW27Zgm9PJ/kgeRdYV+NW8/N0y+2taRnMqWS+Ay8nmQHiMrEfBhKL2w2hhmdoKzvXZ+fZfkK08cjnZF+9E9kv4HEXt+1Llsq1kQw2artdn+bm4FLiBbELFlJ+jwSW23SN/3gl4huwL1HCy3iVf5uXdC9oPoWhJzBbi2qkgrgRsWqbdIEosNUn2K/M1ed3k/HV3Kdkv+p+Tdd0vbL8W3ywB/BbZagqX5NuNz8t/XtB+Qv46eCK/LsPJhi+lfP9rVHj+z8q3+RK4mWx1kDfzfZxW5rktTjafSyJLilxGtlzqNLLXdLPXANlknffkdR+TzRdyfX6cG8iXFy0R39Z5m0T2i/X1+Xm8i2xOgwSsUrTN5Xn5RLLlNK/Kz+GjwP1Ut2zr7GQ9v1L+2jwfOA9Yp6DNwXn9V2Rf+i/Lz2FTzAtU8X4YVelrtJL3ONnKRzfxzVK71+S3T/Oy64FOleyTbInPx/K68Xmsl5B9NjQ931OKtlmR7POs6XPtKrL36dM0f023eK6p3Wu+6fmdwTdLQ19aENNnQL/2/LwpE2cAFxW8tkbn5/vmguv3BM2XE983rxtHNufSxflr4LO8/Iqi9l3JkpRN5/Ge/HxcS/b6TTT/vBpDiferN2/evJW71T0Ab968eesoN1pIaOT1e1ImoZHXz0X2Re0dsi+aL5EtFdqp1B9ptG1Coy/ZF+LHyZIG4/I/Phdp4fn9JP9j/m2yLtrjyCZzOw34SVHbpj/c+3+H871z/gfuZ/kf7c8ChwLdSrTtS/UJjSD7Mn84WXLp9fxcfEH2ZflCCr48lth+lfx5jsmv53iyLxv/RzZmPAraDqHyhEa3gi8A7wOdy7QbRIkv/XndLMAuZF9EJuTn73WypEGzawLMTzYPzDNkXyw+z8/Btfl+5ixo++v8tfI82RfVifm/zwCWqOL8dwL2IftiOIksYXUVWZf0lp7bomRfrsbl1+rxPMayr4H8nB5DlryYnJ+Lo8mSHWMo8wWJrKfIv8neq1+QfZl7gSxxsB3Qpah9Z+CvwIv5cd7Ot5+HFt6vLZyjfmRf+CaQfQFu9hoi6yVwa/76m0z2RfCfwLxVvt9Gldp/C+2H0cp7PD8fe5MldJpeV4+STRLZ7HVdsM/1S9TNnu/r3vx8TCZLZowC/gL8oMQ2vfNz8WJ+/Sbkr7ehwPyVnmtq95qffs7IhnPcl+9rAlkCbIX2/rxpJd5tyZIYH5AlNiaQJeYOoPTn8JJkPfruInvtTyb7/250fg47ldimE9kKYbfzzVw0b+fn5u/A0kXtx2BCw5s3b1XcIqWEJEmS1JYi4r9k80uskVJ6uN7x1FpEDCNLvg1IKY2qbzSS9P3gHBqSJElqU/m8C6uQ9Sx4reXWkiRVxoSGJEmS2kxEXEw2j8TSZJMSV7JUriRJrTKhIUmSpLY0kGwel8vIhmRIklQTzqEhSZIkSZIajj00JEmSJElSw+lc7wA6gl69eqW+ffvWOwxJkiRJklTg0UcfHZdSmq9UnQkNoG/fvjzyyCP1DkOSJEmSJBWIiDfK1TnkRJIkSZIkNZy6JTQiYruIuD4i3o6IiRHxaET8qkS73SLi5Yj4Mm+zYYk2C0XENRHxWUSMi4jTI6Jb+zwTSZIkSZLU3urZQ2N/YCKwH7AVcCdwaUTs3dQgT3CcBVwIbAo8C9wYESsWtJkVGAksCuwI/BnYDjinfZ6GJEmSJElqb/WcQ2PLlNK4gsd3RMSCZImO0/KyIcAFKaWjACJiNLAqMBjYOW+zLbAcsGRK6fW83VfA8Ig4IqX0cps/E0mSJEmS1K7q1kOjKJnR5HFgQYCIWBxYGri8YJtpwBVkvTWabAo83JTMyF0LTAE2qW3UkiRJkiSpI+hok4KuDbyU/3vZ/P6FojbPA/NGxHwF7WZok1KaArxasA9JkiRJkjQT6TAJjXyyz58DJ+ZFPfL7CUVNxxfV9yjRpqldjxLlkiRJkiSpwdVzDo3pIqIvcClwXUppWDsdc3dgd4A+ffowatSo9jisJEmSJEmqgbonNCJiXmAE8AYwsKCqqSfGPMzYA6NHUf34vE2xHsCT5Y6bUjqHfCWUfv36pf79+1cZuSRJkiRJqpe6DjmJiG7AjUAXYIuU0qSC6qZ5MYrnwVgW+Dil9GFBuxnaREQXYHGaz78hSZIkSZJmAnVLaEREZ7IVS5YCNkkpfVBYn1J6jWyC0O0KtpklfzyioOkIYPWIWLSgbCtgNuCWtolekiRJkiTVUz2HnJwBbAb8GegZET0L6h5PKU0GhgAXR8QY4F5gF7IEyE4Fba8E/gZcHRGHkQ0/ORm4NKX0cls/CUmSJEmS1P7qmdDYOL8/tUTdYsCYlNJlETEncBBwGPAs2dCUZ5oappS+iohNgNOBy4HJwHDgwLYMXpIkSZIk1U/dEhoppb4VtjsXOLeVNmPJlnyVJEmSJEnfA3WdFFSSJEmSJOnbMKEhSZIkSZIaTj3n0NB30HfwTfUOoSpjhm5e7xAkSZIkSTMRe2hIkiRJkqSGY0JDkiRJkiQ1HBMakiRJkiSp4ZjQkCRJkiRJDceEhiRJkiRJajgmNCRJkiRJUsMxoSFJkiRJkhqOCQ1JkiRJktRwTGhIkiRJkqSGY0JDkiRJkiQ1HBMakiRJkiSp4ZjQkCRJkiRJDceEhiRJkiRJajgmNCRJkiRJUsMxoSFJkiRJkhqOCQ1JkiRJktRwTGhIkiRJkqSGY0JDkiRJkiQ1HBMakiRJkiSp4ZjQkCRJkiRJDceEhiRJkiRJajgmNCRJkiRJUsMxoSFJkiRJkhqOCQ1JkiRJktRwTGhIkiRJkqSGY0JDkiRJkiQ1HBMakiRJkiSp4ZjQkCRJkiRJDceEhiRJkiRJajgmNCRJkiRJUsMxoSFJkiRJkhpOXRMaEbFkRJwdEU9FxNSIGFVU3z8iUpnbyIJ2g8q02aPdn5QkSZIkSWpznet8/BWAzYAHgFlL1D8GrF1UtgjwX2BEifYbAF8UPH6tBjFKkiRJkqQOpt4JjRtSStcBRMSVQK/CypTSp2TJjuki4sfANODyEvt7OKU0sY1ilSRJkiRJHURdh5yklKZ9i81+BYxOKb1T63gkSZIkSVJjaKhJQSNiaWBV4LIyTV6NiK8j4sWI+EM7hiZJkiRJktpRvYecVGtH4CvgqqLyd4HDgIeATnm7syKiW0rp5PYNUZIkSZIktbVGTGj8L6X0cWFhSmkkMLKgaEREzA4cGhGnlhraEhG7A7sD9OnTh1GjRrVd1G3ggJW+rncIVWm08ytJkiRJ6tgaJqERET8ElgOOrnCTK4Htgb6UWO0kpXQOcA5Av379Uv/+/WsSZ3sZNPimeodQlTED+9c7BEmSJEnSTKSR5tDYkWxJ1usqbJ+K7iVJkiRJ0kyi0RIaN1SxLOu2wDjgjbYLSZIkSZIk1UNdh5xERDdgs/zhQsDcEbFt/vjmlNKkvN1aZENH9iuzn6vIJgR9imxS0B3y2z7fcmlYSZIkSZLUgdV7Do3ewBVFZU2PFwPG5P/eEfgEGFFmPy8CvwUWBgJ4DvhNSumiWgYrSZIkSZI6hromNFJKY8gSEK212xfYt4X6Q4BDahWXJEmSJEnq2BppDg1JkiRJkiTAhIYkSZIkSWpAJjQkSZIkSVLDMaEhSZIkSZIajgkNSZIkSZLUcExoSJIkSZKkhmNCQ5IkSZIkNRwTGpIkSZIkqeGY0JAkSZIkSQ3HhIYkSZIkSWo4JjQkSZIkSVLDMaEhSZIkSZIajgkNSZIkSZLUcExoSJIkSZKkhmNCQ5IkSZIkNRwTGpIkSZIkqeGY0JAkSZIkSQ3HhIYkSZIkSWo4JjQkSZIkSVLD6VzvACRJkqRGcsopf2fChDfrHYakBtO9+yLsu++R9Q5jpmJCQ5IkSarChAlvMmRI33qHIanBDBkypt4hzHQcciJJkiRJkhqOCQ1JkiRJktRwTGhIkiRJkqSGY0JDkiRJkiQ1HBMakiRJkiSp4ZjQkCRJkiRJDceEhiRJkiRJajgmNCRJkiRJUsMxoSFJkiTVyZgxE4g4gkceeafeocyUBgy4gAsvfLLeYTB58tcsssjJXuca6N9/GHvtdXO9w1AHYUJDkiRJagMRR7R4GzTo2nqH2CG8++5n7LLLtcw33z+ZffZ/sPzy/2b06DHT6w877A6WXfZ05pjjGHr0OI4NN7yQ++57q9X93nTTS7z11icMHLjS9LJzznmUAQMuoHv3oUQcwZgxE0puO3LkK6y99v/RrdvRdO8+lA02uGB63ccff8GWW17GnHMew6qrns3jj787w7YHHDCSQw65fYay2WbrzIEHrsNBB91WwRmpn4gjuPLK5+odRouuvnoHjj12w5rtb9Cga9lii0trtj+1r871DkCSJEmaGb377gHT/33jjS+x2243zFDWtWtnxo//sk2OPWXKVLp06dQm+y42bVoipUSnTtX/Vjphwpesu+75rLfeItx0007MN183XnttPL17zzG9zTLL9OLf/96MxRbrwRdffMXJJz/AJptczMsv702fPnOW3feppz7IoEGrzBDXpElfsfHGi7P11suw334jS2537bUvsOuu13H00RswbNjWTJuWeOyxb5IWRx99F599NpnHHvsDZ575MLvtdgOPPLI7AI8++g433/wKjz/+h2b7HThwZf7yl1t59tkPWGGF3lWfq5ldpa/Zeeft2g7RqFHYQ0OSJElqA/PPP+f0W/fuszcrm2ee2ae3feONCfz0pxfRrdvRLL/8v7n11ldn2Ndzz33I5ptfylxzHUvv3v/kV7+6ivfemzi9vulX5uOOu4cf/OAkfvCDkwB4++1P2XHHK+nR4zh69DiOzTe/lJdf/qjFuE866X5WXvlM5pjjGBZa6CR+//vrmTDhm8TLsGFPMOecx3DzzS+z4opn0KXLUTz//DimTJnKQQfdyg9+cBLduh3N6qufy8iRr7R4rOOPv5cFFpiLCy/8BWussRCLLdaDDTdcnOWWm296m513XpkNN1ycxRfvwQor9Oakk37GZ59N4Ykn3iu73w8//JzbbnuNLbdceobyffddi4MP/jHrrbdIye2mTp3GPvuM4PjjN2LPPVdnmWV6sdxy8zFw4MrT2zz//Dh23HFFll66J7vv/iOef34cAF9/PY3ddruBM87YjNlnb/678bzzdmXddRfmssueafGctHbNhgwZxYornsEFFzxB376nMMccx7DrrtcxZcpUzjjjYRZe+GR69jye/fcfybRpafp2ffuewpAho9h556uZc85jmH/+EzjhhPtmqAfYbrsriDiCvn1PYcyYCcwyS/MhUeee+yi9eh3PlClTSz6H/v2HscceN/LnP4+Y/jwOPPB/JeP57W+vo3v3oQwceDUAV1/9PCutdCazzfYPFl74ZI4++i5SSjPsu3DISSWvuxdeGMdWW13GPPMMZc45j2Httf+Pp59+nyFDRnHBBU9y000vT+85NWrUGACOPHI0iy56CrPN9g/mn/8EfvOba1q8bqoPExqSJElSnf3tb3ewzz5r8OSTe7D66gux445XMXHiFCAbkvGTn/yHFVecj4ce+j233fYbJk6cwtZbD5/hC+Lo0W/w1FMfcMstO3P77b9h0qSvGDDgAmafvTOjRw/i/vt/xwILzMlGG13EpElflY1lllmCU07ZhGef3ZNLL92Ghx56m733HjFDmy+//JqjjrqLs8/eguee+xOLLjoPu+56HaNHv8Gll/6SZ57Zk112+SFbbnkZTz5ZPvFw7bUvsOaaC7HDDlfSu/c/WWWVszj99Idm+AJbaMqUqZxzzqPMPfdsrLLK/GX3e889bzLbbJ1ZccXqekI8+ui7vPXWp3Tp0onVVjub+ec/gY03vmiGYSU//GEf7rjjdb7+ehojR77Kyiv3AbJE0Kqrzs+AAYuV3f8aayzE6NFvlK2v9JqNGTOB6657kRtv3Imrr96eK654lq22uoyHH36H//1vZ847b0tOO+0hrrnm+Rn2f9JJ97Pccr147LE/cMQR/TnkkNu5+uqszcMP7wbAueduybvvHsDDD+9G377d+elPl+D88x+fYT/nn/8Ev/71yi32qLjkkqeZNi1x//2/4+yzt+Cccx7jlFMeaBbPssv24pFHdueYYzbg0UffYbvtrmCbbZbl6af/yNChG3Lssfdw+ukPlT1Oa6+7d975jPXWO5+I4NZbf81jj/2BP/1pdaZOTfzlL+uw/fYrsNFGi/Puuwfw7rsHsM46C3PVVc9xwgn3ccYZm/Hyy3tz4407scYaC5WNQfVT1yEnEbEkcCCwNrACcHdKqX9RmzHAokWbvp9Smr+o3fLAafm+JgDnAUeklEqnDSVJkqQOYr/91mLLLZcB4JhjNuDCC5/kiSfeY731FuHMMx/hhz+cn+OO++n09hde+HPmnfd4HnnknelftGafvTPnn78Vs82W/Yl//vmPkxL85z9bExEAnH32FvTufQI33vgS22+/QslY9t13ren/7tu3O8cf/1O23no4F1zwc2aZJdvP1KmJ00/flB/9aEEAXn31Yy677GnGjNmXRRaZB4C99lqD2257jbPPfpQzzti85LFee208Z5zxMPvttxaDB+/ME0+8Nz15stdea0xvd+ONL7HjjlcyadJXLLDAXNx6669bHG7yxhuf0Lv3HFUPg3nttfEA/P3vozjxxI1ZbLHu/PvfD9O//wW88MKfWGCBuRg8eD3++MebWGKJf9G3b3f+7/+24rXXxnPmmY/wwAO/Y++9b+bmm19h2WV7cd55W7LAAnNN3/+CC85Vdt4OgOHDn6nomk2dmvjPf7ZmnnlmZ8UVe7PJJksyevQbvP32r+jSpRPLLTcf6667MHfeOYZf/nL56ftfc80f8Le//QSApZfuycMPv8NJJ93PNtssx3zzZcN8unefnfnn/+bc7rbbauy22w2cdNLPmH32zjz//Ic88MBYzj13yxbP5QILzMm//rUpEcGyy/bipZc+4qST7mf//dee3mb99fvy17+uO/3xwIFXs/76i3LEEQOmx/jyyx9z3HH3svfeazY7RiWvu3//+yHmmKMLV1yx3fQEzNJL95y+j65dOzPbbJ1meM5vvPEJCywwFxtvvASzztqJRRaZh379Fmzx+ao+6j2HxgrAZsADwKwttLuULFnRZEphZUT0AG4DngO2BpYATiTrgXJoDeOVJEmSaq7pV37IvvQCfPDB50DWa+Cuu95gzjmPabbdq69+PD2hseKKvacnM7Lt3uH118cz11zHzrDNpElf8eqrH5eN5Y47XufYY+/h+ec/5JNPJjN16jSmTJnKe+9NnB5b586zzNBD4rHH3iUlWH75f8+wr8mTp7LBBuV7LEyblujXb0GOPXYjAFZddQFefvlj/v3vh2dIaAwY0JcnntiDceMmce65j7L99lfkvRfmKrnfL774quSwj9Y09Xj5299+zLbbZomAc87Zkttue40LL3ySgw5aj3nmmZ1LL/3lDNttvPFFHHvshlxxxXM8++yHPP/8nzjiiFHss88tXHHFdtPbde3amS++KN87ptJrtsgi88wwZKlPnzlYeumeM/SY6NNnzumvoSZrr/2DZo+bemiUs/XWy/CnP93M1Vc/z047rcT55z/OGmss1Grvl7XW+sH0pEzTsQ477E4+/XQyc889GwD9+i0wwzbPP/8hm2++1Axl6623CEccMXqG7ZpU8rp7/PEsMVjNnDLbbbc8p576IIstdio/+9kSbLLJkmy11TIzvL/UMdT7ityQUroOICKuBHqVafduSumBMnUAewBdgW1SSp8Ct0bE3MCQiDg+L5MkSZI6pFln/ebLVtOXwKYv19OmJTbffClOOGHjZtv16fPN5JlzzDHj74PTpiVWWWV+hg/fttl25SZWfOONCWy++aXstttqHHlkf3r27MZjj73Lr3511QzzJcw2W6cZej9Mm5aIyIYtFD4XyL7El7PAAnOx/PLzzVC23HK9OPXUT2Yom2OOLiy55LwsueS8rLXWD1hqqdM477zHOOyw9Uvut1evbowf/0XZ45aPJ/uVvjCmzp1nYamlevLmm5+U3ObCC59k1lk7seOOK7LNNv/ll79cji5dOrHTTivxk58Mm6Htxx9/Mb0nRCmVXrNZZ52x50lElCjLenJ8V7PO2onf/GZlzj//cbbffgUuuugpjjxywHfeL2TXtVIFuZHpvu3rrjULLzwPL764F7ff/hq33fYaBxzwP444YjQPPvj7qmJW26trQiOlNK1Gu9oUGFmUuBgOHAesD9xQo+NIkiRJ7Wq11ebn8sufY9FF52n2pa3l7RbgssueoVevbtMnJW3NI4+8w5QpUzn55J9NT1jceONLrW636qoLkBK8997EFueQKLbuugvz4oszTlL60ksfseii87S43bRpicmTy48sX3XVBfjww0mMGzeJXr26VRzPj360ILPN1okXXxw3feLQadMSr776MT/72RLN2n/44ef8/e93ctddu05v+9VX2VecKVOmMnXqjF93nnnmQ1ZbbYFm+2nyba5ZNR54YGyzx4UTsM466yzNYgb4/e9XY/nlz+CMMx7ms8+msOOOK7Z6rAcffJuU0vQE3QMPjGXBBedq1sui0HLLzce99864JO8997zJD34wN3PN1Xy7Sl53q646Pxdf/HTZVVS6dOlUMvEz++yd2Xzzpdl886UZPHg95p//RO699y023rj560D10yiTgv4uIqZExCcRcWVEFM+psSzwQmFBSulNYFJeJ0mSJDWkP/1pDT755Et22OFKHnxwLK+9Np7bbnuN3Xe/gc8+m1x2u4EDV6ZPnznZeuvhjB49htdfH89dd73BAQeMLLvSyVJL9WTatMQppzzA66+P57LLnm42kWMpSy/dk4EDV2LQoOu48srneO218TzyyDuccMJ9LQ5p2G+/tXjggbEcffRdvPLKx1xxxbP8618P8ac/rQ7Ap59O5tBD7+DBB8fy5puf8Oij7/Db317H2LGflp0DBLIvsb17z8E997w5Q/l7703kiSfe46WXsuf/3HMf8sQT7/Hxx1lvjrnnno099ujH4YePYuTIV3jxxXH8+c8jGD/+S37965WbHWe//Uay335rTZ+/Yb31FuGCC57k+ec/5JRTHmy2msrdd7/BJpuU/0L8ba5ZNR54YCzHHns3L7/8Eeee+ygXXvgU++0345wpt9/+Ou+9N3GGHi7LLNOL9dZbhAMPvJVtt12+xaREk3fe+Yx9972FF18cx5VXPsc//3nfDMcq5YAD1mb06DcYMmQUL730EZdc8hQnnng/f/3rOiXbV/K623PP1Zk4cQrbb38FDz/8Nq+8ks270bRKTt++3XnmmQ948cVxjBs3ia++msqwYU9w3nmP8fTT7/P66+P5z3+eYNZZZ2GppeYF4PTTH2LZZU9v9Ryo7dV7yEklriObY2MssBxwOHB3RKyUUmrq99WDbCLQYuPzOkmSJKkhLbjgXNx77285+ODb2WSTS/jyy69ZZJF52HjjxVsc09+t26zcddcgBg++ne22u4JPPpnMggvOxYABfenRo/SQk5VX7sOpp27Cccfdy6GH3sk66yzMCSdszA47XNlqnP/5z9YcffTd/PWvtzJ27KfMO29X1lhjIQYM6Ft2m9VXX4hrr92BQw65g6OOuotFFpmHo44awJ57ZgmNzp1n4dlnP+T88x/no4++oGfPrqy++kLcddegGeYdKdap0yz89rercMklT/Pzn3/z++ZZZz3CEUeMnv54880vnR77oEGrAPDPf/6ULl06scsu1zJp0lesttoC3HnnLs3m6xg58hVeeukjLrzwF9PL/vSn1XnssXdZc83zWGGF3lxyyTbT6+6//y0++WTy9Lk5Svk216wa+++/Nk899QFHH303c8zRhSOP7D9DPCeeuDH77/8/Fl74ZBZaaC7GjNl3et3vfrcqd931Br/73aoVHWvgwJWYOjWx5prnERH87nertprQWG21Bbjiiu04/PBRHHPM3fTpMyeDB687w3wqxVp73S200NzcddcgDjzwVgYMuICIYKWVenPOOdmkprvtthqjRo2hX79zmThxCnfeuQvdu8/Occfdy1/+8j+++moayy8/H1dfvQOLLZZ9tRw3blKznkWqjyi3JFJ7a5pDo3iVkxLtVgSeAP6SUjolL/sKOLDpcUHbscCFKaVDSuxnd2B3gD59+vxo+PDh3/1JtKOn3y49hq+jWmmhlrsNSpIkNYrhw4cyeHD5JUPVMYwb9wUbb3wd1123OQsvXHry0Pa0556jWGGFefnTn5r39GgP6613Jb/5zbLsvnvrw0VKOeusp/nvf1/hzjt/0WrbHXe8haWX7s6RR7acwPg2ttnmZtZcsw8HHfSjmu+7rQ0d+h477ji43mE0nAEDBjyaUupXqq4RemjMIKX0TES8CKxWUDweKPWNuUdeV2o/5wDnAPTr1y/179+/xpG2rUGDb6p3CFUZM7B/vUOQJEmqiVGjhtG3b996h6FW9O0Lw4bNwrRps9X9ek2e/DVrr70Yf/nLOnTt2tLijm2nc+fOzDvvvFWfi4kTp/DGGxO48MKX+NvfflzR9rPPPjtzzz13Tc/75Mlf8/TTH/DKK5+w//7r1f2afhvzzw+N9r2zo2u4hEYu5bcmL1A0V0ZELAx0o2huDUmSJEnfD1tttUy9QwBgttk6l12RpaPba6+bueyyZ9hqq2X4wx9K/kjeLkaMeIXf/OYattpqGXbYofz8Kfp+abiERj7kZFny3hW5EcCBETFXSumzvGwH4AtgNJIkSZL0PVY4H0Y1hg37OcOG/byqbUaNGvStjtWSn/98WT799OCa71eNra4JjYjoBmyWP1wImDsimhZdvhkYAOwM3Ai8Q5bIOBR4ExhWsKuzgH2AqyPiOGBxYAhwUtFSrpIkSZIkaSZQ7x4avYErisqaHi8GvJW3OQXoDnwE3AIcUpioSCmNj4gNgdOBG8hWPDmZLKkhSZIkSZJmMnVNaKSUxgDRSrMNK9zXc8AG3zUmSZIkSZLU8c1S7wAkSZIkSZKqZUJDkiRJkiQ1HBMakiRJkiSp4ZjQkCRJkiRJDceEhiRJkiRJajgmNCRJkiRJUsOp67KtkiRJUqPp3n0RhgwZU+8wJDWY7t0XqXcIMx0TGpIkSVIV9t33yHqHIEnCISeSJEmSJKkBmdCQJEmSJEkNx4SGJEmSJElqOCY0JEmSJElSw6k4oRERP4mIPxeVDYyIVyPio4g4LSJMkEiSJEmSpDZXTQLiCGDdpgcRsTRwPjAFeAjYE9i7ptFJkiRJkiSVUE1CYwXgwYLHOwFfAmullDYFLgV2rWFskiRJkiRJJVWT0Jgb+Ljg8SbAbSmlT/LHo4HFaxWYJEmSJElSOdUkNN4jT1hERE/gR8DtBfVzAtNqF5okSZIkSVJpnatoOxLYKyImAP3JkhfXF9QvD7xZs8gkSZIkSZLKqCahcSiwLPBPsolA908pjQWIiNmBbYELax6hJEmSJElSkYoTGimlD4H1I2Ju4IuU0lcF1QEMAN6qcXySJEmSJEnNVDSHRkR0i4hHI2KPlNKnRckMUkpfpJSeTCl9XG4fkiRJkiRJtVJRQiOlNIlsQlAn/ZQkSZIkSXVXzSond5JNBipJkiRJklRX1SQ09gF+GBEnR8TSEVHNhKKSJEmSJEk1U01S4vX8flmy5AYRMbWoTUopzVaLwCRJkiRJksqpJqFxCZDaKhBJkiRJkqRKVbNs66A2jEOSJEmSJKli1cyhIUmSJEmS1CFUldCIiCUj4qKIeDsipkTEBnl5r4g4PyLWaJswJUmSJEmSvlFxQiMiVgQeATYDHgc6NdWllMYBKwF71DpASZIkSZKkYtVMCjoU+BhYk2xy0A+K6m8BdqhRXJIkSZIkSWVVM+Tkx8CZKaUPKb3ayZvAgjWJSpIkSZIkqQXVJDQ6AxNbqO8FfP3dwpEkSZIkSWpdNQmNp4ANSlVERADbkM2xIUmSJEmS1KaqSWicBPwyIo4E+uRlXfLJQq8EVgNOrObg+aopZ0fEUxExNSJGFdUvEBH/jIgnI2JiRLwVERdExIJF7fpHRCpxG1pNPJIkSZIkqTFUPCloSumKiFgUOAb4W158U34/FTggpTSiyuOvQLZqygPArCXqfwT8AjgPeJAskTIEuC8iVkwpFQ+BGQi8VvD47SrjkSRJkiRJDaCaVU5IKZ0QEcOBbYGlyHp4vAJclVIa8y2Of0NK6TqAiLiSbB6OQvcAy6aUps/NERGPAS8CvwQuKGr/VErpmW8RhyRJkiRJaiBVJTQAUkpjgVNqcfCU0rRW6ieUKHspIibhiiqSJEmSJH1vVTyHRkR0jYjeRWW9I+KwiDgxItasfXgl41gZ6Aa8VKL6jnwujjERcWhEdGqPmCRJkiRJUvuqpofGOWRzXqwGEBHdyOa+6JvX7x0RA1JK99Y0wgIRMQtwKvAycH1B1SfAUOBuYAqwBXAEMB/w57aKR5IkSZIk1Uc1CY0fA8MKHu9IlszYFHgS+B9wCLB5jWIr5VhgbWD9lNJXTYUppceBxwva3RYRk4H9I+KolNK44h1FxO7A7gB9+vRh1KhRbRh27R2w0tetN+pAGu38SpIkSZI6tmoSGr2BNwsebwE8lFIaCRAR/wH+WsPYZhARewIHAr9KKT1YwSZX5vGsDNxRXJlSOoes1wn9+vVL/fv3r12w7WDQ4Jtab9SBjBnYv94hSJIkSZJmIhXPoQFMAuaC6UM/+gO3FtRPBLrXKrBCEfFL4DTgryml/1a4WSq6lyRJkiRJM4lqEhpPATtHxLzA74B5gBEF9X2BD2oXWiYi+gOXAKellE6oYtNtga/J4pYkSZIkSTORaoacHAXcBHyYPx6dUrqvoH4L4KFqDp5PLLpZ/nAhYO6I2DZ/fDOwKHAt8ALw34hYq2DzD1NKr+b7OTOP62GySUE3A/YCTkkpfVRNTJIkSZIkqeOrOKGRUrozIlYDNgYmANOHfuS9Nu4gSz5UozdwRVFZ0+PFgDXJeoL8ELivqN0FwKD8388Dvwf2A7oArwAHkK2IIkmSJEmSZjLV9NAgpfQCWW+J4vKPyZIJVUkpjQGihSbDmHFllXL7+Rfwr2qPL0mSJEmSGlM1c2hIkiRJkiR1CC0mNCJiWkRMbeE2MSKejYjj8mEnkiRJkiRJba61ISfn0/Kyp92A5YADgW0jYs2U0rhaBSdJkiRJklRKiwmNlNLvK9lJRKwPjAQOA/5cg7gkSZIkSZLKqskcGiml0cD/kS3dKkmSJEmS1KZqOSnoU8BCNdyfJEmSJElSSbVMaMwLfFnD/UmSJEmSJJVUk4RGRHQCtgEer8X+JEmSJEmSWtLipKARsWAr23cFlgX2BFYDtq1RXJIkSZIkSWW1tmzrWFpethUggMnAX1NK19QkKkmSJEmSpBa0ltA4kpYTGl8CY4DbUkof1SooSZIkSZKklrSY0EgpDWmnOCRJkiRJkipWy1VOJEmSJEmS2oUJDUmSJEmS1HBMaEiSJEmSpIZjQkOSJEmSJDUcExqSJEmSJKnhlE1oRMTHEbFtweO/R8SK7ROWJEmSJElSeS310JgT6FrweAiwcptGI0mSJEmSVIGWEhpjgM0iYq6CstS24UiSJEmSJLWupYTGacAOwISImEqWzLg4Iqa2cPu6XaKWJEmSJEnfa53LVaSUTouIp4CNgD7A74DRwKvtFJskSZIkSVJJZRMaACml0WRJDCLi98B5KaVL2yMwSZIkSZKkclpMaBRKKbnEqyRJkiRJ6hAqTmg0iYhNgC2AxfKi14HrU0r/q2VgkiRJkiRJ5VSc0IiILsAVZMmMAMblVZsCf4yIG4DtU0pTah6lJEmSJElSgWqGkRwObAmcCPROKfVOKfUG5gNOALYCDqt9iJIkSZIkSTOqJqGxE3BxSumvKaWm3hmklD5KKR0EXAzsXOsAJUmSJEmSilWT0FgQuK+F+vuBBb5bOJIkSZIkSa2rJqHxDrBWC/VrAu9+t3AkSZIkSZJaV01C4zLg1xHxj4jo0VQYET0i4ijg18CltQ5QkiRJkiSpWDXLth4BrAIcAhwcER/m5fORrXpyC3BkTaOTJEmSJEkqoeKERkppMrBZRGwBbA70zavGADeklG6ueXSSJEmSJEklVDPkBICU0o0ppT+mlDbNb3/8tsmMiFgyIs6OiKciYmpEjCrRJiLikIh4KyK+iIi7ImKVEu2Wj4jbI2JSRLwTEUdGRKdvE5ckSZIkSerYqk5o1NgKwGbAi8BLZdoMBg4DjgO2BCYCt0XE/E0N8jk9bgMSsDXZ0JcDyIbJSJIkSZKkmUy9Exo3pJQWTiltBzxbXBkRs5MlNI5NKZ2eUroN2I4scbFXQdM9gK7ANimlW1NKZ5ElM/aPiLnb/FlIkiRJkqR2VdeERkppWitN1gHmBi4v2OZz4AZg04J2mwIjU0qfFpQNJ0tyrF+baCVJkiRJUkdR7x4arVkWmAq8XFT+fF5X2O6FwgYppTeBSUXtJEmSJEnSTKCjJzR6ABNTSlOLyscD3SKiS0G7CSW2H5/XSZIkSZKkmUhFy7ZGRDfgLWBoSumfbRtS+4iI3YHdAfr06cOoUaPqG1CVDljp63qHUJVGO7+SJEmSpI6tooRGSmlSREwjW2GkPY0H5oyITkW9NHoAk1JKUwrazVNi+x55XTMppXOAcwD69euX+vfvX7Og28OgwTfVO4SqjBnYv94hSJIkSZJmItUMObmBbNnU9vQC0AlYsqi8eM6MFyiaKyMiFga6FbWTJEmSJEkzgWoSGicBC0fE1RGxcUQsERELFt9qHN99wKdkS7UC04e/bAmMKGg3AvhZRMxVULYD8AUwusYxSZIkSZKkOqtoyEnuqfx+BWDrFtp1qnSHeXJis/zhQsDcEbFt/vjmfKjLUOCwiBhP1ttif7JEzGkFuzoL2Ae4OiKOAxYHhgAnFS3lKkmSJEmSZgLVJDSOBFKNj98buKKorOnxYsAYYChZAuNgoCfwCPDTlNL7TRuklMZHxIbA6WRDYyYAJ5MlNSRJkiRJ0kym4oRGSmlIrQ+eUhoDRCttEnB0fmup3XPABjULTpIkSZIkdVjVzKExXUTMHhELRUSXWgckSZIkSZLUmqoSGhGxXkTcDXwGvAmsl5f3iojbI2LjNohRkiRJkiRpBhUnNCJiPeB2YH7gPAqGiqSUxuWPf1vrACVJkiRJkopV00PjH8BzwIrAYSXqRwNr1CIoSZIkSZKkllST0OgHDEspTab0aidvk/XekCRJkiRJalPVJDSm0fKyrQsCk75bOJIkSZIkSa2rJqHxMLBVqYp8tZOBwH21CEqSJEmSJKkl1SQ0jgH6R8SFZMNPABaOiC2Au4DF8jaSJEmSJEltqnOlDVNKt0fETsAZZL0xAM4nW91kArBTSumBmkcoSZIkSZJUpOKEBkBK6fKIuBHYGFiKrIfHK8DIlNLENohPkiRJkiSpmaoSGgAppUnAtbUPRZIkSZIkqTJVJzQi4qfAFmRzZgC8DtyUUvpfLQOTJEmSJEkqp+KERkTMAfwX2JRs3ozxedUWwF4RMRLYLqX0ec2jlCRJkiRJKlDNKif/BDYDjgV6p5R6ppR6Ar2BocAmeRtJkiRJkqQ2VU1CY3vg/JTSoSmlcU2FKaVxKaW/AcPyNpIkSZIkSW2qmoRGF+CRFuofBmb9buFIkiRJkiS1rpqExp3Ahi3Ub5i3kSRJkiRJalPVJDT2BJaJiPMj4ocRMXt++2FE/AdYKm8jSZIkSZLUpsquchIRXwGpqHgWYEVglxKbJLIlXGerWXSSJEmSJEkltLRs6yU0T2hIkiRJkiTVXdmERkppUDvGIUmSJEmSVLFq5tCQJEmSJEnqEFoactJMRMwCbAQsDvQAoqhJSikdW6PYJEmSJEmSSqo4oRER/YArgEVonshokgATGpIkSZIkqU1V00PjbLJeGX8C7gcmtEVAkiRJkiRJrakmobE8MCSldFZbBSNJkiRJklSJaiYFfR34uq0CkSRJkiRJqlQ1CY1jgT9ERPc2ikWSJEmSJKkiFQ85SSldFBGzAi9HxDXAW8DU5s1c5USSJEmSJLWtalY5WQk4AugJ/L5MM1c5kSRJkiRJba6aSUHPAuYC9sRVTiRJkiRJUh1Vk9BYFTjCVU4kSZIkSVK9VTMp6Ju4yokkSZIkSeoAqkloDCVb5aRHWwUjSdL/t3fncZJV9d3HP1/BDRUdwUA0PuJOVIzCREWNgktU3Hc0UTEaYnxcURMkuG8T3ONOfAwaRVDcRcQFBzG4AQY3QEXHHQUEkU0Ef88f5za0Rc909XR137rF5/161aum7r1963e6uqZufe8550qSJEnjWMqQk+sDvwNOTXIYq3SVkyTrgbtvZPWdq+rLSTYANxpZ96uq2n6StUiSJEmSpOmwlEDj5fP+vZpXOXkqsPXIspfS5vT4+rxlBwNvmvf4ognXIUmSJEmSpsRSAo0br1gVm1BV353/OMlVgLXAoVU1f06PX1bVV1a1OEmSJEmS1IuxA42q+vFKFrIE9wXWAO/vuxBJkiRJktSPpUwKOi32BH4GHDOy/ElJLkry2ySHJRmdU0OSJEmSJM2IsXtoJPkRbY6MTamquunyStpkDVsBDwLeUVXza/kY8BVa0PGXwIuAY5LsVFW/Xal6JEmSJElSP5Yyh8bRXD7Q2ALYAbgz8C3gfydS1cY9ELgGI8NNquqZ8x4ek+TYrpYnAm9YaEdJ9gb2Bthuu+1Yv3795KtdQc/Z6eLFN5oiQ/v9SpIkSZKmW/60o8Nm7iTZGTgC2LOqvrDsHW78eT4C3Kaqbj7Gtt8Bjq+qxy+27dq1a+u4446bRImrZod9D++7hCXZsO7+fZcgSZIkSRqYJMdX1dqF1k1kDo2qOgH4T+CASexvIUmuDdyP8ScDLRYfIiNJkiRJkgZokpOC/gy49QT3N+qhwFUZI9BIchtgR+D4FaxHkiRJkiT1ZClzaGxUkivTrj7yq0nsbyP2BE6sqpNGnvv+wN8DnwR+QQsy9gd+Ahy0gvVIkiRJkqSeLOUqJ+/ayKrrALsC2wFPn0BNCz33tsA9gRcssPqnwJ/RJv+8DnAm8Glgv6o6ZyXqkSRJkiRJ/VpKD417cPk5KQo4C/gi7VKqR02qsD95kqozgCtvZN03aWGHJEmSJEm6ghg70KiqHVawDkmSJEmSpLFNclJQSZIkSZKkVWGgIUmSJEmSBmeTQ06S/JHLz5uxKVVVE7lyiiRJkiRJ0sYsFj68i/ECjZ2B2y+/HEmSJEmSpMVtMtCoqidvan2SvwJeQgszzqFdOlWSJEmSJGlFbdbwkCS3Bl4MPBQ4H3gl8JqqOntilUmSJEmSJG3EkgKNJDvSgoxHABcArwEOqKrfTL40SZIkSZKkhY0VaCS5OfAi4NHARcDraUHG6StYmyRJkiRJ0oIWu8rJTYAXAo8FLgbeDKyrql+tQm2SJEmSJEkLWqyHxinAlYBvAOuAXwA3TXLTjf1AVR07ufIkSZIkSZIub7FAY4vufmfg0EW2De0Sr1sssp0kSZIkSdKyLBZoPHFVqpAkSZIkSVqCTQYaVfXu1SpEkiRJkiRpXFfquwBJkiRJkqSlMtCQJEmSJEmDY6AhSZIkSZIGx0BDkiRJkiQNjoGGJEmSJEkaHAMNSZIkSZI0OAYakiRJkiRpcAw0JEmSJEnS4BhoSJIkSZKkwTHQkCRJkiRJg2OgIUmSJEmSBsdAQ5IkSZIkDY6BhiRJkiRJGhwDDUmSJEmSNDgGGpIkSZIkaXAMNCRJkiRJ0uAYaEiSJEmSpMEx0JAkSZIkSYNjoCFJkiRJkgZny74LkEbtsO/hfZcwtg3r7t93CZIkSZJ0hTT1PTSS7JWkFrg9Zd42SbJfkp8muSDJF5PcrseyJUmSJEnSChpSD417ABfMe/zDef/eF3gB8DzgZGAf4HNJblNVp61eiZIkSZIkaTUMKdD4elWdO7owydVogcarqurN3bIvAxuApwH7r2aRkiRJkiRp5U39kJMx3BnYGvjA3IKqOg/4BHC/voqSJEmSJEkrZ0iBxqlJLk5ySpJ/mrd8R+AS4Psj25/UrZMkSZIkSTNmCENOfkmbH+NrwBbAnsDbk2xVVa8H1gDnVtUlIz93FrBVkqtU1UWrWrEkSZIkSVpRUx9oVNWRwJHzFh3RzZuxf5I3bu5+k+wN7A2w3XbbsX79+mXVudqes9PFfZewJEv5/Q6pbUP7u5EkSZKkWTH1gcZGHAY8CtiB1hPjmkm2GOmlsQY4f2O9M6rqQOBAgLVr19Zuu+22ogVP2l77Ht53CUuy4e92G3vbIbVtKe2SJEmSJE3OkObQmK/m3Z9MG4pys5FtduzWSZIkSZKkGTPUQOMRwBnAj4FjgXOAR86tTLIV8EDgiF6qkyRJkiRJK2rqh5wk+RBtQtBv0npiPLq7PaOq/ghcmGQd8IIkZ9F6ZexDC2ve1E/VkiRJkiRpJU19oAGcAvwDcEMgwHeBx1fVf8/bZh0twHg+sA1wHHDvqvrVKtcqSZIkSZJWwdQHGlW1H7DfItsU8IruJkmSJEmSZtxQ59CQJEmSJElXYAYakiRJkiRpcAw0JEmSJEnS4BhoSJIkSZKkwTHQkCRJkiRJg2OgIUmSJEmSBsdAQ5IkSZIkDY6BhiRJkiRJGhwDDUmSJEmSNDgGGpIkSZIkaXAMNCRJkiRJ0uAYaEiSJEmSpMEx0JAkSZIkSYNjoCFJkiRJkgbHQEOSJEmSJA2OgYYkSZIkSRocAw1JkiRJkjQ4BhqSJEmSJGlwDDQkSZIkSdLgGGhIkiRJkqTBMdCQJEmSJEmDY6AhSZIkSZIGx0BDkiRJkiQNjoGGJEmSJEkaHAMNSZIkSZI0OAYakiRJkiRpcAw0JEmSJEnS4BhoSJIkSZKkwTHQkCRJkiRJg2OgIUmSJEmSBsdAQ5IkSZIkDY6BhiRJkiRJGhwDDUmSJEmSNDgGGpIkSZIkaXCmPtBI8sgkH0/y8yTnJjk+yWNGtlmfpBa4Xa2vuiVJkiRJ0srZsu8CxrAP8CPg2cAZwB7AwUm2rao3zdvuC8B+Iz/7+9UpUZIkSZIkraYhBBoPrKoz5j0+Ksn1aUHH/EDjN1X1ldUtTZIkSZIk9WHqh5yMhBlzvgFcf7VrkSRJkiRJ02HqA42N2BX43siyv01yfnc7Mslt+yhMkiRJkiStvMEFGknuCTwEeO28xUcDzwTuA+wN/B/gmCQ7rHZ9kiRJkiRp5aWq+q5hbF1A8VXg2Kp66Ca22x44GTioqp61kW32poUfbLfddrsccsghE693JX3r57/tu4Ql2ekG1x572yG1bSntkiRJkiQtze677358Va1daN1gAo0k1wX+B/gdsFtVnb/I9ocD16qquy2277Vr19Zxxx03mUJXyQ77Ht53CUuyYd39x952SG1bSrskSZIkSUuTZKOBxiCGnCTZCvgkcBXgAYuFGZ3qbpIkSZIkacZMfaCRZEvgg8DNgftW1a/H+JntgbsCx69weZIkSZIkqQdb9l3AGN4K7EGb9HObJNvMW/cN4JbAq2ihx49pE4I+H/gj8IZVrVTahCENpQGH00iSJEmabkMINP62u3/jAutuDJwJhBZqbEObY2M98JCq+slqFChJkiRJklbX1AcaVbXDGJvtsdJ1SJIkSZKk6TH1c2hIkiRJkiSNMtCQJEmSJEmDY6AhSZIkSZIGx0BDkiRJkiQNjoGGJEmSJEkaHAMNSZIkSZI0OAYakiRJkiRpcAw0JEmSJEnS4BhoSJIkSZKkwTHQkCRJkiRJg2OgIUmSJEmSBsdAQ5IkSZIkDY6BhiRJkiRJGhwDDUmSJEmSNDgGGpIkSZIkaXAMNCRJkiRJ0uAYaEiSJEmSpMEx0JAkSZIkSYNjoCFJkiRJkgbHQEOSJEmSJA2OgYYkSZIkSRqcLfsuQNLw7bDv4X2XMLYN6+7fdwmSJEmSJsAeGpIkSZIkaXAMNCRJkiRJ0uAYaEiSJEmSpMEx0JAkSZIkSYNjoCFJkiRJkgbHq5xI0hXMkK5KA16ZRpIkSQuzh4YkSZIkSRocAw1JkiRJkjQ4DjmRpI0Y0tAMh2VIkiTpisYeGpIkSZIkaXDsoSFJmhn2qpEkSbrisIeGJEmSJEkanJkJNJLcKsnnk5yf5BdJXppki77rkiRJkiRJkzcTQ06SrAE+B3wXeDBwU+C1tMBm/x5LkyRJkiRJK2AmAg3gKcDVgYdV1TnAZ5NsDbw4yQHdMkmSBmlW5wYZUrvAeU9mmX+L0sob0vvM99hwzMqQk/sBR44EF4fQQo6791OSJEmSJElaKbPSQ2NH4Kj5C6rqJ0nO79Z9opeqJEmSZohnWIdnVl+zIbULxm/brLZLWimz0kNjDXD2AsvP6tZJkiRJkqQZkqrqu4ZlS/IH4HlV9YaR5T8D3lNV+y3wM3sDe3cPbwmcstJ1DsS2wBl9F7FCZrVttmt4ZrVttmt4ZrVttmt4ZrVttmt4ZrVttmt4ZrltS3WjqrreQitmZcjJWcC1F1i+plt3OVV1IHDgShY1REmOq6q1fdexEma1bbZreGa1bbZreGa1bbZreGa1bbZreGa1bbZreGa5bZM0K0NOTqbNlXGpJDcEturWSZIkSZKkGTIrgcYRwH2SXGveskcDFwBH91OSJEmSJElaKbMSaLwd+D3w4ST36ubHeDHwupFLuWpxszwMZ1bbZruGZ1bbZruGZ1bbZruGZ1bbZruGZ1bbZruGZ5bbNjEzMSkoQJJbAW8GdqVd8eSdwIur6pI+65IkSZIkSZM3M4GGJEmSJEm64piVISdahiQ3S/KOJN9MckmS9X3XNAlJHpnk40l+nuTcJMcneUzfdS1XkkckOTbJmUkuTHJKkv2TXKXv2iYpyQ26162SXLPvepYjyV5dO0ZvT+m7tuVKsmWSfZN8P8nvk/wsyev7rmu5kqzfyGtWSXbtu77lSLJnkhO699fPk7wnyfX7rmu5kjyk+xz7fZIfJdmn75o2xzifyWn2S/LTJBck+WKS261+teMbs11PTXJ49/lWSXZb9UKXaLF2JfnzJK9OcmL3nvtpkncP4T03RtuukuQDSX7Y/R2enuSIJLv0VPJYlnrcm+T13d/ja1apxM0y5ntswwKfaaf1UO6SjPuaJdkpySeT/DbJ75J8bZr/Hsd4j+22iWORI3sqe+rMymVbtTy3BvYAvgJcuedaJmkf4EfAs2nXcN4DODjJtlX1pl4rW55tgKOAV9OGV92BNmfM9sDTeqtq8l4NnAtco+9CJugetMmK5/ywr0Im6CBau15Cu6rUDYFb9VnQhDwV2Hpk2UuB2wNfX/1yJiPJg4D3A28Bngf8OfBy4PAku1TVH/usb3MluQvwYeBdwHOBOwL/nuSPVfWGPmvbDON8Ju8LvID2Gp5M+7z7XJLbVNW0fjkZp12PBwo4EhjKCYjF2rUL8FDaUOivAtvRPrOP7V6vc1epzs2xWNu2oL1erwJOpf2f+WzgqCS3r6pp/Ywb+7g3bUj7k4AhzMk3brsOBuYfB1+0kkVNyKJt60LdY4CP0S4OAfDXwNVXob7NtVi7TqBNpzDf/wEOpV0UQzjkRECSK80dxCY5DNi2qnbrt6rl64KLM0aWHQzsWlU37qmsFZHkFcD/BdbUDLypk9wN+CjwSlqwca0pP+jbpCR7Af/FwNsxKsl9gU8Af1VV3+27npWU1gPqNODQqvrnvuvZXEkOAW5eVbvMW/Yg2gHgrarqpN6KW4buTNVWVfU385a9FngisH1VDeGAHVj8MznJ1YBfAa+tqpd2y64BbADeUVX7r3rRYxjnWGNumyS3Ab4F7F5V61e92CUY4/W6DnBuVV08b9ktgFOAvarq3atb8fg25/gwrUflmcDzq+p1K1/l0i2lXUk+DxwLPA44rKqeu2qFLtGY77ENTHk7FjJm274C/LCqHttDiZtlM99jzwPWATesql+sfJXTzyEnYqhn5BYzGmZ0vgFMfTfPzXAmMBNDTpJsQTtz8FJazxpNr38Ajpr1MKNzX2ANrXfDkF0Z+O3IsrO7+6xuKRN1O+CzI8s+Q3vNBjVEaIzP5DvTzoR/YN7PnEcLF++3gqUtyzjHGkM8Hlms5qo6e36Y0S37HnA+U348spmvx3nAhUzxMcm47UryCGBH2pfHqTfE98+4Fmtb15Pmjvxpz5Opt5mv2WOAow0zLmOgoSuaXYHv9V3EJCTZIslWSe4KPAN42yz0zgCeAlyV1iV+1pya5OK0eU/+qe9iJuCOwPeSvDnJOUnOT/LhIYwN3wx7Aj+jdWcdsncBf5Pk8Um27s4Uv5zhB1NX4/Ldpuce/+Uq17LSdgQuAb4/svykbp2mXJLbAlsxO8cjSZtPaXvgANrf56DD3yRXB14L7NsFhrPkSUku6uaZOCzJjfouaALu2N2vSZuv5uIkpyZ5Uq9VTVj3mX17Bv7+mjQDDV1hJLkn8BDaB9QsOK+7HQMcTRtLPWhJtgFeBuxTVX/ou54J+iVtvPvjgAfSxkq+Pcmze61q+bYH9qKdHd+T1r1/F+AjSYZ8tv9PJNkKeBDwgaGHhlV1OO01O5DWU+MU2jj4h/dY1iT8gDZWer47dPfXXeVaVtoa2hCG0cvSnwVslRmbIHrWJLkS8EZaIPXxnsuZlH8F/kD7rHsCsEdV/bjfkpbt+bT2vLfvQibsY7Q5ou5JO27cFTgmybV7rWr5tu/u3wO8D7g38GngnUn26K2qyduT9l77UN+FTBMnBdUVQpIdaJMgfayqDuq3mom5M+0Mzx2AFwJvpn1IDdkrgK9U1af6LmSSqupI2iR3c47oxsHvn+SNA+4mmu724Ko6EyDJL2kB2z2Az/dY2yQ9kDY57eDPiCTZHXg77QvVEVw2QeFHktxrgS/JQ/F2Wkj4j8BhtP8X565yMtT3l2bTq2hfIu8+Q8H9QcDnaJMMPxX4ZJK7DbXXV5Ib0yYX3n3oIfaoqnrmvIfHJDkW+F/aCYk39FHThMydRHlnVR3Q/fsLSf6SFk7NynHlnsBnquo3fRcyTeyhoZmX5Lq0A/cfA3/XczkTU1UnVNWXukm3ngH8c5Kb9l3X5kpya9qcDC9Ncp1uIrWtutXX7rp/zpLDaGeOd+i5juU4C/jWXJjR+RKtq/8sXOlkzp7AD6rquL4LmYDXAh+vqn+tqvVVdSit59puwIP7LGyZ3gW8rbv9hnbFk5d166b1qh+b6yzgmt18Q/OtAc4f0gSoVzRJnko7K/6Eqvpq3/VMSlWdVlXHVdUnaAHwmbQr8QzVOtpx4ynzjkeuBFy1ezwzPRCr6tu0nno7913LMp3V3X9hZPlRzMjxSJK/og2hHPzJlUkz0NBM67qKf5I2OdUDqur8nktaKSd090O+esvNaRMWfpn2wXQWl82j8TMGNtHTGGrkfohOYuGJJMOMnBXvuuHej9k5gNiRdjbuUlV1Cu1ywoMNRKvqkqp6GnA94La0nidf6VZ/ZaM/OEwn04YJ3Wxk+Y7dOk2hJA+nfY79SxckzqRuAtRvATfpu5ZluCXwMC47FjmLdknyp3X/vkF/pa2IYtjHItCOR+DyxyQzczxCO7lyAW3YkOZxyIlmVpItgQ/Svijfuap+3XNJK+ku3f2Peq1ieb4E7D6y7L60sbl7ANN6PfvN9QjaVVyGPM74k8BLRi6RfDdaMHVif2VN1ENpk9TOSqDxY0bOxHVdcq9Ou+znoFXV3JePubPhx1bVrH3JPxY4B3gkbULXufD+gbS5UTRlkuxGG9f/pqp6Tb/VrKxuOOXOwP/0XcsyPBm45siyQ2jDKd8GnL7qFa2Q7hLJOzL8/zuOpf3ffw/a3Blz7snsHI/sCXyiqs7tu5BpY6ChuQOhuQlzbgBs3V2qCuBTA+7V8FZau54JbNNNODnnG1X1+37KWp4kn6aNVf0ObSbxuwDPAQ6tqlP7rG05ui/E6+cv6+Y+AThmyP+BJ/kQ8DXgm7Qzq4/ubs8Y8PwZ0A6AngF8IskrgWsB/w58rqq+1Gtlk7MncGJVnbTolsPwduD1SX7BZXNovJAWZgx2jHGSOwF3pfU+2Zp2Wbv7dMsGZZzP5CTrgBckOYvWK2MfWq/bqe3JNma71tKG4d2wW373JNsCG6Z1yNdi7QJuBHyU9jod2v2tzjl9mj+3x2jbg2k92D4N/ILL5tD4c+B1q1vt+Mb4W7zc31qSC4GfVtX61aly6cZ4vXYH/p52MuIXtCBjf+AntHlQptaY/3+8FDggydnA12mTXd8NuPtq1zuucb+Ddf9v7AAMfTL5lVFV3q7gN9obpDZy26Hv+pbRrg0z2q6XAd8GzgXOpg03eTpw5b5rW4G27tW9Xtfsu5ZltuOVtDGq59O6Cx4PPK7vuibUtpvRDpTOo50dOQhY03ddE2rbtrTZxPftu5YJtinAP9PCtfOAnwOHAjfpu7ZltmsX2gHsubTeC4cDO/Vd12a2ZdHP5O51/DfacLwLaFe7un3ftU+gXQdtZP1Bfde/ue2a9zk2qHaN2bbbd++104Dfd8ddhwK37rv25f4tLvAzG4DX9F37Ml+v29Im6z69+2w7rXvPXb/v2if1mtHC3R/R5vL6FvCwvmufULveQDvmv2rfNU/jLd0vSZIkSZIkaTCcFFSSJEmSJA2OgYYkSZIkSRocAw1JkiRJkjQ4BhqSJEmSJGlwDDQkSZIkSdLgGGhIkiRJkqTBMdCQJEljSXJQkov7rmMpkjw9yfeT/CHJ2X3Xs1RJdkhSSfbquxZJkqaNgYYkSVMiyV7dl9eLk9xsgfXvTFJ91DZESXYD/gP4BvBkYO9Ftt8xyfuS/DDJhUlOS/LlJOuSXHPlK5YkSUuxZd8FSJKky9kCeCHw+L4LGbh7dfd7V9XZm9owyR2B9cCvgXcDPwG2B24LPA14O3DuShW6CT8Grg78oYfnliRpqhloSJI0fb4BPDbJK6rqlL6LWW1JrlFV501gV38GsFiY0XkBcCHw11X165F6rg38fgL1kOTKQKrqonG2r6rq6pIkSSMcciJJ0vRZRzsj/6LFNkyyIclBCyw/KMmGeY/n5mLYP8kTk5yS5IIkxya5bbfN45Kc1A23OCHJX2/kOf8iyceS/C7JmUnettCQjCT3THJUt915SY5O8jcj27y4q2unJO9Kcgbws0XafLUkr+zaflF3/8okV523TQH/OPfv7vbiTez2ZsDJo2EGQFX9tqr+JFRIctMkByc5Pcnvk3w7yT+ObLNb97yPT/Jv3etxIXD7bljRaxZo29WTnJPkv7vHC86hkWS7JG9J8pPu+X/W1XODedtcuXvek7ttTktyYJLrjuzrdkk+meRX3Wv/8yQfSvIXm/h9SZLUO3toSJI0fX4BvAN4epKXV9V3J7jvhwFbd/vfEtgXOCLJC4HnAwfShrzsCxyW5KZVNX8i0ABHAN8D/hVYCzwFuBGwx6UbJY8C3g8cTev9EGAv4PNJ7lVVXxyp6/20IONFwEbnq0gS4MPA/YD3Al8G7tzVCaVrBgAABpZJREFUvhPwwG7TxwFPAnbr/g3wzU38XjYAd0uytqqO28R2JLlF97y/AV4PnNXVc2CSbapq3ciP/AvtJNJbgIuBXwJHAY9K8ryuF8acBwDXAg7exPNvB3wVuD7wTuBE4HrA/WnBzM+739OHgHsD/69r+02ApwN3SHKnqrowyfWAz3VteB1wRrff+wA3YJFwSZKkPhloSJI0ndbRJrF8MfCoCe73JsDNquoMgLQrf7wVOAC4RVWdObL8XsCn5/38lYATquoJcwuS/BLYL8l9qurIJNcA3gYcUlV/N2+7twPfAl4F3GWkrlOBh4x8uV/I/Wnhwbqqen637K1dDc9NskdVfaqq3ttNCrpbVb13jN/Luq6tX0tyAnAMLYz57ALDX95E++K/87x1b0tyMPCCJG+rqt/O234NsGNV/W5uQbftfwF37Z5rzmO6fX92E7W+ihYg3aOqvjBv+cu7IANgT1q487dVdem+khwFHEkLef6TFgZtA9yvqr4+b18v28TzS5I0FRxyIknSFKqq02iBwiOS7DTBXX9oLszofLm7/+hcmDGy/CYL7OMNG3n8gO7+3sB1gfcm2XbuBlyD1hvgTkm2GtnH28YIM+Y/x+hwjQNG1i9JVa2nhSwfAW4JPKv79+lJnju3XZI1tPZ9ELj6SPuOALYC7jSy+/+eH2Z0PkIbfvKYefvemtbL5YMjvWKYt82VaL1sPjsSZsy1Y+53+Gjgh8A3Rmo8AfgtcI9uu7O7+wfNH7IjSdIQGGhIkjS9DgDOp/XSmJSfjDw+e5HlaxbYx59MVFpVp9OGLNy4W3SL7v5TwOkjt3+iHX9sM7LPUxetvNkBOH0kfJmr4Yx5NSxZVX21qh4OXAe4DbAP7ffw6nlzWNycNnzm37h8297TbfNnI7u+XNu6HhyfogVWcz1mHwpclU0MN6ENLbk2rafLptyCFkaN1nh69/NzNX4ROATYHzgzyaeTPD3J6OsjSdLUcciJJElTqqp+neQtwPOS3G5jm21k+RYbWX7JEpdnI8s3Ze6EyZO4fFAy5/SRxxdsxvOsiKq6BPgO8J0kHwe+T7uE7kFc1rb/AD6xkV18Z+Txxtp2MK23xT1pw0AeQ/t9/c/m1j7PlYCTaXNmLOQsuLRHx2OSvJo2nOdetHlBXphk96r69gRqkSRpRRhoSJI03Q4Angq8hMuHANC+mF5ngeU7rFxJ3JJ2aVkAuokl1wA/6hb9oLs/o6o+N+Hn3gDcp5t889JeGt1wim3n1TARVXVqkt/QJsqEy3pbXDKBth0OnEMLFE6gBRuvW2Tozendzyw2DOkHwB2Bo6rqj4sVUlUn0IajvCztqjfHA88BnrhoKyRJ6olDTiRJmmLdl/b/AB4E7LzAJj8Adh25ZOnOtMkeV8qzNvL48O7+SNpQjf0XmpehC0A211yviH1Glj9vZP2SpF1i9nLHRUnuSBseczJcOrTl88CTktxoge3Hblt3KdiPAA8B/p52omlTw03owokPAfdOsvsCzz/Xo+YQWsDzrAW22WLu0q1J1sz7mTkn0XqVXGfctkiS1Ad7aEiSNP1eAzwNuP0C694BPBL4TJJDaJfa3Bv4Nm2uhEn7I7Bzkg/RvtivpZ3F/0xVfRqgqn6XZG/apVi/leR9tEvR3gC4e7efy30ZH9OnaFdd2S/JX9AuX3on2lU7PllVR2zmft8IbJ3ko7QhI0WbR+MJtC/3L5+37VNpw0JOTPJO2iVsrwvcjhZOXG0Jz/v+7jleApxUVSeO8TP70SYmPbJ7/hO759+DNhfG0cD7gIcDr03yN92yi4GbdstfSBtC8wTa5YE/QgvHtqRdIeVaXW2SJE0tAw1JkqZcVZ2V5A20L6Gj6z6f5BnAc2lzH3yHNhfD44DdVqIc2mVT3wL8O/AH4MDu+efX9cEkP6d9+X4m7eofpwFfA9612U9eVUkeRvtdPJbW1l/SLmX60s3dL63+ufksngBcvav3E7RLxF46l0RVfS/JLl0Ne9Im2DyD1rPhOUt83s8Bv+72scneGfOe/7Qkd6CFIA8Bngz8ihZafL/bppI8gjaHxl7AfYGLgB8DhwJHdbs7mhZKPRzYnjYJ7Xdol9D92BLbIknSqsp4V0iTJEmSJEmaHs6hIUmSJEmSBsdAQ5IkSZIkDY6BhiRJkiRJGhwDDUmSJEmSNDgGGpIkSZIkaXAMNCRJkiRJ0uAYaEiSJEmSpMEx0JAkSZIkSYNjoCFJkiRJkgbHQEOSJEmSJA3O/wcaMUh6P7kUTwAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "fig = plt.figure(figsize=(18, 24))\n", - " \n", - " \n", - "plt.subplot(3, 1, 1)\n", - "plt.hist(services_per_proj, bins=np.arange(-0.5,20.5), rwidth=0.8)\n", - "plt.xticks(np.arange(0,20), fontsize=15)\n", - "plt.yticks(fontsize=15)\n", - "plt.xlabel(\"Number of Services\", fontsize=17)\n", - "plt.ylabel(\"Number of Projects\", fontsize=17)\n", - "plt.gca().yaxis.grid(True)\n", - "plt.title(\"Number of Services per Project\", fontsize=21)\n", - "\n", - "\n", - "plt.subplot(3, 1, 2)\n", - "plt.hist(projects_per_user, bins=np.arange(0.5,10.5), rwidth=0.6)\n", - "plt.xticks(np.arange(1,10), fontsize=15)\n", - "plt.yticks(fontsize=15)\n", - "plt.xlabel(\"Number of Projects\", fontsize=17)\n", - "plt.ylabel(\"Number of Users\", fontsize=17)\n", - "plt.gca().yaxis.grid(True)\n", - "plt.title(\"Number of Projects per User\", fontsize=21)\n", - "plt.text(5, 250, f\"{len(projects_per_user)} users out of 2758 ({int(len(projects_per_user) / 2758 * 100)}%) have at least one project.\", \n", - " bbox={'facecolor': 'yellow', 'alpha': 0.5, 'pad': 10}, fontsize=14)\n", - "\n", - "plt.subplot(3, 1, 3)\n", - "plt.hist(number_of_services_added_to_projects_per_user, bins=np.arange(0.5,18.5), rwidth=0.8)\n", - "plt.xticks(np.arange(1,18), fontsize=15)\n", - "plt.yticks(fontsize=15)\n", - "plt.xlabel(\"Number of Services\", fontsize=17)\n", - "plt.ylabel(\"Number of Users\", fontsize=17)\n", - "plt.gca().yaxis.grid(True)\n", - "plt.title(\"Number of Services added to Projects per User\", fontsize=21)\n", - "plt.text(12, 150, f\"There are {len(projects_per_user) - len(number_of_services_added_to_projects_per_user)} ({int(((len(projects_per_user) - len(number_of_services_added_to_projects_per_user)) / len(projects_per_user) * 100))}%) empty projects.\", \n", - " bbox={'facecolor': 'yellow', 'alpha': 0.5, 'pad': 10}, fontsize=14)\n", - "\n", - "plt.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "eosc_env", - "language": "python", - "name": "venv" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/recommendation_system_app.py b/recommendation_system_app.py index 1fc1700..dc791aa 100644 --- a/recommendation_system_app.py +++ b/recommendation_system_app.py @@ -1,4 +1,4 @@ -from api.main import start_app +from app.main import start_app def main(): diff --git a/requirements.txt b/requirements.txt index 2da3074..f91b172 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ uvicorn==0.15.0 sentence_transformers pyarrow==8.0.0 mlxtend==0.20.0 -sentry-sdk==1.5.10 +sentry-sdk==1.16.0 python-dotenv==0.19.2 cronitor==4.5.0 APScheduler==3.9.1 diff --git a/tests/api_testing/test_autocompletion.py b/tests/api_testing/test_autocompletion.py index c626702..198c738 100644 --- a/tests/api_testing/test_autocompletion.py +++ b/tests/api_testing/test_autocompletion.py @@ -1,7 +1,7 @@ import pytest import requests -BASE_URL = "http://0.0.0.0:4560/v1" +BASE_URL = "http://0.0.0.0:4559/v1" @pytest.mark.api