From 93ff74c4729cf37b4699ef48641607c3db0011f3 Mon Sep 17 00:00:00 2001 From: Cannon Lock Date: Wed, 25 Jan 2023 13:34:07 -0600 Subject: [PATCH] Add name filters (SOFTWARE-5442) Added ability to filter based on the name of a facility, support center, resourcegroup, site, vo, voown, and service Did a little refactoring of the Filter class Added tests for the above --- src/app.py | 94 +------------- src/tests/test_api.py | 114 +++++++++++++++-- src/tests/test_common.py | 267 +++++++++++++++++++++++++++++++++++++++ src/webapp/common.py | 143 ++++++++++++++++++++- src/webapp/topology.py | 22 +++- 5 files changed, 531 insertions(+), 109 deletions(-) create mode 100644 src/tests/test_common.py diff --git a/src/app.py b/src/app.py index 352407cba..1f85d9b26 100755 --- a/src/app.py +++ b/src/app.py @@ -14,11 +14,11 @@ import urllib.parse from webapp import default_config -from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, escape +from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, \ + escape from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService from webapp.forms import GenerateDowntimeForm, GenerateResourceGroupDowntimeForm from webapp.models import GlobalData -from webapp.topology import GRIDTYPE_1, GRIDTYPE_2 from webapp.oasis_managers import get_oasis_manager_endpoint_info @@ -283,6 +283,7 @@ def rgsummary_xml(): return _get_xml_or_fail(global_data.get_topology().get_resource_summary, request.args) + @app.route('/rgdowntime/xml') def rgdowntime_xml(): return _get_xml_or_fail(global_data.get_topology().get_downtimes, request.args) @@ -291,7 +292,7 @@ def rgdowntime_xml(): @app.route('/rgdowntime/ical') def rgdowntime_ical(): try: - filters = get_filters_from_args(request.args) + filters = Filters.from_args(request.args, global_data) except InvalidArgumentsError as e: return Response("Invalid arguments: " + str(e), status=400) response = make_response(global_data.get_topology().get_downtimes_ical(False, filters).to_ical()) @@ -704,94 +705,9 @@ def _make_choices(iterable, select_one=False): return c -def get_filters_from_args(args) -> Filters: - filters = Filters() - def filter_value(filter_key): - filter_value_key = filter_key + "_value" - if filter_key in args: - filter_value_str = args.get(filter_value_key, "") - if filter_value_str == "0": - return False - elif filter_value_str == "1": - return True - else: - raise InvalidArgumentsError("{0} must be 0 or 1".format(filter_value_key)) - filters.active = filter_value("active") - filters.disable = filter_value("disable") - filters.oasis = filter_value("oasis") - - if "gridtype" in args: - gridtype_1, gridtype_2 = args.get("gridtype_1", ""), args.get("gridtype_2", "") - if gridtype_1 == "on" and gridtype_2 == "on": - pass - elif gridtype_1 == "on": - filters.grid_type = GRIDTYPE_1 - elif gridtype_2 == "on": - filters.grid_type = GRIDTYPE_2 - else: - raise InvalidArgumentsError("gridtype_1 or gridtype_2 or both must be \"on\"") - if "service_hidden_value" in args: # note no "service_hidden" args - if args["service_hidden_value"] == "0": - filters.service_hidden = False - elif args["service_hidden_value"] == "1": - filters.service_hidden = True - else: - raise InvalidArgumentsError("service_hidden_value must be 0 or 1") - if "downtime_attrs_showpast" in args: - # doesn't make sense for rgsummary but will be ignored anyway - try: - v = args["downtime_attrs_showpast"] - if v == "all": - filters.past_days = -1 - elif not v: - filters.past_days = 0 - else: - filters.past_days = int(args["downtime_attrs_showpast"]) - except ValueError: - raise InvalidArgumentsError("downtime_attrs_showpast must be an integer, \"\", or \"all\"") - if "has_wlcg" in args: - filters.has_wlcg = True - - # 2 ways to filter by a key like "facility", "service", "sc", "site", etc.: - # - either pass KEY_1=on, KEY_2=on, etc. - # - pass KEY_sel[]=1, KEY_sel[]=2, etc. (multiple KEY_sel[] args). - for filter_key, filter_list, description in [ - ("facility", filters.facility_id, "facility ID"), - ("rg", filters.rg_id, "resource group ID"), - ("service", filters.service_id, "service ID"), - ("sc", filters.support_center_id, "support center ID"), - ("site", filters.site_id, "site ID"), - ("vo", filters.vo_id, "VO ID"), - ("voown", filters.voown_id, "VO owner ID"), - ]: - if filter_key in args: - pat = re.compile(r"{0}_(\d+)".format(filter_key)) - arg_sel = "{0}_sel[]".format(filter_key) - for k, v in args.items(): - if k == arg_sel: - try: - filter_list.append(int(v)) - except ValueError: - raise InvalidArgumentsError("{0}={1}: must be int".format(k,v)) - elif pat.match(k): - m = pat.match(k) - filter_list.append(int(m.group(1))) - if not filter_list: - raise InvalidArgumentsError("at least one {0} must be specified" - " via the syntax {1}_ID=on" - " or {1}_sel[]=ID." - " (These may be specified multiple times for multiple IDs.)"\ - .format(description, filter_key)) - - if filters.voown_id: - filters.populate_voown_name(global_data.get_vos_data().get_vo_id_to_name()) - - return filters - - def _get_xml_or_fail(getter_function, args): try: - filters = get_filters_from_args(args) + filters = Filters.from_args(args, global_data) except InvalidArgumentsError as e: return Response("Invalid arguments: " + str(e), status=400) return Response( diff --git a/src/tests/test_api.py b/src/tests/test_api.py index bb16ec01d..e9efbe142 100644 --- a/src/tests/test_api.py +++ b/src/tests/test_api.py @@ -1,6 +1,8 @@ import re import flask +from flask import testing import pytest +import xmltodict from pytest_mock import MockerFixture # Rewrites the path so the app can be imported like it normally is @@ -64,16 +66,16 @@ def client(): class TestAPI: - def test_sanity(self, client: flask.Flask): + def test_sanity(self, client: testing.FlaskClient): response = client.get('/') assert response.status_code == 200 @pytest.mark.parametrize('endpoint', TEST_ENDPOINTS) - def test_endpoint_existence(self, endpoint, client: flask.Flask): + def test_endpoint_existence(self, endpoint, client: testing.FlaskClient): response = client.get(endpoint) assert response.status_code != 404 - def test_cache_authfile(self, client: flask.Flask, mocker: MockerFixture): + def test_cache_authfile(self, client: testing.FlaskClient, mocker: MockerFixture): mocker.patch("webapp.ldap_data.get_ligo_ldap_dn_list", mocker.MagicMock(return_value=["deadbeef.0"])) resources = client.get('/miscresource/json').json for resource in resources.values(): @@ -84,7 +86,7 @@ def test_cache_authfile(self, client: flask.Flask, mocker: MockerFixture): assert previous_endpoint.status_code == current_endpoint.status_code assert previous_endpoint.data == current_endpoint.data - def test_cache_authfile_public(self, client: flask.Flask): + def test_cache_authfile_public(self, client: testing.FlaskClient): resources = client.get('/miscresource/json').json for resource in resources.values(): resource_fqdn = resource["FQDN"] @@ -94,7 +96,7 @@ def test_cache_authfile_public(self, client: flask.Flask): assert previous_endpoint.status_code == current_endpoint.status_code assert previous_endpoint.data == current_endpoint.data - def test_origin_authfile(self, client: flask.Flask): + def test_origin_authfile(self, client: testing.FlaskClient): resources = client.get('/miscresource/json').json for resource in resources.values(): resource_fqdn = resource["FQDN"] @@ -104,7 +106,7 @@ def test_origin_authfile(self, client: flask.Flask): assert previous_endpoint.status_code == current_endpoint.status_code assert previous_endpoint.data == current_endpoint.data - def test_origin_authfile_public(self, client: flask.Flask): + def test_origin_authfile_public(self, client: testing.FlaskClient): resources = client.get('/miscresource/json').json for resource in resources.values(): resource_fqdn = resource["FQDN"] @@ -114,7 +116,7 @@ def test_origin_authfile_public(self, client: flask.Flask): assert previous_endpoint.status_code == current_endpoint.status_code assert previous_endpoint.data == current_endpoint.data - def test_cache_scitokens(self, client: flask.Flask): + def test_cache_scitokens(self, client: testing.FlaskClient): resources = client.get('/miscresource/json').json for resource in resources.values(): resource_fqdn = resource["FQDN"] @@ -124,7 +126,7 @@ def test_cache_scitokens(self, client: flask.Flask): assert previous_endpoint.status_code == current_endpoint.status_code assert previous_endpoint.data == current_endpoint.data - def test_origin_scitokens(self, client: flask.Flask): + def test_origin_scitokens(self, client: testing.FlaskClient): resources = client.get('/miscresource/json').json for resource in resources.values(): resource_fqdn = resource["FQDN"] @@ -134,7 +136,7 @@ def test_origin_scitokens(self, client: flask.Flask): assert previous_endpoint.status_code == current_endpoint.status_code assert previous_endpoint.data == current_endpoint.data - def test_resource_stashcache_files(self, client: flask.Flask, mocker: MockerFixture): + def test_resource_stashcache_files(self, client: testing.FlaskClient, mocker: MockerFixture): """Tests that the resource table contains the same files as the singular api outputs""" # Disable legacy auth until it's turned back on in Resource.get_stashcache_files() @@ -180,7 +182,7 @@ def test_stashcache_file(key, endpoint, fqdn, resource_stashcache_files): else: app.config["STASHCACHE_LEGACY_AUTH"] = old_legacy_auth - def test_stashcache_namespaces(self, client: flask.Flask): + def test_stashcache_namespaces(self, client: testing.FlaskClient): def validate_cache_schema(cc): assert HOST_PORT_RE.match(cc["auth_endpoint"]) assert HOST_PORT_RE.match(cc["endpoint"]) @@ -704,7 +706,7 @@ class TestEndpointContent: mock_facility.add_site(mock_site) mock_site.add_resource_group(mock_resource_group) - def test_resource_defaults(self, client: flask.Flask): + def test_resource_defaults(self, client: testing.FlaskClient): resources = client.get('/miscresource/json').json # Check that it is not empty @@ -715,7 +717,7 @@ def test_resource_defaults(self, client: flask.Flask): "Description", "FQDN", "FQDNAliases", "VOOwnership", "WLCGInformation", "ContactLists", "IsCCStar"]) - def test_site_defaults(self, client: flask.Flask): + def test_site_defaults(self, client: testing.FlaskClient): sites = client.get('/miscsite/json').json # Check that it is not empty @@ -724,7 +726,7 @@ def test_site_defaults(self, client: flask.Flask): # Check that the site contains the appropriate keys assert set(sites.popitem()[1]).issuperset(["ID", "Name", "IsCCStar"]) - def test_facility_defaults(self, client: flask.Flask): + def test_facility_defaults(self, client: testing.FlaskClient): facilities = client.get('/miscfacility/json').json # Check that it is not empty @@ -733,6 +735,92 @@ def test_facility_defaults(self, client: flask.Flask): # Check that the site contains the appropriate keys assert set(facilities.popitem()[1]).issuperset(["ID", "Name", "IsCCStar"]) + def test_filter_by_service_name(self, client: testing.FlaskClient): + """Checks inclusion of service name filtering on Resource Class""" + + xml = client.get('/rgsummary/xml?service_name[]=CE').data + r = xmltodict.parse(xml) + + resources = [] + for rg in r["ResourceSummary"]["ResourceGroup"]: + if type(rg["Resources"]["Resource"]) == list: + resources.extend(rg["Resources"]["Resource"]) + else: + resources.append(rg["Resources"]["Resource"]) + + services = [] + for r in resources: + if "Services" not in r: + continue + if type(r["Services"]["Service"]) == list: + services.extend(r["Services"]["Service"]) + else: + services.append(r["Services"]["Service"]) + + assert all(list(s["Name"] == "CE" for s in services)) + + def test_rgsummary_filter_by_facility_name(self, client: testing.FlaskClient): + + xml = client.get('/rgsummary/xml?facility_name[]=California%20Institute%20of%20Technology').data + r = xmltodict.parse(xml) + + for rg in r["ResourceSummary"]["ResourceGroup"]: + assert rg["Facility"]["Name"] == "California Institute of Technology" + + def test_rgsummary_filter_by_site_name(self, client: testing.FlaskClient): + + xml = client.get('/rgsummary/xml?site_name[]=Caltech%20CMS%20Tier2').data + r = xmltodict.parse(xml) + + assert r["ResourceSummary"]["ResourceGroup"]["Site"]["Name"] == "Caltech CMS Tier2" + + def test_rgsummary_filter_by_support_center_name(self, client: testing.FlaskClient): + + xml = client.get('/rgsummary/xml?sc_name[]=Self%20Supported').data + r = xmltodict.parse(xml) + + for rg in r["ResourceSummary"]["ResourceGroup"]: + assert rg["SupportCenter"]["Name"] == "Self Supported" + + def test_rgsummary_filter_by_rg_name(self, client: testing.FlaskClient): + + xml = client.get('/rgsummary/xml?rg_name[]=FIUPG').data + r = xmltodict.parse(xml) + + assert r["ResourceSummary"]["ResourceGroup"]["GroupName"] == "FIUPG" + + def test_rgdowntime_filter_by_facility_name(self, client: testing.FlaskClient): + + xml = client.get('/rgdowntime/xml?facility_name[]=Florida%20Institute%20of%20Technology&downtime_attrs_showpast=all').data + r = xmltodict.parse(xml) + + assert r["Downtimes"]["PastDowntimes"]["Downtime"][0]["ResourceGroup"]["GroupName"] == "FLTECH" + + def test_rgdowntime_filter_by_site_name(self, client: testing.FlaskClient): + + xml = client.get('/rgdowntime/xml?site_name[]=Florida%20Tech&downtime_attrs_showpast=all').data + r = xmltodict.parse(xml) + + for downtime in r["Downtimes"]["PastDowntimes"]["Downtime"]: + assert downtime["ResourceGroup"]["GroupName"] == "FLTECH" + + def test_rgdowntime_filter_by_support_center_name(self, client: testing.FlaskClient): + + xml = client.get('/rgdowntime/xml?sc_name[]=Community%20Support%20Center&downtime_attrs_showpast=all').data + r = xmltodict.parse(xml) + + included_resource_groups = [rg['ResourceGroup']['GroupName'] for rg in r['Downtimes']['PastDowntimes']['Downtime']] + + assert "Utah-SLATE-Notchpeak" in included_resource_groups + assert "OSG_IN_IUCAA_SARATHI" not in included_resource_groups # This has a UChicago Support Center + + def test_rgdowntime_filter_by_rg_name(self, client: testing.FlaskClient): + + xml = client.get('/rgdowntime/xml?rg_name[]=FLTECH&downtime_attrs_showpast=all').data + r = xmltodict.parse(xml) + + for downtime in r["Downtimes"]["PastDowntimes"]["Downtime"]: + assert downtime["ResourceGroup"]["GroupName"] == "FLTECH" if __name__ == '__main__': pytest.main() diff --git a/src/tests/test_common.py b/src/tests/test_common.py new file mode 100644 index 000000000..4c7812ca5 --- /dev/null +++ b/src/tests/test_common.py @@ -0,0 +1,267 @@ +import re +import flask +from flask import testing +import pytest +from pytest_mock import MockerFixture + +# Rewrites the path so the app can be imported like it normally is +import os +import sys + +topdir = os.path.join(os.path.dirname(__file__), "..") +sys.path.append(topdir) + +from app import app, global_data +from webapp.common import Filters, InvalidArgumentsError, GRIDTYPE_1, GRIDTYPE_2 + + +@pytest.fixture +def client() -> testing.FlaskClient: + with app.test_client() as client: + yield client + + +class TestFilterFromArgs: + + @staticmethod + def get_request_args(client, url): + request_context = client.application.test_request_context(url) + return request_context.request.args + + def test_empty_args(self, client: testing.FlaskClient): + """Check that no args results in an empty filter""" + + args = self.get_request_args(client, "/test") + arg_informed_filters = Filters.from_args(args, global_data) + + default_filter = Filters() + + # Iterate the class attributes and confirm the are the same in both + for k in Filters().__dict__.keys(): + assert getattr(default_filter, k) == getattr(arg_informed_filters, k) + + def test_get_filter_value_sets_value(self, client: testing.FlaskClient): + """Check we can set values using the get_filter_value method""" + + args = self.get_request_args(client, "/test?active=0&active_value=0") + arg_informed_filters = Filters.from_args(args, global_data) + + assert arg_informed_filters.active is not None + + def test_get_filter_value_correctly_sets_value(self, client: testing.FlaskClient): + """Check we can set values using the get_filter_value method""" + + args = self.get_request_args(client, "/test?active=0&active_value=0") + arg_informed_filters = Filters.from_args(args, global_data) + + assert arg_informed_filters.active is False + + def test_get_filter_value_correctly_sets_value_multi(self, client: testing.FlaskClient): + """Check we can set values using the get_filter_value method""" + + args = self.get_request_args(client, "/test?" + "active=0&active_value=1" + "&disable=0&disable_value=1" + "&oasis=0&oasis_value=0") + arg_informed_filters = Filters.from_args(args, global_data) + + assert arg_informed_filters.active is True + assert arg_informed_filters.disable is True + assert arg_informed_filters.oasis is False + + def test_populate_service_hidden_filter_from_args(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "service_hidden_value=1") + arg_informed_filters = Filters.from_args(args, global_data) + + assert arg_informed_filters.service_hidden is True + + def test_populate_service_hidden_filter_from_args_invalid(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "service_hidden_value=test") + + with pytest.raises(InvalidArgumentsError) as e_info: + Filters.from_args(args, global_data) + + def test_populate_gridtype_filter_from_args_single(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "gridtype=on&" + "gridtype_1=on") + arg_informed_filters = Filters.from_args(args, global_data) + + assert arg_informed_filters.grid_type == GRIDTYPE_1 + + def test_populate_gridtype_filter_from_args_double(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "gridtype=on&" + "gridtype_1=on&" + "gridtype_2=on") + arg_informed_filters = Filters.from_args(args, global_data) + + assert arg_informed_filters.grid_type is None + + def test_populate_gridtype_filter_from_args_error(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "gridtype=on&") + + with pytest.raises(InvalidArgumentsError) as e_info: + Filters.from_args(args, global_data) + + def test_populate_past_days_from_args_all(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "downtime_attrs_showpast=all") + arg_informed_filters = Filters.from_args(args, global_data) + + assert arg_informed_filters.past_days is -1 + + def test_populate_past_days_from_args_none(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "downtime_attrs_showpast") + arg_informed_filters = Filters.from_args(args, global_data) + + assert arg_informed_filters.past_days is 0 + + def test_populate_past_days_from_args_int(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "downtime_attrs_showpast=56") + arg_informed_filters = Filters.from_args(args, global_data) + + assert arg_informed_filters.past_days is 56 + + def test_populate_past_days_from_args_error(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "downtime_attrs_showpast=test") + + with pytest.raises(InvalidArgumentsError) as e_info: + Filters.from_args(args, global_data) + + def test_populate_has_wlcg_from_args(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?has_wlcg") + arg_informed_filters = Filters.from_args(args, global_data) + + assert arg_informed_filters.has_wlcg is True + + def test_add_selector_filter_from_args(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "facility&" + "facility_sel[]=70&" + "facility_311") + arg_informed_filters = Filters.from_args(args, global_data) + + assert set(arg_informed_filters.facility_id) == set([70, 311]) + + def test_add_selector_filter_from_args_multi(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "rg&" + "rg_sel[]=32&" + "rg_123&" + "vo&" + "vo_sel[]=21&" + "vo_45") + arg_informed_filters = Filters.from_args(args, global_data) + + assert set(arg_informed_filters.rg_id) == set([32, 123]) + assert set(arg_informed_filters.vo_id) == set([21, 45]) + + def test_add_selector_filter_from_args_int_error(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "rg&" + "rg_sel[]=test") + + with pytest.raises(InvalidArgumentsError) as e_info: + Filters.from_args(args, global_data) + + def test_add_selector_filter_from_args_no_input_error(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "rg&") + + with pytest.raises(InvalidArgumentsError) as e_info: + Filters.from_args(args, global_data) + + def test_add_id_filter_from_args(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "site_id[]=90&" + "site_id[]=33&" + "service_id[]=21&" + "voown_id[]=4&") + arg_informed_filters = Filters.from_args(args, global_data) + + assert set(arg_informed_filters.site_id) == set([90, 33]) + assert set(arg_informed_filters.service_id) == set([21]) + assert set(arg_informed_filters.voown_id) == set([4]) + + def test_add_id_filter_from_args_int_error(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "site_id[]=temp") + + with pytest.raises(InvalidArgumentsError) as e_info: + Filters.from_args(args, global_data) + + def test_joint_selector_and_id_from_args(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "site_id[]=90&" + "site_id[]=33&" + "service_id[]=21&" + "rg&" + "rg_sel[]=32&" + "rg_123&" + "site&" + "site_sel[]=21&" + "site_45") + arg_informed_filters = Filters.from_args(args, global_data) + + assert set(arg_informed_filters.site_id) == set([90, 33, 21, 45]) + assert set(arg_informed_filters.rg_id) == set([32, 123]) + assert set(arg_informed_filters.service_id) == set([21]) + + def test_add_name_filter_from_args(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "site_name[]=test0&" + "site_name[]=test1") + arg_informed_filters = Filters.from_args(args, global_data) + + assert set(arg_informed_filters.site_name) == set(["test0", "test1"]) + + def test_add_name_filter_from_args_multi(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "site_name[]=test0&" + "rg_name[]=test32&" + "sc_name[]=test1") + arg_informed_filters = Filters.from_args(args, global_data) + + assert set(arg_informed_filters.site_name) == set(["test0"]) + assert set(arg_informed_filters.rg_name) == set(["test32"]) + assert set(arg_informed_filters.support_center_name) == set(["test1"]) + + def test_joint_voown_name_and_id(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "voown_id[]=133&" + "voown_name[]=ANL&" + "voown&" + "voown_69&" + "voown_sel[]=1") + arg_informed_filters = Filters.from_args(args, global_data) + + assert set(arg_informed_filters.voown_name) == set(["ACCRE", "ANL", "Belle", "CDF"]) + + def test_all(self, client: testing.FlaskClient): + args = self.get_request_args(client, "/test?" + "site_name[]=test0&" + "rg_id[]=32&" + "voown_name[]=test1&" + "downtime_attrs_showpast=56&" + "gridtype=on&" + "gridtype_1=on&" + "gridtype_2=on&" + "service_hidden_value=1&" + "has_wlcg") + + arg_informed_filters = Filters.from_args(args, global_data) + + assert set(arg_informed_filters.site_name) == set(["test0"]) + assert set(arg_informed_filters.rg_id) == set([32]) + assert set(arg_informed_filters.voown_name) == set(["test1"]) + assert arg_informed_filters.past_days is 56 + assert arg_informed_filters.grid_type is None + assert arg_informed_filters.service_hidden is True + assert arg_informed_filters.has_wlcg is True diff --git a/src/webapp/common.py b/src/webapp/common.py index eea23856a..0e90c14ab 100644 --- a/src/webapp/common.py +++ b/src/webapp/common.py @@ -24,19 +24,28 @@ RGDOWNTIME_SCHEMA_URL = "https://topology.opensciencegrid.org/schema/rgdowntime.xsd" VOSUMMARY_SCHEMA_URL = "https://topology.opensciencegrid.org/schema/vosummary.xsd" +GRIDTYPE_1 = "OSG Production Resource" +GRIDTYPE_2 = "OSG Integration Test Bed Resource" + SSH_WITH_KEY = os.path.abspath(os.path.dirname(__file__) + "/ssh_with_key.sh") ParsedYaml = NewType("ParsedYaml", Dict[str, Any]) # a complex data structure that's a result of parsing a YAML file PreJSON = NewType("PreJSON", Dict[str, Any]) # a complex data structure that will be converted to JSON in the webapp T = TypeVar("T") +class InvalidArgumentsError(Exception): pass + class Filters(object): def __init__(self): self.facility_id = [] + self.facility_name = [] self.site_id = [] + self.site_name = [] self.support_center_id = [] + self.support_center_name = [] self.service_id = [] + self.service_name = [] self.grid_type = None self.active = None self.disable = None @@ -44,13 +53,145 @@ def __init__(self): self.voown_id = [] self.voown_name = [] self.rg_id = [] + self.rg_name = [] self.service_hidden = None self.oasis = None # for vosummary self.vo_id = [] # for vosummary + self.vo_name = [] self.has_wlcg = None + @classmethod + def from_args(cls, args, global_data): + """Parse http request parameters into a filter object""" + + filters = cls() + + filters.active = cls.get_filter_value(args, "active") + filters.disable = cls.get_filter_value(args, "disable") + filters.oasis = cls.get_filter_value(args, "oasis") + + filters.populate_gridtype_filter_from_args(args) + filters.populate_service_hidden_filter_from_args(args) + filters.populate_past_days_from_args(args) + filters.populate_has_wlcg_from_args(args) + + for filter_key, filter_list, description in [ + ("facility", filters.facility_id, "facility ID"), + ("rg", filters.rg_id, "resource group ID"), + ("service", filters.service_id, "service ID"), + ("sc", filters.support_center_id, "support center ID"), + ("site", filters.site_id, "site ID"), + ("vo", filters.vo_id, "VO ID"), + ("voown", filters.voown_id, "VO owner ID"), + ]: + cls.add_selector_filter_from_args(args, filter_key, filter_list, description) + cls.add_id_filter_from_args(args, filter_key, filter_list) + + for filter_key, filter_list in [ + ("facility", filters.facility_name), + ("rg", filters.rg_name), + ("service", filters.service_name), + ("sc", filters.support_center_name), + ("site", filters.site_name), + ("vo", filters.vo_name), + ("voown", filters.voown_name), + ]: + cls.add_name_filter_from_args(args, filter_key, filter_list) + + filters.populate_voown_name(global_data.get_vos_data().get_vo_id_to_name()) + + return filters + + @staticmethod + def get_filter_value(args, filter_key): + filter_value_key = filter_key + "_value" + if filter_key in args: + filter_value_str = args.get(filter_value_key, "") + if filter_value_str == "0": + return False + elif filter_value_str == "1": + return True + else: + raise InvalidArgumentsError("{0} must be 0 or 1".format(filter_value_key)) + + @staticmethod + def add_selector_filter_from_args(args, filter_key, filter_list, description): + if filter_key in args: + pat = re.compile(r"{0}_(\d+)".format(filter_key)) + arg_sel = "{0}_sel[]".format(filter_key) + for k, v in args.items(): + if k == arg_sel: + try: + filter_list.append(int(v)) + except ValueError: + raise InvalidArgumentsError("{0}={1}: must be int".format(k,v)) + elif pat.match(k): + m = pat.match(k) + filter_list.append(int(m.group(1))) + if not filter_list: + raise InvalidArgumentsError("at least one {0} must be specified" + " via the syntax {1}_ID=on" + " or {1}_sel[]=ID." + " (These may be specified multiple times for multiple IDs.)"\ + .format(description, filter_key)) + + @staticmethod + def add_name_filter_from_args(args, filter_key, filter_list): + arg_sel = "{0}_name[]".format(filter_key) + selected_args = args.getlist(arg_sel) + filter_list.extend(selected_args) + + @staticmethod + def add_id_filter_from_args(args, filter_key, filter_list): + arg_sel = "{0}_id[]".format(filter_key) + selected_args = args.getlist(arg_sel) + for v in selected_args: + try: + filter_list.append(int(v)) + except ValueError: + raise InvalidArgumentsError("{0}={1}: must be int".format(arg_sel, v)) + + def populate_gridtype_filter_from_args(self, args): + if "gridtype" in args: + gridtype_1, gridtype_2 = args.get("gridtype_1", ""), args.get("gridtype_2", "") + if gridtype_1 == "on" and gridtype_2 == "on": + pass + elif gridtype_1 == "on": + self.grid_type = GRIDTYPE_1 + elif gridtype_2 == "on": + self.grid_type = GRIDTYPE_2 + else: + raise InvalidArgumentsError("gridtype_1 or gridtype_2 or both must be \"on\"") + + def populate_service_hidden_filter_from_args(self, args): + if "service_hidden_value" in args: # note no "service_hidden" args + if args["service_hidden_value"] == "0": + self.service_hidden = False + elif args["service_hidden_value"] == "1": + self.service_hidden = True + else: + raise InvalidArgumentsError("service_hidden_value must be 0 or 1") + + def populate_past_days_from_args(self, args): + if "downtime_attrs_showpast" in args: + # doesn't make sense for rgsummary but will be ignored anyway + try: + v = args["downtime_attrs_showpast"] + if v == "all": + self.past_days = -1 + elif not v: + self.past_days = 0 + else: + self.past_days = int(args["downtime_attrs_showpast"]) + except ValueError: + raise InvalidArgumentsError("downtime_attrs_showpast must be an integer, \"\", or \"all\"") + + def populate_has_wlcg_from_args(self, args): + if "has_wlcg" in args: + self.has_wlcg = True + def populate_voown_name(self, vo_id_to_name: Dict): - self.voown_name = [vo_id_to_name.get(i, "") for i in self.voown_id] + self.voown_name.extend([vo_id_to_name.get(i, "") for i in self.voown_id]) def is_null(x, *keys) -> bool: diff --git a/src/webapp/topology.py b/src/webapp/topology.py index 0f6cc85de..e3d5c88c1 100644 --- a/src/webapp/topology.py +++ b/src/webapp/topology.py @@ -8,13 +8,11 @@ import icalendar from .common import RGDOWNTIME_SCHEMA_URL, RGSUMMARY_SCHEMA_URL, Filters, ParsedYaml,\ - is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER + is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER,\ + GRIDTYPE_1, GRIDTYPE_2 from .contacts_reader import ContactsData, User from .exceptions import DataError -GRIDTYPE_1 = "OSG Production Resource" -GRIDTYPE_2 = "OSG Integration Test Bed Resource" - log = getLogger(__name__) @@ -223,6 +221,10 @@ def get_tree(self, authorized=False, filters: Filters = None) -> Optional[Ordere if filters.service_id: filtered_services = [svc for svc in filtered_services if svc["ID"] in filters.service_id] + if filters.service_name: + filtered_services = [svc for svc in filtered_services + if svc["Name"] in filters.service_name] + if filters.service_hidden is not None: filtered_services = [svc for svc in filtered_services if not is_null(svc, "Details", "hidden") @@ -383,9 +385,13 @@ def get_tree(self, authorized=False, filters: Filters = None) -> Optional[Ordere if filters is None: filters = Filters() for filter_list, attribute in [(filters.facility_id, self.site.facility.id), + (filters.facility_name, self.site.facility.name), (filters.site_id, self.site.id), + (filters.site_name, self.site.name), (filters.support_center_id, self.support_center["ID"]), - (filters.rg_id, self.id)]: + (filters.support_center_name, self.support_center["Name"]), + (filters.rg_id, self.id), + (filters.rg_name, self.name)]: if filter_list and attribute not in filter_list: return data_gridtype = GRIDTYPE_1 if self.data.get("Production", None) else GRIDTYPE_2 @@ -490,9 +496,13 @@ def _is_shown(self, filters) -> bool: if filters is None: filters = Filters() for filter_list, attribute in [(filters.facility_id, self.rg.site.facility.id), + (filters.facility_name, self.rg.site.facility.name), (filters.site_id, self.rg.site.id), + (filters.site_name, self.rg.site.name), (filters.support_center_id, self.rg.support_center["ID"]), - (filters.rg_id, self.rg.id)]: + (filters.support_center_name, self.rg.support_center["Name"]), + (filters.rg_id, self.rg.id), + (filters.rg_name, self.rg.name)]: if filter_list and attribute not in filter_list: return False