From 7e5f4523e6f9af95e6c4b9eb0df8f921b565f1aa Mon Sep 17 00:00:00 2001 From: Famlam Date: Sun, 3 Dec 2023 14:35:13 +0100 Subject: [PATCH 1/2] Add analyser to detect duplicate relation members --- ...lyser_osmosis_relation_duplicate_member.py | 94 +++++++++++++++++++ osmose_config.py | 1 + tests/osmosis_relation_duplicate_member.osm | 43 +++++++++ 3 files changed, 138 insertions(+) create mode 100644 analysers/analyser_osmosis_relation_duplicate_member.py create mode 100644 tests/osmosis_relation_duplicate_member.osm diff --git a/analysers/analyser_osmosis_relation_duplicate_member.py b/analysers/analyser_osmosis_relation_duplicate_member.py new file mode 100644 index 000000000..5ae498128 --- /dev/null +++ b/analysers/analyser_osmosis_relation_duplicate_member.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +#-*- coding: utf-8 -*- + +########################################################################### +## ## +## Copyrights Osmose Project 2023 ## +## ## +## This program is free software: you can redistribute it and/or modify ## +## it under the terms of the GNU General Public License as published by ## +## the Free Software Foundation, either version 3 of the License, or ## +## (at your option) any later version. ## +## ## +## This program is distributed in the hope that it will be useful, ## +## but WITHOUT ANY WARRANTY; without even the implied warranty of ## +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## +## GNU General Public License for more details. ## +## ## +## You should have received a copy of the GNU General Public License ## +## along with this program. If not, see . ## +## ## +########################################################################### + +from modules.OsmoseTranslation import T_ +from .Analyser_Osmosis import Analyser_Osmosis + +sql10 = """ +SELECT + id, + ST_AsText(relation_locate(id)), + array_agg(duplicate) +FROM ( + SELECT + id, + member_type || member_id || ' (' || COUNT(*) || ')' AS duplicate + FROM + {0}relations AS relations + JOIN relation_members ON + relations.id = relation_id + WHERE + relations.tags != ''::hstore AND + relations.tags?'type' AND + relations.tags->'type' IN ('multipolygon', 'site', 'waterway', 'enforcement', 'public_transport', 'building') + GROUP BY + id, + member_id, + member_type, + member_role + HAVING + COUNT(*) > 1 +) AS t +GROUP BY + id +""" + +class Analyser_Osmosis_Relation_Duplicate_Member(Analyser_Osmosis): + + def __init__(self, config, logger = None): + Analyser_Osmosis.__init__(self, config, logger) + self.classs_change[3] = self.def_class(item = 1040, level = 1, tags = ['relation', 'fix:chair', 'geom'], + title = T_('Duplicate relation member'), + detail = T_( +'''The relation contains the same member (with the same role) more than once. This is not expected for this type of relations.'''), + fix = T_( +'''Remove the duplicate members until only unique members remain.''')) + + self.callback10 = lambda res: {"class": 3, "data": [self.relation, self.positionAsText], "text": {"en": ', '.join(res[2]).lower()}} + + def analyser_osmosis_full(self): + self.run(sql10.format(""), self.callback10) + + def analyser_osmosis_diff(self): + self.run(sql10.format("touched_"), self.callback10) + + + +########################################################################### + +from .Analyser_Osmosis import TestAnalyserOsmosis + +class Test(TestAnalyserOsmosis): + @classmethod + def setup_class(cls): + from modules import config + TestAnalyserOsmosis.setup_class() + cls.analyser_conf = cls.load_osm("tests/osmosis_relation_duplicate_member.osm", + config.dir_tmp + "/tests/osmosis_relation_duplicate_member.test.xml", {}) + + def test_classes(self): + with Analyser_Osmosis_Relation_Duplicate_Member(self.analyser_conf, self.logger) as a: + a.analyser() + + self.root_err = self.load_errors() + self.check_err(cl="3", elems=[("relation", "10001")]) + self.check_num_err(1) diff --git a/osmose_config.py b/osmose_config.py index 374ff35e4..376a2c99c 100644 --- a/osmose_config.py +++ b/osmose_config.py @@ -197,6 +197,7 @@ def __init__(self, country, polygon_id=None, analyser_options=None, download_url self.analyser["osmosis_polygon_intersects"] = "xxx" self.analyser["osmosis_way_angle"] = "xxx" self.analyser["osmosis_highway_long_crossing"] = "xxx" + self.analyser["osmosis_relation_duplicate_member"] = "xxx" class default_country_simple(default_simple): def __init__(self, part, country, polygon_id=None, analyser_options=None, diff --git a/tests/osmosis_relation_duplicate_member.osm b/tests/osmosis_relation_duplicate_member.osm new file mode 100644 index 000000000..3e0d723d5 --- /dev/null +++ b/tests/osmosis_relation_duplicate_member.osm @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 172b142d7360d2a5510b25695b61dd4a5a4b49e4 Mon Sep 17 00:00:00 2001 From: Famlam Date: Tue, 5 Dec 2023 20:35:07 +0100 Subject: [PATCH 2/2] Also output the duplicate members --- analysers/Analyser_Osmosis.py | 4 ++++ .../analyser_osmosis_relation_duplicate_member.py | 10 ++++++---- doc/3-SQL-basics.md | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/analysers/Analyser_Osmosis.py b/analysers/Analyser_Osmosis.py index 5022144c3..db031003b 100644 --- a/analysers/Analyser_Osmosis.py +++ b/analysers/Analyser_Osmosis.py @@ -622,6 +622,10 @@ def array_full(self, res): for type, id in map(lambda r: (r[0], r[1:]), res): self.typeMapping[type](int(id)) + def array_id(self, res): + for type, id in map(lambda r: (r[0], r[1:]), res): + self.typeMapping_id_only[type](int(id)) + re_points = re.compile(r"[\(,][^\(,\)]*[\),]") def get_points(self, text): diff --git a/analysers/analyser_osmosis_relation_duplicate_member.py b/analysers/analyser_osmosis_relation_duplicate_member.py index 5ae498128..04146ead2 100644 --- a/analysers/analyser_osmosis_relation_duplicate_member.py +++ b/analysers/analyser_osmosis_relation_duplicate_member.py @@ -27,11 +27,13 @@ SELECT id, ST_AsText(relation_locate(id)), - array_agg(duplicate) + array_agg(duplicate_typeid ORDER BY duplicate_typeid), -- sequence doesn't matter as long as it's constant until relation changes + array_agg(duplicate_string ORDER BY duplicate_string) -- keep same sequence as line above FROM ( SELECT id, - member_type || member_id || ' (' || COUNT(*) || ')' AS duplicate + member_type || member_id AS duplicate_typeid, + member_type || member_id || ' (' || COUNT(*) || ')' AS duplicate_string FROM {0}relations AS relations JOIN relation_members ON @@ -63,7 +65,7 @@ def __init__(self, config, logger = None): fix = T_( '''Remove the duplicate members until only unique members remain.''')) - self.callback10 = lambda res: {"class": 3, "data": [self.relation, self.positionAsText], "text": {"en": ', '.join(res[2]).lower()}} + self.callback10 = lambda res: {"class": 3, "data": [self.relation, self.positionAsText, self.array_id], "text": {"en": ', '.join(res[3]).lower()}} def analyser_osmosis_full(self): self.run(sql10.format(""), self.callback10) @@ -90,5 +92,5 @@ def test_classes(self): a.analyser() self.root_err = self.load_errors() - self.check_err(cl="3", elems=[("relation", "10001")]) + self.check_err(cl="3", elems=[("relation", "10001"), ("way", "1")]) self.check_num_err(1) diff --git a/doc/3-SQL-basics.md b/doc/3-SQL-basics.md index 3a5c41fbc..92f284dc0 100644 --- a/doc/3-SQL-basics.md +++ b/doc/3-SQL-basics.md @@ -114,6 +114,7 @@ The available value for the `data`, mapping OSM object id are: From array of (type: N/W/R, ids): * `array_full` +* `array_id` Without the `_full` suffix, only the id is kept in the Osmose issue, not the tags and other attributes.