Skip to content
This repository has been archived by the owner on Oct 5, 2024. It is now read-only.

Commit

Permalink
Merge pull request #486 from hotosm/fix/areaFilterRelations
Browse files Browse the repository at this point in the history 
Fixes and new tests. Draft code for relations.
  • Loading branch information
@emi420
emi420 authored Mar 26, 2024
2 parents 89c3dd4 + 3bef37e commit f3336c7
Show file tree
Hide file tree
Showing 23 changed files with 1,556 additions and 603 deletions.
7 changes: 2 additions & 5 deletions python/dbapi/api/db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,9 @@ async def run(self, query, singleObject = False):
return result[0]
return json.loads((result[0]['result']))
except Exception as e:
# print("\n******* \n" + query + "\n******* \n")
print("\n******* \n" + query + "\n******* \n")
print(e)
if singleObject:
return {}
else:
return []
return None
finally:
await self.pool.release(conn)
return None
100 changes: 50 additions & 50 deletions python/dbapi/api/raw.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,6 @@ def getGeoType(table):
return "LineString"
return "Node"

def getRelationsGeoType(table):
if table == "ways_poly":
return "MultiPolygon"
return "MultiLineString"

def geoFeaturesQuery(
area = None,
tags = None,
Expand All @@ -45,29 +40,21 @@ def geoFeaturesQuery(
table = None):

geoType = getGeoType(table)
query = "with t_data AS ( \
query = "with t_ways AS ( \
SELECT '" + geoType + "' as type, " + table + ".osm_id as id, " + table + ".timestamp, geom as geometry, tags, status, hashtags, editor, created_at FROM " + table + " \
LEFT JOIN validation ON validation.osm_id = " + table + ".osm_id \
LEFT JOIN changesets c ON c.id = " + table + ".changeset"

if table != "nodes":
query += " UNION \
SELECT '" + getRelationsGeoType(table) + "' as type, relations.osm_id as id, relations.timestamp, geom as geometry, tags, status, hashtags, editor, created_at FROM relations \
LEFT JOIN validation ON validation.osm_id = relations.osm_id \
LEFT JOIN changesets c ON c.id = relations.changeset \
WHERE (tags->>'type' = '" + getRelationsGeoType(table).lower() + "')"

query += "), t_results as (select * from t_data WHERE \
LEFT JOIN changesets c ON c.id = " + table + ".changeset \
WHERE \
{0} {1} {2} {3} {4} {5} \
), \
t_features AS ( \
SELECT jsonb_build_object( 'type', 'Feature', 'id', id, 'properties', to_jsonb(t_results) \
- 'geometry' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM t_results \
t_features AS ( \
SELECT jsonb_build_object( 'type', 'Feature', 'id', id, 'properties', to_jsonb(t_ways) \
- 'geometry' , 'geometry', ST_AsGeoJSON(geometry)::jsonb ) AS feature FROM t_ways \
) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(t_features.feature) ) \
as result FROM t_features;".format(
"ST_Intersects(\"geometry\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "1=1 ",
"AND (" + tagsQueryFilter(tags, "t_data") + ")" if tags else "",
"AND " + hashtagQueryFilter(hashtag, "t_data") if hashtag else "",
"ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "1=1 ",
"AND (" + tagsQueryFilter(tags, table) + ")" if tags else "",
"AND " + hashtagQueryFilter(hashtag, table) if hashtag else "",
"AND created_at >= {0} AND created_at <= {1}".format(dateFrom, dateTo) if dateFrom and dateTo else "",
"AND status = '{0}'".format(status) if (status) else "",
"LIMIT " + str(RESULTS_PER_PAGE),
Expand All @@ -87,33 +74,25 @@ def listAllFeaturesQuery(
):

geoType = getGeoType(table)
relationsGeoType = getRelationsGeoType(table)
if table == "nodes":
osmType = "node"
else:
osmType = "way"

query = "with t_data AS ( \
query = "\
( \
SELECT '" + osmType + "' as type, '" + geoType + "' as geotype, " + table + ".osm_id as id, ST_X(ST_Centroid(geom)) as lat, ST_Y(ST_Centroid(geom)) as lon, " + table + ".timestamp, tags, " + table + ".changeset, c.created_at, v.status FROM " + table + " \
LEFT JOIN changesets c ON c.id = " + table + ".changeset \
LEFT JOIN validation v ON v.osm_id = " + table + ".osm_id"

if table != "nodes":
query += " UNION \
SELECT '" + osmType + "' as type, '" + relationsGeoType + "' as geotype, relations.osm_id as id, ST_X(ST_Centroid(geom)) as lat, ST_Y(ST_Centroid(geom)) as lon, relations.timestamp, tags, relations.changeset, c.created_at, v.status FROM relations \
LEFT JOIN validation v ON v.osm_id = relations.osm_id \
LEFT JOIN changesets c ON c.id = relations.changeset \
WHERE (tags->>'type' = '" + getRelationsGeoType(table).lower() + "')"

query += ") select * from t_data \
LEFT JOIN validation v ON v.osm_id = " + table + ".osm_id \
WHERE {0} {1} {2} {3} {4} {5} {6} \
)\
".format(
"created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1",
"AND created_at <= '{0}'".format(dateTo) if (dateTo) else "",
"AND status = '{0}'".format(status) if (status) else "",
"AND " + hashtagQueryFilter(hashtag, "t_data") if hashtag else "",
"AND " + hashtagQueryFilter(hashtag, table) if hashtag else "",
"AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "",
"AND (" + tagsQueryFilter(tags, "t_data") + ")" if tags else "",
"AND (" + tagsQueryFilter(tags, table) + ")" if tags else "",
"AND " + orderBy + " IS NOT NULL ORDER BY " + orderBy + " DESC LIMIT " + str(RESULTS_PER_PAGE_LIST) + (" OFFSET {0}" \
.format(page * RESULTS_PER_PAGE_LIST) if page else ""),
).replace("WHERE 1=1 AND", "WHERE")
Expand Down Expand Up @@ -173,8 +152,19 @@ def getList(
orderBy,
page
)

return self.getPolygonsList(
elif featureType == "polygon":
return self.getPolygonsList(
area,
tags,
hashtag,
dateFrom,
dateTo,
status,
orderBy,
page
)
else:
return self.getAllList(
area,
tags,
hashtag,
Expand Down Expand Up @@ -216,8 +206,18 @@ def getFeatures(
status,
page
)

return self.getPolygons(
elif featureType == "polygon":
return self.getPolygons(
area,
tags,
hashtag,
dateFrom,
dateTo,
status,
page
)
else:
return self.getAll(
area,
tags,
hashtag,
Expand Down Expand Up @@ -362,7 +362,7 @@ async def getPolygonsList(
tags,
hashtag,
status,
orderBy or "id",
orderBy or "ways_poly.osm_id",
page or 0,
dateFrom,
dateTo,
Expand All @@ -372,7 +372,7 @@ async def getPolygonsList(
" UNION ".join([queryPolygons]),
dateFrom,
dateTo,
orderBy or "id"
orderBy or "osm_id"
)
return await self.underpassDB.run(query)

Expand All @@ -393,7 +393,7 @@ async def getLinesList(
tags,
hashtag,
status,
orderBy or "id",
orderBy or "ways_line.osm_id",
page or 0,
dateFrom,
dateTo,
Expand All @@ -403,7 +403,7 @@ async def getLinesList(
" UNION ".join([queryLines]),
dateFrom,
dateTo,
orderBy or "id"
orderBy or "osm_id"
)
return await self.underpassDB.run(query)

Expand All @@ -424,7 +424,7 @@ async def getNodesList(
tags,
hashtag,
status,
orderBy or "id",
orderBy or "nodes.osm_id",
page or 0,
dateFrom,
dateTo,
Expand All @@ -434,7 +434,7 @@ async def getNodesList(
" UNION ".join([queryNodes]),
dateFrom,
dateTo,
orderBy or "id"
orderBy or "osm_id"
)
return await self.underpassDB.run(query)

Expand All @@ -455,7 +455,7 @@ async def getAllList(
tags,
hashtag,
status,
orderBy or "id",
orderBy or "ways_poly.osm_id",
page or 0,
dateFrom,
dateTo,
Expand All @@ -466,7 +466,7 @@ async def getAllList(
tags,
hashtag,
status,
orderBy or "id",
orderBy or "ways_line.osm_id",
page or 0,
dateFrom,
dateTo,
Expand All @@ -477,7 +477,7 @@ async def getAllList(
tags,
hashtag,
status,
orderBy or "id",
orderBy or "nodes.osm_id",
page or 0,
dateFrom,
dateTo,
Expand All @@ -487,6 +487,6 @@ async def getAllList(
" UNION ".join([queryPolygons, queryLines, queryNodes]),
dateFrom,
dateTo,
orderBy or "id"
orderBy or "osm_id"
)
return await self.underpassDB.run(query)
48 changes: 11 additions & 37 deletions python/dbapi/api/stats.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,39 +45,25 @@ async def getCount(
select {0}.osm_id from {0} \
left join changesets c on changeset = c.id \
where {1} {2} {3} {4} {5} \
".format(
), \
count_validated_features as ( \
select count(distinct(all_features.osm_id)) as count from all_features \
left join validation v on all_features.osm_id = v.osm_id \
where v.status = '{6}' \
), count_features as (\
select count(distinct(all_features.osm_id)) as total from all_features \
) \
select count, total from count_validated_features, count_features".format(
table,
"created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1",
"AND created_at <= '{0}'".format(dateTo) if (dateTo) else "",
"AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "",
"AND (" + tagsQueryFilter(tags, table) + ")" if tags else "",
"AND " + hashtagQueryFilter(hashtag, table) if hashtag else "",
status
)

query += " union \
select relations.osm_id from relations \
left join changesets c on changeset = c.id \
where {0} {1} {2} {3} {4} \
".format(
"created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1",
"AND created_at <= '{0}'".format(dateTo) if (dateTo) else "",
"AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "",
"AND (" + tagsQueryFilter(tags, "relations") + ")" if tags else "",
"AND " + hashtagQueryFilter(hashtag, "relations") if hashtag else "",
)

query += "), count_validated_features as ( \
select count(distinct(all_features.osm_id)) as count from all_features \
left join validation v on all_features.osm_id = v.osm_id \
where v.status = '{0}' \
), count_features as (\
select count(distinct(all_features.osm_id)) as total from all_features \
) \
select count, total from count_validated_features, count_features".format(status)

else:
query = "WITH counts AS ("
query += "select distinct {0}.osm_id from {0} \
query = "select count(distinct {0}.osm_id) as count from {0} \
left join changesets c on changeset = c.id \
where {1} {2} {3} {4} {5}".format(
table,
Expand All @@ -87,18 +73,6 @@ async def getCount(
"AND (" + tagsQueryFilter(tags, table) + ")" if tags else "",
"AND " + hashtagQueryFilter(hashtag, table) if hashtag else ""
)
query += " union "
query += "select distinct relations.osm_id from relations \
left join changesets c on changeset = c.id \
where {0} {1} {2} {3} {4}".format(
"created_at >= '{0}'".format(dateFrom) if (dateFrom) else "1=1",
"AND created_at <= '{0}'".format(dateTo) if (dateTo) else "",
"AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "",
"AND (" + tagsQueryFilter(tags, "relations") + ")" if tags else "",
"AND " + hashtagQueryFilter(hashtag, "relations") if hashtag else ""
)
query += ") SELECT COUNT(osm_id) FROM counts;"

return(await self.underpassDB.run(query, True))


3 changes: 3 additions & 0 deletions setup/db/indexes.sql
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id DESC);
CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id DESC);
CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id DESC);
CREATE UNIQUE INDEX relations_id_idx ON public.relations(osm_id DESC);
CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id);
CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id);
CREATE INDEX rel_refs_rel_id_idx ON public.rel_refs (rel_id);
CREATE INDEX rel_refs_way_id_idx ON public.rel_refs (way_id);

CREATE INDEX nodes_version_idx ON public.nodes (version);
CREATE INDEX ways_poly_version_idx ON public.ways_poly (version);
Expand Down
6 changes: 3 additions & 3 deletions src/bootstrap/bootstrap.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ Bootstrap::start(const underpassconfig::UnderpassConfig &config) {

processWays();
processNodes();
processRelations();
// processRelations();

}

Expand Down Expand Up @@ -341,7 +341,7 @@ Bootstrap::threadBootstrapRelationTask(RelationTask relationTask)
BootstrapTask task;
int processed = 0;

auto relationval = std::make_shared<std::vector<std::shared_ptr<ValidateStatus>>>();
// auto relationval = std::make_shared<std::vector<std::shared_ptr<ValidateStatus>>>();

// Proccesing relations
for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) {
Expand All @@ -355,7 +355,7 @@ Bootstrap::threadBootstrapRelationTask(RelationTask relationTask)
++processed;
}
}
// queryvalidate->relations(relval, task.query);
// queryvalidate->relations(relationval, task.query);
task.processed = processed;
const std::lock_guard<std::mutex> lock(tasks_change_mutex);
(*tasks)[taskIndex] = task;
Expand Down
10 changes: 1 addition & 9 deletions src/osm/changeset.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,17 +364,9 @@ ChangeSetFile::on_start_element(const Glib::ustring &name,
bool hashit = false;
bool comhit = false;
bool cbyhit = false;
double min_lat = 0.0;
double min_lon = 0.0;
double max_lat = 0.0;
double max_lon = 0.0;

for (const auto &attr_pair: attributes) {
// std::wcout << "\tPAIR: " << attr_pair.name << " = " << std::endl;
// attr_pair.value << std::endl;
if (attr_pair.name == "k" && attr_pair.value == "max_lat") {
max_lat = std::stod(attr_pair.value);
} else if (attr_pair.name == "k" && attr_pair.value == "hashtags") {
if (attr_pair.name == "k" && attr_pair.value == "hashtags") {
hashit = true;
} else if (attr_pair.name == "k" && attr_pair.value == "comment") {
comhit = true;
Expand Down
Loading

0 comments on commit f3336c7

Please sign in to comment.