From 6b5677a5096cb70a8ff5bb4c5ed6333893b13313 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Wed, 8 Nov 2023 11:39:41 -0300 Subject: [PATCH] Fix for parse tags from DB response, change 'overlaping' to 'overlapping', less verbose --- config/validate/building.yaml | 1440 +---------------- python/dbapi/api/raw.py | 1 - setup/underpass.sql | 2 +- src/bootstrap/bootstrap.cc | 2 +- src/raw/queryraw.cc | 33 +- src/replicator/threads.cc | 4 +- src/testsuite/libunderpass.all/stats-test.cc | 1 - .../libunderpass.all/test-playground.cc | 33 - src/testsuite/libunderpass.all/val-test.cc | 34 +- .../testdata/validation/config/building.yaml | 16 +- src/validate/defaultvalidation.cc | 4 +- src/validate/geospatial.cc | 39 +- src/validate/queryvalidate.cc | 2 +- src/validate/semantic.cc | 69 +- src/validate/validate.hh | 5 +- src/wrappers/python.cc | 2 +- 16 files changed, 120 insertions(+), 1567 deletions(-) diff --git a/config/validate/building.yaml b/config/validate/building.yaml index 041a252a..98b2e609 100644 --- a/config/validate/building.yaml +++ b/config/validate/building.yaml @@ -1,1440 +1,14 @@ config: - - minangle: - - 70 - - maxangle: - - 110 - - anglethreshold: - - 19 - - overlaps: + - badgeom: - yes - - duplicate: + - badgeom_minangle: + - 89 + - badgeom_maxangle: + - 91 + - badvalue: - yes - -tags: - - building: - - apartments - - barracks - - bungalow - - cabin - - detached - - dormitory - - farm - - ger - - hotel - - house - - houseboat - - residential - - semidetached_house - - static_caravan - - stilt_house - - terrace - - tree_house - - commercial - - industrial - - kiosk - - office - - retail - - supermarket - - warehouse - - cathedral - - chapel - - church - - kingdom_hall - - monastery - - mosque - - presbytery - - religious - - shrine - - synagogue - - temple - - bakehouse - - bridge - - civic - - college - - fire_station - - government - - gatehouse - - hospital - - kindergarten - - public - - school - - toilets - - train_station - - transportation - - university - - barn - - conservatory - - cowshed - - farm_auxiliary - - greenhouse - - slurry_tank - - stable - - sty - - grandstand - - pavilion - - riding_hall - - sports_hall - - stadium - - hangar - - hut - - shed - - carport - - garage - - garages - - parking - - digester - - service - - transformer_tower - - water_tower - - storage_tank - - silo - - beach_hut - - bunker - - castle - - construction - - container - - military - - roof - - ruins - - tent - - yes - - - building:levels - - - building:structure: - - confined_masonry - - steel_frame - - wood_frame - - bamboo_frame - - reinforced_masonry - - unreinforced_masonry - - plastered - - RCC_with_beam - - non_engineered_reinforced_concrete - - engineered_reinforced_concrete - - load_bearing_brick_wall_in_cement_mortar - - brick - - cement_blocks - - steel_frame - - concrete - - - building:material: - - cement_block - - brick - - plaster - - wood - - concrete - - metal - - steel - - stone - - glass - - mirror - - mud - - masonry - - tin - - plastic - - timber_framing - - sandstone - - clay - - reed - - loam - - marble - - copper - - slate - - vinyl - - limestone - - tiles - - metal_plates - - bamboo - - adobe - - rammed_earth - - solar_panels - - tyres - - - roof:material: - - roof_tiles - - metal - - concrete - - tar_paper - - asbestos - - eternit - - glass - - metal_sheet - - slate - - tin - - grass - - copper - - thatch - - gravel - - stone - - wood - - plastic - - asphalt - - zinc - - sandstone - - bamboo - - palm_leaves - - banana_leaves - - solar_panels - - titanium - - - roof:shape: - - gabled - - round - - saltbox - - double_saltbox - - quadruple_saltbox - - sawtooth - - cone - - conical - - side_hipped - - lean_to - - shed - - gabled_row - - crosspitched - - parabolic - - many - - gabled - - flat - - hipped - - pyramidal - - skillion - - half-hipped - - gambrel - - mansard - - dome - - onion - - roof:orientation - - along - - across - - roof:height - - roof:angle - - roof:levels - - roof:direction - - roof:material - - roof:colour - - - floor:material: - - plaster - - brick - - wood - - concrete - - glass - - stone - - asphalt - - metal - - marble - - mdf - - timber_framing - - - building:condition: - - good - - average - - poor - - disused - - - building:colour - - building:fireproof: + - incomplete: - yes - - no - - building:flats - - building:levels - - building:min_level - - building:part - - building:soft_storey: - - yes - - no - - reinforced - - - entrance: - - yes - - main - - exit - - service - - emergency - - height - - max_level - - min_level - - non_existent_levels - - start_date - - - shop: - - alcohol - - art - - bakery - - beauty - - beverages - - bicycle - - books - - butcher - - car - - car_parts - - car_repair - - chemist - - clothes - - convenience - - copyshop - - cosmetics - - charcoal - - electronics - - farm - - furniture - - general - - greengrocer - - hairdresser - - hardware - - houseware - - jewellery - - kiosk - - mobile_phone - - pastry - - shoes - - stationary - - supermarket - - tailor - - tea - - department_store - - general - - mall - - hardware - - laundry - - trade - - office: - - accountant - - administrative - - advertising_agency - - architect - - association - - chamber - - charity - - company - - construction_company - - consulting - - courier - - coworking - - diplomatic - - educational_institution - - employment_agency - - energy_supplier - - engineer - - estate_agent - - financial - - financial_advisor - - forestry - - foundation - - geodesist - - government - - graphic_design - - guide - - harbour_master - - insurance - - it - - lawyer - - logistics - - moving_company - - newspaper - - ngo - - notary - - politician - - political_party - - property_management - - quango - - religion - - research - - security - - surveyor - - tax_advisor - - telecommunication - - therapist - - travel_agent - - tutoring - - union - - visa - - water_utility - - yes - - religious: - - church - - mosque - - temple - - shrine - - cathedral - - chapel - - synagogue - - religion: - - muslim - - hindu - - christian - - buddhist - - jewish - - bahai - - capacity_persons: - - <50 - - 50-100 - - 100-250 - - 250-500 - - >500 - - denomination: - - gelug - - jishu - - jodo_shinshu - - jodo_shu - - nichiren - - nyingma - - obaku - - pure_land - - rinzai - - risshu - - shingon_shu - - soto - - tiantai - - tibetan - - vajrayana - - won - - yogacara - - yuzu_nembutsu - - zen - - thai_mahanikaya - - thai_thammayut - - catholic - - armenian_catholic - - chaldean_catholic - - coptic_catholic - - eritrean_catholic - - ethiopian_catholic - - greek_catholic - - hungarian_greek_catholic - - maronite - - polish_catholic - - roman_catholic - - romanian_catholic - - syriac_catholic - - syro-malabar_catholic - - ukrainian_greek_catholic - - orthodox - - antiochian_orthodox - - armenian_apostolic - - bulgarian_orthodox - - eritrean_orthodox - - ethiopian_orthodox - - coptic_orthodox - - georgian_orthodox - - greek_orthodox - - macedonian_orthodox - - old_believers - - romanian_orthodox - - russian_orthodox - - serbian_orthodox - - syriac_orthodox - - ukrainian_orthodox - - protestant - - adventist - - anglican - - baptist - - disciples_of_christ - - episcopal - - evangelical - - evangelical_covenant - - exclusive_brethren - - lutheran - - mennonite - - methodist - - moravian - - pentecostal - - presbyterian - - quaker - - reformed - - uniting - - polish_national_catholic - - african_methodist_episcopal - - african_methodist_episcopal_zion - - alliance - - apostolic_faith - - assemblies_of_god - - calvinistic_methodist - - catholic_apostolic - - church_of_god_in_christ - - church_of_scotland - - churches_of_christ - - czechoslovak_hussite - - disciples_of_christ - - dutch_reformed - - evangelical_covenant - - evangelical_free_church_of_america - - evangelical_lutheran - - evangelical_free_church_of_france - - exclusive_brethren - - foursquare - - free_church_of_scotland - - living_waters_church - - mission_covenant_church_of_sweden - - moravian - - mormon - - nazarene - - new_frontiers - - orthodox_presbyterian_church - - pkn - - remonstrant - - salvation_army - - scottish_episcopal - - seventh_day_adventist - - strict_baptist - - temple_society_australia - - united - - united_free_church_of_scotland - - united_reformed - - uniting - - united_methodist - - united_church_of_christ - - welsh_baptist - - welsh_independent - - apostolic - - assyrian - - catholic_mariavite - - charismatic - - christian_community - - christ_scientist - - church_of_christ - - congregational - - ecumenical - - fsspx - - harrist - - iglesia_ni_cristo - - jehovahs_witness - - kimbanguist - - la_luz_del_mundo - - liberal_catholic - - mariavite - - messianic_jewish - - new_apostolic - - nondenominational - - old_catholic - - philippine_independent - - simultaneum - - spiritist - - shaktism - - hare_krishna - - vaishnavism - - shaivism - - smartism - - nondenominational - - ahmadiyya - - alevi - - bektashi - - ibadi - - ismaili - - shia - - sunni - - sufi - - digambara - - svetambara - - nondenominational - - unaffiliated - - ashkenazi - - buchari - - conservative - - hasidic - - kabbalistic - - karaite - - lubavitch - - mizrachi - - modern_orthodox - - neo_orthodox - - orthodox - - reconstructionist - - reform - - progressive - - liberal - - samaritan - - sephardi - - traditional - - unity - - asatru - - celtic - - greco-roman - - slavic - - wicca - - quanzhen - - zhengyi - - irani - - parsi - - water_source: - - water_works - - manual_pump - - powered_pump - - groundwater - - historic: - - archaeological_site - - building - - fort - - monument - - area - - housing: - - residential - - hotel - - camp_site - - camp_pitch - - apartments - - toilet - - service: - - car_repair - - electronics_repair - - pump_station - - copyshop - - beauty - - service - - tailor - - cuisine: - - afghan - - african - - american - - arab - - argentinian - - armenian - - asian - - australian - - austrian - - balkan - - bangladeshi - - basque - - bavarian - - belgian - - bolivian - - brazilian - - british - - bulgarian - - cajun - - cambodian - - cantonese - - caribbean - - chinese - - colombian - - croatian - - cuban - - czech - - danish - - dutch - - egyptian - - ethiopian - - european - - filipino - - french - - georgian - - german - - greek - - hawaiian - - hungarian - - indian - - indonesian - - irish - - italian - - jamaican - - japanese - - jewish - - korean - - lao - - latin_american - - lebanese - - malagasy - - malaysian - - mediterranean - - mexican - - middle_eastern - - mongolian - - moroccan - - nepalese - - oriental - - pakistani - - persian - - peruvian - - polish - - portuguese - - romanian - - russian - - senegalese - - serbian - - southern - - spanish - - sri_lankan - - swedish - - swiss - - syrian - - taiwanese - - tex-mex - - thai - - tibetan - - turkish - - ukrainian - - uzbek - - venezuelan - - vietnamese - - western - - açaí - - bagel - - beef - - beef_bowl - - beef_noodle - - bougatsa - - bubble_tea - - burger - - cake - - chicken - - chili - - chocolate - - churro - - coffee_shop - - couscous - - crepe - - curry - - donut - - dumpling - - empanada - - falafel - - fish - - fish_and_chips - - fondue - - fried_chicken - - fries - - frozen_yogurt - - gyoza - - gyros - - hot_dog - - hotpot - - ice_cream - - juice - - kebab - - meat - - noodle - - pancake - - pasta - - pastry - - piadina - - pie - - pita - - pizza - - poke - - potato - - pretzel - - ramen - - salad - - sandwich - - sausage - - savory_pancakes - - seafood - - shawarma - - smoothie - - smørrebrød - - soba - - soup - - souvlaki - - steak - - sushi - - tacos - - takoyaki - - tea - - udon - - waffle - - wings - - yakitori - - bakery - - bar&grill - - barbecue - - bistro - - brasserie - - breakfast - - brunch - - buffet - - buschenschank - - cafe - - deli - - dessert - - diner - - fast_food - - fine_dining - - fried_food - - friture - - fusion - - grill - - heuriger - - international - - local - - lunch - - pub - - regional - - snack - - steak_house - - tapas - - yakiniku - - amenity: - - bar - - cafe - - fast_food - - food_court - - ice_cream - - pub - - restaurant - - college - - driving_school - - kindergarten - - language_school - - library - - toy_library - - training - - music_school - - school - - university - - bus_station - - car_rental - - car_sharing - - car_wash - - vehicle_inspection - - ferry_terminal - - fuel - - parking - - bank - - bureau_de_change - - baby_hatch - - clinic - - dentist - - doctors - - hospital - - nursing_home - - pharmacy - - social_facility - - veterinary - - arts_centre - - brothel - - casino - - cinema - - community_centre - - conference_centre - - events_venue - - gambling - - love_hotel - - nightclub - - planetarium - - public_bookcase - - social_centre - - stripclub - - studio - - swingerclub - - theatre - - courthouse - - fire_station - - police - - post_depot - - post_office - - prison - - ranger_station - - townhall - - dressing_room - - give_box - - mailroom - - parcel_locker - - shelter - - toilets - - recycling - - waste_disposal - - waste_transfer_station - - animal_boarding - - animal_breeding - - animal_shelter - - baking_oven - - childcare - - clock - - crematorium - - funeral_hall - - hunting_stand - - internet_cafe - - marketplace - - monastery - - place_of_mourning - - place_of_worship - - public_building - - refugee_site - - tourism: - - camp_site - - taxi - - camp pitch - - caravan_site - - artwork - - gallery - - museum - - zoo - - attraction - - guest_house - - hotel - - hostel - - generator:source: - - grid - - generator - - solar - - hydro - - ref:bag - - addr:housenumber - - addr:housename - - addr:flats - - addr:conscriptionnumber - - addr:street - - addr:place - - addr:postcode - - addr:city - - addr:country - - addr:postbox - - addr:full - - addr:hamlet - - addr:suburb - - addr:subdistrict - - addr:district - - addr:province - - addr:state - - addr:county - - addr:interpolation - - addr:inclusion - - addr:door - - addr:unit - - addr:flats - - addr:floor - - addr:block - - addr:block_number - - roof:colour - - roof:height - - layer - - name - - name:left - - name:right - - int_name - - loc_name - - nat_name - - official_name - - old_name - - reg_name - - short_name - - sorting_name - - alt_name - - noname - - name:ab - - name:ace - - name:ady - - name:af - - name:ak - - name:alt - - name:am - - name:an - - name:ang - - name:ar - - name:arc - - name:as - - name:aus - - name:av - - name:ay - - name:az - - name:ba - - name:be - - name:bg - - name:bh - - name:bi - - name:bm - - name:bn - - name:bo - - name:br - - name:bru - - name:bs - - name:bug - - name:ca - - name:car - - name:cdo - - name:ce - - name:ceb - - name:ch - - name:chm - - name:chr - - name:chy - - name:co - - name:cs - - name:csb - - name:cu - - name:cv - - name:cy - - name:da - - name:de - - name:dsb - - name:dv - - name:dyu - - name:dz - - name:ee - - name:el - - name:en - - name:eo - - name:es - - name:et - - name:etymology - - name:etymology:wikidata - - name:etymology:wikipedia - - name:eu - - name:fa - - name:ff - - name:fi - - name:fo - - name:fr - - name:frr - - name:fur - - name:fy - - name:ga - - name:gcf - - name:gd - - name:gem - - name:gl - - name:gn - - name:grc - - name:gsw - - name:gu - - name:gv - - name:ha - - name:haw - - name:hi - - name:hif - - name:hop - - name:hr - - name:hsb - - name:ht - - name:hu - - name:hy - - name:ia - - name:id - - name:ie - - name:ig - - name:ilo - - name:inh - - name:io - - name:is - - name:it - - name:iu - - name:ja - - name:ja-Latn - - name:jbo - - name:jv - - name:ka - - name:ka-Latn - - name:kaa - - name:kab - - name:kbd - - name:kg - - name:kk - - name:kl - - name:km - - name:kmr - - name:kn - - name:ko - - name:koi - - name:kr - - name:krc - - name:krl - - name:ks - - name:ksh - - name:ku - - name:kum - - name:kv - - name:kw - - name:ky - - name:la - - name:lad - - name:language - - name:lb - - name:lbe - - name:left - - name:lez - - name:lg - - name:li - - name:lmo - - name:ln - - name:lo - - name:lrc - - name:lt - - name:lv - - name:mdf - - name:mg - - name:mhr - - name:mi - - name:min - - name:mk - - name:ml - - name:mn - - name:mos - - name:mr - - name:ms - - name:mt - - name:mwl - - name:my - - name:myv - - name:na - - name:nap - - name:nb - - name:ne - - name:nl - - name:nn - - name:no - - name:nov - - name:nso - - name:oc - - name:om - - name:or - - name:pa - - name:pag - - name:pap - - name:pfl - - name:pih - - name:pjt - - name:pl - - name:pnb - - name:prefix - - name:pronunciation - - name:ps - - name:pt - - name:pwn - - name:qu - - name:right - - name:rm - - name:rn - - name:ru - - name:rue - - name:rw - - name:sa - - name:sah - - name:sc - - name:scn - - name:sco - - name:sd - - name:se - - name:sg - - name:shn - - name:si - - name:signed - - name:sk - - name:skr - - name:sl - - name:sm - - name:sma - - name:sn - - name:so - - name:sq - - name:sr - - name:ss - - name:st - - name:su - - name:sv - - name:sw - - name:syc - - name:szl - - name:ta - - name:te - - name:tet - - name:tg - - name:th - - name:ti - - name:tk - - name:tl - - name:tlh - - name:tn - - name:to - - name:tpi - - name:tr - - name:ts - - name:tt - - name:tw - - name:ty - - name:udm - - name:ug - - name:uk - - name:ur - - name:uz - - name:vi - - name:vi-Hani - - name:vls - - name:vo - - name:wa - - name:war - - name:wikipedia - - name:wo - - name:xmf - - name:yi - - name:yo - - name:za - - name:zu - - - stars - - opening_hours - - operator - - operator:type: - - private - - public - - government - - community - - consortium - - cooperative - - military - - ngo - - religious - - charitable - - leisure: - - adult_gaming_centre - - amusement_arcade - - bandstand - - bird_hide - - bowling_alley - - dance - - escape_game - - fitness_centre - - hackerspace - - ice_rink - - sports_centre - - sports_hall - - stadium - - tanning_salon - - trampoline_park - - wildlife_hide - - phone - - contact:phone - - email - - contact:email - - website - - contact:website - - capacity:persons - - brand - - abandoned - - yes - - branch - - brand:wikidata - - brand:wikipedia - - takeaway - - - payment:cash - - payment:coins - - payment:notes - - payment:cheque - - payment:electronic_purses - - payment:cards - - payment:debit_cards - - payment:credit_cards - - payment:contactless - - payment:electronic_purses - - payment:stored_value_card - - payment:ep_avant - - payment:ep_beep - - payment:ep_brizzi - - payment:ep_bsbcash - - payment:ep_cash - - payment:ep_flazz - - payment:ep_geldkarte - - payment:ep_mach - - payment:ep_mandiri_emoney - - payment:ep_megacash - - payment:ep_mep - - payment:ep_minicash - - payment:ep_minipay - - payment:ep_monedero - - payment:ep_monedero4b - - payment:ep_nobu_emoney - - payment:ep_stwm_legic - - payment:ep_tapcash - - payment:octopus - - payment:mercado_pago - - payment:debit_cards - - payment:bancomat - - payment:bancontact - - payment:bankaxept - - payment:bankaxess - - payment:cb - - payment:coinkite - - payment:cuentarut - - payment:eps - - payment:girocardpayment:girocard_contactless - - payment:gpn_debit - - payment:interac - - payment:laser - - payment:maestro - - payment:mastercard_debit - - payment:mir - - payment:multibanco - - payment:postfinance_card - - payment:unionpay_debit - - payment:v_pay - - payment:visa_debit - - payment:visa_electron - - payment:credit_cards - - payment:american_express - - payment:bca_card - - payment:dinacard - - payment:diners_club - - payment:discover_card - - payment:jcb - - payment:mastercard - - payment:unionpay - - payment:visa - - payment:contactless - - payment:mastercard_contactlesspayment:paypasspayment:mastercard_paypass - - payment:visa_contactlesspayment:visa_paywavepayment:paywave - - payment:american_express_contactlesspayment:expresspay - - payment:girocard_contactless - - payment:interac_contactless - - payment:maestro_contactless - - payment:quickpass - - payment:rupay_contactless - - payment:berlio - - payment:cfn - - payment:circlek - - payment:dci - - payment:dkv - - payment:esso_card - - payment:eurowag - - payment:hoyer_card - - payment:ids - - payment:omv_card - - payment:petrochina - - payment:roadrunner - - payment:routex - - payment:sinopec - - payment:shell - - payment:snnp - - payment:svg - - payment:total_card - - payment:uta - - payment:sms - - payment:phonepayment:toll_number - - payment:app - - payment:mobile_phone - - payment:telephone_cards - - payment:calling_cards - - payment:reverse_charge_calls - - payment:my_pertamina_app - - payment:parkee_app - - payment:payback_app - - payment:payconiq_app - - payment:payme - - payment:spb - - payment:bob - - payment:china_t-union - - payment:clipper - - payment:ic - - payment:icsf - - payment:ep_ipass - - payment:easycardpayment:ep_easycard - - payment:jak_lingko - - payment:ov-chipkaart - - payment:oyster - - payment:prepaid_ticket - - payment:presto - - payment:sube - - payment:license_plate - - payment:touchngo - - payment:touchngo_smarttag - - payment:touchngo_rfid - - payment:autosweep_rfid - - payment:easytrip - - payment:e_pass - - payment:e_zpass - - payment:fastrak - - payment:k-tag - - payment:good_to_go - - payment:i-pass - - payment:nc_quick_pass - - payment:peach_pass - - payment:pikepass - - payment:sunpass - - payment:via_verde - - payment:cryptocurrencies - - payment:bitcoin - - payment:lightning - - payment:lightning_contactless - - payment:bitcoincash - - payment:litecoin - - payment:dogecoin - - payment:ethereum - - payment:dash - - payment:akulaku - - payment:alipay - - payment:apple_pay - - payment:au_pay - - payment:bcapayment:bca:gpn_qris=yespayment:sakuku - - payment:blik - - payment:bluepaypayment:bluepay:gpn_qris=yes - - payment:cashbac - - payment:d_barai - - payment:danapayment:dana:gpn_qris=yes - - payment:e-cny - - payment:edy - - payment:gcash - - payment:google_paypayment:android_pay - - payment:gopay_idpayment:id_gopay:gpn_qris=yes - - payment:gopay_my - - payment:gpn_qris - - payment:grabpay - - payment:huawei_pay - - payment:id - - payment:isakupayment:isaku:gpn_qris - - payment:kaspro - - payment:line_pay - - payment:jko_pay - - payment:linkaja - - payment:maya - - payment:mbway - - payment:merpay - - payment:mipay - - payment:nanaco - - payment:oplati - - payment:ovo - - payment:paypay - - payment:quicpay - - payment:rakuten_pay - - payment:samsung_pay - - payment:satispay - - payment:sgqr - - payment:shopeepaypayment:shopeepay:gpn_qris - - payment:swish - - payment:touchngo - - payment:twint - - payment:venmo - - payment:vipps - - payment:waon - - payment:wechat - - payment:yukkpayment:yukk:gpn_qris - - payment:zip - - payment:ebt - - payment:snap - - payment:wic - - payment:wire_transfer - - payment:bank_transfer - - payment:invoice - - payment:paypal - - payment:service_voucher - - payment:meal_voucher - - payment:token - - payment:token_coin - - payment:gift_card - - payment:account_cards - - payment:szep - - payment:posta_paletta - - payment:edenred_card - - payment:edenred_voucher - - payment:otp_cafeteria - - payment:multipay - - payment:museumkaart - - payment:u-key - - payment:none - - safety:mask:covid19: - - yes - - no - - given - - sold - - note - - healthcare - - clinic - - hospital - - dentist - - doctor - - pharmacy - - source - - destroyed:building - - yes diff --git a/python/dbapi/api/raw.py b/python/dbapi/api/raw.py index 6c207d8f..9a80539a 100644 --- a/python/dbapi/api/raw.py +++ b/python/dbapi/api/raw.py @@ -116,7 +116,6 @@ def listFeaturesQuery( "AND status = '{0}'".format(status) if (status) else "", "ORDER BY " + orderBy + " DESC LIMIT " + str(RESULTS_PER_PAGE_LIST) + (" OFFSET {0}".format(page * RESULTS_PER_PAGE_LIST) if page else ""), ) - print("status", status) return query class Raw: diff --git a/setup/underpass.sql b/setup/underpass.sql index 145f21c2..33be7d98 100644 --- a/setup/underpass.sql +++ b/setup/underpass.sql @@ -42,7 +42,7 @@ ALTER TABLE ONLY public.changesets DROP TYPE IF EXISTS public.objtype; CREATE TYPE public.objtype AS ENUM ('node', 'way', 'relation'); DROP TYPE IF EXISTS public.status; -CREATE TYPE public.status AS ENUM ('notags', 'complete', 'incomplete', 'badvalue', 'correct', 'badgeom', 'orphan', 'overlaping', 'duplicate'); +CREATE TYPE public.status AS ENUM ('notags', 'complete', 'incomplete', 'badvalue', 'correct', 'badgeom', 'orphan', 'overlapping', 'duplicate'); CREATE TABLE IF NOT EXISTS public.validation ( osm_id int8, diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 0b8699df..05a7b059 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -75,7 +75,7 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { }; for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) { - std::cout << "Counting geometries ... " << std::endl; + std::cout << std::endl << "Counting geometries ... " << std::endl; int total = queryraw->getWaysCount(*table_it); std::cout << "Total: " << total << std::endl; if (total > 0) { diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index bd156173..16078244 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include #include #include "utils/log.hh" @@ -369,21 +371,24 @@ QueryRaw::getNodeCacheFromWays(std::shared_ptr> ways, std::m } std::map parseTagsString(std::string input) { - input.pop_back(); - std::map result; - std::stringstream ss(input); - std::string token; - while (std::getline(ss, token, ',')) { - size_t arrowPos = token.find_last_of(":"); - if (arrowPos != std::string::npos) { - std::string key = token.substr(1, arrowPos - 1); - std::string value = token.substr(arrowPos + 2); - key.erase(std::remove( key.begin(), key.end(), '\"' ),key.end()); - value.erase(std::remove( value.begin(), value.end(), '\"' ),value.end()); - result[key] = value; - } + // std::cout << "[INPUT] " << input << std::endl; + std::map tags; + boost::property_tree::ptree pt; + try { + std::istringstream jsonStream(input); + boost::property_tree::read_json(jsonStream, pt); + } catch (const boost::property_tree::json_parser::json_parser_error& e) { + std::cerr << "Error parsing JSON: " << e.what() << std::endl; + return tags; + } + for (const auto& pair : pt) { + tags[pair.first] = pair.second.get_value(); } - return result; + // for (const auto& pair : tags) { + // std::cout << pair.first << "=" << pair.second << std::endl; + // } + // std::cout << std::endl; + return tags; } std::list> diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index f756caeb..0686f70a 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -535,8 +535,8 @@ threadOsmChange(OsmChangeTask osmChangeTask) for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { task.query += queryvalidate->applyChange(*it->get(), *status_it); } - if (!it->get()->hasStatus(overlaping)) { - task.query += queryvalidate->updateValidation(it->get()->osm_id, overlaping, "building"); + if (!it->get()->hasStatus(overlapping)) { + task.query += queryvalidate->updateValidation(it->get()->osm_id, overlapping, "building"); } if (!it->get()->hasStatus(duplicate)) { task.query += queryvalidate->updateValidation(it->get()->osm_id, duplicate, "building"); diff --git a/src/testsuite/libunderpass.all/stats-test.cc b/src/testsuite/libunderpass.all/stats-test.cc index b119670c..d9caf1c8 100644 --- a/src/testsuite/libunderpass.all/stats-test.cc +++ b/src/testsuite/libunderpass.all/stats-test.cc @@ -237,7 +237,6 @@ class TestStats { if (this->verbose) { std::cout << "changeset: " << changestats->changeset << std::endl; } - // TODO: get values to test from YAML validation file testStat(changestats, validation, "highway"); testStat(changestats, validation, "building"); testStat(changestats, validation, "humanitarian_building"); diff --git a/src/testsuite/libunderpass.all/test-playground.cc b/src/testsuite/libunderpass.all/test-playground.cc index 8522f893..15f51951 100644 --- a/src/testsuite/libunderpass.all/test-playground.cc +++ b/src/testsuite/libunderpass.all/test-playground.cc @@ -18,44 +18,11 @@ // #include -#include -#include - -std::map parseTagsString(const std::string& input) { - std::map result; - std::stringstream ss(input); - std::string token; - - while (std::getline(ss, token, ',')) { - // std::cout << token << std::endl; - // Remove leading and trailing whitespaces - // token.erase(token.find_first_not_of(" "), token.find_last_not_of(" ") + 1); - // std::cout << token << std::endl; - // Find the position of the arrow - size_t arrowPos = token.find("=>"); - if (arrowPos != std::string::npos) { - std::string key = token.substr(0, arrowPos); - std::string value = token.substr(arrowPos + 2); - result[key] = value; - } - } - - return result; -} int main() { - std::string input = "\"fee\"=>\"no\", \"access\"=>\"yes\""; - std::map resultMap = parseTagsString(input); - - // Iterate over the map and print the key-value pairs - for (const auto& pair : resultMap) { - std::cout << pair.first << ": " << pair.second << std::endl; - } - return 0; } - // local Variables: // mode: C++ // indent-tabs-mode: nil diff --git a/src/testsuite/libunderpass.all/val-test.cc b/src/testsuite/libunderpass.all/val-test.cc index 0b0dfc9b..e6300f7d 100644 --- a/src/testsuite/libunderpass.all/val-test.cc +++ b/src/testsuite/libunderpass.all/val-test.cc @@ -91,6 +91,7 @@ osmobjects::OsmWay readOsmWayFromFile(std::string filename) { } else { log_debug("Couldn't load ! %1%", filespec); }; + osmchange.buildGeometriesFromNodeCache(); return *osmchange.changes.front().get()->ways.front().get(); } @@ -195,7 +196,7 @@ test_semantic(std::shared_ptr &plugin) { // But it's complete status = plugin->checkNode(node, "building"); - if (status->hasStatus(complete)) { + if (!status->hasStatus(incomplete)) { runtest.pass("Validate::checkNode(complete) [semantic building]"); } else { runtest.fail("Validate::checkNode(complete) [semantic building]"); @@ -243,7 +244,7 @@ test_semantic(std::shared_ptr &plugin) { // But it's complete way.addTag("building:material", "sponge"); status = plugin->checkWay(way, "building"); - if (status->hasStatus(complete)) { + if (!status->hasStatus(incomplete)) { runtest.pass("Validate::checkWay(bad value) [semantic building]"); } else { runtest.fail("Validate::checkWay(bad value) [semantic building]"); @@ -418,29 +419,30 @@ test_geospatial(std::shared_ptr &plugin) // } // Overlapping, duplicate - osmchange::OsmChangeFile osmfoverlaping; + osmchange::OsmChangeFile osmfoverlapping; const multipolygon_t poly; filespec = DATADIR; filespec += "/testdata/validation/rect-overlap-and-duplicate-building.osc"; if (boost::filesystem::exists(filespec)) { - osmfoverlaping.readChanges(filespec); + osmfoverlapping.readChanges(filespec); + osmfoverlapping.buildGeometriesFromNodeCache(); } else { log_debug("Couldn't load ! %1%", filespec); } - for (auto it = std::begin(osmfoverlaping.changes); it != std::end(osmfoverlaping.changes); ++it) { + for (auto it = std::begin(osmfoverlapping.changes); it != std::end(osmfoverlapping.changes); ++it) { osmchange::OsmChange *change = it->get(); for (auto nit = std::begin(change->ways); nit != std::end(change->ways); ++nit) { osmobjects::OsmWay *way = nit->get(); way->priority = true; } } - auto wayval = osmfoverlaping.validateWays(poly, plugin); + auto wayval = osmfoverlapping.validateWays(poly, plugin); for (auto sit = wayval->begin(); sit != wayval->end(); ++sit) { auto status = *sit->get(); - if (status.hasStatus(overlaping)) { - runtest.pass("Validate::validateWays(overlaping) [geometry building]"); + if (status.hasStatus(overlapping)) { + runtest.pass("Validate::validateWays(overlapping) [geometry building]"); } else { - runtest.fail("Validate::validateWays(overlaping) [geometry building]"); + runtest.fail("Validate::validateWays(overlapping) [geometry building]"); } if (status.hasStatus(duplicate)) { runtest.pass("Validate::validateWays(duplicate) [geometry building]"); @@ -450,28 +452,28 @@ test_geospatial(std::shared_ptr &plugin) } // No overlapping, no duplicate - osmchange::OsmChangeFile osmfnooverlaping; + osmchange::OsmChangeFile osmfnooverlapping; filespec = DATADIR; filespec += "/testsuite/testdata/validation/rect-no-overlap-and-duplicate-building.osc"; if (boost::filesystem::exists(filespec)) { - osmfnooverlaping.readChanges(filespec); + osmfnooverlapping.readChanges(filespec); } else { log_debug("Couldn't load ! %1%", filespec); } - for (auto it = std::begin(osmfnooverlaping.changes); it != std::end(osmfnooverlaping.changes); ++it) { + for (auto it = std::begin(osmfnooverlapping.changes); it != std::end(osmfnooverlapping.changes); ++it) { osmchange::OsmChange *change = it->get(); for (auto nit = std::begin(change->ways); nit != std::end(change->ways); ++nit) { osmobjects::OsmWay *way = nit->get(); way->priority = true; } } - wayval = osmfnooverlaping.validateWays(poly, plugin); + wayval = osmfnooverlapping.validateWays(poly, plugin); for (auto sit = wayval->begin(); sit != wayval->end(); ++sit) { auto status = *sit->get(); - if (!status.hasStatus(overlaping)) { - runtest.pass("Validate::validateWays(no overlaping) [geometry building]"); + if (!status.hasStatus(overlapping)) { + runtest.pass("Validate::validateWays(no overlapping) [geometry building]"); } else { - runtest.fail("Validate::validateWays(no overlaping) [geometry building]"); + runtest.fail("Validate::validateWays(no overlapping) [geometry building]"); } if (!status.hasStatus(duplicate)) { runtest.pass("Validate::validateWays(no duplicate) [geometry building]"); diff --git a/src/testsuite/testdata/validation/config/building.yaml b/src/testsuite/testdata/validation/config/building.yaml index 519b6f9f..b63b58ee 100644 --- a/src/testsuite/testdata/validation/config/building.yaml +++ b/src/testsuite/testdata/validation/config/building.yaml @@ -1,16 +1,12 @@ config: - - minangle: - - 70 - - maxangle: - - 110 - - anglethreshold: - - 19 - - overlaps: + - badgeom: - yes - - duplicate: - - yes - - enable_tag_values: + - badgeom_minangle: + - 89 + - badgeom_maxangle: + - 91 + - badvalue: - yes tags: diff --git a/src/validate/defaultvalidation.cc b/src/validate/defaultvalidation.cc index 8655ee31..5a95502c 100644 --- a/src/validate/defaultvalidation.cc +++ b/src/validate/defaultvalidation.cc @@ -70,7 +70,9 @@ DefaultValidation::checkWay(const osmobjects::OsmWay &way, const std::string &ty yaml::Yaml tests = yamls[type]; semantic::Semantic::checkWay(way, type, tests, status); geospatial::Geospatial::checkWay(way, type, tests, status); - boost::geometry::centroid(way.linestring, status->center); + if (way.linestring.size() > 2) { + boost::geometry::centroid(way.linestring, status->center); + } status->source = type; return status; } diff --git a/src/validate/geospatial.cc b/src/validate/geospatial.cc index 79bbbaf2..3ee110fe 100644 --- a/src/validate/geospatial.cc +++ b/src/validate/geospatial.cc @@ -51,37 +51,44 @@ Geospatial::Geospatial() {} std::shared_ptr Geospatial::checkWay(const osmobjects::OsmWay &way, const std::string &type, yaml::Yaml &tests, std::shared_ptr &status) { - // On non-english numeric locales using decimal separator different than '.' - // this is necessary to parse double strings with std::stod correctly - // without loosing precision - // std::setlocale(LC_NUMERIC, "C"); - setlocale(LC_NUMERIC, "C"); - if (way.action == osmobjects::remove) { return status; } - // These values are in the config section of the YAML file auto config = tests.get("config"); + bool check_badgeom = config.get_value("badgeom") == "yes"; + // bool check_overlapping = config.get_value("overlapping") == "yes"; + // bool check_duplicate = config.get_value("duplicate") == "yes"; + + if (check_badgeom) { + auto badgeom_minangle = config.get_value("badgeom_minangle"); + auto badgeom_maxangle = config.get_value("badgeom_maxangle"); + if (badgeom_minangle != "" && badgeom_maxangle != "") { + if (unsquared(way.linestring, std::stod(badgeom_minangle), std::stod(badgeom_maxangle))) { + status->status.insert(badgeom); + } + } else { + if (unsquared(way.linestring)) { + status->status.insert(badgeom); + } + } - // if (config.contains_value("overlaps", "yes")) { + } + + // if (check_overlapping) { // auto allWays = context.getOverlappingWays(); // if (overlaps(allWays, way)) { - // status->status.insert(overlaping); + // status->status.insert(overlapping); // } // } - // if (config.contains_value("duplicate", "yes")) { + // if (check_duplicate) { // auto allWays = context.getOverlappingWays(); // if (overlaps(allWays, way)) { - // status->status.insert(overlaping); + // status->status.insert(overlapping); // } // } - if (unsquared(way.linestring)) { - status->status.insert(badgeom); - } - return status; } @@ -107,8 +114,6 @@ Geospatial::overlaps(const std::list> &allwa bool Geospatial::duplicate(const std::list> &allways, osmobjects::OsmWay &way) { - // This test only applies to buildings, as highways often overlap. - // TODO: move logic to a config file #ifdef TIMING_DEBUG_X boost::timer::auto_cpu_timer timer("validate::duplicate: took %w seconds\n"); #endif diff --git a/src/validate/queryvalidate.cc b/src/validate/queryvalidate.cc index dcbe8933..499beae6 100644 --- a/src/validate/queryvalidate.cc +++ b/src/validate/queryvalidate.cc @@ -69,7 +69,7 @@ std::map status_list = { {badvalue, "badvalue"}, {correct, "correct"}, {orphan, "orphan"}, - {overlaping, "overlaping"}, + {overlapping, "overlapping"}, {duplicate, "duplicate"}, {badgeom, "badgeom"} }; diff --git a/src/validate/semantic.cc b/src/validate/semantic.cc index 1c06af4f..8f9feb94 100644 --- a/src/validate/semantic.cc +++ b/src/validate/semantic.cc @@ -66,21 +66,25 @@ Semantic::checkTag(const std::string &key, const std::string &value, std::shared if (!key.empty() && value.empty()) { log_debug("WARNING: empty value for tag \"%1%\"", key); status->status.insert(badvalue); + status->values.insert(key + "=" + value); } // Check for a space in the tag key if (key.find(' ') != std::string::npos) { log_error("WARNING: spaces in tag key \"%1%\"", key); status->status.insert(badvalue); + status->values.insert(key + "=" + value); } // Check for single quotes in the tag value - if (value.find('\'') != std::string::npos) { - log_error("WARNING: single quote in tag value \"%1%\"", value); + if (key.find('\'') != std::string::npos) { + log_error("WARNING: single quote in tag key \"%1%\"", value); status->status.insert(badvalue); + status->values.insert(key + "=" + value); } - // Check for single quotes in the tag value - if (value.find('\"') != std::string::npos) { - log_error("WARNING: double quote in tag value \"%1%\"", value); + // Check for double quotes in the tag key + if (key.find('\"') != std::string::npos) { + log_error("WARNING: double quote in tag key \"%1%\"", value); status->status.insert(badvalue); + status->values.insert(key + "=" + value); } } @@ -147,48 +151,49 @@ Semantic::checkNode(const osmobjects::OsmNode &node, const std::string &type, ya std::shared_ptr Semantic::checkWay(const osmobjects::OsmWay &way, const std::string &type, yaml::Yaml &tests, std::shared_ptr &status) { - // On non-english numeric locales using decimal separator different than '.' - // this is necessary to parse double strings with std::stod correctly - // without loosing precision - // std::setlocale(LC_NUMERIC, "C"); - setlocale(LC_NUMERIC, "C"); - - if (way.tags.size() == 0) { - status->status.insert(notags); - return status; - } if (way.action == osmobjects::remove) { return status; } - std::string key; - // List of valid tags to be validated - auto tags = tests.get("tags"); - // These values are in the config section of the YAML file auto config = tests.get("config"); + bool check_badvalue = config.get_value("badvalue") == "yes"; + bool check_incomplete = config.get_value("incomplete") == "yes"; - // Not using required_tags disables writing features flagged for not being tag complete - // from being written to the database thus reducing the size of the results. - auto required_tags = tests.get("required_tags"); + // List of required tags to be validated + yaml::Node required_tags; + if (check_incomplete) { + required_tags = tests.get("required_tags"); + } - // This enables/disables writing features flagged for not having values - // in range as defined in the YAML config file. This prevents those - // from being written to the database to reduce the size of the results. + // List of valid tags to be validated + yaml::Node tags; + if (check_badvalue) { + tags = tests.get("tags"); + } + + if (check_badvalue && way.tags.size() == 0) { + status->status.insert(notags); + return status; + } int tagexists = 0; for (auto vit = std::begin(way.tags); vit != std::end(way.tags); ++vit) { - if (!isValidTag(vit->first, vit->second, tags)) { - status->status.insert(badvalue); - status->values.insert(vit->first + "=" + vit->second); + if (check_badvalue) { + if (tags.children.size() > 0 && !isValidTag(vit->first, vit->second, tags)) { + status->status.insert(badvalue); + status->values.insert(vit->first + "=" + vit->second); + } + checkTag(vit->first, vit->second, status); } - if (isRequiredTag(vit->first, required_tags)) { - tagexists++; + if (check_incomplete) { + if (isRequiredTag(vit->first, required_tags)) { + tagexists++; + } } - // checkTag(vit->first, vit->second, status); } - if (tagexists != required_tags.children.size()) { + if (check_incomplete && tagexists != required_tags.children.size()) { status->status.insert(incomplete); } diff --git a/src/validate/validate.hh b/src/validate/validate.hh index 6f96b674..a03439e0 100644 --- a/src/validate/validate.hh +++ b/src/validate/validate.hh @@ -88,7 +88,7 @@ typedef enum { correct, badgeom, orphan, - overlaping, + overlapping, duplicate, valid } valerror_t; @@ -141,7 +141,7 @@ class ValidateStatus { results[correct] = "Correct tag value"; results[badgeom] = "Bad geometry"; results[orphan] = "Orphan"; - results[overlaping] = "Overlap"; + results[overlapping] = "Overlap"; results[duplicate] = "Duplicate"; for (const auto &stat: std::as_const(status)) { std::cerr << "\tResult: " << results[stat] << std::endl; @@ -183,7 +183,6 @@ class BOOST_SYMBOL_VISIBLE Validate { for (auto &file: std::filesystem::recursive_directory_iterator(path)) { std::filesystem::path config = file.path(); if (config.extension() == ".yaml") { - std::cout << "Loading: " << config.stem() << std::endl; yaml::Yaml yaml; yaml.read(config.string()); if (!config.stem().empty()) { diff --git a/src/wrappers/python.cc b/src/wrappers/python.cc index c3b0efbb..7bc666d3 100644 --- a/src/wrappers/python.cc +++ b/src/wrappers/python.cc @@ -47,7 +47,7 @@ std::map results = { {correct, "correct"}, {badgeom, "badgeom"}, {orphan, "orphan"}, - {overlaping, "overlaping"}, + {overlapping, "overlapping"}, {duplicate, "duplicate"} };