Skip to content

Commit

Permalink
Use the new JSON tag2links #230 #264 #335 #418 #475 #498 #405
Browse files Browse the repository at this point in the history
  • Loading branch information
@frodrigo
frodrigo committed Jun 4, 2024
1 parent 642a745 commit 16a3a51
Show file tree
Hide file tree
Showing 6 changed files with 2,791 additions and 428 deletions.
24 changes: 8 additions & 16 deletions api/issue.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from modules.query import fixes_default
from modules.utils import LangsNegociation

from .issue_utils import _expand_tags, _get, t2l
from .issue_utils import _get, _keys, t2l

router = APIRouter()

Expand Down Expand Up @@ -218,7 +218,7 @@ def _error(
data_type[elem["type"]]: True,
"type": data_type[elem["type"]],
"id": elem["id"],
"tags": _expand_tags(tags, t2l.checkTags(tags)),
"tags": t2l.addLinks(tags),
"fixes": [],
}
for fix_index, fix_group in enumerate(marker["fixes"] or []):
Expand All @@ -231,13 +231,9 @@ def _error(
tmp_elem["fixes"].append(
{
"num": fix_index,
"add": _expand_tags(
fix["create"], t2l.checkTags(fix["create"])
),
"mod": _expand_tags(
fix["modify"], t2l.checkTags(fix["modify"])
),
"del": _expand_tags(fix["delete"], {}, True),
"add": t2l.addLinks(fix["create"]),
"mod": t2l.addLinks(fix["modify"]),
"del": _keys(fix["delete"]),
}
)
elems.append(tmp_elem)
Expand All @@ -255,13 +251,9 @@ def _error(
new_elems.append(
{
"num": fix_index,
"add": _expand_tags(
fix["create"], t2l.checkTags(fix["create"])
),
"mod": _expand_tags(
fix["modify"], t2l.checkTags(fix["modify"])
),
"del": _expand_tags(fix["delete"], {}, True),
"add": t2l.addLinks(fix["create"]),
"mod": t2l.addLinks(fix["modify"]),
"del": _keys(fix["delete"]),
}
)

Expand Down
18 changes: 3 additions & 15 deletions api/issue_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from .tool import tag2link

t2l = tag2link.tag2link(
os.path.dirname(os.path.realpath(__file__)) + "/tool/tag2link_sources.xml"
os.path.dirname(os.path.realpath(__file__)) + "/tool/tag2link_sources.json"
)


Expand Down Expand Up @@ -92,17 +92,5 @@ async def _get(
}


def _expand_tags(
tags: Dict[str, str], links: Dict[str, str], short: bool = False
) -> List[Dict[str, str]]:
t = []
if short:
for k in tags:
t.append({"k": k})
else:
for k, v in sorted(tags.items()):
if links and k in links:
t.append({"k": k, "v": v, "vlink": links[k]})
else:
t.append({"k": k, "v": v})
return t
def _keys(tags: Dict[str, str]) -> List[Dict[str, str]]:
return list(map(lambda k: {"k": k}, tags.keys()))
150 changes: 31 additions & 119 deletions api/tool/tag2link.py
View file
Original file line number Diff line number Diff line change
@@ -1,132 +1,44 @@
# https://trac.openstreetmap.org/browser/subversion/applications/editors/josm/plugins/tag2link/resources/tag2link_sources.xml?rev=30720&format=txt
# https://raw.githubusercontent.com/JOSM/tag2link/master/index.json

import re
import xml.sax
from typing import Any, Dict, List
from xml.sax.xmlreader import AttributesImpl


class Exact(xml.sax.handler.ContentHandler):
def __init__(self):
self.rules: List[Dict[str, Any]] = []
self.val: Regex = re.compile("(%[^%]+%|\{[^}]+})") # noqa

def unposix(self, regex: str) -> str:
regex = regex.replace("\p{Lower}", "[a-z]") # noqa
regex = regex.replace("\p{Upper}", "[A-Z]") # noqa
regex = regex.replace("\p{Digit}", "[0-9]") # noqa
return regex

def startElement(self, name: str, attrs: AttributesImpl) -> None:
if name == "rule":
self.rules.append({"conditions": [], "link": None})
elif name == "condition":
self.rules[-1]["conditions"].append(
{
"kk": attrs["k"],
"k": re.compile("^" + self.unposix(attrs["k"]) + "$"),
"v": (
re.compile("^" + self.unposix(attrs["v"]) + "$")
if "v" in attrs
else None
),
"id": attrs["id"] if "id" in attrs else None,
}
)
elif name == "link":
link = attrs["href"]
subs = []
if "%" in link or "{" in link:
for valmatch in self.val.finditer(link):
sub = []
v = valmatch.group(1)
link = link.replace(v, "%s", 1)
var = v[1:-1].split(":")
for ll in var:
if "." not in ll and ll != "v" and ll != "k":
vv = ll
else:
vv = ll.split(".")
if vv[0] in ("k", "v"):
vv = [None] + vv
if len(vv) == 2:
vv.append(0)
else:
vv[2] = int(vv[2])
sub.append(vv)
subs.append(sub)
self.rules[-1]["link"] = {"url": link, "subs": subs}
import json
from typing import Dict, List


class tag2link:
def __init__(self, rulesFiles: str):
parser = xml.sax.make_parser()
handler = Exact()
parser.setContentHandler(handler)
parser.parse(open(rulesFiles, "r", encoding="utf-8"))
self.rules = handler.rules
self.all = re.compile(".*")
rules_json = json.load(open(rulesFiles, "r", encoding="utf-8"))
self.rules: Dict[str, str] = dict(
map(
lambda rule: [
rule["key"][4:],
rule["url"],
],
filter(lambda rule: rule["key"].startswith("Key:"), rules_json),
)
)

def checkTags(self, tags: Dict[str, str]) -> Dict[str, str]:
try:
urls: Dict[str, str] = {}
for rule in self.rules:
valid = True
id = {}
for condition in rule["conditions"]:
match = False
for key in tags.keys():
kmatch = condition["k"].match(key)
if kmatch:
id[condition["id"]] = {"k": kmatch}
if condition["v"] is None:
vmatch = self.all.match(tags[key])
id[condition["id"]]["v"] = vmatch
match = True
break
else:
vmatch = condition["v"].match(tags[key])
if vmatch:
id[condition["id"]]["v"] = vmatch
match = True
break
if not match:
valid = False
break
if valid:
replace = []
for sub in rule["link"]["subs"]:
for v in sub:
if isinstance(v, str):
replace.append(v)
break
else:
val = id[v[0]][v[1]].group(v[2])
if val:
replace.append(val)
break
ret = rule["link"]["url"] % tuple(replace)
if "://" not in ret:
ret = "http://" + ret
urls[id[rule["link"]["subs"][0][0][0]]["k"].group(0)] = ret
return urls
except Exception:
return {}
def addLinks(self, tags: Dict[str, str]) -> List[Dict[str, str]]:
links = []
for key, value in tags.items():
links.append({"k": key, "v": value})
if key in self.rules:
links[-1]["vlink"] = self.rules[key].replace("$1", tags[key])
return links


if __name__ == "__main__":
t2l = tag2link("tag2link_sources.xml")
print(t2l.checkTags({"oneway": "yes"}))
print(t2l.checkTags({"url": "plop.com"}))
print(t2l.checkTags({"url": "http://plop.com"}))
print(t2l.checkTags({"ref:UAI": "123"}))
t2l = tag2link("tag2link_sources.json")
print(t2l.addLinks({"oneway": "yes"}))
print(t2l.addLinks({"url": "plop.com"}))
print(t2l.addLinks({"url": "http://plop.com"}))
print(t2l.addLinks({"ref:UAI": "123"}))
print(
t2l.checkTags(
t2l.addLinks(
{"man_made": "survey_point", "source": "©IGN 2012", "ref": "1234567 - A"}
)
)
print(
t2l.checkTags(
t2l.addLinks(
{
"url": "span://bad",
"man_made": "survey_point",
Expand All @@ -135,7 +47,7 @@ def checkTags(self, tags: Dict[str, str]) -> Dict[str, str]:
}
)
)
print(t2l.checkTags({"wikipedia:fr": "toto"}))
print(t2l.checkTags({"wikipedia": "fr:toto"}))
print(t2l.checkTags({"wikipedia": "toto"}))
print(t2l.checkTags({"source": "source", "source:url": "http://example.com"}))
print(t2l.addLinks({"wikipedia:fr": "toto"}))
print(t2l.addLinks({"wikipedia": "fr:toto"}))
print(t2l.addLinks({"wikipedia": "toto"}))
print(t2l.addLinks({"source": "source", "source:url": "http://example.com"}))
Loading

0 comments on commit 16a3a51

Please sign in to comment.