From 1a64e45e9ffc8050a914f19324b2724453e9cef1 Mon Sep 17 00:00:00 2001
From: xxyzz <gitpull@protonmail.com>
Date: Mon, 9 Oct 2023 17:17:34 +0800
Subject: [PATCH] Add validate JSON script

---
 json_schema/validate.py | 36 ++++++++++++++++++++++++++++++++++++
 pyproject.toml          | 10 +++++++++-
 2 files changed, 45 insertions(+), 1 deletion(-)
 create mode 100644 json_schema/validate.py

diff --git a/json_schema/validate.py b/json_schema/validate.py
new file mode 100644
index 00000000..1fb53e46
--- /dev/null
+++ b/json_schema/validate.py
@@ -0,0 +1,36 @@
+import argparse
+import json
+from concurrent.futures import ProcessPoolExecutor
+from functools import partial
+from pathlib import Path
+
+
+def worker(line, schema={}):
+    from jsonschema import validate
+
+    validate(instance=json.loads(line), schema=schema)
+
+
+def main():
+    """
+    Validate extracted JSONL file with JSON schema.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("jsonl_path", type=Path)
+    parser.add_argument("schema_path", type=Path)
+    args = parser.parse_args()
+
+    with (
+        args.jsonl_path.open(encoding="utf-8") as jsonl_f,
+        args.schema_path.open(encoding="utf-8") as schema_f,
+        ProcessPoolExecutor() as executor,
+    ):
+        schema = json.load(schema_f)
+        for _ in executor.map(
+            partial(worker, schema=schema), jsonl_f, chunksize=1000
+        ):
+            pass
+
+
+if __name__ == "__main__":
+    main()
diff --git a/pyproject.toml b/pyproject.toml
index 8556bbcd..32afc44b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -39,6 +39,7 @@ dependencies = [
 [project.optional-dependencies]
 dev = [
     "black",
+    "jsonschema",
     "mypy",
     "nose2[coverage_plugin]",
     "ruff",
@@ -55,7 +56,14 @@ homepage = "https://github.com/tatuylonen/wiktextract"
 zip-safe = false
 
 [tool.setuptools.packages.find]
-exclude = ["languages", "overrides", "tests", "tools", "usertools"]
+exclude = [
+    "languages",
+    "overrides",
+    "tests",
+    "tools",
+    "usertools",
+    "json_schema"
+]
 
 [tool.setuptools.package-data]
 wiktextract = [