Merge pull request #167 from nationalarchives/improve-test-fixtures

Add more feedback to test fixtures load script

Author: jacksonj04

README.md

@@ -43,9 +43,9 @@ You'll then need to deploy the configuration (see Deployment, above)
 
 Ensure that `MARKLOGIC_HOST` in `.env` in the editor and public ui is set to `host.docker.internal` in `.env` and that the username and password are both `admin` if you want to use them with the local instance.
 
-### 4. (Optional) Populate data in the local database
+### 4. (Optional) Populate test fixtures in the local database
 
-To get some example documents onto the local database, there are `development_scripts/populate_top_judgments_and_neighbours.py` and `development_scripts/populate_from_caselaw.py` which copy documents from the live caselaw site (they don't import or fake properties) into your local database. (Check https://caselaw.nationalarchives.gov.uk/terms-of-use and get in touch if you intend to download many more than these.)
+To load test fixtures, run `development_scripts/populate_from_caselaw.py`. This will load a variety of documents.
 
 There are also other ways other importing data as detailed further down the readme but haven't been tested for a while.
 

development_scripts/populate_from_caselaw.py

@@ -33,43 +33,55 @@ def test_response(response):
     try:
         response.raise_for_status()
     except Exception:
+        print("🔴 EXCEPTION")
         print("Content of server response:")
         print(response.content)
         raise
 
 
 def load_fixture_from_url(url: str) -> None:
+    print(f"> Loading {url} from URL…")
+
     ml_url = f"/{url}.xml"
-    print(url)
+
     response = requests.get(f"https://caselaw.nationalarchives.gov.uk/{url}/data.xml")
     test_response(response)
     xml = response.content
-    print("got xml")
+    print(">> Downloaded XML…")
 
     response = requests.put(
         f"http://admin:admin@localhost:8011/LATEST/documents?uri={ml_url}&collection=judgment",
         data=xml,
     )
     test_response(response)
-    print("added to local Marklogic db")
+    print(">> ✅ Saved to MarkLogic")
 
 
 def load_fixture_from_files(file_prefix: str) -> None:
+    print(f"> Loading {filename} from filesystem…")
+
     ml_url = "/" + filename.replace("-", "/") + ".xml"
+
     with Path("development_scripts", "fixture_data", f"{filename}-content.xml").open() as f:
         content = f.read()
     with Path("development_scripts", "fixture_data", f"{filename}-properties.xml").open() as f:
         properties = f.read()
+
+    print(">> Source files read to memory…")
+
     response = requests.put(
         f"http://admin:admin@localhost:8011/LATEST/documents?uri={ml_url}&collection=judgment",
         data=content,
     )
     test_response(response)
+    print(">> Saved source document to MarkLogic…")
+
     response = requests.put(
         f"http://admin:admin@localhost:8011/LATEST/documents?uri={ml_url}&category=properties",
         data=properties,
     )
     test_response(response)
+    print(">> Saved properties document to MarkLogic…")
 
     xquery = f"""
     xquery version "1.0-ml";
@@ -78,13 +90,18 @@ def load_fixture_from_files(file_prefix: str) -> None:
     """
 
     response = requests.post("http://admin:admin@localhost:8011/LATEST/eval", data={"xquery": xquery})
+    test_response(response)
+    print(">> Set as managed document…")
+
+    print(">> ✅ Saved to MarkLogic")
 
-    print(response.content)
-    print(f"added {filename} to local Marklogic db")
 
+print("Beginning fixture import…")
 
+print("Loading fixtures from URLs…")
 for url in URLS_TO_IMPORT:
     load_fixture_from_url(url)
 
+print("Loading fixtures from files…")
 for filename in FIXTURE_FILES_TO_IMPORT:
     load_fixture_from_files(file_prefix=filename)

development_scripts/populate_top_judgments_and_neighbours.py

@@ -45,4 +45,3 @@ unfixable = ["ERA"]
        "S113",   # requests no timeout : TODO
       ]
 "tests/*" = ["S101"]  # assert fine in tests
-"development_scripts/populate_top_judgments_and_neighbours.py" = ["S314", "C400"] # TODO