82 introduce adm types (#84)

* minor [skip ci]

* adds administrative_division and mgm to update/populate this filed

* readme updates [skip ci]

.gitignore

@@ -43,3 +43,4 @@ hansi.csv
 hansi.json
 1824-12-05.json
 1914-12-05.json
+sources.csv

README.md

@@ -12,6 +12,23 @@ HistoGIS is based upon django, geodjango and [djangobaseproject](https://github.
 2. Adapt the information in `webpage/metadata.py` according to your needs.
 3. Create and activate a virtual environment and run `pip install -r requirements.txt`.
 
+
+## useful commands
+
+### cleanup
+removes not used vocabs and sources without shapes
+```bash
+python manage.py cleanup
+```
+
+### update administrative divisions
+updates administrative_division fields according to [this google spreadsheet](https://docs.google.com/spreadsheets/d/14WNuiB7KnnezWndKJslw-j-EdHRF04CH2M1HqVSmPGg)
+
+```bash
+python manage.py update_amd
+```
+
+
 ### building the image
 
 `docker build -t histogis:latest .`

histogis/settings.py

@@ -72,7 +72,6 @@
     "news",
     "analyze",
     "archeutils",
-
 ]
 
 CRISPY_ALLOWED_TEMPLATE_PACKS = "bootstrap5"

shps/custom_api_views.py

@@ -24,7 +24,5 @@ def shapes_geojson(request):
         lambda geom: round_coords(geom, precision=2)
     )
     data = json.loads(gdf.to_json())
-    data["metadata"] = {
-        "number of objects": len(gdf)
-    }
+    data["metadata"] = {"number of objects": len(gdf)}
     return JsonResponse(data)

shps/forms.py

@@ -88,7 +88,12 @@ def __init__(self, *args, **kwargs):
         self.form_tag = False
         self.layout = Layout(
             BS5Accordion(
-                AccordionGroup("Basic Search", "name", css_id="basic_search_fields"),
+                AccordionGroup(
+                    "Basic Search",
+                    "name",
+                    "administrative_division",
+                    css_id="basic_search_fields",
+                ),
                 AccordionGroup(
                     "Advanced search",
                     "description",
@@ -133,6 +138,7 @@ def __init__(self, *args, **kwargs):
                     "all_name",
                     "name",
                     "alt_name",
+                    "administrative_division",
                     "in_between",
                     css_id="basic_search_fields",
                 ),

shps/management/commands/update_amd.py

@@ -0,0 +1,23 @@
+from django.core.management.base import BaseCommand
+
+from shps.models import Source, TempSpatial
+from shps.utils import gsheet_to_df
+
+
+class Command(BaseCommand):
+
+    help = """updates administrative_division fields according to
+    https://docs.google.com/spreadsheets/d/14WNuiB7KnnezWndKJslw-j-EdHRF04CH2M1HqVSmPGg"""
+
+    def handle(self, *args, **options):
+        df = gsheet_to_df("14WNuiB7KnnezWndKJslw-j-EdHRF04CH2M1HqVSmPGg")
+        for i, row in df.iterrows():
+            adm = row["adm"].lower().replace(" ", "")
+            source = Source.objects.get(id=row["id"])
+            source.administrative_division = adm
+            source.save()
+            print(f"processing objects related to {source}")
+            for x in TempSpatial.objects.filter(source=source):
+                x.administrative_division = adm
+                x.save()
+        print("done")

shps/migrations/0002_source_administrative_division_and_more.py

@@ -0,0 +1,79 @@
+# Generated by Django 5.1.3 on 2024-12-04 11:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("shps", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="source",
+            name="administrative_division",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    (
+                        "adm1",
+                        "ADM 1: a primary administrative division of a country, such as a state in the United States",
+                    ),
+                    (
+                        "adm2",
+                        "ADM 2: a subdivision of a first-order administrative division",
+                    ),
+                    (
+                        "adm3",
+                        "ADM 3: a subdivision of a second-order administrative division",
+                    ),
+                    (
+                        "adm4",
+                        "ADM 4: a subdivision of a third-order administrative division",
+                    ),
+                    (
+                        "adm5",
+                        "ADM 5: a subdivision of a fourth-order administrative division",
+                    ),
+                ],
+                help_text="Mainly used to group objects by similar hierarchy level; (http://www.geonames.org/export/codes.html)",
+                max_length=5,
+                null=True,
+                verbose_name="administrative division",
+            ),
+        ),
+        migrations.AddField(
+            model_name="tempspatial",
+            name="administrative_division",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    (
+                        "adm1",
+                        "ADM 1: a primary administrative division of a country, such as a state in the United States",
+                    ),
+                    (
+                        "adm2",
+                        "ADM 2: a subdivision of a first-order administrative division",
+                    ),
+                    (
+                        "adm3",
+                        "ADM 3: a subdivision of a second-order administrative division",
+                    ),
+                    (
+                        "adm4",
+                        "ADM 4: a subdivision of a third-order administrative division",
+                    ),
+                    (
+                        "adm5",
+                        "ADM 5: a subdivision of a fourth-order administrative division",
+                    ),
+                ],
+                help_text="Mainly used to group objects by similar hierarchy level; (http://www.geonames.org/export/codes.html)",
+                max_length=5,
+                null=True,
+                verbose_name="administrative division",
+            ),
+        ),
+    ]

shps/models.py

@@ -18,6 +18,16 @@
 
 ARCHE = Namespace("https://vocabs.acdh.oeaw.ac.at/schema#")
 ACDH = Namespace("https://id.acdh.oeaw.ac.at/")
+ADM_CHOICES = (
+    (
+        "adm1",
+        "ADM 1: a primary administrative division of a country, such as a state in the United States",
+    ),
+    ("adm2", "ADM 2: a subdivision of a first-order administrative division"),
+    ("adm3", "ADM 3: a subdivision of a second-order administrative division"),
+    ("adm4", "ADM 4: a subdivision of a third-order administrative division"),
+    ("adm5", "ADM 5: a subdivision of a fourth-order administrative division"),
+)
 
 curent_date = datetime.now().strftime("%Y-%m-%d")
 
@@ -69,6 +79,14 @@ class Source(models.Model):
     quote = models.TextField(
         blank=True, null=True, verbose_name="Quote", help_text="How to quote."
     )
+    administrative_division = models.CharField(
+        blank=True,
+        null=True,
+        choices=ADM_CHOICES,
+        max_length=5,
+        verbose_name="administrative division",
+        help_text="Mainly used to group objects by similar hierarchy level; (http://www.geonames.org/export/codes.html)",
+    )
     original_url = models.TextField(
         blank=True,
         null=True,
@@ -160,6 +178,14 @@ class TempSpatial(IdProvider):
         verbose_name="Alternative Names",
         help_text="Alternative Names, use '; ' as separator in case of more names",
     )
+    administrative_division = models.CharField(
+        blank=True,
+        null=True,
+        choices=ADM_CHOICES,
+        max_length=5,
+        verbose_name="administrative division",
+        help_text="Mainly used to group objects by similar hierarchy level; (http://www.geonames.org/export/codes.html)",
+    )
     wikidata_id = models.CharField(
         max_length=500,
         blank=True,

shps/utils.py

@@ -1,4 +1,7 @@
+import requests
+import pandas as pd
 from shapely.geometry import mapping, shape
+from io import BytesIO
 
 
 def round_coords(geometry, precision=2):
@@ -21,3 +24,13 @@ def _round_coords_recursive(coords, precision):
     ):  # Handle nested lists (e.g., multipolygons, polygons)
         return [_round_coords_recursive(c, precision) for c in coords]
     return [round(coord, precision) for coord in coords]
+
+
+def gsheet_to_df(sheet_id):
+    GDRIVE_BASE_URL = "https://docs.google.com/spreadsheet/ccc?key="
+    url = f"{GDRIVE_BASE_URL}{sheet_id}&output=csv"
+    r = requests.get(url)
+    print(r.status_code)
+    data = r.content
+    df = pd.read_csv(BytesIO(data))
+    return df

vocabs/skos.py

@@ -15,7 +15,9 @@ def __init__(self, csv_file):
         self.data = [x for x in csv.reader(self.csv_file)]
         self.headers = self.data[0]
         try:
-            self.alt_lang = (self.headers[1])[(self.headers[1]).index("@") + 1 :]  # noqa: E203
+            self.alt_lang = (self.headers[1])[
+                (self.headers[1]).index("@") + 1 :  # noqa: E203
+            ]  # noqa: E203
         except:  # noqa: E722
             self.alt_lang = None
         self.schemes = set([x[0] for x in self.data[1:]])