Fulltext refactor, including dupes checking

.dockerignore

@@ -7,3 +7,5 @@
 *talks*
 *data-backups*
 *certs*
+*.bak*
+*pre-fulltext-backups*

.envs/local

@@ -20,6 +20,8 @@ REDIS_URL=redis://redis:6379/0
 CELERY_FLOWER_USER=aePSKkcANYKRklVrFMzjQPlsHXUrQEzV
 CELERY_FLOWER_PASSWORD=Ykr75MuBE22sphvBlNzqLkLWRFD9NcnIbAnYkGbQARde4MViE1z6dKQDDafMtmCf
 
+VIRTUAL_HOST=20.90.118.144
+
 # HTTPS
 # ------------------------------------------------------------------------------
 # VIRTUAL_HOST=lwmdb-test.local

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ on:
     paths-ignore: ['docs/**']
 
   push:
-    branches: ['main', 'geocensus', 'production-deploy']
+    branches: ['main', 'geocensus', 'fulltext-hmd']
     paths-ignore: ['docs/**']
 
 concurrency:

.pre-commit-config.yaml

@@ -1,11 +1,13 @@
 exclude: '^docs/|/migrations/'
 default_stages: [commit]
+default_language_version:
+  python: python3.11
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
     hooks:
-      - id: check-docstring-first
+      # - id: check-docstring-first
       - id: check-merge-conflict
       - id: check-toml
       - id: check-yaml
@@ -77,7 +79,7 @@ repos:
       - id: black
 
   - repo: https://github.com/pycqa/flake8
-    rev: 6.1.0
+    rev: 7.0.0
     hooks:
       - id: flake8
         additional_dependencies:
@@ -130,6 +132,7 @@ repos:
           - tqdm
           - psycopg[binary]
           - validators
+          - django-admin-cursor-paginator
 # sets up .pre-commit-ci.yaml to ensure pre-commit dependencies stay up to date
 ci:
   autoupdate_schedule: weekly

README.md

@@ -1,4 +1,4 @@
-# Living With Machines Database: `lmwdb`
+# Living With Machines Database: `lwmdb`
 
 <!-- prettier-ignore-start -->
 [![DOI](https://zenodo.org/badge/460939495.svg)](https://zenodo.org/badge/latestdoi/460939495)

census/migrations/0001_initial.py

@@ -1,106 +1,194 @@
-# Generated by Django 3.2.13 on 2022-08-18 15:02
+# Generated by Django 4.2.7 on 2024-01-17 11:55
 
 from django.db import migrations, models
 import django.db.models.deletion
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
     dependencies = [
-        ('gazetteer', '0001_initial'),
+        ("gazetteer", "0001_initial"),
     ]
 
     operations = [
         migrations.CreateModel(
-            name='Record',
+            name="Record",
             fields=[
-                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('created_at', models.DateTimeField(auto_now_add=True)),
-                ('updated_at', models.DateTimeField(auto_now=True)),
-                ('CENSUS_YEAR', models.IntegerField()),
-                ('CEN', models.IntegerField()),
-                ('REGCNTY', models.CharField(max_length=40, null=True)),
-                ('REGDIST', models.CharField(max_length=40, null=True)),
-                ('SUBDIST', models.CharField(max_length=40, null=True)),
-                ('TYPE', models.CharField(max_length=20, null=True)),
-                ('POP_DENS', models.FloatField(null=True)),
-                ('POP', models.FloatField(null=True)),
-                ('ACRES', models.FloatField(null=True)),
-                ('TFR', models.FloatField(null=True)),
-                ('ASFR_20_24', models.FloatField(null=True)),
-                ('ASFR_25_29', models.FloatField(null=True)),
-                ('ASFR_30_34', models.FloatField(null=True)),
-                ('ASFR_35_39', models.FloatField(null=True)),
-                ('ASFR_40_44', models.FloatField(null=True)),
-                ('ASFR_45_49', models.FloatField(null=True)),
-                ('TMFR', models.FloatField(null=True)),
-                ('TMFR_25', models.FloatField(null=True)),
-                ('ASMFR_20_24', models.FloatField(null=True)),
-                ('ASMFR_25_29', models.FloatField(null=True)),
-                ('ASMFR_30_34', models.FloatField(null=True)),
-                ('ASMFR_35_39', models.FloatField(null=True)),
-                ('ASMFR_40_44', models.FloatField(null=True)),
-                ('ASMFR_45_49', models.FloatField(null=True)),
-                ('LEGIT_RATE', models.FloatField(null=True)),
-                ('ILLEG_RATE', models.FloatField(null=True)),
-                ('ILLEG_RATIO', models.FloatField(null=True)),
-                ('F_SMAM', models.FloatField(null=True)),
-                ('M_SMAM', models.FloatField(null=True)),
-                ('F_CEL_4554', models.FloatField(null=True)),
-                ('M_CEL_4554', models.FloatField(null=True)),
-                ('IMR', models.FloatField(null=True)),
-                ('ECMR', models.FloatField(null=True)),
-                ('DOC', models.FloatField(null=True)),
-                ('LP_FAM', models.FloatField(null=True)),
-                ('SINGLE_PER', models.FloatField(null=True)),
-                ('HOUSE_SERV', models.FloatField(null=True)),
-                ('BOARD', models.FloatField(null=True)),
-                ('HH_KIN', models.FloatField(null=True)),
-                ('AV_AGE', models.FloatField(null=True)),
-                ('AV_AGE_F', models.FloatField(null=True)),
-                ('AV_AGE_M', models.FloatField(null=True)),
-                ('DEPEND', models.FloatField(null=True)),
-                ('C_WORK_AGE', models.FloatField(null=True)),
-                ('ELD_WORK_AGE', models.FloatField(null=True)),
-                ('IRISH_BORN', models.FloatField(null=True)),
-                ('SR', models.FloatField(null=True)),
-                ('HC1', models.FloatField(null=True)),
-                ('HC2', models.FloatField(null=True)),
-                ('HC3', models.FloatField(null=True)),
-                ('HC4', models.FloatField(null=True)),
-                ('HC5', models.FloatField(null=True)),
-                ('SC1', models.FloatField(null=True)),
-                ('SC2', models.FloatField(null=True)),
-                ('SC3', models.FloatField(null=True)),
-                ('SC4', models.FloatField(null=True)),
-                ('SC5', models.FloatField(null=True)),
-                ('SC6', models.FloatField(null=True)),
-                ('SC7', models.FloatField(null=True)),
-                ('SC8', models.FloatField(null=True)),
-                ('FMAR_PRATE', models.FloatField(null=True)),
-                ('FNM_PRATE', models.FloatField(null=True)),
-                ('FWID_PRATE', models.FloatField(null=True)),
-                ('F_DOM', models.FloatField(null=True)),
-                ('F_TEX', models.FloatField(null=True)),
-                ('C_TEACHER', models.FloatField(null=True)),
-                ('F_CL_1013', models.FloatField(null=True)),
-                ('M_CL_1013', models.FloatField(null=True)),
-                ('F_CL_1418', models.FloatField(null=True)),
-                ('M_CL_1418', models.FloatField(null=True)),
-                ('REGCNTY_admin_county', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='REGCNTY_census_records', related_query_name='REGCNTY_census_record', to='gazetteer.admincounty')),
-                ('REGCNTY_historic_county', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='REGCNTY_census_records', related_query_name='REGCNTY_census_record', to='gazetteer.historiccounty')),
-                ('REGCNTY_place', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='REGCNTY_census_records', related_query_name='REGCNTY_census_record', to='gazetteer.place')),
-                ('REGDIST_admin_county', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='REGDIST_census_records', related_query_name='REGDIST_census_record', to='gazetteer.admincounty')),
-                ('REGDIST_historic_county', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='REGDIST_census_records', related_query_name='REGDIST_census_record', to='gazetteer.historiccounty')),
-                ('REGDIST_place', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='REGDIST_census_records', related_query_name='REGDIST_census_record', to='gazetteer.place')),
-                ('SUBDIST_admin_county', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='SUBDIST_census_records', related_query_name='SUBDIST_census_record', to='gazetteer.admincounty')),
-                ('SUBDIST_historic_county', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='SUBDIST_census_records', related_query_name='SUBDIST_census_record', to='gazetteer.historiccounty')),
-                ('SUBDIST_place', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='SUBDIST_census_records', related_query_name='SUBDIST_census_record', to='gazetteer.place')),
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("CENSUS_YEAR", models.IntegerField()),
+                ("CEN", models.IntegerField()),
+                ("REGCNTY", models.CharField(max_length=40, null=True)),
+                ("REGDIST", models.CharField(max_length=40, null=True)),
+                ("SUBDIST", models.CharField(max_length=40, null=True)),
+                ("TYPE", models.CharField(max_length=20, null=True)),
+                ("POP_DENS", models.FloatField(null=True)),
+                ("POP", models.FloatField(null=True)),
+                ("ACRES", models.FloatField(null=True)),
+                ("TFR", models.FloatField(null=True)),
+                ("ASFR_20_24", models.FloatField(null=True)),
+                ("ASFR_25_29", models.FloatField(null=True)),
+                ("ASFR_30_34", models.FloatField(null=True)),
+                ("ASFR_35_39", models.FloatField(null=True)),
+                ("ASFR_40_44", models.FloatField(null=True)),
+                ("ASFR_45_49", models.FloatField(null=True)),
+                ("TMFR", models.FloatField(null=True)),
+                ("TMFR_25", models.FloatField(null=True)),
+                ("ASMFR_20_24", models.FloatField(null=True)),
+                ("ASMFR_25_29", models.FloatField(null=True)),
+                ("ASMFR_30_34", models.FloatField(null=True)),
+                ("ASMFR_35_39", models.FloatField(null=True)),
+                ("ASMFR_40_44", models.FloatField(null=True)),
+                ("ASMFR_45_49", models.FloatField(null=True)),
+                ("LEGIT_RATE", models.FloatField(null=True)),
+                ("ILLEG_RATE", models.FloatField(null=True)),
+                ("ILLEG_RATIO", models.FloatField(null=True)),
+                ("F_SMAM", models.FloatField(null=True)),
+                ("M_SMAM", models.FloatField(null=True)),
+                ("F_CEL_4554", models.FloatField(null=True)),
+                ("M_CEL_4554", models.FloatField(null=True)),
+                ("IMR", models.FloatField(null=True)),
+                ("ECMR", models.FloatField(null=True)),
+                ("DOC", models.FloatField(null=True)),
+                ("LP_FAM", models.FloatField(null=True)),
+                ("SINGLE_PER", models.FloatField(null=True)),
+                ("HOUSE_SERV", models.FloatField(null=True)),
+                ("BOARD", models.FloatField(null=True)),
+                ("HH_KIN", models.FloatField(null=True)),
+                ("AV_AGE", models.FloatField(null=True)),
+                ("AV_AGE_F", models.FloatField(null=True)),
+                ("AV_AGE_M", models.FloatField(null=True)),
+                ("DEPEND", models.FloatField(null=True)),
+                ("C_WORK_AGE", models.FloatField(null=True)),
+                ("ELD_WORK_AGE", models.FloatField(null=True)),
+                ("IRISH_BORN", models.FloatField(null=True)),
+                ("SR", models.FloatField(null=True)),
+                ("HC1", models.FloatField(null=True)),
+                ("HC2", models.FloatField(null=True)),
+                ("HC3", models.FloatField(null=True)),
+                ("HC4", models.FloatField(null=True)),
+                ("HC5", models.FloatField(null=True)),
+                ("SC1", models.FloatField(null=True)),
+                ("SC2", models.FloatField(null=True)),
+                ("SC3", models.FloatField(null=True)),
+                ("SC4", models.FloatField(null=True)),
+                ("SC5", models.FloatField(null=True)),
+                ("SC6", models.FloatField(null=True)),
+                ("SC7", models.FloatField(null=True)),
+                ("SC8", models.FloatField(null=True)),
+                ("FMAR_PRATE", models.FloatField(null=True)),
+                ("FNM_PRATE", models.FloatField(null=True)),
+                ("FWID_PRATE", models.FloatField(null=True)),
+                ("F_DOM", models.FloatField(null=True)),
+                ("F_TEX", models.FloatField(null=True)),
+                ("C_TEACHER", models.FloatField(null=True)),
+                ("F_CL_1013", models.FloatField(null=True)),
+                ("M_CL_1013", models.FloatField(null=True)),
+                ("F_CL_1418", models.FloatField(null=True)),
+                ("M_CL_1418", models.FloatField(null=True)),
+                (
+                    "REGCNTY_admin_county",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="REGCNTY_census_records",
+                        related_query_name="REGCNTY_census_record",
+                        to="gazetteer.admincounty",
+                    ),
+                ),
+                (
+                    "REGCNTY_historic_county",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="REGCNTY_census_records",
+                        related_query_name="REGCNTY_census_record",
+                        to="gazetteer.historiccounty",
+                    ),
+                ),
+                (
+                    "REGCNTY_place",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="REGCNTY_census_records",
+                        related_query_name="REGCNTY_census_record",
+                        to="gazetteer.place",
+                    ),
+                ),
+                (
+                    "REGDIST_admin_county",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="REGDIST_census_records",
+                        related_query_name="REGDIST_census_record",
+                        to="gazetteer.admincounty",
+                    ),
+                ),
+                (
+                    "REGDIST_historic_county",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="REGDIST_census_records",
+                        related_query_name="REGDIST_census_record",
+                        to="gazetteer.historiccounty",
+                    ),
+                ),
+                (
+                    "REGDIST_place",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="REGDIST_census_records",
+                        related_query_name="REGDIST_census_record",
+                        to="gazetteer.place",
+                    ),
+                ),
+                (
+                    "SUBDIST_admin_county",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="SUBDIST_census_records",
+                        related_query_name="SUBDIST_census_record",
+                        to="gazetteer.admincounty",
+                    ),
+                ),
+                (
+                    "SUBDIST_historic_county",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="SUBDIST_census_records",
+                        related_query_name="SUBDIST_census_record",
+                        to="gazetteer.historiccounty",
+                    ),
+                ),
+                (
+                    "SUBDIST_place",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="SUBDIST_census_records",
+                        related_query_name="SUBDIST_census_record",
+                        to="gazetteer.place",
+                    ),
+                ),
             ],
             options={
-                'abstract': False,
+                "abstract": False,
             },
         ),
     ]

compose/local/django/start

@@ -5,9 +5,11 @@ set -o pipefail
 set -o nounset
 
 
+# python manage.py makemigrations --merge --update
 python manage.py makemigrations
 python manage.py migrate
 # python manage.py runserver_plus 0.0.0.0:8000
 # python manage.py shell_plus --notebook
+
 # use if serving async
 exec uvicorn config.asgi:application --host 0.0.0.0 --reload --reload-include '*.html'

compose/local/docs/Dockerfile

@@ -28,9 +28,9 @@ COPY . .
 
 RUN pip install poetry
 
-EXPOSE 9000
+EXPOSE 8000
 
 RUN poetry config virtualenvs.create false && poetry install --with docs
 
 ENTRYPOINT [ "mkdocs" ]
-CMD ["serve", "--dev-addr=0.0.0.0:9000"]
+CMD ["serve", "-f", "/app/mkdocs.yml", "--dev-addr=0.0.0.0:8000"]

compose/local/docs/start

@@ -5,4 +5,4 @@ set -o pipefail
 set -o nounset
 
 # make livehtml
-mkdocs serve --dev-addr=0.0.0.0:9000
+mkdocs serve -a --dev-addr=0.0.0.0:9000

compose/production/django/Dockerfile

@@ -48,6 +48,7 @@ FROM python as python-run-stage
 
 ARG BUILD_ENVIRONMENT=production
 ARG APP_HOME=/app
+ARG STATIC_FILES=${APP_HOME}/staticfiles
 
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1
@@ -116,6 +117,10 @@ COPY --from=client-builder --chown=django:django ${APP_HOME} ${APP_HOME}
 # make django owner of the WORKDIR directory as well.
 RUN chown django:django ${APP_HOME}
 
+
+WORKDIR ${APP_HOME}
+
+
 USER django
 
 ENTRYPOINT ["/entrypoint"]