Merge pull request #181 from legiongis/land_manager_update

Land Manager account improvements; site-filter refactor; and more...

fpan/admin.py

@@ -1,10 +1,9 @@
 from django.contrib.gis import admin
-from fpan.models.region import Region
-from fpan.models.managedarea import ManagedArea
+from fpan.models import Region, ManagedArea
 
 class ManagedAreaAdmin(admin.GeoModelAdmin):
 
     search_fields = ["name"]
 
-admin.site.register(Region, admin.GeoModelAdmin)
-admin.site.register(ManagedArea, ManagedAreaAdmin)
+# admin.site.register(Region, admin.GeoModelAdmin)
+# admin.site.register(ManagedArea, ManagedAreaAdmin)

fpan/fixtures/agencies.json

@@ -0,0 +1,37 @@
+[
+{
+ "model": "fpan.agency",
+ "fields": {
+  "code": "fsp",
+  "name": "FL Dept. of Environmental Protection, Div. of Recreation and Parks"
+ }
+},
+{
+ "model": "fpan.agency",
+ "fields": {
+  "code": "ffs",
+  "name": "FL Dept. of Agriculture and Consumer Services, Florida Forest Service"
+ }
+},
+{
+ "model": "fpan.agency",
+ "fields": {
+  "code": "fwcc",
+  "name": "FL Fish and Wildlife Conservation Commission"
+ }
+},
+{
+ "model": "fpan.agency",
+ "fields": {
+  "code": "fco",
+  "name": "FL Dept. of Environmental Protection, Florida Coastal Office"
+ }
+},
+{
+ "model": "fpan.agency",
+ "fields": {
+  "code": "owp",
+  "name": "FL Dept. of Environmental Protection, Office of Water Policy"
+ }
+}
+]

fpan/fixtures/management_area_types.json

@@ -0,0 +1,26 @@
+[
+{
+ "model": "fpan.managementareatype",
+ "fields": {
+  "name": "State Park"
+ }
+},
+{
+ "model": "fpan.managementareatype",
+ "fields": {
+  "name": "County Park"
+ }
+},
+{
+ "model": "fpan.managementareatype",
+ "fields": {
+  "name": "State Forest"
+ }
+},
+{
+ "model": "fpan.managementareatype",
+ "fields": {
+  "name": "Aquatic Preserve"
+ }
+}
+]

fpan/management/commands/add_districts_to_managed_areas.py

@@ -1,7 +1,7 @@
 import os
 import csv
 from django.core.management.base import BaseCommand, CommandError
-from fpan.models.managedarea import ManagedArea
+from fpan.models import ManagedArea
 
 class Command(BaseCommand):
 

fpan/management/commands/check_indexes.py

@@ -0,0 +1,106 @@
+import os
+import csv
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from arches.app.models.resource import Resource
+from arches.app.models.graph import Graph
+from arches.app.search.search_engine_factory import SearchEngineInstance as se
+
+class Command(BaseCommand):
+
+    help = 'Checks the current ElasticSearch resource index against the ORM '\
+    'objects and prints a list of missing resources to the logs directory.'
+
+    def add_arguments(self, parser):
+
+        parser.add_argument("--index",
+            action="store_true",
+            default=False,
+            help="attempt to index resources that aren't in index."
+        )
+
+    def handle(self, *args, **options):
+
+        es_contents = self.get_es_contents()
+
+        graphs = Graph.objects.filter(isresource=True).exclude(name="Arches System Settings")
+
+        for graph in graphs:
+
+            print(graph.name)
+
+            missing = []
+            uuid_resids = Resource.objects.filter(graph=graph).values_list('resourceinstanceid', flat=True)
+            db_resourceids = set([str(i) for i in uuid_resids])
+
+            try:
+                es_resourceids = es_contents[str(graph.pk)]
+            except KeyError:
+                es_resourceids = set()
+
+            print(f"- in db: {len(db_resourceids)}")
+            print(f"- in index: {len(es_resourceids)}")
+            if db_resourceids != es_resourceids:
+                es_diff = list(es_resourceids - db_resourceids)
+                if len(es_diff) > 0:
+                    print(f"  {len(es_diff)} indexed resources not in db:")
+                    [print("    " + i) for i in es_diff[:5]]
+                    if len(es_diff) > 5:
+                        print("    ...")
+                db_diff = list(db_resourceids - es_resourceids)
+                if len(db_diff) > 0:
+                    print(f"  {len(db_diff)} db resources not in index:")
+                    [print("    " + i) for i in db_diff[:5]]
+                    if len(db_diff) > 5:
+                        print("    ...")
+                    if options["index"]:
+                        print("    indexing these resources now...")
+                        for id in db_diff:
+                            r = Resource.objects.get(pk=id)
+                            try:
+                                r.index()
+                            except Exception as e:
+                                print(e)
+                                break
+
+
+
+    def get_es_contents(self):
+
+        summary = dict()
+        for resinfo in self.iterate_all_documents(se, 'resources'):
+            resid, graphid = resinfo
+            if graphid != 'None':
+                if graphid in summary:
+                    summary[graphid].add(resid)
+                else:
+                    summary[graphid] = set([resid])
+        return summary
+
+    def iterate_all_documents(self, se, index, pagesize=250, scroll_timeout="1m"):
+        """
+        Helper to iterate ALL values from a single index. Yields all the documents.
+        https://techoverflow.net/2019/05/07/elasticsearch-how-to-iterate-scroll-through-all-documents-in-index/
+        """
+        is_first = True
+        while True:
+            # Scroll next
+            if is_first: # Initialize scroll
+                result = se.search(index=index, scroll="1m", body={
+                    "size": pagesize
+                })
+                is_first = False
+            else:
+                ## note: need to access the ElasticSearch() instance directly
+                ## here, (.es), because the Arches se object doesn't inherit .scroll()
+                result = se.es.scroll(body={
+                    "scroll_id": scroll_id,
+                    "scroll": scroll_timeout
+                })
+            scroll_id = result["_scroll_id"]
+            hits = result["hits"]["hits"]
+            # Stop after no more docs
+            if not hits:
+                break
+            # Yield each entry
+            yield from ((hit['_source']['resourceinstanceid'], hit['_source']['graph_id']) for hit in hits)

fpan/media/js/views/components/search/site-filter.js

@@ -1,37 +1,19 @@
 define([
     'knockout',
     'views/components/search/base-filter',
-    'fpan'
-], function(ko, BaseFilter, FPAN) {
+], function(ko, BaseFilter) {
     var componentName = 'site-filter';
-    console.log("at the top of the file");
     return ko.components.register(componentName, {
         viewModel: BaseFilter.extend({
             initialize: function(options) {
                 options.name = 'Site Filter';
-
-                console.log("initializing site filter");
                 BaseFilter.prototype.initialize.call(this, options);
-
                 this.restoreState();
             },
 
-            // this doesn't seem to be used anywhere and can likely be removed.
-            updateQuery: function() {
-                console.log("updating query");
-                var queryObj = this.query();
-                queryObj[componentName] = 'enabled';
-                this.query(queryObj);
-            },
-
             restoreState: function() {
-                console.log("restoring state");
                 var queryObj = this.query();
-                if (FPAN.full_site_access == "True") {
-                  queryObj[componentName] = 'disabled';
-                } else {
-                  queryObj[componentName] = 'enabled';
-                }
+                queryObj[componentName] = 'enabled';
                 this.query(queryObj);
             },
 

fpan/models/managedarea.py → fpan/models.py

@@ -1,26 +1,41 @@
 from django.contrib.gis.db import models
 
+from arches.app.models.models import UserProfile
+
+## this model will be deprecated in favor of ManagementArea and ManagementAreaGroup
+class Region(models.Model):
+    name = models.CharField(max_length=254)
+    region_code = models.CharField(max_length=4)
+    geom = models.MultiPolygonField()
+
+    # Returns the string representation of the model.
+    def __str__(self):              # __unicode__ on Python 2
+        return self.name
+
+from django.contrib.gis.db import models
+
+## this model will be deprecated in favor of ManagementArea and ManagementAreaGroup
 class ManagedArea(models.Model):
     """this model functions similar to the Region model. it should be moved to a new file
     or something... preferably Region and MananagedArea would be just put in models.py
     to improve the clarity of import statements, etc."""
-    
+
     AGENCY_CHOICES = (
         ("FL Dept. of Environmental Protection, Div. of Recreation and Parks","FL Dept. of Environmental Protection, Div. of Recreation and Parks"),
         ("FL Dept. of Agriculture and Consumer Services, Florida Forest Service","FL Dept. of Agriculture and Consumer Services, Florida Forest Service"),
         ("FL Fish and Wildlife Conservation Commission","FL Fish and Wildlife Conservation Commission"),
         ("FL Dept. of Environmental Protection, Florida Coastal Office","FL Dept. of Environmental Protection, Florida Coastal Office"),
         ("FL Dept. of Environmental Protection, Office of Water Policy","FL Dept. of Environmental Protection, Office of Water Policy"),
     )
-        
+
     CATEGORY_CHOICES = (
         ("State Park","State Park"),
         ("State Forest","State Forest"),
         ("Fish and Wildlife Conservation Commission","Fish and Wildlife Conservation Commission"),
         ("Aquatic Preserve","Aquatic Preserve"),
         ("Water Management District","Water Management District"),
     )
-    
+
     WMD_DISTRICT_CHOICES = (
         ("North","North"),
         ("North Central","North Central"),
@@ -31,6 +46,7 @@ class ManagedArea(models.Model):
     )
 
     name = models.CharField(max_length=254)
+
     agency = models.CharField(max_length=254,choices=AGENCY_CHOICES)
     category = models.CharField(max_length=254,choices=CATEGORY_CHOICES)
     nickname = models.CharField(max_length=30,null=True,blank=True)