Merge pull request mozboz#8 from webisteme/master

Django integration

Makefile

@@ -0,0 +1,23 @@
+
+virtualenv:
+	[ -d .env ] || virtualenv .env 
+	.env/bin/pip install -r requirements.txt
+	echo "Now type \"source .env/bin/activate\" to active python env"
+
+
+installpostgres:
+	sudo apt-get install postgresql
+	sudo -u postgres psql -c "CREATE USER tfm LOGIN ENCRYPTED PASSWORD 'password';"
+	sudo -u postgres createdb tfm -O tfm
+
+resetdb:
+	sudo -u postgres dropdb tfm
+	sudo -u postgres createdb tfm -O tfm
+	make setupdb
+
+setupdb:
+	.env/bin/python manage.py migrate
+	.env/bin/python manage.py createsuperuser
+
+runserver:
+	.env/bin/python runserver_plus

README.md

@@ -1,12 +1,35 @@
-# transformapetl
-#
+# Transformap ETL
 
-1. Setup
+## Setup dev:
 
-pip install -r requirements.txt  
-cp transformap.example.conf transformap.conf [+ add database credentials]
+    make installpostgres
+    make virtualenv
+    make setupdb
+    source .env/bin/activate
+    make runserver
+
+Refer to Makefile for further details; it should serve as a guide to how to
+manage the local Python development environment.
+
+## Postgresql JSONB field support in Django
+
+The JSON field support appears to be quite good, django's queryset API supports
+several operations for querying data inside JSONB fields.
+
+See: https://docs.djangoproject.com/en/1.10/ref/contrib/postgres/fields/#containment-and-key-operations
+
+## Usage
+
+    python manage.py transformap -i jobs/jobfile.json
+
+Admin interface: /admin
+API URL: /places-api/
+New jobs should be created as .YAML files in jobs/
+
+## Misc
+
+ETL business logic: places/management/commands
+REST API configuration: places/api.py
 
-2. Usage
 
-python transformap.py -i jobs/charlotte.yaml
 

jobs/osm.yaml

@@ -27,5 +27,5 @@ transform:
       source_field_name: wheelchair
       target_field_name: wheelchair_accessible
 load:
-    definition: "Convergir map"
-    owner: "Convergir"
+    definition: "OSM map"
+    owner: "OSM"

manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tfm.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError:
+        # The above import may fail for some other reason. Ensure that the
+        # issue is really that Django is missing to avoid masking other
+        # exceptions on Python 2.
+        try:
+            import django
+        except ImportError:
+            raise ImportError(
+                "Couldn't import Django. Are you sure it's installed and "
+                "available on your PYTHONPATH environment variable? Did you "
+                "forget to activate a virtual environment?"
+            )
+        raise
+    execute_from_command_line(sys.argv)

places/admin.py

@@ -0,0 +1,51 @@
+from django.contrib import admin
+
+from places.models import MapOwner, MapInstance, MapObject, MapData, \
+    MapDefinition, SchemaField, SchemaVersion
+
+
+
+class MapObjectAdmin(admin.ModelAdmin):
+    list_display = ('id', 'map_instance', 'longitude', 'latitude', 'date_created')
+    list_display_links = ('id', )
+
+
+class MapInstanceAdmin(admin.ModelAdmin):
+    list_display = ('id', 'schema', 'map_definition', 'date_created')
+    list_display_links = ('id',)
+
+
+class MapOwnerAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name', 'date_created',)
+    list_display_links = ('id',)
+
+
+class MapDefinitionAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name', 'owner', 'date_created')
+    list_display_links = ('id',)
+
+
+class SchemaFieldAdmin(admin.ModelAdmin):
+    list_display = ('id', 'schema', 'field_name', 'field_type')
+    list_display_links = ('id',)
+
+
+class SchemaVersionAdmin(admin.ModelAdmin):
+    list_display = ('id', 'map_definition', 'map_owner', 'date_created')
+    list_display_links = ('id',)
+
+
+class MapDataAdmin(admin.ModelAdmin):
+    list_display = ('id', 'map_instance', 'map_object', 'schema_field')
+    list_display_links = ('id',)
+
+
+
+admin.site.register(MapObject, MapObjectAdmin)
+admin.site.register(MapInstance, MapInstanceAdmin)
+admin.site.register(MapOwner, MapOwnerAdmin)
+admin.site.register(MapDefinition, MapDefinitionAdmin)
+admin.site.register(SchemaField, SchemaFieldAdmin)
+admin.site.register(SchemaVersion, SchemaVersionAdmin)
+admin.site.register(MapData, MapDataAdmin)
+

places/api.py

@@ -0,0 +1,96 @@
+from places.models import MapOwner, MapInstance, MapObject, MapData, \
+    MapDefinition, SchemaField, SchemaVersion
+
+from rest_framework import filters, routers, serializers, viewsets
+
+
+# Map Owner
+
+class MapOwnerSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = MapOwner
+        fields = ('name', 'date_created', 'date_modified')
+
+class MapOwnerViewSet(viewsets.ModelViewSet):
+    queryset = MapOwner.objects.all()
+    serializer_class = MapOwnerSerializer
+
+# Map Instance
+
+class MapInstanceSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = MapInstance
+        fields = ('schema', 'map_definition', 'date_created', 'date_modified')
+
+class MapInstanceViewSet(viewsets.ModelViewSet):
+    queryset = MapInstance.objects.all()
+    serializer_class = MapInstanceSerializer
+
+# Map Object
+
+class MapObjectSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = MapObject
+        fields = ('guid', 'map_instance', 'longitude', 'latitude', 'date_created', 'date_modified')
+
+class MapObjectViewSet(viewsets.ModelViewSet):
+    queryset = MapObject.objects.all()
+    serializer_class = MapObjectSerializer
+
+# Map Data
+
+class MapDataSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = MapData
+        fields = ('map_instance', 'map_object', 'schema_field', 'field_value', 'date_created', 'date_modified')
+
+class MapDataViewSet(viewsets.ModelViewSet):
+    queryset = MapData.objects.all()
+    serializer_class = MapDataSerializer
+
+# Map Definition
+
+class MapDefinitionSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = MapDefinition
+        fields = ('name', 'owner', 'date_created', 'date_modified')
+
+class MapDefinitionViewSet(viewsets.ModelViewSet):
+    queryset = MapDefinition.objects.all()
+    serializer_class = MapDefinitionSerializer
+
+# Schema Field
+
+class SchemaFieldSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = SchemaField
+        fields = ('is_base_field', 'schema', 'field_name', 'field_type', 'field_description', 'date_created', 'date_modified')
+
+class SchemaFieldViewSet(viewsets.ModelViewSet):
+    queryset = SchemaField.objects.all()
+    serializer_class = SchemaFieldSerializer
+
+# Schema Version
+
+class SchemaVersionSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = SchemaVersion
+        fields = ('map_definition', 'map_owner', 'date_created', 'date_modified')
+
+class SchemaVersionViewSet(viewsets.ModelViewSet):
+    queryset = SchemaVersion.objects.all()
+    serializer_class = SchemaVersionSerializer
+
+
+
+
+router = routers.DefaultRouter()
+
+router.register('map_owners', MapOwnerViewSet)
+router.register('map_instance', MapInstanceViewSet)
+router.register('map_object', MapObjectViewSet)
+router.register('map_data', MapDataViewSet)
+router.register('map_definition', MapDefinitionViewSet)
+router.register('schema_field', SchemaFieldViewSet)
+router.register('schema_version', SchemaVersionViewSet)
+

places/apps.py

@@ -0,0 +1,7 @@
+from __future__ import unicode_literals
+
+from django.apps import AppConfig
+
+
+class PlacesConfig(AppConfig):
+    name = 'places'

transformap/extract.py → places/management/commands/_extract.py

@@ -2,9 +2,11 @@
 # coding: utf-8
 
 import urllib
-from jobs import Extract
 import time
 
+from _jobs import Extract
+
+
 class ExtractHttp(Extract):
 
     '''

transformap/jobs.py → places/management/commands/_jobs.py

@@ -1,8 +1,7 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-import orm
-import logging, logging.config
+import logging
 import json
 
 
@@ -11,19 +10,15 @@ class TMJob(object):
     ''' Base Job class '''
 
     def __init__(self, module_config):
-
-        self.orm = orm
-        self.session = orm.db_connect()
         self.config = module_config
 
     def setup_logging(self, name):
-
-        logging.config.fileConfig("logging.conf")
-        logger = logging.getLogger(name)
-        self.logger = logger
+        self.logger = logging.getLogger(name)
 
 
 class Extract(TMJob):
+
+    ''' Base Extract class '''
 
     def __init__(self, config):
         self.setup_logging('Extrac')
@@ -32,6 +27,8 @@ def __init__(self, config):
 
 class Transform(TMJob):
 
+    ''' Base Transform class '''
+
     def __init__(self, config):
         self.setup_logging('Transf')
         super(Transform, self).__init__(config)