Skip to content

Commit

Permalink
merge release/2.14
Browse files Browse the repository at this point in the history
  • Loading branch information
CI committed Apr 27, 2020
2 parents 40458f8 + bbeb2cb commit d30fddd
Show file tree
Hide file tree
Showing 21 changed files with 331 additions and 42 deletions.
7 changes: 7 additions & 0 deletions CHANGES
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
2.14
----
* Add geonameid to locations endpoint
* Add pd_url to pd indicator endpoint
* Set budget values in process_country and not through mapping
* Fix InterventionBudget loader queryset

2.13
----
* Add new office endpoint
Expand Down
6 changes: 3 additions & 3 deletions docker/Dockerfile.alpine.base
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ ENV TEST ${TEST}


RUN apk add --no-cache --virtual .build-deps \
--repository http://dl-cdn.alpinelinux.org/alpine/edge/main \
--repository http://dl-cdn.alpinelinux.org/alpine/edge/testing \
--repository http://dl-3.alpinelinux.org/alpine/edge/community/ \
--repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/main \
--repository http://dl-cdn.alpinelinux.org/alpine/latest-stable/testing \
--repository http://dl-3.alpinelinux.org/alpine/latest-stable/community/ \
gcc \
g++ \
gdal-dev \
Expand Down
4 changes: 2 additions & 2 deletions docker/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?=
DEVELOP?=1
DOCKER_PASS?=
DOCKER_USER?=
TARGET?=2.13.1a
TARGET?=2.14.1a
PUSH_BASE?=1
BASE?=$(shell echo "${TARGET}" | sed "s/\([0-9]*\)\.\([0-9]*\)\.\(.*\)/\1.\2/g" )
#BASE?=$(shell echo "${TARGET}" | sed "s/\([0-9]*\)\.\([0-9]*\)\.\(.*\)/\1.\2/g" | echo "`xargs`xx" )
Expand Down Expand Up @@ -55,7 +55,7 @@ build-test:
TARGET=dev $(MAKE) .build test

build:
$(MAKE) .build test
$(MAKE) .build


.run:
Expand Down
2 changes: 1 addition & 1 deletion src/etools_datamart/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import warnings

NAME = 'etools-datamart'
VERSION = __version__ = '2.13'
VERSION = __version__ = '2.14'
__author__ = ''

# UserWarning: The psycopg2 wheel package will be renamed from release 2.11;
Expand Down
7 changes: 7 additions & 0 deletions src/etools_datamart/api/endpoints/datamart/location.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,18 @@ class Meta:


class LocationSerializer(serializers.ModelSerializer):
geonameid = serializers.SerializerMethodField()

class Meta:
model = models.Location
exclude = ('schema_name', 'tree_id', 'lft', 'rght', 'level', 'source_id',
'geom', 'point', 'latitude', 'longitude')

def get_geonameid(self, obj):
if obj.geoname:
return obj.geoname.geoname_id
return None


class LocationViewSet(common.DataMartViewSet):
serializer_class = LocationSerializer
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Generated by Django 2.2.9 on 2020-04-20 17:55

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('data', '0113_auto_20200408_1747'),
]

operations = [
migrations.AlterField(
model_name='section',
name='name',
field=models.CharField(blank=True, max_length=128, null=True),
),
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Generated by Django 2.2.9 on 2020-04-21 13:39

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('data', '0114_auto_20200420_1755'),
]

operations = [
migrations.AddField(
model_name='pdindicator',
name='pd_url',
field=models.CharField(blank=True, max_length=254, null=True),
),
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Generated by Django 2.2.11 on 2020-04-17 17:34

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('data', '0115_pdindicator_pd_url'),
]

operations = [
migrations.CreateModel(
name='GeoName',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('toponym_name', models.CharField(max_length=254, null=True)),
('name', models.CharField(max_length=254, null=True)),
('lat', models.FloatField()),
('lng', models.FloatField()),
('geoname_id', models.CharField(max_length=50, null=True)),
('country_code', models.CharField(max_length=20, null=True)),
('country_name', models.CharField(max_length=150, null=True)),
('fcl', models.CharField(max_length=50, null=True)),
('fcode', models.CharField(max_length=50, null=True)),
('distance', models.FloatField(null=True)),
],
options={
'unique_together': {('lat', 'lng')},
},
),
migrations.AddField(
model_name='location',
name='geoname',
field=models.ForeignKey(blank=True, null=True, on_delete=models.deletion.DO_NOTHING, to='data.GeoName'),
),
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Generated by Django 2.2.9 on 2020-04-21 15:44

import django.db.models.deletion
from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('data', '0116_auto_20200417_1734'),
]

operations = [
migrations.AlterField(
model_name='location',
name='geoname',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='data.GeoName'),
),
]
101 changes: 99 additions & 2 deletions src/etools_datamart/apps/mart/data/models/location.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
from xml.etree import ElementTree

from django.conf import settings
from django.contrib.gis.db import models as geomodels
from django.contrib.gis.db.models.functions import Centroid
from django.db import connection, models

import requests

from etools_datamart.apps.core.models import DataMartManager, DataMartQuerySet
from etools_datamart.apps.mart.data.models.base import EtoolsDataMartModel
from etools_datamart.apps.sources.etools.models import LocationsGatewaytype, LocationsLocation
Expand Down Expand Up @@ -39,12 +44,29 @@ def batch_update_centroid(self):
with connection.cursor() as cursor:
cursor.execute(sql)

# need to update geoname
for record in super().filter(
latitude__isnull=False,
longitude__isnull=False,
).all():
geoname = GeoName.objects.get_or_add(
lat=record.latitude,
lng=record.longitude,
)
if record.geoname != geoname:
record.geoname = geoname
record.save()

def update_centroid(self):
clone = self._chain()
for each in clone.annotate(cent=Centroid('geom')):
each.point = each.cent
each.latitude = each.point.y
each.longitude = each.point.x
each.geoname = GeoName.objects.get_or_add(
lat=each.point.y,
lng=each.point.x,
)
each.save()


Expand All @@ -69,6 +91,7 @@ class Location(EtoolsDataMartModel):
point = geomodels.PointField(blank=True, null=True)
gateway = models.ForeignKey(GatewayType, models.DO_NOTHING, blank=True, null=True)
geom = geomodels.MultiPolygonField(blank=True, null=True)
geoname = models.ForeignKey("GeoName", models.DO_NOTHING, blank=True, null=True)
level = models.IntegerField(db_index=True)
lft = models.IntegerField()
parent = models.ForeignKey('self', models.DO_NOTHING, blank=True, null=True)
Expand All @@ -90,13 +113,87 @@ class Options:
source = LocationsLocation
queryset = lambda: LocationsLocation.objects.order_by('-parent')
last_modify_field = 'modified'
exclude_from_compare = ['latitude', 'longitude', 'point']
exclude_from_compare = ['latitude', 'longitude', 'point', 'geoname']
# sync_deleted_records = False
mapping = {'source_id': 'id',
# 'area_code': lambda loader, record: loader.context['country'].business_area_code,
'parent': '__self__',
'gateway': GatewayType
'gateway': GatewayType,
}

def __str__(self):
return self.name


# store geo names pulled from http://api.geonames.org
# Using lat and lng get geoname data with
# http://api.geonames.org/findNearby?lat=47.3&lng=9&username=ntrncic
class GeoNameManager(models.Manager):
def get_or_add(self, lat, lng):
# check if we have a matching lat, lng record
# if so, return that record
# otherwise create a new record based on results
# of request to geonames.org
if not lat or not lng:
return None
try:
geoname = self.get_queryset().get(lat=lat, lng=lng)
except GeoName.DoesNotExist:
payload = {
"lat": lat,
"lng": lng,
"username": settings.GEONAMES_USERNAME,
}
res = requests.get(
settings.GEONAMES_URL,
params=payload,
timeout=settings.REQUEST_TIMEOUT,
)
geoname = ElementTree.fromstring(res.content)[0]
mapping = [
("toponym_name", "toponymName"),
("name", "name"),
("lat", "lat"),
("lng", "lng"),
("geoname_id", "geonameId"),
("country_code", "countryCode"),
("country_name", "countryName"),
("fcl", "fcl"),
("fcode", "fcode"),
("distance", "distance"),
]
data = {}
for k, f in mapping:
try:
data[k] = geoname.find(f).text
except AttributeError:
return None
lat = data.pop("lat")
lng = data.pop("lng")
geoname, __ = GeoName.objects.get_or_create(
lat=lat,
lng=lng,
defaults=data,
)
return geoname


class GeoName(models.Model):
toponym_name = models.CharField(max_length=254, null=True)
name = models.CharField(max_length=254, null=True)
lat = models.FloatField()
lng = models.FloatField()
geoname_id = models.CharField(max_length=50, null=True)
country_code = models.CharField(max_length=20, null=True)
country_name = models.CharField(max_length=150, null=True)
fcl = models.CharField(max_length=50, null=True)
fcode = models.CharField(max_length=50, null=True)
distance = models.FloatField(null=True)

objects = GeoNameManager()

class Meta:
unique_together = ('lat', 'lng')

def __str__(self):
return f"{self.name} ({self.lat}, {self.lng})"
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,24 @@
from etools_datamart.apps.mart.data.models import Location
from etools_datamart.apps.mart.data.models.base import EtoolsDataMartModel
from etools_datamart.apps.mart.data.models.intervention import InterventionAbstract, InterventionLoader
from etools_datamart.apps.mart.data.models.mixins import extend
from etools_datamart.apps.sources.etools.models import (FundsFundsreservationheader, models,
PartnersIntervention, PartnersInterventionbudget,)


class InterventionBudgetLoader(InterventionLoader):
def get_queryset(self):
return PartnersInterventionbudget.objects

def process_country(self):
qs = PartnersInterventionbudget.objects.all()
for record in qs.all():
record.intervention.budget = record
for record in self.get_queryset().all():
filters = self.config.key(self, record)
values = self.get_values(record.intervention)
values['source_id'] = record.id
values['budget_cso_contribution'] = record.partner_contribution_local
values['budget_unicef_cash'] = record.unicef_cash_local
values['budget_total'] = record.total_local
values['budget_currency'] = record.currency
values['budget_unicef_supply'] = record.in_kind_amount_local
op = self.process_record(filters, values)
self.increment_counter(op)

Expand Down Expand Up @@ -57,13 +62,4 @@ class Options(InterventionAbstract.Options):
model = PartnersInterventionbudget
depends = (Location,)
key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name,
source_id=record.intervention.budget.pk)

mapping = extend(InterventionAbstract.Options.mapping,
dict(
budget_cso_contribution='budget.partner_contribution_local',
budget_unicef_cash='budget.unicef_cash_local',
budget_total='budget.total_local',
budget_currency='budget.currency',
budget_unicef_supply='budget.in_kind_amount_local',
))
source_id=record.pk)
10 changes: 10 additions & 0 deletions src/etools_datamart/apps/mart/data/models/pd_indicator.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from django.shortcuts import reverse

from etools_datamart.apps.mart.data.fields import SafeDecimal
from etools_datamart.apps.mart.data.loader import EtoolsLoader
from etools_datamart.apps.mart.data.models import Location
Expand Down Expand Up @@ -30,6 +32,12 @@ def process_country(self):
op = self.process_record(filters, values)
self.increment_counter(op)

def get_pd_url(self, record: ReportsAppliedindicator, values: dict, **kwargs):
return reverse(
"api:intervention-detail",
args=["latest", record.lower_result.result_link.intervention.pk],
)


class PDIndicator(LocationMixin, EtoolsDataMartModel):
context_code = models.CharField(max_length=50, blank=True, null=True)
Expand Down Expand Up @@ -73,6 +81,7 @@ class PDIndicator(LocationMixin, EtoolsDataMartModel):
# from lower_result
lower_result_name = models.CharField(max_length=500, blank=True, null=True)
result_link_intervention = models.IntegerField(blank=True, null=True)
pd_url = models.CharField(max_length=254, blank=True, null=True)

# from section
section_name = models.CharField(max_length=45, blank=True, null=True)
Expand Down Expand Up @@ -126,6 +135,7 @@ class Options:
section_name='section.name',
lower_result_name='lower_result.name',
result_link_intervention='lower_result.result_link.intervention.pk',
pd_url='-',

disaggregation_name='disaggregation.name',
disaggregation_active='disaggregation.active',
Expand Down
2 changes: 1 addition & 1 deletion src/etools_datamart/apps/mart/data/models/report_sector.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@


class Section(EtoolsDataMartModel):
name = models.CharField(max_length=45, blank=True, null=True)
name = models.CharField(max_length=128, blank=True, null=True)
description = models.CharField(max_length=256, blank=True, null=True)
alternate_id = models.IntegerField(blank=True, null=True)
alternate_name = models.CharField(max_length=255, blank=True, null=True)
Expand Down
Loading

0 comments on commit d30fddd

Please sign in to comment.