Merge pull request #32 from Qabel/m/phn2

New search semantics, Plivo backend

defaults.yaml

@@ -41,6 +41,16 @@ qabel:
         #
         # Notable available backends (for reference see https://github.com/stefanfoulis/django-sendsms/ ):
         #
+        # Plivo
+        # SENDSMS_BACKEND: 'sendsms_backends.plivo.SmsBackend'
+        # PLIVO_AUTH_ID: '...'
+        # PLIVO_AUTH_TOKEN: '...'
+        #
+        # Plivo can optionally send message sending reports to an API. To do that, set these:
+        # PLIVO_REPORT_URL: 'https://...'
+        # PLIVO_REPORT_METHOD: 'GET' or 'POST' or ...
+        # See https://www.plivo.com/docs/getting-started/sms-delivery-reports/ for details.
+        #
         # Twilio
         # SENDSMS_BACKEND = 'sendsms.backends.twiliorest.SmsBackend'
         # SENDSMS_TWILIO_ACCOUNT_SID = '...'
@@ -55,6 +65,9 @@ qabel:
 
         # When sending real SMS, this has to be the phone number you got from your SMS gateway. You probably paid money for it.
         # This setting doesn't matter for the dummy backends.
+        # Note that for successful delivery this is _required_. With an invalid value here delivery _will_ fail,
+        # without producing any application log (because the SMS gateways have internal queues, a successful request
+        # does not imply successful delivery). These failures will only show up through reporting hooks and dashboards.
         SENDSMS_DEFAULT_FROM_PHONE: '+15005550006'
 
         # Configure countries allowed for phone number registration. Check the applicable charges before

index_service/__init__.py

@@ -1,3 +1,5 @@
 
 # Import in __init__ to register checks
 from . import crypto
+
+default_app_config = 'index_service.apps.IndexConfig'

index_service/admin.py

@@ -1,3 +1,22 @@
 from django.contrib import admin
 
-# Register your models here.
+from . import models
+
+
+@admin.register(models.Identity)
+class IdentityAdmin(admin.ModelAdmin):
+    pass
+
+
+@admin.register(models.Entry)
+class EntryAdmin(admin.ModelAdmin):
+    pass
+
+
+class PendingVerificationInline(admin.TabularInline):
+    model = models.PendingVerification
+
+
+@admin.register(models.PendingUpdateRequest)
+class PendingUpdateRequestAdmin(admin.ModelAdmin):
+    inlines = (PendingVerificationInline,)

index_service/apps.py

@@ -0,0 +1,7 @@
+
+from django.apps import AppConfig
+
+
+class IndexConfig(AppConfig):
+    name = 'index_service'
+    verbose_name = 'Index service'

index_service/migrations/0002_add_creation_timestamps.py

@@ -0,0 +1,32 @@
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+from django.utils import timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('index_service', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='entry',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, default=timezone.now()),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='identity',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, default=timezone.now()),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='pendingupdaterequest',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, default=timezone.now()),
+            preserve_default=False,
+        ),
+    ]

index_service/models.py

@@ -10,7 +10,14 @@
 from .utils import short_id
 
 
-class Identity(ExportModelOperationsMixin('Identity'), models.Model):
+class CreationTimestampModel(models.Model):
+    created = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        abstract = True
+
+
+class Identity(ExportModelOperationsMixin('Identity'), CreationTimestampModel):
     """
     An identity, composed of the public key, drop URL and alias.
 
@@ -25,17 +32,20 @@ class Meta:
         # Index over the whole triplet; we'll access this way when processing update requests.
         index_together = ('public_key', 'alias', 'drop_url')
         unique_together = index_together
+        verbose_name_plural = 'Identities'
 
     def delete_if_garbage(self):
         """Clean up this identity if there are no entries referring to it."""
         if not self.entry_set.count():
             self.delete()
 
     def __repr__(self):
-        return u'alias: {0} public_key: {1}'.format(self.alias, repr(self.public_key))
+        return 'alias: {} public_key: {}'.format(self.alias, repr(self.public_key))
+
+    __str__ = __repr__
 
 
-class Entry(ExportModelOperationsMixin('Entry'), models.Model):
+class Entry(ExportModelOperationsMixin('Entry'), CreationTimestampModel):
     """
     An Entry connects a piece of private data (email, phone, ...) to an identity.
 
@@ -54,12 +64,16 @@ class Entry(ExportModelOperationsMixin('Entry'), models.Model):
     value = models.CharField(max_length=200)
     identity = models.ForeignKey(Identity)
 
+    def __str__(self):
+        return '{}: {}'.format(self.field, self.value)
+
     class Meta:
         # Note that there is no uniqueness of anything
         index_together = ('field', 'value')
+        verbose_name_plural = 'Entries'
 
 
-class PendingUpdateRequest(ExportModelOperationsMixin('PendingUpdateRequest'), models.Model):
+class PendingUpdateRequest(ExportModelOperationsMixin('PendingUpdateRequest'), CreationTimestampModel):
     """
     Pending update request: When additional user-asynchronous authorization is required a request has
     to be stored in the database (and all verifications have to complete) before it can be executed.

index_service/serializers.py

@@ -55,6 +55,28 @@ def validate_public_key(self, value):
         return value
 
 
+class FieldSerializer(serializers.Serializer):
+    field = serializers.ChoiceField(Entry.FIELDS)
+    value = serializers.CharField()
+
+    def create(self, validated_data):
+        return validated_data
+
+
+class SearchResultSerializer(IdentitySerializer):
+    matches = FieldSerializer(many=True)
+
+    class Meta(IdentitySerializer.Meta):
+        fields = IdentitySerializer.Meta.fields + ('matches',)
+
+
+class SearchSerializer(serializers.Serializer):
+    query = FieldSerializer(many=True, required=True)
+
+    def create(self, validated_data):
+        return validated_data
+
+
 class UpdateItemSerializer(serializers.Serializer):
     action = serializers.ChoiceField(('create', 'delete'))
     field = serializers.ChoiceField(Entry.FIELDS)

index_service/views.py

@@ -1,6 +1,7 @@
 
 from django.conf import settings
 from django.db import transaction
+from django.db.models import Q
 from django.utils.decorators import method_decorator
 from rest_framework.decorators import api_view
 from rest_framework.parsers import JSONParser
@@ -10,7 +11,7 @@
 
 from .crypto import NoiseBoxParser, KeyPair, encode_key
 from .models import Identity, Entry, PendingUpdateRequest, PendingVerification
-from .serializers import IdentitySerializer, UpdateRequestSerializer, scrub_field
+from .serializers import SearchSerializer, UpdateRequestSerializer, SearchResultSerializer, scrub_field
 from .verification import execute_if_complete
 from .utils import authorized_api
 
@@ -24,9 +25,12 @@
 Public keys are represented by their hexadecimal string encoding, since JSON cannot transport binary data.
 """
 
+class RaisableResponse(Response, RuntimeError):
+    pass
+
 
 def error(description):
-    return Response({'error': description}, 400)
+    return RaisableResponse({'error': description}, 400)
 
 
 @api_view(('GET',))
@@ -61,23 +65,94 @@ def key(request, format=None):
     })
 
 
-@api_view(('GET',))
-@authorized_api
-def search(request, format=None):
+@method_decorator(authorized_api, 'dispatch')
+class SearchView(APIView):
     """
     Search for identities registered for private data.
     """
-    data = request.query_params
-    identities = Identity.objects
-    if not data or set(data.keys()) > Entry.FIELDS:
-        return error('No or unknown fields specified: ' + ', '.join(data.keys()))
-    for field, value in data.items():
+
+    parser_classes = (JSONParser,)
+
+    def parse_value(self, field, value):
         try:
-            value = scrub_field(field, value)
+            return scrub_field(field, value)
         except ValueError as exc:
-            return error('Failed to parse field %r: %s' % (field, exc))
-        identities = identities.filter(entry__field=field, entry__value=value)
-    return Response({'identities': IdentitySerializer(identities, many=True).data})
+            raise error('Failed to parse field %r: %s' % (field, exc)) from exc
+
+    def parse_query_params(self, query_params):
+        """Return {field-name -> set-of-values}."""
+        query_fields = query_params.keys()
+        if not query_params or not set(query_fields) <= Entry.FIELDS:  # Note: (not <=) != (>=) for sets!
+            raise error('No or unknown fields specified: ' + ', '.join(query_fields))
+        fields = {}
+        for field in query_fields:
+            for value in query_params.getlist(field):
+                value = self.parse_value(field, value)
+                fields.setdefault(field, set()).add(value)
+        return fields
+
+    def parse_json(self, json):
+        serializer = SearchSerializer(data=json)
+        serializer.is_valid(True)
+        query = serializer.save()['query']
+        if not query:
+            raise error('No fields specified.')
+        fields = {}
+        for field_value in query:
+            field, value = field_value['field'], field_value['value']
+            if field not in Entry.FIELDS:
+                raise error('Unknown field %r specified.' % field)
+            value = self.parse_value(field, value)
+            fields.setdefault(field, set()).add(value)
+        return fields
+
+    def get_identities(self, fields):
+        query = Q()
+        for field, values in fields.items():
+            for value in values:
+                query |= Q(entry__field=field, entry__value=value)
+        identities = Identity.objects.filter(query).distinct()
+        return identities
+
+    def mark_matching_fields(self, identity, fields):
+        """identity.matches = list(fields that matched this identity)."""
+        matches = []
+        for field, search_values in fields.items():
+            entries = identity.entry_set.filter(field=field)
+            for entry in entries:
+                identity_value = entry.value
+                if identity_value in search_values:
+                    matches.append({
+                        'field': field,
+                        'value': identity_value,
+                    })
+        # Makes query result reproducible, easier for tests and caching
+        matches.sort(key=lambda match: (match['field'], match['value']))
+        identity.matches = matches
+
+    def process_search(self, fields):
+        identities = self.get_identities(fields)
+        for identity in identities:
+            self.mark_matching_fields(identity, fields)
+        return Response({
+            'identities': SearchResultSerializer(identities, many=True).data
+        })
+
+    def get(self, request, format=None):
+        try:
+            fields = self.parse_query_params(request.query_params)
+            return self.process_search(fields)
+        except RaisableResponse as rr:
+            return rr
+
+    def post(self, request, format=None):
+        try:
+            fields = self.parse_json(request.data)
+            return self.process_search(fields)
+        except RaisableResponse as rr:
+            return rr
+
+search = SearchView.as_view()
 
 
 @method_decorator(authorized_api, 'dispatch')

requirements.txt

@@ -4,7 +4,7 @@ django-rest-auth==0.7.0
 djangorestframework==3.3.3
 django-mail-templated==2.6.2
 django-sendsms==0.2.3
-twilio==5.4.0
+plivo==0.11.1
 flake8==2.5.4
 psycopg2==2.6.1
 py==1.4.31

sendsms_backends/plivo.py

@@ -0,0 +1,54 @@
+"""
+http://pypi.python.org/pypi/plivo/
+"""
+import logging
+
+from plivo import RestAPI
+
+from django.conf import settings
+
+from sendsms.backends.base import BaseSmsBackend
+
+PLIVO_AUTH_ID = getattr(settings, 'PLIVO_AUTH_ID', '')
+PLIVO_AUTH_TOKEN = getattr(settings, 'PLIVO_AUTH_TOKEN', '')
+
+PLIVO_REPORT_URL = getattr(settings, 'PLIVO_REPORT_URL', '')
+PLIVO_REPORT_METHOD = getattr(settings, 'PLIVO_REPORT_METHOD', 'GET')
+
+logger = logging.getLogger(__name__)
+
+report_params = {}
+if PLIVO_REPORT_URL:
+    report_params.update({
+        'url': PLIVO_REPORT_URL,
+        'method': PLIVO_REPORT_METHOD,
+    })
+    logger.info('Picked up report URL: %s, HTTP verb: %s', PLIVO_REPORT_URL, PLIVO_REPORT_METHOD)
+
+
+class SmsBackend(BaseSmsBackend):
+    def send_messages(self, messages):
+        api = RestAPI(PLIVO_AUTH_ID, PLIVO_AUTH_TOKEN)
+        for message in messages:
+            params = {
+                'src': message.from_phone,
+                'text': message.body,
+            }
+            params.update(report_params)
+
+            for to in message.to:
+                try:
+                    params.update({
+                        'dst': to,
+                    })
+                    logger.debug('Sending message from %s to %s (%d bytes body)',
+                                 message.from_phone, to,
+                                 len(message.body.encode()))
+                    status_code, response = api.send_message(params)
+                    ok = status_code in (200, 202, 204)
+                    log = logger.info if ok else logger.error
+                    log('Message status: %d\nResponse is: %s', status_code, response)
+                except Exception:
+                    logger.exception('Exception while sending message (fail_silently=%s)', self.fail_silently)
+                    if not self.fail_silently:
+                        raise