From c144cf9ef7fb2004679811b98e07c882ff980f35 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 18 Sep 2016 14:21:35 +0200 Subject: [PATCH 1/6] Implement new search semantics --- index_service/serializers.py | 22 +++++++ index_service/views.py | 103 ++++++++++++++++++++++++++----- tests/api.py | 113 ++++++++++++++++++++++++++++------- 3 files changed, 202 insertions(+), 36 deletions(-) diff --git a/index_service/serializers.py b/index_service/serializers.py index 1332b65..e507967 100644 --- a/index_service/serializers.py +++ b/index_service/serializers.py @@ -55,6 +55,28 @@ def validate_public_key(self, value): return value +class FieldSerializer(serializers.Serializer): + field = serializers.ChoiceField(Entry.FIELDS) + value = serializers.CharField() + + def create(self, validated_data): + return validated_data + + +class SearchResultSerializer(IdentitySerializer): + matches = FieldSerializer(many=True) + + class Meta(IdentitySerializer.Meta): + fields = IdentitySerializer.Meta.fields + ('matches',) + + +class SearchSerializer(serializers.Serializer): + query = FieldSerializer(many=True, required=True) + + def create(self, validated_data): + return validated_data + + class UpdateItemSerializer(serializers.Serializer): action = serializers.ChoiceField(('create', 'delete')) field = serializers.ChoiceField(Entry.FIELDS) diff --git a/index_service/views.py b/index_service/views.py index 7e4b5f3..cd7f04a 100644 --- a/index_service/views.py +++ b/index_service/views.py @@ -1,6 +1,7 @@ from django.conf import settings from django.db import transaction +from django.db.models import Q from django.utils.decorators import method_decorator from rest_framework.decorators import api_view from rest_framework.parsers import JSONParser @@ -10,7 +11,7 @@ from .crypto import NoiseBoxParser, KeyPair, encode_key from .models import Identity, Entry, PendingUpdateRequest, PendingVerification -from .serializers import IdentitySerializer, UpdateRequestSerializer, scrub_field +from .serializers import SearchSerializer, UpdateRequestSerializer, SearchResultSerializer, scrub_field from .verification import execute_if_complete from .utils import authorized_api @@ -24,9 +25,12 @@ Public keys are represented by their hexadecimal string encoding, since JSON cannot transport binary data. """ +class RaisableResponse(Response, RuntimeError): + pass + def error(description): - return Response({'error': description}, 400) + return RaisableResponse({'error': description}, 400) @api_view(('GET',)) @@ -61,23 +65,94 @@ def key(request, format=None): }) -@api_view(('GET',)) -@authorized_api -def search(request, format=None): +@method_decorator(authorized_api, 'dispatch') +class SearchView(APIView): """ Search for identities registered for private data. """ - data = request.query_params - identities = Identity.objects - if not data or set(data.keys()) > Entry.FIELDS: - return error('No or unknown fields specified: ' + ', '.join(data.keys())) - for field, value in data.items(): + + parser_classes = (JSONParser,) + + def parse_value(self, field, value): try: - value = scrub_field(field, value) + return scrub_field(field, value) except ValueError as exc: - return error('Failed to parse field %r: %s' % (field, exc)) - identities = identities.filter(entry__field=field, entry__value=value) - return Response({'identities': IdentitySerializer(identities, many=True).data}) + raise error('Failed to parse field %r: %s' % (field, exc)) from exc + + def parse_query_params(self, query_params): + """Return {field-name -> set-of-values}.""" + query_fields = query_params.keys() + if not query_params or not set(query_fields) <= Entry.FIELDS: # Note: (not <=) != (>=) for sets! + raise error('No or unknown fields specified: ' + ', '.join(query_fields)) + fields = {} + for field in query_fields: + for value in query_params.getlist(field): + value = self.parse_value(field, value) + fields.setdefault(field, set()).add(value) + return fields + + def parse_json(self, json): + serializer = SearchSerializer(data=json) + serializer.is_valid(True) + query = serializer.save()['query'] + if not query: + raise error('No fields specified.') + fields = {} + for field_value in query: + field, value = field_value['field'], field_value['value'] + if field not in Entry.FIELDS: + raise error('Unknown field %r specified.' % field) + value = self.parse_value(field, value) + fields.setdefault(field, set()).add(value) + return fields + + def get_identities(self, fields): + query = Q() + for field, values in fields.items(): + for value in values: + query |= Q(entry__field=field, entry__value=value) + identities = Identity.objects.filter(query).distinct() + return identities + + def mark_matching_fields(self, identity, fields): + """identity.matches = list(fields that matched this identity).""" + matches = [] + for field, search_values in fields.items(): + entries = identity.entry_set.filter(field=field) + for entry in entries: + identity_value = entry.value + if identity_value in search_values: + matches.append({ + 'field': field, + 'value': identity_value, + }) + # Makes query result reproducible, easier for tests and caching + matches.sort(key=lambda match: (match['field'], match['value'])) + identity.matches = matches + + def process_search(self, fields): + identities = self.get_identities(fields) + for identity in identities: + self.mark_matching_fields(identity, fields) + return Response({ + 'identities': SearchResultSerializer(identities, many=True).data + }) + + def get(self, request, format=None): + try: + fields = self.parse_query_params(request.query_params) + return self.process_search(fields) + except RaisableResponse as rr: + return rr + + def post(self, request, format=None): + try: + fields = self.parse_json(request.data) + return self.process_search(fields) + except RaisableResponse as rr: + return rr + +search = SearchView.as_view() @method_decorator(authorized_api, 'dispatch') diff --git a/tests/api.py b/tests/api.py index 1997c04..980ff6b 100644 --- a/tests/api.py +++ b/tests/api.py @@ -8,7 +8,7 @@ from rest_framework import status from index_service.crypto import decode_key -from index_service.models import Entry +from index_service.models import Entry, Identity from index_service.logic import UpdateRequest from index_service.utils import AccountingAuthorization @@ -32,32 +32,101 @@ def test_get_key(self, api_client): class SearchTest: path = '/api/v0/search/' - def test_get_identity(self, api_client, email_entry): - response = api_client.get(self.path, {'email': email_entry.value}) - assert response.status_code == status.HTTP_200_OK - result = response.data['identities'] - assert len(result) == 1 - assert result[0]['alias'] == 'qabel_user' - assert result[0]['drop_url'] == 'http://127.0.0.1:6000/qabel_user' - - def test_get_no_identity(self, api_client): - response = api_client.get(self.path, {'email': 'no_such_email@example.com'}) - assert response.status_code == status.HTTP_200_OK + @pytest.fixture(params=('get', 'post')) + def search_client(self, request, api_client): + def client(query): + if request.param == 'get': + return api_client.get(self.path, query) + else: + transformed_query = [] + for field, value in query.items(): + if isinstance(value, (list, tuple)): + for v in value: + transformed_query.append({'field': field, 'value': v}) + else: + transformed_query.append({'field': field, 'value': value}) + q = json.dumps({'query': transformed_query}) + return api_client.post(self.path, q, content_type='application/json') + return client + + def test_get_identity(self, search_client, email_entry): + response = search_client({'email': email_entry.value}) + assert response.status_code == status.HTTP_200_OK, response.json() + identities = response.data['identities'] + assert len(identities) == 1 + identity = identities[0] + assert identity['alias'] == 'qabel_user' + assert identity['drop_url'] == 'http://127.0.0.1:6000/qabel_user' + matches = identity['matches'] + assert len(matches) == 1 + assert {'field': 'email', 'value': email_entry.value} in matches + + def test_get_no_identity(self, search_client): + response = search_client({'email': 'no_such_email@example.com'}) + assert response.status_code == status.HTTP_200_OK, response.json() assert len(response.data['identities']) == 0 - def test_no_full_match(self, api_client, email_entry): - response = api_client.get(self.path, {'email': email_entry.value, - 'phone': '123456789'}) - assert not response.data['identities'] - - def test_match_is_exact(self, api_client, email_entry): - response = api_client.get(self.path, {'email': email_entry.value + "a"}) + def test_multiple_fields_are_ORed(self, search_client, email_entry): + response = search_client({'email': email_entry.value, 'phone': '123456789'}) + assert response.status_code == status.HTTP_200_OK, response.json() + identities = response.data['identities'] + assert len(identities) == 1 + identity = identities[0] + assert identity['alias'] == 'qabel_user' + assert identity['drop_url'] == 'http://127.0.0.1:6000/qabel_user' + matches = identity['matches'] + assert len(matches) == 1 + assert {'field': 'email', 'value': email_entry.value} in matches + + def test_match_is_exact(self, search_client, email_entry): + response = search_client({'email': email_entry.value + "a"}) + assert response.status_code == status.HTTP_200_OK, response.json() assert not response.data['identities'] - response = api_client.get(self.path, {'email': "a" + email_entry.value}) + response = search_client({'email': "a" + email_entry.value}) + assert response.status_code == status.HTTP_200_OK, response.json() assert not response.data['identities'] - # XXX phone number tests - # XXX check that phone numbers are normalized (always have a cc/country code e.g. +49...) + def test_cross_identity(self, search_client, email_entry, identity): + identity2 = Identity(alias='1234', drop_url='http://127.0.0.1:6000/qabel_1234', public_key=identity.public_key) + identity2.save() + phone1, phone2 = '+491234', '+491235' + email = 'bar@example.net' + Entry(identity=identity2, field='phone', value=phone1).save() + Entry(identity=identity2, field='phone', value=phone2).save() + Entry(identity=identity2, field='email', value=email).save() + + response = search_client({ + 'email': (email_entry.value, email), + 'phone': phone1, + }) + assert response.status_code == status.HTTP_200_OK, response.json() + identities = response.data['identities'] + assert len(identities) == 2 + + expected1 = { + 'alias': '1234', + 'drop_url': 'http://127.0.0.1:6000/qabel_1234', + 'public_key': identity.public_key, + 'matches': [ + {'field': 'email', 'value': email}, + {'field': 'phone', 'value': phone1}, + ] + } + assert expected1 in identities + + def test_unknown_field(self, search_client): + response = search_client({'no such field': '...'}) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json() + + def test_missing_query(self, api_client): + response = api_client.post(self.path, '{}', content_type='application/json') + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json() + + def test_empty_query(self, search_client): + response = search_client({}) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json() + # "No or unknown field spec'd" or "No fields spec'd" + assert 'fields specified' in response.json()['error'] class UpdateTest: From da50702874eb26597705abc2685a622b2950056d Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 18 Sep 2016 14:21:38 +0200 Subject: [PATCH 2/6] Add plivo backend --- defaults.yaml | 13 ++++++++++ requirements.txt | 2 +- sendsms_backends/plivo.py | 54 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 sendsms_backends/plivo.py diff --git a/defaults.yaml b/defaults.yaml index ee7c478..3a0d506 100644 --- a/defaults.yaml +++ b/defaults.yaml @@ -41,6 +41,16 @@ qabel: # # Notable available backends (for reference see https://github.com/stefanfoulis/django-sendsms/ ): # + # Plivo + # SENDSMS_BACKEND: 'sendsms_backends.plivo.SmsBackend' + # PLIVO_AUTH_ID: '...' + # PLIVO_AUTH_TOKEN: '...' + # + # Plivo can optionally send message sending reports to an API. To do that, set these: + # PLIVO_REPORT_URL: 'https://...' + # PLIVO_REPORT_METHOD: 'GET' or 'POST' or ... + # See https://www.plivo.com/docs/getting-started/sms-delivery-reports/ for details. + # # Twilio # SENDSMS_BACKEND = 'sendsms.backends.twiliorest.SmsBackend' # SENDSMS_TWILIO_ACCOUNT_SID = '...' @@ -55,6 +65,9 @@ qabel: # When sending real SMS, this has to be the phone number you got from your SMS gateway. You probably paid money for it. # This setting doesn't matter for the dummy backends. + # Note that for successful delivery this is _required_. With an invalid value here delivery _will_ fail, + # without producing any application log (because the SMS gateways have internal queues, a successful request + # does not imply successful delivery). These failures will only show up through reporting hooks and dashboards. SENDSMS_DEFAULT_FROM_PHONE: '+15005550006' # Configure countries allowed for phone number registration. Check the applicable charges before diff --git a/requirements.txt b/requirements.txt index da3482e..2a855b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ django-rest-auth==0.7.0 djangorestframework==3.3.3 django-mail-templated==2.6.2 django-sendsms==0.2.3 -twilio==5.4.0 +plivo==0.11.1 flake8==2.5.4 psycopg2==2.6.1 py==1.4.31 diff --git a/sendsms_backends/plivo.py b/sendsms_backends/plivo.py new file mode 100644 index 0000000..0dbe293 --- /dev/null +++ b/sendsms_backends/plivo.py @@ -0,0 +1,54 @@ +""" +http://pypi.python.org/pypi/plivo/ +""" +import logging + +from plivo import RestAPI + +from django.conf import settings + +from sendsms.backends.base import BaseSmsBackend + +PLIVO_AUTH_ID = getattr(settings, 'PLIVO_AUTH_ID', '') +PLIVO_AUTH_TOKEN = getattr(settings, 'PLIVO_AUTH_TOKEN', '') + +PLIVO_REPORT_URL = getattr(settings, 'PLIVO_REPORT_URL', '') +PLIVO_REPORT_METHOD = getattr(settings, 'PLIVO_REPORT_METHOD', 'GET') + +logger = logging.getLogger(__name__) + +report_params = {} +if PLIVO_REPORT_URL: + report_params.update({ + 'url': PLIVO_REPORT_URL, + 'method': PLIVO_REPORT_METHOD, + }) + logger.info('Picked up report URL: %s, HTTP verb: %s', PLIVO_REPORT_URL, PLIVO_REPORT_METHOD) + + +class SmsBackend(BaseSmsBackend): + def send_messages(self, messages): + api = RestAPI(PLIVO_AUTH_ID, PLIVO_AUTH_TOKEN) + for message in messages: + params = { + 'src': message.from_phone, + 'text': message.body, + } + params.update(report_params) + + for to in message.to: + try: + params.update({ + 'dst': to, + }) + logger.debug('Sending message from %s to %s (%d bytes body)', + message.from_phone, to, + len(message.body.encode())) + status_code, response = api.send_message(params) + ok = status_code in (200, 202, 204) + log = logger.info if ok else logger.error + log('Message status: %d\nResponse is: %s', status_code, response) + except Exception: + logger.exception('Exception while sending message (fail_silently=%s)', self.fail_silently) + if not self.fail_silently: + raise From 57def564ca3e03a8c25e65c07b851bd46d4c71ae Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 18 Sep 2016 14:21:39 +0200 Subject: [PATCH 3/6] test_phone_item_blatantly_invalid --- tests/serializers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/serializers.py b/tests/serializers.py index 21bf4b8..c902b48 100644 --- a/tests/serializers.py +++ b/tests/serializers.py @@ -113,6 +113,13 @@ def test_phone_item_international_request_not_allowed(): assert 'is not available at this time' in str(serializer.errors) +def test_phone_item_blatantly_invalid(): + serializer = UpdateItemSerializer(data=make_update_item('phone', 'abcdef')) + assert not serializer.is_valid(), serializer.errors + # It's a bit unfortunate that this currently is a non_field_error, something for later. + assert 'did not seem to be a phone number' in serializer.errors['non_field_errors'][0] + + @pytest.mark.parametrize('invalid', [ {}, { @@ -170,7 +177,7 @@ def test_similar_items(simple_identity): def test_identity_deserialize_multiple(simple_identity): def deserialize_and_save(data): - idser = IdentitySerializer(data=simple_identity) + idser = IdentitySerializer(data=data) idser.is_valid(True) return idser.save() identity1 = deserialize_and_save(simple_identity) From 024a6967cd3ef6bd252e242d760c4860f6d95832 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 18 Sep 2016 14:21:39 +0200 Subject: [PATCH 4/6] Add creation timestamp to Identity, Entry, PendingUpdateRequest --- .../0002_add_creation_timestamps.py | 32 +++++++++++++++++++ index_service/models.py | 13 ++++++-- tests/models.py | 12 +++++++ 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 index_service/migrations/0002_add_creation_timestamps.py diff --git a/index_service/migrations/0002_add_creation_timestamps.py b/index_service/migrations/0002_add_creation_timestamps.py new file mode 100644 index 0000000..f47f7b5 --- /dev/null +++ b/index_service/migrations/0002_add_creation_timestamps.py @@ -0,0 +1,32 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +from django.utils import timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('index_service', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='entry', + name='created', + field=models.DateTimeField(auto_now_add=True, default=timezone.now()), + preserve_default=False, + ), + migrations.AddField( + model_name='identity', + name='created', + field=models.DateTimeField(auto_now_add=True, default=timezone.now()), + preserve_default=False, + ), + migrations.AddField( + model_name='pendingupdaterequest', + name='created', + field=models.DateTimeField(auto_now_add=True, default=timezone.now()), + preserve_default=False, + ), + ] diff --git a/index_service/models.py b/index_service/models.py index a2b48c4..7090a59 100644 --- a/index_service/models.py +++ b/index_service/models.py @@ -10,7 +10,14 @@ from .utils import short_id -class Identity(ExportModelOperationsMixin('Identity'), models.Model): +class CreationTimestampModel(models.Model): + created = models.DateTimeField(auto_now_add=True) + + class Meta: + abstract = True + + +class Identity(ExportModelOperationsMixin('Identity'), CreationTimestampModel): """ An identity, composed of the public key, drop URL and alias. @@ -35,7 +42,7 @@ def __repr__(self): return u'alias: {0} public_key: {1}'.format(self.alias, repr(self.public_key)) -class Entry(ExportModelOperationsMixin('Entry'), models.Model): +class Entry(ExportModelOperationsMixin('Entry'), CreationTimestampModel): """ An Entry connects a piece of private data (email, phone, ...) to an identity. @@ -59,7 +66,7 @@ class Meta: index_together = ('field', 'value') -class PendingUpdateRequest(ExportModelOperationsMixin('PendingUpdateRequest'), models.Model): +class PendingUpdateRequest(ExportModelOperationsMixin('PendingUpdateRequest'), CreationTimestampModel): """ Pending update request: When additional user-asynchronous authorization is required a request has to be stored in the database (and all verifications have to complete) before it can be executed. diff --git a/tests/models.py b/tests/models.py index 039bb3c..5fe4c1d 100644 --- a/tests/models.py +++ b/tests/models.py @@ -37,3 +37,15 @@ def test_pending_verification_duplicate_id(db, dumb_request): verification1 = factory('1234') verification2 = factory('1234') assert verification1.id != verification2.id + + +def test_identity_creation_timestamp(identity): + assert (identity.created - timezone.now()) < datetime.timedelta(seconds=1) + + +def test_entry_creation_timestamp(email_entry): + assert (email_entry.created - timezone.now()) < datetime.timedelta(seconds=1) + + +def test_request_creation_timestamp(dumb_request): + assert (dumb_request.created - timezone.now()) < datetime.timedelta(seconds=1) From 30febb21aa3ad7c75e46bad32564bb398fbe78b8 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 18 Sep 2016 14:21:39 +0200 Subject: [PATCH 5/6] Add admin interface --- index_service/__init__.py | 2 ++ index_service/admin.py | 21 ++++++++++++++++++++- index_service/apps.py | 7 +++++++ index_service/models.py | 9 ++++++++- 4 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 index_service/apps.py diff --git a/index_service/__init__.py b/index_service/__init__.py index fe9e58a..37b105d 100644 --- a/index_service/__init__.py +++ b/index_service/__init__.py @@ -1,3 +1,5 @@ # Import in __init__ to register checks from . import crypto + +default_app_config = 'index_service.apps.IndexConfig' diff --git a/index_service/admin.py b/index_service/admin.py index 8c38f3f..35f9f5b 100644 --- a/index_service/admin.py +++ b/index_service/admin.py @@ -1,3 +1,22 @@ from django.contrib import admin -# Register your models here. +from . import models + + +@admin.register(models.Identity) +class IdentityAdmin(admin.ModelAdmin): + pass + + +@admin.register(models.Entry) +class EntryAdmin(admin.ModelAdmin): + pass + + +class PendingVerificationInline(admin.TabularInline): + model = models.PendingVerification + + +@admin.register(models.PendingUpdateRequest) +class PendingUpdateRequestAdmin(admin.ModelAdmin): + inlines = (PendingVerificationInline,) diff --git a/index_service/apps.py b/index_service/apps.py new file mode 100644 index 0000000..9148be3 --- /dev/null +++ b/index_service/apps.py @@ -0,0 +1,7 @@ + +from django.apps import AppConfig + + +class IndexConfig(AppConfig): + name = 'index_service' + verbose_name = 'Index service' diff --git a/index_service/models.py b/index_service/models.py index 7090a59..4a07ea2 100644 --- a/index_service/models.py +++ b/index_service/models.py @@ -32,6 +32,7 @@ class Meta: # Index over the whole triplet; we'll access this way when processing update requests. index_together = ('public_key', 'alias', 'drop_url') unique_together = index_together + verbose_name_plural = 'Identities' def delete_if_garbage(self): """Clean up this identity if there are no entries referring to it.""" @@ -39,7 +40,9 @@ def delete_if_garbage(self): self.delete() def __repr__(self): - return u'alias: {0} public_key: {1}'.format(self.alias, repr(self.public_key)) + return 'alias: {} public_key: {}'.format(self.alias, repr(self.public_key)) + + __str__ = __repr__ class Entry(ExportModelOperationsMixin('Entry'), CreationTimestampModel): @@ -61,9 +64,13 @@ class Entry(ExportModelOperationsMixin('Entry'), CreationTimestampModel): value = models.CharField(max_length=200) identity = models.ForeignKey(Identity) + def __str__(self): + return '{}: {}'.format(self.field, self.value) + class Meta: # Note that there is no uniqueness of anything index_together = ('field', 'value') + verbose_name_plural = 'Entries' class PendingUpdateRequest(ExportModelOperationsMixin('PendingUpdateRequest'), CreationTimestampModel): From 9d846ee2e5c3829c2612d8b38b3661455bd4cd7d Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 18 Sep 2016 14:21:40 +0200 Subject: [PATCH 6/6] Add tests for system check --- tests/crypto.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/crypto.py diff --git a/tests/crypto.py b/tests/crypto.py new file mode 100644 index 0000000..3299395 --- /dev/null +++ b/tests/crypto.py @@ -0,0 +1,25 @@ + +from django.core.checks import Error + +import pytest + +from index_service.crypto import check_server_private_key + + +@pytest.mark.parametrize('private_key', ( + '123', b'1234', bytes(31), bytes(31).hex() +)) +def test_check_server_private_key_invalid(settings, private_key): + settings.SERVER_PRIVATE_KEY = private_key + errors = check_server_private_key(None) + assert errors + assert isinstance(errors[0], Error) + + +@pytest.mark.parametrize('private_key', ( + None, bytes(32), bytes(32).hex() +)) +def test_check_server_private_key(settings, private_key): + settings.SERVER_PRIVATE_KEY = private_key + errors = check_server_private_key(None) + assert not errors