Feature/status board

api/serializers.py

@@ -5,10 +5,14 @@
 from rest_framework_json_api import serializers
 
 from share import models
-from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult
 
-from api import fields
 
+from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, HarvestLog, SourceConfig
+
+
+from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, logs
+
+from api import fields
 
 class BaseShareSerializer(serializers.ModelSerializer):
 
@@ -132,12 +136,21 @@ class Meta:
             'is_active', 'gravatar', 'locale', 'time_zone', 'is_trusted'
         )
 
-
 class SourceSerializer(ShareModelSerializer):
     class Meta:
         model = models.Source
         fields = ('name', 'home_page', 'long_title', 'icon')
 
+class HarvestLogSerializer(ShareModelSerializer):
+    class Meta:
+        model = models.HarvestLog
+        fields = '__all__'
+
+class SourceConfigSerializer(ShareModelSerializer):
+    class Meta:
+        model = models.SourceConfig
+        fields = '__all__'
+
 
 class SiteBannerSerializer(ShareModelSerializer):
     color = serializers.SerializerMethodField()

api/urls.py

@@ -14,6 +14,8 @@
 from api.serializers import BaseShareSerializer
 from api.views.share import ShareObjectViewSet
 
+
+
 app_name = 'api'
 
 router = DefaultRouter()
@@ -86,6 +88,8 @@ def register_url(self, subclass, viewset):
 register_route(r'rawdata', views.RawDatumViewSet)
 register_route(r'user', views.ShareUserViewSet)
 register_route(r'sources', views.SourceViewSet)
+register_route(r'HarvestLog', views.HarvestLogViewSet)
+register_route(r'SourceConfig', views.SourceConfigViewSet)
 
 router.register(r'normalizeddata', views.NormalizedDataViewSet, base_name='normalizeddata')
 

api/views/__init__.py

@@ -5,3 +5,5 @@
 from .registration import *  # noqa
 from .schema import *  # noqa
 from .banner import *  # noqa
+from .harvestlogs import *  # noqa
+from .sourceConfig import * # noqa

api/views/harvestlogs.py

@@ -0,0 +1,23 @@
+from rest_framework import viewsets
+from rest_framework import filters
+from django_filters.filters import MultipleChoiceFilter
+from api.views import ShareObjectViewSet
+from share.util import IDObfuscator
+
+from api.serializers import HarvestLogSerializer
+from share.models import HarvestLog
+
+class SourceConfigFilterBackend(MultipleChoiceFilter):
+    def filter_queryset(self, request, queryset, view, conjoined=True):
+        if 'source_config_id' in request.GET:
+            decoded = IDObfuscator.decode_id(request.GET['source_config_id'])
+            queryset = queryset.filter(source_config_id=decoded)
+        if 'status' in request.GET:
+            queryset = queryset.filter(status__in=request.GET.getlist('status'))
+        return queryset
+
+class HarvestLogViewSet(ShareObjectViewSet):
+    serializer_class = HarvestLogSerializer
+    queryset = HarvestLog.objects.all()
+    filter_backends = (SourceConfigFilterBackend, )
+    filter_fields = ('source_config_id', 'status',)

api/views/share.py

@@ -28,13 +28,15 @@ class ShareObjectViewSet(viewsets.ReadOnlyModelViewSet):
 
     # Override get_queryset to handle items marked as deleted.
     def get_queryset(self, list=True):
+        # import ipdb; ipdb.set_trace()
         queryset = super().get_queryset()
         if list and hasattr(queryset.model, 'is_deleted'):
             return queryset.exclude(is_deleted=True)
         return queryset
 
     # Override to convert encoded pk to an actual pk
     def get_object(self):
+        import ipdb; ipdb.set_trace()
         queryset = self.filter_queryset(self.get_queryset(False))
 
         # Perform the lookup filtering.

api/views/sourceConfig.py

@@ -0,0 +1,18 @@
+from rest_framework import viewsets
+from api.views import ShareObjectViewSet
+from rest_framework import filters
+from api.serializers import SourceConfigSerializer
+from share.models import SourceConfig
+from api.pagination import FuzzyPageNumberPagination
+from django.db.models import Count
+from django.db.models import Aggregate
+from share.models import HarvestLog
+from django.db.models import OuterRef, Subquery
+from django.db.models import IntegerField
+
+class SourceConfigViewSet(ShareObjectViewSet):
+    serializer_class = SourceConfigSerializer
+    pagination_class = FuzzyPageNumberPagination
+    queryset = SourceConfig.objects.all()
+
+

api/views/workflow.py

@@ -20,8 +20,8 @@
 from api.pagination import CursorPagination
 from api.authentication import APIV1TokenBackPortAuthentication
 from api.permissions import ReadOnlyOrTokenHasScopeOrIsAuthenticated
-from api.serializers import FullNormalizedDataSerializer, BasicNormalizedDataSerializer, \
-    RawDatumSerializer, ShareUserSerializer, SourceSerializer
+from api.serializers import FullNormalizedDataSerializer, BasicNormalizedDataSerializer,RawDatumSerializer, ShareUserSerializer, SourceSerializer
+
 from share.models import RawDatum, NormalizedData, Source, SourceConfig, Transformer, ShareUser
 from share.tasks import disambiguate
 from share.harvest.serialization import DictSerializer
@@ -56,6 +56,7 @@ class SourceViewSet(viewsets.ReadOnlyModelViewSet):
     def get_queryset(self):
         return Source.objects.exclude(icon='').exclude(is_deleted=True)
 
+
     def create(self, request, *args, **kwargs):
         try:
             long_title = request.data['long_title']
@@ -129,6 +130,7 @@ def create(self, request, *args, **kwargs):
         )
 
 
+
 class NormalizedDataViewSet(viewsets.ModelViewSet):
     """View showing all normalized data in the SHARE Dataset.
 

docs/elasticsearch.rst

@@ -26,6 +26,7 @@ Elasticsearch can be used to search the following fields in the normalized data:
     'contributors'
     'funders'
     'publishers'
+    'id'
 
 
 Accessing the Search API