From 5cbac4da7dc720c2a27584f053295fff9c38782b Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Tue, 5 Nov 2019 17:03:08 -0500 Subject: [PATCH 01/27] Made new models branch It has the code from the models.py we've been using rather than the old models.py --- backend/apps/main/models.py | 104 +++++++++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/backend/apps/main/models.py b/backend/apps/main/models.py index f5e67b9..c3584a5 100644 --- a/backend/apps/main/models.py +++ b/backend/apps/main/models.py @@ -3,7 +3,10 @@ """ import json from pathlib import Path -# from django.db import models +from django.db import models +import pandas as pd +import pickle +from collections import Counter from config.settings.base import BACKEND_DIR @@ -19,6 +22,99 @@ def __init__(self, pk: int, name: str, docs: int, words: int): self.docs = docs self.words = words +MAX_LENGTH = 250 + +class Document(models.Model): + au = models.CharField(blank=True, max_length=MAX_LENGTH) + au_org = models.CharField(blank=True, max_length=MAX_LENGTH) + au_person = models.CharField(blank=True, max_length=MAX_LENGTH) + cc = models.CharField(blank=True, max_length=MAX_LENGTH) + cc_org = models.CharField(blank=True, max_length=MAX_LENGTH) + collection = models.CharField(blank=True, max_length=MAX_LENGTH) + date = models.CharField(blank=True, max_length=MAX_LENGTH) + doc_type = models.CharField(blank=True, max_length=MAX_LENGTH) + pages = models.IntegerField(blank=True) + rc = models.CharField(blank=True, max_length=MAX_LENGTH) + rc_org = models.CharField(blank=True, max_length=MAX_LENGTH) + rc_person = models.CharField(blank=True, max_length=MAX_LENGTH) + text = models.TextField(blank=True) + tid = models.CharField(unique=True, max_length=MAX_LENGTH) + title = models.CharField(blank=True, max_length=MAX_LENGTH) + + recipients = models.ManyToManyField(Person) + authors = models.ManyToManyField(Person) + + def __str__(self): + return f'tid: {self.tid}, title: {self.title}, date: {self.date}' + +def import_csv_to_document_model(path): + df = pd.read_csv(path).fillna('') + for _, row in df.iterrows(): + d = Document(au=row['au'], + au_org=row['au_org'], + au_person=row['au_person'], + cc=row['cc'], + cc_org=row['cc_org'], + collection=row['collection'], + date=row['date'], + doc_type=row['doc_type'], + pages=int(row['pages']), + rc=row['rc'], + rc_org=row['rc_org'], + rc_person=row['rc_person'], + text=row['text'], + tid=row['tid'], + title=row['title'] + ) + d.save() + +class Person(models.Model): + # Maybe should be blank = True?? + last = models.CharField(max_length=255) + first = models.CharField(max_length=255) + middle = models.CharField(max_length=255) + position = models.CharField(max_length=MAX_LENGTH) + # TODO: Make counter into textfield as json + positions = models.TextField() + aliases = models.TextField() + count = models.IntegerField() + + # TODO: write getter to parse json in textfield + # See rereading/.../backend, analysis + # Look into JSON Fields -- they won't work here, but it's + # an example of how to do this stuff + def get_parsed_positions(self): + self.positions + + def __str__(self): + s = f'{self.first} {self.middle} {self.last}' + s = s + ", Position: " + str(self.positions) + ", Aliases: " + \ + str(self.aliases) + ", count: " + str(self.count) + return s + + @property + def positions_counter(self): + return Counter(json.loads(self.positions)) + + # TODO: write json getter here + @property + def aliases_list(self): + +def import_peopledb_to_person_model(file_path): + with open(str(file_path), 'rb') as infile: + db = pickle.load(infile) + for person in db.people: + p = Person(last=person.last, + first=person.first, + middle=person.middle, + position=person.position, + positions=json.dumps(person.positions), + aliases=json.dumps(person.aliases), + count=person.count + ) + p.save() + + class Edge: """ @@ -66,3 +162,9 @@ def load_network_json_data(return_type: str): words = int(person_dict.get('words')) people.append(Person(pk, name, docs, words)) return people + + +if __name__ == '__main__': + path = Path('..', 'data', 'name_disambiguation', 'dunn_docs.csv') + import_csv_to_document_model(path) + Document.objects.all() From aa50d4bf778c5a6455944fdb55b98442a43d2089 Mon Sep 17 00:00:00 2001 From: Christina Wang Date: Thu, 7 Nov 2019 10:21:03 -0500 Subject: [PATCH 02/27] updated models with import and manytomany fields --- backend/apps/main/models.py | 214 ++++++++++++++++++++---------------- 1 file changed, 120 insertions(+), 94 deletions(-) diff --git a/backend/apps/main/models.py b/backend/apps/main/models.py index c3584a5..35114ca 100644 --- a/backend/apps/main/models.py +++ b/backend/apps/main/models.py @@ -7,24 +7,85 @@ import pandas as pd import pickle from collections import Counter +# from django.db import models from config.settings.base import BACKEND_DIR +MAX_LENGTH = 250 -class Person: - """ - Python class for representing people in the data; - probably to be eventually replaced with a Django model + +class Person(models.Model): + """Django database to represent Person objects + Fields: + last: CharField, last name + first: CharField, first name + middle: CharField, middle name + most_likely_org: CharField, org that appeared most times for the person + positions: TextField, string json representation of positions Counter (all related + annotations on the person). DO NOT DIRECTLY ACCESS (use property positions_counter) + aliases: TextField, string json representation of aliases Counter + (all raw strings used to refer to this person). DO NOT DIRECTLY ACCESS (use property + aliases_counter) + # TODO figure out if this is as author or as recipient + count: IntegerField, number of documents the person appeared in + Properties: + positions_counter: Counter of related annotations on the person + aliases_counter: Counter of all raw strings used to refer to this person """ - def __init__(self, pk: int, name: str, docs: int, words: int): - self.pk = pk # pylint: disable=C0103 - self.name = name - self.docs = docs - self.words = words + # Maybe should be blank = True?? + last = models.CharField(max_length=255) + first = models.CharField(max_length=255) + middle = models.CharField(max_length=255) + most_likely_org = models.CharField(max_length=MAX_LENGTH) + # positions & aliases are json strings that need to be parsed as Counter every time + positions = models.TextField() + aliases = models.TextField() + count = models.IntegerField() + + def __str__(self): + s = f'{self.first} {self.middle} {self.last}' + s = s + ", Position: " + str(self.positions) + ", Aliases: " + \ + str(self.aliases) + ", count: " + str(self.count) + return s + + @property + def positions_counter(self): + """ + :return: positions Counter (from string json) + """ + return Counter(json.loads(self.positions)) + + @property + def aliases_counter(self): + """ + :return: aliases Counter (from string json) + """ + return Counter(json.loads(self.aliases)) -MAX_LENGTH = 250 class Document(models.Model): + """Django database to represent Person objects + # TODO: check if the field descriptions are correct + Fields: + au: CharField, author(s) + au_org: CharField, author organizations + au_person: CharField, author(s) + cc: CharField, cc-ed names + cc_org: CharField, organization of cc-ed names + collection: CharField, collection that the document was released from + date: CharField, string representation of the date of the document + doc_type: CharField, type of the document (e.g. letter, memo) + pages: IntegerField, number of pages in document + rc: CharField, recipient(s) + rc_org: CharField, recipient organizations + rc_person: CharField, recipient(s) + text: TextField, text of the document (is it full text??) + tid: CharField, document ID + title: CharField, title of the document + + authors: ManyToManyField, connect Document and its authors' Person objects in Django + recipients: ManyToManyField, connect Document and its recipients' Person objects in Django + """ au = models.CharField(blank=True, max_length=MAX_LENGTH) au_org = models.CharField(blank=True, max_length=MAX_LENGTH) au_person = models.CharField(blank=True, max_length=MAX_LENGTH) @@ -41,14 +102,20 @@ class Document(models.Model): tid = models.CharField(unique=True, max_length=MAX_LENGTH) title = models.CharField(blank=True, max_length=MAX_LENGTH) - recipients = models.ManyToManyField(Person) authors = models.ManyToManyField(Person) + recipients = models.ManyToManyField(Person) def __str__(self): return f'tid: {self.tid}, title: {self.title}, date: {self.date}' -def import_csv_to_document_model(path): - df = pd.read_csv(path).fillna('') + +def import_csv_to_document_model(csv_path): + """ + Reads csv of docs and create Document model + :param csv_path: Path to csv file + :return: + """ + df = pd.read_csv(csv_path).fillna('') for _, row in df.iterrows(): d = Document(au=row['au'], au_org=row['au_org'], @@ -68,46 +135,53 @@ def import_csv_to_document_model(path): ) d.save() -class Person(models.Model): - # Maybe should be blank = True?? - last = models.CharField(max_length=255) - first = models.CharField(max_length=255) - middle = models.CharField(max_length=255) - position = models.CharField(max_length=MAX_LENGTH) - # TODO: Make counter into textfield as json - positions = models.TextField() - aliases = models.TextField() - count = models.IntegerField() - - # TODO: write getter to parse json in textfield - # See rereading/.../backend, analysis - # Look into JSON Fields -- they won't work here, but it's - # an example of how to do this stuff - def get_parsed_positions(self): - self.positions - - def __str__(self): - s = f'{self.first} {self.middle} {self.last}' - s = s + ", Position: " + str(self.positions) + ", Aliases: " + \ - str(self.aliases) + ", count: " + str(self.count) - return s + # for au/au_person, and rc/rc_person, parse it into list of individual raw names + parsed_au = [] + if row['au_person']: + parsed_au = parse_column_person(row['au_person']) + elif row['au']: + parsed_au = parse_column_person(row['au']) + # for each raw name, search in Person model by aliases + # add connection to authors (ManyToManyField) + for name in parsed_au: + person = Person.objects.filter(aliases__contains=name) + d.authors.add(person) + + parsed_rc = [] + if row['rc_person']: + parsed_rc = parse_column_person(row['rc_person']) + elif row['rc']: + parsed_rc = parse_column_person(row['rc']) + for name in parsed_rc: + person = Person.objects.filter(aliases__contains=name) + d.recipients.add(person) + + +def parse_column_person(column_name): + """ + Splits individual names by semicolon or bar (|) - @property - def positions_counter(self): - return Counter(json.loads(self.positions)) + :param column_name: list, taken from csv with doc info + :return: list, names of people in column + """ + names = [] + for name_split_semicolon in [n.strip() for n in column_name.split(';')]: + for name_split_bar in [m.strip() for m in name_split_semicolon.split('|')]: + if 0 < len(name_split_bar) < 100: # pylint: disable=C1801 + names.append(name_split_bar) + return names - # TODO: write json getter here - @property - def aliases_list(self): def import_peopledb_to_person_model(file_path): with open(str(file_path), 'rb') as infile: db = pickle.load(infile) + for person in db.people: p = Person(last=person.last, first=person.first, middle=person.middle, - position=person.position, + most_likely_org=person.most_likely_org, + # convert Counter object into json string positions=json.dumps(person.positions), aliases=json.dumps(person.aliases), count=person.count @@ -115,56 +189,8 @@ def import_peopledb_to_person_model(file_path): p.save() - -class Edge: - """ - Python object to store edges, should be replaced if the Person class is replaced - """ - def __init__(self, pk: int, node1: str, node2: str, # pylint: disable-msg=R0913 - docs: int, words: int): - self.pk = pk # pylint: disable=C0103 - self.node1 = node1 # could be replaced with Person - self.node2 = node2 - self.docs = docs - self.words = words - - -def load_network_json_data(return_type: str): - """ - Loads test json data for initial prototyping - :param return_type: string, determines which list needs to be determined, has the - precondition that it either must be 'nodes' or 'edges' - :return: list, contains all of the edges or all of the nodes (people at this point in time) - from network_test_data.json - """ - if return_type not in ['nodes', 'edges']: - raise ValueError("Specified return type needs to be nodes or edges") - - json_path = Path(BACKEND_DIR, 'data', 'network_test_data.json') - with open(json_path) as json_file: - data = json.load(json_file) - if return_type == "edges": - edges = [] - for edges_dict in data['links']: - pk = int(edges_dict.get('id')) # pylint: disable=C0103 - node1 = edges_dict.get('node1') - node2 = edges_dict.get('node2') - docs = edges_dict.get('docs') - words = int(edges_dict.get('words')) - edges.append(Edge(pk, node1, node2, docs, words)) - return edges - else: - people = [] - for person_dict in data['nodes']: - pk = int(person_dict.get('id')) # pylint: disable=C0103 - name = person_dict.get('name') - docs = person_dict.get('docs') - words = int(person_dict.get('words')) - people.append(Person(pk, name, docs, words)) - return people - - if __name__ == '__main__': - path = Path('..', 'data', 'name_disambiguation', 'dunn_docs.csv') - import_csv_to_document_model(path) + import_peopledb_to_person_model(Path('..', 'data', 'name_disambiguation', 'names_db_10.pickle')) + import_csv_to_document_model(Path('..', 'data', 'name_disambiguation', 'dunn_docs.csv')) Document.objects.all() + From b5eebe6b9d68202273ca1d92faeeb94566f6b68e Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Fri, 8 Nov 2019 17:02:08 -0500 Subject: [PATCH 03/27] Too many blank spaces --- backend/apps/main/serializers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/apps/main/serializers.py b/backend/apps/main/serializers.py index 925ab82..7dcd22e 100644 --- a/backend/apps/main/serializers.py +++ b/backend/apps/main/serializers.py @@ -32,7 +32,6 @@ class EdgeSerializer(serializers.Serializer): docs = serializers.IntegerField(read_only=True) words = serializers.IntegerField(read_only=True) - def create(self, validated_data): """ We will not create new objects using this serializer """ From 0c23a59a465bb4cb0092247b34a411dc776178db Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Fri, 8 Nov 2019 17:05:30 -0500 Subject: [PATCH 04/27] Copying Edge, Person classes and load_network_json_data from master serializers We need to copy these over --- backend/apps/main/serializers.py | 63 +++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/backend/apps/main/serializers.py b/backend/apps/main/serializers.py index 925ab82..fea0f39 100644 --- a/backend/apps/main/serializers.py +++ b/backend/apps/main/serializers.py @@ -4,6 +4,8 @@ allow the frontend to suggest changes to the backend/database. """ from rest_framework import serializers +import json +from pathlib import Path class PersonSerializer(serializers.Serializer): @@ -32,9 +34,68 @@ class EdgeSerializer(serializers.Serializer): docs = serializers.IntegerField(read_only=True) words = serializers.IntegerField(read_only=True) - def create(self, validated_data): """ We will not create new objects using this serializer """ def update(self, instance, validated_data): """ We will not update data using this serializer """ + + +class Person: + """ + Python class for representing people in the data; + probably to be eventually replaced with a Django model + """ + def __init__(self, pk: int, name: str, docs: int, words: int): + self.pk = pk # pylint: disable=C0103 + self.name = name + self.docs = docs + self.words = words + + +class Edge: + """ + Python object to store edges, should be replaced if the Person class is replaced + """ + def __init__(self, pk: int, node1: str, node2: str, # pylint: disable-msg=R0913 + docs: int, words: int): + self.pk = pk # pylint: disable=C0103 + self.node1 = node1 # could be replaced with Person + self.node2 = node2 + self.docs = docs + self.words = words + + +def load_network_json_data(return_type: str): + """ + Loads test json data for initial prototyping + :param return_type: string, determines which list needs to be determined, has the + precondition that it either must be 'nodes' or 'edges' + :return: list, contains all of the edges or all of the nodes (people at this point in time) + from network_test_data.json + """ + if return_type not in ['nodes', 'edges']: + raise ValueError("Specified return type needs to be nodes or edges") + + json_path = Path(BACKEND_DIR, 'data', 'network_test_data.json') + with open(json_path) as json_file: + data = json.load(json_file) + if return_type == "edges": + edges = [] + for edges_dict in data['links']: + pk = int(edges_dict.get('id')) # pylint: disable=C0103 + node1 = edges_dict.get('node1') + node2 = edges_dict.get('node2') + docs = edges_dict.get('docs') + words = int(edges_dict.get('words')) + edges.append(Edge(pk, node1, node2, docs, words)) + return edges + else: + people = [] + for person_dict in data['nodes']: + pk = int(person_dict.get('id')) # pylint: disable=C0103 + name = person_dict.get('name') + docs = person_dict.get('docs') + words = int(person_dict.get('words')) + people.append(Person(pk, name, docs, words)) + return people From 69e7ca44c659fc89b06f06aaac8adcf9e0a698ac Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Mon, 11 Nov 2019 12:21:43 -0500 Subject: [PATCH 05/27] Figuring out some TODOs for next session --- backend/apps/main/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index 8d0eef1..4cafdab 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -51,3 +51,5 @@ def get_network_data(request): link['source'] = link['node1'] link['target'] = link['node2'] return JsonResponse(data) + +# TODO write function that takes an info request from the user, accesses relevant info, returns info From 7d8923f06e9970e7bb7d4667766ff1768f0d2e2d Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Tue, 12 Nov 2019 16:41:27 -0500 Subject: [PATCH 06/27] Added PersonInfoSerializer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PersonSerializer was a dummy to help create a temporary json– this serializer will actually be used when a user requests info about a Person --- backend/apps/main/serializers.py | 57 ++++++++++++++++++++------------ backend/apps/main/views.py | 32 ++++++++++-------- backend/config/urls.py | 3 ++ 3 files changed, 58 insertions(+), 34 deletions(-) diff --git a/backend/apps/main/serializers.py b/backend/apps/main/serializers.py index fea0f39..2081fcf 100644 --- a/backend/apps/main/serializers.py +++ b/backend/apps/main/serializers.py @@ -6,14 +6,32 @@ from rest_framework import serializers import json from pathlib import Path +from .models import Person -class PersonSerializer(serializers.Serializer): +# class PersonSerializer(serializers.Serializer): #this was a dummy to help serialize network info +# """ +# Serializer for the Person model +# """ +# pk = serializers.IntegerField(read_only=True) # pylint: disable=C0103 +# name = serializers.CharField(read_only=True) +# docs = serializers.IntegerField(read_only=True) +# words = serializers.IntegerField(read_only=True) +# +# def create(self, validated_data): +# """ We will not create new objects using this serializer """ +# +# def update(self, instance, validated_data): +# """ We will not update data using this serializer """ + + +class EdgeSerializer(serializers.Serializer): """ - Serializer for the Person model + Serializer for the Edge model """ pk = serializers.IntegerField(read_only=True) # pylint: disable=C0103 - name = serializers.CharField(read_only=True) + node1 = serializers.CharField(read_only=True) + node2 = serializers.CharField(read_only=True) docs = serializers.IntegerField(read_only=True) words = serializers.IntegerField(read_only=True) @@ -24,15 +42,13 @@ def update(self, instance, validated_data): """ We will not update data using this serializer """ -class EdgeSerializer(serializers.Serializer): +class PersonInfoSerializer(serializers.ModelSerializer): """ - Serializer for the Edge model + Serializer for the Person Info model """ - pk = serializers.IntegerField(read_only=True) # pylint: disable=C0103 - node1 = serializers.CharField(read_only=True) - node2 = serializers.CharField(read_only=True) - docs = serializers.IntegerField(read_only=True) - words = serializers.IntegerField(read_only=True) + class Meta: + model = Person + fields = ["last", "first", "middle", "most_likely_org", "positions", "aliases", "count"] def create(self, validated_data): """ We will not create new objects using this serializer """ @@ -40,17 +56,16 @@ def create(self, validated_data): def update(self, instance, validated_data): """ We will not update data using this serializer """ - -class Person: - """ - Python class for representing people in the data; - probably to be eventually replaced with a Django model - """ - def __init__(self, pk: int, name: str, docs: int, words: int): - self.pk = pk # pylint: disable=C0103 - self.name = name - self.docs = docs - self.words = words +# class Person: #this was just a dummy to help serialize the network info +# """ +# Python class for representing people in the data; +# probably to be eventually replaced with a Django model +# """ +# def __init__(self, pk: int, name: str, docs: int, words: int): +# self.pk = pk # pylint: disable=C0103 +# self.name = name +# self.docs = docs +# self.words = words class Edge: diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index 4cafdab..d2ce5e0 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -5,24 +5,15 @@ from pathlib import Path import random +from collections import Counter from django.http import JsonResponse from rest_framework.decorators import api_view from rest_framework.response import Response from config.settings.base import BACKEND_DIR -from .serializers import PersonSerializer, EdgeSerializer +from .serializers import PersonInfoSerializer, EdgeSerializer from .models import load_network_json_data - - - - -@api_view(['GET']) -def list_people(request): - """ - Return a list of all Person objects, serialized. - """ - serializer = PersonSerializer(instance=load_network_json_data("nodes"), many=True) - return Response(serializer.data) +from .models import Person @api_view(['GET']) @@ -52,4 +43,19 @@ def get_network_data(request): link['target'] = link['node2'] return JsonResponse(data) -# TODO write function that takes an info request from the user, accesses relevant info, returns info + +@api_view(['GET']) +def get_person_info(request): + dummy = Person(last="Lastname", + first="Firstname", + middle="Middlename", + most_likely_org="Philip Morris", + # convert Counter object into json string + positions=json.dumps(Counter()), + aliases=json.dumps([]), + count=5 + ) + serializer = PersonInfoSerializer(instance=load_network_json_data("nodes"), many=True) + return Response(serializer.data) + + diff --git a/backend/config/urls.py b/backend/config/urls.py index ef105ba..953bb8f 100644 --- a/backend/config/urls.py +++ b/backend/config/urls.py @@ -36,4 +36,7 @@ # React views url('', render_react_view, {'component_name': 'MainView'}), + + # Person_info views + url('api/person_info/', main_views.get_person_info), ] From ead38bf223f6838dc2307fdfbacb7c131fb04b9c Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Tue, 12 Nov 2019 16:49:51 -0500 Subject: [PATCH 07/27] Temporary fix for bug with document document.recipients clashing with document.authors --- backend/apps/main/models.py | 3 ++- backend/apps/main/views.py | 6 +++--- backend/config/urls.py | 2 -- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/backend/apps/main/models.py b/backend/apps/main/models.py index 35114ca..836ecdf 100644 --- a/backend/apps/main/models.py +++ b/backend/apps/main/models.py @@ -103,7 +103,8 @@ class Document(models.Model): title = models.CharField(blank=True, max_length=MAX_LENGTH) authors = models.ManyToManyField(Person) - recipients = models.ManyToManyField(Person) + # recipients = models.ManyToManyField(Person) #they both need to be done, but they clash + # right now def __str__(self): return f'tid: {self.tid}, title: {self.title}, date: {self.date}' diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index d2ce5e0..e67ecf3 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -11,8 +11,7 @@ from rest_framework.decorators import api_view from rest_framework.response import Response from config.settings.base import BACKEND_DIR -from .serializers import PersonInfoSerializer, EdgeSerializer -from .models import load_network_json_data +from .serializers import PersonInfoSerializer, EdgeSerializer, load_network_json_data from .models import Person @@ -46,6 +45,7 @@ def get_network_data(request): @api_view(['GET']) def get_person_info(request): + print("HI!!!") dummy = Person(last="Lastname", first="Firstname", middle="Middlename", @@ -55,7 +55,7 @@ def get_person_info(request): aliases=json.dumps([]), count=5 ) - serializer = PersonInfoSerializer(instance=load_network_json_data("nodes"), many=True) + serializer = PersonInfoSerializer(instance=dummy, many=False) return Response(serializer.data) diff --git a/backend/config/urls.py b/backend/config/urls.py index 953bb8f..1748459 100644 --- a/backend/config/urls.py +++ b/backend/config/urls.py @@ -27,8 +27,6 @@ path('admin/', admin.site.urls), # API endpoints - url('api/people/', main_views.list_people), - # might want to change name later url('api/edges/', main_views.list_edges), # temporary json endpoint for network data From 25d06a9ff2b26efa3238cbcc50a95f05c0b98cd4 Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Tue, 12 Nov 2019 16:55:44 -0500 Subject: [PATCH 08/27] Moved the url for 'api/person_info/' above When it was below the '' Url, it wouldn't load because the '' matched everything --- backend/config/urls.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/config/urls.py b/backend/config/urls.py index 1748459..3b1d8c3 100644 --- a/backend/config/urls.py +++ b/backend/config/urls.py @@ -32,9 +32,11 @@ # temporary json endpoint for network data url('get_network_data', main_views.get_network_data), + # Person_info views + url('api/person_info/', main_views.get_person_info), + # React views url('', render_react_view, {'component_name': 'MainView'}), - # Person_info views - url('api/person_info/', main_views.get_person_info), + ] From e79b5680fc6d83467dab5f2877e01f6a6162d87b Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Thu, 14 Nov 2019 11:33:54 -0500 Subject: [PATCH 09/27] Updated get_person_info to return person info Now it accepts a query, extracts the full_name from it, and returns a person with that last name --- .../migrations/0002_auto_20191114_1533.py | 31 +++++++++++++++++++ .../main/migrations/0003_person_full_name.py | 19 ++++++++++++ backend/apps/main/models.py | 1 + backend/apps/main/serializers.py | 3 +- backend/apps/main/views.py | 16 +++++----- 5 files changed, 62 insertions(+), 8 deletions(-) create mode 100644 backend/apps/main/migrations/0002_auto_20191114_1533.py create mode 100644 backend/apps/main/migrations/0003_person_full_name.py diff --git a/backend/apps/main/migrations/0002_auto_20191114_1533.py b/backend/apps/main/migrations/0002_auto_20191114_1533.py new file mode 100644 index 0000000..96e6766 --- /dev/null +++ b/backend/apps/main/migrations/0002_auto_20191114_1533.py @@ -0,0 +1,31 @@ +# Generated by Django 2.2.4 on 2019-11-14 15:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Person', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('last', models.CharField(max_length=255)), + ('first', models.CharField(max_length=255)), + ('middle', models.CharField(max_length=255)), + ('most_likely_org', models.CharField(max_length=250)), + ('positions', models.TextField()), + ('aliases', models.TextField()), + ('count', models.IntegerField()), + ], + ), + migrations.AddField( + model_name='document', + name='authors', + field=models.ManyToManyField(to='main.Person'), + ), + ] diff --git a/backend/apps/main/migrations/0003_person_full_name.py b/backend/apps/main/migrations/0003_person_full_name.py new file mode 100644 index 0000000..85b5b95 --- /dev/null +++ b/backend/apps/main/migrations/0003_person_full_name.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.4 on 2019-11-14 16:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0002_auto_20191114_1533'), + ] + + operations = [ + migrations.AddField( + model_name='person', + name='full_name', + field=models.CharField(default=None, max_length=255), + preserve_default=False, + ), + ] diff --git a/backend/apps/main/models.py b/backend/apps/main/models.py index 836ecdf..519a195 100644 --- a/backend/apps/main/models.py +++ b/backend/apps/main/models.py @@ -36,6 +36,7 @@ class Person(models.Model): last = models.CharField(max_length=255) first = models.CharField(max_length=255) middle = models.CharField(max_length=255) + full_name = models.CharField(max_length=255) most_likely_org = models.CharField(max_length=MAX_LENGTH) # positions & aliases are json strings that need to be parsed as Counter every time positions = models.TextField() diff --git a/backend/apps/main/serializers.py b/backend/apps/main/serializers.py index 2081fcf..9493e3e 100644 --- a/backend/apps/main/serializers.py +++ b/backend/apps/main/serializers.py @@ -48,7 +48,8 @@ class PersonInfoSerializer(serializers.ModelSerializer): """ class Meta: model = Person - fields = ["last", "first", "middle", "most_likely_org", "positions", "aliases", "count"] + fields = ["last", "first", "middle", "full_name", "most_likely_org", "positions", + "aliases", "count"] def create(self, validated_data): """ We will not create new objects using this serializer """ diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index e67ecf3..b0caa6b 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -45,17 +45,19 @@ def get_network_data(request): @api_view(['GET']) def get_person_info(request): - print("HI!!!") - dummy = Person(last="Lastname", - first="Firstname", - middle="Middlename", - most_likely_org="Philip Morris", + dummy = Person(last="LAB", + first="MIT", + middle="DH", + full_name="MIT DH LAB", + most_likely_org="MIT YAY!!!!!", # convert Counter object into json string positions=json.dumps(Counter()), aliases=json.dumps([]), - count=5 + count=3 ) - serializer = PersonInfoSerializer(instance=dummy, many=False) + dummy.save() + queryset = Person.objects.filter(full_name=request.query_params['full_name']) + serializer = PersonInfoSerializer(instance=queryset, many=True) return Response(serializer.data) From 4d1f3afea550820093d18256f6d987634369b335 Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Fri, 15 Nov 2019 16:46:31 -0500 Subject: [PATCH 10/27] Deleted the dummy person in views.py We no longer need this --- backend/apps/main/views.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index b0caa6b..ba1d558 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -45,17 +45,13 @@ def get_network_data(request): @api_view(['GET']) def get_person_info(request): - dummy = Person(last="LAB", - first="MIT", - middle="DH", - full_name="MIT DH LAB", - most_likely_org="MIT YAY!!!!!", - # convert Counter object into json string - positions=json.dumps(Counter()), - aliases=json.dumps([]), - count=3 - ) - dummy.save() + """ + Finds a person matching the full name requested by the user, serializes this person, and + returns the serialized person + :param request: request from the user; + request.query_params is a dict {'full name': FULL NAME OF RELEVANT PERSON} + :return: serialized person matching full name imbain request + """ queryset = Person.objects.filter(full_name=request.query_params['full_name']) serializer = PersonInfoSerializer(instance=queryset, many=True) return Response(serializer.data) From 389bf48d0994a4002ee79bd7cb90dea57524a26f Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Sat, 16 Nov 2019 12:35:15 -0500 Subject: [PATCH 11/27] I deleted some imports that were never used No need for imports that aren't used --- backend/apps/main/models.py | 3 --- backend/apps/main/serializers.py | 27 --------------------------- backend/apps/main/views.py | 1 - 3 files changed, 31 deletions(-) diff --git a/backend/apps/main/models.py b/backend/apps/main/models.py index 519a195..77917ef 100644 --- a/backend/apps/main/models.py +++ b/backend/apps/main/models.py @@ -7,9 +7,6 @@ import pandas as pd import pickle from collections import Counter -# from django.db import models - -from config.settings.base import BACKEND_DIR MAX_LENGTH = 250 diff --git a/backend/apps/main/serializers.py b/backend/apps/main/serializers.py index 9493e3e..ddb5467 100644 --- a/backend/apps/main/serializers.py +++ b/backend/apps/main/serializers.py @@ -9,22 +9,6 @@ from .models import Person -# class PersonSerializer(serializers.Serializer): #this was a dummy to help serialize network info -# """ -# Serializer for the Person model -# """ -# pk = serializers.IntegerField(read_only=True) # pylint: disable=C0103 -# name = serializers.CharField(read_only=True) -# docs = serializers.IntegerField(read_only=True) -# words = serializers.IntegerField(read_only=True) -# -# def create(self, validated_data): -# """ We will not create new objects using this serializer """ -# -# def update(self, instance, validated_data): -# """ We will not update data using this serializer """ - - class EdgeSerializer(serializers.Serializer): """ Serializer for the Edge model @@ -57,17 +41,6 @@ def create(self, validated_data): def update(self, instance, validated_data): """ We will not update data using this serializer """ -# class Person: #this was just a dummy to help serialize the network info -# """ -# Python class for representing people in the data; -# probably to be eventually replaced with a Django model -# """ -# def __init__(self, pk: int, name: str, docs: int, words: int): -# self.pk = pk # pylint: disable=C0103 -# self.name = name -# self.docs = docs -# self.words = words - class Edge: """ diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index ba1d558..6d089fb 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -5,7 +5,6 @@ from pathlib import Path import random -from collections import Counter from django.http import JsonResponse from rest_framework.decorators import api_view From 69c88befdd23440b5f6e77283fdad5d2a5b47bd5 Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Sat, 16 Nov 2019 12:50:47 -0500 Subject: [PATCH 12/27] Imported BACKEND_DIR It may throw an error if I don't do this --- backend/apps/main/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/apps/main/models.py b/backend/apps/main/models.py index 77917ef..bc98816 100644 --- a/backend/apps/main/models.py +++ b/backend/apps/main/models.py @@ -8,6 +8,8 @@ import pickle from collections import Counter +from config.settings.base import BACKEND_DIR + MAX_LENGTH = 250 From c02a853849a941ce9f9bc77e25d961b9190726c5 Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Sat, 16 Nov 2019 13:28:58 -0500 Subject: [PATCH 13/27] I commented out things I thought we were going to delete Not sure if list_edges and get_network_data are ok to delete yet --- backend/apps/main/models.py | 4 -- backend/apps/main/serializers.py | 71 ++++++++++++++++---------------- backend/apps/main/views.py | 70 +++++++++++++++---------------- 3 files changed, 70 insertions(+), 75 deletions(-) diff --git a/backend/apps/main/models.py b/backend/apps/main/models.py index bc98816..46f7dd8 100644 --- a/backend/apps/main/models.py +++ b/backend/apps/main/models.py @@ -8,8 +8,6 @@ import pickle from collections import Counter -from config.settings.base import BACKEND_DIR - MAX_LENGTH = 250 @@ -161,7 +159,6 @@ def import_csv_to_document_model(csv_path): def parse_column_person(column_name): """ Splits individual names by semicolon or bar (|) - :param column_name: list, taken from csv with doc info :return: list, names of people in column """ @@ -194,4 +191,3 @@ def import_peopledb_to_person_model(file_path): import_peopledb_to_person_model(Path('..', 'data', 'name_disambiguation', 'names_db_10.pickle')) import_csv_to_document_model(Path('..', 'data', 'name_disambiguation', 'dunn_docs.csv')) Document.objects.all() - diff --git a/backend/apps/main/serializers.py b/backend/apps/main/serializers.py index ddb5467..110fb4c 100644 --- a/backend/apps/main/serializers.py +++ b/backend/apps/main/serializers.py @@ -3,9 +3,10 @@ in ways that can be transported across the backend/frontend divide, or allow the frontend to suggest changes to the backend/database. """ +# import json +# from pathlib import Path from rest_framework import serializers -import json -from pathlib import Path +# from config.settings.base import BACKEND_DIR from .models import Person @@ -55,36 +56,36 @@ def __init__(self, pk: int, node1: str, node2: str, # pylint: disable-msg=R0913 self.words = words -def load_network_json_data(return_type: str): - """ - Loads test json data for initial prototyping - :param return_type: string, determines which list needs to be determined, has the - precondition that it either must be 'nodes' or 'edges' - :return: list, contains all of the edges or all of the nodes (people at this point in time) - from network_test_data.json - """ - if return_type not in ['nodes', 'edges']: - raise ValueError("Specified return type needs to be nodes or edges") - - json_path = Path(BACKEND_DIR, 'data', 'network_test_data.json') - with open(json_path) as json_file: - data = json.load(json_file) - if return_type == "edges": - edges = [] - for edges_dict in data['links']: - pk = int(edges_dict.get('id')) # pylint: disable=C0103 - node1 = edges_dict.get('node1') - node2 = edges_dict.get('node2') - docs = edges_dict.get('docs') - words = int(edges_dict.get('words')) - edges.append(Edge(pk, node1, node2, docs, words)) - return edges - else: - people = [] - for person_dict in data['nodes']: - pk = int(person_dict.get('id')) # pylint: disable=C0103 - name = person_dict.get('name') - docs = person_dict.get('docs') - words = int(person_dict.get('words')) - people.append(Person(pk, name, docs, words)) - return people +# def load_network_json_data(return_type: str): +# """ +# Loads test json data for initial prototyping +# :param return_type: string, determines which list needs to be determined, has the +# precondition that it either must be 'nodes' or 'edges' +# :return: list, contains all of the edges or all of the nodes (people at this point in time) +# from network_test_data.json +# """ +# if return_type not in ['nodes', 'edges']: +# raise ValueError("Specified return type needs to be nodes or edges") +# +# json_path = Path(BACKEND_DIR, 'data', 'network_test_data.json') +# with open(json_path) as json_file: +# data = json.load(json_file) +# if return_type == "edges": +# edges = [] +# for edges_dict in data['links']: +# pk = int(edges_dict.get('id')) # pylint: disable=C0103 +# node1 = edges_dict.get('node1') +# node2 = edges_dict.get('node2') +# docs = edges_dict.get('docs') +# words = int(edges_dict.get('words')) +# edges.append(Edge(pk, node1, node2, docs, words)) +# return edges +# else: +# people = [] +# for person_dict in data['nodes']: +# pk = int(person_dict.get('id')) # pylint: disable=C0103 +# name = person_dict.get('name') +# docs = person_dict.get('docs') +# words = int(person_dict.get('words')) +# people.append(Person(pk, name, docs, words)) +# return people diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index 6d089fb..e07e707 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -1,45 +1,45 @@ """ Views that define API endpoints for the site """ -import json -from pathlib import Path - -import random - -from django.http import JsonResponse +# import json +# from pathlib import Path +# +# import random +# +# from django.http import JsonResponse from rest_framework.decorators import api_view from rest_framework.response import Response -from config.settings.base import BACKEND_DIR -from .serializers import PersonInfoSerializer, EdgeSerializer, load_network_json_data +# from backend.config.settings.base import BACKEND_DIR +from .serializers import PersonInfoSerializer#, EdgeSerializer , load_network_json_data from .models import Person -@api_view(['GET']) -def list_edges(request): - """ - Return a list of all Edge objects, serialized. - """ - serializer = EdgeSerializer(instance=load_network_json_data("edges"), many=True) - return Response(serializer.data) - - -def get_network_data(request): - """ - Temporary view to get network test data json - """ - json_path = Path(BACKEND_DIR, 'data', 'network_test_data.json') - with open(json_path) as json_file: - data = json.load(json_file) - nodes = data['nodes'] - for node in nodes: - node['x'] = random.random()*500 - node['y'] = random.random()*500 - node['weight'] = node['docs'] - links = data['links'] - for link in links: - link['source'] = link['node1'] - link['target'] = link['node2'] - return JsonResponse(data) +# @api_view(['GET']) +# def list_edges(request): +# """ +# Return a list of all Edge objects, serialized. +# """ +# serializer = EdgeSerializer(instance=load_network_json_data("edges"), many=True) +# return Response(serializer.data) + + +# def get_network_data(request): +# """ +# Temporary view to get network test data json +# """ +# json_path = Path(BACKEND_DIR, 'data', 'network_test_data.json') +# with open(json_path) as json_file: +# data = json.load(json_file) +# nodes = data['nodes'] +# for node in nodes: +# node['x'] = random.random()*500 +# node['y'] = random.random()*500 +# node['weight'] = node['docs'] +# links = data['links'] +# for link in links: +# link['source'] = link['node1'] +# link['target'] = link['node2'] +# return JsonResponse(data) @api_view(['GET']) @@ -54,5 +54,3 @@ def get_person_info(request): queryset = Person.objects.filter(full_name=request.query_params['full_name']) serializer = PersonInfoSerializer(instance=queryset, many=True) return Response(serializer.data) - - From 2959c52ea5271dc6d7583d5f7c17f58f2bfb8f21 Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Thu, 21 Nov 2019 11:53:15 -0500 Subject: [PATCH 14/27] Added a test case that checks models in tests.py Going to add more test cases soon; this is the first. We also tried adding a test case for serializer but then commented it out because it wasn't working. --- backend/apps/main/serializers.py | 74 ++++++++++++++++---------------- backend/apps/main/tests.py | 74 +++++++++++++++++++++++++++++++- backend/apps/main/views.py | 68 ++++++++++++++--------------- 3 files changed, 145 insertions(+), 71 deletions(-) diff --git a/backend/apps/main/serializers.py b/backend/apps/main/serializers.py index 110fb4c..7b8fb2c 100644 --- a/backend/apps/main/serializers.py +++ b/backend/apps/main/serializers.py @@ -3,10 +3,9 @@ in ways that can be transported across the backend/frontend divide, or allow the frontend to suggest changes to the backend/database. """ -# import json -# from pathlib import Path +import json +from pathlib import Path from rest_framework import serializers -# from config.settings.base import BACKEND_DIR from .models import Person @@ -56,36 +55,39 @@ def __init__(self, pk: int, node1: str, node2: str, # pylint: disable-msg=R0913 self.words = words -# def load_network_json_data(return_type: str): -# """ -# Loads test json data for initial prototyping -# :param return_type: string, determines which list needs to be determined, has the -# precondition that it either must be 'nodes' or 'edges' -# :return: list, contains all of the edges or all of the nodes (people at this point in time) -# from network_test_data.json -# """ -# if return_type not in ['nodes', 'edges']: -# raise ValueError("Specified return type needs to be nodes or edges") -# -# json_path = Path(BACKEND_DIR, 'data', 'network_test_data.json') -# with open(json_path) as json_file: -# data = json.load(json_file) -# if return_type == "edges": -# edges = [] -# for edges_dict in data['links']: -# pk = int(edges_dict.get('id')) # pylint: disable=C0103 -# node1 = edges_dict.get('node1') -# node2 = edges_dict.get('node2') -# docs = edges_dict.get('docs') -# words = int(edges_dict.get('words')) -# edges.append(Edge(pk, node1, node2, docs, words)) -# return edges -# else: -# people = [] -# for person_dict in data['nodes']: -# pk = int(person_dict.get('id')) # pylint: disable=C0103 -# name = person_dict.get('name') -# docs = person_dict.get('docs') -# words = int(person_dict.get('words')) -# people.append(Person(pk, name, docs, words)) -# return people +def load_network_json_data(return_type: str): + """ + Loads test json data for initial prototyping + :param return_type: string, determines which list needs to be determined, has the + precondition that it either must be 'nodes' or 'edges' + :return: list, contains all of the edges or all of the nodes (people at this point in time) + from network_test_data.json + """ + if return_type not in ['nodes', 'edges']: + raise ValueError("Specified return type needs to be nodes or edges") + + json_path = Path('/Users/kimba/Documents/GitHub/tobacco_networks/backend', 'data', + 'network_test_data.json') + with open(json_path) as json_file: + data = json.load(json_file) + if return_type == "edges": + edges = [] + for edges_dict in data['links']: + pk = int(edges_dict.get('id')) # pylint: disable=C0103 + node1 = edges_dict.get('node1') + node2 = edges_dict.get('node2') + docs = edges_dict.get('docs') + words = int(edges_dict.get('words')) + edges.append(Edge(pk, node1, node2, docs, words)) + return edges + else: + people = [] + for person_dict in data['nodes']: + pk = int(person_dict.get('id')) # pylint: disable=C0103 + name = person_dict.get('name') + docs = person_dict.get('docs') + words = int(person_dict.get('words')) + people.append(Person(pk, name, docs, words)) + return people + + diff --git a/backend/apps/main/tests.py b/backend/apps/main/tests.py index 37b07ee..9236b66 100644 --- a/backend/apps/main/tests.py +++ b/backend/apps/main/tests.py @@ -1,11 +1,83 @@ """ Tests for the main app. """ - +import json from django.test import TestCase +from collections import Counter +from rest_framework.test import APIRequestFactory +from rest_framework.response import Response +from .serializers import PersonInfoSerializer +from .models import Person +from .views import get_person_info +# from rest_framework.test import APIRequestFactory +import os +os.environ.setdefault("DJANGO_SETTINGS_MODULE", __file__) +import django +django.setup() class MainTests(TestCase): + def setUp(self): + Person.objects.create( + last="LAB", + first="MIT", + middle="DH", + full_name="MIT DH LAB", + most_likely_org="MIT YAY", + positions=json.dumps(Counter()), + aliases=json.dumps(Counter()), + count=5) + Person.objects.create( + last="LAR", + first="MOT", + middle="DJ", + full_name="MOT DJ LAR", + most_likely_org="KIT BAY", + positions=json.dumps(Counter()), + aliases=json.dumps(Counter()), + count=3) + self.uri = '/api/' + def test_is_this_on(self): """ Trivial test to make sure the testing system is working """ self.assertTrue(2+2 == 4) + + def test_models_01(self): + dummy_1 = Person.objects.get(last='LAB') + dummy_2 = Person.objects.get(last='LAR') + s = f'{dummy_1.first} {dummy_1.middle} {dummy_1.last}' + s = s + ", Position: " + str(dummy_1.positions) + ", Aliases: " + \ + str(dummy_1.aliases) + ", count: " + str(dummy_1.count) + + s2 = f'{dummy_2.first} {dummy_2.middle} {dummy_2.last}' + s2 = s2 + ", Position: " + str(dummy_2.positions) + ", Aliases: " + \ + str(dummy_2.aliases) + ", count: " + str(dummy_2.count) + self.assertEqual( + str(dummy_1), s) + self.assertEqual( + str(dummy_2), s2) + + # def test_api_views(self): + # Person.objects.create( + # last="LAB", + # first="MIT", + # middle="DH", + # full_name="MIT DH LAB", + # most_likely_org="MIT YAY", + # positions=json.dumps(Counter()), + # aliases=json.dumps(Counter()), + # count=5) + # self.factory = APIRequestFactory() + # dummy_1 = Person.objects.get(full_name='MIT DH LAB') + # dummy_1.save() + # serializer = PersonInfoSerializer(instance=dummy_1, many=False) + # expected = Response(serializer.data) + # request = self.factory.get(self.uri) + # result = get_person_info(request) + # self.assertEqual(expected, result) + + + + + + diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index e07e707..0532743 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -1,45 +1,45 @@ """ Views that define API endpoints for the site """ -# import json -# from pathlib import Path -# -# import random -# -# from django.http import JsonResponse +import json +from pathlib import Path + +import random + +from django.http import JsonResponse from rest_framework.decorators import api_view +from collections import Counter from rest_framework.response import Response -# from backend.config.settings.base import BACKEND_DIR -from .serializers import PersonInfoSerializer#, EdgeSerializer , load_network_json_data +from .serializers import PersonInfoSerializer, EdgeSerializer , load_network_json_data from .models import Person -# @api_view(['GET']) -# def list_edges(request): -# """ -# Return a list of all Edge objects, serialized. -# """ -# serializer = EdgeSerializer(instance=load_network_json_data("edges"), many=True) -# return Response(serializer.data) - - -# def get_network_data(request): -# """ -# Temporary view to get network test data json -# """ -# json_path = Path(BACKEND_DIR, 'data', 'network_test_data.json') -# with open(json_path) as json_file: -# data = json.load(json_file) -# nodes = data['nodes'] -# for node in nodes: -# node['x'] = random.random()*500 -# node['y'] = random.random()*500 -# node['weight'] = node['docs'] -# links = data['links'] -# for link in links: -# link['source'] = link['node1'] -# link['target'] = link['node2'] -# return JsonResponse(data) +@api_view(['GET']) +def list_edges(request): + """ + Return a list of all Edge objects, serialized. + """ + serializer = EdgeSerializer(instance=load_network_json_data("edges"), many=True) + return Response(serializer.data) + + +def get_network_data(request): + """ + Temporary view to get network test data json + """ + json_path = Path('/Users/kimba/Documents/GitHub/tobacco_networks/backend', 'data', 'network_test_data.json') + with open(json_path) as json_file: + data = json.load(json_file) + nodes = data['nodes'] + for node in nodes: + node['x'] = random.random()*500 + node['y'] = random.random()*500 + node['weight'] = node['docs'] + links = data['links'] + for link in links: + link['source'] = link['node1'] + link['target'] = link['node2'] + return JsonResponse(data) @api_view(['GET']) From 61191a081fff1ea95bcdc6962b28f98a0edc10d5 Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Fri, 22 Nov 2019 16:46:02 -0500 Subject: [PATCH 15/27] Added test cases for views.py They correctly test whether views will return "person not found" dummy person instance if full_name requested not in database; also tests if returns correctly serialized person if the full_name IS in the database --- README.md | 16 +++-- .../migrations/0004_auto_20191122_2141.py | 18 +++++ backend/apps/main/models.py | 2 +- backend/apps/main/serializers.py | 2 - backend/apps/main/tests.py | 69 +++++++++++-------- backend/apps/main/views.py | 23 +++++-- 6 files changed, 89 insertions(+), 41 deletions(-) create mode 100644 backend/apps/main/migrations/0004_auto_20191122_2141.py diff --git a/README.md b/README.md index 0a00c09..400db30 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,15 @@ just like you did with the rereading project. See the tutorial here: https://uro Finally, from the main directory, run `python setup.py develop` to set up the name_disambiguation package -Note: -- the api does not work at (http://127.0.0.1:8000/api/) as it's not set up for this repo. You can skip the step. -- there is no `frontend/src/App.js`. Use `frontend/src/main/main.js` -instead. +- the api is functional: (http://127.0.0.1:8000/api/person_info/) + +If you would like to find a person through the API, +paste the above url, followed by: + +?full_name=Relevant Person Name + +If the person is not in the database, +the API will return an "empty" person whose full name will be +shown as: Relevant Person Name not found. +Otherwise, a dictionary with the information in the +database associated with that person will be returned. diff --git a/backend/apps/main/migrations/0004_auto_20191122_2141.py b/backend/apps/main/migrations/0004_auto_20191122_2141.py new file mode 100644 index 0000000..35a7e22 --- /dev/null +++ b/backend/apps/main/migrations/0004_auto_20191122_2141.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-11-22 21:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0003_person_full_name'), + ] + + operations = [ + migrations.AlterField( + model_name='person', + name='full_name', + field=models.CharField(max_length=255, unique=True), + ), + ] diff --git a/backend/apps/main/models.py b/backend/apps/main/models.py index 46f7dd8..c6d127a 100644 --- a/backend/apps/main/models.py +++ b/backend/apps/main/models.py @@ -33,7 +33,7 @@ class Person(models.Model): last = models.CharField(max_length=255) first = models.CharField(max_length=255) middle = models.CharField(max_length=255) - full_name = models.CharField(max_length=255) + full_name = models.CharField(max_length=255, unique=True) most_likely_org = models.CharField(max_length=MAX_LENGTH) # positions & aliases are json strings that need to be parsed as Counter every time positions = models.TextField() diff --git a/backend/apps/main/serializers.py b/backend/apps/main/serializers.py index 7b8fb2c..810c2ab 100644 --- a/backend/apps/main/serializers.py +++ b/backend/apps/main/serializers.py @@ -89,5 +89,3 @@ def load_network_json_data(return_type: str): words = int(person_dict.get('words')) people.append(Person(pk, name, docs, words)) return people - - diff --git a/backend/apps/main/tests.py b/backend/apps/main/tests.py index 9236b66..4a14544 100644 --- a/backend/apps/main/tests.py +++ b/backend/apps/main/tests.py @@ -1,6 +1,7 @@ """ Tests for the main app. """ +import django import json from django.test import TestCase from collections import Counter @@ -9,10 +10,8 @@ from .serializers import PersonInfoSerializer from .models import Person from .views import get_person_info -# from rest_framework.test import APIRequestFactory import os os.environ.setdefault("DJANGO_SETTINGS_MODULE", __file__) -import django django.setup() @@ -36,11 +35,16 @@ def setUp(self): positions=json.dumps(Counter()), aliases=json.dumps(Counter()), count=3) - self.uri = '/api/' - - def test_is_this_on(self): - """ Trivial test to make sure the testing system is working """ - self.assertTrue(2+2 == 4) + Person.objects.create( + last="BOBBERT", + first="BOB", + middle="BOBSON", + full_name="BOB BOBSON BOBBERT", + most_likely_org="MIT", + positions=json.dumps(Counter()), + aliases=json.dumps(Counter()), + count=5) + self.factory = APIRequestFactory() def test_models_01(self): dummy_1 = Person.objects.get(last='LAB') @@ -57,27 +61,32 @@ def test_models_01(self): self.assertEqual( str(dummy_2), s2) - # def test_api_views(self): - # Person.objects.create( - # last="LAB", - # first="MIT", - # middle="DH", - # full_name="MIT DH LAB", - # most_likely_org="MIT YAY", - # positions=json.dumps(Counter()), - # aliases=json.dumps(Counter()), - # count=5) - # self.factory = APIRequestFactory() - # dummy_1 = Person.objects.get(full_name='MIT DH LAB') - # dummy_1.save() - # serializer = PersonInfoSerializer(instance=dummy_1, many=False) - # expected = Response(serializer.data) - # request = self.factory.get(self.uri) - # result = get_person_info(request) - # self.assertEqual(expected, result) - - - - - + def test_api_views_01(self): + """tests that get_person_info returns correct person info when the person is in the + database""" + self.factory = APIRequestFactory() + request = self.factory.get('/api/person_info/', {'full_name': 'BOB BOBSON BOBBERT'}) + dummy_1 = Person.objects.filter(full_name='BOB BOBSON BOBBERT') + serializer = PersonInfoSerializer(instance=dummy_1, many=True) + expected = Response(serializer.data) + result = get_person_info(request) + self.assertEqual(expected.data, result.data) + def test_api_views_02(self): + """tests that get_person_info returns correct person info when the person is NOT in the + database""" + Person.objects.create( + last="", + first="", + middle="", + full_name="JANE DOE DEERE not available.", + most_likely_org="", + positions=json.dumps(Counter()), + aliases=json.dumps(Counter()), + count=0) + request = self.factory.get('/api/person_info/', {'full_name': 'JANE DOE DEERE'}) + dummy_1 = Person.objects.get(full_name='JANE DOE DEERE not available.') + serializer = PersonInfoSerializer(instance=dummy_1, many=False) + expected = Response(serializer.data) + result = get_person_info(request) + self.assertEqual(expected.data, result.data) diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index 0532743..9b46e5d 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -10,7 +10,7 @@ from rest_framework.decorators import api_view from collections import Counter from rest_framework.response import Response -from .serializers import PersonInfoSerializer, EdgeSerializer , load_network_json_data +from .serializers import PersonInfoSerializer, EdgeSerializer, load_network_json_data from .models import Person @@ -27,7 +27,8 @@ def get_network_data(request): """ Temporary view to get network test data json """ - json_path = Path('/Users/kimba/Documents/GitHub/tobacco_networks/backend', 'data', 'network_test_data.json') + json_path = Path('/Users/kimba/Documents/GitHub/tobacco_networks/backend', 'data', + 'network_test_data.json') with open(json_path) as json_file: data = json.load(json_file) nodes = data['nodes'] @@ -52,5 +53,19 @@ def get_person_info(request): :return: serialized person matching full name imbain request """ queryset = Person.objects.filter(full_name=request.query_params['full_name']) - serializer = PersonInfoSerializer(instance=queryset, many=True) - return Response(serializer.data) + if not list(queryset): + full_name = request.query_params['full_name'] + new = Person( + last="", + first="", + middle="", + full_name=full_name + " not available.", + most_likely_org="", + positions=json.dumps(Counter()), + aliases=json.dumps(Counter()), + count=0) + serializer = PersonInfoSerializer(instance=new, many=False) + return Response(serializer.data) + else: + serializer = PersonInfoSerializer(instance=queryset, many=True) + return Response(serializer.data) From 89b7b3049f32c98761ffaddf6968f2dd1c144007 Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Mon, 25 Nov 2019 13:46:13 -0500 Subject: [PATCH 16/27] Trying to render person_research_directors --- backend/apps/main/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index 9b46e5d..65493cf 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -28,7 +28,7 @@ def get_network_data(request): Temporary view to get network test data json """ json_path = Path('/Users/kimba/Documents/GitHub/tobacco_networks/backend', 'data', - 'network_test_data.json') + 'person_research_directors.json') with open(json_path) as json_file: data = json.load(json_file) nodes = data['nodes'] From 753026207dfd0eed793028991d31d92f1e0b66e6 Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Thu, 5 Dec 2019 11:42:16 -0500 Subject: [PATCH 17/27] Trying to merge views Temporary changes in order to merge --- backend/apps/main/views.py | 68 ++++++++++++++------------------------ 1 file changed, 25 insertions(+), 43 deletions(-) diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index 65493cf..6ef5003 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -7,28 +7,28 @@ import random from django.http import JsonResponse -from rest_framework.decorators import api_view -from collections import Counter -from rest_framework.response import Response -from .serializers import PersonInfoSerializer, EdgeSerializer, load_network_json_data -from .models import Person - - -@api_view(['GET']) -def list_edges(request): - """ - Return a list of all Edge objects, serialized. - """ - serializer = EdgeSerializer(instance=load_network_json_data("edges"), many=True) - return Response(serializer.data) +from backend.config.settings.base import BACKEND_DIR def get_network_data(request): """ Temporary view to get network test data json """ - json_path = Path('/Users/kimba/Documents/GitHub/tobacco_networks/backend', 'data', - 'person_research_directors.json') + + datasets = { + 'lawyers': 'person_lawyers.json', + 'research_directors': 'person_research_directors.json', + 'sterling': 'person_sterling.json', + 'top_100_edges': 'top_100_edges.json', + 'test': 'network_test_data.json' + } + + if request.GET and 'dataset' in request.GET: + json_filename = datasets[request.GET['dataset']] + else: + json_filename = 'network_test_data.json' + + json_path = Path(BACKEND_DIR, 'data', json_filename) with open(json_path) as json_file: data = json.load(json_file) nodes = data['nodes'] @@ -36,36 +36,18 @@ def get_network_data(request): node['x'] = random.random()*500 node['y'] = random.random()*500 node['weight'] = node['docs'] + node['affiliation'] = random.choice(['Phillip Morris International', 'British American ' + 'Tobacco', + 'Japan Tobacco', 'Imperial Tobacco']) links = data['links'] + adjacent_nodes = {} for link in links: link['source'] = link['node1'] link['target'] = link['node2'] - return JsonResponse(data) + adjacent_nodes[link['node1'] + "-" + link['node2']] = True + adjacent_nodes[link['node2'] + "-" + link['node1']] = True + data["adjacent_nodes"] = adjacent_nodes + # TODO: Need to add adjacent_nodes and add False value : talk to rest of group about this -@api_view(['GET']) -def get_person_info(request): - """ - Finds a person matching the full name requested by the user, serializes this person, and - returns the serialized person - :param request: request from the user; - request.query_params is a dict {'full name': FULL NAME OF RELEVANT PERSON} - :return: serialized person matching full name imbain request - """ - queryset = Person.objects.filter(full_name=request.query_params['full_name']) - if not list(queryset): - full_name = request.query_params['full_name'] - new = Person( - last="", - first="", - middle="", - full_name=full_name + " not available.", - most_likely_org="", - positions=json.dumps(Counter()), - aliases=json.dumps(Counter()), - count=0) - serializer = PersonInfoSerializer(instance=new, many=False) - return Response(serializer.data) - else: - serializer = PersonInfoSerializer(instance=queryset, many=True) - return Response(serializer.data) + return JsonResponse(data) From 9acdf0399e21068b9f9b6a3c7f982df9a033037d Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Fri, 6 Dec 2019 14:17:52 -0500 Subject: [PATCH 18/27] Changing Person to DjangoPerson DjangoPerson is the model we're using now (name changed), so just updating the code to reflect that --- backend/apps/main/serializers.py | 12 ++++++------ backend/apps/main/tests.py | 18 ++++++++---------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/backend/apps/main/serializers.py b/backend/apps/main/serializers.py index 810c2ab..d7f620c 100644 --- a/backend/apps/main/serializers.py +++ b/backend/apps/main/serializers.py @@ -6,7 +6,7 @@ import json from pathlib import Path from rest_framework import serializers -from .models import Person +from .models import DjangoPerson class EdgeSerializer(serializers.Serializer): @@ -28,10 +28,10 @@ def update(self, instance, validated_data): class PersonInfoSerializer(serializers.ModelSerializer): """ - Serializer for the Person Info model + Serializer for the DjangoPerson Info model """ class Meta: - model = Person + model = DjangoPerson fields = ["last", "first", "middle", "full_name", "most_likely_org", "positions", "aliases", "count"] @@ -44,12 +44,12 @@ def update(self, instance, validated_data): class Edge: """ - Python object to store edges, should be replaced if the Person class is replaced + Python object to store edges, should be replaced if the DjangoPerson class is replaced """ def __init__(self, pk: int, node1: str, node2: str, # pylint: disable-msg=R0913 docs: int, words: int): self.pk = pk # pylint: disable=C0103 - self.node1 = node1 # could be replaced with Person + self.node1 = node1 # could be replaced with DjangoPerson self.node2 = node2 self.docs = docs self.words = words @@ -87,5 +87,5 @@ def load_network_json_data(return_type: str): name = person_dict.get('name') docs = person_dict.get('docs') words = int(person_dict.get('words')) - people.append(Person(pk, name, docs, words)) + people.append(DjangoPerson(pk, name, docs, words)) return people diff --git a/backend/apps/main/tests.py b/backend/apps/main/tests.py index 86417b7..9eb2ab8 100644 --- a/backend/apps/main/tests.py +++ b/backend/apps/main/tests.py @@ -24,7 +24,7 @@ class MainTests(TestCase): def setUp(self): - Person.objects.create( + DjangoPerson.objects.create( last="LAB", first="MIT", middle="DH", @@ -33,7 +33,7 @@ def setUp(self): positions=json.dumps(Counter()), aliases=json.dumps(Counter()), count=5) - Person.objects.create( + DjangoPerson.objects.create( last="LAR", first="MOT", middle="DJ", @@ -42,7 +42,7 @@ def setUp(self): positions=json.dumps(Counter()), aliases=json.dumps(Counter()), count=3) - Person.objects.create( + DjangoPerson.objects.create( last="BOBBERT", first="BOB", middle="BOBSON", @@ -54,8 +54,8 @@ def setUp(self): self.factory = APIRequestFactory() def test_models_01(self): - dummy_1 = Person.objects.get(last='LAB') - dummy_2 = Person.objects.get(last='LAR') + dummy_1 = DjangoPerson.objects.get(last='LAB') + dummy_2 = DjangoPerson.objects.get(last='LAR') s = f'{dummy_1.first} {dummy_1.middle} {dummy_1.last}' s = s + ", Position: " + str(dummy_1.positions) + ", Aliases: " + \ str(dummy_1.aliases) + ", count: " + str(dummy_1.count) @@ -73,7 +73,7 @@ def test_api_views_01(self): database""" self.factory = APIRequestFactory() request = self.factory.get('/api/person_info/', {'full_name': 'BOB BOBSON BOBBERT'}) - dummy_1 = Person.objects.filter(full_name='BOB BOBSON BOBBERT') + dummy_1 = DjangoPerson.objects.filter(full_name='BOB BOBSON BOBBERT') serializer = PersonInfoSerializer(instance=dummy_1, many=True) expected = Response(serializer.data) result = get_person_info(request) @@ -82,7 +82,7 @@ def test_api_views_01(self): def test_api_views_02(self): """tests that get_person_info returns correct person info when the person is NOT in the database""" - Person.objects.create( + DjangoPerson.objects.create( last="", first="", middle="", @@ -92,15 +92,13 @@ def test_api_views_02(self): aliases=json.dumps(Counter()), count=0) request = self.factory.get('/api/person_info/', {'full_name': 'JANE DOE DEERE'}) - dummy_1 = Person.objects.get(full_name='JANE DOE DEERE not available.') + dummy_1 = DjangoPerson.objects.get(full_name='JANE DOE DEERE not available.') serializer = PersonInfoSerializer(instance=dummy_1, many=False) expected = Response(serializer.data) result = get_person_info(request) self.assertEqual(expected.data, result.data) - - class ModelsTests(TestCase): """ Tests import methods in models.py From 38ba2099d87908479e9fea37b153cc12a0de1e90 Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Fri, 6 Dec 2019 14:22:02 -0500 Subject: [PATCH 19/27] Updating DjangoPerson and pasting in api_views pasting back in what we deleted in order to merge correctly --- backend/apps/main/tests.py | 1 - backend/apps/main/views.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/backend/apps/main/tests.py b/backend/apps/main/tests.py index 9eb2ab8..d4c934c 100644 --- a/backend/apps/main/tests.py +++ b/backend/apps/main/tests.py @@ -8,7 +8,6 @@ from rest_framework.test import APIRequestFactory from rest_framework.response import Response from .serializers import PersonInfoSerializer -from .models import Person from .views import get_person_info import os from pathlib import Path diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index 6ef5003..461eeac 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -5,8 +5,9 @@ from pathlib import Path import random - +from rest_framework.decorators import api_view from django.http import JsonResponse +from rest_framework.response import Response from backend.config.settings.base import BACKEND_DIR @@ -51,3 +52,31 @@ def get_network_data(request): # TODO: Need to add adjacent_nodes and add False value : talk to rest of group about this return JsonResponse(data) + + +@api_view(['GET']) +def get_person_info(request): + """ + Finds a person matching the full name requested by the user, serializes this person, and + returns the serialized person + :param request: request from the user; + request.query_params is a dict {'full name': FULL NAME OF RELEVANT PERSON} + :return: serialized person matching full name imbain request + """ + queryset = DjangoPerson.objects.filter(full_name=request.query_params['full_name']) + if not list(queryset): + full_name = request.query_params['full_name'] + new = DjangoPerson( + last="", + first="", + middle="", + full_name=full_name + " not available.", + most_likely_org="", + positions=json.dumps(Counter()), + aliases=json.dumps(Counter()), + count=0) + serializer = PersonInfoSerializer(instance=new, many=False) + return Response(serializer.data) + else: + serializer = PersonInfoSerializer(instance=queryset, many=True) + return Response(serializer.data) From b3531c0e62ca2e13e4db0c7fe3ddf8d12132f12f Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Fri, 6 Dec 2019 14:23:14 -0500 Subject: [PATCH 20/27] Imported counter --- backend/apps/main/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index 461eeac..ec5bd6b 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -5,6 +5,7 @@ from pathlib import Path import random +from collections import Counter from rest_framework.decorators import api_view from django.http import JsonResponse from rest_framework.response import Response From 0ed1ccfe36cb7b938610030cecdabe2cd2a0f4ba Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Fri, 6 Dec 2019 14:23:36 -0500 Subject: [PATCH 21/27] importing DjangoPerson --- backend/apps/main/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index ec5bd6b..058d038 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -8,6 +8,7 @@ from collections import Counter from rest_framework.decorators import api_view from django.http import JsonResponse +from .models import DjangoPerson from rest_framework.response import Response from backend.config.settings.base import BACKEND_DIR From 2ef991c0bce85477406901f9844dfb088acdc70a Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Fri, 6 Dec 2019 14:26:15 -0500 Subject: [PATCH 22/27] importing personinfoserializers --- backend/apps/main/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index 058d038..4ed7bc3 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -9,6 +9,7 @@ from rest_framework.decorators import api_view from django.http import JsonResponse from .models import DjangoPerson +from .serializers import import PersonInfoSerializer from rest_framework.response import Response from backend.config.settings.base import BACKEND_DIR From da57436c803c1a47b1e05de55cf2100615c6721c Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Fri, 6 Dec 2019 14:28:19 -0500 Subject: [PATCH 23/27] import personinfoserializer --- backend/apps/main/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index 4ed7bc3..838eacf 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -9,7 +9,7 @@ from rest_framework.decorators import api_view from django.http import JsonResponse from .models import DjangoPerson -from .serializers import import PersonInfoSerializer +from .serializers import PersonInfoSerializer from rest_framework.response import Response from backend.config.settings.base import BACKEND_DIR From 00fd55524760a41ed8c16d79bced2841f154129a Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Fri, 6 Dec 2019 14:40:57 -0500 Subject: [PATCH 24/27] Fixed pylint issues --- backend/apps/main/tests.py | 33 +++++++++++++++++++-------------- backend/apps/main/views.py | 4 ++-- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/backend/apps/main/tests.py b/backend/apps/main/tests.py index d4c934c..5aa11a8 100644 --- a/backend/apps/main/tests.py +++ b/backend/apps/main/tests.py @@ -1,27 +1,31 @@ """ Tests for the main app. """ -import django + import json -from django.test import TestCase from collections import Counter +from pathlib import Path +import os +from django.test import TestCase +import django from rest_framework.test import APIRequestFactory from rest_framework.response import Response -from .serializers import PersonInfoSerializer -from .views import get_person_info -import os -from pathlib import Path -from name_disambiguation.people_db import PeopleDatabase from name_disambiguation.config import DATA_PATH -from apps.main.models import DjangoPerson +from name_disambiguation.people_db import PeopleDatabase from apps.main.models import Document +from apps.main.models import DjangoPerson from apps.main.models import import_peopledb_to_person_model from apps.main.models import import_csv_to_document_model +from .views import get_person_info +from .serializers import PersonInfoSerializer os.environ.setdefault("DJANGO_SETTINGS_MODULE", __file__) django.setup() class MainTests(TestCase): + """Test that models creates people correctly, + get_person_info returns correct info when person is in the database + get_person_info returns correct message when person is NOT in the database""" def setUp(self): DjangoPerson.objects.create( last="LAB", @@ -53,19 +57,20 @@ def setUp(self): self.factory = APIRequestFactory() def test_models_01(self): + """Tests that the person created with DjangoPerson model has the correct attributes""" dummy_1 = DjangoPerson.objects.get(last='LAB') dummy_2 = DjangoPerson.objects.get(last='LAR') - s = f'{dummy_1.first} {dummy_1.middle} {dummy_1.last}' - s = s + ", Position: " + str(dummy_1.positions) + ", Aliases: " + \ + s_1 = f'{dummy_1.first} {dummy_1.middle} {dummy_1.last}' + s_1 = s_1 + ", Position: " + str(dummy_1.positions) + ", Aliases: " + \ str(dummy_1.aliases) + ", count: " + str(dummy_1.count) - s2 = f'{dummy_2.first} {dummy_2.middle} {dummy_2.last}' - s2 = s2 + ", Position: " + str(dummy_2.positions) + ", Aliases: " + \ + s_2 = f'{dummy_2.first} {dummy_2.middle} {dummy_2.last}' + s_2 = s_2 + ", Position: " + str(dummy_2.positions) + ", Aliases: " + \ str(dummy_2.aliases) + ", count: " + str(dummy_2.count) self.assertEqual( - str(dummy_1), s) + str(dummy_1), s_1) self.assertEqual( - str(dummy_2), s2) + str(dummy_2), s_2) def test_api_views_01(self): """tests that get_person_info returns correct person info when the person is in the diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index 838eacf..16153aa 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -7,11 +7,11 @@ import random from collections import Counter from rest_framework.decorators import api_view +from rest_framework.response import Response from django.http import JsonResponse +from backend.config.settings.base import BACKEND_DIR from .models import DjangoPerson from .serializers import PersonInfoSerializer -from rest_framework.response import Response -from backend.config.settings.base import BACKEND_DIR def get_network_data(request): From 52659e7d2f313547e18a6d92b354664032e32126 Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Fri, 6 Dec 2019 15:46:59 -0500 Subject: [PATCH 25/27] Added list_edges Forgot to add it back; now it's back! --- backend/apps/main/views.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index 16153aa..c0f8e2d 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -11,7 +11,16 @@ from django.http import JsonResponse from backend.config.settings.base import BACKEND_DIR from .models import DjangoPerson -from .serializers import PersonInfoSerializer +from .serializers import PersonInfoSerializer, EdgeSerializer + + +@api_view(['GET']) +def list_edges(request): + """ + Return a list of all Edge objects, serialized. + """ + serializer = EdgeSerializer(instance=load_network_json_data("edges"), many=True) + return Response(serializer.data) def get_network_data(request): From 8428734154f6307891aaaa9f548db71d2a59e388 Mon Sep 17 00:00:00 2001 From: Raquel Garcia Date: Fri, 6 Dec 2019 17:07:42 -0500 Subject: [PATCH 26/27] We made migrations Everything is working!!!! --- backend/apps/main/migrations/0001_initial.py | 18 ++++++++- .../migrations/0002_auto_20191108_2333.py | 36 ------------------ .../migrations/0002_auto_20191114_1533.py | 31 --------------- .../migrations/0003_auto_20191112_1537.py | 18 --------- .../main/migrations/0003_person_full_name.py | 19 --------- .../migrations/0004_auto_20191122_2141.py | 18 --------- .../migrations/0004_djangoperson_full_name.py | 18 --------- .../migrations/0005_auto_20191117_2156.py | 18 --------- .../migrations/0006_auto_20191121_1535.py | 33 ---------------- backend/apps/main/tests.py | 4 +- data/django/test_peopledb.pickle | Bin 691 -> 691 bytes 11 files changed, 19 insertions(+), 194 deletions(-) delete mode 100644 backend/apps/main/migrations/0002_auto_20191108_2333.py delete mode 100644 backend/apps/main/migrations/0002_auto_20191114_1533.py delete mode 100644 backend/apps/main/migrations/0003_auto_20191112_1537.py delete mode 100644 backend/apps/main/migrations/0003_person_full_name.py delete mode 100644 backend/apps/main/migrations/0004_auto_20191122_2141.py delete mode 100644 backend/apps/main/migrations/0004_djangoperson_full_name.py delete mode 100644 backend/apps/main/migrations/0005_auto_20191117_2156.py delete mode 100644 backend/apps/main/migrations/0006_auto_20191121_1535.py diff --git a/backend/apps/main/migrations/0001_initial.py b/backend/apps/main/migrations/0001_initial.py index 0ef292b..bfb2971 100644 --- a/backend/apps/main/migrations/0001_initial.py +++ b/backend/apps/main/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.4 on 2019-10-25 20:15 +# Generated by Django 2.2.4 on 2019-12-06 22:05 from django.db import migrations, models @@ -11,6 +11,20 @@ class Migration(migrations.Migration): ] operations = [ + migrations.CreateModel( + name='DjangoPerson', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('last', models.CharField(max_length=250)), + ('first', models.CharField(max_length=250)), + ('middle', models.CharField(max_length=250)), + ('full_name', models.CharField(max_length=250, unique=True)), + ('most_likely_org', models.CharField(max_length=250)), + ('positions', models.TextField()), + ('aliases', models.TextField()), + ('count', models.IntegerField()), + ], + ), migrations.CreateModel( name='Document', fields=[ @@ -30,6 +44,8 @@ class Migration(migrations.Migration): ('text', models.TextField(blank=True)), ('tid', models.CharField(max_length=250, unique=True)), ('title', models.CharField(blank=True, max_length=250)), + ('authors', models.ManyToManyField(related_name='document_by_authors', to='main.DjangoPerson')), + ('recipients', models.ManyToManyField(related_name='document_by_recipients', to='main.DjangoPerson')), ], ), ] diff --git a/backend/apps/main/migrations/0002_auto_20191108_2333.py b/backend/apps/main/migrations/0002_auto_20191108_2333.py deleted file mode 100644 index ac22798..0000000 --- a/backend/apps/main/migrations/0002_auto_20191108_2333.py +++ /dev/null @@ -1,36 +0,0 @@ -# Generated by Django 2.2.4 on 2019-11-08 23:33 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('main', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='DjangoPerson', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('last', models.CharField(max_length=255)), - ('first', models.CharField(max_length=255)), - ('middle', models.CharField(max_length=255)), - ('position', models.CharField(max_length=250)), - ('positions', models.TextField()), - ('aliases', models.TextField()), - ('count', models.IntegerField()), - ], - ), - migrations.AddField( - model_name='document', - name='authors', - field=models.ManyToManyField(related_name='document_by_authors', to='main.DjangoPerson'), - ), - migrations.AddField( - model_name='document', - name='recipients', - field=models.ManyToManyField(related_name='document_by_recipients', to='main.DjangoPerson'), - ), - ] diff --git a/backend/apps/main/migrations/0002_auto_20191114_1533.py b/backend/apps/main/migrations/0002_auto_20191114_1533.py deleted file mode 100644 index 96e6766..0000000 --- a/backend/apps/main/migrations/0002_auto_20191114_1533.py +++ /dev/null @@ -1,31 +0,0 @@ -# Generated by Django 2.2.4 on 2019-11-14 15:33 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('main', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='Person', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('last', models.CharField(max_length=255)), - ('first', models.CharField(max_length=255)), - ('middle', models.CharField(max_length=255)), - ('most_likely_org', models.CharField(max_length=250)), - ('positions', models.TextField()), - ('aliases', models.TextField()), - ('count', models.IntegerField()), - ], - ), - migrations.AddField( - model_name='document', - name='authors', - field=models.ManyToManyField(to='main.Person'), - ), - ] diff --git a/backend/apps/main/migrations/0003_auto_20191112_1537.py b/backend/apps/main/migrations/0003_auto_20191112_1537.py deleted file mode 100644 index ded5608..0000000 --- a/backend/apps/main/migrations/0003_auto_20191112_1537.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.4 on 2019-11-12 15:37 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('main', '0002_auto_20191108_2333'), - ] - - operations = [ - migrations.RenameField( - model_name='djangoperson', - old_name='position', - new_name='most_likely_org', - ), - ] diff --git a/backend/apps/main/migrations/0003_person_full_name.py b/backend/apps/main/migrations/0003_person_full_name.py deleted file mode 100644 index 85b5b95..0000000 --- a/backend/apps/main/migrations/0003_person_full_name.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 2.2.4 on 2019-11-14 16:19 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('main', '0002_auto_20191114_1533'), - ] - - operations = [ - migrations.AddField( - model_name='person', - name='full_name', - field=models.CharField(default=None, max_length=255), - preserve_default=False, - ), - ] diff --git a/backend/apps/main/migrations/0004_auto_20191122_2141.py b/backend/apps/main/migrations/0004_auto_20191122_2141.py deleted file mode 100644 index 35a7e22..0000000 --- a/backend/apps/main/migrations/0004_auto_20191122_2141.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.4 on 2019-11-22 21:41 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('main', '0003_person_full_name'), - ] - - operations = [ - migrations.AlterField( - model_name='person', - name='full_name', - field=models.CharField(max_length=255, unique=True), - ), - ] diff --git a/backend/apps/main/migrations/0004_djangoperson_full_name.py b/backend/apps/main/migrations/0004_djangoperson_full_name.py deleted file mode 100644 index 6478e13..0000000 --- a/backend/apps/main/migrations/0004_djangoperson_full_name.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.4 on 2019-11-15 20:38 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('main', '0003_auto_20191112_1537'), - ] - - operations = [ - migrations.AddField( - model_name='djangoperson', - name='full_name', - field=models.CharField(default=None, max_length=255), - ), - ] diff --git a/backend/apps/main/migrations/0005_auto_20191117_2156.py b/backend/apps/main/migrations/0005_auto_20191117_2156.py deleted file mode 100644 index e30c9ef..0000000 --- a/backend/apps/main/migrations/0005_auto_20191117_2156.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.4 on 2019-11-17 21:56 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('main', '0004_djangoperson_full_name'), - ] - - operations = [ - migrations.AlterField( - model_name='djangoperson', - name='full_name', - field=models.CharField(default=None, max_length=255, unique=True), - ), - ] diff --git a/backend/apps/main/migrations/0006_auto_20191121_1535.py b/backend/apps/main/migrations/0006_auto_20191121_1535.py deleted file mode 100644 index 9575311..0000000 --- a/backend/apps/main/migrations/0006_auto_20191121_1535.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 2.2.4 on 2019-11-21 15:35 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('main', '0005_auto_20191117_2156'), - ] - - operations = [ - migrations.AlterField( - model_name='djangoperson', - name='first', - field=models.CharField(max_length=250), - ), - migrations.AlterField( - model_name='djangoperson', - name='full_name', - field=models.CharField(max_length=250, unique=True), - ), - migrations.AlterField( - model_name='djangoperson', - name='last', - field=models.CharField(max_length=250), - ), - migrations.AlterField( - model_name='djangoperson', - name='middle', - field=models.CharField(max_length=250), - ), - ] diff --git a/backend/apps/main/tests.py b/backend/apps/main/tests.py index 5aa11a8..849e8a8 100644 --- a/backend/apps/main/tests.py +++ b/backend/apps/main/tests.py @@ -61,11 +61,11 @@ def test_models_01(self): dummy_1 = DjangoPerson.objects.get(last='LAB') dummy_2 = DjangoPerson.objects.get(last='LAR') s_1 = f'{dummy_1.first} {dummy_1.middle} {dummy_1.last}' - s_1 = s_1 + ", Position: " + str(dummy_1.positions) + ", Aliases: " + \ + s_1 = s_1 + ", Positions: " + str(dummy_1.positions) + ", Aliases: " + \ str(dummy_1.aliases) + ", count: " + str(dummy_1.count) s_2 = f'{dummy_2.first} {dummy_2.middle} {dummy_2.last}' - s_2 = s_2 + ", Position: " + str(dummy_2.positions) + ", Aliases: " + \ + s_2 = s_2 + ", Positions: " + str(dummy_2.positions) + ", Aliases: " + \ str(dummy_2.aliases) + ", count: " + str(dummy_2.count) self.assertEqual( str(dummy_1), s_1) diff --git a/data/django/test_peopledb.pickle b/data/django/test_peopledb.pickle index 62e8e4cf2fe4009dcbd432162701ea3c02f4b4e7..89690795f32ae82b042e18cd7d2a90193469086f 100644 GIT binary patch delta 391 zcmYL^%T5A85JeeYqDb(8FGL+d5fl;Mx?klup9G>Oq%hglNhV)hcg#PAPp z{TtWzv=iOks#{g()+8_qypNN9kH>S{Gz`qJfRK;9SF?{Wn-aQJA*52QjXq)u8GW{f zHewV@(rQ2T+Gf|pGG7>ACBvfbm7MA>sRXOKpN0PW1Z%q2vJt0Pm#Bvq>$#7F9<(|< zn0>>>0Gk;;&4Oa5p%u$b4O=|S6j7F-^2kt14ck1*=lHxsPBO1>s}w2SXWH1I*yS=( zn$7-QQA?UCM)u@Lh9b)oy3b|q4{+cxo!1*$O|K*8BJXg>Ba%pvm&AfZ91!NqGE90KpjO96mh}zh}aOA(k3IwTtP5Hf|}QYH$pu0uHmQfFZ@+b zTh!Zp@3%b9i~^&;=P(=ecsvy~SE#C(;sIGawcpGxrny(X^9XZ$;JMvwc3PNmLiOvr zHsFekOAL$DXIfZdSmqNlsdw6Svt?pM%=fX%!!mM@le|jg_?k1m&JcB2kFUL#E@ITz zYlvX#0~>v8CdH)FtEgpaU`x0r_aMT>YuFZ%te6#ZHe-h`NJidxDC&iZp_2r;tujp( zh|`cX+hy1jj!j}{C<*WW@Lh=yl9Fg4#c&{EG&F`E_K~*XetGb&s_QtCZ2MESqNWvd zWn}&gjz#2G;Kafy! Date: Tue, 10 Dec 2019 10:29:22 -0500 Subject: [PATCH 27/27] fixed pylint and django backend test --- .github/workflows/ci.yml | 4 ++-- backend/apps/main/tests.py | 8 ++++---- backend/apps/main/views.py | 2 +- data/django/test_peopledb.pickle | Bin 691 -> 691 bytes 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b3c1cb..17a0ae7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: - name: Run Django tests run: | cd backend - python manage.py test + python manage.py test apps.main analysis_tests: runs-on: ubuntu-latest @@ -63,7 +63,7 @@ jobs: run: | pip install -r requirements.txt python setup.py develop - + - name: Run analysis tests run: | cd analysis diff --git a/backend/apps/main/tests.py b/backend/apps/main/tests.py index 849e8a8..12559ed 100644 --- a/backend/apps/main/tests.py +++ b/backend/apps/main/tests.py @@ -12,10 +12,10 @@ from rest_framework.response import Response from name_disambiguation.config import DATA_PATH from name_disambiguation.people_db import PeopleDatabase -from apps.main.models import Document -from apps.main.models import DjangoPerson -from apps.main.models import import_peopledb_to_person_model -from apps.main.models import import_csv_to_document_model +from .models import Document +from .models import DjangoPerson +from .models import import_peopledb_to_person_model +from .models import import_csv_to_document_model from .views import get_person_info from .serializers import PersonInfoSerializer os.environ.setdefault("DJANGO_SETTINGS_MODULE", __file__) diff --git a/backend/apps/main/views.py b/backend/apps/main/views.py index c0f8e2d..0245ea1 100644 --- a/backend/apps/main/views.py +++ b/backend/apps/main/views.py @@ -12,7 +12,7 @@ from backend.config.settings.base import BACKEND_DIR from .models import DjangoPerson from .serializers import PersonInfoSerializer, EdgeSerializer - +from .serializers import load_network_json_data @api_view(['GET']) def list_edges(request): diff --git a/data/django/test_peopledb.pickle b/data/django/test_peopledb.pickle index 89690795f32ae82b042e18cd7d2a90193469086f..1d3939ae323ad3039431d276318fb6f856c47467 100644 GIT binary patch delta 386 zcmYL_OHaZ;6opH9h@!!_qN1ZHf+D_Amxv9)DQz-Roh=y5kPywW)Eyx%y4Ubii2u)N zX>@mP?z!hXxufVP`ZY{P6-8-ib!uXYd-|pGZugMjA*mm}f4U{UAXUYDtQpmYX^ z*98)s5gq$9!sE(d94i+(a#WR7@(c$eO=CfU!#)aLftZYFXyvM<;Ye2S9#BCsfNMDZ who5j&F7FvmX;{ACjNyD-R delta 355 zcmYMvNl(H+90l+Wn^iC_K@dc#-~x)cZ%2ZS!6}%GAakV#GbBXwEIoK=jECMe{URoO zKpkq0w>LBYdB2%)bR7Ng3K2z7t~#w2W_VP_$H9v=LYz;@+-(>}qt^WV8fIK18J7Ir z-f+;f`W8}R(Lvh#CaJe&8sTctQ_;ExGBjkn$TF<>r4P^c(+D{lvrQ2zzhl+GT1iav zn7q^0Yu6nec@gJCF6+?5teR