-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
0 parents
commit 56333fd
Showing
9 changed files
with
940 additions
and
0 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
from sqlite3 import connect | ||
|
||
DATABASE_NAME = './mirror/mirror.db' | ||
START_URL = "https://ratsinformation.leipzig.de/allris_leipzig_public/oparl/papers?body=2387&page=1" | ||
|
||
|
||
class FakeResponse: | ||
def __init__(self, state, content=None): | ||
self.state = state | ||
if content: | ||
self.content = content | ||
|
||
|
||
def get(url): | ||
with connect(DATABASE_NAME) as con: | ||
res = con.execute('SELECT data FROM mirror WHERE id=?', (url, )) | ||
data = res.fetchone() | ||
if data: | ||
return FakeResponse(state=200, content=data[0]) | ||
else: | ||
return FakeResponse(state=400) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
from configparser import ConfigParser | ||
from neo4j import GraphDatabase, Session | ||
from contextlib import contextmanager | ||
|
||
|
||
def neo4j_config(): | ||
config = ConfigParser() | ||
config.read('config') | ||
section = config['Neo4j'] | ||
return dict(uri=section['NEO4J_URI'], | ||
user=section['NEO4J_USERNAME'], | ||
password=section['NEO4J_PASSWORD']) | ||
|
||
|
||
class Database: | ||
def __init__(self, uri, user, password): | ||
self.driver = GraphDatabase.driver(uri, auth=(user, password)) | ||
self.driver.verify_connectivity() | ||
self.session = self.driver.session | ||
|
||
|
||
@contextmanager | ||
def database_connection(): | ||
database = Database(**neo4j_config()) | ||
exception: (Exception, None) = None | ||
try: | ||
yield database | ||
except Warning as w: | ||
print(w) | ||
except Exception as e: | ||
exception = e | ||
finally: | ||
database.driver.close() | ||
if exception: | ||
raise exception | ||
|
||
if __name__ == '__main__': | ||
database = Database(**neo4j_config()) | ||
res = database.driver.execute_query('match (n) return count(n)') | ||
exit() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
from neo4j.graph import Node | ||
|
||
from nodes_scheme import \ | ||
ATTRIBUTES, \ | ||
AbcNodeInterface, \ | ||
AbcOparlPaperInterface, \ | ||
AbcOparlPersonInterface, \ | ||
AbcOparlOrganizationInterface, \ | ||
AbcOparlLocationInterface | ||
|
||
|
||
class Paper(AbcOparlPaperInterface): | ||
_content: Node | ||
|
||
@ATTRIBUTES.OPARL_ID.as_primary | ||
def oparl_id(self): | ||
return self._content.get('oparl_id') | ||
|
||
@ATTRIBUTES.MODIFIED | ||
def modified(self): | ||
return self._content.get('modified') | ||
|
||
@ATTRIBUTES.REFERENCE.as_primary | ||
def reference(self): | ||
return self._content.get('reference') | ||
|
||
@ATTRIBUTES.PAPER_TYPE | ||
def paper_type(self): | ||
return self._content.get('paper_type') | ||
|
||
@ATTRIBUTES.WEB_URL | ||
def web_url(self): | ||
return self._content.get('web_url') | ||
|
||
@ATTRIBUTES.ORIGIN_DATE | ||
def origin_date(self): | ||
return self._content.get('origin_date') | ||
|
||
def directors(self): | ||
pass | ||
|
||
|
||
factory_mapping = [Paper, | ||
AbcOparlPersonInterface, | ||
AbcOparlOrganizationInterface, | ||
AbcOparlLocationInterface] | ||
|
||
|
||
def node_factory(result): | ||
for cls in factory_mapping: | ||
if cls._labels == list(result.labels): | ||
obj = cls(result) | ||
return obj |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,164 @@ | ||
from oparl_objects import BasicOparl as OparlBasic | ||
from oparl_objects import Paper as OparlPaper | ||
from oparl_objects import Person as OparlPerson | ||
from oparl_objects import Organization as OparlOrganization | ||
from oparl_objects import Location as OparlLocation | ||
from oparl_objects import oparl_factory as oparl_factory | ||
import fakerequest as request | ||
import json | ||
|
||
from nodes_scheme import \ | ||
RELATIONS, \ | ||
ATTRIBUTES, \ | ||
AbcNodeInterface, \ | ||
AbcOparlPaperInterface, \ | ||
AbcOparlPersonInterface, \ | ||
AbcOparlOrganizationInterface, \ | ||
AbcOparlLocationInterface | ||
|
||
|
||
class UnknownOparlNode(AbcNodeInterface): | ||
_content: OparlBasic | ||
_labels = [] | ||
|
||
@ATTRIBUTES.OPARL_ID.as_primary | ||
def oparl_id(self): | ||
return self._content.oparl_id | ||
|
||
|
||
def converted_get_request(node: UnknownOparlNode): | ||
url = node.oparl_id.value() | ||
response = request.get(url) | ||
if response.state == 200: | ||
content = json.loads(response.content) | ||
return node_factory(oparl_factory(content)) | ||
|
||
|
||
class OparlPaperNode(AbcOparlPaperInterface): | ||
_content: OparlPaper | ||
|
||
@ATTRIBUTES.OPARL_ID.as_primary | ||
def oparl_id(self): | ||
return self._content.oparl_id | ||
|
||
@ATTRIBUTES.MODIFIED | ||
def modified(self): | ||
return self._content.modified | ||
|
||
@ATTRIBUTES.REFERENCE.as_primary | ||
def reference(self): | ||
return self._content.reference | ||
|
||
@ATTRIBUTES.PAPER_TYPE | ||
def paper_type(self): | ||
return self._content.paper_type | ||
|
||
@ATTRIBUTES.WEB_URL | ||
def web_url(self): | ||
return self._content.web_url | ||
|
||
@ATTRIBUTES.ORIGIN_DATE | ||
def origin_date(self): | ||
return self._content.origin_date | ||
|
||
@RELATIONS.DIRECTED.as_generator | ||
def directors(self): | ||
for director in self._content.under_direction_of: | ||
if isinstance(director, UnknownOparlNode): | ||
director = converted_get_request(director) | ||
yield node_factory(director), self | ||
|
||
@RELATIONS.SUBMITTED.as_generator | ||
def originators(self): | ||
for originator in self._content.originator_persons: | ||
if isinstance(originator, UnknownOparlNode): | ||
originator = converted_get_request(originator) | ||
yield node_factory(originator), self | ||
|
||
|
||
#_content.consultations | ||
|
||
|
||
class OparlPersonNode(AbcOparlPersonInterface): | ||
_content: OparlPerson | ||
|
||
@ATTRIBUTES.OPARL_ID.as_primary | ||
def oparl_id(self): | ||
return self._content.oparl_id | ||
|
||
@ATTRIBUTES.MODIFIED | ||
def modified(self): | ||
return self._content.modified | ||
|
||
@ATTRIBUTES.NAME | ||
def name(self): | ||
return self._content.name | ||
|
||
@ATTRIBUTES.WEB_URL | ||
def web_url(self): | ||
return self._content.web_url | ||
|
||
|
||
class OparlOrganizationNode(AbcOparlOrganizationInterface): | ||
_content: OparlOrganization | ||
|
||
@ATTRIBUTES.OPARL_ID.as_primary | ||
def oparl_id(self): | ||
return self._content.oparl_id | ||
|
||
@ATTRIBUTES.MODIFIED | ||
def modified(self): | ||
return self._content.modified | ||
|
||
@ATTRIBUTES.NAME | ||
def name(self): | ||
return self._content.name | ||
|
||
@ATTRIBUTES.START_DATE | ||
def start_date(self): | ||
return self._content.start_date | ||
|
||
@ATTRIBUTES.END_DATE | ||
def end_date(self): | ||
return self._content.end_date | ||
|
||
|
||
class OparlLocationNode(AbcOparlLocationInterface): | ||
_content: OparlLocation | ||
|
||
@ATTRIBUTES.OPARL_ID.as_primary | ||
def oparl_id(self): | ||
return self._content.oparl_id | ||
|
||
@ATTRIBUTES.MODIFIED | ||
def modified(self): | ||
return self._content.modified | ||
|
||
@ATTRIBUTES.LOCALITY | ||
def locality(self): | ||
return self._content.locality | ||
|
||
@ATTRIBUTES.POSTAL_CODE | ||
def postal_code(self): | ||
return self._content.postal_code | ||
|
||
@ATTRIBUTES.DESCRIPTION | ||
def description(self): | ||
return self._content.description | ||
|
||
@ATTRIBUTES.STREET_ADDRESS | ||
def street_address(self): | ||
return self._content.street_address | ||
|
||
|
||
factory_mapping = {OparlPerson: OparlPersonNode, | ||
OparlPaper: OparlPaperNode, | ||
OparlOrganization: OparlOrganizationNode, | ||
OparlLocation: OparlLocationNode, | ||
OparlBasic: UnknownOparlNode} | ||
|
||
|
||
def node_factory(oparl_obj): | ||
assert isinstance(oparl_obj, OparlBasic) | ||
node_cls = factory_mapping.get(oparl_obj.__class__) | ||
return node_cls(oparl_obj) |
Oops, something went wrong.