initial commit

fakerequest.py

@@ -0,0 +1,21 @@
+from sqlite3 import connect
+
+DATABASE_NAME = './mirror/mirror.db'
+START_URL = "https://ratsinformation.leipzig.de/allris_leipzig_public/oparl/papers?body=2387&page=1"
+
+
+class FakeResponse:
+    def __init__(self, state, content=None):
+        self.state = state
+        if content:
+            self.content = content
+
+
+def get(url):
+    with connect(DATABASE_NAME) as con:
+        res = con.execute('SELECT data FROM mirror WHERE id=?', (url, ))
+        data = res.fetchone()
+        if data:
+            return FakeResponse(state=200, content=data[0])
+        else:
+            return FakeResponse(state=400)

neo_connector.py

@@ -0,0 +1,40 @@
+from configparser import ConfigParser
+from neo4j import GraphDatabase, Session
+from contextlib import contextmanager
+
+
+def neo4j_config():
+    config = ConfigParser()
+    config.read('config')
+    section = config['Neo4j']
+    return dict(uri=section['NEO4J_URI'],
+                user=section['NEO4J_USERNAME'],
+                password=section['NEO4J_PASSWORD'])
+
+
+class Database:
+    def __init__(self, uri, user, password):
+        self.driver = GraphDatabase.driver(uri, auth=(user, password))
+        self.driver.verify_connectivity()
+        self.session = self.driver.session
+
+
+@contextmanager
+def database_connection():
+    database = Database(**neo4j_config())
+    exception: (Exception, None) = None
+    try:
+        yield database
+    except Warning as w:
+        print(w)
+    except Exception as e:
+        exception = e
+    finally:
+        database.driver.close()
+        if exception:
+            raise exception
+
+if __name__ == '__main__':
+    database = Database(**neo4j_config())
+    res = database.driver.execute_query('match (n) return count(n)')
+    exit()

nodes_from_neo4j.py

@@ -0,0 +1,53 @@
+from neo4j.graph import Node
+
+from nodes_scheme import \
+    ATTRIBUTES, \
+    AbcNodeInterface, \
+    AbcOparlPaperInterface, \
+    AbcOparlPersonInterface, \
+    AbcOparlOrganizationInterface, \
+    AbcOparlLocationInterface
+
+
+class Paper(AbcOparlPaperInterface):
+    _content: Node
+
+    @ATTRIBUTES.OPARL_ID.as_primary
+    def oparl_id(self):
+        return self._content.get('oparl_id')
+
+    @ATTRIBUTES.MODIFIED
+    def modified(self):
+        return self._content.get('modified')
+
+    @ATTRIBUTES.REFERENCE.as_primary
+    def reference(self):
+        return self._content.get('reference')
+
+    @ATTRIBUTES.PAPER_TYPE
+    def paper_type(self):
+        return self._content.get('paper_type')
+
+    @ATTRIBUTES.WEB_URL
+    def web_url(self):
+        return self._content.get('web_url')
+
+    @ATTRIBUTES.ORIGIN_DATE
+    def origin_date(self):
+        return self._content.get('origin_date')
+
+    def directors(self):
+        pass
+
+
+factory_mapping = [Paper,
+                   AbcOparlPersonInterface,
+                   AbcOparlOrganizationInterface,
+                   AbcOparlLocationInterface]
+
+
+def node_factory(result):
+    for cls in factory_mapping:
+        if cls._labels == list(result.labels):
+            obj = cls(result)
+            return obj

nodes_from_oparl.py

@@ -0,0 +1,164 @@
+from oparl_objects import BasicOparl as OparlBasic
+from oparl_objects import Paper as OparlPaper
+from oparl_objects import Person as OparlPerson
+from oparl_objects import Organization as OparlOrganization
+from oparl_objects import Location as OparlLocation
+from oparl_objects import oparl_factory as oparl_factory
+import fakerequest as request
+import json
+
+from nodes_scheme import \
+    RELATIONS, \
+    ATTRIBUTES, \
+    AbcNodeInterface, \
+    AbcOparlPaperInterface, \
+    AbcOparlPersonInterface, \
+    AbcOparlOrganizationInterface, \
+    AbcOparlLocationInterface
+
+
+class UnknownOparlNode(AbcNodeInterface):
+    _content: OparlBasic
+    _labels = []
+
+    @ATTRIBUTES.OPARL_ID.as_primary
+    def oparl_id(self):
+        return self._content.oparl_id
+
+
+def converted_get_request(node: UnknownOparlNode):
+    url = node.oparl_id.value()
+    response = request.get(url)
+    if response.state == 200:
+        content = json.loads(response.content)
+        return node_factory(oparl_factory(content))
+
+
+class OparlPaperNode(AbcOparlPaperInterface):
+    _content: OparlPaper
+
+    @ATTRIBUTES.OPARL_ID.as_primary
+    def oparl_id(self):
+        return self._content.oparl_id
+
+    @ATTRIBUTES.MODIFIED
+    def modified(self):
+        return self._content.modified
+
+    @ATTRIBUTES.REFERENCE.as_primary
+    def reference(self):
+        return self._content.reference
+
+    @ATTRIBUTES.PAPER_TYPE
+    def paper_type(self):
+        return self._content.paper_type
+
+    @ATTRIBUTES.WEB_URL
+    def web_url(self):
+        return self._content.web_url
+
+    @ATTRIBUTES.ORIGIN_DATE
+    def origin_date(self):
+        return self._content.origin_date
+
+    @RELATIONS.DIRECTED.as_generator
+    def directors(self):
+        for director in self._content.under_direction_of:
+            if isinstance(director, UnknownOparlNode):
+                director = converted_get_request(director)
+            yield node_factory(director), self
+
+    @RELATIONS.SUBMITTED.as_generator
+    def originators(self):
+        for originator in self._content.originator_persons:
+            if isinstance(originator, UnknownOparlNode):
+                originator = converted_get_request(originator)
+            yield node_factory(originator), self
+
+
+    #_content.consultations
+
+
+class OparlPersonNode(AbcOparlPersonInterface):
+    _content: OparlPerson
+
+    @ATTRIBUTES.OPARL_ID.as_primary
+    def oparl_id(self):
+        return self._content.oparl_id
+
+    @ATTRIBUTES.MODIFIED
+    def modified(self):
+        return self._content.modified
+
+    @ATTRIBUTES.NAME
+    def name(self):
+        return self._content.name
+
+    @ATTRIBUTES.WEB_URL
+    def web_url(self):
+        return self._content.web_url
+
+
+class OparlOrganizationNode(AbcOparlOrganizationInterface):
+    _content: OparlOrganization
+
+    @ATTRIBUTES.OPARL_ID.as_primary
+    def oparl_id(self):
+        return self._content.oparl_id
+
+    @ATTRIBUTES.MODIFIED
+    def modified(self):
+        return self._content.modified
+
+    @ATTRIBUTES.NAME
+    def name(self):
+        return self._content.name
+
+    @ATTRIBUTES.START_DATE
+    def start_date(self):
+        return self._content.start_date
+
+    @ATTRIBUTES.END_DATE
+    def end_date(self):
+        return self._content.end_date
+
+
+class OparlLocationNode(AbcOparlLocationInterface):
+    _content: OparlLocation
+
+    @ATTRIBUTES.OPARL_ID.as_primary
+    def oparl_id(self):
+        return self._content.oparl_id
+
+    @ATTRIBUTES.MODIFIED
+    def modified(self):
+        return self._content.modified
+
+    @ATTRIBUTES.LOCALITY
+    def locality(self):
+        return self._content.locality
+
+    @ATTRIBUTES.POSTAL_CODE
+    def postal_code(self):
+        return self._content.postal_code
+
+    @ATTRIBUTES.DESCRIPTION
+    def description(self):
+        return self._content.description
+
+    @ATTRIBUTES.STREET_ADDRESS
+    def street_address(self):
+        return self._content.street_address
+
+
+factory_mapping = {OparlPerson: OparlPersonNode,
+                   OparlPaper: OparlPaperNode,
+                   OparlOrganization: OparlOrganizationNode,
+                   OparlLocation: OparlLocationNode,
+                   OparlBasic: UnknownOparlNode}
+
+
+def node_factory(oparl_obj):
+    assert isinstance(oparl_obj, OparlBasic)
+    node_cls = factory_mapping.get(oparl_obj.__class__)
+    return node_cls(oparl_obj)