Support subdirectories

quit/conf.py

@@ -414,25 +414,32 @@ def get_blobs_from_repository(self, rev):
             dict: containing names rdf files plus format and oid.
 
         """
+        def find_blobs(tree, prefix=''):
+            # Collect graph files, rdf files and config files
+            for entry in tree:
+                if entry.type == 'blob':
+                    format = guess_format(entry.name)
+                    if format is None and entry.name.endswith('.graph'):
+                        graph_file_blobs[join(prefix, entry.name)] = entry.id
+                    elif format is not None and format == 'nt':
+                        rdf_file_blobs[join(prefix, entry.name)] = (entry.id, format)
+                    elif format is not None and entry.name == 'config.ttl':
+                        config_files.append(str(entry.id))
+                elif entry.type == 'tree':
+                    tree_obj = self.repository[entry.id]
+                    find_blobs(tree_obj, join(prefix, entry.name))
+
         config_files = []
         graph_files = {}
         graph_file_blobs = {}
         rdf_file_blobs = {}
+
         try:
             commit = self.repository.revparse_single(rev)
         except Exception:
             return graph_files, config_files, rdf_file_blobs
 
-        # Collect graph files, rdf files and config files
-        for entry in commit.tree:
-            if entry.type == 'blob':
-                format = guess_format(entry.name)
-                if format is None and entry.name.endswith('.graph'):
-                    graph_file_blobs[entry.name] = entry.id
-                elif format is not None and format == 'nt':
-                    rdf_file_blobs[entry.name] = (entry.id, format)
-                elif format is not None and entry.name == 'config.ttl':
-                    config_files.append(str(entry.id))
+        find_blobs(commit.tree)
 
         # collect pairs of rdf files and graph files
         for filename in rdf_file_blobs.keys():

tests/helpers.py

@@ -1,6 +1,6 @@
 from tempfile import TemporaryDirectory
 from pygit2 import init_repository, clone_repository, Signature
-from os import path, walk
+from os import path, walk, makedirs
 from os.path import join
 from rdflib import Graph
 from urllib.parse import quote_plus
@@ -156,7 +156,7 @@ def withNoConfigInformation(self):
 
         return tmpRepo
 
-    def withGraphs(self, graphUriContentDict, mode='graphfiles'):
+    def withGraphs(self, graphUriContentDict, mode='graphfiles', subDirectory=False):
         """Give a TemporaryRepository() initialized with a dictionary of graphUris and content (nt)."""
         uristring = ''
         configFileContent = """@base <http://quit.aksw.org/vocab/> .
@@ -176,29 +176,41 @@ def withGraphs(self, graphUriContentDict, mode='graphfiles'):
         index.read()
 
         i = 0
+
         for graphUri, graphContent in sorted(graphUriContentDict.items()):
+            subdir = ''
+
+            if subDirectory:
+                subdir = 'sub{}'.format(i)
+                abs_subdir = path.join(tmpRepo.repo.workdir, subdir)
+                makedirs(abs_subdir, exist_ok=True)
+
             filename = 'graph_{}.nt'.format(i)
-            with open(path.join(tmpRepo.repo.workdir, filename), "w") as graphFile:
+
+            with open(path.join(tmpRepo.repo.workdir, subdir, filename), "w") as graphFile:
                 if graphContent:
                     graphFile.write(graphContent)
 
             if mode == 'graphfiles':
                 # Set Graph URI to http://example.org/
-                with open(path.join(tmpRepo.repo.workdir, filename + ".graph"), "w") as graphFile:
+                with open(path.join(tmpRepo.repo.workdir, subdir, filename + ".graph"), "w") as graphFile:
                     graphFile.write(graphUri)
-                index.add(filename + '.graph')
+
+                index.add(path.join(subdir, filename + '.graph'))
             elif mode == 'configfile':
-                uristring += graphResource.format(i, graphUri, filename)
+                uristring += graphResource.format(i, graphUri, join(subdir, filename))
 
             # Add and Commit the empty graph
-            index.add(filename)
+            index.add(path.join(subdir, filename))
             i += 1
         if mode == 'configfile':
             graph = Graph()
+
             with open(path.join(tmpRepo.repo.workdir, "config.ttl"), "w") as configFile:
                 rdf_content = configFileContent.format(tmpRepo.repo.workdir, uristring)
                 graph.parse(format='turtle', data=rdf_content)
                 configFile.write(graph.serialize(format='turtle').decode())
+
             index.add('config.ttl')
 
         index.write()

tests/test_app.py

@@ -2797,6 +2797,149 @@ def testStartApp(self):
             response = app.post('/provenance', data=dict(query=query))
             self.assertEqual(response.status, '404 NOT FOUND')
 
+    def testSubdirectoriesGraphfile(self):
+        """Test if subdirectories are recognized and commits are working using graphfiles."""
+        # Prepare a Repository with subdirectories
+        repo_content = {'urn:graph0': '<urn:0> <urn:0> <urn:0> .\n',
+                        'urn:graph1': '<urn:1> <urn:1> <urn:1> .\n'}
+
+        with TemporaryRepositoryFactory().withGraphs(repo_content, 'graphfiles', True) as repo:
+            select = 'SELECT ?s ?p ?o WHERE {{ GRAPH <urn:graph{}> {{ ?s ?p ?o }} }}'
+            update = """
+                DELETE DATA {{
+                    GRAPH <urn:graph{i}> {{
+                        <urn:{i}> <urn:{i}> <urn:{i}> . }} }} ;
+                INSERT DATA {{
+                    GRAPH <urn:graph{i}> {{
+                        <urn:{i}{i}> <urn:{i}{i}> <urn:{i}{i}> . }} }}"""
+
+            # Start Quit
+            args = quitApp.parseArgs(['-t', repo.workdir])
+            objects = quitApp.initialize(args)
+            config = objects['config']
+            app = create_app(config).test_client()
+
+            # check states after init
+            for i in [0, 1]:
+                self.assertTrue(
+                    path.isfile(path.join(repo.workdir,
+                                          'sub{}'.format(i),
+                                          'graph_{}.nt.graph'.format(i))))
+                # check store content
+                res = app.post('/sparql',
+                               data=dict(query=select.format(i)),
+                               headers=dict(accept='application/sparql-results+json'))
+                obj = json.loads(res.data.decode("utf-8"))
+                self.assertEqual(len(obj["results"]["bindings"]), 1)
+                self.assertDictEqual(obj["results"]["bindings"][0], {
+                    "s": {'type': 'uri', 'value': 'urn:{}'.format(i)},
+                    "p": {'type': 'uri', 'value': 'urn:{}'.format(i)},
+                    "o": {'type': 'uri', 'value': 'urn:{}'.format(i)}})
+
+                # check file existence
+                with open(path.join(repo.workdir,
+                                    'sub{}'.format(i),
+                                    'graph_{}.nt'.format(i)), 'r') as f:
+                    self.assertEqual(
+                        '<urn:{i}> <urn:{i}> <urn:{i}> .\n'.format(i=i),
+                        f.read())
+
+            # check states after update
+            for i in [0, 1]:
+                # perform update
+                app.post('/sparql', data=dict(update=update.format(i=i)))
+
+                # check store content
+                res = app.post('/sparql',
+                               data=dict(query=select.format(i)),
+                               headers=dict(accept='application/sparql-results+json'))
+                obj = json.loads(res.data.decode("utf-8"))
+
+                # check file existence
+                with open(path.join(repo.workdir,
+                                    'sub{}'.format(i),
+                                    'graph_{}.nt'.format(i)), 'r') as f:
+                    self.assertEqual(
+                        '<urn:{i}{i}> <urn:{i}{i}> <urn:{i}{i}> .\n'.format(i=i),
+                        f.read())
+
+                self.assertEqual(len(obj["results"]["bindings"]), 1)
+                self.assertDictEqual(obj["results"]["bindings"][0], {
+                    "s": {'type': 'uri', 'value': 'urn:{i}{i}'.format(i=i)},
+                    "p": {'type': 'uri', 'value': 'urn:{i}{i}'.format(i=i)},
+                    "o": {'type': 'uri', 'value': 'urn:{i}{i}'.format(i=i)}})
+
+    def testSubdirectoriesConfigfile(self):
+        """Test if subdirectories are recognized and commits are working using configfile."""
+        # Prepare a Repository with subdirectories
+        repo_content = {'urn:graph0': '<urn:0> <urn:0> <urn:0> .\n',
+                        'urn:graph1': '<urn:1> <urn:1> <urn:1> .\n'}
+
+        with TemporaryRepositoryFactory().withGraphs(repo_content, 'configfile', True) as repo:
+            select = 'SELECT ?s ?p ?o WHERE {{ GRAPH <urn:graph{}> {{ ?s ?p ?o }} }}'
+            update = """
+                DELETE DATA {{
+                    GRAPH <urn:graph{i}> {{
+                        <urn:{i}> <urn:{i}> <urn:{i}> . }} }} ;
+                INSERT DATA {{
+                    GRAPH <urn:graph{i}> {{
+                        <urn:{i}{i}> <urn:{i}{i}> <urn:{i}{i}> . }} }}"""
+
+            # Start Quit
+            args = quitApp.parseArgs(['-t', repo.workdir])
+            objects = quitApp.initialize(args)
+            config = objects['config']
+            app = create_app(config).test_client()
+
+            # check states after init
+            for i in [0, 1]:
+                self.assertFalse(path.isfile(path.join(repo.workdir,
+                                             'sub{}'.format(i),
+                                             'graph_{}.nt.graph'.format(i))))
+                # check store content
+                res = app.post('/sparql',
+                               data=dict(query=select.format(i)),
+                               headers=dict(accept='application/sparql-results+json'))
+                obj = json.loads(res.data.decode("utf-8"))
+                self.assertEqual(len(obj["results"]["bindings"]), 1)
+                self.assertDictEqual(obj["results"]["bindings"][0], {
+                    "s": {'type': 'uri', 'value': 'urn:{}'.format(i)},
+                    "p": {'type': 'uri', 'value': 'urn:{}'.format(i)},
+                    "o": {'type': 'uri', 'value': 'urn:{}'.format(i)}})
+
+                # check file existence
+                with open(path.join(repo.workdir,
+                                    'sub{}'.format(i),
+                                    'graph_{}.nt'.format(i)), 'r') as f:
+                    self.assertEqual(
+                        '<urn:{i}> <urn:{i}> <urn:{i}> .\n'.format(i=i),
+                        f.read())
+
+            # check states after update
+            for i in [0, 1]:
+                # perform update
+                app.post('/sparql', data=dict(update=update.format(i=i)))
+
+                # check store content
+                res = app.post('/sparql',
+                               data=dict(query=select.format(i)),
+                               headers=dict(accept='application/sparql-results+json'))
+                obj = json.loads(res.data.decode("utf-8"))
+
+                # check file existence
+                with open(path.join(repo.workdir,
+                                    'sub{}'.format(i),
+                                    'graph_{}.nt'.format(i)), 'r') as f:
+                    self.assertEqual(
+                        '<urn:{i}{i}> <urn:{i}{i}> <urn:{i}{i}> .\n'.format(i=i),
+                        f.read())
+
+                self.assertEqual(len(obj["results"]["bindings"]), 1)
+                self.assertDictEqual(obj["results"]["bindings"][0], {
+                    "s": {'type': 'uri', 'value': 'urn:{i}{i}'.format(i=i)},
+                    "p": {'type': 'uri', 'value': 'urn:{i}{i}'.format(i=i)},
+                    "o": {'type': 'uri', 'value': 'urn:{i}{i}'.format(i=i)}})
+
     def testWithOnDeleteAndInsert(self):
         """Test WITH on DELETE and INSERT plus USING.