From 68c17e38970c0e98311317d282aad82db8cb1f2d Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Fri, 2 Nov 2018 07:54:23 +0100 Subject: [PATCH 01/22] Add stub for resolution methods --- quit/web/modules/endpoint.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index b1ef4857..2aa401fd 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -99,6 +99,27 @@ def sparql(parent_commit_ref): return make_response('No branch or reference given.', 400) if queryType in ['InsertData', 'DeleteData', 'Modify', 'DeleteWhere']: + parent_commit_id = request.values.get('parent_commit_id', None) or None + if parent_commit_id and parent_commit_id != commitid: + resolution_method = request.values.get('resolution_method', None) or None + if resolution_method = "reject": + logger.debug("rejecting update because {} is at {} but {} was expected".format( + parent_commit_ref, commitid, parent_commit_id)) + return make_response('reject', 409) # alternative 412 + elif resolution_method = "merge": + logger.debug("going to merge update into {} because it is at {} but {} was expected".format( + parent_commit_ref, commitid, parent_commit_id)) + # TODO + return make_response('reject', 409) # alternative 412 + elif resolution_method = "branch": + logger.debug("writing update to a branch of {} because it is at {} but {} was expected".format( + parent_commit_ref, commitid, parent_commit_id)) + # TODO + return make_response('reject', 409) # alternative 412 + + + # TODO check resolution method + res, exception = graph.update(parsedQuery) try: From 0e97b36764c892fd83d0d91eef9045bef08fa188 Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Fri, 2 Nov 2018 11:20:55 +0100 Subject: [PATCH 02/22] More scatch of QEI resolution methods --- quit/core.py | 9 +++++ quit/web/modules/endpoint.py | 70 +++++++++++++++++++----------------- 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/quit/core.py b/quit/core.py index 04c5189a..c5fa69f1 100644 --- a/quit/core.py +++ b/quit/core.py @@ -411,6 +411,15 @@ def getFileReferenceAndContext(self, blob, commit): return quitWorkingData return self._blobs.get(blob) + def applyQueryOnCommit(self, parsedQuery, parent_commit_ref, target_ref, query=query, + default_graph=default_graph, named_graph=named_graph): + """Apply an update query on the graph and the git repository.""" + graph, commitid = self.instance(parent_commit_ref) + resultingChanges, exception = graph.update(parsedQuery) + oid = self.commit(graph, resultingChanges, 'New Commit from QuitStore', parent_commit_ref, + target_ref, query=query, default_graph=default_graph, + named_graph=named_graph) + def commit(self, graph, delta, message, parent_commit_ref, target_ref, query=None, default_graph=[], named_graph=[], **kwargs): """Commit changes after applying deltas to the blobs. diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index 2aa401fd..d6f87c35 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -92,54 +92,58 @@ def sparql(parent_commit_ref): except SparqlProtocolError as e: return make_response('Sparql Protocol Error', 400) - try: - graph, commitid = quit.instance(parent_commit_ref) - except Exception as e: - logger.exception(e) - return make_response('No branch or reference given.', 400) + commitid = None # TODO remove when restructuring mimetypes if queryType in ['InsertData', 'DeleteData', 'Modify', 'DeleteWhere']: + commit = quit.repository.revision(parent_commit_ref) + parent_commit_id = request.values.get('parent_commit_id', None) or None - if parent_commit_id and parent_commit_id != commitid: + if parent_commit_id and parent_commit_id != commit.id: resolution_method = request.values.get('resolution_method', None) or None - if resolution_method = "reject": + if resolution_method == "reject": logger.debug("rejecting update because {} is at {} but {} was expected".format( - parent_commit_ref, commitid, parent_commit_id)) - return make_response('reject', 409) # alternative 412 - elif resolution_method = "merge": - logger.debug("going to merge update into {} because it is at {} but {} was expected".format( - parent_commit_ref, commitid, parent_commit_id)) - # TODO + parent_commit_ref, commit.id, parent_commit_id)) return make_response('reject', 409) # alternative 412 - elif resolution_method = "branch": + elif resolution_method in ("merge", "branch"): logger.debug("writing update to a branch of {} because it is at {} but {} was expected".format( - parent_commit_ref, commitid, parent_commit_id)) - # TODO - return make_response('reject', 409) # alternative 412 + parent_commit_ref, commit.id, parent_commit_id)) + # TODO apply update on graph, commitid + target_ref = "some temorary reference" + oid = quit.applyQueryOnCommit(parsedQuery, parent_commit_id, target_ref, query=query, + default_graph=default_graph, named_graph=named_graph) + if resolution_method == "merge": + logger.debug("going to merge update into {} because it is at {} but {} was expected".format( + parent_commit_ref, commit.id, parent_commit_id)) + #TODO merge graph, commitid with original graph, commitid and commit to parent_commit_ref - # TODO check resolution method + return make_response('branched', 200) - res, exception = graph.update(parsedQuery) + return make_response('branched', 200) + # Add info about temporary branch + else: + graph, commitid = quit.instance(parent_commit_id) - try: target_ref = request.values.get('target_ref', None) or default_branch target_ref = 'refs/heads/{}'.format(target_ref) - oid = quit.commit(graph, res, 'New Commit from QuitStore', parent_commit_ref, - target_ref, query=query, default_graph=default_graph, - named_graph=named_graph) - if exception is not None: - logger.exception(exception) - return 'Update query not executed (completely), (detected USING NAMED)', 400 - response = make_response('', 200) - response.headers["X-CurrentBranch"] = target_ref - response.headers["X-CurrentCommit"] = oid - return response + try: + oid = quit.applyQueryOnCommit(parsedQuery, parent_commit_ref, target_ref, query=query, default_graph=default_graph, + named_graph=named_graph) + response = make_response('', 200) + response.headers["X-CurrentBranch"] = target_ref + response.headers["X-CurrentCommit"] = oid + return response + except Exception as e: + # query ok, but unsupported query type or other problem during commit + logger.exception(e) + return make_response('Error after executing the update query.', 400) + elif queryType in ['SelectQuery', 'DescribeQuery', 'AskQuery', 'ConstructQuery']: + try: + graph, commitid = quit.instance(parent_commit_ref) except Exception as e: - # query ok, but unsupported query type or other problem during commit logger.exception(e) - return make_response('Error after executing the update query.', 400) - elif queryType in ['SelectQuery', 'DescribeQuery', 'AskQuery', 'ConstructQuery']: + return make_response('No branch or reference given.', 400) + try: res = graph.query(parsedQuery) except FromNamedError as e: From 2fbb3f0a2ad1599749bc7005b8756308dc38d737 Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Fri, 2 Nov 2018 16:43:09 +0100 Subject: [PATCH 03/22] Fix pylava --- quit/web/modules/endpoint.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index d6f87c35..ca57d719 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -92,7 +92,7 @@ def sparql(parent_commit_ref): except SparqlProtocolError as e: return make_response('Sparql Protocol Error', 400) - commitid = None # TODO remove when restructuring mimetypes + commitid = None # TODO remove when restructuring mimetypes if queryType in ['InsertData', 'DeleteData', 'Modify', 'DeleteWhere']: commit = quit.repository.revision(parent_commit_ref) @@ -103,19 +103,21 @@ def sparql(parent_commit_ref): if resolution_method == "reject": logger.debug("rejecting update because {} is at {} but {} was expected".format( parent_commit_ref, commit.id, parent_commit_id)) - return make_response('reject', 409) # alternative 412 + return make_response('reject', 409) # alternative 412 elif resolution_method in ("merge", "branch"): - logger.debug("writing update to a branch of {} because it is at {} but {} was expected".format( - parent_commit_ref, commit.id, parent_commit_id)) + logger.debug("writing update to a branch of {} because it is at {} but {} was " + + "expected".format(parent_commit_ref, commit.id, parent_commit_id)) # TODO apply update on graph, commitid target_ref = "some temorary reference" - oid = quit.applyQueryOnCommit(parsedQuery, parent_commit_id, target_ref, query=query, - default_graph=default_graph, named_graph=named_graph) + oid = quit.applyQueryOnCommit(parsedQuery, parent_commit_id, target_ref, + query=query, default_graph=default_graph, + named_graph=named_graph) if resolution_method == "merge": - logger.debug("going to merge update into {} because it is at {} but {} was expected".format( - parent_commit_ref, commit.id, parent_commit_id)) - #TODO merge graph, commitid with original graph, commitid and commit to parent_commit_ref + logger.debug("going to merge update into {} because it is at {} but {} was " + + "expected".format(parent_commit_ref, commit.id, parent_commit_id)) + # TODO merge graph, commitid with original graph, commitid and commit to + # parent_commit_ref return make_response('branched', 200) @@ -127,7 +129,8 @@ def sparql(parent_commit_ref): target_ref = request.values.get('target_ref', None) or default_branch target_ref = 'refs/heads/{}'.format(target_ref) try: - oid = quit.applyQueryOnCommit(parsedQuery, parent_commit_ref, target_ref, query=query, default_graph=default_graph, + oid = quit.applyQueryOnCommit(parsedQuery, parent_commit_ref, target_ref, + query=query, default_graph=default_graph, named_graph=named_graph) response = make_response('', 200) response.headers["X-CurrentBranch"] = target_ref From b87dadb4f733f835ff717496e9b1a3eded48c600 Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Fri, 2 Nov 2018 16:52:15 +0100 Subject: [PATCH 04/22] Fix parameters --- quit/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quit/core.py b/quit/core.py index c5fa69f1..7ddb7799 100644 --- a/quit/core.py +++ b/quit/core.py @@ -411,8 +411,8 @@ def getFileReferenceAndContext(self, blob, commit): return quitWorkingData return self._blobs.get(blob) - def applyQueryOnCommit(self, parsedQuery, parent_commit_ref, target_ref, query=query, - default_graph=default_graph, named_graph=named_graph): + def applyQueryOnCommit(self, parsedQuery, parent_commit_ref, target_ref, query=None, + default_graph=[], named_graph=[]): """Apply an update query on the graph and the git repository.""" graph, commitid = self.instance(parent_commit_ref) resultingChanges, exception = graph.update(parsedQuery) From 5a1839168efd1ad14e9b4e90dabd2eb7e828f6ac Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Fri, 2 Nov 2018 17:08:24 +0100 Subject: [PATCH 05/22] Reestablish old behavior --- quit/core.py | 6 ++++++ quit/web/modules/endpoint.py | 13 ++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/quit/core.py b/quit/core.py index 7ddb7799..f3dc6c9e 100644 --- a/quit/core.py +++ b/quit/core.py @@ -416,9 +416,15 @@ def applyQueryOnCommit(self, parsedQuery, parent_commit_ref, target_ref, query=N """Apply an update query on the graph and the git repository.""" graph, commitid = self.instance(parent_commit_ref) resultingChanges, exception = graph.update(parsedQuery) + if exception: + # TODO need to revert or invalidate the graph at this point. + pass oid = self.commit(graph, resultingChanges, 'New Commit from QuitStore', parent_commit_ref, target_ref, query=query, default_graph=default_graph, named_graph=named_graph) + if exception: + raise exception + def commit(self, graph, delta, message, parent_commit_ref, target_ref, query=None, default_graph=[], named_graph=[], **kwargs): diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index ca57d719..c121e933 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -95,18 +95,21 @@ def sparql(parent_commit_ref): commitid = None # TODO remove when restructuring mimetypes if queryType in ['InsertData', 'DeleteData', 'Modify', 'DeleteWhere']: - commit = quit.repository.revision(parent_commit_ref) + if parent_commit_ref: + commit_id = quit.repository.revision(parent_commit_ref).id + else: + commit_id = None parent_commit_id = request.values.get('parent_commit_id', None) or None - if parent_commit_id and parent_commit_id != commit.id: + if parent_commit_id and parent_commit_id != commit_id: resolution_method = request.values.get('resolution_method', None) or None if resolution_method == "reject": logger.debug("rejecting update because {} is at {} but {} was expected".format( - parent_commit_ref, commit.id, parent_commit_id)) + parent_commit_ref, commit_id, parent_commit_id)) return make_response('reject', 409) # alternative 412 elif resolution_method in ("merge", "branch"): logger.debug("writing update to a branch of {} because it is at {} but {} was " + - "expected".format(parent_commit_ref, commit.id, parent_commit_id)) + "expected".format(parent_commit_ref, commit_id, parent_commit_id)) # TODO apply update on graph, commitid target_ref = "some temorary reference" oid = quit.applyQueryOnCommit(parsedQuery, parent_commit_id, target_ref, @@ -115,7 +118,7 @@ def sparql(parent_commit_ref): if resolution_method == "merge": logger.debug("going to merge update into {} because it is at {} but {} was " + - "expected".format(parent_commit_ref, commit.id, parent_commit_id)) + "expected".format(parent_commit_ref, commit_id, parent_commit_id)) # TODO merge graph, commitid with original graph, commitid and commit to # parent_commit_ref From 2dbf6a746230f4fab50224fa6fefb43d580cb3ad Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Fri, 2 Nov 2018 17:24:42 +0100 Subject: [PATCH 06/22] Remove newline --- quit/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/quit/core.py b/quit/core.py index f3dc6c9e..eee96f74 100644 --- a/quit/core.py +++ b/quit/core.py @@ -425,7 +425,6 @@ def applyQueryOnCommit(self, parsedQuery, parent_commit_ref, target_ref, query=N if exception: raise exception - def commit(self, graph, delta, message, parent_commit_ref, target_ref, query=None, default_graph=[], named_graph=[], **kwargs): """Commit changes after applying deltas to the blobs. From fc01beed7255efb793012b04423c3cac80c87766 Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Fri, 2 Nov 2018 17:24:51 +0100 Subject: [PATCH 07/22] Add shortUUID --- quit/web/modules/endpoint.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index c121e933..a14e2f81 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -11,6 +11,8 @@ from quit.web.app import render_template, feature_required from quit.exceptions import UnSupportedQuery, SparqlProtocolError, NonAbsoluteBaseError from quit.exceptions import FromNamedError +import datetime +import uuid logger = logging.getLogger('quit.modules.endpoint') @@ -111,7 +113,9 @@ def sparql(parent_commit_ref): logger.debug("writing update to a branch of {} because it is at {} but {} was " + "expected".format(parent_commit_ref, commit_id, parent_commit_id)) # TODO apply update on graph, commitid - target_ref = "some temorary reference" + time = datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S') + shortUUID = uuid.uuid1().bytes.encode('base64').rstrip('=\n').replace('/', '_') + target_ref = "refs/heads/tmp/" + time + shortUUID oid = quit.applyQueryOnCommit(parsedQuery, parent_commit_id, target_ref, query=query, default_graph=default_graph, named_graph=named_graph) From a5c6e1b981012e231285929a5e7740e0a4d46416 Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Fri, 2 Nov 2018 17:36:07 +0100 Subject: [PATCH 08/22] Merge, lets see --- quit/git.py | 2 +- quit/web/modules/endpoint.py | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/quit/git.py b/quit/git.py index 012fe659..a1aa6ed5 100644 --- a/quit/git.py +++ b/quit/git.py @@ -335,7 +335,7 @@ def merge(self, reference=None, target=None, branch=None): 'reference' to the resulting commit Keyword arguments: - reference -- The reference which should point ot the result of the merge + reference -- The reference which should point to the result of the merge target -- The target of the merge operation (if omitted, 'branch' will be merged into 'reference') branch -- The branche which should be merged into 'target' respective 'reference' diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index a14e2f81..faa1331b 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -112,7 +112,6 @@ def sparql(parent_commit_ref): elif resolution_method in ("merge", "branch"): logger.debug("writing update to a branch of {} because it is at {} but {} was " + "expected".format(parent_commit_ref, commit_id, parent_commit_id)) - # TODO apply update on graph, commitid time = datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S') shortUUID = uuid.uuid1().bytes.encode('base64').rstrip('=\n').replace('/', '_') target_ref = "refs/heads/tmp/" + time + shortUUID @@ -125,10 +124,22 @@ def sparql(parent_commit_ref): "expected".format(parent_commit_ref, commit_id, parent_commit_id)) # TODO merge graph, commitid with original graph, commitid and commit to # parent_commit_ref + try: + quit.repository.merge(target=parent_commit_ref, branch=target_ref) + except QuitMergeConflict as e: + response = make_response('merge failed', 400) + response.headers["X-CurrentBranch"] = target_ref + response.headers["X-CurrentCommit"] = oid + return response + response = make_response('success', 200) + response.headers["X-CurrentBranch"] = parent_commit_ref + response.headers["X-CurrentCommit"] = oid + return response + response = make_response('branched', 200) + response.headers["X-CurrentBranch"] = target_ref + response.headers["X-CurrentCommit"] = oid + return response - return make_response('branched', 200) - - return make_response('branched', 200) # Add info about temporary branch else: graph, commitid = quit.instance(parent_commit_id) From d8a695e310ed1d329d45cddc5c227d05ee233be9 Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Fri, 2 Nov 2018 17:42:23 +0100 Subject: [PATCH 09/22] Simplify response building --- quit/web/modules/endpoint.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index faa1331b..767efda1 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -126,16 +126,12 @@ def sparql(parent_commit_ref): # parent_commit_ref try: quit.repository.merge(target=parent_commit_ref, branch=target_ref) + response = make_response('success', 200) + target_ref = parent_commit_ref except QuitMergeConflict as e: response = make_response('merge failed', 400) - response.headers["X-CurrentBranch"] = target_ref - response.headers["X-CurrentCommit"] = oid - return response - response = make_response('success', 200) - response.headers["X-CurrentBranch"] = parent_commit_ref - response.headers["X-CurrentCommit"] = oid - return response - response = make_response('branched', 200) + else: + response = make_response('branched', 200) response.headers["X-CurrentBranch"] = target_ref response.headers["X-CurrentCommit"] = oid return response From 467fb5378e914cef4251639d8a844cf4f4e09f27 Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Fri, 2 Nov 2018 21:36:01 +0100 Subject: [PATCH 10/22] Fix some issues --- quit/core.py | 1 + quit/git.py | 14 +++++++++----- quit/web/modules/endpoint.py | 16 ++++++++++------ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/quit/core.py b/quit/core.py index eee96f74..00e393d3 100644 --- a/quit/core.py +++ b/quit/core.py @@ -424,6 +424,7 @@ def applyQueryOnCommit(self, parsedQuery, parent_commit_ref, target_ref, query=N named_graph=named_graph) if exception: raise exception + return oid def commit(self, graph, delta, message, parent_commit_ref, target_ref, query=None, default_graph=[], named_graph=[], **kwargs): diff --git a/quit/git.py b/quit/git.py index a1aa6ed5..1a919095 100644 --- a/quit/git.py +++ b/quit/git.py @@ -3,7 +3,7 @@ import re import logging -from _pygit2 import GitError +from _pygit2 import GitError, Oid from os.path import expanduser, join from quit.exceptions import RepositoryNotFound, RevisionNotFound, NodeNotFound, RemoteNotFound from quit.exceptions import QuitMergeConflict, QuitGitRefNotFound, QuitGitRepoError @@ -272,9 +272,9 @@ def pull(self, remote_name=None, refspec=None): local_branch = groups.group("dst") logger.debug("pull: parsed refspec is: {}, {}, {}".format(plus, remote_branch, local_branch)) - remote_master_id = self.fetch(remote_name=remote_name, remote_branch=remote_branch) - if remote_master_id is not None: - self.merge(reference=local_branch, branch=remote_master_id) + remote_reference = self.fetch(remote_name=remote_name, remote_branch=remote_branch) + if remote_reference is not None: + self.merge(reference=local_branch, branch=remote_reference) def push(self, remote_name=None, refspec=None): """Push changes on a local repository to a remote repository. @@ -350,7 +350,11 @@ def merge(self, reference=None, target=None, branch=None): logger.debug("merge: {}, {}, {}".format(reference, target, branch)) - merge_result, _ = self._repository.merge_analysis(branch) + if not isinstance(branch, Oid): + branch_id = self._repository.lookup_reference(branch).target + else: + branch_id = branch + merge_result, _ = self._repository.merge_analysis(branch_id) if reference != "master": raise Exception("We first have to implement switching branches") diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index 767efda1..d1f3ddb9 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -10,9 +10,10 @@ from quit.helpers import parse_sparql_request, parse_query_type from quit.web.app import render_template, feature_required from quit.exceptions import UnSupportedQuery, SparqlProtocolError, NonAbsoluteBaseError -from quit.exceptions import FromNamedError +from quit.exceptions import FromNamedError, QuitMergeConflict import datetime import uuid +import base64 logger = logging.getLogger('quit.modules.endpoint') @@ -110,11 +111,14 @@ def sparql(parent_commit_ref): parent_commit_ref, commit_id, parent_commit_id)) return make_response('reject', 409) # alternative 412 elif resolution_method in ("merge", "branch"): - logger.debug("writing update to a branch of {} because it is at {} but {} was " + - "expected".format(parent_commit_ref, commit_id, parent_commit_id)) + logger.debug(("writing update to a branch of {} because it is at {} but {} was " + "expected").format(parent_commit_ref, commit_id, parent_commit_id)) time = datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S') - shortUUID = uuid.uuid1().bytes.encode('base64').rstrip('=\n').replace('/', '_') - target_ref = "refs/heads/tmp/" + time + shortUUID + shortUUID = (base64.urlsafe_b64encode(uuid.uuid1().bytes).decode("utf-8") + ).rstrip('=\n').replace('/', '_') + target_branch = "tmp/{}_{}".format(time, shortUUID) + target_ref = "refs/heads/" + target_branch + logger.debug("target ref is: {}".format(target_ref)) oid = quit.applyQueryOnCommit(parsedQuery, parent_commit_id, target_ref, query=query, default_graph=default_graph, named_graph=named_graph) @@ -125,7 +129,7 @@ def sparql(parent_commit_ref): # TODO merge graph, commitid with original graph, commitid and commit to # parent_commit_ref try: - quit.repository.merge(target=parent_commit_ref, branch=target_ref) + quit.repository.merge(reference=parent_commit_ref, branch=target_ref) response = make_response('success', 200) target_ref = parent_commit_ref except QuitMergeConflict as e: From fc96af25211a388363aed06799742b26cbc7817c Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Mon, 5 Nov 2018 18:25:13 +0100 Subject: [PATCH 11/22] Check for current branch before merge, only checkout if on current branch after update, use branch_id in merge --- quit/core.py | 2 +- quit/git.py | 19 ++++++++++--------- quit/web/modules/endpoint.py | 5 +++-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/quit/core.py b/quit/core.py index 00e393d3..2da01ede 100644 --- a/quit/core.py +++ b/quit/core.py @@ -567,7 +567,7 @@ def _applyUnknownGraphs(delta, known_blobs): if oid: self._commits.set(oid.hex, blobs_new) commit = self.repository.revision(oid.hex) - if not self.repository.is_bare: + if not self.repository.is_bare and target_ref == self.repository.current_head: self.repository._repository.checkout( target_ref, strategy=pygit2.GIT_CHECKOUT_FORCE) self.syncSingle(commit) diff --git a/quit/git.py b/quit/git.py index 1a919095..ec00d138 100644 --- a/quit/git.py +++ b/quit/git.py @@ -342,7 +342,7 @@ def merge(self, reference=None, target=None, branch=None): """ if reference is None: - reference = "master" + reference = self.current_head if target is None: target = "HEAD" if branch is None: @@ -356,8 +356,9 @@ def merge(self, reference=None, target=None, branch=None): branch_id = branch merge_result, _ = self._repository.merge_analysis(branch_id) - if reference != "master": - raise Exception("We first have to implement switching branches") + if reference != self.current_head: + raise Exception(("Try to merge into {} currently on {}. We first have to implement to " + "switch branches").format(reference, self.current_head)) if target != "HEAD": raise Exception("We currently are only able to merge into the HEAD of a branch") @@ -368,19 +369,19 @@ def merge(self, reference=None, target=None, branch=None): # We can just fastforward elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: - self._repository.checkout_tree(self._repository.get(branch)) + self._repository.checkout_tree(self._repository.get(branch_id)) try: master_ref = self._repository.lookup_reference( 'refs/heads/{}'.format(reference)) - master_ref.set_target(branch) + master_ref.set_target(branch_id) except KeyError: self._repository.create_branch( - reference, self._repository.get(branch)) - self._repository.head.set_target(branch) + reference, self._repository.get(branch_id)) + self._repository.head.set_target(branch_id) return elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: - self._repository.merge(branch) + self._repository.merge(branch_id) if self._repository.index.conflicts is not None: for conflict in self._repository.index.conflicts: @@ -391,7 +392,7 @@ def merge(self, reference=None, target=None, branch=None): tree = self._repository.index.write_tree() self._repository.create_commit( 'HEAD', user, user, 'Merge!', tree, - [self._repository.head.target, branch] + [self._repository.head.target, branch_id] ) # We need to do this or git CLI will think we are still merging. self._repository.state_cleanup() diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index d1f3ddb9..67a251bb 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -124,8 +124,9 @@ def sparql(parent_commit_ref): named_graph=named_graph) if resolution_method == "merge": - logger.debug("going to merge update into {} because it is at {} but {} was " + - "expected".format(parent_commit_ref, commit_id, parent_commit_id)) + logger.debug(("going to merge update into {} because it is at {} but {} was " + "expected").format(parent_commit_ref, commit_id, + parent_commit_id)) # TODO merge graph, commitid with original graph, commitid and commit to # parent_commit_ref try: From 5aa8db6f9bfee0925a47b714f258944e178a810a Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Tue, 13 Nov 2018 14:29:35 +0100 Subject: [PATCH 12/22] Remove TODO --- quit/web/modules/endpoint.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index 09660021..3eedc928 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -126,8 +126,6 @@ def sparql(branch_or_ref): if resolution_method == "merge": logger.debug(("going to merge update into {} because it is at {} but {} was " "expected").format(branch_or_ref, commit_id, parent_commit_id)) - # TODO merge graph, commitid with original graph, commitid and commit to - # branch_or_ref try: quit.repository.merge(reference=branch_or_ref, branch=target_ref) response = make_response('success', 200) From 7b1833beed0293e5eca2d6af0668a407f78d0985 Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Tue, 13 Nov 2018 14:42:51 +0100 Subject: [PATCH 13/22] Add branch header on select and fix merge argument to target --- quit/web/modules/endpoint.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index 3eedc928..07e0b3bb 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -127,7 +127,7 @@ def sparql(branch_or_ref): logger.debug(("going to merge update into {} because it is at {} but {} was " "expected").format(branch_or_ref, commit_id, parent_commit_id)) try: - quit.repository.merge(reference=branch_or_ref, branch=target_ref) + quit.repository.merge(target=branch_or_ref, branch=target_ref) response = make_response('success', 200) target_ref = branch_or_ref except QuitMergeConflict as e: @@ -184,6 +184,7 @@ def sparql(branch_or_ref): elif queryType in ['ConstructQuery', 'DescribeQuery']: response = create_result_response(res, rdfMimetypes[mimetype]) if commitid: + response.headers["X-CurrentBranch"] = branch_or_ref response.headers["X-CurrentCommit"] = commitid return response except KeyError as e: From be12c901751e1733313b53e5f4d3fdcccd6c02ed Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Sun, 25 Nov 2018 19:02:02 +0100 Subject: [PATCH 14/22] Move result creation into elif block --- quit/web/modules/endpoint.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index d372fef5..2cb32904 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -91,8 +91,6 @@ def sparql(branch_or_ref): except SparqlProtocolError: return make_response('Sparql Protocol Error', 400) - commitid = None # TODO remove when restructuring mimetypes - if queryType in ['InsertData', 'DeleteData', 'Modify', 'DeleteWhere', 'Load']: if branch_or_ref: commit_id = quit.repository.revision(branch_or_ref).id @@ -168,20 +166,21 @@ def sparql(branch_or_ref): return make_response('FROM NAMED not supported, yet', 400) except UnSupportedQuery: return make_response('Unsupported Query', 400) - else: - logger.debug("Unsupported Type: {}".format(queryType)) - return make_response("Unsupported Query Type: {}".format(queryType), 400) - mimetype = _getBestMatchingMimeType(request, queryType) + mimetype = _getBestMatchingMimeType(request, queryType) - if not mimetype: - return make_response("Mimetype: {} not acceptable".format(mimetype), 406) + if not mimetype: + return make_response("Mimetype: {} not acceptable".format(mimetype), 406) - response = create_result_response(res, mimetype, serializations[mimetype]) - if commitid: - response.headers["X-CurrentBranch"] = branch_or_ref - response.headers["X-CurrentCommit"] = commitid - return response + response = create_result_response(res, mimetype, serializations[mimetype]) + if branch_or_ref: + response.headers["X-CurrentBranch"] = branch_or_ref + if commitid: + response.headers["X-CurrentCommit"] = commitid + return response + else: + logger.debug("Unsupported Type: {}".format(queryType)) + return make_response("Unsupported Query Type: {}".format(queryType), 400) @endpoint.route("/provenance", methods=['POST', 'GET']) From 3e9ea5fbfdd06929d273f328b981510782b95cce Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Sun, 25 Nov 2018 21:58:02 +0100 Subject: [PATCH 15/22] Delete temporary branch anfter merge and check if parent commit exists --- quit/git.py | 8 +++++++- quit/web/modules/endpoint.py | 11 ++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/quit/git.py b/quit/git.py index 408ac8c8..d3d95bf4 100644 --- a/quit/git.py +++ b/quit/git.py @@ -117,7 +117,13 @@ def lookup(self, name): except KeyError: pass except ValueError: - return self._repository.get(name).id + pass + try: + revison = self._repository.get(name) + if revison: + return revison.id + except Exception as e: + logger.exception(e) raise RevisionNotFound(name) def revision(self, id='HEAD'): diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index 2cb32904..ef120c79 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -8,7 +8,7 @@ from quit.helpers import parse_sparql_request, parse_query_type from quit.web.app import render_template, feature_required from quit.exceptions import UnSupportedQuery, SparqlProtocolError, NonAbsoluteBaseError -from quit.exceptions import FromNamedError, QuitMergeConflict +from quit.exceptions import FromNamedError, QuitMergeConflict, RevisionNotFound import datetime import uuid import base64 @@ -107,6 +107,12 @@ def sparql(branch_or_ref): elif resolution_method in ("merge", "branch"): logger.debug(("writing update to a branch of {} because it is at {} but {} was " "expected").format(branch_or_ref, commit_id, parent_commit_id)) + try: + quit.repository.lookup(parent_commit_id) + except RevisionNotFound: + return make_response("The provided parent commit (parent_commit_id={}) " + "could not be found.".format(parent_commit_id), 400) + time = datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S') shortUUID = (base64.urlsafe_b64encode(uuid.uuid1().bytes).decode("utf-8") ).rstrip('=\n').replace('/', '_') @@ -122,6 +128,9 @@ def sparql(branch_or_ref): "expected").format(branch_or_ref, commit_id, parent_commit_id)) try: quit.repository.merge(target=branch_or_ref, branch=target_ref) + # delete temporary branch + tmp_branch = quit.repository._repository.branches.get(target_branch) + tmp_branch.delete() response = make_response('success', 200) target_ref = branch_or_ref except QuitMergeConflict as e: From 1dfa6e3979f63c8c76dd24fe5d0ce899969dab5c Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Mon, 18 Feb 2019 22:29:47 +0100 Subject: [PATCH 16/22] Add first test for the non QEI case --- tests/test_endpoint.py | 91 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/tests/test_endpoint.py b/tests/test_endpoint.py index bdc2af39..0851e7d4 100644 --- a/tests/test_endpoint.py +++ b/tests/test_endpoint.py @@ -2,11 +2,100 @@ import unittest from context import quit +import quit.application as quitApp +from quit.web.app import create_app +from tempfile import TemporaryDirectory +import json class EndpointTests(unittest.TestCase): """Test endpoint features.""" - pass + def setUp(self): + return + + def tearDown(self): + return + + def testInsertDataNoSnapshotIsolation(self): + """Test inserting data and selecting it, starting with an empty directory. + + 1. Prepare an empty directory + 2. Start Quit + 3. execute INSERT DATA query + 4. execute SELECT query + """ + # Prepate a git Repository + with TemporaryDirectory() as repo: + + # Start Quit + args = quitApp.parseArgs(['-t', repo]) + objects = quitApp.initialize(args) + config = objects['config'] + app = create_app(config).test_client() + + # execute INSERT DATA query + update = """INSERT DATA { + GRAPH { + a ; + "Take out the organic waste" . + }} + """ + app.post('/sparql', data=dict(update=update)) + + # execute SELECT query + select = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" + select_resp = app.post('/sparql', data=dict(query=select), headers=dict(accept="application/sparql-results+json")) + + # execute SELECT query + select = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" + select_resp = app.post('/sparql', data=dict(query=select), headers=dict(accept="application/sparql-results+json")) + + # execute INSERT DATA query + update = """INSERT DATA { + GRAPH { + . + }} + """ + app.post('/sparql', data=dict(update=update)) + + # execute INSERT DATA query + update = """DELETE { + GRAPH { + ?todo ?task . + }} + INSERT { + GRAPH { + ?todo "Take out the organic waste and the residual waste" . + }} + WHERE { + BIND ("Take out the organic waste" as ?task) + GRAPH { + ?todo ?task + } + } + """ + app.post('/sparql', data=dict(update=update)) + + # execute SELECT query + select = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" + select_resp = app.post('/sparql', data=dict(query=select), headers=dict(accept="application/sparql-results+json")) + + obj = json.loads(select_resp.data.decode("utf-8")) + + self.assertEqual(len(obj["results"]["bindings"]), 3) + + self.assertDictEqual(obj["results"]["bindings"][0], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://ex.org/status'}, + "o": {'type': 'uri', 'value': 'http://ex.org/completed'}}) + self.assertDictEqual(obj["results"]["bindings"][1], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://ex.org/task'}, + "o": {'type': 'literal', 'value': 'Take out the organic waste and the residual waste'}}) + self.assertDictEqual(obj["results"]["bindings"][2], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'}, + "o": {'type': 'uri', 'value': 'http://ex.org/Todo'}}) if __name__ == '__main__': From 1ac9ca8bc3ca50370d69857963dfc1c8046d0fb3 Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Tue, 19 Feb 2019 18:09:17 +0100 Subject: [PATCH 17/22] Test status codes and add test for reject case --- tests/test_endpoint.py | 109 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 100 insertions(+), 9 deletions(-) diff --git a/tests/test_endpoint.py b/tests/test_endpoint.py index 0851e7d4..fbe52526 100644 --- a/tests/test_endpoint.py +++ b/tests/test_endpoint.py @@ -17,12 +17,7 @@ def tearDown(self): return def testInsertDataNoSnapshotIsolation(self): - """Test inserting data and selecting it, starting with an empty directory. - - 1. Prepare an empty directory - 2. Start Quit - 3. execute INSERT DATA query - 4. execute SELECT query + """Test inserting data without checking the snapshot isolation using the commit id. """ # Prepate a git Repository with TemporaryDirectory() as repo: @@ -40,15 +35,18 @@ def testInsertDataNoSnapshotIsolation(self): "Take out the organic waste" . }} """ - app.post('/sparql', data=dict(update=update)) + response = app.post('/sparql', data=dict(update=update)) + self.assertEqual(response.status_code, 200) # execute SELECT query select = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" select_resp = app.post('/sparql', data=dict(query=select), headers=dict(accept="application/sparql-results+json")) + self.assertEqual(select_resp.status_code, 200) # execute SELECT query select = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" select_resp = app.post('/sparql', data=dict(query=select), headers=dict(accept="application/sparql-results+json")) + self.assertEqual(select_resp.status_code, 200) # execute INSERT DATA query update = """INSERT DATA { @@ -56,7 +54,8 @@ def testInsertDataNoSnapshotIsolation(self): . }} """ - app.post('/sparql', data=dict(update=update)) + response = app.post('/sparql', data=dict(update=update)) + self.assertEqual(response.status_code, 200) # execute INSERT DATA query update = """DELETE { @@ -74,11 +73,13 @@ def testInsertDataNoSnapshotIsolation(self): } } """ - app.post('/sparql', data=dict(update=update)) + response = app.post('/sparql', data=dict(update=update)) + self.assertEqual(response.status_code, 200) # execute SELECT query select = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" select_resp = app.post('/sparql', data=dict(query=select), headers=dict(accept="application/sparql-results+json")) + self.assertEqual(select_resp.status_code, 200) obj = json.loads(select_resp.data.decode("utf-8")) @@ -97,6 +98,96 @@ def testInsertDataNoSnapshotIsolation(self): "p": {'type': 'uri', 'value': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'}, "o": {'type': 'uri', 'value': 'http://ex.org/Todo'}}) + def testInsertDataOverlappingWithReject(self): + """Test inserting data from two clients (simulated) with overlapping update requests. + """ + # Prepate a git Repository + with TemporaryDirectory() as repo: + + # Start Quit + args = quitApp.parseArgs(['-t', repo]) + objects = quitApp.initialize(args) + config = objects['config'] + app = create_app(config).test_client() + + # execute INSERT DATA query + update = """INSERT DATA { + GRAPH { + a ; + "Take out the organic waste" . + }} + """ + response = app.post('/sparql', data=dict(update=update)) + self.assertEqual(response.status_code, 200) + + # Client A: execute SELECT query + selectA = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" + selectA_resp = app.post('/sparql', data=dict(query=selectA), headers=dict(accept="application/sparql-results+json")) + self.assertEqual(selectA_resp.status_code, 200) + branchA = selectA_resp.headers['X-CurrentBranch'] + commitA = selectA_resp.headers['X-CurrentCommit'] + + # Client B: execute SELECT query + selectB = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" + selectB_resp = app.post('/sparql', data=dict(query=selectB), headers=dict(accept="application/sparql-results+json")) + self.assertEqual(selectB_resp.status_code, 200) + branchB = selectB_resp.headers['X-CurrentBranch'] + commitB = selectB_resp.headers['X-CurrentCommit'] + self.assertEqual(commitA, commitB) + self.assertEqual(branchA, branchB) + + # Client B: update operation + updateB = """INSERT DATA { + GRAPH { + . + }} + """ + response = app.post('/sparql', data=dict(update=updateB, parent_commit_id=commitB, resolution_method='reject')) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200) + + # Client A: update operation + updateA = """DELETE { + GRAPH { + ?todo ?task . + }} + INSERT { + GRAPH { + ?todo "Take out the organic waste and the residual waste" . + }} + WHERE { + BIND ("Take out the organic waste" as ?task) + GRAPH { + ?todo ?task + } + } + """ + response = app.post('/sparql', data=dict(update=updateA, parent_commit_id=commitA, resolution_method='reject')) + # FAILURE. The second request should be rejected because it asumes a different commit + self.assertEqual(response.status_code, 409) + + # check the result + select = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" + select_resp = app.post('/sparql', data=dict(query=select), headers=dict(accept="application/sparql-results+json")) + self.assertEqual(select_resp.status_code, 200) + + obj = json.loads(select_resp.data.decode("utf-8")) + + self.assertEqual(len(obj["results"]["bindings"]), 3) + + self.assertDictEqual(obj["results"]["bindings"][0], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://ex.org/status'}, + "o": {'type': 'uri', 'value': 'http://ex.org/completed'}}) + self.assertDictEqual(obj["results"]["bindings"][1], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://ex.org/task'}, + "o": {'type': 'literal', 'value': 'Take out the organic waste'}}) + self.assertDictEqual(obj["results"]["bindings"][2], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'}, + "o": {'type': 'uri', 'value': 'http://ex.org/Todo'}}) + if __name__ == '__main__': unittest.main() From 01d080302a899fd87b59bbfa1640992435760add Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Tue, 19 Feb 2019 18:27:57 +0100 Subject: [PATCH 18/22] Fix returning the branch name properly as X-CurrentBranch --- quit/web/modules/endpoint.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index ef120c79..8e94a7a4 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -132,12 +132,12 @@ def sparql(branch_or_ref): tmp_branch = quit.repository._repository.branches.get(target_branch) tmp_branch.delete() response = make_response('success', 200) - target_ref = branch_or_ref + target_branch = branch_or_ref except QuitMergeConflict as e: response = make_response('merge failed', 400) else: response = make_response('branched', 200) - response.headers["X-CurrentBranch"] = target_ref + response.headers["X-CurrentBranch"] = target_branch response.headers["X-CurrentCommit"] = oid return response @@ -152,7 +152,7 @@ def sparql(branch_or_ref): query=query, default_graph=default_graph, named_graph=named_graph) response = make_response('', 200) - response.headers["X-CurrentBranch"] = target_ref + response.headers["X-CurrentBranch"] = target_head if oid is not None: response.headers["X-CurrentCommit"] = oid else: From 9da2deb996967431684c04c8ca4426ae2556b7d9 Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Tue, 19 Feb 2019 18:38:59 +0100 Subject: [PATCH 19/22] Add branch and commit assertions to reject test --- tests/test_endpoint.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_endpoint.py b/tests/test_endpoint.py index fbe52526..ae74627e 100644 --- a/tests/test_endpoint.py +++ b/tests/test_endpoint.py @@ -98,6 +98,7 @@ def testInsertDataNoSnapshotIsolation(self): "p": {'type': 'uri', 'value': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'}, "o": {'type': 'uri', 'value': 'http://ex.org/Todo'}}) + def testInsertDataOverlappingWithReject(self): """Test inserting data from two clients (simulated) with overlapping update requests. """ @@ -144,7 +145,8 @@ def testInsertDataOverlappingWithReject(self): """ response = app.post('/sparql', data=dict(update=updateB, parent_commit_id=commitB, resolution_method='reject')) self.assertEqual(response.status_code, 200) - self.assertEqual(response.status_code, 200) + self.assertEqual(branchB, response.headers['X-CurrentBranch']) + self.assertNotEqual(commitB, response.headers['X-CurrentCommit']) # Client A: update operation updateA = """DELETE { @@ -170,6 +172,8 @@ def testInsertDataOverlappingWithReject(self): select = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" select_resp = app.post('/sparql', data=dict(query=select), headers=dict(accept="application/sparql-results+json")) self.assertEqual(select_resp.status_code, 200) + self.assertEqual(branchA, select_resp.headers['X-CurrentBranch']) + self.assertNotEqual(commitA, select_resp.headers['X-CurrentCommit']) obj = json.loads(select_resp.data.decode("utf-8")) From 7e66c5250398ac6871d942997ee57e3ab62175bd Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Tue, 19 Feb 2019 18:39:16 +0100 Subject: [PATCH 20/22] Add branch resolution scenario --- tests/test_endpoint.py | 120 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/tests/test_endpoint.py b/tests/test_endpoint.py index ae74627e..b8d99013 100644 --- a/tests/test_endpoint.py +++ b/tests/test_endpoint.py @@ -193,5 +193,125 @@ def testInsertDataOverlappingWithReject(self): "o": {'type': 'uri', 'value': 'http://ex.org/Todo'}}) + def testInsertDataOverlappingWithBranch(self): + """Test inserting data from two clients (simulated) with overlapping update requests and branch resolution + """ + # Prepate a git Repository + with TemporaryDirectory() as repo: + + # Start Quit + args = quitApp.parseArgs(['-t', repo]) + objects = quitApp.initialize(args) + config = objects['config'] + app = create_app(config).test_client() + + # execute INSERT DATA query + update = """INSERT DATA { + GRAPH { + a ; + "Take out the organic waste" . + }} + """ + response = app.post('/sparql', data=dict(update=update)) + self.assertEqual(response.status_code, 200) + + # Client A: execute SELECT query + selectA = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" + selectA_resp = app.post('/sparql', data=dict(query=selectA), headers=dict(accept="application/sparql-results+json")) + self.assertEqual(selectA_resp.status_code, 200) + branchA = selectA_resp.headers['X-CurrentBranch'] + commitA = selectA_resp.headers['X-CurrentCommit'] + + # Client B: execute SELECT query + selectB = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" + selectB_resp = app.post('/sparql', data=dict(query=selectB), headers=dict(accept="application/sparql-results+json")) + self.assertEqual(selectB_resp.status_code, 200) + branchB = selectB_resp.headers['X-CurrentBranch'] + commitB = selectB_resp.headers['X-CurrentCommit'] + self.assertEqual(commitA, commitB) + self.assertEqual(branchA, branchB) + + # Client B: update operation + updateB = """INSERT DATA { + GRAPH { + . + }} + """ + response = app.post('/sparql', data=dict(update=updateB, parent_commit_id=commitB, resolution_method='branch')) + self.assertEqual(response.status_code, 200) + self.assertEqual(branchB, response.headers['X-CurrentBranch']) + newCommitB = response.headers['X-CurrentCommit'] + self.assertNotEqual(commitB, newCommitB) + + # Client A: update operation + updateA = """DELETE { + GRAPH { + ?todo ?task . + }} + INSERT { + GRAPH { + ?todo "Take out the organic waste and the residual waste" . + }} + WHERE { + BIND ("Take out the organic waste" as ?task) + GRAPH { + ?todo ?task + } + } + """ + response = app.post('/sparql', data=dict(update=updateA, parent_commit_id=commitA, resolution_method='branch')) + # FAILURE. The second request should be rejected because it asumes a different commit + self.assertEqual(response.status_code, 200) + newBranchA = response.headers['X-CurrentBranch'] + self.assertNotEqual(branchA, newBranchA) + newCommitA = response.headers['X-CurrentCommit'] + self.assertNotEqual(commitA, newCommitA) + self.assertNotEqual(newCommitB, newCommitA) + + # check the result on the master branch + select = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" + select_resp = app.post('/sparql', data=dict(query=select), headers=dict(accept="application/sparql-results+json")) + self.assertEqual(select_resp.status_code, 200) + self.assertEqual(branchB, select_resp.headers['X-CurrentBranch']) + self.assertEqual(newCommitB, select_resp.headers['X-CurrentCommit']) + + obj = json.loads(select_resp.data.decode("utf-8")) + + self.assertEqual(len(obj["results"]["bindings"]), 3) + + self.assertDictEqual(obj["results"]["bindings"][0], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://ex.org/status'}, + "o": {'type': 'uri', 'value': 'http://ex.org/completed'}}) + self.assertDictEqual(obj["results"]["bindings"][1], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://ex.org/task'}, + "o": {'type': 'literal', 'value': 'Take out the organic waste'}}) + self.assertDictEqual(obj["results"]["bindings"][2], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'}, + "o": {'type': 'uri', 'value': 'http://ex.org/Todo'}}) + + + # check the result on the newly created branch + select = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" + select_resp = app.post('/sparql/{}'.format(newBranchA), data=dict(query=select), headers=dict(accept="application/sparql-results+json")) + self.assertEqual(select_resp.status_code, 200) + self.assertEqual(newBranchA, select_resp.headers['X-CurrentBranch']) + self.assertEqual(newCommitA, select_resp.headers['X-CurrentCommit']) + + obj = json.loads(select_resp.data.decode("utf-8")) + + self.assertEqual(len(obj["results"]["bindings"]), 2) + + self.assertDictEqual(obj["results"]["bindings"][0], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://ex.org/task'}, + "o": {'type': 'literal', 'value': 'Take out the organic waste and the residual waste'}}) + self.assertDictEqual(obj["results"]["bindings"][1], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'}, + "o": {'type': 'uri', 'value': 'http://ex.org/Todo'}}) + if __name__ == '__main__': unittest.main() From fb438869df2ae0f06161757bb74270dd0d22fbfb Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Tue, 19 Feb 2019 19:06:22 +0100 Subject: [PATCH 21/22] Return oid of merge commit in merge case --- quit/web/modules/endpoint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/quit/web/modules/endpoint.py b/quit/web/modules/endpoint.py index 8e94a7a4..af504f41 100644 --- a/quit/web/modules/endpoint.py +++ b/quit/web/modules/endpoint.py @@ -128,6 +128,7 @@ def sparql(branch_or_ref): "expected").format(branch_or_ref, commit_id, parent_commit_id)) try: quit.repository.merge(target=branch_or_ref, branch=target_ref) + oid = quit.repository.revision(branch_or_ref).id # delete temporary branch tmp_branch = quit.repository._repository.branches.get(target_branch) tmp_branch.delete() From 1d17973b630fac4c0b5f236dd3586e79a02ab493 Mon Sep 17 00:00:00 2001 From: Natanael Arndt Date: Tue, 19 Feb 2019 19:06:37 +0100 Subject: [PATCH 22/22] Add merge test --- tests/test_endpoint.py | 100 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/tests/test_endpoint.py b/tests/test_endpoint.py index b8d99013..16b19d44 100644 --- a/tests/test_endpoint.py +++ b/tests/test_endpoint.py @@ -313,5 +313,105 @@ def testInsertDataOverlappingWithBranch(self): "p": {'type': 'uri', 'value': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'}, "o": {'type': 'uri', 'value': 'http://ex.org/Todo'}}) + + def testInsertDataOverlappingWithMerge(self): + """Test inserting data from two clients (simulated) with overlapping update requests and merge resolution + """ + # Prepate a git Repository + with TemporaryDirectory() as repo: + + # Start Quit + args = quitApp.parseArgs(['-t', repo]) + objects = quitApp.initialize(args) + config = objects['config'] + app = create_app(config).test_client() + + # execute INSERT DATA query + update = """INSERT DATA { + GRAPH { + a ; + "Take out the organic waste" . + }} + """ + response = app.post('/sparql', data=dict(update=update)) + self.assertEqual(response.status_code, 200) + + # Client A: execute SELECT query + selectA = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" + selectA_resp = app.post('/sparql', data=dict(query=selectA), headers=dict(accept="application/sparql-results+json")) + self.assertEqual(selectA_resp.status_code, 200) + branchA = selectA_resp.headers['X-CurrentBranch'] + commitA = selectA_resp.headers['X-CurrentCommit'] + + # Client B: execute SELECT query + selectB = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" + selectB_resp = app.post('/sparql', data=dict(query=selectB), headers=dict(accept="application/sparql-results+json")) + self.assertEqual(selectB_resp.status_code, 200) + branchB = selectB_resp.headers['X-CurrentBranch'] + commitB = selectB_resp.headers['X-CurrentCommit'] + self.assertEqual(commitA, commitB) + self.assertEqual(branchA, branchB) + + # Client B: update operation + updateB = """INSERT DATA { + GRAPH { + . + }} + """ + response = app.post('/sparql', data=dict(update=updateB, parent_commit_id=commitB, resolution_method='merge')) + self.assertEqual(response.status_code, 200) + self.assertEqual(branchB, response.headers['X-CurrentBranch']) + newCommitB = response.headers['X-CurrentCommit'] + self.assertNotEqual(commitB, newCommitB) + + # Client A: update operation + updateA = """DELETE { + GRAPH { + ?todo ?task . + }} + INSERT { + GRAPH { + ?todo "Take out the organic waste and the residual waste" . + }} + WHERE { + BIND ("Take out the organic waste" as ?task) + GRAPH { + ?todo ?task + } + } + """ + response = app.post('/sparql', data=dict(update=updateA, parent_commit_id=commitA, resolution_method='merge')) + # FAILURE. The second request should be rejected because it asumes a different commit + self.assertEqual(response.status_code, 200) + self.assertEqual(branchA, response.headers['X-CurrentBranch']) + newCommitA = response.headers['X-CurrentCommit'] + self.assertNotEqual(commitA, newCommitA) + self.assertNotEqual(newCommitB, newCommitA) + + # check the result on the master branch + select = "SELECT * WHERE {graph {?s a ; ?p ?o .}} ORDER BY ?s ?p ?o" + select_resp = app.post('/sparql', data=dict(query=select), headers=dict(accept="application/sparql-results+json")) + self.assertEqual(select_resp.status_code, 200) + self.assertEqual(branchB, select_resp.headers['X-CurrentBranch']) + self.assertEqual(newCommitA, select_resp.headers['X-CurrentCommit']) + + obj = json.loads(select_resp.data.decode("utf-8")) + + self.assertEqual(len(obj["results"]["bindings"]), 3) + + self.assertDictEqual(obj["results"]["bindings"][0], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://ex.org/status'}, + "o": {'type': 'uri', 'value': 'http://ex.org/completed'}}) + self.assertDictEqual(obj["results"]["bindings"][1], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://ex.org/task'}, + "o": {'type': 'literal', 'value': 'Take out the organic waste and the residual waste'}}) + self.assertDictEqual(obj["results"]["bindings"][2], { + "s": {'type': 'uri', 'value': 'http://ex.org/garbage'}, + "p": {'type': 'uri', 'value': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'}, + "o": {'type': 'uri', 'value': 'http://ex.org/Todo'}}) + + if __name__ == '__main__': unittest.main()