From 89a65b6bc39d805345a04f4a5c9c98ccf032f467 Mon Sep 17 00:00:00 2001 From: Jacob Wegner Date: Mon, 22 Nov 2021 08:40:27 -0600 Subject: [PATCH 1/3] Update to latest releases --- corpus.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/corpus.json b/corpus.json index 5ef07d2..9428d9b 100644 --- a/corpus.json +++ b/corpus.json @@ -10,9 +10,9 @@ "tarball_url": "https://api.github.com/repos/PerseusDL/canonical-greekLit/tarball/0.0.2940" }, "OpenGreekAndLatin/csel-dev": { - "ref": "1.0.250", - "sha": "41e8aadc5fab7c7af58e0b540d3a2c59ebd7c0c1", - "tarball_url": "https://api.github.com/repos/OpenGreekAndLatin/csel-dev/tarball/1.0.250" + "ref": "1.0.267", + "sha": "f4a3d9d1f78a46de137a33a9ffbe0de126324cc5", + "tarball_url": "https://api.github.com/repos/OpenGreekAndLatin/csel-dev/tarball/1.0.267" }, "PerseusDL/canonical-farsiLit": { "ref": "0.0.11", @@ -25,9 +25,9 @@ "tarball_url": "https://api.github.com/repos/PerseusDL/canonical-pdlpsci/tarball/0.0.56" }, "OpenGreekAndLatin/First1KGreek": { - "ref": "1.1.5669", - "sha": "20d1b7b971af52e5de3d4769eb05a8e142b5e159", - "tarball_url": "https://api.github.com/repos/OpenGreekAndLatin/First1KGreek/tarball/1.1.5669" + "ref": "1.1.5714", + "sha": "afda8cd313f5aac9169c27253a28af6e7156cfad", + "tarball_url": "https://api.github.com/repos/OpenGreekAndLatin/First1KGreek/tarball/1.1.5714" }, "OpenGreekAndLatin/Latin": { "ref": "v1.12.0", From 56b06aee245213aa2a0130b4d106be97f1d2295a Mon Sep 17 00:00:00 2001 From: Jacob Wegner Date: Mon, 22 Nov 2021 08:45:57 -0600 Subject: [PATCH 2/3] Add the ability to override releases for repos Supports an override for PerseusDL/canonical-pdlpsci --- corpus.json | 40 ++++++---- scaife_cts_api/update_corpus_shas.py | 108 +++++++++++++++++---------- 2 files changed, 96 insertions(+), 52 deletions(-) diff --git a/corpus.json b/corpus.json index 9428d9b..a094927 100644 --- a/corpus.json +++ b/corpus.json @@ -2,61 +2,73 @@ "PerseusDL/canonical-latinLit": { "ref": "0.0.963", "sha": "fd5bb4b2fc675a1f9111e783440f90c856e5a61e", - "tarball_url": "https://api.github.com/repos/PerseusDL/canonical-latinLit/tarball/0.0.963" + "tarball_url": "https://api.github.com/repos/PerseusDL/canonical-latinLit/tarball/0.0.963", + "prefer_default_branch": false }, "PerseusDL/canonical-greekLit": { "ref": "0.0.2940", "sha": "382365f5aca95c733d7e521bdcc7a4c2fc25ac6c", - "tarball_url": "https://api.github.com/repos/PerseusDL/canonical-greekLit/tarball/0.0.2940" + "tarball_url": "https://api.github.com/repos/PerseusDL/canonical-greekLit/tarball/0.0.2940", + "prefer_default_branch": false }, "OpenGreekAndLatin/csel-dev": { "ref": "1.0.267", "sha": "f4a3d9d1f78a46de137a33a9ffbe0de126324cc5", - "tarball_url": "https://api.github.com/repos/OpenGreekAndLatin/csel-dev/tarball/1.0.267" + "tarball_url": "https://api.github.com/repos/OpenGreekAndLatin/csel-dev/tarball/1.0.267", + "prefer_default_branch": false }, "PerseusDL/canonical-farsiLit": { "ref": "0.0.11", "sha": "a26b9a0f62aabd970a9fac26afa60d83e8a38823", - "tarball_url": "https://api.github.com/repos/PerseusDL/canonical-farsiLit/tarball/0.0.11" + "tarball_url": "https://api.github.com/repos/PerseusDL/canonical-farsiLit/tarball/0.0.11", + "prefer_default_branch": false }, "PerseusDL/canonical-pdlpsci": { - "ref": "0.0.56", - "sha": "2bd7e1d2acec2c442d3c676f50d15eac565e9457", - "tarball_url": "https://api.github.com/repos/PerseusDL/canonical-pdlpsci/tarball/0.0.56" + "ref": "master", + "sha": "f02ac7533c5e757985e630a4c6e3b6824c1c7f00", + "tarball_url": "https://api.github.com/repos/PerseusDL/canonical-pdlpsci/tarball/f02ac7533c5e757985e630a4c6e3b6824c1c7f00", + "prefer_default_branch": true }, "OpenGreekAndLatin/First1KGreek": { "ref": "1.1.5714", "sha": "afda8cd313f5aac9169c27253a28af6e7156cfad", - "tarball_url": "https://api.github.com/repos/OpenGreekAndLatin/First1KGreek/tarball/1.1.5714" + "tarball_url": "https://api.github.com/repos/OpenGreekAndLatin/First1KGreek/tarball/1.1.5714", + "prefer_default_branch": false }, "OpenGreekAndLatin/Latin": { "ref": "v1.12.0", "sha": "034ac92b345cb55c24e7923395637c602ba428d6", - "tarball_url": "https://api.github.com/repos/OpenGreekAndLatin/Latin/tarball/v1.12.0" + "tarball_url": "https://api.github.com/repos/OpenGreekAndLatin/Latin/tarball/v1.12.0", + "prefer_default_branch": false }, "lascivaroma/priapeia": { "ref": "1.1.18", "sha": "0a75c5c2fba85cef125555ea35029409585c54cb", - "tarball_url": "https://api.github.com/repos/lascivaroma/priapeia/tarball/1.1.18" + "tarball_url": "https://api.github.com/repos/lascivaroma/priapeia/tarball/1.1.18", + "prefer_default_branch": false }, "hlapin/ancJewLitCTS": { "ref": "0.1.50", "sha": "f3534557c6cae678bccbbf0d97ac2b2cdf71cc2e", - "tarball_url": "https://api.github.com/repos/hlapin/ancJewLitCTS/tarball/0.1.50" + "tarball_url": "https://api.github.com/repos/hlapin/ancJewLitCTS/tarball/0.1.50", + "prefer_default_branch": false }, "nevenjovanovic/sophocles-ajax-latinus": { "ref": "master", "sha": "aee7611f65a30b8fe17f1833154988c811e886b3", - "tarball_url": "https://api.github.com/repos/nevenjovanovic/sophocles-ajax-latinus/tarball/aee7611f65a30b8fe17f1833154988c811e886b3" + "tarball_url": "https://api.github.com/repos/nevenjovanovic/sophocles-ajax-latinus/tarball/aee7611f65a30b8fe17f1833154988c811e886b3", + "prefer_default_branch": false }, "jacobwegner/PDL": { "ref": "v0.0.2", "sha": "eca3d09f9516c85173bece362749d0cb40c261df", - "tarball_url": "https://api.github.com/repos/jacobwegner/PDL/tarball/v0.0.2" + "tarball_url": "https://api.github.com/repos/jacobwegner/PDL/tarball/v0.0.2", + "prefer_default_branch": false }, "jacobwegner/Multepal-CTS": { "ref": "v0.0.1-rc4", "sha": "f101dd39a7f98eea0442678cac4f83306707d51a", - "tarball_url": "https://api.github.com/repos/jacobwegner/Multepal-CTS/tarball/v0.0.1-rc4" + "tarball_url": "https://api.github.com/repos/jacobwegner/Multepal-CTS/tarball/v0.0.1-rc4", + "prefer_default_branch": false } } \ No newline at end of file diff --git a/scaife_cts_api/update_corpus_shas.py b/scaife_cts_api/update_corpus_shas.py index bfdfb32..de89735 100644 --- a/scaife_cts_api/update_corpus_shas.py +++ b/scaife_cts_api/update_corpus_shas.py @@ -4,6 +4,71 @@ from github import Github, UnknownObjectException +class ReleaseResolver: + def __init__(self, client, repo_name, data): + self.client = client + self.repo_name = repo_name + self.prefer_default_branch = data.get("prefer_default", False) + self.sha = data["sha"] + + def fetch_latest_release(self, repo): + latest_release = repo.get_latest_release() + self.ref = latest_release.tag_name + # NOTE: latest_commit_sha will differ from latest_release.target_committish, because + # the release was created and then the tag was advanced if a HookSet was used + self.latest_commit_sha = repo.get_commit(self.ref).sha + self.tarball_url = latest_release.tarball_url + + def fetch_latest_commit(self, repo): + default_branch = repo.get_branch(repo.default_branch) + self.ref = default_branch.name + self.latest_commit_sha = default_branch.commit.sha + self.tarball_url = ( + f"https://api.github.com/repos/{self.repo_name}/tarball/{self.latest_commit_sha}" + ) + + def resolve_release(self): + diff_url = "" + self.repo = self.client.get_repo(self.repo_name) + + if self.prefer_default_branch: + self.fetch_latest_commit(self.repo) + else: + try: + self.fetch_latest_release(self.repo) + except UnknownObjectException: + print( + f'{self.repo_name} has no release data. retreiving latest SHA from "{repo.default_branch}"' + ) + self.fetch_latest_commit(self.repo) + + should_update = self.latest_commit_sha != self.sha + if should_update: + compared = self.repo.compare(self.sha, self.latest_commit_sha) + diff_url = compared.html_url + return should_update, diff_url + + def emit_status(self, should_update, diff_url): + return [ + self.repo.full_name, + should_update, + self.sha, + self.latest_commit_sha, + diff_url, + ] + + def update_corpus(self, corpus_dict): + should_update, diff_url = self.resolve_release() + + corpus_dict[self.repo_name] = dict( + ref=self.ref, + sha=self.latest_commit_sha, + tarball_url=self.tarball_url, + prefer_default_branch=self.prefer_default_branch, + ) + return self.emit_status(should_update, diff_url) + + def main(): """ Small helper script used to update to latest releases @@ -18,48 +83,15 @@ def main(): else: client = Github() - status = [] + statuses = [] corpus = json.load(open("corpus.json")) new_corpus = dict() for repo_name, data in corpus.items(): - sha = data["sha"] - diff_url = "" - repo = client.get_repo(repo_name) - try: - latest_release = repo.get_latest_release() - ref = latest_release.tag_name - # NOTE: latest_commit_sha will differ from latest_release.target_committish, because - # the release was created and then the tag was advanced if a HookSet was used - latest_commit_sha = repo.get_commit(ref).sha - tarball_url = latest_release.tarball_url - except UnknownObjectException: - print( - f'{repo_name} has no release data. retreiving latest SHA from "{repo.default_branch}"' - ) - default_branch = repo.get_branch(repo.default_branch) - ref = default_branch.name - latest_commit_sha = default_branch.commit.sha - tarball_url = f"https://api.github.com/repos/{repo_name}/tarball/{latest_commit_sha}" - - should_update = latest_commit_sha != sha - if should_update: - compared = repo.compare(sha, latest_commit_sha) - diff_url = compared.html_url + resolver = ReleaseResolver(client, repo_name, data) + + status_result = resolver.update_corpus(new_corpus) + statuses.append(status_result) - new_corpus[repo_name] = dict( - ref=ref, - sha=latest_commit_sha, - tarball_url=tarball_url - ) - status.append( - [ - repo.full_name, - should_update, - sha, - latest_commit_sha, - diff_url, - ] - ) json.dump(new_corpus, open("corpus.json", "w"), indent=2) From 65174d88e1c8d5b37d3a09a51adb33887383be02 Mon Sep 17 00:00:00 2001 From: Jacob Wegner Date: Sun, 28 Nov 2021 15:51:01 -0600 Subject: [PATCH 3/3] Bump SHAs and bugfix for log output --- corpus.json | 14 +++++++------- scaife_cts_api/update_corpus_shas.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/corpus.json b/corpus.json index a094927..d43eff1 100644 --- a/corpus.json +++ b/corpus.json @@ -24,10 +24,10 @@ "prefer_default_branch": false }, "PerseusDL/canonical-pdlpsci": { - "ref": "master", - "sha": "f02ac7533c5e757985e630a4c6e3b6824c1c7f00", - "tarball_url": "https://api.github.com/repos/PerseusDL/canonical-pdlpsci/tarball/f02ac7533c5e757985e630a4c6e3b6824c1c7f00", - "prefer_default_branch": true + "ref": "v0.2.0", + "sha": "e00733153e9e25a0d7263b0c37a9eefe742c71b9", + "tarball_url": "https://api.github.com/repos/PerseusDL/canonical-pdlpsci/tarball/v0.2.0", + "prefer_default_branch": false }, "OpenGreekAndLatin/First1KGreek": { "ref": "1.1.5714", @@ -36,9 +36,9 @@ "prefer_default_branch": false }, "OpenGreekAndLatin/Latin": { - "ref": "v1.12.0", - "sha": "034ac92b345cb55c24e7923395637c602ba428d6", - "tarball_url": "https://api.github.com/repos/OpenGreekAndLatin/Latin/tarball/v1.12.0", + "ref": "v1.14.0", + "sha": "9a3e833b515adad490bdd11977eaf1a84d24b246", + "tarball_url": "https://api.github.com/repos/OpenGreekAndLatin/Latin/tarball/v1.14.0", "prefer_default_branch": false }, "lascivaroma/priapeia": { diff --git a/scaife_cts_api/update_corpus_shas.py b/scaife_cts_api/update_corpus_shas.py index de89735..def5dc9 100644 --- a/scaife_cts_api/update_corpus_shas.py +++ b/scaife_cts_api/update_corpus_shas.py @@ -38,7 +38,7 @@ def resolve_release(self): self.fetch_latest_release(self.repo) except UnknownObjectException: print( - f'{self.repo_name} has no release data. retreiving latest SHA from "{repo.default_branch}"' + f'{self.repo_name} has no release data. retreiving latest SHA from "{self.repo.default_branch}"' ) self.fetch_latest_commit(self.repo)