From b37f55549461cfe0731b57623f315860b3db49d0 Mon Sep 17 00:00:00 2001 From: Adam Jonas Date: Sun, 29 Nov 2020 18:51:08 -0500 Subject: [PATCH] github-to-sqlite pull-requests command (#48) Thanks, @adamjonas --- README.md | 13 ++ github_to_sqlite/cli.py | 34 ++++ github_to_sqlite/utils.py | 75 ++++++++ tests/pull_requests.json | 370 ++++++++++++++++++++++++++++++++++++ tests/test_pull_requests.py | 128 +++++++++++++ 5 files changed, 620 insertions(+) create mode 100644 tests/pull_requests.json create mode 100644 tests/test_pull_requests.py diff --git a/README.md b/README.md index 4c56694..c670aa2 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Save data from GitHub to a SQLite database. - [How to install](#how-to-install) - [Authentication](#authentication) - [Fetching issues for a repository](#fetching-issues-for-a-repository) +- [Fetching pull requests for a repository](#fetching-pull-requests-for-a-repository) - [Fetching issue comments for a repository](#fetching-issue-comments-for-a-repository) - [Fetching commits for a repository](#fetching-commits-for-a-repository) - [Fetching tags for a repository](#fetching-tags-for-a-repository) @@ -64,6 +65,18 @@ You can use the `--issue` option to only load just one specific issue: $ github-to-sqlite issues github.db simonw/datasette --issue=1 +## Fetching pull-requests for a repository + +While pull-requests are a type of issue, you will get more information on pull-requests by pulling them separately. For example, whether a pull-request has been merged and when. + +Following the API of issues, the `pull-requests` command retrieves all of the pull-requests belonging to a specified repository. + + $ github-to-sqlite pull-requests github.db simonw/datasette + +You can use the `--pull-request` option to only load just one specific pull-request: + + $ github-to-sqlite pull-requests github.db simonw/datasette --pull-request=81 + ## Fetching issue comments for a repository The `issue-comments` command retrieves all of the comments on all of the issues in a repository. diff --git a/github_to_sqlite/cli.py b/github_to_sqlite/cli.py index 757fc7c..f51d4d8 100644 --- a/github_to_sqlite/cli.py +++ b/github_to_sqlite/cli.py @@ -70,6 +70,40 @@ def issues(db_path, repo, issue, auth, load): utils.save_issues(db, issues, repo_full) utils.ensure_db_shape(db) +@cli.command(name="pull-requests") +@click.argument( + "db_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("repo", required=False) +@click.option("--pull-request", help="Just pull this pull-request number") +@click.option( + "-a", + "--auth", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=True), + default="auth.json", + help="Path to auth.json token file", +) +@click.option( + "--load", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=True, exists=True), + help="Load pull-requests JSON from this file instead of the API", +) +def pull_requests(db_path, repo, pull_request, auth, load): + "Save pull_requests for a specified repository, e.g. simonw/datasette" + db = sqlite_utils.Database(db_path) + token = load_token(auth) + repo_full = utils.fetch_repo(repo, token) + if load: + pull_requests = json.load(open(load)) + else: + pull_requests = utils.fetch_pull_requests(repo, token, pull_request) + + pull_requests = list(pull_requests) + utils.save_pull_requests(db, pull_requests, repo_full) + utils.ensure_db_shape(db) + @cli.command(name="issue-comments") @click.argument( diff --git a/github_to_sqlite/utils.py b/github_to_sqlite/utils.py index 5c699dd..7be3e57 100644 --- a/github_to_sqlite/utils.py +++ b/github_to_sqlite/utils.py @@ -8,6 +8,7 @@ "commits": ["message"], "issue_comments": ["body"], "issues": ["title", "body"], + "pull_requests": ["title", "body"], "labels": ["name", "description"], "licenses": ["name"], "milestones": ["title", "description"], @@ -149,6 +150,70 @@ def save_issues(db, issues, repo): table.m2m("labels", label, pk="id") +def save_pull_requests(db, pull_requests, repo): + if "milestones" not in db.table_names(): + if "users" not in db.table_names(): + # So we can define the foreign key from milestones: + db["users"].create({"id": int}, pk="id") + db["milestones"].create( + {"id": int, "title": str, "description": str, "creator": int, "repo": int}, + pk="id", + foreign_keys=(("repo", "repos", "id"), ("creator", "users", "id")), + ) + for original in pull_requests: + # Ignore all of the _url fields + pull_request = { + key: value for key, value in original.items() if not key.endswith("url") + } + # Add repo key + pull_request["repo"] = repo["id"] + # Pull request _links can be flattened to just their URL + pull_request["url"] = pull_request["_links"]["html"]["href"] + pull_request.pop("_links") + # Extract user + pull_request["user"] = save_user(db, pull_request["user"]) + labels = pull_request.pop("labels") + # Head sha + pull_request["head"] = pull_request["head"]["sha"] + pull_request["base"] = pull_request["base"]["sha"] + # Extract milestone + if pull_request["milestone"]: + pull_request["milestone"] = save_milestone(db, pull_request["milestone"], repo["id"]) + # For the moment we ignore the assignees=[] array but we DO turn assignee + # singular into a foreign key reference + pull_request.pop("assignees", None) + if original["assignee"]: + pull_request["assignee"] = save_user(db, pull_request["assignee"]) + pull_request.pop("active_lock_reason") + # ignore requested_reviewers and requested_teams + pull_request.pop("requested_reviewers", None) + pull_request.pop("requested_teams", None) + # Insert record + table = db["pull_requests"].insert( + pull_request, + pk="id", + foreign_keys=[ + ("user", "users", "id"), + ("assignee", "users", "id"), + ("milestone", "milestones", "id"), + ("repo", "repos", "id"), + ], + alter=True, + replace=True, + columns={ + "user": int, + "assignee": int, + "milestone": int, + "repo": int, + "title": str, + "body": str, + }, + ) + # m2m for labels + for label in labels: + table.m2m("labels", label, pk="id") + + def save_user(db, user): # Remove all url fields except avatar_url and html_url to_save = { @@ -274,6 +339,16 @@ def fetch_issues(repo, token=None, issue=None): for issues in paginate(url, headers): yield from issues +def fetch_pull_requests(repo, token=None, pull_request=None): + headers = make_headers(token) + if pull_request is not None: + url = "https://api.github.com/repos/{}/pulls/{}".format(repo, pull_request) + yield from [requests.get(url).json()] + else: + url = "https://api.github.com/repos/{}/pulls?state=all&filter=all".format(repo) + for pull_requests in paginate(url, headers): + yield from pull_requests + def fetch_issue_comments(repo, token=None, issue=None): assert "/" in repo diff --git a/tests/pull_requests.json b/tests/pull_requests.json new file mode 100644 index 0000000..3768245 --- /dev/null +++ b/tests/pull_requests.json @@ -0,0 +1,370 @@ +[ + { + "url": "https://api.github.com/repos/simonw/datasette/pulls/571", + "id": 313384926, + "node_id": "MDExOlB1bGxSZXF1ZXN0MzEzMzg0OTI2", + "html_url": "https://github.com/simonw/datasette/pull/571", + "diff_url": "https://github.com/simonw/datasette/pull/571.diff", + "patch_url": "https://github.com/simonw/datasette/pull/571.patch", + "issue_url": "https://api.github.com/repos/simonw/datasette/issues/571", + "number": 571, + "state": "closed", + "locked": false, + "title": "detect_fts now works with alternative table escaping", + "user": { + "login": "simonw", + "id": 9599, + "node_id": "MDQ6VXNlcjk1OTk=", + "avatar_url": "https://avatars0.githubusercontent.com/u/9599?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/simonw", + "html_url": "https://github.com/simonw", + "followers_url": "https://api.github.com/users/simonw/followers", + "following_url": "https://api.github.com/users/simonw/following{/other_user}", + "gists_url": "https://api.github.com/users/simonw/gists{/gist_id}", + "starred_url": "https://api.github.com/users/simonw/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/simonw/subscriptions", + "organizations_url": "https://api.github.com/users/simonw/orgs", + "repos_url": "https://api.github.com/users/simonw/repos", + "events_url": "https://api.github.com/users/simonw/events{/privacy}", + "received_events_url": "https://api.github.com/users/simonw/received_events", + "type": "User", + "site_admin": false + }, + "body": "Fixes #570", + "created_at": "2019-09-03T00:23:39Z", + "updated_at": "2019-09-03T00:32:28Z", + "closed_at": "2019-09-03T00:32:28Z", + "merged_at": "2019-09-03T00:32:28Z", + "merge_commit_sha": "2dc5c8dc259a0606162673d394ba8cc1c6f54428", + "assignee": null, + "assignees": [ + + ], + "requested_reviewers": [ + + ], + "requested_teams": [ + + ], + "labels": [ + + ], + "milestone": null, + "draft": false, + "commits_url": "https://api.github.com/repos/simonw/datasette/pulls/571/commits", + "review_comments_url": "https://api.github.com/repos/simonw/datasette/pulls/571/comments", + "review_comment_url": "https://api.github.com/repos/simonw/datasette/pulls/comments{/number}", + "comments_url": "https://api.github.com/repos/simonw/datasette/issues/571/comments", + "statuses_url": "https://api.github.com/repos/simonw/datasette/statuses/a85239f69261c10f1a9f90514c8b5d113cb94585", + "head": { + "label": "simonw:detect-fts", + "ref": "detect-fts", + "sha": "a85239f69261c10f1a9f90514c8b5d113cb94585", + "user": { + "login": "simonw", + "id": 9599, + "node_id": "MDQ6VXNlcjk1OTk=", + "avatar_url": "https://avatars0.githubusercontent.com/u/9599?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/simonw", + "html_url": "https://github.com/simonw", + "followers_url": "https://api.github.com/users/simonw/followers", + "following_url": "https://api.github.com/users/simonw/following{/other_user}", + "gists_url": "https://api.github.com/users/simonw/gists{/gist_id}", + "starred_url": "https://api.github.com/users/simonw/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/simonw/subscriptions", + "organizations_url": "https://api.github.com/users/simonw/orgs", + "repos_url": "https://api.github.com/users/simonw/repos", + "events_url": "https://api.github.com/users/simonw/events{/privacy}", + "received_events_url": "https://api.github.com/users/simonw/received_events", + "type": "User", + "site_admin": false + }, + "repo": { + "id": 107914493, + "node_id": "MDEwOlJlcG9zaXRvcnkxMDc5MTQ0OTM=", + "name": "datasette", + "full_name": "simonw/datasette", + "private": false, + "owner": { + "login": "simonw", + "id": 9599, + "node_id": "MDQ6VXNlcjk1OTk=", + "avatar_url": "https://avatars0.githubusercontent.com/u/9599?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/simonw", + "html_url": "https://github.com/simonw", + "followers_url": "https://api.github.com/users/simonw/followers", + "following_url": "https://api.github.com/users/simonw/following{/other_user}", + "gists_url": "https://api.github.com/users/simonw/gists{/gist_id}", + "starred_url": "https://api.github.com/users/simonw/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/simonw/subscriptions", + "organizations_url": "https://api.github.com/users/simonw/orgs", + "repos_url": "https://api.github.com/users/simonw/repos", + "events_url": "https://api.github.com/users/simonw/events{/privacy}", + "received_events_url": "https://api.github.com/users/simonw/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://github.com/simonw/datasette", + "description": "An open source multi-tool for exploring and publishing data", + "fork": false, + "url": "https://api.github.com/repos/simonw/datasette", + "forks_url": "https://api.github.com/repos/simonw/datasette/forks", + "keys_url": "https://api.github.com/repos/simonw/datasette/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/simonw/datasette/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/simonw/datasette/teams", + "hooks_url": "https://api.github.com/repos/simonw/datasette/hooks", + "issue_events_url": "https://api.github.com/repos/simonw/datasette/issues/events{/number}", + "events_url": "https://api.github.com/repos/simonw/datasette/events", + "assignees_url": "https://api.github.com/repos/simonw/datasette/assignees{/user}", + "branches_url": "https://api.github.com/repos/simonw/datasette/branches{/branch}", + "tags_url": "https://api.github.com/repos/simonw/datasette/tags", + "blobs_url": "https://api.github.com/repos/simonw/datasette/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/simonw/datasette/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/simonw/datasette/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/simonw/datasette/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/simonw/datasette/statuses/{sha}", + "languages_url": "https://api.github.com/repos/simonw/datasette/languages", + "stargazers_url": "https://api.github.com/repos/simonw/datasette/stargazers", + "contributors_url": "https://api.github.com/repos/simonw/datasette/contributors", + "subscribers_url": "https://api.github.com/repos/simonw/datasette/subscribers", + "subscription_url": "https://api.github.com/repos/simonw/datasette/subscription", + "commits_url": "https://api.github.com/repos/simonw/datasette/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/simonw/datasette/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/simonw/datasette/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/simonw/datasette/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/simonw/datasette/contents/{+path}", + "compare_url": "https://api.github.com/repos/simonw/datasette/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/simonw/datasette/merges", + "archive_url": "https://api.github.com/repos/simonw/datasette/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/simonw/datasette/downloads", + "issues_url": "https://api.github.com/repos/simonw/datasette/issues{/number}", + "pulls_url": "https://api.github.com/repos/simonw/datasette/pulls{/number}", + "milestones_url": "https://api.github.com/repos/simonw/datasette/milestones{/number}", + "notifications_url": "https://api.github.com/repos/simonw/datasette/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/simonw/datasette/labels{/name}", + "releases_url": "https://api.github.com/repos/simonw/datasette/releases{/id}", + "deployments_url": "https://api.github.com/repos/simonw/datasette/deployments", + "created_at": "2017-10-23T00:39:03Z", + "updated_at": "2020-07-27T20:42:15Z", + "pushed_at": "2020-07-26T01:21:05Z", + "git_url": "git://github.com/simonw/datasette.git", + "ssh_url": "git@github.com:simonw/datasette.git", + "clone_url": "https://github.com/simonw/datasette.git", + "svn_url": "https://github.com/simonw/datasette", + "homepage": "http://datasette.readthedocs.io/", + "size": 3487, + "stargazers_count": 3642, + "watchers_count": 3642, + "language": "Python", + "has_issues": true, + "has_projects": false, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 206, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 190, + "license": { + "key": "apache-2.0", + "name": "Apache License 2.0", + "spdx_id": "Apache-2.0", + "url": "https://api.github.com/licenses/apache-2.0", + "node_id": "MDc6TGljZW5zZTI=" + }, + "forks": 206, + "open_issues": 190, + "watchers": 3642, + "default_branch": "master" + } + }, + "base": { + "label": "simonw:master", + "ref": "master", + "sha": "f04deebec4f3842f7bd610cd5859de529f77d50e", + "user": { + "login": "simonw", + "id": 9599, + "node_id": "MDQ6VXNlcjk1OTk=", + "avatar_url": "https://avatars0.githubusercontent.com/u/9599?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/simonw", + "html_url": "https://github.com/simonw", + "followers_url": "https://api.github.com/users/simonw/followers", + "following_url": "https://api.github.com/users/simonw/following{/other_user}", + "gists_url": "https://api.github.com/users/simonw/gists{/gist_id}", + "starred_url": "https://api.github.com/users/simonw/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/simonw/subscriptions", + "organizations_url": "https://api.github.com/users/simonw/orgs", + "repos_url": "https://api.github.com/users/simonw/repos", + "events_url": "https://api.github.com/users/simonw/events{/privacy}", + "received_events_url": "https://api.github.com/users/simonw/received_events", + "type": "User", + "site_admin": false + }, + "repo": { + "id": 107914493, + "node_id": "MDEwOlJlcG9zaXRvcnkxMDc5MTQ0OTM=", + "name": "datasette", + "full_name": "simonw/datasette", + "private": false, + "owner": { + "login": "simonw", + "id": 9599, + "node_id": "MDQ6VXNlcjk1OTk=", + "avatar_url": "https://avatars0.githubusercontent.com/u/9599?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/simonw", + "html_url": "https://github.com/simonw", + "followers_url": "https://api.github.com/users/simonw/followers", + "following_url": "https://api.github.com/users/simonw/following{/other_user}", + "gists_url": "https://api.github.com/users/simonw/gists{/gist_id}", + "starred_url": "https://api.github.com/users/simonw/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/simonw/subscriptions", + "organizations_url": "https://api.github.com/users/simonw/orgs", + "repos_url": "https://api.github.com/users/simonw/repos", + "events_url": "https://api.github.com/users/simonw/events{/privacy}", + "received_events_url": "https://api.github.com/users/simonw/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://github.com/simonw/datasette", + "description": "An open source multi-tool for exploring and publishing data", + "fork": false, + "url": "https://api.github.com/repos/simonw/datasette", + "forks_url": "https://api.github.com/repos/simonw/datasette/forks", + "keys_url": "https://api.github.com/repos/simonw/datasette/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/simonw/datasette/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/simonw/datasette/teams", + "hooks_url": "https://api.github.com/repos/simonw/datasette/hooks", + "issue_events_url": "https://api.github.com/repos/simonw/datasette/issues/events{/number}", + "events_url": "https://api.github.com/repos/simonw/datasette/events", + "assignees_url": "https://api.github.com/repos/simonw/datasette/assignees{/user}", + "branches_url": "https://api.github.com/repos/simonw/datasette/branches{/branch}", + "tags_url": "https://api.github.com/repos/simonw/datasette/tags", + "blobs_url": "https://api.github.com/repos/simonw/datasette/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/simonw/datasette/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/simonw/datasette/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/simonw/datasette/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/simonw/datasette/statuses/{sha}", + "languages_url": "https://api.github.com/repos/simonw/datasette/languages", + "stargazers_url": "https://api.github.com/repos/simonw/datasette/stargazers", + "contributors_url": "https://api.github.com/repos/simonw/datasette/contributors", + "subscribers_url": "https://api.github.com/repos/simonw/datasette/subscribers", + "subscription_url": "https://api.github.com/repos/simonw/datasette/subscription", + "commits_url": "https://api.github.com/repos/simonw/datasette/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/simonw/datasette/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/simonw/datasette/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/simonw/datasette/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/simonw/datasette/contents/{+path}", + "compare_url": "https://api.github.com/repos/simonw/datasette/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/simonw/datasette/merges", + "archive_url": "https://api.github.com/repos/simonw/datasette/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/simonw/datasette/downloads", + "issues_url": "https://api.github.com/repos/simonw/datasette/issues{/number}", + "pulls_url": "https://api.github.com/repos/simonw/datasette/pulls{/number}", + "milestones_url": "https://api.github.com/repos/simonw/datasette/milestones{/number}", + "notifications_url": "https://api.github.com/repos/simonw/datasette/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/simonw/datasette/labels{/name}", + "releases_url": "https://api.github.com/repos/simonw/datasette/releases{/id}", + "deployments_url": "https://api.github.com/repos/simonw/datasette/deployments", + "created_at": "2017-10-23T00:39:03Z", + "updated_at": "2020-07-27T20:42:15Z", + "pushed_at": "2020-07-26T01:21:05Z", + "git_url": "git://github.com/simonw/datasette.git", + "ssh_url": "git@github.com:simonw/datasette.git", + "clone_url": "https://github.com/simonw/datasette.git", + "svn_url": "https://github.com/simonw/datasette", + "homepage": "http://datasette.readthedocs.io/", + "size": 3487, + "stargazers_count": 3642, + "watchers_count": 3642, + "language": "Python", + "has_issues": true, + "has_projects": false, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 206, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 190, + "license": { + "key": "apache-2.0", + "name": "Apache License 2.0", + "spdx_id": "Apache-2.0", + "url": "https://api.github.com/licenses/apache-2.0", + "node_id": "MDc6TGljZW5zZTI=" + }, + "forks": 206, + "open_issues": 190, + "watchers": 3642, + "default_branch": "master" + } + }, + "_links": { + "self": { + "href": "https://api.github.com/repos/simonw/datasette/pulls/571" + }, + "html": { + "href": "https://github.com/simonw/datasette/pull/571" + }, + "issue": { + "href": "https://api.github.com/repos/simonw/datasette/issues/571" + }, + "comments": { + "href": "https://api.github.com/repos/simonw/datasette/issues/571/comments" + }, + "review_comments": { + "href": "https://api.github.com/repos/simonw/datasette/pulls/571/comments" + }, + "review_comment": { + "href": "https://api.github.com/repos/simonw/datasette/pulls/comments{/number}" + }, + "commits": { + "href": "https://api.github.com/repos/simonw/datasette/pulls/571/commits" + }, + "statuses": { + "href": "https://api.github.com/repos/simonw/datasette/statuses/a85239f69261c10f1a9f90514c8b5d113cb94585" + } + }, + "author_association": "OWNER", + "active_lock_reason": null, + "merged": true, + "mergeable": null, + "rebaseable": null, + "mergeable_state": "unknown", + "merged_by": { + "login": "simonw", + "id": 9599, + "node_id": "MDQ6VXNlcjk1OTk=", + "avatar_url": "https://avatars0.githubusercontent.com/u/9599?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/simonw", + "html_url": "https://github.com/simonw", + "followers_url": "https://api.github.com/users/simonw/followers", + "following_url": "https://api.github.com/users/simonw/following{/other_user}", + "gists_url": "https://api.github.com/users/simonw/gists{/gist_id}", + "starred_url": "https://api.github.com/users/simonw/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/simonw/subscriptions", + "organizations_url": "https://api.github.com/users/simonw/orgs", + "repos_url": "https://api.github.com/users/simonw/repos", + "events_url": "https://api.github.com/users/simonw/events{/privacy}", + "received_events_url": "https://api.github.com/users/simonw/received_events", + "type": "User", + "site_admin": false + }, + "comments": 0, + "review_comments": 0, + "maintainer_can_modify": false, + "commits": 1, + "additions": 7, + "deletions": 3, + "changed_files": 2 + } +] diff --git a/tests/test_pull_requests.py b/tests/test_pull_requests.py new file mode 100644 index 0000000..1d3fd6e --- /dev/null +++ b/tests/test_pull_requests.py @@ -0,0 +1,128 @@ +from github_to_sqlite import utils +import pytest +import pathlib +import sqlite_utils +from sqlite_utils.db import ForeignKey +import json + + +@pytest.fixture +def pull_requests(): + return json.load(open(pathlib.Path(__file__).parent / "pull_requests.json")) + + +@pytest.fixture +def db(pull_requests): + db = sqlite_utils.Database(memory=True) + db["repos"].insert( + {"id": 1}, + pk="id", + columns={"organization": int, "topics": str, "name": str, "description": str}, + ) + utils.save_pull_requests(db, pull_requests, {"id": 1}) + return db + + +def test_tables(db): + assert {"pull_requests", "users", "repos", "milestones"} == set(db.table_names()) + assert { + ForeignKey( + table="pull_requests", column="repo", other_table="repos", other_column="id" + ), + ForeignKey( + table="pull_requests", + column="milestone", + other_table="milestones", + other_column="id", + ), + ForeignKey( + table="pull_requests", + column="assignee", + other_table="users", + other_column="id", + ), + ForeignKey( + table="pull_requests", column="user", other_table="users", other_column="id" + ), + } == set(db["pull_requests"].foreign_keys) + + +def test_pull_requests(db): + pull_request_rows = list(db["pull_requests"].rows) + assert [ + { + "id": 313384926, + "node_id": "MDExOlB1bGxSZXF1ZXN0MzEzMzg0OTI2", + "number": 571, + "state": "closed", + "locked": 0, + "title": "detect_fts now works with alternative table escaping", + "user": 9599, + "body": "Fixes #570", + "created_at": "2019-09-03T00:23:39Z", + "updated_at": "2019-09-03T00:32:28Z", + "closed_at": "2019-09-03T00:32:28Z", + "merged_at": "2019-09-03T00:32:28Z", + "merge_commit_sha": "2dc5c8dc259a0606162673d394ba8cc1c6f54428", + "assignee": None, + "milestone": None, + "draft": 0, + "head": "a85239f69261c10f1a9f90514c8b5d113cb94585", + "base": "f04deebec4f3842f7bd610cd5859de529f77d50e", + "author_association": "OWNER", + "merged": 1, + "mergeable": None, + "rebaseable": None, + "mergeable_state": "unknown", + "merged_by": '{"login": "simonw", "id": 9599, "node_id": "MDQ6VXNlcjk1OTk=", "avatar_url": "https://avatars0.githubusercontent.com/u/9599?v=4", "gravatar_id": "", "url": "https://api.github.com/users/simonw", "html_url": "https://github.com/simonw", "followers_url": "https://api.github.com/users/simonw/followers", "following_url": "https://api.github.com/users/simonw/following{/other_user}", "gists_url": "https://api.github.com/users/simonw/gists{/gist_id}", "starred_url": "https://api.github.com/users/simonw/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/simonw/subscriptions", "organizations_url": "https://api.github.com/users/simonw/orgs", "repos_url": "https://api.github.com/users/simonw/repos", "events_url": "https://api.github.com/users/simonw/events{/privacy}", "received_events_url": "https://api.github.com/users/simonw/received_events", "type": "User", "site_admin": false}', + "comments": 0, + "review_comments": 0, + "maintainer_can_modify": 0, + "commits": 1, + "additions": 7, + "deletions": 3, + "changed_files": 2, + "repo": 1, + "url": "https://github.com/simonw/datasette/pull/571", + } + ] == pull_request_rows + + +def test_users(db): + user_rows = list(db["users"].rows) + assert [ + { + "login": "simonw", + "id": 9599, + "node_id": "MDQ6VXNlcjk1OTk=", + "avatar_url": "https://avatars0.githubusercontent.com/u/9599?v=4", + "gravatar_id": "", + "html_url": "https://github.com/simonw", + "type": "User", + "site_admin": 0, + "name": "simonw", + } + ] == user_rows + + +def test_foreign_keys(db): + assert [ + ForeignKey( + table="pull_requests", column="repo", other_table="repos", other_column="id" + ), + ForeignKey( + table="pull_requests", + column="milestone", + other_table="milestones", + other_column="id", + ), + ForeignKey( + table="pull_requests", + column="assignee", + other_table="users", + other_column="id", + ), + ForeignKey( + table="pull_requests", column="user", other_table="users", other_column="id" + ), + ] == db["pull_requests"].foreign_keys