From 2b7f396782cdfe8d977a555cbf2579e1dfd8f8c8 Mon Sep 17 00:00:00 2001 From: Sean Sunoo <97641529+Ssunoo2@users.noreply.github.com> Date: Sat, 27 Apr 2024 20:32:45 -1000 Subject: [PATCH 1/3] Create github.R --- exec/github.R | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 exec/github.R diff --git a/exec/github.R b/exec/github.R new file mode 100644 index 00000000..29106d2e --- /dev/null +++ b/exec/github.R @@ -0,0 +1,100 @@ +#!/usr/local/bin/Rscript + +# Kaiaulu - https://github.com/sailuh/kaiaulu +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +require(yaml,quietly=TRUE) +require(cli,quietly=TRUE) +require(docopt,quietly=TRUE) +require(kaiaulu,quietly=TRUE) +require(data.table,quietly=TRUE) +require(jsonlite,quietly=TRUE) +require(knitr,quietly=TRUE) +require(magrittr,quietly=TRUE) +require(gt,quietly=TRUE) + + +# Rscript github.R refresh issues ../../kaiaulu/conf/kaiaulu.yml ../../rawdata/github/kaiaulu/issue_search/ +# Rscript github.R refresh comments ../../kaiaulu/conf/kaiaulu.yml ../../rawdata/github/kaiaulu/issue_search/ + +doc <- " +USAGE: + github refresh help + github refresh issues + github refresh comments + github download Github help + github (-h | --help) + github --version + +DESCRIPTION: + Provides functions to refresh Github issue data. Please see + Kaiaulu's README.md for instructions on how to create . + + +OPTIONS: + -h --help Show this screen. + --version Show version. +" + + + +arguments <- docopt::docopt(doc, version = 'Kaiaulu 0.0.0.9600') +if(arguments[["refresh"]] & arguments[["help"]]){ + cli_alert_info("Downloads new data from Github Rest API") +}else if(arguments[["refresh"]] & arguments [["issues"]]){ + + conf_path <- arguments[[""]] + save_path <- arguments[[""]] + + conf <- yaml::read_yaml(conf_path) + + save_path <- path.expand(conf[["issue_tracker"]][["github"]][["replies"]]) + save_path_issue_refresh <- paste0(save_path,"/issue_search/") + save_path_issue <- paste0(save_path,"/issue/") + save_path_issue_or_pr_comments <- paste0(save_path,"/issue_or_pr_comment/") + # Path you wish to save all raw data. A folder with the repo name and sub-folders will be created. + owner <- conf[["issue_tracker"]][["github"]][["owner"]] # Has to match github organization (e.g. github.com/sailuh) + repo <- conf[["issue_tracker"]][["github"]][["repo"]] # Has to match github repository (e.g. github.com/sailuh/perceive) + # your file github_token (a text file) contains the GitHub token API + token <- scan("~/.ssh/github_token",what="character",quiet=TRUE) + + gh_response <- github_api_project_issue_refresh(owner, + repo, + token, + save_path_issue_refresh, + verbose=TRUE) + github_api_iterate_pages(token,gh_response, + save_path_issue_refresh, + prefix="issue", + verbose=TRUE) + + cli_alert_success(paste0("Downloaded new Github issues saved at: ",save_path_issue_refresh)) +}else if(arguments[["refresh"]] & arguments [["comments"]]){ + + conf <- yaml::read_yaml("../conf/kaiaulu.yml") + save_path <- path.expand(conf[["issue_tracker"]][["github"]][["replies"]]) + save_path_issue_or_pr_comments <- paste0(save_path,"/issue_or_pr_comment/") + # Path you wish to save all raw data. A folder with the repo name and sub-folders will be created. + owner <- conf[["issue_tracker"]][["github"]][["owner"]] # Has to match github organization (e.g. github.com/sailuh) + repo <- conf[["issue_tracker"]][["github"]][["repo"]] # Has to match github repository (e.g. github.com/sailuh/perceive) + # your file github_token (a text file) contains the GitHub token API + token <- scan("~/.ssh/github_token",what="character",quiet=TRUE) + + gh_response_issue_or_pr_comment <- github_api_project_issue_or_pr_comment_refresh(owner, + repo, + token, save_path_issue_or_pr_comments, verbose=TRUE) + + # create directory and iterate over data + #dir.create(save_path_issue_or_pr_comments) + github_api_iterate_pages(token,gh_response_issue_or_pr_comment, + save_path_issue_or_pr_comments, + prefix="issue_or_pr_comment", + verbose=TRUE) +} + + + + From 1ec8ca13c38247ce368139857c85e1a3f6f64f59 Mon Sep 17 00:00:00 2001 From: Carlos Paradis Date: Sun, 28 Apr 2024 02:43:31 -0700 Subject: [PATCH 2/3] fix github actions --- .github/workflows/R-CMD-check.yml | 4 ++-- .github/workflows/test-coverage.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/R-CMD-check.yml b/.github/workflows/R-CMD-check.yml index 39b5968f..2350bf81 100644 --- a/.github/workflows/R-CMD-check.yml +++ b/.github/workflows/R-CMD-check.yml @@ -12,7 +12,7 @@ name: R-CMD-check jobs: R-CMD-check: - runs-on: macOS-latest + runs-on: macOS-13 strategy: matrix: r-version: ['4.2'] @@ -65,7 +65,7 @@ jobs: - name: Install UCtags and Update tools.yml if: always() run: | - brew tap universal-ctags/universal-ctags + brew tap homebrew/core brew install --HEAD universal-ctags utags_head=$(ls /usr/local/Cellar/universal-ctags | tail -n 1) sed -i -e "s|utags: \/usr\/local\/Cellar\/universal-ctags\/HEAD-62f0144\/bin\/ctags|utags: \/usr\/local\/Cellar\/universal-ctags\/${utags_head}\/bin\/ctags|g" tools.yml diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index cfc2fecd..e70821a2 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -10,7 +10,7 @@ name: test-coverage jobs: test-coverage: - runs-on: macOS-latest + runs-on: macOS-13 strategy: matrix: r-version: ['4.2'] @@ -57,7 +57,7 @@ jobs: - name: Install UCtags and Update tools.yml if: always() run: | - brew tap universal-ctags/universal-ctags + brew tap homebrew/core brew install --HEAD universal-ctags utags_head=$(ls /usr/local/Cellar/universal-ctags | tail -n 1) sed -i -e "s|utags: \/usr\/local\/Cellar\/universal-ctags\/HEAD-62f0144\/bin\/ctags|utags: \/usr\/local\/Cellar\/universal-ctags\/${utags_head}\/bin\/ctags|g" tools.yml From d75251eae871fdfb60026e11ed5a9a725a71508e Mon Sep 17 00:00:00 2001 From: RavenMarQ Date: Tue, 10 Dec 2024 12:17:28 -1000 Subject: [PATCH 3/3] Update github.R To standardize github.R to match mailinglist.R executable. Also updated implementation to utilize the configuration get functions. --- exec/github.R | 138 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 91 insertions(+), 47 deletions(-) diff --git a/exec/github.R b/exec/github.R index 29106d2e..51102e15 100644 --- a/exec/github.R +++ b/exec/github.R @@ -22,10 +22,10 @@ require(gt,quietly=TRUE) doc <- " USAGE: + github parse help + github parse [--issues | --comments | --pr] github refresh help - github refresh issues - github refresh comments - github download Github help + github refresh [--issues | --comments | --pr] github (-h | --help) github --version @@ -37,62 +37,106 @@ DESCRIPTION: OPTIONS: -h --help Show this screen. --version Show version. + --issues Refreshes/Parses issues + --comments Refreshes/Parses comments + --pr Refreshes/Parses Pull Requests " +arguments <- docopt::docopt(doc, version = 'Kaiaulu 0.0.0.9700') - -arguments <- docopt::docopt(doc, version = 'Kaiaulu 0.0.0.9600') -if(arguments[["refresh"]] & arguments[["help"]]){ - cli_alert_info("Downloads new data from Github Rest API") -}else if(arguments[["refresh"]] & arguments [["issues"]]){ +if(arguments[["refresh"]] & arguments[["help"]]) { + cli_alert_info("Downloads new data (whose type is specified by flags) from Github Rest API") +} else if(arguments[["refresh"]]) { conf_path <- arguments[[""]] - save_path <- arguments[[""]] + project_key <- arguments[[""]] conf <- yaml::read_yaml(conf_path) - - save_path <- path.expand(conf[["issue_tracker"]][["github"]][["replies"]]) - save_path_issue_refresh <- paste0(save_path,"/issue_search/") - save_path_issue <- paste0(save_path,"/issue/") - save_path_issue_or_pr_comments <- paste0(save_path,"/issue_or_pr_comment/") # Path you wish to save all raw data. A folder with the repo name and sub-folders will be created. - owner <- conf[["issue_tracker"]][["github"]][["owner"]] # Has to match github organization (e.g. github.com/sailuh) - repo <- conf[["issue_tracker"]][["github"]][["repo"]] # Has to match github repository (e.g. github.com/sailuh/perceive) + owner <- get_github_owner(conf, "project_key_1") # Has to match github organization (e.g. github.com/sailuh) + repo <- get_github_repo(conf, "project_key_1") # Has to match github repository (e.g. github.com/sailuh/perceive) # your file github_token (a text file) contains the GitHub token API token <- scan("~/.ssh/github_token",what="character",quiet=TRUE) - gh_response <- github_api_project_issue_refresh(owner, - repo, - token, - save_path_issue_refresh, - verbose=TRUE) - github_api_iterate_pages(token,gh_response, - save_path_issue_refresh, - prefix="issue", - verbose=TRUE) - - cli_alert_success(paste0("Downloaded new Github issues saved at: ",save_path_issue_refresh)) -}else if(arguments[["refresh"]] & arguments [["comments"]]){ - - conf <- yaml::read_yaml("../conf/kaiaulu.yml") - save_path <- path.expand(conf[["issue_tracker"]][["github"]][["replies"]]) - save_path_issue_or_pr_comments <- paste0(save_path,"/issue_or_pr_comment/") - # Path you wish to save all raw data. A folder with the repo name and sub-folders will be created. - owner <- conf[["issue_tracker"]][["github"]][["owner"]] # Has to match github organization (e.g. github.com/sailuh) - repo <- conf[["issue_tracker"]][["github"]][["repo"]] # Has to match github repository (e.g. github.com/sailuh/perceive) - # your file github_token (a text file) contains the GitHub token API - token <- scan("~/.ssh/github_token",what="character",quiet=TRUE) - - gh_response_issue_or_pr_comment <- github_api_project_issue_or_pr_comment_refresh(owner, - repo, - token, save_path_issue_or_pr_comments, verbose=TRUE) + if (arguments[["--issues"]]) { + save_path_issue <- get_github_issue_path(conf, project_key) + + gh_response <- github_api_project_issue(owner,repo,token) + dir.create(save_path_issue) + github_api_iterate_pages(token,gh_response, + save_path_issue, + prefix="issue", + verbose = TRUE) + + cli_alert_success(paste0("Downloaded new Github issues saved at: ", save_path_issue)) + } else if(arguments [["--pr"]]){ + save_path_pull_request <- get_github_pull_request_path(conf, project_key) + + gh_response <- github_api_project_pull_request(owner,repo,token) + dir.create(save_path_pull_request) + github_api_iterate_pages(token,gh_response, + save_path_pull_request, + prefix="pull_request", + verbose = TRUE) + + cli_alert_success(paste0("Downloaded new Github Pull Requests saved at: ", save_path_pull_request)) + } else if(arguments [["--comments"]]){ + save_path_issue_or_pr_comments <- path.expand(get_github_issue_or_pr_comment_path(conf, project_key)) + + dir.create(save_path_issue_or_pr_comments) + github_api_iterate_pages(token,gh_response, + save_path_issue_or_pr_comments, + prefix="issue_or_pr_comment", + verbose= TRUE) + + cli_alert_success(paste0("Downloaded new Github issue or PR comments saved at: ", save_path_issue_or_pr_comments)) + } +} else if(arguments[["parse"]] & arguments[["help"]]) { + cli_alert_info("Parses downloaded data (whose type is specified by flags) and saves into a csv denoted by ") +} else if(arguments[["parse"]]) { + conf_path <- arguments[[""]] + save_path <- arguments[[""]] - # create directory and iterate over data - #dir.create(save_path_issue_or_pr_comments) - github_api_iterate_pages(token,gh_response_issue_or_pr_comment, - save_path_issue_or_pr_comments, - prefix="issue_or_pr_comment", - verbose=TRUE) + if (arguments[["--issues"]]) { + save_path_issue <- get_github_issue_path(conf, project_key) + + all_issue <- lapply(list.files(save_path_issue, + full.names = TRUE),read_json) + all_issue <- lapply(all_issue, + github_parse_project_issue) + all_issue <- rbindlist(all_issue,fill=TRUE) + + data.table::fwrite(all_issue, save_path) + cli::cli_alert_success(paste0("Dependencies table was saved at: ", save_path)) + } else if(arguments [["--pr"]]){ + save_path_pull_request <- get_github_pull_request_path(conf, project_key) + + all_pr <- lapply(list.files(save_path_pull_request, + full.names = TRUE),read_json) + all_pr <- lapply(all_pr, + github_parse_project_pull_request) + all_pr <- rbindlist(all_pr,fill=TRUE) + + data.table::fwrite(all_pr, save_path) + cli::cli_alert_success(paste0("Dependencies table was saved at: ", save_path)) + } else if(arguments [["--comments"]]){ + save_path_issue_or_pr_comments <- path.expand(get_github_issue_or_pr_comment_path(conf, project_key)) + + all_issue_or_pr_comments <- lapply(list.files(save_path_issue_or_pr_comments, + full.names = TRUE),read_json) + all_issue_or_pr_comments <- lapply(all_issue_or_pr_comments, + github_parse_project_issue_or_pr_comments) + all_issue_or_pr_comments <- rbindlist(all_issue_or_pr_comments,fill=TRUE) + + data.table::fwrite(all_issue_or_pr_comments, save_path) + cli::cli_alert_success(paste0("Dependencies table was saved at: ", save_path)) + } +} else if (arguments[["-h"]] || arguments[["--help"]]) { + cli::cli_alert_info(doc) +} else if (arguments[["--version"]]) { + cli::cli_alert_info('Kaiaulu 0.0.0.9700') +} else { + stop("No/invalid option(s) provided.") }