From f37c2e58672cbb995a76c0be63d62b864f4c50cb Mon Sep 17 00:00:00 2001 From: munja Date: Thu, 20 Oct 2022 17:11:43 +0100 Subject: [PATCH] feat: two new macros (mp_gitadd and mp_gitstatus) with corresponding tests, also a new utility program for deploying the library as a SAS PACKAGE --- .github/workflows/main.yml | 7 + base/mp_gitadd.sas | 46 ++++++ base/mp_gitstatus.sas | 67 +++++++++ sasjs/sasjsconfig.json | 14 ++ sasjs/utils/create_sas_package.sas | 224 +++++++++++++++++++++++++++++ tests/base/mp_gitadd.test.sas | 53 +++++++ tests/base/mp_gitstatus.test.sas | 39 +++++ 7 files changed, 450 insertions(+) create mode 100644 base/mp_gitadd.sas create mode 100644 base/mp_gitstatus.sas create mode 100644 sasjs/utils/create_sas_package.sas create mode 100644 tests/base/mp_gitadd.test.sas create mode 100644 tests/base/mp_gitstatus.test.sas diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b626a6d1..0de0fd1c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,3 +19,10 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + - name: SAS Packages Release + run: | + sasjs compile job -s sasjs/utils/create_sas_package.sas -o sasjsbuild/makepak.sas + # this part depends on https://github.com/sasjs/server/issues/307 + # sasjs run sasjsbuild/makepak.sas -t sas9 + + diff --git a/base/mp_gitadd.sas b/base/mp_gitadd.sas new file mode 100644 index 00000000..369535b9 --- /dev/null +++ b/base/mp_gitadd.sas @@ -0,0 +1,46 @@ +/** + @file + @brief Stages files in a GIT repo + @details Uses the output dataset from mp_gitstatus.sas to determine the files + that should be staged. + + If STAGED != `"TRUE"` then the file is staged (so you could provide an empty + char column if staging all observations). + + Usage: + + %let dir=%sysfunc(pathname(work))/core; + %let repo=https://github.com/sasjs/core; + %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir)); + %mf_writefile(&dir/somefile.txt,l1=some content) + %mf_deletefile(&dir/package.json) + %mp_gitstatus(&dir,outds=work.gitstatus) + + %mp_gitadd(&dir,inds=work.gitstatus) + + @param [in] gitdir The directory containing the GIT repository + @param [in] inds= (work.mp_gitadd) The input dataset with the list of files + to stage. Will accept the output from mp_gitstatus(), else just use a table + with the following columns: + @li path $1024 - relative path to the file in the repo + @li staged $32 - whether the file is staged (TRUE or FALSE) + @li status $64 - either new, deleted, or modified + + @param [in] mdebug= (0) Set to 1 to enable DEBUG messages + +

Related Files

+ @li mp_gitadd.test.sas + @li mp_gitstatus.sas + +**/ + +%macro mp_gitadd(gitdir,inds=work.mp_gitadd,mdebug=0); + +data _null_; + set &inds; + if STAGED ne "TRUE"; + rc=git_index_add("&gitdir",cats(path),status); + if rc ne 0 or &mdebug=1 then put rc=; +run; + +%mend mp_gitadd; diff --git a/base/mp_gitstatus.sas b/base/mp_gitstatus.sas new file mode 100644 index 00000000..6c3aa435 --- /dev/null +++ b/base/mp_gitstatus.sas @@ -0,0 +1,67 @@ +/** + @file + @brief Creates a dataset with the output from `GIT_STATUS()` + @details Uses `git_status()` to fetch the number of changed files, then + iterates through with `git_status_get()` and `git_index_add()` for each + change - which is created in an output dataset. + + Usage: + + %let dir=%sysfunc(pathname(work))/core; + %let repo=https://github.com/sasjs/core; + %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir)); + %mf_writefile(&dir/somefile.txt,l1=some content) + %mf_deletefile(&dir/package.json) + + %mp_gitstatus(&dir,outds=work.gitstatus) + + More info on these functions is in this [helpful paper]( +https://www.sas.com/content/dam/SAS/support/en/sas-global-forum-proceedings/2019/3057-2019.pdf + ) by Danny Zimmerman. + + @param [in] gitdir The directory containing the GIT repository + @param [out] outds= (work.git_status) The output dataset to create. Vars: + @li gitdir $1024 - directory of repo + @li path $1024 - relative path to the file in the repo + @li staged $32 - whether the file is staged (TRUE or FALSE) + @li status $64 - either new, deleted, or modified + @li cnt - number of files + @li n - the "nth" file in the list from git_status() + + @param [in] mdebug= (0) Set to 1 to enable DEBUG messages + +

Related Files

+ @li mp_gitstatus.test.sas + @li mp_gitadd.sas + +**/ + +%macro mp_gitstatus(gitdir,outds=work.mp_gitstatus,mdebug=0); + +data &outds; + LENGTH gitdir path $ 1024 STATUS $ 64 STAGED $ 32; + call missing (of _all_); + gitdir=symget('gitdir'); + cnt=git_status(trim(gitdir)); + if cnt=-1 then do; + put "The libgit2 library is unavailable and no Git operations can be used."; + put "See: https://stackoverflow.com/questions/74082874"; + end; + else if cnt=-2 then do; + put "The libgit2 library is available, but the status function failed."; + put "See the log for details."; + end; + else do n=1 to cnt; + rc=GIT_STATUS_GET(n,gitdir,'PATH',path); + rc=GIT_STATUS_GET(n,gitdir,'STAGED',staged); + rc=GIT_STATUS_GET(n,gitdir,'STATUS',status); + output; + %if &mdebug=1 %then %do; + putlog (_all_)(=); + %end; + end; + rc=git_status_free(gitdir); + drop rc cnt; +run; + +%mend mp_gitstatus; diff --git a/sasjs/sasjsconfig.json b/sasjs/sasjsconfig.json index ae89c0e0..b420ce87 100644 --- a/sasjs/sasjsconfig.json +++ b/sasjs/sasjsconfig.json @@ -73,6 +73,10 @@ "allowInsecureRequests": false }, "appLoc": "/sasjs/core", + "deployConfig": { + "deployServicePack": true, + "deployScripts": [] + }, "macroFolders": [ "server", "tests/serveronly" @@ -105,6 +109,16 @@ "deployServicePack": true }, "contextName": "SAS Job Execution compute context" + }, + { + "name": "sasjs9", + "serverUrl": "https://sas9.4gl.io", + "serverType": "SASJS", + "appLoc": "/Public/app/sasjs9", + "deployConfig": { + "deployServicePack": true, + "deployScripts": [] + } } ] } \ No newline at end of file diff --git a/sasjs/utils/create_sas_package.sas b/sasjs/utils/create_sas_package.sas new file mode 100644 index 00000000..69238b01 --- /dev/null +++ b/sasjs/utils/create_sas_package.sas @@ -0,0 +1,224 @@ +/** + @file + @brief Deploy repo as a SAS PACKAGES module + @details After every release, this program is executed to update the SASPAC + repo with the latest macros (and same version number). + The program is first compiled using sasjs compile, then executed using + sasjs run. + + Requires the server to have SSH keys. + +

SAS Macros

+ @li mp_gitadd.sas + @li mp_gitreleaseinfo.sas + @li mp_gitstatus.sas + +**/ + + +/* get package version */ +%mp_gitreleaseinfo(GITHUB,sasjs/core,outlib=splib) +data _null_; + set splib.root; + call symputx('version',TAG_NAME); +run; + +/* clone the source repo */ +%let dir = %sysfunc(pathname(work))/core; +%put source clone rc=%sysfunc(GITFN_CLONE(https://github.com/sasjs/core,&dir)); + + +/* + clone the target repo. + If you have issues, see: https://stackoverflow.com/questions/74082874 +*/ +options dlcreatedir; +libname _ "&dirOut."; +%let dirOut = %sysfunc(pathname(work))/package; +%put tgt clone rc=%sysfunc(GITFN_CLONE( + git@github.com:allanbowe/sasjscore.git, + &dirOut, + git, + %str( ), + /home/sasjssrv/.ssh/id_ecdsa.pub, + /home/sasjssrv/.ssh/id_ecdsa +)); + + +/* + Prepare Package Metadata +*/ +data _null_; + infile CARDS4; + file "&dirOut./description.sas"; + input; + if _infile_ =: 'Version:' then put "Version: &version."; + else put _infile_; +CARDS4; +Type: Package +Package: SASjsCore +Title: SAS Macros for Application Development +Version: $(PLACEHOLDER) +Author: Allan Bowe +Maintainer: 4GL Ltd +License: MIT +Encoding: UTF8 + +DESCRIPTION START: + +The SASjs Macro Core library is a component of the SASjs framework, the +source for which is avaible here: https://github.com/sasjs + +Macros are divided by: + +* Macro Functions (prefix mf_) +* Macro Procedures (prefix mp_) +* Macros for Metadata (prefix mm_) +* Macros for SASjs Server (prefix ms_) +* Macros for Viya (prefix mv_) + +DESCRIPTION END: +;;;; +run; + +/* + Prepare Package License +*/ +data _null_; + file "&dirOut./license.sas"; + infile "&dir/LICENSE"; + input; + put _infile_; +run; + +/* + Extract Core files into MacroCore Package location +*/ +data members(compress=char); + length dref dref2 $ 8 name name2 $ 32 path $ 2048; + rc = filename(dref, "&dir."); + put dref=; + did = dopen(dref); + if did then + do i = 1 to dnum(did); + name = dread(did, i); + if name in + ("base" "ddl" "fcmp" "lua" "meta" "metax" "server" "viya" "xplatform") + then do; + rc = filename(dref2,catx("/", "&dir.", name)); + put dref2= name; + did2 = dopen(dref2); + + if did2 then + do j = 1 to dnum(did2); + name2 = dread(did2, j); + path = catx("/", "&dir.", name, name2); + if "sas" = scan(name2, -1, ".") then output; + end; + rc = dclose(did2); + rc = filename(dref2); + end; + end; + rc = dclose(did); + rc = filename(dref); + keep name name2 path; +run; + +%let temp_options = %sysfunc(getoption(source)) %sysfunc(getoption(notes)); +options nosource nonotes; +data _null_; + set members; + by name notsorted; + + ord + first.name; + + if first.name then + do; + call execute('libname _ ' + !! quote(catx("/", "&dirOut.", put(ord, z3.)!!"_macros")) + !! ";" + ); + put @1 "./" ord z3. "_macros/"; + end; + + put @10 name2; + call execute(" + data _null_; + infile " !! quote(strip(path)) !! "; + file " !! quote(catx("/", "&dirOut.", put(ord, z3.)!!"_macros", name2)) !!"; + input; + select; + when (2 = trigger) put _infile_; + when (_infile_ = '/**') do; put '/*** HELP START ***//**'; trigger+1; end; + when (_infile_ = '**/') do; put '**//*** HELP END ***/'; trigger+1; end; + otherwise put _infile_; + end; + run;"); + +run; +options &temp_options.; + +/* + Generate SASjsCore Package +*/ +%GeneratePackage( + filesLocation=&dirOut +) + +/** + * apply new version in a github action + * 1. create folder + * 2. create template yaml + * 3. replace version number + */ + +%mf_mkdir(&dirout/.github/workflows) + +%let desc=Version &version of sasjs/core is now on SAS PACKAGES :ok_hand:; +data _null_; + file "&dirout/.github/workflows/release.yml"; + put "name: SASjs Core Package Publish Tag"; + put "on:"; + put " push:"; + put " branches:"; + put " - main"; + put "jobs:"; + put " update:"; + put " runs-on: ubuntu-latest"; + put " steps:"; + put " - uses: actions/checkout@master"; + put " - name: Make Release"; + put " uses: alice-biometrics/release-creator/@v1.0.5"; + put " with:"; + put " github_token: ${{ secrets.GH_TOKEN }}"; + put " branch: main"; + put " draft: false"; + put " version: &version"; + put " description: '&desc'"; +run; + + +/** + * Add, Commit & Push! + */ +%mp_gitstatus(&dirout,outds=work.gitstatus,mdebug=1) +%mp_gitadd(&dirout,inds=work.gitstatus,mdebug=1) + +data _null_; + rc=gitfn_commit("&dirout" + ,"HEAD","&sysuserid","sasjs@core" + ,"FEAT: Releasing &version" + ); + put rc=; + rc=git_push( + "&dirout" + ,"git" + ,"" + ,"/home/sasjssrv/.ssh/id_ecdsa.pub" + ,"/home/sasjssrv/.ssh/id_ecdsa" + ); +run; + + + + diff --git a/tests/base/mp_gitadd.test.sas b/tests/base/mp_gitadd.test.sas new file mode 100644 index 00000000..f351536c --- /dev/null +++ b/tests/base/mp_gitadd.test.sas @@ -0,0 +1,53 @@ +/** + @file + @brief Testing mp_gitadd.sas macro + +

SAS Macros

+ @li mf_deletefile.sas + @li mf_writefile.sas + @li mp_gitadd.sas + @li mp_gitstatus.sas + @li mp_assert.sas + +**/ + +/* clone the source repo */ +%let dir = %sysfunc(pathname(work))/core; +%put source clone rc=%sysfunc(GITFN_CLONE(https://github.com/sasjs/core,&dir)); + +/* add a file */ +%mf_writefile(&dir/somefile.txt,l1=some content) +/* change a file */ +%mf_writefile(&dir/readme.md,l1=new readme) +/* delete a file */ +%mf_deletefile(&dir/package.json) + +/* Run git status */ +%mp_gitstatus(&dir,outds=work.gitstatus) + +%let test1=0; +proc sql noprint; +select count(*) into: test1 from work.gitstatus where staged='FALSE'; + +/* should be three unstaged changes now */ +%mp_assert( + iftrue=(&test1=3), + desc=3 changes are ready to add, + outds=work.test_results +) + +/* add them */ +%mp_gitadd(&dir,inds=work.gitstatus,mdebug=&sasjs_mdebug) + +/* check status */ +%mp_gitstatus(&dir,outds=work.gitstatus2) +%let test2=0; +proc sql noprint; +select count(*) into: test2 from work.gitstatus2 where staged='TRUE'; + +/* should be three staged changes now */ +%mp_assert( + iftrue=(&test2=3), + desc=3 changes were added, + outds=work.test_results +) diff --git a/tests/base/mp_gitstatus.test.sas b/tests/base/mp_gitstatus.test.sas new file mode 100644 index 00000000..1ebcfeac --- /dev/null +++ b/tests/base/mp_gitstatus.test.sas @@ -0,0 +1,39 @@ +/** + @file + @brief Testing mp_gitstatus.sas macro + +

SAS Macros

+ @li mf_deletefile.sas + @li mf_writefile.sas + @li mp_gitstatus.sas + @li mp_assertdsobs.sas + +**/ + +/* clone the source repo */ +%let dir = %sysfunc(pathname(work))/core; +%put source clone rc=%sysfunc(GITFN_CLONE(https://github.com/sasjs/core,&dir)); + +%mp_gitstatus(&dir,outds=work.gitstatus) + +%mp_assert( + iftrue=(&syscc=0), + desc=Initial mp_gitstatus runs without errors, + outds=work.test_results +) + +/* should be empty as there are no changes yet */ +%mp_assertdsobs(work.gitstatus,test=EMPTY) + +/* add a file */ +%mf_writefile(&dir/somefile.txt,l1=some content) +/* change a file */ +%mf_writefile(&dir/readme.md,l1=new readme) +/* delete a file */ +%mf_deletefile(&dir/package.json) + +/* re-run git status */ +%mp_gitstatus(&dir,outds=work.gitstatus) + +/* should be three changes now */ +%mp_assertdsobs(work.gitstatus,test=EQUALS 3)