From 965cc0c71da520d423978a0e3f4b6d7fc519862a Mon Sep 17 00:00:00 2001 From: Erik Pearson Date: Thu, 23 Jan 2020 06:27:42 -0800 Subject: [PATCH 01/47] make generated python code pep8 compliant changes to templates, code parser, and test data --- .../kbase/mobu/compiler/PrevCodeParser.java | 2 +- .../test/DockerClientServerTester.java | 20 +++++++------- .../kbase/templates/python_impl.vm.properties | 26 ++++++++++--------- .../templates/python_server.vm.properties | 13 +++------- 4 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/java/us/kbase/mobu/compiler/PrevCodeParser.java b/src/java/us/kbase/mobu/compiler/PrevCodeParser.java index f59a7a49..b4a4ed33 100644 --- a/src/java/us/kbase/mobu/compiler/PrevCodeParser.java +++ b/src/java/us/kbase/mobu/compiler/PrevCodeParser.java @@ -25,7 +25,7 @@ public class PrevCodeParser { public static HashMap parsePrevCode(File implFile, String commentPrefix, List funcs, boolean withClassHeader) throws IOException, ParseException { - commentPrefix = Pattern.quote(commentPrefix); + commentPrefix = Pattern.quote(commentPrefix) + "[ ]?"; Pattern PAT_HEADER = Pattern.compile(".*" + commentPrefix + "BEGIN_HEADER\n(.*\n)?[ \t]*" + commentPrefix + "END_HEADER\n.*", Pattern.DOTALL); Pattern PAT_CLASS_HEADER = Pattern.compile(".*" + commentPrefix + diff --git a/src/java/us/kbase/mobu/initializer/test/DockerClientServerTester.java b/src/java/us/kbase/mobu/initializer/test/DockerClientServerTester.java index 5f72c4e3..25f2f7c7 100644 --- a/src/java/us/kbase/mobu/initializer/test/DockerClientServerTester.java +++ b/src/java/us/kbase/mobu/initializer/test/DockerClientServerTester.java @@ -153,22 +153,22 @@ protected File initJava(String moduleName) throws Exception { protected File initPython(String moduleName) throws Exception { File moduleDir = new File(moduleName); String implInit = "" + - "#BEGIN_HEADER\n" + - "#END_HEADER\n" + + "# BEGIN_HEADER\n" + + "# END_HEADER\n" + "\n" + - " #BEGIN_CLASS_HEADER\n" + - " #END_CLASS_HEADER\n" + + " # BEGIN_CLASS_HEADER\n" + + " # END_CLASS_HEADER\n" + "\n" + - " #BEGIN_CONSTRUCTOR\n" + - " #END_CONSTRUCTOR\n" + + " # BEGIN_CONSTRUCTOR\n" + + " # END_CONSTRUCTOR\n" + "\n" + - " #BEGIN run_test\n" + + " # BEGIN run_test\n" + " returnVal = input\n" + - " #END run_test\n" + + " # END run_test\n" + "\n" + - " #BEGIN throw_error\n" + + " # BEGIN throw_error\n" + " raise ValueError(input)\n" + - " #END throw_error\n"; + " # END throw_error\n"; File implFile = new File(moduleDir, "lib/" + moduleName + "/" + moduleName + "Impl.py"); init("python", moduleName, implFile, implInit); diff --git a/src/java/us/kbase/templates/python_impl.vm.properties b/src/java/us/kbase/templates/python_impl.vm.properties index ea22c9f7..ac5a3c5b 100644 --- a/src/java/us/kbase/templates/python_impl.vm.properties +++ b/src/java/us/kbase/templates/python_impl.vm.properties @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -#BEGIN_HEADER -${module.py_module_header}#END_HEADER +# BEGIN_HEADER +${module.py_module_header}# END_HEADER #set( $specToPy = {} ) #set( $void = $specToPy.put("list", "list") ) #set( $void = $specToPy.put("tuple", "list") ) @@ -31,14 +31,14 @@ class ${module.module_name}: GIT_URL = "${module.git_url}" GIT_COMMIT_HASH = "${module.git_commit_hash}" - #BEGIN_CLASS_HEADER -${module.py_module_class_header} #END_CLASS_HEADER + # BEGIN_CLASS_HEADER +${module.py_module_class_header} # END_CLASS_HEADER # config contains contents of config file in a hash or None if it couldn't # be found def __init__(self, config): - #BEGIN_CONSTRUCTOR -${module.py_module_constructor} #END_CONSTRUCTOR + # BEGIN_CONSTRUCTOR +${module.py_module_constructor} # END_CONSTRUCTOR pass #set( $status_in_kidl = false ) @@ -74,8 +74,8 @@ ${module.py_module_constructor} #END_CONSTRUCTOR #if( $method.ret_count > 0 ) # return variables are: $display.join($retlist, ", ") #end - #BEGIN ${method.name} -${method.py_user_code} #[[#END]]# ${method.name} + # BEGIN ${method.name} +${method.py_user_code} #[[# END]]# ${method.name} #if( $method.ret_count > 0 ) # At some point might do deeper type checking... @@ -83,8 +83,9 @@ ${method.py_user_code} #[[#END]]# ${method.name} #set( $spectype = $ret.baretype ) #set( $type = $specToPy.get($spectype) ) if not isinstance(${ret.name}, ${type}): - raise ValueError('Method ${method.name} return value ' + - '${ret.name} is not type ${type} as required.') + raise ValueError('Method ${method.name} ' + + 'return value ${ret.name} ' + + 'is not type ${type} as required.') #end # return the results return [$display.join($retlist, ', ')] @@ -93,8 +94,9 @@ ${method.py_user_code} #[[#END]]# ${method.name} #end #end #if( !$status_in_kidl ) + def status(self, ctx): - #BEGIN_STATUS + # BEGIN_STATUS #if( ${module.py_module_status} ) ${module.py_module_status}#else returnVal = {'state': "OK", @@ -103,6 +105,6 @@ ${module.py_module_status}#else 'git_url': self.GIT_URL, 'git_commit_hash': self.GIT_COMMIT_HASH} #end - #END_STATUS + # END_STATUS return [returnVal] #end diff --git a/src/java/us/kbase/templates/python_server.vm.properties b/src/java/us/kbase/templates/python_server.vm.properties index 136f476a..990f1aee 100644 --- a/src/java/us/kbase/templates/python_server.vm.properties +++ b/src/java/us/kbase/templates/python_server.vm.properties @@ -51,6 +51,7 @@ def get_config(): retconfig[nameval[0]] = nameval[1] return retconfig + config = get_config() #set( $specToPy = {} ) @@ -146,12 +147,6 @@ class JSONRPCServiceCustom(JSONRPCService): debugging purposes. """ rdata = jsondata - # we already deserialize the json string earlier in the server code, no - # need to do it again -# try: -# rdata = json.loads(jsondata) -# except ValueError: -# raise ParseError # set some default values for error handling request = self._get_default_vals() @@ -526,6 +521,7 @@ class Application(object): 60) return "%s%+02d:%02d" % (dtnow.isoformat(), hh, mm) + application = Application() # This is the uwsgi application dictionary. On startup uwsgi will look @@ -639,6 +635,7 @@ def process_async_cli(input_file_path, output_file_path, token): f.write(json.dumps(resp, cls=JSONObjectEncoder)) return exit_code + if __name__ == "__main__": if (len(sys.argv) >= 3 and len(sys.argv) <= 4 and os.path.isfile(sys.argv[1])): @@ -668,7 +665,3 @@ if __name__ == "__main__": assert False, "unhandled option" start_server(host=host, port=port) -# print("Listening on port %s" % port) -# httpd = make_server( host, port, application) -# -# httpd.serve_forever() From e3684edf9228d5b9289234dfb262f60c16338a23 Mon Sep 17 00:00:00 2001 From: Erik Pearson Date: Thu, 23 Jan 2020 06:54:09 -0800 Subject: [PATCH 02/47] revert test file change, not necessary and can't be tested. Also, there are a few other test files that should all be changed and tested together... --- .../test/DockerClientServerTester.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/java/us/kbase/mobu/initializer/test/DockerClientServerTester.java b/src/java/us/kbase/mobu/initializer/test/DockerClientServerTester.java index 25f2f7c7..5f72c4e3 100644 --- a/src/java/us/kbase/mobu/initializer/test/DockerClientServerTester.java +++ b/src/java/us/kbase/mobu/initializer/test/DockerClientServerTester.java @@ -153,22 +153,22 @@ protected File initJava(String moduleName) throws Exception { protected File initPython(String moduleName) throws Exception { File moduleDir = new File(moduleName); String implInit = "" + - "# BEGIN_HEADER\n" + - "# END_HEADER\n" + + "#BEGIN_HEADER\n" + + "#END_HEADER\n" + "\n" + - " # BEGIN_CLASS_HEADER\n" + - " # END_CLASS_HEADER\n" + + " #BEGIN_CLASS_HEADER\n" + + " #END_CLASS_HEADER\n" + "\n" + - " # BEGIN_CONSTRUCTOR\n" + - " # END_CONSTRUCTOR\n" + + " #BEGIN_CONSTRUCTOR\n" + + " #END_CONSTRUCTOR\n" + "\n" + - " # BEGIN run_test\n" + + " #BEGIN run_test\n" + " returnVal = input\n" + - " # END run_test\n" + + " #END run_test\n" + "\n" + - " # BEGIN throw_error\n" + + " #BEGIN throw_error\n" + " raise ValueError(input)\n" + - " # END throw_error\n"; + " #END throw_error\n"; File implFile = new File(moduleDir, "lib/" + moduleName + "/" + moduleName + "Impl.py"); init("python", moduleName, implFile, implInit); From 4d7bdb7bc7f093f3da8b6f6a4244e74eac67ec54 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 29 Jan 2020 09:01:35 -0800 Subject: [PATCH 03/47] Allow KBASE_TEST_TOKEN env var to be used as well as hardcoding the test token in the test_local folder --- .travis.yml | 5 ++- .../us/kbase/mobu/tester/ConfigLoader.java | 41 +++++++++++-------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index fd30344f..4e18eb02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,9 @@ language: java python: - "2.7" +jdk: + - openjdk8 + sudo: required services: @@ -42,4 +45,4 @@ script: - make - kb-sdk validate - + diff --git a/src/java/us/kbase/mobu/tester/ConfigLoader.java b/src/java/us/kbase/mobu/tester/ConfigLoader.java index fe947478..05933993 100644 --- a/src/java/us/kbase/mobu/tester/ConfigLoader.java +++ b/src/java/us/kbase/mobu/tester/ConfigLoader.java @@ -29,7 +29,7 @@ public class ConfigLoader { private final String catalogUrl; private final Map secureCfgParams; - public ConfigLoader(Properties props, boolean testMode, + public ConfigLoader(Properties props, boolean testMode, String configPathInfo, boolean tryHomeCfg) throws Exception { if (configPathInfo == null) { configPathInfo = "test_local/test.cfg"; @@ -47,7 +47,12 @@ public ConfigLoader(Properties props, boolean testMode, String tokenString = props.getProperty(testPrefix + "token"); if (tokenString != null && tokenString.trim().isEmpty()) { tokenString = null; + String test_token = System.getenv("KBASE_TEST_TOKEN"); + if (test_token != null && !test_token.trim().isEmpty()) { + tokenString = test_token; + } } + if (user == null && tokenString == null) { throw new IllegalStateException("Error: KBase account credentials are not set in " + configPathInfo); @@ -60,7 +65,7 @@ public ConfigLoader(Properties props, boolean testMode, token = auth.validateToken(tokenString); } else { if (password == null || password.trim().isEmpty()) { - System.out.println("You haven't preset your password in " +configPathInfo + + System.out.println("You haven't preset your password in " +configPathInfo + " file. Please enter it now."); password = new String(System.console().readPassword("Password: ")); } @@ -86,51 +91,51 @@ public ConfigLoader(Properties props, boolean testMode, } } } - + public String getAuthUrl() { return authUrl; } - + public String getAuthAllowInsecure() { return authAllowInsecure; } - + public AuthToken getToken() { return token; } - + public String getEndPoint() { return endPoint; } - + public String getCatalogUrl() { return catalogUrl; } - + public String getHandleUrl() { return handleUrl; } - + public String getJobSrvUrl() { return jobSrvUrl; } - + public String getNjswUrl() { return njswUrl; } - + public String getShockUrl() { return shockUrl; } - + public String getSrvWizUrl() { return srvWizUrl; } - + public String getWsUrl() { return wsUrl; } - + public void generateConfigProperties(File configPropsFile) throws Exception { PrintWriter pw = new PrintWriter(configPropsFile); try { @@ -143,7 +148,7 @@ public void generateConfigProperties(File configPropsFile) throws Exception { pw.println("srv_wiz_url = " + srvWizUrl); pw.println("njsw_url = " + njswUrl); pw.println("auth_service_url = " + authUrl); - pw.println("auth_service_url_allow_insecure = " + + pw.println("auth_service_url_allow_insecure = " + (authAllowInsecure == null ? "false" : authAllowInsecure)); for (String param : secureCfgParams.keySet()) { pw.println(param + " = " + secureCfgParams.get(param)); @@ -152,7 +157,7 @@ public void generateConfigProperties(File configPropsFile) throws Exception { pw.close(); } } - + public CallbackServerConfig buildCallbackServerConfig( URL callbackUrl, Path workDir, LineLogger logger) throws Exception { return new CallbackServerConfigBuilder( @@ -160,8 +165,8 @@ public CallbackServerConfig buildCallbackServerConfig( new URL(handleUrl), new URL(srvWizUrl), new URL(njswUrl), new URL(authUrl), authAllowInsecure, new URL(catalogUrl), callbackUrl, workDir, null, logger).build(); } - - private static String getConfigUrl(Properties props, String key, String endPoint, + + private static String getConfigUrl(Properties props, String key, String endPoint, String defaultUrlSuffix) { String ret = props.getProperty(key); return ret == null ? (endPoint + "/" + defaultUrlSuffix) : ret; From fa01d62002640944a971c0223179b4975aaab43f Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Fri, 31 Jan 2020 14:18:31 -0800 Subject: [PATCH 04/47] Adding KBASE_TEST_TOKEN to entrypoint --- entrypoint | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entrypoint b/entrypoint index e12d17f4..9fad6bca 100755 --- a/entrypoint +++ b/entrypoint @@ -5,7 +5,7 @@ if [ "z$1" = "zshell" ] ; then elif [ "z$1" = "zsetup" ] ; then cat << EOF G=\$(docker run -i -v /var/run/docker.sock:/var/run/docker.sock --entrypoint ls kbase/kb-sdk -l /var/run/docker.sock|awk '{print \$4}'); -alias kb-sdk='docker run -it --rm -v \$HOME:\$HOME -u \$(id -u) -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL --group-add \$G kbase/kb-sdk' +alias kb-sdk='docker run -it --rm -v \$HOME:\$HOME -u \$(id -u) -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e DKBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$G kbase/kb-sdk' EOF elif [ "z$1" = "zgenscript" ] ; then cat << EOF @@ -16,7 +16,7 @@ if [ ! -e \$HOME/.kbsdk.cache ] ; then docker run -i -v /var/run/docker.sock:/var/run/docker.sock --entrypoint ls kbase/kb-sdk -l /var/run/docker.sock|awk '{print \$4}' > \$HOME/.kbsdk.cache fi -exec docker run -it --rm -v \$HOME:\$HOME -u \$(id -u) -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL --group-add \$(cat \$HOME/.kbsdk.cache) kbase/kb-sdk \$@ +exec docker run -it --rm -v \$HOME:\$HOME -u \$(id -u) -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e DKBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$(cat \$HOME/.kbsdk.cache) kbase/kb-sdk \$@ EOF elif [ "z$1" = "zsdkbase" ] || [ "z$1" = "zpull-base-image" ] ; then echo "Pulling and tagging the base image" From ef102684ea363ca863e7860df1c38b904f160961 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Fri, 7 Feb 2020 08:29:20 -0800 Subject: [PATCH 05/47] Removing stray "D" from env var --- entrypoint | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/entrypoint b/entrypoint index 9fad6bca..65251b46 100755 --- a/entrypoint +++ b/entrypoint @@ -1,11 +1,10 @@ #!/bin/sh - if [ "z$1" = "zshell" ] ; then exec $DSHELL -l elif [ "z$1" = "zsetup" ] ; then cat << EOF G=\$(docker run -i -v /var/run/docker.sock:/var/run/docker.sock --entrypoint ls kbase/kb-sdk -l /var/run/docker.sock|awk '{print \$4}'); -alias kb-sdk='docker run -it --rm -v \$HOME:\$HOME -u \$(id -u) -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e DKBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$G kbase/kb-sdk' +alias kb-sdk='docker run -it --rm -v \$HOME:\$HOME -u \$(id -u) -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e KBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$G kbase/kb-sdk' EOF elif [ "z$1" = "zgenscript" ] ; then cat << EOF @@ -16,7 +15,7 @@ if [ ! -e \$HOME/.kbsdk.cache ] ; then docker run -i -v /var/run/docker.sock:/var/run/docker.sock --entrypoint ls kbase/kb-sdk -l /var/run/docker.sock|awk '{print \$4}' > \$HOME/.kbsdk.cache fi -exec docker run -it --rm -v \$HOME:\$HOME -u \$(id -u) -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e DKBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$(cat \$HOME/.kbsdk.cache) kbase/kb-sdk \$@ +exec docker run -it --rm -v \$HOME:\$HOME -u \$(id -u) -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e KBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$(cat \$HOME/.kbsdk.cache) kbase/kb-sdk \$@ EOF elif [ "z$1" = "zsdkbase" ] || [ "z$1" = "zpull-base-image" ] ; then echo "Pulling and tagging the base image" From 35e74791f043adc991f73af1ed1d48575b6cf95d Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 11 Jun 2020 07:18:35 -0700 Subject: [PATCH 06/47] Whitespace changes only: removing trailing spaces, replacing tabs with spaces --- .travis.yml | 2 - Makefile | 2 +- build.xml | 4 +- src/java/us/kbase/mobu/ModuleBuilder.java | 701 +++++++++--------- .../mobu/initializer/ModuleInitializer.java | 390 +++++----- .../kbase/mobu/installer/ClientInstaller.java | 48 +- .../us/kbase/mobu/tester/ModuleTester.java | 46 +- .../mobu/tester/test/ModuleTesterTest.java | 194 ++--- .../us/kbase/mobu/util/ProcessHelper.java | 14 +- .../kbase/mobu/validator/ModuleValidator.java | 372 +++++----- .../kbase/scripts/test/Test8.perl.properties | 4 +- 11 files changed, 887 insertions(+), 890 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4e18eb02..6b83fb97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,5 +44,3 @@ script: - cd MyContigFilter - make - kb-sdk validate - - diff --git a/Makefile b/Makefile index 2ecd55e3..b1e51fb5 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ bin: jars-submodule-init echo '#!/bin/bash' > bin/kb-sdk echo 'DIR=$(DIR)' >> bin/kb-sdk echo 'KBASE_JARS_DIR=$$DIR/submodules/jars/lib/jars' >> bin/kb-sdk - @# Next command processes links in JAR_DEPS_BIN file and has 5 parts (one on each line): + @# Next command processes links in JAR_DEPS_BIN file and has 5 parts (one on each line): @# (1) removing comments @# (2) trimming each line (picking first word actually) @# (3) skipping empty lines diff --git a/build.xml b/build.xml index abbaa391..f1125bc1 100644 --- a/build.xml +++ b/build.xml @@ -49,7 +49,7 @@ - + @@ -205,7 +205,7 @@ echo $pid > $DIR/pid.txt Successfully built: ${ee.dir}/${ee.start.script} - + #!/bin/bash DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) kill $(<"$DIR/pid.txt") diff --git a/src/java/us/kbase/mobu/ModuleBuilder.java b/src/java/us/kbase/mobu/ModuleBuilder.java index 9d6a0e06..54837280 100644 --- a/src/java/us/kbase/mobu/ModuleBuilder.java +++ b/src/java/us/kbase/mobu/ModuleBuilder.java @@ -31,7 +31,7 @@ import us.kbase.mobu.validator.ModuleValidator; public class ModuleBuilder { - + private static final String defaultParentPackage = "us.kbase"; private static final String MODULE_BUILDER_SH_NAME = "kb-sdk"; @@ -45,36 +45,35 @@ public class ModuleBuilder { private static final String INSTALL_COMMAND = "install"; private static final String RUN_COMMAND = "run"; //private static final String SUBMIT_COMMAND = "submit"; - + public static final String GLOBAL_SDK_HOME_ENV_VAR = "KB_SDK_HOME"; public static final String DEFAULT_METHOD_STORE_URL = "https://appdev.kbase.us/services/narrative_method_store/rpc"; - + public static final String VERSION = "1.2.0"; - - + public static void main(String[] args) throws Exception { - - // setup the basic CLI argument parser with global -h and --help commands - GlobalArgs gArgs = new GlobalArgs(); - JCommander jc = new JCommander(gArgs); - jc.setProgramName(MODULE_BUILDER_SH_NAME); - - - // add the 'init' command - InitCommandArgs initArgs = new InitCommandArgs(); - jc.addCommand(INIT_COMMAND, initArgs); - - // add the 'validate' command - ValidateCommandArgs validateArgs = new ValidateCommandArgs(); - jc.addCommand(VALIDATE_COMMAND, validateArgs); - - // add the 'compile' command - CompileCommandArgs compileArgs = new CompileCommandArgs(); - jc.addCommand(COMPILE_COMMAND, compileArgs); - - // add the 'help' command - HelpCommandArgs help = new HelpCommandArgs(); - jc.addCommand(HELP_COMMAND, help); + + // set up the basic CLI argument parser with global -h and --help commands + GlobalArgs gArgs = new GlobalArgs(); + JCommander jc = new JCommander(gArgs); + jc.setProgramName(MODULE_BUILDER_SH_NAME); + + + // add the 'init' command + InitCommandArgs initArgs = new InitCommandArgs(); + jc.addCommand(INIT_COMMAND, initArgs); + + // add the 'validate' command + ValidateCommandArgs validateArgs = new ValidateCommandArgs(); + jc.addCommand(VALIDATE_COMMAND, validateArgs); + + // add the 'compile' command + CompileCommandArgs compileArgs = new CompileCommandArgs(); + jc.addCommand(COMPILE_COMMAND, compileArgs); + + // add the 'help' command + HelpCommandArgs help = new HelpCommandArgs(); + jc.addCommand(HELP_COMMAND, help); // add the 'test' command TestCommandArgs testArgs = new TestCommandArgs(); @@ -91,84 +90,84 @@ public static void main(String[] args) throws Exception { // add the 'install' command InstallCommandArgs installArgs = new InstallCommandArgs(); jc.addCommand(INSTALL_COMMAND, installArgs); - + // add the 'run' command RunCommandArgs runArgs = new RunCommandArgs(); jc.addCommand(RUN_COMMAND, runArgs); - // parse the arguments and gracefully catch any errors - try { - jc.parse(args); - } catch (RuntimeException e) { - showError("Command Line Argument Error", e.getMessage()); - System.exit(1); - } - - // if the help flag is set, then show some help and exit - if(gArgs.help) { - showBriefHelp(jc, System.out); - return; - } - - // no command entered, just print brief info and exit - if(jc.getParsedCommand()==null) { - showBriefHelp(jc, System.out); - return; - } - - // if we get here, we have a valid command, so process it and do stuff - int returnCode = 0; - if(jc.getParsedCommand().equals(HELP_COMMAND)) { - showCommandUsage(jc,help,System.out); - - } else if(jc.getParsedCommand().equals(INIT_COMMAND)) { - returnCode = runInitCommand(initArgs,jc); - } else if(jc.getParsedCommand().equals(VALIDATE_COMMAND)) { - returnCode = runValidateCommand(validateArgs,jc); - } else if(jc.getParsedCommand().equals(COMPILE_COMMAND)) { - returnCode = runCompileCommand(compileArgs,jc); + // parse the arguments and gracefully catch any errors + try { + jc.parse(args); + } catch (RuntimeException e) { + showError("Command Line Argument Error", e.getMessage()); + System.exit(1); + } + + // if the help flag is set, then show some help and exit + if(gArgs.help) { + showBriefHelp(jc, System.out); + return; + } + + // no command entered, just print brief info and exit + if(jc.getParsedCommand()==null) { + showBriefHelp(jc, System.out); + return; + } + + // if we get here, we have a valid command, so process it and do stuff + int returnCode = 0; + if(jc.getParsedCommand().equals(HELP_COMMAND)) { + showCommandUsage(jc,help,System.out); + + } else if(jc.getParsedCommand().equals(INIT_COMMAND)) { + returnCode = runInitCommand(initArgs,jc); + } else if(jc.getParsedCommand().equals(VALIDATE_COMMAND)) { + returnCode = runValidateCommand(validateArgs,jc); + } else if(jc.getParsedCommand().equals(COMPILE_COMMAND)) { + returnCode = runCompileCommand(compileArgs,jc); } else if(jc.getParsedCommand().equals(TEST_COMMAND)) { returnCode = runTestCommand(testArgs,jc); - } else if (jc.getParsedCommand().equals(VERSION_COMMAND)) { - returnCode = runVersionCommand(versionArgs, jc); - } else if (jc.getParsedCommand().equals(RENAME_COMMAND)) { - returnCode = runRenameCommand(renameArgs, jc); - } else if (jc.getParsedCommand().equals(INSTALL_COMMAND)) { - returnCode = runInstallCommand(installArgs, jc); - } else if (jc.getParsedCommand().equals(RUN_COMMAND)) { - returnCode = runRunCommand(runArgs, jc); - } - - if(returnCode!=0) { - System.exit(returnCode); - } + } else if (jc.getParsedCommand().equals(VERSION_COMMAND)) { + returnCode = runVersionCommand(versionArgs, jc); + } else if (jc.getParsedCommand().equals(RENAME_COMMAND)) { + returnCode = runRenameCommand(renameArgs, jc); + } else if (jc.getParsedCommand().equals(INSTALL_COMMAND)) { + returnCode = runInstallCommand(installArgs, jc); + } else if (jc.getParsedCommand().equals(RUN_COMMAND)) { + returnCode = runRunCommand(runArgs, jc); + } + + if(returnCode!=0) { + System.exit(returnCode); + } } - + private static int runValidateCommand(ValidateCommandArgs validateArgs, JCommander jc) { - // initialize - if(validateArgs.modules==null) { - validateArgs.modules = new ArrayList(); - } - if(validateArgs.modules.size()==0) { - validateArgs.modules.add("."); - } - try { - ModuleValidator mv = new ModuleValidator(validateArgs.modules,validateArgs.verbose, - validateArgs.methodStoreUrl, validateArgs.allowSyncMethods); - return mv.validateAll(); - } catch (Exception e) { + // initialize + if(validateArgs.modules==null) { + validateArgs.modules = new ArrayList(); + } + if(validateArgs.modules.size()==0) { + validateArgs.modules.add("."); + } + try { + ModuleValidator mv = new ModuleValidator(validateArgs.modules,validateArgs.verbose, + validateArgs.methodStoreUrl, validateArgs.allowSyncMethods); + return mv.validateAll(); + } catch (Exception e) { if (validateArgs.verbose) e.printStackTrace(); showError("Error while validating module", e.getMessage()); return 1; } - } + } /** * Runs the module initialization command - this creates a new module in the relative directory name given. * There's only a couple of possible arguments here in the initArgs: * userName (required) - the user's Github user name, used to set up some optional fields - * moduleNames (required) - this catchall represents the module's name. Any whitespace (e.g. token breaks) + * moduleNames (required) - this catchall represents the module's name. Any whitespace (e.g. token breaks) * are replaced with underscores. So if a user runs: * kb-sdk init my new module * they get a module called "my_new_module" in a directory of the same name. @@ -176,80 +175,80 @@ private static int runValidateCommand(ValidateCommandArgs validateArgs, JCommand * @param jc * @return */ - private static int runInitCommand(InitCommandArgs initArgs, JCommander jc) { - // Figure out module name. - // Join together spaced out names with underscores if necessary. - if (initArgs.moduleNames == null || initArgs.moduleNames.size() == 0) { - ModuleBuilder.showError("Init Error", "A module name is required."); - return 1; - } - String moduleName = StringUtils.join(initArgs.moduleNames, "_"); - - // Get username if available - String userName = null; - if (initArgs.userName != null) - userName = initArgs.userName; - - // Get chosen language - String language = ModuleInitializer.DEFAULT_LANGUAGE; - if (initArgs.language != null) { - language = initArgs.language; - } - try { - ModuleInitializer initer = new ModuleInitializer(moduleName, userName, language, initArgs.verbose); - initer.initialize(initArgs.example); - } - catch (Exception e) { + private static int runInitCommand(InitCommandArgs initArgs, JCommander jc) { + // Figure out module name. + // Join together spaced out names with underscores if necessary. + if (initArgs.moduleNames == null || initArgs.moduleNames.size() == 0) { + ModuleBuilder.showError("Init Error", "A module name is required."); + return 1; + } + String moduleName = StringUtils.join(initArgs.moduleNames, "_"); + + // Get username if available + String userName = null; + if (initArgs.userName != null) + userName = initArgs.userName; + + // Get chosen language + String language = ModuleInitializer.DEFAULT_LANGUAGE; + if (initArgs.language != null) { + language = initArgs.language; + } + try { + ModuleInitializer initer = new ModuleInitializer(moduleName, userName, language, initArgs.verbose); + initer.initialize(initArgs.example); + } + catch (Exception e) { if (initArgs.verbose) e.printStackTrace(); - showError("Error while initializing module", e.getMessage()); - return 1; - } - return 0; - } - - public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { - printVersion(); - // Step 1: convert list of args to a Files (this must be defined because it is required) - File specFile = null; - try { - if(a.specFileNames.size()>1) { - // right now we only support compiling one file at a time + showError("Error while initializing module", e.getMessage()); + return 1; + } + return 0; + } + + public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { + printVersion(); + // Step 1: convert list of args to a Files (this must be defined because it is required) + File specFile = null; + try { + if(a.specFileNames.size()>1) { + // right now we only support compiling one file at a time ModuleBuilder.showError("Compile Error","Too many input KIDL spec files provided", - "Currently there is only support for compiling one module at a time, which \n"+ - "is required to register data types and KIDL specs with the Workspace. This \n"+ - "code may be extended in the future to allow multiple modules to be compiled \n"+ - "together from separate files into the same client/server."); + "Currently there is only support for compiling one module at a time, which \n"+ + "is required to register data types and KIDL specs with the Workspace. This \n"+ + "code may be extended in the future to allow multiple modules to be compiled \n"+ + "together from separate files into the same client/server."); return 1; - } - List specFiles = convertToFiles(a.specFileNames); - specFile = specFiles.get(0); - } catch (IOException | RuntimeException e) { + } + List specFiles = convertToFiles(a.specFileNames); + specFile = specFiles.get(0); + } catch (IOException | RuntimeException e) { showError("Error accessing input KIDL spec file", e.getMessage()); if (a.verbose) { e.printStackTrace(); } return 1; - } - + } + if (a.clAsyncVer != null && a.dynservVer != null) { showError("Bad arguments", "clasyncver and dynserver cannot both be specified"); return 1; } - - // Step 2: check or create the output directory - File outDir = a.outDir == null ? new File(".") : new File(a.outDir); + + // Step 2: check or create the output directory + File outDir = a.outDir == null ? new File(".") : new File(a.outDir); try { - outDir = outDir.getCanonicalFile(); - } catch (IOException e) { + outDir = outDir.getCanonicalFile(); + } catch (IOException e) { showError("Error accessing output directory", e.getMessage()); return 1; - } + } if (!outDir.exists()) outDir.mkdirs(); - - // Step 3: validate the URL option if it is defined + + // Step 3: validate the URL option if it is defined if (a.url != null) { try { new URL(a.url); @@ -258,7 +257,7 @@ public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { return 1; } } - + // Step 4: info collection for status method File moduleDir = specFile.getAbsoluteFile().getParentFile(); String semanticVersion = null; @@ -290,13 +289,13 @@ public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { } } try { - RunCompileCommand.generate(specFile, a.url, a.jsClientSide, a.jsClientName, a.perlClientSide, - a.perlClientName, a.perlServerSide, a.perlServerName, a.perlImplName, - a.perlPsgiName, a.perlEnableRetries, a.pyClientSide, a.pyClientName, - a.pyServerSide, a.pyServerName, a.pyImplName, a.javaClientSide, - a.javaServerSide, a.javaPackageParent, a.javaSrcDir, a.javaLibDir, - a.javaBuildXml, a.javaGwtPackage, a.rClientSide, a.rClientName, - a.rServerSide, a.rServerName, a.rImplName, outDir, a.jsonSchema, + RunCompileCommand.generate(specFile, a.url, a.jsClientSide, a.jsClientName, a.perlClientSide, + a.perlClientName, a.perlServerSide, a.perlServerName, a.perlImplName, + a.perlPsgiName, a.perlEnableRetries, a.pyClientSide, a.pyClientName, + a.pyServerSide, a.pyServerName, a.pyImplName, a.javaClientSide, + a.javaServerSide, a.javaPackageParent, a.javaSrcDir, a.javaLibDir, + a.javaBuildXml, a.javaGwtPackage, a.rClientSide, a.rClientName, + a.rServerSide, a.rServerName, a.rImplName, outDir, a.jsonSchema, a.makefile, a.clAsyncVer, a.dynservVer, a.html, semanticVersion, gitUrl, gitCommitHash); } catch (Throwable e) { @@ -310,30 +309,30 @@ public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { return 0; } - private static String getCmdOutput(File workDir, String... cmd) throws Exception { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - ProcessHelper.cmd(cmd).exec(workDir, null, pw); - return sw.toString().trim(); - } - + private static String getCmdOutput(File workDir, String... cmd) throws Exception { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + ProcessHelper.cmd(cmd).exec(workDir, null, pw); + return sw.toString().trim(); + } + static List convertToFiles(List filenames) throws IOException { - List files = new ArrayList(filenames.size()); - for(String filename: filenames) { - File f = new File(filename); - if(!f.exists()) { - throw new IOException("KIDL file \""+filename+"\" does not exist"); - } + List files = new ArrayList(filenames.size()); + for(String filename: filenames) { + File f = new File(filename); + if(!f.exists()) { + throw new IOException("KIDL file \""+filename+"\" does not exist"); + } files.add(f); - } - return files; + } + return files; } - + public static class GlobalArgs { - @Parameter(names = {"-h","--help"}, help = true, description="Display help and full usage information.") - boolean help; + @Parameter(names = {"-h","--help"}, help = true, description="Display help and full usage information.") + boolean help; } - + /** * Runs the module test command - this runs tests in local docker container. * @param testArgs @@ -375,7 +374,7 @@ public static String getGitCommit() { } catch (Exception ignore) {} return gitCommit; } - + private static int runVersionCommand(VersionCommandArgs testArgs, JCommander jc) { printVersion(); return 0; @@ -398,14 +397,14 @@ private static int runRenameCommand(RenameCommandArgs renameArgs, JCommander jc) private static int runInstallCommand(InstallCommandArgs installArgs, JCommander jc) { if (installArgs.moduleName == null || installArgs.moduleName.size() != 1) { - ModuleBuilder.showError("Command Line Argument Error", + ModuleBuilder.showError("Command Line Argument Error", "One and only one module name should be provided"); return 1; } try { return new ClientInstaller().install(installArgs.lang, installArgs.async, - installArgs.core || installArgs.sync, installArgs.dynamic, - installArgs.tagVer, installArgs.verbose, installArgs.moduleName.get(0), + installArgs.core || installArgs.sync, installArgs.dynamic, + installArgs.tagVer, installArgs.verbose, installArgs.moduleName.get(0), null, installArgs.clientName); } catch (Exception e) { if (installArgs.verbose) @@ -417,17 +416,17 @@ private static int runInstallCommand(InstallCommandArgs installArgs, JCommander private static int runRunCommand(RunCommandArgs runArgs, JCommander jc) { if (runArgs.methodName == null || runArgs.methodName.size() != 1) { - ModuleBuilder.showError("Command Line Argument Error", + ModuleBuilder.showError("Command Line Argument Error", "One and only one method name should be provided"); return 1; } try { if (runArgs.inputFile == null && runArgs.inputJson == null && !runArgs.stdin) throw new IllegalStateException("No one input method is used " + - "(should be one of '-i', '-j' or '-s')"); - return new ModuleRunner(runArgs.sdkHome).run(runArgs.methodName.get(0), - runArgs.inputFile, runArgs.stdin, runArgs.inputJson, runArgs.output, - runArgs.tagVer, runArgs.verbose, runArgs.keepTempFiles, runArgs.provRefs, + "(should be one of '-i', '-j' or '-s')"); + return new ModuleRunner(runArgs.sdkHome).run(runArgs.methodName.get(0), + runArgs.inputFile, runArgs.stdin, runArgs.inputJson, runArgs.output, + runArgs.tagVer, runArgs.verbose, runArgs.keepTempFiles, runArgs.provRefs, runArgs.mountPoints); } catch (Exception e) { if (runArgs.verbose) @@ -439,150 +438,150 @@ private static int runRunCommand(RunCommandArgs runArgs, JCommander jc) { @Parameters(commandDescription = "Validate a module or modules.") private static class ValidateCommandArgs { - @Parameter(names={"-v","--verbose"}, description="Show verbose output") + @Parameter(names={"-v","--verbose"}, description="Show verbose output") boolean verbose = false; - - @Parameter(names={"-m", "--method_store"}, description="Narrative Method Store URL " + - "(default is " + DEFAULT_METHOD_STORE_URL + ")") - String methodStoreUrl = DEFAULT_METHOD_STORE_URL; - - @Parameter(names={"-a","--allow_sync_method"}, description="Allow synchonous methods " + - "(advanced option, default is false)") + + @Parameter(names={"-m", "--method_store"}, description="Narrative Method Store URL " + + "(default is " + DEFAULT_METHOD_STORE_URL + ")") + String methodStoreUrl = DEFAULT_METHOD_STORE_URL; + + @Parameter(names={"-a","--allow_sync_method"}, description="Allow synchonous methods " + + "(advanced option, default is false)") boolean allowSyncMethods = false; - - @Parameter(description="[path to the module directories]") + + @Parameter(description="[path to the module directories]") List modules; } - + @Parameters(commandDescription = "Initialize a module in the current directory.") private static class InitCommandArgs { - @Parameter(names={"-v","--verbose"}, description="Show verbose output") - boolean verbose = false; - - @Parameter(required=true, names={"-u","--user"}, - description="(Required) provide a username to serve as the owner of this module") - String userName; - - @Parameter(names={"-e","--example"}, - description="Include a fully featured example in your module. " + - "This generates an example set of code and configurations that can " + - "be used to demonstrate a very simple example.") - boolean example = false; - - @Parameter(names={"-l","--language"}, description="Choose a language for developing " + - " code in your module. You can currently choose from Python, Perl, and Java " + - "(default=Python)") - String language = ModuleInitializer.DEFAULT_LANGUAGE; - - @Parameter(required=true, description="") - List moduleNames; + @Parameter(names={"-v","--verbose"}, description="Show verbose output") + boolean verbose = false; + + @Parameter(required=true, names={"-u","--user"}, + description="(Required) provide a username to serve as the owner of this module") + String userName; + + @Parameter(names={"-e","--example"}, + description="Include a fully featured example in your module. " + + "This generates an example set of code and configurations that can " + + "be used to demonstrate a very simple example.") + boolean example = false; + + @Parameter(names={"-l","--language"}, description="Choose a language for developing " + + " code in your module. You can currently choose from Python, Perl, and Java " + + "(default=Python)") + String language = ModuleInitializer.DEFAULT_LANGUAGE; + + @Parameter(required=true, description="") + List moduleNames; } - - + + @Parameters(commandDescription = "Get help and usage information.") private static class HelpCommandArgs { - @Parameter(description="Show usage for this command") + @Parameter(description="Show usage for this command") List command; - @Parameter(names={"-a","--all"}, description="Show usage for all commands") + @Parameter(names={"-a","--all"}, description="Show usage for all commands") boolean showAll = false; } - + @Parameters(commandDescription = "Compile a KIDL file into client and server code") public static class CompileCommandArgs { - - @Parameter(names="--out",description="Set the output folder name (default is the current directory)") - //, metaVar="") + + @Parameter(names="--out",description="Set the output folder name (default is the current directory)") + //, metaVar="") String outDir = null; - - @Parameter(names="--js", description="Generate a JavaScript client with a standard default name") + + @Parameter(names="--js", description="Generate a JavaScript client with a standard default name") boolean jsClientSide = false; - @Parameter(names="--jsclname", description="Generate a JavaScript client with the name provided by this option" + - "(overrides --js option)")//, metaVar = "") + @Parameter(names="--jsclname", description="Generate a JavaScript client with the name provided by this option" + + "(overrides --js option)")//, metaVar = "") String jsClientName = null; - @Parameter(names="--pl", description="Generate a Perl client with the default name") + @Parameter(names="--pl", description="Generate a Perl client with the default name") boolean perlClientSide = false; - @Parameter(names="--plclname", description="Generate a Perl client with the name provided optionally " + - "prefixed by subdirectories separated by :: as in the standard perl module syntax (e.g. "+ - "Bio::KBase::MyModule::Client, overrides the --pl option)")//, metaVar = "") + @Parameter(names="--plclname", description="Generate a Perl client with the name provided optionally " + + "prefixed by subdirectories separated by :: as in the standard perl module syntax (e.g. "+ + "Bio::KBase::MyModule::Client, overrides the --pl option)")//, metaVar = "") String perlClientName = null; - @Parameter(names="--plsrv", description="Generate a Perl server with a " + - "standard default name. If set, Perl clients will automatically be generated too.") + @Parameter(names="--plsrv", description="Generate a Perl server with a " + + "standard default name. If set, Perl clients will automatically be generated too.") boolean perlServerSide = false; - @Parameter(names="--plsrvname", description="Generate a Perl server with with the " + - "name provided, optionally prefixed by subdirectories separated by :: as "+ - "in the standard perl module syntax (e.g. Bio::KBase::MyModule::Server, "+ - "overrides the --plserv option). If set, Perl clients will be generated too.") - //, metaVar = "") + @Parameter(names="--plsrvname", description="Generate a Perl server with with the " + + "name provided, optionally prefixed by subdirectories separated by :: as "+ + "in the standard perl module syntax (e.g. Bio::KBase::MyModule::Server, "+ + "overrides the --plserv option). If set, Perl clients will be generated too.") + //, metaVar = "") String perlServerName = null; - @Parameter(names="--plimplname", description="Generate a Perl server implementation with the " + - "name provided, optionally prefixed by subdirectories separated by :: as "+ - "in the standard Perl module syntax (e.g. Bio::KBase::MyModule::Impl). "+ - "If set, Perl server and client code will be generated too.")//, metaVar = "") + @Parameter(names="--plimplname", description="Generate a Perl server implementation with the " + + "name provided, optionally prefixed by subdirectories separated by :: as "+ + "in the standard Perl module syntax (e.g. Bio::KBase::MyModule::Impl). "+ + "If set, Perl server and client code will be generated too.")//, metaVar = "") String perlImplName = null; - @Parameter(names="--plpsginame", description="Generate a perl PSGI file with the name provided.")//, metaVar = "") + @Parameter(names="--plpsginame", description="Generate a perl PSGI file with the name provided.")//, metaVar = "") String perlPsgiName = null; - @Parameter(names="--plenableretries", description="When set, generated Perl client code will enable "+ - "reconnection retries with the server") + @Parameter(names="--plenableretries", description="When set, generated Perl client code will enable "+ + "reconnection retries with the server") boolean perlEnableRetries = false; - @Parameter(names="--py", description="Generate a Python client with a standard default name") + @Parameter(names="--py", description="Generate a Python client with a standard default name") boolean pyClientSide = false; - @Parameter(names="--pyclname", description="Generate a Python client with with the " + - "name provided, optionally prefixed by subdirectories separated by '.' as "+ - "in the standard Python module syntax (e.g. biokbase.mymodule.client,"+ - "overrides the --py option).")//, metaVar = "") + @Parameter(names="--pyclname", description="Generate a Python client with with the " + + "name provided, optionally prefixed by subdirectories separated by '.' as "+ + "in the standard Python module syntax (e.g. biokbase.mymodule.client,"+ + "overrides the --py option).")//, metaVar = "") String pyClientName = null; - @Parameter(names="--pysrv", description="Generate a Python server with a " + - "standard default name. If set, Python clients will automatically be generated too.") + @Parameter(names="--pysrv", description="Generate a Python server with a " + + "standard default name. If set, Python clients will automatically be generated too.") boolean pyServerSide = false; - @Parameter(names="--pysrvname", description="Generate a Python server with the " + - "name provided, optionally prefixed by subdirectories separated by '.' as "+ - "in the standard Python module syntax (e.g. biokbase.mymodule.server,"+ - "overrides the --pysrv option).")//, metaVar = "") + @Parameter(names="--pysrvname", description="Generate a Python server with the " + + "name provided, optionally prefixed by subdirectories separated by '.' as "+ + "in the standard Python module syntax (e.g. biokbase.mymodule.server,"+ + "overrides the --pysrv option).")//, metaVar = "") String pyServerName = null; - @Parameter(names="--pyimplname", description="Generate a Python server implementation with the " + - "name provided, optionally prefixed by subdirectories separated by '.' as "+ - "in the standard Python module syntax (e.g. biokbase.mymodule.impl)." + - " If set, Python server and client code will be generated too.")//, metaVar = "") + @Parameter(names="--pyimplname", description="Generate a Python server implementation with the " + + "name provided, optionally prefixed by subdirectories separated by '.' as "+ + "in the standard Python module syntax (e.g. biokbase.mymodule.impl)." + + " If set, Python server and client code will be generated too.")//, metaVar = "") String pyImplName = null; - @Parameter(names="--java", description="Generate Java client code in the directory set by --javasrc") + @Parameter(names="--java", description="Generate Java client code in the directory set by --javasrc") boolean javaClientSide = false; - @Parameter(names="--javasrc",description="Set the output folder for generated Java code")//, metaVar = - //"") + @Parameter(names="--javasrc",description="Set the output folder for generated Java code")//, metaVar = + //"") String javaSrcDir = "src"; - @Parameter(names="--javalib",description="Set the output folder for jar files (if defined then --java will be " + - "treated as true automatically)")//, metaVar = - //"") + @Parameter(names="--javalib",description="Set the output folder for jar files (if defined then --java will be " + + "treated as true automatically)")//, metaVar = + //"") String javaLibDir = null; - @Parameter(names="--url", description="Set the default url for the service in generated client code")//, metaVar = "") + @Parameter(names="--url", description="Set the default url for the service in generated client code")//, metaVar = "") String url = null; - @Parameter(names="--javapackage",description="Set the Java package for generated code (module subpackages are " + - "created in this package), default value is " + defaultParentPackage)//, - //metaVar = "") + @Parameter(names="--javapackage",description="Set the Java package for generated code (module subpackages are " + + "created in this package), default value is " + defaultParentPackage)//, + //metaVar = "") String javaPackageParent = defaultParentPackage; - @Parameter(names="--javasrv", description="Generate Java server code in the directory set by --javasrc") + @Parameter(names="--javasrv", description="Generate Java server code in the directory set by --javasrc") boolean javaServerSide = false; - @Parameter(names="--javagwt",description="Generate a GWT client Java package (useful if you need " + - "copies of generated classes for GWT clients)")//, metaVar="") + @Parameter(names="--javagwt",description="Generate a GWT client Java package (useful if you need " + + "copies of generated classes for GWT clients)")//, metaVar="") String javaGwtPackage = null; @Parameter(names="--r", description="DEPRECATED Generate a R client with a standard default name") @@ -610,20 +609,20 @@ public static class CompileCommandArgs { " If set, R server and client code will be generated too.") String rImplName = null; - @Parameter(names="--jsonschema",description="Generate JSON schema documents for the types in the output folder specified.")//, metaVar="") + @Parameter(names="--jsonschema",description="Generate JSON schema documents for the types in the output folder specified.")//, metaVar="") String jsonSchema = null; - - @Parameter(names="--javabuildxml",description="Will generate build.xml template for Ant") + + @Parameter(names="--javabuildxml",description="Will generate build.xml template for Ant") boolean javaBuildXml; - - @Parameter(names="--makefile",description="Will generate makefile templates for servers and/or java client") + + @Parameter(names="--makefile",description="Will generate makefile templates for servers and/or java client") boolean makefile = false; @Parameter(names="--clasyncver",description="Will set in client code version of service for asyncronous calls " + - "(it could be git commit hash of version registered in catalog or one of version tags: dev/beta/release)") + "(it could be git commit hash of version registered in catalog or one of version tags: dev/beta/release)") String clAsyncVer = null; - + @Parameter(names="--dynservver", description="Clients will be built " + "for use with KBase dynamic services (e.g. with URL lookup " + "via the Service Wizard) with the specified version " + @@ -631,11 +630,11 @@ public static class CompileCommandArgs { "clasyncver may not be specified if " + "dynservver is specified.") String dynservVer = null; - + @Parameter(names="--html", description="Generate HTML version(s) of " + "the input spec file(s)") boolean html = false; - + @Parameter(names={"-v", "--verbose"}, description="Print full stack " + "trace on a compile failure") boolean verbose = false; @@ -643,16 +642,16 @@ public static class CompileCommandArgs { @Parameter(required=true, description="") List specFileNames; } - + @Parameters(commandDescription = "Test a module with local Docker.") private static class TestCommandArgs { @Parameter(names={"-m", "--method_store"}, description="Narrative Method Store URL used in validation " + "(default is " + DEFAULT_METHOD_STORE_URL + ")") String methodStoreUrl = DEFAULT_METHOD_STORE_URL; - + @Parameter(names={"-s", "--skip_validation"}, description="Will skip validation step (default is false)") boolean skipValidation = false; - + @Parameter(names={"-a","--allow_sync_method"}, description="Allow synchonous methods " + "(advanced option, part of validation settings, default value is false)") boolean allowSyncMethods = false; @@ -660,11 +659,11 @@ private static class TestCommandArgs { @Parameter(names={"-v","--verbose"}, description="Print more details including error stack traces") boolean verbose = false; } - + @Parameters(commandDescription = "Print current version of kb-sdk.") private static class VersionCommandArgs { } - + @Parameters(commandDescription = "Rename a module name.") private static class RenameCommandArgs { @Parameter(names={"-v","--verbose"}, description="Print more details including error stack traces") @@ -679,16 +678,16 @@ private static class InstallCommandArgs { @Parameter(names={"-l", "--language"}, description="Language of generated client code " + "(default is one defined in kbase.yml)") String lang; - + @Parameter(names={"-a", "--async"}, description="Force generation of asynchronous calls " + - "(default is chosen based on information registered in catalog)") + "(default is chosen based on information registered in catalog)") boolean async = false; @Parameter(names={"-s", "--sync"}, description="Depricated flag, means the same as -c (--core)") boolean sync = false; @Parameter(names={"-c", "--core"}, description="Force generation of calls to core service" + - "(WARNING: please use it only for core services not registered in catalog)") + "(WARNING: please use it only for core services not registered in catalog)") boolean core = false; @Parameter(names={"-d", "--dynamic"}, description="Force generation of dynamic service calls" + @@ -701,7 +700,7 @@ private static class InstallCommandArgs { @Parameter(names={"-v","--verbose"}, description="Print more details including error stack traces") boolean verbose = false; - + @Parameter(names={"-n","--clientname"}, description="Optional parameter defining custom client name " + "(default is module name)") String clientName = null; @@ -713,19 +712,19 @@ private static class InstallCommandArgs { @Parameters(commandDescription = "Run a method of registered KBase module locally.") private static class RunCommandArgs { @Parameter(names={"-i", "--input"}, description="Input JSON file " + - "(optional, if not set -s or -j should be used)") + "(optional, if not set -s or -j should be used)") File inputFile = null; @Parameter(names={"-s","--stdin"}, description="Flag for reading input data from STDIN " + - "(default is false, used if neither -i or -j is set)") + "(default is false, used if neither -i or -j is set)") boolean stdin = false; @Parameter(names={"-j","--json"}, description="Input JSON string " + - "(optional, if not set -s or -i should be used)") + "(optional, if not set -s or -i should be used)") String inputJson = null; @Parameter(names={"-o", "--output"}, description="Output JSON file " + - "(optional, if not set output will be printed to STDOUT)") + "(optional, if not set output will be printed to STDOUT)") File output = null; @Parameter(names={"-t","--tag-or-ver"}, description="Tag or version " + @@ -733,95 +732,95 @@ private static class RunCommandArgs { String tagVer = null; @Parameter(names={"-v","--verbose"}, description="Print more details including error " + - "stack traces") + "stack traces") boolean verbose = false; @Parameter(names={"-k","--keep-tmp"}, description="Keep temporary files/folders at the " + - "end (default value is false)") + "end (default value is false)") boolean keepTempFiles = false; - + @Parameter(names={"-h","--sdk-home"}, description="Home folder of kb-sdk where sdk.cfg " + - "file and run_local folder are expected to be found or created if absent " + + "file and run_local folder are expected to be found or created if absent " + "(default path is loaded from 'KB_SDK_HOME' system environment variable)") String sdkHome = null; @Parameter(names={"-r","--prov-refs"}, description="Optional comma-separated list of " + - "references workspace objects that will be refered from provenance") + "references workspace objects that will be refered from provenance") String provRefs = null; @Parameter(names={"-m","--mount-points"}, description="Optional comma-separated list of " + - "mount point pairs for docker container (this parameter contains of local folder" + - " and inner folder separated by ':', local folder points to place in local file " + - "system, inner folder appears inside docker container, if inner path is not " + - "absolute it's treated as relative to /kb/module/work folder; if some mount " + - "point path doesn't have ':' and inner part then it appears as " + - "/kb/module/work/tmp inside docker)") + "mount point pairs for docker container (this parameter contains of local folder" + + " and inner folder separated by ':', local folder points to place in local file " + + "system, inner folder appears inside docker container, if inner path is not " + + "absolute it's treated as relative to /kb/module/work folder; if some mount " + + "point path doesn't have ':' and inner part then it appears as " + + "/kb/module/work/tmp inside docker)") String mountPoints = null; @Parameter(required=true, description="") + "(with SDK module prefix followed by '.')>") List methodName; } private static void showBriefHelp(JCommander jc, PrintStream out) { - Map commands = jc.getCommands(); - out.println(""); - out.println(MODULE_BUILDER_SH_NAME + " - a developer tool for building and validating KBase modules"); - out.println(""); - out.println("usage: "+MODULE_BUILDER_SH_NAME+" [options]"); - out.println(""); - out.println(" The available commands are:"); - for (Map.Entry command : commands.entrySet()) { - out.println(" " + command.getKey() +" - "+jc.getCommandDescription(command.getKey())); - } - out.println(""); - out.println(" For help on a specific command, see \"kb-sdk help \"."); - out.println(" For full usage information, see \"kb-sdk help -a\"."); - out.println(""); + Map commands = jc.getCommands(); + out.println(""); + out.println(MODULE_BUILDER_SH_NAME + " - a developer tool for building and validating KBase modules"); + out.println(""); + out.println("usage: "+MODULE_BUILDER_SH_NAME+" [options]"); + out.println(""); + out.println(" The available commands are:"); + for (Map.Entry command : commands.entrySet()) { + out.println(" " + command.getKey() +" - "+jc.getCommandDescription(command.getKey())); + } + out.println(""); + out.println(" For help on a specific command, see \"kb-sdk help \"."); + out.println(" For full usage information, see \"kb-sdk help -a\"."); + out.println(""); }; - + private static void showCommandUsage(JCommander jc, HelpCommandArgs helpArgs, PrintStream out) { - if(helpArgs.showAll) { - showFullUsage(jc, out); - } else if(helpArgs.command !=null && helpArgs.command.size()>0) { - String indent = ""; - StringBuilder usage = new StringBuilder(); - for(String cmd:helpArgs.command) { - if(jc.getCommands().containsKey(cmd)) { - usage.append(MODULE_BUILDER_SH_NAME + " "+cmd+"\n"); - jc.usage(cmd,usage,indent+" "); - usage.append("\n"); - } else { - out.println("Command \""+cmd+"\" is not a valid command. To view available commands, run:"); - out.println(" "+MODULE_BUILDER_SH_NAME+" "+HELP_COMMAND); - return; - } - } - out.print(usage.toString()); - } else { - showBriefHelp(jc, out); - } + if(helpArgs.showAll) { + showFullUsage(jc, out); + } else if(helpArgs.command !=null && helpArgs.command.size()>0) { + String indent = ""; + StringBuilder usage = new StringBuilder(); + for(String cmd:helpArgs.command) { + if(jc.getCommands().containsKey(cmd)) { + usage.append(MODULE_BUILDER_SH_NAME + " "+cmd+"\n"); + jc.usage(cmd,usage,indent+" "); + usage.append("\n"); + } else { + out.println("Command \""+cmd+"\" is not a valid command. To view available commands, run:"); + out.println(" "+MODULE_BUILDER_SH_NAME+" "+HELP_COMMAND); + return; + } + } + out.print(usage.toString()); + } else { + showBriefHelp(jc, out); + } }; - + private static void showFullUsage(JCommander jc, PrintStream out) { - String indent = ""; - StringBuilder usage = new StringBuilder(); - jc.usage(usage,indent); - out.print(usage.toString()); + String indent = ""; + StringBuilder usage = new StringBuilder(); + jc.usage(usage,indent); + out.print(usage.toString()); }; - - + + private static void showError(String error, String message, String extraHelp) { - System.err.println(error + ": " + message+"\n"); - if(!extraHelp.isEmpty()) - System.err.println(extraHelp+"\n"); - System.err.println("For more help and usage information, run:"); - System.err.println(" "+MODULE_BUILDER_SH_NAME+" "+HELP_COMMAND); - System.err.println(""); + System.err.println(error + ": " + message+"\n"); + if(!extraHelp.isEmpty()) + System.err.println(extraHelp+"\n"); + System.err.println("For more help and usage information, run:"); + System.err.println(" "+MODULE_BUILDER_SH_NAME+" "+HELP_COMMAND); + System.err.println(""); } - + private static void showError(String error, String message) { - showError(error,message,""); + showError(error,message,""); } } diff --git a/src/java/us/kbase/mobu/initializer/ModuleInitializer.java b/src/java/us/kbase/mobu/initializer/ModuleInitializer.java index 43141980..248a93c4 100644 --- a/src/java/us/kbase/mobu/initializer/ModuleInitializer.java +++ b/src/java/us/kbase/mobu/initializer/ModuleInitializer.java @@ -17,111 +17,111 @@ import us.kbase.templates.TemplateFormatter; public class ModuleInitializer { - public static final String DEFAULT_LANGUAGE = "python"; + public static final String DEFAULT_LANGUAGE = "python"; - private String moduleName; - private String userName; - private String language; - private boolean verbose; - private File dir; - - private static String[] subdirs = {"data", - "scripts", - "lib", - "test", - "ui", - "ui/narrative", - "ui/narrative/methods"}; - - public ModuleInitializer(String moduleName, String userName, boolean verbose) { - this(moduleName, userName, DEFAULT_LANGUAGE, verbose); - } - - public ModuleInitializer(String moduleName, String userName, String language, boolean verbose) { - this(moduleName, userName, language, verbose, null); - } - - public ModuleInitializer(String moduleName, String userName, String language, + private String moduleName; + private String userName; + private String language; + private boolean verbose; + private File dir; + + private static String[] subdirs = {"data", + "scripts", + "lib", + "test", + "ui", + "ui/narrative", + "ui/narrative/methods"}; + + public ModuleInitializer(String moduleName, String userName, boolean verbose) { + this(moduleName, userName, DEFAULT_LANGUAGE, verbose); + } + + public ModuleInitializer(String moduleName, String userName, String language, boolean verbose) { + this(moduleName, userName, language, verbose, null); + } + + public ModuleInitializer(String moduleName, String userName, String language, boolean verbose, File dir) { - this.moduleName = moduleName; - this.userName = userName; - this.language = language == null ? DEFAULT_LANGUAGE : language; - this.verbose = verbose; - this.dir = dir; - } - - /** - * - * @param example - * @throws IOException - */ - public void initialize(boolean example) throws Exception { - if (this.moduleName == null) { - throw new RuntimeException("Unable to create a null directory!"); - } - String moduleDir = dir == null ? this.moduleName : + this.moduleName = moduleName; + this.userName = userName; + this.language = language == null ? DEFAULT_LANGUAGE : language; + this.verbose = verbose; + this.dir = dir; + } + + /** + * + * @param example + * @throws IOException + */ + public void initialize(boolean example) throws Exception { + if (this.moduleName == null) { + throw new RuntimeException("Unable to create a null directory!"); + } + String moduleDir = dir == null ? this.moduleName : new File(dir, this.moduleName).getPath(); - this.language = qualifyLanguage(this.language); - - if (this.verbose) { - String msg = "Initializing module \"" + this.moduleName + "\""; - if (example) - msg += " with example methods"; - System.out.println(msg); - } + this.language = qualifyLanguage(this.language); + + if (this.verbose) { + String msg = "Initializing module \"" + this.moduleName + "\""; + if (example) + msg += " with example methods"; + System.out.println(msg); + } - List subdirList = new ArrayList(Arrays.asList(subdirs)); - if (example && this.language.equals("r")) { - subdirList.add("ui/narrative/methods/count_contigs_in_set"); - subdirList.add("ui/narrative/methods/count_contigs_in_set/img"); - } - else { - subdirList.add("ui/narrative/methods/run_" + this.moduleName); - subdirList.add("ui/narrative/methods/run_" + this.moduleName + "/img"); - } - - // 1. build dir with moduleName - initDirectory(Paths.get(moduleDir), true); - - // 2. build skeleton subdirs - for (String dir : subdirList) { - initDirectory(Paths.get(moduleDir, dir), true); - } + List subdirList = new ArrayList(Arrays.asList(subdirs)); + if (example && this.language.equals("r")) { + subdirList.add("ui/narrative/methods/count_contigs_in_set"); + subdirList.add("ui/narrative/methods/count_contigs_in_set/img"); + } + else { + subdirList.add("ui/narrative/methods/run_" + this.moduleName); + subdirList.add("ui/narrative/methods/run_" + this.moduleName + "/img"); + } + + // 1. build dir with moduleName + initDirectory(Paths.get(moduleDir), true); + + // 2. build skeleton subdirs + for (String dir : subdirList) { + initDirectory(Paths.get(moduleDir, dir), true); + } + + /* + * 3. Fill in templated files and write them + * + * Set up the context - the set of variables used to flesh out the templates */ + String specFile = Paths.get(this.moduleName + ".spec").toString(); - /* - * 3. Fill in templated files and write them - * - * Set up the context - the set of variables used to flesh out the templates */ - String specFile = Paths.get(this.moduleName + ".spec").toString(); - - Map moduleContext = new HashMap(); - moduleContext.put("module_name", this.moduleName); - moduleContext.put("user_name", this.userName); - moduleContext.put("spec_file", specFile); - moduleContext.put("language", this.language); - moduleContext.put("module_root_path", Paths.get(moduleDir).toAbsolutePath()); - moduleContext.put("example", example); - moduleContext.put("dollar_sign", "$"); + Map moduleContext = new HashMap(); + moduleContext.put("module_name", this.moduleName); + moduleContext.put("user_name", this.userName); + moduleContext.put("spec_file", specFile); + moduleContext.put("language", this.language); + moduleContext.put("module_root_path", Paths.get(moduleDir).toAbsolutePath()); + moduleContext.put("example", example); + moduleContext.put("dollar_sign", "$"); moduleContext.put("os_name", System.getProperty("os.name")); - Map templateFiles = new HashMap(); - templateFiles.put("module_typespec", Paths.get(moduleDir, specFile)); - templateFiles.put("module_travis", Paths.get(moduleDir, ".travis.yml")); - templateFiles.put("module_dockerfile", Paths.get(moduleDir, "Dockerfile")); - templateFiles.put("module_readme", Paths.get(moduleDir, "README.md")); - templateFiles.put("module_release_notes", Paths.get(moduleDir, "RELEASE_NOTES.md")); - templateFiles.put("module_makefile", Paths.get(moduleDir, "Makefile")); - templateFiles.put("module_deploy_cfg", Paths.get(moduleDir, "deploy.cfg")); - templateFiles.put("module_license", Paths.get(moduleDir, "LICENSE")); - templateFiles.put("module_docker_entrypoint", Paths.get(moduleDir, "scripts", "entrypoint.sh")); + Map templateFiles = new HashMap(); + templateFiles.put("module_typespec", Paths.get(moduleDir, specFile)); + templateFiles.put("module_travis", Paths.get(moduleDir, ".travis.yml")); + templateFiles.put("module_dockerfile", Paths.get(moduleDir, "Dockerfile")); + templateFiles.put("module_readme", Paths.get(moduleDir, "README.md")); + templateFiles.put("module_release_notes", Paths.get(moduleDir, "RELEASE_NOTES.md")); + templateFiles.put("module_makefile", Paths.get(moduleDir, "Makefile")); + templateFiles.put("module_deploy_cfg", Paths.get(moduleDir, "deploy.cfg")); + templateFiles.put("module_license", Paths.get(moduleDir, "LICENSE")); + templateFiles.put("module_docker_entrypoint", Paths.get(moduleDir, "scripts", "entrypoint.sh")); templateFiles.put("module_prepare_deploy_cfg", Paths.get(moduleDir, "scripts", "prepare_deploy_cfg.py")); templateFiles.put("module_run_async", Paths.get(moduleDir, "scripts", "run_async.sh")); - templateFiles.put("module_readme_lib", Paths.get(moduleDir, "lib", "README.md")); - templateFiles.put("module_readme_ui", Paths.get(moduleDir, "ui", "README.md")); - templateFiles.put("module_readme_test", Paths.get(moduleDir, "test", "README.md")); - templateFiles.put("module_readme_data", Paths.get(moduleDir, "data", "README.md")); - templateFiles.put("module_config_yaml", Paths.get(moduleDir, "kbase.yml")); + templateFiles.put("module_readme_lib", Paths.get(moduleDir, "lib", "README.md")); + templateFiles.put("module_readme_ui", Paths.get(moduleDir, "ui", "README.md")); + templateFiles.put("module_readme_test", Paths.get(moduleDir, "test", "README.md")); + templateFiles.put("module_readme_data", Paths.get(moduleDir, "data", "README.md")); + templateFiles.put("module_config_yaml", Paths.get(moduleDir, "kbase.yml")); templateFiles.put("module_gitignore", Paths.get(moduleDir, ".gitignore")); templateFiles.put("module_dockerignore", Paths.get(moduleDir, ".dockerignore")); templateFiles.put("module_readme_test_local", Paths.get(moduleDir, "test_local", "readme.txt")); @@ -129,13 +129,13 @@ public void initialize(boolean example) throws Exception { templateFiles.put("module_run_tests", Paths.get(moduleDir, "test_local", "run_tests.sh")); templateFiles.put("module_run_bash", Paths.get(moduleDir, "test_local", "run_bash.sh")); templateFiles.put("module_run_docker", Paths.get(moduleDir, "test_local", "run_docker.sh")); - - switch (language) { - case "java": + + switch (language) { + case "java": templateFiles.put("module_build_xml", Paths.get(moduleDir, "build.xml")); - templateFiles.put("module_web_xml", Paths.get(moduleDir, "scripts", "web.xml")); + templateFiles.put("module_web_xml", Paths.get(moduleDir, "scripts", "web.xml")); templateFiles.put("module_jetty_xml", Paths.get(moduleDir, "scripts", "jetty.xml")); - fillTemplate(moduleContext, "module_typespec", templateFiles.remove("module_typespec")); + fillTemplate(moduleContext, "module_typespec", templateFiles.remove("module_typespec")); String javaPackageParent = "."; // Special value meaning top level package. moduleContext.put("java_package_parent", javaPackageParent); JavaData data = JavaTypeGenerator.parseSpec(new File(moduleDir, specFile)); @@ -148,7 +148,7 @@ public void initialize(boolean example) throws Exception { File testJavaFile = new File(testSrcDir, modulePackage.replace('.', '/') + "/test/" + javaModuleName + "ServerTest.java"); fillTemplate(moduleContext, "module_test_java_client", testJavaFile.toPath()); break; - case "python": + case "python": templateFiles.put("module_test_python_client", Paths.get(moduleDir, "test", this.moduleName + "_server_test.py")); templateFiles.put("module_tox", Paths.get(moduleDir, "tox.ini")); break; @@ -158,12 +158,12 @@ public void initialize(boolean example) throws Exception { case "r": templateFiles.put("module_test_r_client", Paths.get(moduleDir, "test", this.moduleName + "_server_test.r")); break; - } - - if (example && this.language.equals("r")) { - templateFiles.put("module_method_spec_json", Paths.get(moduleDir, "ui", "narrative", "methods", "count_contigs_in_set", "spec.json")); - templateFiles.put("module_method_spec_yaml", Paths.get(moduleDir, "ui", "narrative", "methods", "count_contigs_in_set", "display.yaml")); - } else { + } + + if (example && this.language.equals("r")) { + templateFiles.put("module_method_spec_json", Paths.get(moduleDir, "ui", "narrative", "methods", "count_contigs_in_set", "spec.json")); + templateFiles.put("module_method_spec_yaml", Paths.get(moduleDir, "ui", "narrative", "methods", "count_contigs_in_set", "display.yaml")); + } else { templateFiles.put("module_method_spec_json", Paths.get(moduleDir, "ui", "narrative", "methods", "run_"+this.moduleName, "spec.json")); templateFiles.put("module_method_spec_yaml", Paths.get(moduleDir, "ui", "narrative", "methods", "run_"+this.moduleName, "display.yaml")); } @@ -201,12 +201,12 @@ public void initialize(boolean example) throws Exception { break; } - for (String templateName : templateFiles.keySet()) { - fillTemplate(moduleContext, templateName, templateFiles.get(templateName)); - } - - if (example) { - // Generated examples require some other SDK dependencies + for (String templateName : templateFiles.keySet()) { + fillTemplate(moduleContext, templateName, templateFiles.get(templateName)); + } + + if (example) { + // Generated examples require some other SDK dependencies new ClientInstaller(new File(moduleDir), false).install( this.language, true, // async clients @@ -218,9 +218,9 @@ public void initialize(boolean example) throws Exception { null, null // clientName ); - } - // Let's install fresh workspace client in any cases (we need it at least in tests): - new ClientInstaller(new File(moduleDir), false).install( + } + // Let's install fresh workspace client in any cases (we need it at least in tests): + new ClientInstaller(new File(moduleDir), false).install( this.language, false, // async clients true, // core or sync clients @@ -243,82 +243,82 @@ public void initialize(boolean example) throws Exception { null // clientName ); - System.out.println("Done! Your module is available in the " + moduleDir + " directory."); - System.out.println("Compile and run the example methods with the following inputs:"); - System.out.println(" cd " + moduleDir); - System.out.println(" make (required after making changes to " + new File(specFile).getName() + ")"); - System.out.println(" kb-sdk test (will require setting test user account credentials in test_local/test.cfg)"); - System.out.println(); - } - - /** - * - * @param dirPath - * @throws IOException - */ - private void initDirectory(Path dirPath, boolean failOnExist) throws IOException { - if (this.verbose) System.out.println("Making directory \"" + dirPath.toString() + "\""); - File newDir = dirPath.toFile(); - if (!newDir.exists()) { - newDir.mkdirs(); - } - else if (failOnExist) { - throw new IOException("Error while creating module - " + dirPath + " already exists!"); - } - } - - private void initFile(Path filePath, boolean failOnExist) throws IOException { - if (this.verbose) System.out.println("Building empty file \"" + filePath.toString() + "\""); - boolean done = filePath.toFile().createNewFile(); - if (!done && failOnExist) - throw new IOException("Unable to create file \"" + filePath.toString() + "\" - file already exists!"); - } - /** - * - * @param context - * @param templateName - * @param outfile - * @throws IOException - */ - private void fillTemplate(Map context, String templateName, Path outfilePath) throws IOException { - if (this.verbose) System.out.println("Building file \"" + outfilePath.toString() + "\""); - initDirectory(outfilePath.getParent(), false); - TemplateFormatter.formatTemplate(templateName, context, outfilePath.toFile()); - } - - /** - * Takes a language string and returns a "qualified" form. E.g. "perl", "Perl", "pl", ".pl", should all - * return "perl", "Python", "python", ".py", and "py" should all return Python, etc. - * - * Right now, we support Perl, Python and Java for implementation languages - * @param language - * @return - */ - public static String qualifyLanguage(String language) { - String lang = language.toLowerCase(); - - String[] perlNames = {"perl", ".pl", "pl"}; - if (Arrays.asList(perlNames).contains(lang)) - return "perl"; - - String[] pythonNames = {"python", ".py", "py"}; - if (Arrays.asList(pythonNames).contains(lang)) - return "python"; - - String[] javaNames = {"java", ".java"}; - if (Arrays.asList(javaNames).contains(lang)) - return "java"; - - String[] rNames = {"r", ".r"}; - if (Arrays.asList(rNames).contains(lang)) { - System.out.println( - "************************************************************************\n" + - "WARNING: R support is deprecated and will be removed in a future release\n" + - "************************************************************************"); - return "r"; - } - - // If we get here, then we don't recognize it! throw a runtime exception - throw new RuntimeException("Unrecognized language: " + language + "\n\tWe currently only support Python, Perl, and Java."); - } -} \ No newline at end of file + System.out.println("Done! Your module is available in the " + moduleDir + " directory."); + System.out.println("Compile and run the example methods with the following inputs:"); + System.out.println(" cd " + moduleDir); + System.out.println(" make (required after making changes to " + new File(specFile).getName() + ")"); + System.out.println(" kb-sdk test (will require setting test user account credentials in test_local/test.cfg)"); + System.out.println(); + } + + /** + * + * @param dirPath + * @throws IOException + */ + private void initDirectory(Path dirPath, boolean failOnExist) throws IOException { + if (this.verbose) System.out.println("Making directory \"" + dirPath.toString() + "\""); + File newDir = dirPath.toFile(); + if (!newDir.exists()) { + newDir.mkdirs(); + } + else if (failOnExist) { + throw new IOException("Error while creating module - " + dirPath + " already exists!"); + } + } + + private void initFile(Path filePath, boolean failOnExist) throws IOException { + if (this.verbose) System.out.println("Building empty file \"" + filePath.toString() + "\""); + boolean done = filePath.toFile().createNewFile(); + if (!done && failOnExist) + throw new IOException("Unable to create file \"" + filePath.toString() + "\" - file already exists!"); + } + /** + * + * @param context + * @param templateName + * @param outfile + * @throws IOException + */ + private void fillTemplate(Map context, String templateName, Path outfilePath) throws IOException { + if (this.verbose) System.out.println("Building file \"" + outfilePath.toString() + "\""); + initDirectory(outfilePath.getParent(), false); + TemplateFormatter.formatTemplate(templateName, context, outfilePath.toFile()); + } + + /** + * Takes a language string and returns a "qualified" form. E.g. "perl", "Perl", "pl", ".pl", should all + * return "perl", "Python", "python", ".py", and "py" should all return Python, etc. + * + * Right now, we support Perl, Python and Java for implementation languages + * @param language + * @return + */ + public static String qualifyLanguage(String language) { + String lang = language.toLowerCase(); + + String[] perlNames = {"perl", ".pl", "pl"}; + if (Arrays.asList(perlNames).contains(lang)) + return "perl"; + + String[] pythonNames = {"python", ".py", "py"}; + if (Arrays.asList(pythonNames).contains(lang)) + return "python"; + + String[] javaNames = {"java", ".java"}; + if (Arrays.asList(javaNames).contains(lang)) + return "java"; + + String[] rNames = {"r", ".r"}; + if (Arrays.asList(rNames).contains(lang)) { + System.out.println( + "************************************************************************\n" + + "WARNING: R support is deprecated and will be removed in a future release\n" + + "************************************************************************"); + return "r"; + } + + // If we get here, then we don't recognize it! throw a runtime exception + throw new RuntimeException("Unrecognized language: " + language + "\n\tWe currently only support Python, Perl, and Java."); + } +} diff --git a/src/java/us/kbase/mobu/installer/ClientInstaller.java b/src/java/us/kbase/mobu/installer/ClientInstaller.java index bdb30951..3df58d91 100644 --- a/src/java/us/kbase/mobu/installer/ClientInstaller.java +++ b/src/java/us/kbase/mobu/installer/ClientInstaller.java @@ -47,7 +47,7 @@ public class ClientInstaller { private final String language; private final Properties sdkConfig; private final URL catalogUrl; - + public ClientInstaller() throws Exception { this(null, true); } @@ -84,13 +84,13 @@ public ClientInstaller(File dir, boolean showWarnings) throws Exception { catalogUrl = new URL(catalogUrlText); } - public int install(String lang, boolean async, boolean core, boolean dynamic, String tagVer, + public int install(String lang, boolean async, boolean core, boolean dynamic, String tagVer, boolean verbose, String moduleName, String libDirName) throws Exception { return install(lang, async, core, dynamic, tagVer, verbose, moduleName, libDirName, null); } - - public int install(String lang, boolean async, boolean core, boolean dynamic, String tagVer, - boolean verbose, String moduleName, String libDirName, String clientName) + + public int install(String lang, boolean async, boolean core, boolean dynamic, String tagVer, + boolean verbose, String moduleName, String libDirName, String clientName) throws Exception { if (core && (dynamic || async)) { throw new IllegalStateException("It's not allowed to set 'core' mode and one " + @@ -142,7 +142,7 @@ public String loadIncludedSpec(String specFileName) throws Exception { }; } else { throw new IllegalStateException("Path " + moduleName + " is not recognized as " + - "existing local file or URL"); + "existing local file or URL"); } moduleName = null; } else { @@ -159,11 +159,11 @@ public String loadIncludedSpec(String specFileName) throws Exception { } if (modVer.getCompilationReport() == null) throw new IllegalStateException("Compilation report is not found for this " + - "version of [" + moduleName + "] module."); + "version of [" + moduleName + "] module."); List specFiles = modVer.getCompilationReport().getSpecFiles(); if (specFiles == null) throw new IllegalStateException("Compilation report returned from catalog is " + - "out of date. [" + moduleName + "] module should be reregistered again."); + "out of date. [" + moduleName + "] module should be reregistered again."); final List mainSpec = new ArrayList(); final Map deps = new LinkedHashMap(); for (SpecFile spec : specFiles) { @@ -183,22 +183,22 @@ public String loadIncludedSpec(String specFileName) throws Exception { }; } if (async && dynamic) { - compile(lang, async, false, tagVer, verbose, moduleName, libDirName, fp, + compile(lang, async, false, tagVer, verbose, moduleName, libDirName, fp, semanticVersion, gitUrl, gitCommitHash, clientName, filePath); if (clientName == null) { clientName = moduleName; } - compile(lang, false, dynamic, tagVer, verbose, moduleName, libDirName, fp, + compile(lang, false, dynamic, tagVer, verbose, moduleName, libDirName, fp, semanticVersion, gitUrl, gitCommitHash, clientName + "Service", filePath); } else { - compile(lang, async, dynamic, tagVer, verbose, moduleName, libDirName, fp, + compile(lang, async, dynamic, tagVer, verbose, moduleName, libDirName, fp, semanticVersion, gitUrl, gitCommitHash, clientName, filePath); } return 0; } - private void compile(String lang, boolean async, boolean dynamic, String tagVer, - boolean verbose, String moduleName, String libDirName, FileProvider fp, + private void compile(String lang, boolean async, boolean dynamic, String tagVer, + boolean verbose, String moduleName, String libDirName, FileProvider fp, String semanticVersion, String gitUrl, String gitCommitHash, String clientName, String filePath) throws Exception { String url = null; @@ -216,7 +216,7 @@ private void compile(String lang, boolean async, boolean dynamic, String tagVer, final FileProvider fp2 = fp; IncludeProvider ip = new IncludeProvider() { @Override - public Map parseInclude(String includeLine) + public Map parseInclude(String includeLine) throws KidlParseException { String specPath = includeLine.trim(); if (specPath.startsWith("#include")) @@ -285,8 +285,8 @@ public Map parseInclude(String includeLine) String javaPackageParent = "."; String customClientClassName = TextUtils.capitalize(clientName) + "Client"; URL urlEndpoint = url == null ? null : new URL(url); - JavaTypeGenerator.processSpec(services, javaSrcDir, javaPackageParent, false, null, - null, urlEndpoint, null, null, clientAsyncVer, dynservVer, semanticVersion, + JavaTypeGenerator.processSpec(services, javaSrcDir, javaPackageParent, false, null, + null, urlEndpoint, null, null, clientAsyncVer, dynservVer, semanticVersion, gitUrl, gitCommitHash, null, customClientClassName); } else { String perlClientName = null; @@ -302,9 +302,9 @@ public Map parseInclude(String includeLine) if (isJS) jsClientName = "installed_clients/" + clientName + "Client"; FileSaver output = new DiskFileSaver(libDir); - TemplateBasedGenerator.generate(services, url, isJS, jsClientName, isPerl, + TemplateBasedGenerator.generate(services, url, isJS, jsClientName, isPerl, perlClientName, false, null, null, null, isPython, pyClientName, false, null, - null, isR, rClientName, false, null, null, false, ip, output, null, null, + null, isR, rClientName, false, null, null, false, ip, output, null, null, async, clientAsyncVer, dynservVer, semanticVersion, gitUrl, gitCommitHash); } // Now let's add record about this client to dependencies.json file @@ -313,8 +313,8 @@ public Map parseInclude(String includeLine) FileSaver depsDir = new DiskFileSaver(moduleDir); addDependency(moduleName, isSdk, versionTag, filePath, depsDir); } - - public static void addDependency(String moduleName, boolean isSdk, String versionTag, + + public static void addDependency(String moduleName, boolean isSdk, String versionTag, String filePath, FileSaver depsDir) throws Exception { Map depMap = new TreeMap(); ObjectMapper mapper = new ObjectMapper(); @@ -322,7 +322,7 @@ public static void addDependency(String moduleName, boolean isSdk, String versio File depsFile = depsDir.getAsFileOrNull("dependencies.json"); if (depsFile != null && depsFile.exists()) { try { - List deps = mapper.readValue(depsFile, + List deps = mapper.readValue(depsFile, new TypeReference>() {}); for (Dependency dep : deps) { depMap.put(dep.moduleName.toLowerCase(), dep); @@ -343,7 +343,7 @@ public static void addDependency(String moduleName, boolean isSdk, String versio mapper.writeValue(depsWr, deps); } } - + private static boolean isUrl(String url) { try { new URL(url); @@ -352,7 +352,7 @@ private static boolean isUrl(String url) { return false; } } - + private static boolean isLocalFile(String path) { try { File f = new File(path); @@ -362,7 +362,7 @@ private static boolean isLocalFile(String path) { return false; } } - + private static interface FileProvider { public String loadMainSpec() throws Exception; public String loadIncludedSpec(String specFileName) throws Exception; diff --git a/src/java/us/kbase/mobu/tester/ModuleTester.java b/src/java/us/kbase/mobu/tester/ModuleTester.java index 7600b8d0..f9c068f9 100644 --- a/src/java/us/kbase/mobu/tester/ModuleTester.java +++ b/src/java/us/kbase/mobu/tester/ModuleTester.java @@ -53,7 +53,7 @@ public class ModuleTester { public ModuleTester() throws Exception { this(null); } - + public ModuleTester(File dir) throws Exception { moduleDir = dir == null ? DirUtils.findModuleDir() : DirUtils.findModuleDir(dir); String kbaseYml = TextUtils.readFileText(new File(moduleDir, "kbase.yml")); @@ -70,7 +70,7 @@ public ModuleTester(File dir) throws Exception { ModuleInitializer.qualifyLanguage((String) kbaseYmlConfig.get("service-language")); moduleContext.put("os_name", System.getProperty("os.name")); } - + private static void checkIgnoreLine(File f, String line) throws IOException { List lines = new ArrayList(); if (f.exists()) @@ -82,18 +82,18 @@ private static void checkIgnoreLine(File f, String line) throws IOException { FileUtils.writeLines(f, lines); } } - + public int runTests(String methodStoreUrl, boolean skipValidation, boolean allowSyncMethods) throws Exception { if (skipValidation) { System.out.println("Validation step is skipped"); } else { - ModuleValidator mv = new ModuleValidator(Arrays.asList(moduleDir.getCanonicalPath()), + ModuleValidator mv = new ModuleValidator(Arrays.asList(moduleDir.getCanonicalPath()), false, methodStoreUrl, allowSyncMethods); int returnCode = mv.validateAll(); if (returnCode!=0) { System.out.println("You can skip validation step using -s (or --skip_validation)" + - " flag"); + " flag"); System.exit(returnCode); } } @@ -114,7 +114,7 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow if (kbaseYmlConfig.get("data-version") != null) { File refDataDir = new File(tlDir, "refdata"); if (!refDataDir.exists()) { - TemplateFormatter.formatTemplate("module_run_tests", moduleContext, + TemplateFormatter.formatTemplate("module_run_tests", moduleContext, runTestsSh); refDataDir.mkdir(); } @@ -128,7 +128,7 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow if (!testCfg.exists()) { TemplateFormatter.formatTemplate("module_test_cfg", moduleContext, testCfg); System.out.println("Set KBase account credentials in test_local/test.cfg and then " + - "test again"); + "test again"); return 1; } Properties props = new Properties(); @@ -138,10 +138,10 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow } finally { is.close(); } - + ConfigLoader cfgLoader = new ConfigLoader(props, true, "test_local/test.cfg", true); - - + + File workDir = new File(tlDir, "workdir"); workDir.mkdir(); File tokenFile = new File(workDir, "token"); @@ -175,7 +175,7 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow String callbackNetworksText = props.getProperty("callback_networks"); if (callbackNetworksText != null) { callbackNetworks = callbackNetworksText.trim().split("\\s*,\\s*"); - System.out.println("Custom network instarface list is defined: " + + System.out.println("Custom network instarface list is defined: " + Arrays.asList(callbackNetworks)); } URL callbackUrl = CallbackServer.getCallbackUrl(callbackPort, callbackNetworks); @@ -185,7 +185,7 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow JsonServerSyslog.setStaticUseSyslog(false); JsonServerSyslog.setStaticMlogFile("callback.log"); } - CallbackServerConfig cfg = cfgLoader.buildCallbackServerConfig(callbackUrl, + CallbackServerConfig cfg = cfgLoader.buildCallbackServerConfig(callbackUrl, tlDir.toPath(), new LineLogger() { @Override public void logNextLine(String line, boolean isError) { @@ -217,7 +217,7 @@ public void logNextLine(String line, boolean isError) { } else { if (callbackNetworks != null && callbackNetworks.length > 0) { throw new IllegalStateException("No proper callback IP was found, " + - "please check callback_networks parameter in test.cfg"); + "please check callback_networks parameter in test.cfg"); } System.out.println("WARNING: No callback URL was received " + "by the job runner. Local callbacks are disabled."); @@ -227,7 +227,7 @@ public void logNextLine(String line, boolean isError) { System.out.println(); ProcessHelper.cmd("chmod", "+x", runTestsSh.getCanonicalPath()).exec(tlDir); int exitCode = ProcessHelper.cmd("bash", DirUtils.getFilePath(runTestsSh), - callbackUrl == null ? "http://fakecallbackurl" : + callbackUrl == null ? "http://fakecallbackurl" : callbackUrl.toExternalForm()).exec(tlDir).getExitCode(); return exitCode; } finally { @@ -251,7 +251,7 @@ public static boolean buildNewDockerImageWithCleanup(File moduleDir, File tlDir, ProcessHelper.cmd("bash", runDockerPath, "rm", "-v", "-f", cntId).exec(tlDir); } } - String oldImageId = findImageIdByName(tlDir, imageName, runDockerSh); + String oldImageId = findImageIdByName(tlDir, imageName, runDockerSh); System.out.println(); System.out.println("Build Docker image"); boolean ok = buildImage(moduleDir, imageName, runDockerSh); @@ -267,7 +267,7 @@ public static boolean buildNewDockerImageWithCleanup(File moduleDir, File tlDir, } return true; } - + public static String findImageIdByName(File tlDir, String imageName, File runDockerSh) throws Exception { List lines; @@ -295,7 +295,7 @@ public static String[] splitByWhiteSpaces(String line) { String[] parts = line.split("\\s+"); return parts; } - + private static List exec(File workDir, String... cmd) throws Exception { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); @@ -312,13 +312,13 @@ private static List exec(File workDir, String... cmd) throws Exception { br.close(); return ret; } - - public static boolean buildImage(File repoDir, String targetImageName, + + public static boolean buildImage(File repoDir, String targetImageName, File runDockerSh) throws Exception { String scriptPath = DirUtils.getFilePath(runDockerSh); String repoPath = DirUtils.getFilePath(repoDir); - Process p = Runtime.getRuntime().exec(new String[] {"bash", - scriptPath, "build", "--rm", "-t", + Process p = Runtime.getRuntime().exec(new String[] {"bash", + scriptPath, "build", "--rm", "-t", targetImageName, repoPath}); List workers = new ArrayList(); InputStream[] inputStreams = new InputStream[] {p.getInputStream(), p.getErrorStream()}; @@ -358,7 +358,7 @@ public void run() { } catch (Exception e) { e.printStackTrace(); throw new IllegalStateException("Error reading data from executed " + - "container", e); + "container", e); } } }); @@ -374,7 +374,7 @@ public void run() { if (cntIdToDelete[0] != null) { System.out.println("Cleaning up building container: " + cntIdToDelete[0]); Thread.sleep(1000); - ProcessHelper.cmd("bash", scriptPath, + ProcessHelper.cmd("bash", scriptPath, "rm", "-v", "-f", cntIdToDelete[0]).exec(repoDir); } } catch (Exception ex) { diff --git a/src/java/us/kbase/mobu/tester/test/ModuleTesterTest.java b/src/java/us/kbase/mobu/tester/test/ModuleTesterTest.java index ae1f2488..3cbd4ee9 100644 --- a/src/java/us/kbase/mobu/tester/test/ModuleTesterTest.java +++ b/src/java/us/kbase/mobu/tester/test/ModuleTesterTest.java @@ -21,44 +21,44 @@ public class ModuleTesterTest { - private static final String SIMPLE_MODULE_NAME = "ASimpleModule_for_unit_testing"; - private static final boolean cleanupAfterTests = true; - - private static List createdModuleNames = new ArrayList(); - private static AuthToken token; - + private static final String SIMPLE_MODULE_NAME = "ASimpleModule_for_unit_testing"; + private static final boolean cleanupAfterTests = true; + + private static List createdModuleNames = new ArrayList(); + private static AuthToken token; + @BeforeClass public static void beforeClass() throws Exception { token = TestConfigHelper.getToken(); } - @AfterClass - public static void tearDownModule() throws Exception { - if (cleanupAfterTests) - for (String moduleName : createdModuleNames) - try { - deleteDir(moduleName); - } catch (Exception ex) { - System.err.println("Error cleaning up module [" + - moduleName + "]: " + ex.getMessage()); - } - } - - @After - public void afterText() { - System.out.println(); - } - - private static void deleteDir(String moduleName) throws Exception { - File module = new File(moduleName); - if (module.exists() && module.isDirectory()) - FileUtils.deleteDirectory(module); - } - + @AfterClass + public static void tearDownModule() throws Exception { + if (cleanupAfterTests) + for (String moduleName : createdModuleNames) + try { + deleteDir(moduleName); + } catch (Exception ex) { + System.err.println("Error cleaning up module [" + + moduleName + "]: " + ex.getMessage()); + } + } + + @After + public void afterText() { + System.out.println(); + } + + private static void deleteDir(String moduleName) throws Exception { + File module = new File(moduleName); + if (module.exists() && module.isDirectory()) + FileUtils.deleteDirectory(module); + } + private void init(String lang, String moduleName) throws Exception { deleteDir(moduleName); createdModuleNames.add(moduleName); - ModuleInitializer initer = new ModuleInitializer(moduleName, token.getUserName(), + ModuleInitializer initer = new ModuleInitializer(moduleName, token.getUserName(), lang, false); initer.initialize(true); } @@ -68,13 +68,13 @@ private int runTestsInDocker(String moduleName) throws Exception { return runTestsInDocker(moduleDir, token); } - public static int runTestsInDocker(File moduleDir, AuthToken token) throws Exception { - return runTestsInDocker(moduleDir, token, false); - } - + public static int runTestsInDocker(File moduleDir, AuthToken token) throws Exception { + return runTestsInDocker(moduleDir, token, false); + } + public static int runTestsInDocker( final File moduleDir, - final AuthToken token, + final AuthToken token, final boolean skipValidation) throws Exception { DockerClientServerTester.correctDockerfile(moduleDir); @@ -83,7 +83,7 @@ public static int runTestsInDocker( "test_token=" + token.getToken() + "\n" + "kbase_endpoint=" + TestConfigHelper.getKBaseEndpoint() + "\n" + "auth_service_url=" + TestConfigHelper.getAuthServiceUrl() + "\n" + - "auth_service_url_allow_insecure=" + + "auth_service_url_allow_insecure=" + TestConfigHelper.getAuthServiceUrlInsecure() + "\n"; FileUtils.writeStringToFile(testCfgFile, testCfgText); int exitCode = new ModuleTester(moduleDir).runTests(ModuleBuilder.DEFAULT_METHOD_STORE_URL, @@ -92,84 +92,84 @@ public static int runTestsInDocker( return exitCode; } - @Test - public void testPerlModuleExample() throws Exception { - System.out.println("Test [testPerlModuleExample]"); - String lang = "perl"; - String moduleName = SIMPLE_MODULE_NAME + "Perl"; - init(lang, moduleName); - int exitCode = runTestsInDocker(moduleName); - Assert.assertEquals(0, exitCode); - } - - @Test - public void testPerlModuleError() throws Exception { - System.out.println("Test [testPerlModuleError]"); - String lang = "perl"; - String moduleName = SIMPLE_MODULE_NAME + "PerlError"; - init(lang, moduleName); - File implFile = new File(moduleName + "/lib/" + moduleName + "/" + moduleName + "Impl.pm"); - String implText = FileUtils.readFileToString(implFile); - implText = implText.replace(" #BEGIN filter_contigs", - " #BEGIN filter_contigs\n" + - " die \"Special error\";"); - FileUtils.writeStringToFile(implFile, implText); - int exitCode = runTestsInDocker(moduleName); - Assert.assertEquals(2, exitCode); - } - - @Test - public void testPythonModuleExample() throws Exception { - System.out.println("Test [testPythonModuleExample]"); - String lang = "python"; - String moduleName = SIMPLE_MODULE_NAME + "Python"; - init(lang, moduleName); - int exitCode = runTestsInDocker(moduleName); + @Test + public void testPerlModuleExample() throws Exception { + System.out.println("Test [testPerlModuleExample]"); + String lang = "perl"; + String moduleName = SIMPLE_MODULE_NAME + "Perl"; + init(lang, moduleName); + int exitCode = runTestsInDocker(moduleName); Assert.assertEquals(0, exitCode); - } + } - @Test - public void testPythonModuleError() throws Exception { + @Test + public void testPerlModuleError() throws Exception { + System.out.println("Test [testPerlModuleError]"); + String lang = "perl"; + String moduleName = SIMPLE_MODULE_NAME + "PerlError"; + init(lang, moduleName); + File implFile = new File(moduleName + "/lib/" + moduleName + "/" + moduleName + "Impl.pm"); + String implText = FileUtils.readFileToString(implFile); + implText = implText.replace(" #BEGIN filter_contigs", + " #BEGIN filter_contigs\n" + + " die \"Special error\";"); + FileUtils.writeStringToFile(implFile, implText); + int exitCode = runTestsInDocker(moduleName); + Assert.assertEquals(2, exitCode); + } + + @Test + public void testPythonModuleExample() throws Exception { + System.out.println("Test [testPythonModuleExample]"); + String lang = "python"; + String moduleName = SIMPLE_MODULE_NAME + "Python"; + init(lang, moduleName); + int exitCode = runTestsInDocker(moduleName); + Assert.assertEquals(0, exitCode); + } + + @Test + public void testPythonModuleError() throws Exception { System.out.println("Test [testPythonModuleError]"); - String lang = "python"; + String lang = "python"; String moduleName = SIMPLE_MODULE_NAME + "PythonError"; init(lang, moduleName); File implFile = new File(moduleName + "/lib/" + moduleName + "/" + moduleName + "Impl.py"); String implText = FileUtils.readFileToString(implFile); - implText = implText.replace(" #BEGIN filter_contigs", + implText = implText.replace(" #BEGIN filter_contigs", " #BEGIN filter_contigs\n" + " raise ValueError('Special error')"); FileUtils.writeStringToFile(implFile, implText); int exitCode = runTestsInDocker(moduleName); Assert.assertEquals(2, exitCode); - } + } - @Test - public void testJavaModuleExample() throws Exception { + @Test + public void testJavaModuleExample() throws Exception { System.out.println("Test [testJavaModuleExample]"); - String lang = "java"; + String lang = "java"; String moduleName = SIMPLE_MODULE_NAME + "Java"; init(lang, moduleName); int exitCode = runTestsInDocker(moduleName); Assert.assertEquals(0, exitCode); - } - - @Test - public void testJavaModuleError() throws Exception { - System.out.println("Test [testJavaModuleError]"); - String lang = "java"; - String moduleName = SIMPLE_MODULE_NAME + "JavaError"; - init(lang, moduleName); + } + + @Test + public void testJavaModuleError() throws Exception { + System.out.println("Test [testJavaModuleError]"); + String lang = "java"; + String moduleName = SIMPLE_MODULE_NAME + "JavaError"; + init(lang, moduleName); File implFile = new File(moduleName + "/lib/src/" + - "asimplemoduleforunittestingjavaerror/ASimpleModuleForUnitTestingJavaErrorServer.java"); + "asimplemoduleforunittestingjavaerror/ASimpleModuleForUnitTestingJavaErrorServer.java"); String implText = FileUtils.readFileToString(implFile); - implText = implText.replace(" //BEGIN filter_contigs", + implText = implText.replace(" //BEGIN filter_contigs", " //BEGIN filter_contigs\n" + " if (true) throw new IllegalStateException(\"Special error\");"); FileUtils.writeStringToFile(implFile, implText); - int exitCode = runTestsInDocker(moduleName); - Assert.assertEquals(2, exitCode); - } + int exitCode = runTestsInDocker(moduleName); + Assert.assertEquals(2, exitCode); + } @Test public void testRModuleError() throws Exception { @@ -179,14 +179,14 @@ public void testRModuleError() throws Exception { init(lang, moduleName); File implFile = new File(moduleName + "/lib/" + moduleName + "/" + moduleName + "Impl.r"); String implText = FileUtils.readFileToString(implFile); - implText = implText.replace(" #BEGIN count_contigs", + implText = implText.replace(" #BEGIN count_contigs", " #BEGIN count_contigs\n" + " stop(\"Special error\")"); FileUtils.writeStringToFile(implFile, implText); int exitCode = runTestsInDocker(moduleName); Assert.assertEquals(2, exitCode); } - + @Test public void testSelfCalls() throws Exception { System.out.println("Test [testSelfCalls]"); @@ -214,17 +214,17 @@ public void testSelfCalls() throws Exception { " returnVal = input * input\n" + " #END calc_square\n"; File moduleDir = new File(moduleName); - File implFile = new File(moduleDir, "lib/" + moduleName + "/" + + File implFile = new File(moduleDir, "lib/" + moduleName + "/" + moduleName + "Impl.py"); ModuleInitializer initer = new ModuleInitializer(moduleName, token.getUserName(), lang, false); initer.initialize(false); File specFile = new File(moduleDir, moduleName + ".spec"); - String specText = FileUtils.readFileToString(specFile).replace("};", + String specText = FileUtils.readFileToString(specFile).replace("};", "funcdef run_local(int input) returns (int) authentication required;\n" + "funcdef calc_square(int input) returns (int) authentication required;\n" + "};"); File testFile = new File(moduleDir, "test/" + moduleName + "_server_test.py"); - String testCode = FileUtils.readFileToString(testFile).replace(" def test_your_method(self):", + String testCode = FileUtils.readFileToString(testFile).replace(" def test_your_method(self):", " def test_your_method(self):\n" + " self.assertEqual(25, self.getImpl().run_local(self.getContext(), 5)[0])" ); @@ -234,4 +234,4 @@ public void testSelfCalls() throws Exception { int exitCode = runTestsInDocker(moduleDir, token, true); Assert.assertEquals(0, exitCode); } -} \ No newline at end of file +} diff --git a/src/java/us/kbase/mobu/util/ProcessHelper.java b/src/java/us/kbase/mobu/util/ProcessHelper.java index c28e75f5..60d2bc16 100644 --- a/src/java/us/kbase/mobu/util/ProcessHelper.java +++ b/src/java/us/kbase/mobu/util/ProcessHelper.java @@ -61,7 +61,7 @@ public static ProcessHelper exec(CommandHolder cmd, File workDir, BufferedReader } public static ProcessHelper exec(CommandHolder cmd, File workDir, BufferedReader input, boolean saveOutput, boolean saveErrors, boolean waitFor) throws IOException { - return new ProcessHelper(cmd, workDir, saveOutput ? OutType.StringOut : OutType.SystemOut, + return new ProcessHelper(cmd, workDir, saveOutput ? OutType.StringOut : OutType.SystemOut, saveErrors ? OutType.StringErr : OutType.SystemErr, input, null, null, waitFor); } @@ -108,9 +108,9 @@ public ProcessHelper(CommandHolder cmd, File workDir, OutType outType, OutType e } public Process getProcess() { - return process; - } - + return process; + } + private Thread readInNewThread(final InputStream is, final OutType outType) { Thread ret = new Thread(new Runnable() { public void run() { @@ -189,10 +189,10 @@ public CommandHolder add(String... newCmdParts) { } public CommandHolder dontWaitFor() { - waitFor = false; - return this; + waitFor = false; + return this; } - + public ProcessHelper exec(File workDir) throws IOException { return ProcessHelper.exec(this, workDir, waitFor); } diff --git a/src/java/us/kbase/mobu/validator/ModuleValidator.java b/src/java/us/kbase/mobu/validator/ModuleValidator.java index 9363d4b0..2f1a3d75 100644 --- a/src/java/us/kbase/mobu/validator/ModuleValidator.java +++ b/src/java/us/kbase/mobu/validator/ModuleValidator.java @@ -39,44 +39,44 @@ public class ModuleValidator { - - - private static final String KBASE_YML_FILE = "kbase.yml"; - - - protected List modulePaths; - protected boolean verbose; - protected String methodStoreUrl; - protected boolean allowSyncMethods; - - public ModuleValidator(List modulePaths, boolean verbose, - String defaultMethodStoreUrl, boolean allowSyncMethods) throws Exception { - this.modulePaths = modulePaths; - this.verbose = verbose; - if (modulePaths.size() == 1) { - File module = new File(modulePaths.get(0)); - File testCfg = new File(new File(module, "test_local"), "test.cfg"); - if (testCfg.exists()) { - Properties props = new Properties(); - InputStream is = new FileInputStream(testCfg); - try { - props.load(is); - } finally { - is.close(); - } - String endPoint = props.getProperty("kbase_endpoint"); - if (endPoint != null) - this.methodStoreUrl = endPoint + "/narrative_method_store/rpc"; - } - } - if (this.methodStoreUrl == null) { - this.methodStoreUrl = defaultMethodStoreUrl; - System.out.println("WARNING! 'kbase_endpoint' property was not found in " + - "/test_local/test.cfg so validation is done against NMS in appdev"); - } - this.allowSyncMethods = allowSyncMethods; - } - + + + private static final String KBASE_YML_FILE = "kbase.yml"; + + + protected List modulePaths; + protected boolean verbose; + protected String methodStoreUrl; + protected boolean allowSyncMethods; + + public ModuleValidator(List modulePaths, boolean verbose, + String defaultMethodStoreUrl, boolean allowSyncMethods) throws Exception { + this.modulePaths = modulePaths; + this.verbose = verbose; + if (modulePaths.size() == 1) { + File module = new File(modulePaths.get(0)); + File testCfg = new File(new File(module, "test_local"), "test.cfg"); + if (testCfg.exists()) { + Properties props = new Properties(); + InputStream is = new FileInputStream(testCfg); + try { + props.load(is); + } finally { + is.close(); + } + String endPoint = props.getProperty("kbase_endpoint"); + if (endPoint != null) + this.methodStoreUrl = endPoint + "/narrative_method_store/rpc"; + } + } + if (this.methodStoreUrl == null) { + this.methodStoreUrl = defaultMethodStoreUrl; + System.out.println("WARNING! 'kbase_endpoint' property was not found in " + + "/test_local/test.cfg so validation is done against NMS in appdev"); + } + this.allowSyncMethods = allowSyncMethods; + } + private static boolean isModuleDir(File dir) { return new File(dir, "Dockerfile").exists() && new File(dir, "Makefile").exists() && @@ -86,52 +86,52 @@ private static boolean isModuleDir(File dir) { new File(dir, "test").exists() && new File(dir, "ui").exists(); } - - public int validateAll() { - - int errors = 0; - - for(String modulePathString : modulePaths) { - File module = new File(modulePathString); - System.out.println("\nValidating module in ("+module+")"); - - if(!module.exists()) { - System.err.println(" **ERROR** - the module does not exist"); - errors++; continue; - } - if(!module.isDirectory()) { - System.err.println(" **ERROR** - the module location is not a directory."); - errors++; continue; - } - - try { - if(verbose) System.out.println(" - canonical path = "+module.getCanonicalPath()+""); - File dir = module.getCanonicalFile(); - while (!isModuleDir(dir)) { - dir = dir.getParentFile(); - if (dir == null) - throw new IllegalStateException(" **ERROR** - cannot find folder with module structure"); - } - module = dir; - } catch (IOException e) { - System.err.println(" **ERROR** - unable to extract module canonical path:"); - System.err.println(" "+e.getMessage()); - } - - - // 1) Validate the configuration file - try { - int status = validateKBaseYmlConfig(module); - if(status!=0) { - errors++; continue; - } - } catch (Exception e) { - System.err.println(" **ERROR** - configuration file validation failed:"); - System.err.println(" "+e.getMessage()); - errors++; continue; - } - - KbModule parsedKidl = null; + + public int validateAll() { + + int errors = 0; + + for(String modulePathString : modulePaths) { + File module = new File(modulePathString); + System.out.println("\nValidating module in ("+module+")"); + + if(!module.exists()) { + System.err.println(" **ERROR** - the module does not exist"); + errors++; continue; + } + if(!module.isDirectory()) { + System.err.println(" **ERROR** - the module location is not a directory."); + errors++; continue; + } + + try { + if(verbose) System.out.println(" - canonical path = "+module.getCanonicalPath()+""); + File dir = module.getCanonicalFile(); + while (!isModuleDir(dir)) { + dir = dir.getParentFile(); + if (dir == null) + throw new IllegalStateException(" **ERROR** - cannot find folder with module structure"); + } + module = dir; + } catch (IOException e) { + System.err.println(" **ERROR** - unable to extract module canonical path:"); + System.err.println(" "+e.getMessage()); + } + + + // 1) Validate the configuration file + try { + int status = validateKBaseYmlConfig(module); + if(status!=0) { + errors++; continue; + } + } catch (Exception e) { + System.err.println(" **ERROR** - configuration file validation failed:"); + System.err.println(" "+e.getMessage()); + errors++; continue; + } + + KbModule parsedKidl = null; try { Map config = parseKBaseYaml(new File(module, KBASE_YML_FILE)); String moduleName = (String)config.get("module-name"); @@ -153,78 +153,78 @@ public int validateAll() { errors++; continue; } - // 2) Validate UI components - - // 2a) Validate Narrative Methods - - File uiNarrativeMethodsDir = new File(new File(new File(module, "ui"), "narrative"), "methods"); - if (uiNarrativeMethodsDir.exists()) { - for (File methodDir : uiNarrativeMethodsDir.listFiles()) { - if (methodDir.isDirectory()) { - System.out.println("\nValidating method in ("+methodDir+")"); - try { - int status = validateMethodSpec(methodDir, parsedKidl, this.allowSyncMethods); - if (status != 0) { - errors++; - continue; - } - } catch (Exception e) { - System.err.println(" **ERROR** - method-spec validation failed:"); - System.err.println(" "+e.getMessage()); - errors++; continue; - } - } - } - } - - } - - - - - if(errors>0) { - if(modulePaths.size()==1) { - System.out.println("\n\nThis module contains errors.\n"); - } else { - System.out.println("\n\nErrors detected in "+errors +" of "+modulePaths.size()+" modules.\n"); - } - return 1; - } - if(modulePaths.size()==1) { - System.out.println("\n\nCongrats- this module is valid.\n"); - } else { - System.out.println("\n\nCongrats- all "+modulePaths.size()+" modules are valid.\n"); - } - return 0; - } - - protected int validateKBaseYmlConfig(File module) throws IOException { - File kbaseYmlFile = new File(module.getCanonicalPath()+File.separator+KBASE_YML_FILE); - if(verbose) System.out.println(" - configuration file = "+kbaseYmlFile); - - if(!kbaseYmlFile.exists()) { - System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file does not exist in module directory."); - return 1; - } - if(kbaseYmlFile.isDirectory()) { - System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file location is a directory, not a file."); - return 1; - } - - try { - parseKBaseYaml(kbaseYmlFile); - if(verbose) System.out.println(" - configuration file is valid YAML"); - } catch(Exception e) { - System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file location is invalid:"); - System.err.println(" "+e.getMessage()); - return 1; - - } - - - - return 0; - } + // 2) Validate UI components + + // 2a) Validate Narrative Methods + + File uiNarrativeMethodsDir = new File(new File(new File(module, "ui"), "narrative"), "methods"); + if (uiNarrativeMethodsDir.exists()) { + for (File methodDir : uiNarrativeMethodsDir.listFiles()) { + if (methodDir.isDirectory()) { + System.out.println("\nValidating method in ("+methodDir+")"); + try { + int status = validateMethodSpec(methodDir, parsedKidl, this.allowSyncMethods); + if (status != 0) { + errors++; + continue; + } + } catch (Exception e) { + System.err.println(" **ERROR** - method-spec validation failed:"); + System.err.println(" "+e.getMessage()); + errors++; continue; + } + } + } + } + + } + + + + + if(errors>0) { + if(modulePaths.size()==1) { + System.out.println("\n\nThis module contains errors.\n"); + } else { + System.out.println("\n\nErrors detected in "+errors +" of "+modulePaths.size()+" modules.\n"); + } + return 1; + } + if(modulePaths.size()==1) { + System.out.println("\n\nCongrats- this module is valid.\n"); + } else { + System.out.println("\n\nCongrats- all "+modulePaths.size()+" modules are valid.\n"); + } + return 0; + } + + protected int validateKBaseYmlConfig(File module) throws IOException { + File kbaseYmlFile = new File(module.getCanonicalPath()+File.separator+KBASE_YML_FILE); + if(verbose) System.out.println(" - configuration file = "+kbaseYmlFile); + + if(!kbaseYmlFile.exists()) { + System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file does not exist in module directory."); + return 1; + } + if(kbaseYmlFile.isDirectory()) { + System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file location is a directory, not a file."); + return 1; + } + + try { + parseKBaseYaml(kbaseYmlFile); + if(verbose) System.out.println(" - configuration file is valid YAML"); + } catch(Exception e) { + System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file location is invalid:"); + System.err.println(" "+e.getMessage()); + return 1; + + } + + + + return 0; + } @SuppressWarnings("unchecked") public Map parseKBaseYaml(File kbaseYmlFile) throws IOException { @@ -232,14 +232,14 @@ public Map parseKBaseYaml(File kbaseYmlFile) throws IOException { String kbaseYml = TextUtils.readFileText(kbaseYmlFile); return (Map) yaml.load(kbaseYml); } - - - protected int validateMethodSpec(File methodDir, KbModule parsedKidl, - boolean allowSyncMethods) throws IOException { - NarrativeMethodStoreClient nms = new NarrativeMethodStoreClient(new URL(methodStoreUrl)); - nms.setAllSSLCertificatesTrusted(true); - nms.setIsInsecureHttpConnectionAllowed(true); - String spec = FileUtils.readFileToString(new File(methodDir, "spec.json")); + + + protected int validateMethodSpec(File methodDir, KbModule parsedKidl, + boolean allowSyncMethods) throws IOException { + NarrativeMethodStoreClient nms = new NarrativeMethodStoreClient(new URL(methodStoreUrl)); + nms.setAllSSLCertificatesTrusted(true); + nms.setIsInsecureHttpConnectionAllowed(true); + String spec = FileUtils.readFileToString(new File(methodDir, "spec.json")); String display = FileUtils.readFileToString(new File(methodDir, "display.yaml")); Map extraFiles = new LinkedHashMap(); for (File f : methodDir.listFiles()) { @@ -272,11 +272,11 @@ protected int validateMethodSpec(File methodDir, KbModule parsedKidl, System.err.println(" "+e.getMessage()); return 1; } - } + } - public static void validateMethodSpecMapping(String specText, KbModule parsedKidl, - boolean allowSyncMethods) throws IOException { - JsonNode spec = new ObjectMapper().readTree(specText); + public static void validateMethodSpecMapping(String specText, KbModule parsedKidl, + boolean allowSyncMethods) throws IOException { + JsonNode spec = new ObjectMapper().readTree(specText); JsonNode behaviorNode = get("/", spec, "behavior"); if (behaviorNode.get("none") != null) return; // Don't pay attention at viewer methods (since they don't use back-end) @@ -336,7 +336,7 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid } } if (func == null) { - throw new IllegalStateException(" **ERROR** - unknown method \"" + + throw new IllegalStateException(" **ERROR** - unknown method \"" + methodName + "\" defined within path " + "[behavior/service-mapping/method] in spec.json"); } @@ -361,9 +361,9 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid targetArgPos = targetArgPosNode.asInt(); if (targetArgPos >= func.getParameters().size()) { throw new IllegalStateException(" **ERROR** - value " + targetArgPos + " within " + - "path [" + path + "/target_argument_position] in spec.json is out of " + - "bounds (number of arguments defined for function \"" + methodName + "\" " + - "is " + func.getParameters().size() + ")"); + "path [" + path + "/target_argument_position] in spec.json is out of " + + "bounds (number of arguments defined for function \"" + methodName + "\" " + + "is " + func.getParameters().size() + ")"); } argsUsed.add(targetArgPos); KbType argType = func.getParameters().get(targetArgPos).getType(); @@ -378,7 +378,7 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid argType instanceof KbTuple) { throw new IllegalStateException(" **ERROR** - value " + targetProp + " within " + "path [" + path + "/target_property] in spec.json can't be applied to " + - "type " + argType.getClass().getSimpleName() + " (defined for argument " + + "type " + argType.getClass().getSimpleName() + " (defined for argument " + targetArgPos + ")"); } if (argType instanceof KbStruct) { @@ -392,8 +392,8 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid } if (!found) { System.err.println(" **WARNINGS** - value \"" + targetProp + "\" within " + - "path [" + path + "/target_property] in spec.json doesn't match " + - "any field of structure defined as argument type" + + "path [" + path + "/target_property] in spec.json doesn't match " + + "any field of structure defined as argument type" + (struct.getName() != null ? (" (" + struct.getName() + ")") : "")); } } @@ -411,22 +411,22 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid } if (func.getParameters().size() != argsUsed.size()) { throw new IllegalStateException(" **ERROR** - not all arguments are set for function " + - "\"" + func.getName() + "\", list of defined arguments is: " + argsUsed); + "\"" + func.getName() + "\", list of defined arguments is: " + argsUsed); } if (inputParamIdToType.size() != paramsUsed.size()) { Set paramsNotUsed = new TreeSet(inputParamIdToType.keySet()); paramsNotUsed.removeAll(paramsUsed); - System.err.println(" **WARNINGS** - some of input parameters are not used: " + + System.err.println(" **WARNINGS** - some of input parameters are not used: " + paramsNotUsed); } - } - - private static JsonNode get(String nodePath, JsonNode node, String childName) { - JsonNode ret = node.get(childName); - if (ret == null) - throw new IllegalStateException(" **ERROR** - can't find sub-node [" + childName + - "] within path [" + nodePath + "] in spec.json"); - return ret; - } + } + + private static JsonNode get(String nodePath, JsonNode node, String childName) { + JsonNode ret = node.get(childName); + if (ret == null) + throw new IllegalStateException(" **ERROR** - can't find sub-node [" + childName + + "] within path [" + nodePath + "] in spec.json"); + return ret; + } } diff --git a/src/java/us/kbase/scripts/test/Test8.perl.properties b/src/java/us/kbase/scripts/test/Test8.perl.properties index 31565d25..68e90e6b 100644 --- a/src/java/us/kbase/scripts/test/Test8.perl.properties +++ b/src/java/us/kbase/scripts/test/Test8.perl.properties @@ -2,7 +2,7 @@ package StoringImpl; use strict; use Bio::KBase::Exceptions; # Use Semantic Versioning (2.0.0-rc.1) -# http://semver.org +# http://semver.org our $VERSION = "0.1.0"; =head1 NAME @@ -226,7 +226,7 @@ sub m2second -=head2 version +=head2 version $return = $obj->version() From 8d0e981048defdc10d15458b490f6af856b90ef5 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 11 Jun 2020 07:24:48 -0700 Subject: [PATCH 07/47] Typo in error message --- src/java/us/kbase/mobu/installer/ClientInstaller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/us/kbase/mobu/installer/ClientInstaller.java b/src/java/us/kbase/mobu/installer/ClientInstaller.java index 3df58d91..6533e223 100644 --- a/src/java/us/kbase/mobu/installer/ClientInstaller.java +++ b/src/java/us/kbase/mobu/installer/ClientInstaller.java @@ -328,7 +328,7 @@ public static void addDependency(String moduleName, boolean isSdk, String versio depMap.put(dep.moduleName.toLowerCase(), dep); } } catch (Exception ex) { - throw new IllegalStateException("Error parsing depedencies file [" + depsFile + + throw new IllegalStateException("Error parsing dependencies file [" + depsFile + "]: " + ex.getMessage(), ex); } } From 2a7474a70d008729276b382e5a625029ecd86984 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 11 Jun 2020 07:33:24 -0700 Subject: [PATCH 08/47] Adding build date/time to kb-sdk. Failing earlier in kb-sdk test. --- build.xml | 5 ++- src/java/us/kbase/mobu/ModuleBuilder.java | 17 +++++++- .../mobu/initializer/ModuleInitializer.java | 41 ++++++++----------- .../us/kbase/mobu/tester/ConfigLoader.java | 13 ++++-- .../us/kbase/mobu/tester/ModuleTester.java | 10 +++-- .../us/kbase/mobu/util/ProcessHelper.java | 5 ++- 6 files changed, 56 insertions(+), 35 deletions(-) diff --git a/build.xml b/build.xml index f1125bc1..6b82a7fa 100644 --- a/build.xml +++ b/build.xml @@ -102,7 +102,7 @@ - + @@ -116,7 +116,8 @@ ### PLEASE DO NOT CHANGE THIS FILE MANUALLY! ### giturl=${git.url} branch=${git.branch} -commit=${git.commit} +commit=${git.commit} +build_timestamp=${DSTAMP}-${TSTAMP} Compiling with jar directory: ${jardir} diff --git a/src/java/us/kbase/mobu/ModuleBuilder.java b/src/java/us/kbase/mobu/ModuleBuilder.java index 54837280..4801efbb 100644 --- a/src/java/us/kbase/mobu/ModuleBuilder.java +++ b/src/java/us/kbase/mobu/ModuleBuilder.java @@ -360,7 +360,22 @@ private static int runTestCommand(TestCommandArgs testArgs, JCommander jc) { private static void printVersion() { String gitCommit = getGitCommit(); - System.out.println("KBase SDK version " + VERSION + (gitCommit == null ? "" : (" (commit " + gitCommit + ")"))); + String timestamp = getBuildTimestamp(); + System.out.println("KBase SDK version " + VERSION + + (timestamp == null ? "" : ("_" + timestamp)) + + (gitCommit == null ? "" : (" (commit " + gitCommit + ")"))); + } + + public static String getBuildTimestamp() { + String buildTimestamp = null; + try { + Properties gitProps = new Properties(); + InputStream is = ModuleBuilder.class.getResourceAsStream("git.properties"); + gitProps.load(is); + is.close(); + buildTimestamp = gitProps.getProperty("build_timestamp"); + } catch (Exception ignore) {} + return buildTimestamp; } public static String getGitCommit() { diff --git a/src/java/us/kbase/mobu/initializer/ModuleInitializer.java b/src/java/us/kbase/mobu/initializer/ModuleInitializer.java index 248a93c4..f8bb566a 100644 --- a/src/java/us/kbase/mobu/initializer/ModuleInitializer.java +++ b/src/java/us/kbase/mobu/initializer/ModuleInitializer.java @@ -169,19 +169,16 @@ public void initialize(boolean example) throws Exception { } switch(this.language) { - // Perl just needs an impl file and a start server script + // start_server scripts are generated by the Makefile case "perl": - //templateFiles.put("module_start_perl_server", Paths.get(moduleDir, "scripts", "start_server.sh")); - // start_server script is now made in Makefile + // Perl needs an impl file templateFiles.put("module_perl_impl", Paths.get(moduleDir, "lib", this.moduleName, this.moduleName + "Impl.pm")); break; - // Python needs some empty __init__.py files and the impl file (Done, see TemplateBasedGenerator.initPyhtonPackages) case "python": + // python requires impl and __init__.py files initDirectory(Paths.get(moduleDir, "lib", this.moduleName), false); initFile(Paths.get(moduleDir, "lib", this.moduleName, "__init__.py"), false); templateFiles.put("module_python_impl", Paths.get(moduleDir, "lib", this.moduleName, this.moduleName + "Impl.py")); - //templateFiles.put("module_start_python_server", Paths.get(moduleDir, "scripts", "start_server.sh")); - // start_server script is now made in Makefile break; case "r": templateFiles.put("module_r_impl", Paths.get(moduleDir, "lib", this.moduleName, this.moduleName + "Impl.r")); @@ -194,8 +191,6 @@ public void initialize(boolean example) throws Exception { File serverJavaFile = new File(srcDir, modulePackage.replace('.', '/') + "/" + javaModuleName + "Server.java"); fillTemplate(moduleContext, "module_java_impl", serverJavaFile.toPath()); JavaTypeGenerator.processSpec(new File(moduleDir, specFile), srcDir, javaPackageParent, true, null, null, null); - //templateFiles.put("module_start_java_server", Paths.get(moduleDir, "scripts", "start_server.sh")); - // start_server script is now made in Makefile break; default: break; @@ -209,38 +204,38 @@ public void initialize(boolean example) throws Exception { // Generated examples require some other SDK dependencies new ClientInstaller(new File(moduleDir), false).install( this.language, - true, // async clients - false, // core or sync clients - false, // dynamic client - null, //tagVer + true, // async clients + false, // core or sync clients + false, // dynamic client + null, // tagVer this.verbose, "AssemblyUtil", null, - null // clientName + null // clientName ); } // Let's install fresh workspace client in any cases (we need it at least in tests): new ClientInstaller(new File(moduleDir), false).install( this.language, - false, // async clients - true, // core or sync clients - false, // dynamic client - null, //tagVer + false, // async clients + true, // core or sync clients + false, // dynamic client + null, // tagVer this.verbose, "https://raw.githubusercontent.com/kbase/workspace_deluxe/master/workspace.spec", null, - null // clientName + null // clientName ); new ClientInstaller(new File(moduleDir), false).install( this.language, - true, // async clients - false, // core or sync clients - false, // dynamic client - null, //tagVer + true, // async clients + false, // core or sync clients + false, // dynamic client + null, // tagVer this.verbose, "KBaseReport", null, - null // clientName + null // clientName ); System.out.println("Done! Your module is available in the " + moduleDir + " directory."); diff --git a/src/java/us/kbase/mobu/tester/ConfigLoader.java b/src/java/us/kbase/mobu/tester/ConfigLoader.java index 05933993..5832da96 100644 --- a/src/java/us/kbase/mobu/tester/ConfigLoader.java +++ b/src/java/us/kbase/mobu/tester/ConfigLoader.java @@ -38,6 +38,11 @@ public ConfigLoader(Properties props, boolean testMode, if (authUrl == null) throw new IllegalStateException("Error: 'auth_service_url' parameter is not set in " + configPathInfo); + endPoint = props.getProperty("kbase_endpoint"); + if (endPoint == null) + throw new IllegalStateException("Error: KBase services end-point is not set in " + + configPathInfo); + String testPrefix = testMode ? "test_" : ""; String user = props.getProperty(testPrefix + "user"); if (user != null && user.trim().isEmpty()) { @@ -47,9 +52,13 @@ public ConfigLoader(Properties props, boolean testMode, String tokenString = props.getProperty(testPrefix + "token"); if (tokenString != null && tokenString.trim().isEmpty()) { tokenString = null; + } + if (tokenString == null) { + System.out.println("No token found in test.cfg file; checking environment"); String test_token = System.getenv("KBASE_TEST_TOKEN"); if (test_token != null && !test_token.trim().isEmpty()) { tokenString = test_token; + System.out.println("Using token from KBASE_TEST_TOKEN env var"); } } @@ -71,10 +80,6 @@ public ConfigLoader(Properties props, boolean testMode, } token = auth.login(user.trim(), password.trim()).getToken(); } - endPoint = props.getProperty("kbase_endpoint"); - if (endPoint == null) - throw new IllegalStateException("Error: KBase services end-point is not set in " + - configPathInfo); jobSrvUrl = getConfigUrl(props, "job_service_url", endPoint, "userandjobstate"); wsUrl = getConfigUrl(props, "workspace_url", endPoint, "ws"); shockUrl = getConfigUrl(props, "shock_url", endPoint, "shock-api"); diff --git a/src/java/us/kbase/mobu/tester/ModuleTester.java b/src/java/us/kbase/mobu/tester/ModuleTester.java index f9c068f9..67662776 100644 --- a/src/java/us/kbase/mobu/tester/ModuleTester.java +++ b/src/java/us/kbase/mobu/tester/ModuleTester.java @@ -226,9 +226,13 @@ public void logNextLine(String line, boolean isError) { try { System.out.println(); ProcessHelper.cmd("chmod", "+x", runTestsSh.getCanonicalPath()).exec(tlDir); - int exitCode = ProcessHelper.cmd("bash", DirUtils.getFilePath(runTestsSh), - callbackUrl == null ? "http://fakecallbackurl" : - callbackUrl.toExternalForm()).exec(tlDir).getExitCode(); + int exitCode = ProcessHelper.cmd( + "bash", + DirUtils.getFilePath(runTestsSh), + callbackUrl == null + ? "http://fakecallbackurl" + : callbackUrl.toExternalForm() + ).exec(tlDir).getExitCode(); return exitCode; } finally { if (jettyServer != null) { diff --git a/src/java/us/kbase/mobu/util/ProcessHelper.java b/src/java/us/kbase/mobu/util/ProcessHelper.java index 60d2bc16..0f102886 100644 --- a/src/java/us/kbase/mobu/util/ProcessHelper.java +++ b/src/java/us/kbase/mobu/util/ProcessHelper.java @@ -75,8 +75,9 @@ public ProcessHelper(CommandHolder cmd, File workDir, OutType outType, OutType e outPw = output; if (error != null) errPw = error; - process = cmd.cmdLine != null ? Runtime.getRuntime().exec(cmd.cmdLine, null, workDir) : - Runtime.getRuntime().exec(cmd.cmdParts, null, workDir); + process = cmd.cmdLine != null + ? Runtime.getRuntime().exec(cmd.cmdLine, null, workDir) + : Runtime.getRuntime().exec(cmd.cmdParts, null, workDir); Thread outTh = readInNewThread(process.getInputStream(), outType); Thread errTh = readInNewThread(process.getErrorStream(), errType); if (input != null) { From 7e58f55ad589fa64f83804c2dc9e380620210c45 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 11 Jun 2020 07:38:10 -0700 Subject: [PATCH 09/47] Adding actions file --- .github/workflows/kb_sdk-actions.yml | 46 ++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/kb_sdk-actions.yml diff --git a/.github/workflows/kb_sdk-actions.yml b/.github/workflows/kb_sdk-actions.yml new file mode 100644 index 00000000..afd88b49 --- /dev/null +++ b/.github/workflows/kb_sdk-actions.yml @@ -0,0 +1,46 @@ +name: Basic kb-sdk tests + +on: + [push, pull_request] + +jobs: + + docker_build_and_test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Add bin to $PATH + run: | + env + echo "::add-path::$GITHUB_WORKSPACE/bin" + - name: Build with Ant + run: | + make + env + - name: checking kb-sdk functions + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + run: | + kb-sdk help + kb-sdk version + kb-sdk init -l perl -u user PerlBase + cd PerlBase + kb-sdk test + cd .. + rm -rf PerlBase + kb-sdk init -l perl -u user --example PerlExample + cd PerlExample + kb-sdk test + cd .. + rm -rf PerlExample + kb-sdk init -l python -u user --example MyContigFilter + cd MyContigFilter + make + kb-sdk test + cd .. From 835b9cc0489d8484966d0143ce000390bf180d7b Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Fri, 12 Jun 2020 07:02:39 -0700 Subject: [PATCH 10/47] Adding in Java example generate/test to github actions; DRYing out retrieval of information from git.properties file. --- .github/workflows/kb_sdk-actions.yml | 20 ++++++++++--- src/java/us/kbase/mobu/ModuleBuilder.java | 28 +++++++------------ .../compiler/report/CompilationReporter.java | 18 ++++++------ 3 files changed, 35 insertions(+), 31 deletions(-) diff --git a/.github/workflows/kb_sdk-actions.yml b/.github/workflows/kb_sdk-actions.yml index afd88b49..668ff0b5 100644 --- a/.github/workflows/kb_sdk-actions.yml +++ b/.github/workflows/kb_sdk-actions.yml @@ -5,7 +5,7 @@ on: jobs: - docker_build_and_test: + kb_sdk_build: runs-on: ubuntu-latest steps: - name: Checkout @@ -15,14 +15,22 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 + - name: Add bin to $PATH run: | - env echo "::add-path::$GITHUB_WORKSPACE/bin" + - name: Build with Ant run: | make env + + - name: Upload kb-sdk build + uses: actions/upload-artifact@v2 + with: + name: kb-sdk_build + path: bin/kb-sdk + - name: checking kb-sdk functions env: KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} @@ -39,8 +47,12 @@ jobs: kb-sdk test cd .. rm -rf PerlExample - kb-sdk init -l python -u user --example MyContigFilter - cd MyContigFilter + kb-sdk init -l python -u user --example PythonExample + cd PythonExample make kb-sdk test cd .. + rm -rf PythonExample + kb-sdk init -l java -u user --example JavaExample + cd JavaExample + kb-sdk test diff --git a/src/java/us/kbase/mobu/ModuleBuilder.java b/src/java/us/kbase/mobu/ModuleBuilder.java index 4801efbb..659abb86 100644 --- a/src/java/us/kbase/mobu/ModuleBuilder.java +++ b/src/java/us/kbase/mobu/ModuleBuilder.java @@ -359,37 +359,29 @@ private static int runTestCommand(TestCommandArgs testArgs, JCommander jc) { } private static void printVersion() { - String gitCommit = getGitCommit(); - String timestamp = getBuildTimestamp(); + String gitCommit = getGitProp("commit"); + String timestamp = getGitProp("build_timestamp"); System.out.println("KBase SDK version " + VERSION + (timestamp == null ? "" : ("_" + timestamp)) + (gitCommit == null ? "" : (" (commit " + gitCommit + ")"))); } - public static String getBuildTimestamp() { - String buildTimestamp = null; - try { - Properties gitProps = new Properties(); - InputStream is = ModuleBuilder.class.getResourceAsStream("git.properties"); - gitProps.load(is); - is.close(); - buildTimestamp = gitProps.getProperty("build_timestamp"); - } catch (Exception ignore) {} - return buildTimestamp; - } - public static String getGitCommit() { - String gitCommit = null; + public static String getGitProp(String propertyToGet) { + String propertyValue = null; try { Properties gitProps = new Properties(); InputStream is = ModuleBuilder.class.getResourceAsStream("git.properties"); gitProps.load(is); is.close(); - gitCommit = gitProps.getProperty("commit"); - } catch (Exception ignore) {} - return gitCommit; + propertyValue = gitProps.getProperty(propertyToGet); + } catch (Exception e) { + showError("Error while retrieving version information", e.getMessage()); + } + return propertyValue; } + private static int runVersionCommand(VersionCommandArgs testArgs, JCommander jc) { printVersion(); return 0; diff --git a/src/java/us/kbase/mobu/compiler/report/CompilationReporter.java b/src/java/us/kbase/mobu/compiler/report/CompilationReporter.java index afef7a0e..535031ed 100644 --- a/src/java/us/kbase/mobu/compiler/report/CompilationReporter.java +++ b/src/java/us/kbase/mobu/compiler/report/CompilationReporter.java @@ -34,12 +34,12 @@ public class CompilationReporter { public static void prepareCompileReport(File codeDir, List services, - boolean perlServerSide, String perlImplName, boolean pyServerSide, + boolean perlServerSide, String perlImplName, boolean pyServerSide, String pyImplName, boolean rServerSide, String rImplName, - boolean javaServerSide, String javaPackageParent, String javaSrcPath, + boolean javaServerSide, String javaPackageParent, String javaSrcPath, JavaData javaParsingData, List specFiles, File reportFile) throws Exception { String sdkVersion = ModuleBuilder.VERSION; - String sdkGitCommit = ModuleBuilder.getGitCommit(); + String sdkGitCommit = ModuleBuilder.getGitProp("commit"); String moduleName = null; KbModule module = null; for (KbService srv : services) @@ -74,11 +74,11 @@ public static void prepareCompileReport(File codeDir, List services, FileSaver javaSrcDir = new DiskFileSaver(javaSrc); for (JavaModule jmodule : javaParsingData.getModules()) { if (jmodule.getOriginal().getModuleName().equals(moduleName)) { - String moduleDir = JavaTypeGenerator.sub(javaPackageParent, + String moduleDir = JavaTypeGenerator.sub(javaPackageParent, jmodule.getModulePackage()).replace('.', '/'); - String serverClassName = TextUtils.capitalize(jmodule.getModuleName()) + + String serverClassName = TextUtils.capitalize(jmodule.getModuleName()) + "Server"; - implFile = javaSrcDir.getAsFileOrNull(moduleDir + "/" + + implFile = javaSrcDir.getAsFileOrNull(moduleDir + "/" + serverClassName + ".java"); } } @@ -102,7 +102,7 @@ public static void prepareCompileReport(File codeDir, List services, public static Report createReport(List specFiles, String sdkVersion, String sdkGitCommit, String moduleName, - KbModule module, String implFilePath, String implCommentPrefix, + KbModule module, String implFilePath, String implCommentPrefix, String implText) throws Exception, IOException { Map funcPositions = new LinkedHashMap(); String commentPrefix = Pattern.quote(implCommentPrefix); @@ -111,8 +111,8 @@ public static Report createReport(List specFiles, if (comp instanceof KbFuncdef) { KbFuncdef func = (KbFuncdef)comp; String funcName = func.getName(); - Pattern p = Pattern.compile(MessageFormat.format(".*" + commentPrefix + - "BEGIN {0}\n(.*\n)?[ \t]*" + commentPrefix + "END {0}\n.*", + Pattern p = Pattern.compile(MessageFormat.format(".*" + commentPrefix + + "BEGIN {0}\n(.*\n)?[ \t]*" + commentPrefix + "END {0}\n.*", funcName), Pattern.DOTALL); FunctionPlace place = checkMatch(funcPositions, p, implText); if (place != null) From 9ac1aba13cbf1878be827f5cb44139de997dd85c Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 16 Jun 2020 09:28:59 -0700 Subject: [PATCH 11/47] Updating actions to create a build/test matrix. Report version when starting kb-sdk. Add release notes. Fix KBase report class name error in Java version. --- .github/workflows/kb_sdk-actions.yml | 132 ++++++++++++++---- RELEASE_NOTES.txt | 20 ++- src/java/us/kbase/mobu/ModuleBuilder.java | 8 +- .../templates/module_java_impl.vm.properties | 26 ++-- .../module_test_java_client.vm.properties | 34 +++-- 5 files changed, 157 insertions(+), 63 deletions(-) diff --git a/.github/workflows/kb_sdk-actions.yml b/.github/workflows/kb_sdk-actions.yml index 668ff0b5..86ac9bba 100644 --- a/.github/workflows/kb_sdk-actions.yml +++ b/.github/workflows/kb_sdk-actions.yml @@ -5,8 +5,16 @@ on: jobs: - kb_sdk_build: - runs-on: ubuntu-latest + test_kb-sdk_builds: + if: "!contains(github.event.head_commit.message, 'skip ci')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] + language: [perl, python, java] + test_type: [base, example] + steps: - name: Checkout uses: actions/checkout@v2 @@ -16,43 +24,113 @@ jobs: with: java-version: 1.8 + - name: Build with Ant + run: | + make + - name: Add bin to $PATH run: | echo "::add-path::$GITHUB_WORKSPACE/bin" + - name: checking kb-sdk functions + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + env + kb-sdk help + kb-sdk version + + - name: init base repo + if: matrix.test_type == 'base' + env: + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + kb-sdk init -l $LANGUAGE_TOKEN -u user SampleApp + + - name: init example repo + if: matrix.test_type == 'example' + env: + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + kb-sdk init -l $LANGUAGE_TOKEN -u user --example SampleApp + + - name: test ${{ matrix.test_type }} repo + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + cd SampleApp + kb-sdk test || true + sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg + kb-sdk test + + - name: make resulting app available as artefact in case of failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: kbaseapp-${{ matrix.language }}-${{ matrix.test_type }}-${{ matrix.os }} + path: SampleApp + + + test_existing_repos: + if: "!contains(github.event.head_commit.message, 'skip ci')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] + app: [WsLargeDataIO, KBaseReport] + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build with Ant run: | make + + - name: Add bin to $PATH + run: | + echo "::add-path::$GITHUB_WORKSPACE/bin" + + - name: checking basic kb-sdk functions + run: | env + kb-sdk help + kb-sdk version - - name: Upload kb-sdk build - uses: actions/upload-artifact@v2 + - name: Checkout existing kbase module ${{ matrix.app }} + uses: actions/checkout@v2 with: - name: kb-sdk_build - path: bin/kb-sdk + repository: kbaseapps/${{ matrix.app }} + path: kbase_app - - name: checking kb-sdk functions + - name: prep for repo tests + run: | + cd kbase_app + kb-sdk test || true + + - name: edit test.cfg env: KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} run: | - kb-sdk help - kb-sdk version - kb-sdk init -l perl -u user PerlBase - cd PerlBase - kb-sdk test - cd .. - rm -rf PerlBase - kb-sdk init -l perl -u user --example PerlExample - cd PerlExample - kb-sdk test - cd .. - rm -rf PerlExample - kb-sdk init -l python -u user --example PythonExample - cd PythonExample - make - kb-sdk test - cd .. - rm -rf PythonExample - kb-sdk init -l java -u user --example JavaExample - cd JavaExample + cd kbase_app + sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg + + - name: run test + run: | + cd kbase_app kb-sdk test + + - name: make resulting app available as artefact in case of failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: kbaseapp-${{ matrix.app }}-${{ matrix.os }} + path: kbase_app diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 40808007..601fd2ae 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,11 +1,21 @@ OVERVIEW ----------------------------------------- -The KBase SDK is a set of tools for developing new modules -that can be dynamically registered and run on the KBase -platform. Modules include all code, specification files, and -documentation needed to define and run a set of methods +The KBase SDK is a set of tools for developing new modules +that can be dynamically registered and run on the KBase +platform. Modules include all code, specification files, and +documentation needed to define and run a set of methods in the KBase Narrative interface. +VERSION: 1.2.2 (Released 06/xx/2020) +------------------------------------ +NEW FEATURES: +- kb-sdk version reports the build time and date of the instance +- kb-sdk can use the environment variable KBASE_TEST_TOKEN if a test token is not present in the test.cfg file +- GitHub Actions now used to run basic tests on initialising and testing new base and sample repos. + +BUG FIXES: +- Java client uses the correct name for KBase report + VERSION: 1.2.0 (Released 11/14/2018) ------------------------------------ NEW FEATURES: @@ -74,7 +84,7 @@ VERSION: 1.0.0 (Released 11/19/2015) ------------------------------------------ NEW FEATURES: - R language is now supported for generation of clients and server stubs. -- Test sub-command is fully implemented for all 4 languages (perl, +- Test sub-command is fully implemented for all 4 languages (perl, python, java and R). - Version sub-command was added diff --git a/src/java/us/kbase/mobu/ModuleBuilder.java b/src/java/us/kbase/mobu/ModuleBuilder.java index 659abb86..d7a18836 100644 --- a/src/java/us/kbase/mobu/ModuleBuilder.java +++ b/src/java/us/kbase/mobu/ModuleBuilder.java @@ -49,7 +49,7 @@ public class ModuleBuilder { public static final String GLOBAL_SDK_HOME_ENV_VAR = "KB_SDK_HOME"; public static final String DEFAULT_METHOD_STORE_URL = "https://appdev.kbase.us/services/narrative_method_store/rpc"; - public static final String VERSION = "1.2.0"; + public static final String VERSION = "1.2.2"; public static void main(String[] args) throws Exception { @@ -58,7 +58,6 @@ public static void main(String[] args) throws Exception { JCommander jc = new JCommander(gArgs); jc.setProgramName(MODULE_BUILDER_SH_NAME); - // add the 'init' command InitCommandArgs initArgs = new InitCommandArgs(); jc.addCommand(INIT_COMMAND, initArgs); @@ -95,6 +94,9 @@ public static void main(String[] args) throws Exception { RunCommandArgs runArgs = new RunCommandArgs(); jc.addCommand(RUN_COMMAND, runArgs); + // print name and version + printVersion(); + // parse the arguments and gracefully catch any errors try { jc.parse(args); @@ -383,7 +385,7 @@ public static String getGitProp(String propertyToGet) { private static int runVersionCommand(VersionCommandArgs testArgs, JCommander jc) { - printVersion(); + // version was printed at start up, so just exit return 0; } diff --git a/src/java/us/kbase/templates/module_java_impl.vm.properties b/src/java/us/kbase/templates/module_java_impl.vm.properties index 97ee23b5..ee13035f 100644 --- a/src/java/us/kbase/templates/module_java_impl.vm.properties +++ b/src/java/us/kbase/templates/module_java_impl.vm.properties @@ -22,7 +22,7 @@ import assemblyutil.GetAssemblyParams; import assemblyutil.SaveAssemblyParams; import kbasereport.CreateParams; import kbasereport.KBaseReportClient; -import kbasereport.Report; +import kbasereport.SimpleReport; import kbasereport.ReportInfo; import kbasereport.WorkspaceObject; import net.sf.jfasta.FASTAElement; @@ -73,11 +73,11 @@ public class ${java_module_name}Server extends JsonServerServlet { public FilterContigsResults filterContigs(FilterContigsParams params, AuthToken authPart, RpcContext jsonRpcContext) throws Exception { FilterContigsResults returnVal = null; //BEGIN run_${module_name} - + // Print statements to stdout/stderr are captured and available as the App log System.out.println("Starting filter contigs. Parameters:"); System.out.println(params); - + /* Step 1 - Parse/examine the parameters and catch any errors * It is important to check that parameters exist and are defined, and that nice error * messages are returned to users. Parameter values go through basic validation when @@ -103,7 +103,7 @@ public class ${java_module_name}Server extends JsonServerServlet { throw new IllegalArgumentException("min_length parameter cannot be negative (" + minLength + ")"); } - + /* Step 2 - Download the input data as a Fasta file * We can use the AssemblyUtils module to download a FASTA file from our Assembly data * object. The return object gives us the path to the file that was created. @@ -142,26 +142,26 @@ public class ${java_module_name}Server extends JsonServerServlet { final String resultText = String.format("Filtered assembly to %s contigs out of %s", remaining, total); System.out.println(resultText); - + // Step 4 - Save the new Assembly back to the system - + final String newAssyRef = assyUtil.saveAssemblyFromFasta(new SaveAssemblyParams() .withAssemblyName(fileobj.getAssemblyName()) .withWorkspaceName(workspaceName) .withFile(new FastaAssemblyFile().withPath(out.toString()))); - + // Step 5 - Build a Report and return - + final KBaseReportClient kbr = new KBaseReportClient(callbackURL, authPart); // see note above about bad practice kbr.setIsInsecureHttpConnectionAllowed(true); final ReportInfo report = kbr.create(new CreateParams().withWorkspaceName(workspaceName) - .withReport(new Report().withTextMessage(resultText) + .withReport(new SimpleReport().withTextMessage(resultText) .withObjectsCreated(Arrays.asList(new WorkspaceObject() .withDescription("Filtered contigs") .withRef(newAssyRef))))); // Step 6: contruct the output to send back - + returnVal = new ReportResults() .withReportName(report.getName()) .withReportRef(report.getRef()); @@ -195,7 +195,7 @@ import java.net.MalformedURLException; import kbasereport.CreateParams; import kbasereport.KBaseReportClient; -import kbasereport.Report; +import kbasereport.SimpleReport; import kbasereport.ReportInfo; import kbasereport.WorkspaceObject; import ${java_package}.ReportResults; @@ -221,7 +221,7 @@ import ${java_package}.ReportResults; final String parameter_1 = params.get("parameter_1").asInstance(); final ReportInfo report = kbr.create(new CreateParams() .withWorkspaceName(ws_name) - .withReport(new Report() + .withReport(new SimpleReport() .withTextMessage(parameter_1) .withObjectsCreated(new java.util.ArrayList()))); @@ -230,4 +230,4 @@ import ${java_package}.ReportResults; .withReportRef(report.getRef()); //END run_${module_name} -#end \ No newline at end of file +#end diff --git a/src/java/us/kbase/templates/module_test_java_client.vm.properties b/src/java/us/kbase/templates/module_test_java_client.vm.properties index 51bdce43..0dbc2fd8 100644 --- a/src/java/us/kbase/templates/module_test_java_client.vm.properties +++ b/src/java/us/kbase/templates/module_test_java_client.vm.properties @@ -46,7 +46,7 @@ public class ${java_module_name}ServerTest { private static ${java_module_name}Server impl = null; private static Path scratch; private static URL callbackURL; - + @BeforeClass public static void init() throws Exception { // Config loading @@ -72,7 +72,7 @@ public class ${java_module_name}ServerTest { .getAbsolutePath()); impl = new ${java_module_name}Server(); } - + private static String getWsName() throws Exception { if (wsName == null) { long suffix = System.currentTimeMillis(); @@ -81,13 +81,13 @@ public class ${java_module_name}ServerTest { } return wsName; } - + private static RpcContext getContext() { return new RpcContext().withProvenance(Arrays.asList(new ProvenanceAction() .withService("${module_name}").withMethod("please_never_use_it_in_production") .withMethodParams(new ArrayList()))); } - + @AfterClass public static void cleanup() { if (wsName != null) { @@ -99,7 +99,7 @@ public class ${java_module_name}ServerTest { } } } - + #if ($example) private String loadFASTA( final Path filename, @@ -116,9 +116,9 @@ public class ${java_module_name}ServerTest { .withAssemblyName(objectName) .withWorkspaceName(getWsName()) .withFile(new FastaAssemblyFile().withPath(filename.toString()))); - + } - + @Test public void test_run${java_module_name}_ok() throws Exception { // First load a test FASTA file as an KBase Assembly @@ -128,9 +128,9 @@ public class ${java_module_name}ServerTest { "agctt\n" + ">seq3\n" + "agcttttcatgg"; - + final String ref = loadFASTA(scratch.resolve("test1.fasta"), "TestAssembly", fastaContent); - + // second, call the implementation Map params = new HashMap<>(); params.put("workspace_name", new UObject(getWsName())); @@ -139,7 +139,7 @@ public class ${java_module_name}ServerTest { impl.run${java_module_name}(params, token, getContext()); } - + @Test public void test_run${java_module_name}_err1() throws Exception { try { @@ -153,7 +153,7 @@ public class ${java_module_name}ServerTest { ex.getMessage()); } } - + @Test public void test_run${java_module_name}_err2() throws Exception { try { @@ -167,7 +167,7 @@ public class ${java_module_name}ServerTest { Assert.assertEquals("min_length parameter cannot be negative (-10)", ex.getMessage()); } } - + @Test public void test_run${java_module_name}_err3() throws Exception { try { @@ -178,8 +178,12 @@ public class ${java_module_name}ServerTest { impl.run${java_module_name}(params, token, getContext()); Assert.fail("Error is expected above"); } catch (ServerException ex) { - Assert.assertEquals("Error on ObjectSpecification #1: Illegal number of separators " + - "/ in object reference fake", ex.getMessage()); + // ex.getMessage() returns the error string quoted + // so just test that the message is the same + Assert.assertTrue( ex.getMessage().contains( + "Error on ObjectSpecification #1: Illegal number of separators " + + "/ in object reference fake" + ) ); } } #else @@ -200,4 +204,4 @@ public class ${java_module_name}ServerTest { final ReportResults ret = impl.run${java_module_name}(params, token, getContext()); } #end -} \ No newline at end of file +} From 2384e3b0321e2cb1db494f0d35846d96b4e97a62 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 17 Jun 2020 11:56:41 -0700 Subject: [PATCH 12/47] Updating release notes with more specific details --- RELEASE_NOTES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 601fd2ae..fe41bf2e 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -14,7 +14,7 @@ NEW FEATURES: - GitHub Actions now used to run basic tests on initialising and testing new base and sample repos. BUG FIXES: -- Java client uses the correct name for KBase report +- Java client generated by kb-sdk uses the correct name for KBase report VERSION: 1.2.0 (Released 11/14/2018) ------------------------------------ From 48f8e29270d2f3dff215d03db0f7bea3c50bf76c Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 17 Jun 2020 16:40:18 -0700 Subject: [PATCH 13/47] Reworded release notes message. Let's skip ci this time. --- RELEASE_NOTES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index fe41bf2e..d71d78db 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -14,7 +14,7 @@ NEW FEATURES: - GitHub Actions now used to run basic tests on initialising and testing new base and sample repos. BUG FIXES: -- Java client generated by kb-sdk uses the correct name for KBase report +- Corrected a class name in the generated tests for the Java SDK module. VERSION: 1.2.0 (Released 11/14/2018) ------------------------------------ From c3e81fbc04f6929db26b6b304bc4633f45ad18ac Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 18 Jun 2020 13:12:34 -0700 Subject: [PATCH 14/47] Die if the git.properties file cannot be found --- src/java/us/kbase/mobu/ModuleBuilder.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/java/us/kbase/mobu/ModuleBuilder.java b/src/java/us/kbase/mobu/ModuleBuilder.java index d7a18836..58b54125 100644 --- a/src/java/us/kbase/mobu/ModuleBuilder.java +++ b/src/java/us/kbase/mobu/ModuleBuilder.java @@ -371,14 +371,15 @@ private static void printVersion() { public static String getGitProp(String propertyToGet) { String propertyValue = null; + Properties gitProps = new Properties(); try { - Properties gitProps = new Properties(); InputStream is = ModuleBuilder.class.getResourceAsStream("git.properties"); gitProps.load(is); is.close(); propertyValue = gitProps.getProperty(propertyToGet); } catch (Exception e) { - showError("Error while retrieving version information", e.getMessage()); + showError("Error while retrieving version information from git.properties file: ", e.getMessage()); + System.exit(1); } return propertyValue; } From 2813c80604ca7ee5ffe3a858a43529fd7ca9a5a4 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Mon, 22 Jun 2020 11:25:37 -0700 Subject: [PATCH 15/47] Update dockerfile with updated dependencies --- Dockerfile | 32 +++++++++++++++++++------------- RELEASE_NOTES.txt | 7 ++++++- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 63eb7f27..46ac1bd2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,30 @@ -FROM ubuntu:14.04 +FROM ubuntu:20.04 MAINTAINER Shane Canon # Update apt and install jdk and docker engine to get docker clients +# Docker installation instructions from https://docs.docker.com/engine/install/ubuntu/ RUN apt-get -y update && \ - apt-get -y install openjdk-7-jdk make git ant && \ - apt-get -y install apt-transport-https ca-certificates && \ - apt-key adv \ - --keyserver hkp://ha.pool.sks-keyservers.net:80 \ - --recv-keys 58118E89F3A912897C070ADBF76221572C52609D && \ - echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" > /etc/apt/sources.list.d/docker.list && \ - apt-get -y update && apt-get -y install docker-engine=1.11.2-0~trusty + DEBIAN_FRONTEND=noninteractive apt-get -y install tzdata && \ + apt-get -y install openjdk-8-jdk make git ant curl gnupg-agent && \ + apt-get -y install apt-transport-https ca-certificates software-properties-common && \ + update-java-alternatives -s java-1.8.0-openjdk-amd64 +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" && \ + apt-get -y update && \ + apt-get -y install docker-ce docker-ce-cli containerd.io + +# Add kb-sdk src and fix CallbackServer interface ADD . /src -# Add kb_sdk src and fix CallbackServer interface RUN \ - cd /src && \ - sed -i 's/en0/eth0/' src/java/us/kbase/common/executionengine/CallbackServer.java && \ - make && \ - /src/entrypoint prune && rm -rf /src/.git + cd /src && \ + sed -i 's/en0/eth0/' src/java/us/kbase/common/executionengine/CallbackServer.java && \ + make && \ + /src/entrypoint prune && rm -rf /src/.git ENV PATH=$PATH:/src/bin diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index d71d78db..d81971d0 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -6,7 +6,12 @@ platform. Modules include all code, specification files, and documentation needed to define and run a set of methods in the KBase Narrative interface. -VERSION: 1.2.2 (Released 06/xx/2020) +VERSION: 1.2.3 (Released 06/2020) +------------------------------------ +NEW FEATURES: +- updated kb-sdk Dockerfile to use Java 8, the most recent Docker engine, and Ubuntu 20.04. + +VERSION: 1.2.2 (Released 06/2020) ------------------------------------ NEW FEATURES: - kb-sdk version reports the build time and date of the instance From 965b8407f6b8386a041da3230779351b8d87d1c2 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 20 Aug 2020 12:11:47 -0700 Subject: [PATCH 16/47] Bump version to 1.2.3 --- src/java/us/kbase/mobu/ModuleBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/us/kbase/mobu/ModuleBuilder.java b/src/java/us/kbase/mobu/ModuleBuilder.java index 58b54125..94e8f386 100644 --- a/src/java/us/kbase/mobu/ModuleBuilder.java +++ b/src/java/us/kbase/mobu/ModuleBuilder.java @@ -49,7 +49,7 @@ public class ModuleBuilder { public static final String GLOBAL_SDK_HOME_ENV_VAR = "KB_SDK_HOME"; public static final String DEFAULT_METHOD_STORE_URL = "https://appdev.kbase.us/services/narrative_method_store/rpc"; - public static final String VERSION = "1.2.2"; + public static final String VERSION = "1.2.3"; public static void main(String[] args) throws Exception { From 76e37289899412781d0434c22031d16a70968ec0 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 16 Jun 2020 12:23:42 -0700 Subject: [PATCH 17/47] try removing sdkbase dir --- .gitignore | 2 - .travis.yml | 1 - Makefile | 7 +- doc/building_sdk.md | 13 +- doc/codebase_anatomy.md | 1 - entrypoint | 3 - sdkbase/Dockerfile | 44 --- sdkbase/create_certs | 20 - sdkbase/lets_encript/lets-encript-readme.txt | 1 - .../lets-encrypt-x3-cross-signed.der | Bin 1174 -> 0 bytes sdkbase/makeconfig | 4 - sdkbase/openssl.cnf | 355 ------------------ sdkbase/sdkbase.build.tag | 1 - .../templates/module_travis.vm.properties | 3 +- 14 files changed, 6 insertions(+), 449 deletions(-) delete mode 100644 sdkbase/Dockerfile delete mode 100755 sdkbase/create_certs delete mode 100644 sdkbase/lets_encript/lets-encript-readme.txt delete mode 100644 sdkbase/lets_encript/lets-encrypt-x3-cross-signed.der delete mode 100755 sdkbase/makeconfig delete mode 100644 sdkbase/openssl.cnf delete mode 100644 sdkbase/sdkbase.build.tag diff --git a/.gitignore b/.gitignore index 1cf1ed55..5705f26d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,6 @@ /temp /dist /eclipse-classes/ -/sdkbase/ssl -/sdkbase/cluster.ini /classes/ /temp_* /ASimpleModule* diff --git a/.travis.yml b/.travis.yml index 6b83fb97..4087b9a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,7 +35,6 @@ install: script: - make - - make sdkbase - export PATH=$(pwd)/bin:$PATH - source src/sh/sdk-completion.sh - kb-sdk help diff --git a/Makefile b/Makefile index b1e51fb5..773931f0 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ KBASE_COMMON_JAR = kbase/common/kbase-common-0.0.23.jar QUOTE = '\'' # make sure our make test works -.PHONY : test test-python sdkbase +.PHONY : test test-python default: compile @@ -103,11 +103,6 @@ deploy-scripts: fi; $(ANT) deploy_bin -DBIN_TARGET=$(TARGET)/bin -DBIN_LIB_TARGET=$(TARGET)/lib -DKBASE_COMMON_JAR=$(KBASE_COMMON_JAR) -sdkbase: - # docker rmi -f kbase/deplbase:latest - cd sdkbase && ./makeconfig - docker build --no-cache -t kbase/kbase:sdkbase2.latest sdkbase - test: submodule-init @echo "Running unit tests" make test-python diff --git a/doc/building_sdk.md b/doc/building_sdk.md index 04a17f8b..cc6d1866 100644 --- a/doc/building_sdk.md +++ b/doc/building_sdk.md @@ -1,9 +1,9 @@ -#### Building SDK +## Building the SDK System Dependencies: - Mac OS X 10.8+ or Linux. kb-sdk does not run natively in Windows, but see [here](doc/FAQ.md#windows) for more details. - - Java JRE 7 or 8 (9 is currently incompatible) http://www.oracle.com/technetwork/java/javase/downloads/index.html + - Java JRE 8: http://www.oracle.com/technetwork/java/javase/downloads/index.html (9 is currently incompatible; the SDK will run on Java 7, but using the more modern Java 8 is recommended) - (Mac only) Xcode https://developer.apple.com/xcode - git https://git-scm.com - Docker https://www.docker.com (for local testing) @@ -11,24 +11,20 @@ System Dependencies: Get the SDK: git clone https://github.com/kbase/kb_sdk + git clone https://github.com/kbase/jars Pull dependencies and configure the SDK: cd kb_sdk make bin -Download the local KBase SDK base Docker image: - - docker pull kbase/sdkbase2:latest - Add the kb-sdk tool to your PATH and enable command completion. From the kb_sdk directory: # for bash export PATH=$(pwd)/bin:$PATH source src/sh/sdk-completion.sh - -#### Build from source +### Build from source Additional System Dependencies: @@ -39,4 +35,3 @@ Follow basic instructions above. Instead of running `make bin` you can run `mak cd kb_sdk make - diff --git a/doc/codebase_anatomy.md b/doc/codebase_anatomy.md index 91ea0e9e..3ef5aa2c 100644 --- a/doc/codebase_anatomy.md +++ b/doc/codebase_anatomy.md @@ -16,7 +16,6 @@ This document describes the the file structure of the `kb_sdk` codebase. * `Makefile` - commands for compiling, building docker images, and initializing submodules * `Pipfile` and `Pipfile.lock` - python dependencies for pipenv * `reports/` - files generated for JaCoCo test coverage reports -* `sdkbase/` - Docker files for the image used inside actual SDK apps * `src/` - the main source code for this project; see below * `submodules/` and `submodules_hacks` - jars and other dependencies * `test_scripts/` - test helpers in perl, python, and js diff --git a/entrypoint b/entrypoint index 65251b46..afa74c85 100755 --- a/entrypoint +++ b/entrypoint @@ -17,9 +17,6 @@ fi exec docker run -it --rm -v \$HOME:\$HOME -u \$(id -u) -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e KBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$(cat \$HOME/.kbsdk.cache) kbase/kb-sdk \$@ EOF -elif [ "z$1" = "zsdkbase" ] || [ "z$1" = "zpull-base-image" ] ; then - echo "Pulling and tagging the base image" - docker pull kbase/kbase:sdkbase2.latest elif [ "z$1" = "zprune" ] ; then echo "Used during build to shrink image. Not needed by the user." for f in $(find /src/submodules/jars/lib/jars -type f -name '*.jar') ; do diff --git a/sdkbase/Dockerfile b/sdkbase/Dockerfile deleted file mode 100644 index a4045629..00000000 --- a/sdkbase/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -FROM kbase/deplbase:latest - -COPY ./sdkbase.build.tag /tmp/ - -# Update certs -RUN apt-get update -RUN apt-get install ca-certificates - -# Fix Python SSL warnings for python < 2.7.9 (system python on Trusty is 2.7.6) -# https://github.com/pypa/pip/issues/4098 -RUN pip install pip==8.1.2 -RUN pip install --disable-pip-version-check requests requests_toolbelt pyopenssl --upgrade - -#install coverage tool -RUN pip install coverage - -RUN \ - . /kb/dev_container/user-env.sh && \ - cd /kb/dev_container/modules && \ - rm -rf auth && \ - git clone -b auth2 https://github.com/kbase/auth && \ - cd /kb/dev_container/modules/auth && \ - make && make deploy - -COPY ./lets_encript/lets-encrypt-x3-cross-signed.der /tmp/ -RUN keytool -import -keystore /usr/lib/jvm/java-7-oracle/jre/lib/security/cacerts \ - -storepass changeit -noprompt -trustcacerts -alias letsencryptauthorityx31 \ - -file /tmp/lets-encrypt-x3-cross-signed.der -RUN rm /tmp/lets-encrypt-x3-cross-signed.der - -# Update kb_sdk at build time -RUN \ - . /kb/dev_container/user-env.sh && \ - rm /kb/runtime/java && \ - ln -s /usr/lib/jvm/java-7-oracle /kb/runtime/java && \ - cd /kb/dev_container/modules && \ - rm -rf jars && \ - git clone https://github.com/kbase/jars && \ - rm -rf kb_sdk && \ - git clone https://github.com/kbase/kb_sdk -b develop && \ - cd /kb/dev_container/modules/jars && \ - make && make deploy && \ - cd /kb/dev_container/modules/kb_sdk && \ - make && make deploy diff --git a/sdkbase/create_certs b/sdkbase/create_certs deleted file mode 100755 index c78e39d3..00000000 --- a/sdkbase/create_certs +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -[ -e ./ssl ] || mkdir ./ssl - -[ -z $PUBLIC_ADDRESS ] && PUBLIC_ADDRESS="kbase.us" -export PUBLIC_ADDRESS - -# Create certs for development deploy -if [ -e ./ssl/proxy.key ] ; then - echo "Skipping proxy cert" -else - openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout ./ssl/proxy.key -out ./ssl/proxy.crt -config openssl.cnf -batch -fi - -if [ -e ./ssl/narrative.key ] ; then - echo "Skipping narrative cert" -else - cp ./ssl/proxy.key ./ssl/narrative.key - cp ./ssl/proxy.crt ./ssl/narrative.crt -fi diff --git a/sdkbase/lets_encript/lets-encript-readme.txt b/sdkbase/lets_encript/lets-encript-readme.txt deleted file mode 100644 index c0f188be..00000000 --- a/sdkbase/lets_encript/lets-encript-readme.txt +++ /dev/null @@ -1 +0,0 @@ -Certificate was loaded from https://letsencrypt.org/certs/lets-encrypt-x3-cross-signed.der on 2016-12-19. \ No newline at end of file diff --git a/sdkbase/lets_encript/lets-encrypt-x3-cross-signed.der b/sdkbase/lets_encript/lets-encrypt-x3-cross-signed.der deleted file mode 100644 index e08466c5a7cc64b2bf97e9e7e658a00543bf3211..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1174 zcmXqLVwq&n#9Xz2nTe5!Nq~#d(TRb9F}Srji@WvBat;GtHcqWJkGAi;jEvl@36s;oISRp<>3NAIrA4Uh&j1wXVrpV! zWSDcA=R?#}J@@TJvF6R2Gm;HnT{l&eRDHO2pXtYfN&4G&cssJLUOCT{BjU|Qts@!j z+>0NVK6&l0_q|9qh$z3@aP-_(ic|2>=eGQq~-^kmKfP`JhC5R4vw5eA)gIAakh*Cmi&<$y_8PCx(i1!@^cPL@*HTc` zWI0xH;DX`VenX3?V!Kut?{NOOQU0(&{h~@kVOw3ZACLchGS1p*X?S$~kzQWe6fxsf zzgjf6UEw}(qxSG*CT2zk#>GvHwZIUmG!Oy?sH`v}<9`+o12!PV#K>U42NL54iLn6F zN1H)C8;3R2P#h-gA@BM&1Jqk%meH_+KEj7|4} z{$n&SFwld!fr(K}3F?Mqpc6oL=~fu(<)oGrr$UmNUVc%!fepx1c@|RxV*|ql`U`a1 zv|*Zxa?sojOz%Pd{vpne5ys9zK6=STIR>&I-Fz%!EFvqsvXWV9@9uhb(Q^AT*VOl) zSG-w>oJ4_H446b28SWZi7yO(YVR2VN^fzbdzOt~L1s+Q?-PNlPsc=mC7$xr|q-m~F z5-5|DvE%I2g7l}~m+bM=2s}Ic#k%&nuWsy$TF5x-&WF2MvROCm!j@;Oxdt4HR-{@kf)2a)+!u%`E2){6W_KTHd;L6O!Z+>x=&G>hD&N+uyM)Tw+#W)@;$%&5TiZ?#vJAaME2mNZDq`5a2Ps~@9)}OFZ z(#`f;o%FJ*_MMBXawUGLE}hse`rwn8e#8bgeVq)oW8O2)-kQ(zvwp%`XD=Zazx?7V bjaN%nhllF4Ox`})Hs(OaAVarCE7 diff --git a/sdkbase/makeconfig b/sdkbase/makeconfig deleted file mode 100755 index 85ac92d9..00000000 --- a/sdkbase/makeconfig +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -docker run --rm --entrypoint cat kbase/deplbase:latest /kb/deployment/deployment.cfg > cluster.ini -./create_certs diff --git a/sdkbase/openssl.cnf b/sdkbase/openssl.cnf deleted file mode 100644 index 2f072a40..00000000 --- a/sdkbase/openssl.cnf +++ /dev/null @@ -1,355 +0,0 @@ -# -# OpenSSL example configuration file. -# This is mostly being used for generation of certificate requests. -# - -# This definition stops the following lines choking if HOME isn't -# defined. -HOME = . -RANDFILE = $ENV::HOME/.rnd - -# Extra OBJECT IDENTIFIER info: -#oid_file = $ENV::HOME/.oid -oid_section = new_oids - -# To use this configuration file with the "-extfile" option of the -# "openssl x509" utility, name here the section containing the -# X.509v3 extensions to use: -# extensions = -# (Alternatively, use a configuration file that has only -# X.509v3 extensions in its main [= default] section.) - -[ new_oids ] - -# We can add new OIDs in here for use by 'ca', 'req' and 'ts'. -# Add a simple OID like this: -# testoid1=1.2.3.4 -# Or use config file substitution like this: -# testoid2=${testoid1}.5.6 - -# Policies used by the TSA examples. -tsa_policy1 = 1.2.3.4.1 -tsa_policy2 = 1.2.3.4.5.6 -tsa_policy3 = 1.2.3.4.5.7 - -#################################################################### -[ ca ] -default_ca = CA_default # The default ca section - -#################################################################### -[ CA_default ] - -dir = ./demoCA # Where everything is kept -certs = $dir/certs # Where the issued certs are kept -crl_dir = $dir/crl # Where the issued crl are kept -database = $dir/index.txt # database index file. -#unique_subject = no # Set to 'no' to allow creation of - # several ctificates with same subject. -new_certs_dir = $dir/newcerts # default place for new certs. - -certificate = $dir/cacert.pem # The CA certificate -serial = $dir/serial # The current serial number -crlnumber = $dir/crlnumber # the current crl number - # must be commented out to leave a V1 CRL -crl = $dir/crl.pem # The current CRL -private_key = $dir/private/cakey.pem# The private key -RANDFILE = $dir/private/.rand # private random number file - -x509_extensions = usr_cert # The extentions to add to the cert - -# Comment out the following two lines for the "traditional" -# (and highly broken) format. -name_opt = ca_default # Subject Name options -cert_opt = ca_default # Certificate field options - -# Extension copying option: use with caution. -# copy_extensions = copy - -# Extensions to add to a CRL. Note: Netscape communicator chokes on V2 CRLs -# so this is commented out by default to leave a V1 CRL. -# crlnumber must also be commented out to leave a V1 CRL. -# crl_extensions = crl_ext - -default_days = 365 # how long to certify for -default_crl_days= 30 # how long before next CRL -default_md = default # use public key default MD -preserve = no # keep passed DN ordering - -# A few difference way of specifying how similar the request should look -# For type CA, the listed attributes must be the same, and the optional -# and supplied fields are just that :-) -policy = policy_match - -# For the CA policy -[ policy_match ] -countryName = match -stateOrProvinceName = match -organizationName = match -organizationalUnitName = optional -commonName = supplied -emailAddress = optional - -# For the 'anything' policy -# At this point in time, you must list all acceptable 'object' -# types. -[ policy_anything ] -countryName = optional -stateOrProvinceName = optional -localityName = optional -organizationName = optional -organizationalUnitName = optional -commonName = supplied -emailAddress = optional - -#################################################################### -[ req ] -default_bits = 1024 -default_keyfile = privkey.pem -distinguished_name = req_distinguished_name -attributes = req_attributes -x509_extensions = v3_ca # The extentions to add to the self signed cert - -# Passwords for private keys if not present they will be prompted for -# input_password = secret -# output_password = secret - -# This sets a mask for permitted string types. There are several options. -# default: PrintableString, T61String, BMPString. -# pkix : PrintableString, BMPString (PKIX recommendation before 2004) -# utf8only: only UTF8Strings (PKIX recommendation after 2004). -# nombstr : PrintableString, T61String (no BMPStrings or UTF8Strings). -# MASK:XXXX a literal mask value. -# WARNING: ancient versions of Netscape crash on BMPStrings or UTF8Strings. -string_mask = utf8only - -req_extensions = v3_req # The extensions to add to a certificate request - -[ req_distinguished_name ] -countryName = Country Name (2 letter code) -countryName_default = US -countryName_min = 2 -countryName_max = 2 - -stateOrProvinceName = State or Province Name (full name) -stateOrProvinceName_default = California - -localityName = Locality Name (eg, city) - -0.organizationName = Organization Name (eg, company) -0.organizationName_default = KBase - -# we can do this but it is not needed normally :-) -#1.organizationName = Second Organization Name (eg, company) -#1.organizationName_default = World Wide Web Pty Ltd - -organizationalUnitName = Organizational Unit Name (eg, section) -#organizationalUnitName_default = - -commonName = Common Name (e.g. server FQDN or YOUR name) -commonName_max = 64 -commonName_default = ${ENV::PUBLIC_ADDRESS} - -emailAddress = Email Address -emailAddress_max = 64 - -# SET-ex3 = SET extension number 3 - -[ req_attributes ] -challengePassword = A challenge password -challengePassword_min = 4 -challengePassword_max = 20 - -unstructuredName = An optional company name - -[ usr_cert ] - -# These extensions are added when 'ca' signs a request. - -# This goes against PKIX guidelines but some CAs do it and some software -# requires this to avoid interpreting an end user certificate as a CA. - -basicConstraints=CA:FALSE - -# Here are some examples of the usage of nsCertType. If it is omitted -# the certificate can be used for anything *except* object signing. - -# This is OK for an SSL server. -# nsCertType = server - -# For an object signing certificate this would be used. -# nsCertType = objsign - -# For normal client use this is typical -# nsCertType = client, email - -# and for everything including object signing: -# nsCertType = client, email, objsign - -# This is typical in keyUsage for a client certificate. -# keyUsage = nonRepudiation, digitalSignature, keyEncipherment - -# This will be displayed in Netscape's comment listbox. -nsComment = "OpenSSL Generated Certificate" - -# PKIX recommendations harmless if included in all certificates. -subjectKeyIdentifier=hash -authorityKeyIdentifier=keyid,issuer - -# This stuff is for subjectAltName and issuerAltname. -# Import the email address. -# subjectAltName=email:copy -# An alternative to produce certificates that aren't -# deprecated according to PKIX. -# subjectAltName=email:move - -# Copy subject details -# issuerAltName=issuer:copy - -#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem -#nsBaseUrl -#nsRevocationUrl -#nsRenewalUrl -#nsCaPolicyUrl -#nsSslServerName - -# This is required for TSA certificates. -# extendedKeyUsage = critical,timeStamping - -[ v3_req ] - -# Extensions to add to a certificate request - -basicConstraints = CA:TRUE -keyUsage = digitalSignature, keyEncipherment -subjectAltName = @alt_names - -[alt_names] -DNS.1 = $ENV::PUBLIC_ADDRESS - -[ v3_ca ] - - -# Extensions for a typical CA - - -# PKIX recommendation. - -subjectKeyIdentifier=hash - -authorityKeyIdentifier=keyid:always,issuer - -# This is what PKIX recommends but some broken software chokes on critical -# extensions. -#basicConstraints = critical,CA:true -# So we do this instead. -basicConstraints = CA:true - -# Key usage: this is typical for a CA certificate. However since it will -# prevent it being used as an test self-signed certificate it is best -# left out by default. -# keyUsage = cRLSign, keyCertSign - -# Some might want this also -# nsCertType = sslCA, emailCA - -# Include email address in subject alt name: another PKIX recommendation -# subjectAltName=email:copy -# Copy issuer details -# issuerAltName=issuer:copy - -# DER hex encoding of an extension: beware experts only! -# obj=DER:02:03 -# Where 'obj' is a standard or added object -# You can even override a supported extension: -# basicConstraints= critical, DER:30:03:01:01:FF - -[ crl_ext ] - -# CRL extensions. -# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL. - -# issuerAltName=issuer:copy -authorityKeyIdentifier=keyid:always - -[ proxy_cert_ext ] -# These extensions should be added when creating a proxy certificate - -# This goes against PKIX guidelines but some CAs do it and some software -# requires this to avoid interpreting an end user certificate as a CA. - -basicConstraints=CA:FALSE - -# Here are some examples of the usage of nsCertType. If it is omitted -# the certificate can be used for anything *except* object signing. - -# This is OK for an SSL server. -# nsCertType = server - -# For an object signing certificate this would be used. -# nsCertType = objsign - -# For normal client use this is typical -# nsCertType = client, email - -# and for everything including object signing: -# nsCertType = client, email, objsign - -# This is typical in keyUsage for a client certificate. -# keyUsage = nonRepudiation, digitalSignature, keyEncipherment - -# This will be displayed in Netscape's comment listbox. -nsComment = "OpenSSL Generated Certificate" - -# PKIX recommendations harmless if included in all certificates. -subjectKeyIdentifier=hash -authorityKeyIdentifier=keyid,issuer - -# This stuff is for subjectAltName and issuerAltname. -# Import the email address. -# subjectAltName=email:copy -# An alternative to produce certificates that aren't -# deprecated according to PKIX. -# subjectAltName=email:move - -# Copy subject details -# issuerAltName=issuer:copy - -#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem -#nsBaseUrl -#nsRevocationUrl -#nsRenewalUrl -#nsCaPolicyUrl -#nsSslServerName - -# This really needs to be in place for it to be a proxy certificate. -proxyCertInfo=critical,language:id-ppl-anyLanguage,pathlen:3,policy:foo - -#################################################################### -[ tsa ] - -default_tsa = tsa_config1 # the default TSA section - -[ tsa_config1 ] - -# These are used by the TSA reply generation only. -dir = ./demoCA # TSA root directory -serial = $dir/tsaserial # The current serial number (mandatory) -crypto_device = builtin # OpenSSL engine to use for signing -signer_cert = $dir/tsacert.pem # The TSA signing certificate - # (optional) -certs = $dir/cacert.pem # Certificate chain to include in reply - # (optional) -signer_key = $dir/private/tsakey.pem # The TSA private key (optional) - -default_policy = tsa_policy1 # Policy if request did not specify it - # (optional) -other_policies = tsa_policy2, tsa_policy3 # acceptable policies (optional) -digests = md5, sha1 # Acceptable message digests (mandatory) -accuracy = secs:1, millisecs:500, microsecs:100 # (optional) -clock_precision_digits = 0 # number of digits after dot. (optional) -ordering = yes # Is ordering defined for timestamps? - # (optional, default: no) -tsa_name = yes # Must the TSA name be included in the reply? - # (optional, default: no) -ess_cert_id_chain = no # Must the ESS cert id chain be included? - # (optional, default: no) diff --git a/sdkbase/sdkbase.build.tag b/sdkbase/sdkbase.build.tag deleted file mode 100644 index c65fbcf7..00000000 --- a/sdkbase/sdkbase.build.tag +++ /dev/null @@ -1 +0,0 @@ -4/7/2017 diff --git a/src/java/us/kbase/templates/module_travis.vm.properties b/src/java/us/kbase/templates/module_travis.vm.properties index a7c1b256..e51dcae5 100644 --- a/src/java/us/kbase/templates/module_travis.vm.properties +++ b/src/java/us/kbase/templates/module_travis.vm.properties @@ -20,7 +20,6 @@ install: - git clone https://github.com/kbase/kb_sdk - cd kb_sdk - make bin - - make sdkbase - export PATH=$(pwd)/bin:$PATH - source src/sh/sdk-completion.sh - popd @@ -28,4 +27,4 @@ install: script: - kb-sdk validate -after_script: \ No newline at end of file +after_script: From 5be329c5a0e5b7fe7a3f5761c68fb30dfab85c1c Mon Sep 17 00:00:00 2001 From: Jason S Fillman <6155956+jsfillman@users.noreply.github.com> Date: Fri, 11 Sep 2020 14:41:09 -0700 Subject: [PATCH 18/47] jsf-nose-mod --- src/java/us/kbase/templates/module_makefile.vm.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/us/kbase/templates/module_makefile.vm.properties b/src/java/us/kbase/templates/module_makefile.vm.properties index 19e103ce..e0ddc179 100644 --- a/src/java/us/kbase/templates/module_makefile.vm.properties +++ b/src/java/us/kbase/templates/module_makefile.vm.properties @@ -124,7 +124,7 @@ build-test-script: #if($language == "python") echo 'export PYTHONPATH=$$script_dir/../$(LIB_DIR):$$PATH:$$PYTHONPATH' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) echo 'cd $$script_dir/../$(TEST_DIR)' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) - echo 'python -m nose --with-coverage --cover-package=$(SERVICE_CAPS) --cover-html --cover-html-dir=/kb/module/work/test_coverage --nocapture --nologcapture .' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) + echo 'python -m nose --with-coverage --cover-package=$(SERVICE_CAPS) --cover-html --cover-html-dir=/kb/module/work/test_coverage --cover-xml --cover-xml-file=/kb/module/work/test_coverage/coverage.xml --nocapture --nologcapture .' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) #end #if($language == "java") echo 'export JAVA_HOME=$(JAVA_HOME)' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) From b7b7d16966f4fe8b4f4c41932c496d75728866f6 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 10 Sep 2020 14:57:40 -0700 Subject: [PATCH 19/47] Adding kb-sdk image build and upload to Dockerhub to GitHub actions --- .github/workflows/kb_sdk-actions.yml | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/kb_sdk-actions.yml b/.github/workflows/kb_sdk-actions.yml index 86ac9bba..ddf67d9f 100644 --- a/.github/workflows/kb_sdk-actions.yml +++ b/.github/workflows/kb_sdk-actions.yml @@ -134,3 +134,33 @@ jobs: with: name: kbaseapp-${{ matrix.app }}-${{ matrix.os }} path: kbase_app + + dockerhub_upload: + runs-on: ubuntu-latest + if: | + (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master') && + github.event_name == 'push' && + !contains(github.event.head_commit.message, 'skip docker') && + !contains(github.event.head_commit.message, 'skip ci') + needs: + - test_existing_repos + - test_kb-sdk_builds + + steps: + - name: checkout git repo + uses: actions/checkout@v2 + + - name: set current branch name as TAG_NAME for docker image + shell: bash + run: echo "::set-env name=TAG_NAME::${GITHUB_REF#refs/heads/}" + + - name: build and push to dockerhub + uses: opspresso/action-docker@master + with: + args: --docker + env: + USERNAME: ${{ secrets.DOCKER_USERNAME }} + PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKERFILE: "Dockerfile" + IMAGE_NAME: "kbase/kb-sdk" + From 55a2d35f478b2d9f7f35ec439582e5e94419dbb4 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Mon, 21 Sep 2020 16:36:10 -0700 Subject: [PATCH 20/47] Remove return statement that stops kb-sdk test after adding the test.cfg file Remove extra step of adding in the test token to the test.cfg file in the github actions Improve error message text with information about where the kbase test token may be found Update release notes; bump version number. --- .github/workflows/kb_sdk-actions.yml | 14 +------------- RELEASE_NOTES.txt | 5 +++++ src/java/us/kbase/mobu/ModuleBuilder.java | 2 +- src/java/us/kbase/mobu/tester/ConfigLoader.java | 2 +- src/java/us/kbase/mobu/tester/ModuleTester.java | 5 ----- 5 files changed, 8 insertions(+), 20 deletions(-) diff --git a/.github/workflows/kb_sdk-actions.yml b/.github/workflows/kb_sdk-actions.yml index ddf67d9f..5ff19ec4 100644 --- a/.github/workflows/kb_sdk-actions.yml +++ b/.github/workflows/kb_sdk-actions.yml @@ -61,8 +61,6 @@ jobs: LANGUAGE_TOKEN: ${{ matrix.language }} run: | cd SampleApp - kb-sdk test || true - sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg kb-sdk test - name: make resulting app available as artefact in case of failure @@ -111,19 +109,9 @@ jobs: repository: kbaseapps/${{ matrix.app }} path: kbase_app - - name: prep for repo tests - run: | - cd kbase_app - kb-sdk test || true - - - name: edit test.cfg + - name: run repo tests env: KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} - run: | - cd kbase_app - sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg - - - name: run test run: | cd kbase_app kb-sdk test diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index d81971d0..3ba0bac8 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -6,6 +6,11 @@ platform. Modules include all code, specification files, and documentation needed to define and run a set of methods in the KBase Narrative interface. +VERSION: 1.2.4 (Released 09/2020) +--------------------------------- +CHANGES: +- `kb-sdk test` no longer dies on its first run; if the user has the KBASE_TEST_TOKEN environment variable set with their developer token, tests can be run without having to edit the test.cfg file. + VERSION: 1.2.3 (Released 06/2020) ------------------------------------ NEW FEATURES: diff --git a/src/java/us/kbase/mobu/ModuleBuilder.java b/src/java/us/kbase/mobu/ModuleBuilder.java index 94e8f386..52ba366c 100644 --- a/src/java/us/kbase/mobu/ModuleBuilder.java +++ b/src/java/us/kbase/mobu/ModuleBuilder.java @@ -49,7 +49,7 @@ public class ModuleBuilder { public static final String GLOBAL_SDK_HOME_ENV_VAR = "KB_SDK_HOME"; public static final String DEFAULT_METHOD_STORE_URL = "https://appdev.kbase.us/services/narrative_method_store/rpc"; - public static final String VERSION = "1.2.3"; + public static final String VERSION = "1.2.4"; public static void main(String[] args) throws Exception { diff --git a/src/java/us/kbase/mobu/tester/ConfigLoader.java b/src/java/us/kbase/mobu/tester/ConfigLoader.java index 5832da96..b9d6a71d 100644 --- a/src/java/us/kbase/mobu/tester/ConfigLoader.java +++ b/src/java/us/kbase/mobu/tester/ConfigLoader.java @@ -64,7 +64,7 @@ public ConfigLoader(Properties props, boolean testMode, if (user == null && tokenString == null) { throw new IllegalStateException("Error: KBase account credentials are not set in " + - configPathInfo); + configPathInfo + " or the KBASE_TEST_TOKEN environment variable"); } authAllowInsecure = props.getProperty("auth_service_url_allow_insecure"); ConfigurableAuthService auth = new ConfigurableAuthService( diff --git a/src/java/us/kbase/mobu/tester/ModuleTester.java b/src/java/us/kbase/mobu/tester/ModuleTester.java index 67662776..289a7693 100644 --- a/src/java/us/kbase/mobu/tester/ModuleTester.java +++ b/src/java/us/kbase/mobu/tester/ModuleTester.java @@ -127,9 +127,6 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow TemplateFormatter.formatTemplate("module_run_docker", moduleContext, runDockerSh); if (!testCfg.exists()) { TemplateFormatter.formatTemplate("module_test_cfg", moduleContext, testCfg); - System.out.println("Set KBase account credentials in test_local/test.cfg and then " + - "test again"); - return 1; } Properties props = new Properties(); InputStream is = new FileInputStream(testCfg); @@ -140,8 +137,6 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow } ConfigLoader cfgLoader = new ConfigLoader(props, true, "test_local/test.cfg", true); - - File workDir = new File(tlDir, "workdir"); workDir.mkdir(); File tokenFile = new File(workDir, "token"); From 03256514ffbf884e42dfce8ae875c1a1462fb7d4 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 22 Sep 2020 06:55:00 -0700 Subject: [PATCH 21/47] Test both env var and test.cfg-based auth --- .github/workflows/kb_sdk-actions.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/kb_sdk-actions.yml b/.github/workflows/kb_sdk-actions.yml index 5ff19ec4..3ddf2593 100644 --- a/.github/workflows/kb_sdk-actions.yml +++ b/.github/workflows/kb_sdk-actions.yml @@ -14,6 +14,7 @@ jobs: os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] language: [perl, python, java] test_type: [base, example] + auth: [token, env_var] steps: - name: Checkout @@ -55,10 +56,25 @@ jobs: run: | kb-sdk init -l $LANGUAGE_TOKEN -u user --example SampleApp - - name: test ${{ matrix.test_type }} repo + - name: test ${{ matrix.test_type }} repo, using test.cfg for auth + if: matrix.auth == 'token' + run: | + cd SampleApp + kb-sdk test || true + + - name: test ${{ matrix.test_type }} repo, using test.cfg for auth + if: matrix.auth == 'token' + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + run: | + cd SampleApp + sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg + kb-sdk test + + - name: test ${{ matrix.test_type }} repo, using env var for auth + if: matrix.auth == 'env_var' env: KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} - LANGUAGE_TOKEN: ${{ matrix.language }} run: | cd SampleApp kb-sdk test From 23401e521d3ab1afff41361afd55f86b21222628 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 23 Sep 2020 09:35:45 -0700 Subject: [PATCH 22/47] Use both auth methods for existing repo tests --- .github/workflows/kb_sdk-actions.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/kb_sdk-actions.yml b/.github/workflows/kb_sdk-actions.yml index 3ddf2593..346ea3ff 100644 --- a/.github/workflows/kb_sdk-actions.yml +++ b/.github/workflows/kb_sdk-actions.yml @@ -95,6 +95,7 @@ jobs: matrix: os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] app: [WsLargeDataIO, KBaseReport] + auth: [token, env_var] steps: - name: Checkout @@ -125,7 +126,23 @@ jobs: repository: kbaseapps/${{ matrix.app }} path: kbase_app - - name: run repo tests + - name: run repo tests, using test.cfg for auth + if: matrix.auth == 'token' + run: | + cd kbase_app + kb-sdk test || true + + - name: run repo tests, using test.cfg for auth + if: matrix.auth == 'token' + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + run: | + cd kbase_app + sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg + kb-sdk test + + - name: run repo tests, using env var for auth + if: matrix.auth == 'env_var' env: KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} run: | From 6d0d36f5f8d0996859857c0e6cb0d805198b948e Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 3 Jun 2020 10:59:37 -0700 Subject: [PATCH 23/47] Perl tidy -- mostly whitespace changes --- .gitignore | 1 + lib/Bio/KBase/DeploymentConfig.pm | 68 +- lib/Bio/KBase/Exceptions.pm | 74 +- lib/Bio/KBase/Log.pm | 409 ++++---- .../templates/module_perl_impl.vm.properties | 210 ++-- .../module_test_perl_client.vm.properties | 108 ++- .../kbase/templates/perl_client.vm.properties | 651 +++++++------ .../kbase/templates/perl_impl.vm.properties | 68 +- .../kbase/templates/perl_psgi.vm.properties | 18 +- .../kbase/templates/perl_server.vm.properties | 900 +++++++++--------- test_scripts/perl/lib/TestImpl.pm | 284 +++--- test_scripts/perl/run-client.pl | 118 ++- test_scripts/perl/test-client.pl | 280 +++--- 13 files changed, 1720 insertions(+), 1469 deletions(-) diff --git a/.gitignore b/.gitignore index 5705f26d..be10695a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ /test_scripts/test.cfg *.pyc env-*/ +submodules/auth diff --git a/lib/Bio/KBase/DeploymentConfig.pm b/lib/Bio/KBase/DeploymentConfig.pm index 4bb54573..8faa7a31 100644 --- a/lib/Bio/KBase/DeploymentConfig.pm +++ b/lib/Bio/KBase/DeploymentConfig.pm @@ -4,7 +4,7 @@ use strict; use base 'Class::Accessor'; use Config::Simple; -__PACKAGE__->mk_accessors(qw(service_name settings)); +__PACKAGE__->mk_accessors( qw( service_name settings ) ); =head1 NAME @@ -13,9 +13,9 @@ Bio::KBase::DeploymentConfig =head1 DESCRIPTION The C class wraps the access to a KBase deployment.cfg -file. It tests for the existence of the KB_DEPLOYMENT_CONFIG and -KB_SERVICE_NAME environment variables; if both are present, the -configuration parameters for the given service will be loaded +file. It tests for the existence of the KB_DEPLOYMENT_CONFIG and +KB_SERVICE_NAME environment variables; if both are present, the +configuration parameters for the given service will be loaded from that config file. If they are not present, the module supports fallback to defaults as defined by the module. @@ -46,42 +46,36 @@ A hash reference containing the default values for the service parameters. =back =cut - -sub new -{ - my($class, $service_name, $defaults) = @_; - - if ((my $n = $ENV{KB_SERVICE_NAME}) ne "") - { - $service_name = $n; + +sub new { + my ( $class, $service_name, $defaults ) = @_; + + if ( ( my $n = $ENV{ KB_SERVICE_NAME } ) ne "" ) { + $service_name = $n; } my $settings = {}; - if (ref($defaults)) - { - %$settings = %$defaults; + if ( ref( $defaults ) ) { + %$settings = %$defaults; } - my $cfg_file = $ENV{KB_DEPLOYMENT_CONFIG}; - if (-e $cfg_file) - { - my $cfg = Config::Simple->new(); - $cfg->read($cfg_file); - - my %cfg = $cfg->vars; - - for my $k (keys %cfg) - { - if ($k =~ /^$service_name\.(.*)/) - { - $settings->{$1} = $cfg{$k}; - } - } + my $cfg_file = $ENV{ KB_DEPLOYMENT_CONFIG }; + if ( -e $cfg_file ) { + my $cfg = Config::Simple->new(); + $cfg->read( $cfg_file ); + + my %config = $cfg->vars; + + for my $k ( keys %config ) { + if ( $k =~ /^$service_name\.(.*)/ ) { + $settings->{ $1 } = $config{ $k }; + } + } } my $self = { - settings => $settings, - service_name => $service_name, + settings => $settings, + service_name => $service_name, }; return bless $self, $class; @@ -91,14 +85,13 @@ sub new Retrieve a setting from the configuration. - my $value = $obj->setting("key-name"); + my $value = $obj->setting( "key-name" ); =cut -sub setting -{ - my($self, $key) = @_; - return $self->{settings}->{$key}; +sub setting { + my ( $self, $key ) = @_; + return $self->{ settings }->{ $key }; } =item C @@ -110,3 +103,4 @@ Return the name of the service currently configured. =cut 1; + diff --git a/lib/Bio/KBase/Exceptions.pm b/lib/Bio/KBase/Exceptions.pm index b7fdf876..d09411fe 100644 --- a/lib/Bio/KBase/Exceptions.pm +++ b/lib/Bio/KBase/Exceptions.pm @@ -1,50 +1,52 @@ package Bio::KBase::Exceptions; -use Exception::Class - ( - Bio::KBase::Exceptions::KBaseException => { - description => 'KBase exception', - fields => ['method_name'], - }, - - Bio::KBase::Exceptions::JSONRPC => { - description => 'JSONRPC error', - fields => ['code', 'data'], - isa => 'Bio::KBase::Exceptions::KBaseException', - }, - - Bio::KBase::Exceptions::HTTP => { - description => 'HTTP error', - fields => ['status_line'], - isa => 'Bio::KBase::Exceptions::KBaseException', - }, - - Bio::KBase::Exceptions::ArgumentValidationError => { - description => 'argument validation error', - fields => ['method_name'], - isa => 'Bio::KBase::Exceptions::KBaseException', - }, - - ); - -Bio::KBase::Exceptions::KBaseException->Trace(1); +use Exception::Class ( + Bio::KBase::Exceptions::KBaseException => { + description => 'KBase exception', + fields => [ 'method_name' ], + }, + + Bio::KBase::Exceptions::JSONRPC => { + description => 'JSONRPC error', + fields => [ 'code', 'data' ], + isa => 'Bio::KBase::Exceptions::KBaseException', + }, + + Bio::KBase::Exceptions::HTTP => { + description => 'HTTP error', + fields => [ 'status_line' ], + isa => 'Bio::KBase::Exceptions::KBaseException', + }, + + Bio::KBase::Exceptions::ArgumentValidationError => { + description => 'argument validation error', + fields => [ 'method_name' ], + isa => 'Bio::KBase::Exceptions::KBaseException', + }, +); + +Bio::KBase::Exceptions::KBaseException->Trace( 1 ); package Bio::KBase::Exceptions::HTTP; use strict; -sub full_message -{ - my($self) = @_; - return $self->message . "\nHTTP status: " . $self->status_line . "\nFunction invoked: " . $self->method_name; +sub full_message { + my ( $self ) = @_; + return + $self->message + . "\nHTTP status: " + . $self->status_line + . "\nFunction invoked: " + . $self->method_name; } package Bio::KBase::Exceptions::JSONRPC; use strict; -sub full_message -{ - my($self) = @_; - return sprintf("JSONRPC error:\n%s\nJSONRPC error code: %s\nJSONRPC error data:%s\n", $self->message, $self->code, $self->data); +sub full_message { + my ( $self ) = @_; + return sprintf( "JSONRPC error:\n%s\nJSONRPC error code: %s\nJSONRPC error data:%s\n", + $self->message, $self->code, $self->data ); } 1; diff --git a/lib/Bio/KBase/Log.pm b/lib/Bio/KBase/Log.pm index ede10352..14f20b81 100644 --- a/lib/Bio/KBase/Log.pm +++ b/lib/Bio/KBase/Log.pm @@ -14,51 +14,52 @@ use Time::HiRes; use Sys::Syslog qw( :DEFAULT setlogsock); # $ENV{'MLOG_CONFIG_FILE'} should point to an INI-formatted file, or an empty string, or should not exist. -our $MLOG_ENV_FILE = "MLOG_CONFIG_FILE"; -my $_GLOBAL = "global"; -our $MLOG_LOG_LEVEL = "mlog_log_level"; -our $MLOG_API_URL = "mlog_api_url"; -our $MLOG_LOG_FILE = "mlog_log_file"; - -our $DEFAULT_LOG_LEVEL = 6; -our $MSG_FACILITY = 'local1'; -our $EMERG_FACILITY = 'local0'; - -our $EMERG = 0; -our $ALERT = 1; -our $CRIT = 2; -our $ERR = 3; +our $MLOG_ENV_FILE = "MLOG_CONFIG_FILE"; +my $_GLOBAL = "global"; +our $MLOG_LOG_LEVEL = "mlog_log_level"; +our $MLOG_API_URL = "mlog_api_url"; +our $MLOG_LOG_FILE = "mlog_log_file"; + +our $DEFAULT_LOG_LEVEL = 6; +our $MSG_FACILITY = 'local1'; +our $EMERG_FACILITY = 'local0'; + +our $EMERG = 0; +our $ALERT = 1; +our $CRIT = 2; +our $ERR = 3; our $WARNING = 4; -our $NOTICE = 5; -our $INFO = 6; -our $DEBUG = 7; -our $DEBUG2 = 8; -our $DEBUG3 = 9; - -my $_MLOG_TEXT_TO_LEVEL = { 'EMERG' => $EMERG, - 'ALERT' => $ALERT, - 'CRIT' => $CRIT, - 'ERR' => $ERR, - 'WARNING' => $WARNING, - 'NOTICE' => $NOTICE, - 'INFO' => $INFO, - 'DEBUG' => $DEBUG, - 'DEBUG2' => $DEBUG2, - 'DEBUG3' => $DEBUG3, - }; - -my @_MLOG_TO_SYSLOG = (0, 1, 2, 3, 4, 5, 6, 7, 7, 7); +our $NOTICE = 5; +our $INFO = 6; +our $DEBUG = 7; +our $DEBUG2 = 8; +our $DEBUG3 = 9; + +my $_MLOG_TEXT_TO_LEVEL = { + 'EMERG' => $EMERG, + 'ALERT' => $ALERT, + 'CRIT' => $CRIT, + 'ERR' => $ERR, + 'WARNING' => $WARNING, + 'NOTICE' => $NOTICE, + 'INFO' => $INFO, + 'DEBUG' => $DEBUG, + 'DEBUG2' => $DEBUG2, + 'DEBUG3' => $DEBUG3, +}; + +my @_MLOG_TO_SYSLOG = ( 0, 1, 2, 3, 4, 5, 6, 7, 7, 7 ); my $_MLOG_LEVEL_TO_TEXT = {}; -foreach my $k (keys %{$_MLOG_TEXT_TO_LEVEL}) { - $_MLOG_LEVEL_TO_TEXT->{$_MLOG_TEXT_TO_LEVEL->{$k}} = $k; +foreach my $k ( keys %{ $_MLOG_TEXT_TO_LEVEL } ) { + $_MLOG_LEVEL_TO_TEXT->{ $_MLOG_TEXT_TO_LEVEL->{ $k } } = $k; } -our $LOG_LEVEL_MIN = min(keys %{$_MLOG_LEVEL_TO_TEXT}); -our $LOG_LEVEL_MAX = max(keys %{$_MLOG_LEVEL_TO_TEXT}); +our $LOG_LEVEL_MIN = min( keys %{ $_MLOG_LEVEL_TO_TEXT } ); +our $LOG_LEVEL_MAX = max( keys %{ $_MLOG_LEVEL_TO_TEXT } ); -our $USER = $ENV{'USER'}; -our $PARENT_FILE = abs_path($0); +our $USER = $ENV{ 'USER' }; +our $PARENT_FILE = abs_path( $0 ); =pod @@ -174,275 +175,303 @@ clear_user_log_level() : Removes the user-defined log level. =cut sub _get_option { - my ($opts, $key) = @_; - if(!(defined $opts)) { + my ( $opts, $key ) = @_; + if ( !( defined $opts ) ) { return undef; } - return defined $opts->{$key} ? $opts->{$key} : undef; + return defined $opts->{ $key } ? $opts->{ $key } : undef; } sub new { - my ($class, $sub, $lc, $options) = @_; - unless(defined $sub) { + my ( $class, $sub, $lc, $options ) = @_; + unless ( defined $sub ) { die "ERROR: You must define a subsystem when calling init_log()\n"; } my $self = {}; bless $self, $class; - $self->{authuser} = _get_option($options, 'authuser'); - $self->{module} = _get_option($options, 'module'); - $self->{method} = _get_option($options, 'method'); - $self->{call_id} = _get_option($options, 'call_id'); - $self->{ip_address} = _get_option($options, 'ip_address'); - $self->{tag} = _get_option($options, 'tag'); - $self->{_callback} = defined $options->{"changecallback"} ? - $options->{"changecallback"} : sub {}; - $self->{_subsystem} = $sub; - $self->{_mlog_config_file} = _get_option($options, 'config'); - if(!(defined $self->{_mlog_config_file})) { - $self->{_mlog_config_file} = defined $ENV{$MLOG_ENV_FILE} ? - $ENV{$MLOG_ENV_FILE} : undef; + $self->{ authuser } = _get_option( $options, 'authuser' ); + $self->{ module } = _get_option( $options, 'module' ); + $self->{ method } = _get_option( $options, 'method' ); + $self->{ call_id } = _get_option( $options, 'call_id' ); + $self->{ ip_address } = _get_option( $options, 'ip_address' ); + $self->{ tag } = _get_option( $options, 'tag' ); + $self->{ _callback } + = defined $options->{ "changecallback" } + ? $options->{ "changecallback" } + : sub { }; + $self->{ _subsystem } = $sub; + $self->{ _mlog_config_file } = _get_option( $options, 'config' ); + + if ( !( defined $self->{ _mlog_config_file } ) ) { + $self->{ _mlog_config_file } + = defined $ENV{ $MLOG_ENV_FILE } ? $ENV{ $MLOG_ENV_FILE } : undef; } - $self->{_user_log_level} = -1; - $self->{_config_log_level} = -1; - $self->{_user_log_file} = _get_option($options, 'logfile'); - $self->{_config_log_file} = undef; - $self->{_api_log_level} = -1; - $self->{_msgs_since_config_update} = 0; - $self->{_time_at_config_update} = ""; - $self->{msg_count} = 0; - $self->{_recheck_api_msg} = 100; - $self->{_recheck_api_time} = 300; # 5 mins - if(defined $lc) { - $self->{_log_constraints} = $lc; + $self->{ _user_log_level } = -1; + $self->{ _config_log_level } = -1; + $self->{ _user_log_file } = _get_option( $options, 'logfile' ); + $self->{ _config_log_file } = undef; + $self->{ _api_log_level } = -1; + $self->{ _msgs_since_config_update } = 0; + $self->{ _time_at_config_update } = ""; + $self->{ msg_count } = 0; + $self->{ _recheck_api_msg } = 100; + $self->{ _recheck_api_time } = 300; # 5 mins + + if ( defined $lc ) { + $self->{ _log_constraints } = $lc; } - $self->{_init} = 1; + $self->{ _init } = 1; $self->update_config(); - $self->{_init} = undef; + $self->{ _init } = undef; return $self; } sub _get_time_since_start { - my ($self) = @_; - my $now = time; - my $seconds_duration = $now - $self->{_time_at_config_update}; + my ( $self ) = @_; + my $now = time; + my $seconds_duration = $now - $self->{ _time_at_config_update }; return $seconds_duration; } sub get_log_level { - my ($self) = @_; - if($self->{_user_log_level} != -1) { - return $self->{_user_log_level}; - } elsif($self->{_config_log_level} != -1) { - return $self->{_config_log_level}; - } elsif($self->{_api_log_level} != -1) { - return $self->{_api_log_level}; - } else { + my ( $self ) = @_; + if ( $self->{ _user_log_level } != -1 ) { + return $self->{ _user_log_level }; + } + elsif ( $self->{ _config_log_level } != -1 ) { + return $self->{ _config_log_level }; + } + elsif ( $self->{ _api_log_level } != -1 ) { + return $self->{ _api_log_level }; + } + else { return $DEFAULT_LOG_LEVEL; } } sub update_config { - my ($self) = @_; + my ( $self ) = @_; my $loglevel = $self->get_log_level(); - my $logfile = $self->get_log_file(); - - $self->{_api_log_level} = -1; - $self->{_msgs_since_config_update} = 0; - $self->{_time_at_config_update} = time; - + my $logfile = $self->get_log_file(); + + $self->{ _api_log_level } = -1; + $self->{ _msgs_since_config_update } = 0; + $self->{ _time_at_config_update } = time; + # Retrieving config variables. my $api_url = ""; - if(defined $self->{_mlog_config_file} && -e $self->{_mlog_config_file} && - -s $self->{_mlog_config_file} > 0) { - my $cfg = new Config::Simple($self->{_mlog_config_file}); - my $cfgitems = $cfg->get_block($_GLOBAL); - my $subitems = $cfg->get_block($self->{_subsystem}); - foreach my $k (keys %{$subitems}) { - $cfgitems->{$k} = $subitems->{$k}; + if ( defined $self->{ _mlog_config_file } + && -e $self->{ _mlog_config_file } + && -s $self->{ _mlog_config_file } > 0 ) { + my $cfg = new Config::Simple( $self->{ _mlog_config_file } ); + my $cfgitems = $cfg->get_block( $_GLOBAL ); + my $subitems = $cfg->get_block( $self->{ _subsystem } ); + foreach my $k ( keys %{ $subitems } ) { + $cfgitems->{ $k } = $subitems->{ $k }; } - if(defined $cfgitems->{$MLOG_LOG_LEVEL}) { - if($cfgitems->{$MLOG_LOG_LEVEL} !~ /^\d+$/) { - warn "Cannot parse log level " . $cfgitems->{$MLOG_LOG_LEVEL} . - " from file " . $self->{_mlog_config_file} . - " to int. Keeping current log level."; - } else { - $self->{_config_log_level} = $cfgitems->{$MLOG_LOG_LEVEL}; + if ( defined $cfgitems->{ $MLOG_LOG_LEVEL } ) { + if ( $cfgitems->{ $MLOG_LOG_LEVEL } !~ /^\d+$/ ) { + warn "Cannot parse log level " + . $cfgitems->{ $MLOG_LOG_LEVEL } + . " from file " + . $self->{ _mlog_config_file } + . " to int. Keeping current log level."; + } + else { + $self->{ _config_log_level } = $cfgitems->{ $MLOG_LOG_LEVEL }; } } - if(defined $cfgitems->{$MLOG_API_URL}) { - $api_url = $cfgitems->{$MLOG_API_URL}; + if ( defined $cfgitems->{ $MLOG_API_URL } ) { + $api_url = $cfgitems->{ $MLOG_API_URL }; } - if(defined $cfgitems->{$MLOG_LOG_FILE}) { - $self->{_config_log_file} = $cfgitems->{$MLOG_LOG_FILE} + if ( defined $cfgitems->{ $MLOG_LOG_FILE } ) { + $self->{ _config_log_file } = $cfgitems->{ $MLOG_LOG_FILE }; } - } elsif (defined $self->{_mlog_config_file}) { - warn "Cannot read config file " . $self->{_mlog_config_file}; + } + elsif ( defined $self->{ _mlog_config_file } ) { + warn "Cannot read config file " . $self->{ _mlog_config_file }; } - unless($api_url eq "") { - my $subsystem_api_url = $api_url."/".$self->{_subsystem}; - my $json = get($subsystem_api_url); - if(defined $json) { - my $decoded_json = decode_json($json); + unless ( $api_url eq "" ) { + my $subsystem_api_url = $api_url . "/" . $self->{ _subsystem }; + my $json = get( $subsystem_api_url ); + if ( defined $json ) { + my $decoded_json = decode_json( $json ); my $max_matching_level = -1; - foreach my $constraint_set (@{$decoded_json->{'log_levels'}}) { - my $level = $constraint_set->{'level'}; - my $constraints = $constraint_set->{'constraints'}; - if($level <= $max_matching_level) { + foreach my $constraint_set ( @{ $decoded_json->{ 'log_levels' } } ) { + my $level = $constraint_set->{ 'level' }; + my $constraints = $constraint_set->{ 'constraints' }; + if ( $level <= $max_matching_level ) { next; } my $matches = 1; - foreach my $constraint (keys %{$constraints}) { - if(! exists $self->{_log_constraints}->{$constraint}) { + foreach my $constraint ( keys %{ $constraints } ) { + if ( !exists $self->{ _log_constraints }->{ $constraint } ) { $matches = 0; - } elsif($self->{_log_constraints}->{$constraint} ne $constraints->{$constraint}) { + } + elsif ( $self->{ _log_constraints }->{ $constraint } ne + $constraints->{ $constraint } ) + { $matches = 0; } } - if($matches == 1) { + if ( $matches == 1 ) { $max_matching_level = $level; } } - $self->{_api_log_level} = $max_matching_level; - } else { - warn "Could not retrieve Log subsystem from control API at: $subsystem_api_url"; + $self->{ _api_log_level } = $max_matching_level; + } + else { + warn + "Could not retrieve Log subsystem from control API at: $subsystem_api_url"; } } } sub _resolve_log_level { - my ($self, $level) = @_; - if(defined $_MLOG_TEXT_TO_LEVEL->{$level}) { - $level = $_MLOG_TEXT_TO_LEVEL->{$level}; - } elsif (!(defined $_MLOG_LEVEL_TO_TEXT->{$level})) { + my ( $self, $level ) = @_; + if ( defined $_MLOG_TEXT_TO_LEVEL->{ $level } ) { + $level = $_MLOG_TEXT_TO_LEVEL->{ $level }; + } + elsif ( !( defined $_MLOG_LEVEL_TO_TEXT->{ $level } ) ) { die "Illegal log level"; } return $level; } sub set_log_level { - my ($self, $level) = @_; - $self->{_user_log_level} = $self->_resolve_log_level($level); - $self->{_callback}(); + my ( $self, $level ) = @_; + $self->{ _user_log_level } = $self->_resolve_log_level( $level ); + $self->{ _callback }(); } sub get_log_file { - my ($self) = @_; - if ($self->{_user_log_file}) { - return $self->{_user_log_file}; + my ( $self ) = @_; + if ( $self->{ _user_log_file } ) { + return $self->{ _user_log_file }; } - if ($self->{_config_log_file}) { - return $self->{_config_log_file}; + if ( $self->{ _config_log_file } ) { + return $self->{ _config_log_file }; } return undef; } sub set_log_file { - my ($self, $filename) = @_; - $self->{_user_log_file} = $filename; - $self->{_callback}(); + my ( $self, $filename ) = @_; + $self->{ _user_log_file } = $filename; + $self->{ _callback }(); } sub set_log_msg_check_count { - my ($self, $count) = @_; - if($count !~ /^\d+$/ || $count < 0) { - die "Format for calling set_log_msg_check_count is set_log_msg_check_count(integer count)\n"; + my ( $self, $count ) = @_; + if ( $count !~ /^\d+$/ || $count < 0 ) { + die + "Format for calling set_log_msg_check_count is set_log_msg_check_count(integer count)\n"; } - $self->{_recheck_api_msg} = $count; + $self->{ _recheck_api_msg } = $count; } sub set_log_msg_check_interval { - my ($self, $interval) = @_; - if($interval !~ /^\d+$/ || $interval < 0) { - die "Format for calling set_log_msg_check_interval is set_log_msg_check_interval(integer seconds)\n"; + my ( $self, $interval ) = @_; + if ( $interval !~ /^\d+$/ || $interval < 0 ) { + die + "Format for calling set_log_msg_check_interval is set_log_msg_check_interval(integer seconds)\n"; } - $self->{_recheck_api_time} = $interval; + $self->{ _recheck_api_time } = $interval; } sub clear_user_log_level { - my ($self) = @_; - $self->{_user_log_level} = -1; + my ( $self ) = @_; + $self->{ _user_log_level } = -1; } sub _get_ident { - my ($self, $level, $authuser, $module, $method, $call_id, $ip_address, $tag) = @_; - my @infos = ($self->{_subsystem}, $_MLOG_LEVEL_TO_TEXT->{$level}, - Time::HiRes::time(), $USER, $PARENT_FILE, $$); - if ($self->{ip_address}) { + my ( $self, $level, $authuser, $module, $method, $call_id, $ip_address, $tag ) = @_; + my @infos = ( + $self->{ _subsystem }, + $_MLOG_LEVEL_TO_TEXT->{ $level }, + Time::HiRes::time(), $USER, $PARENT_FILE, $$ + ); + if ( $self->{ ip_address } ) { push @infos, $ip_address || '-'; } - if ($self->{authuser}) { + if ( $self->{ authuser } ) { push @infos, $authuser || '-'; } - if ($self->{module}) { + if ( $self->{ module } ) { push @infos, $module || '-'; } - if ($self->{method}) { + if ( $self->{ method } ) { push @infos, $method || '-'; } - if ($self->{call_id}) { + if ( $self->{ call_id } ) { push @infos, $call_id || '-'; } - if ($self->{tag}) { + if ( $self->{ tag } ) { push @infos, $tag || '-'; } - return "[" . join("] [", @infos). "]"; + return "[" . join( "] [", @infos ) . "]"; } sub _syslog { - my ($self, $facility, $level, $ident, $message) = @_; - openlog($ident, "", $facility); - if(ref($message) eq 'ARRAY') { - foreach my $m (@{$message}) { - syslog($_MLOG_TO_SYSLOG[$level], "$m"); + my ( $self, $facility, $level, $ident, $message ) = @_; + openlog( $ident, "", $facility ); + if ( ref( $message ) eq 'ARRAY' ) { + foreach my $m ( @{ $message } ) { + syslog( $_MLOG_TO_SYSLOG[ $level ], "$m" ); } - } else { - syslog($_MLOG_TO_SYSLOG[$level], "$message"); + } + else { + syslog( $_MLOG_TO_SYSLOG[ $level ], "$message" ); } closelog(); } sub _log { - my ($self, $ident, $message) = @_; - my $time = POSIX::strftime("%Y-%m-%d %H:%M:%S", localtime); - my $msg = join(" ", $time, hostname(), $ident . ": "); - open LOG, ">>" . $self->get_log_file() || - warn "Could not print log message $msg to " . $self->get_log_file() . "\n"; - if(ref($message) eq 'ARRAY') { - foreach my $m (@{$message}) { + my ( $self, $ident, $message ) = @_; + my $time = POSIX::strftime( "%Y-%m-%d %H:%M:%S", localtime ); + my $msg = join( " ", $time, hostname(), $ident . ": " ); + open LOG, ">>" . $self->get_log_file() + || warn "Could not print log message $msg to " . $self->get_log_file() . "\n"; + if ( ref( $message ) eq 'ARRAY' ) { + foreach my $m ( @{ $message } ) { print LOG $msg . "$m\n"; } - } else { + } + else { print LOG $msg . "$message\n"; } close LOG; } sub log_message { - my ($self, $level, $message, $authuser, $module, $method, $call_id, - $ip_address, $tag) = @_; - $level = $self->_resolve_log_level($level); + my ( + $self, $level, $message, $authuser, $module, + $method, $call_id, $ip_address, $tag + ) = @_; + $level = $self->_resolve_log_level( $level ); - ++$self->{msg_count}; - ++$self->{_msgs_since_config_update}; + ++$self->{ msg_count }; + ++$self->{ _msgs_since_config_update }; - if($self->{_msgs_since_config_update} >= $self->{_recheck_api_msg} || - $self->_get_time_since_start() >= $self->{_recheck_api_time}) { + if ( $self->{ _msgs_since_config_update } >= $self->{ _recheck_api_msg } + || $self->_get_time_since_start() >= $self->{ _recheck_api_time } ) { $self->update_config(); } - my $ident = $self->_get_ident($level, $authuser, $module, $method, $call_id, - $ip_address, $tag); + my $ident = $self->_get_ident( $level, $authuser, $module, $method, $call_id, + $ip_address, $tag ); + # If this message is an emergency, send a copy to the emergency facility first. - if($level == 0) { - $self->_syslog($EMERG_FACILITY, $level, $ident, $message); + if ( $level == 0 ) { + $self->_syslog( $EMERG_FACILITY, $level, $ident, $message ); } - if($level <= $self->get_log_level()) { - $self->_syslog($MSG_FACILITY, $level, $ident, $message); - if($self->get_log_file()) { - $self->_log($ident, $message); + if ( $level <= $self->get_log_level() ) { + $self->_syslog( $MSG_FACILITY, $level, $ident, $message ); + if ( $self->get_log_file() ) { + $self->_log( $ident, $message ); } } } diff --git a/src/java/us/kbase/templates/module_perl_impl.vm.properties b/src/java/us/kbase/templates/module_perl_impl.vm.properties index ffbf4e07..09a06d81 100644 --- a/src/java/us/kbase/templates/module_perl_impl.vm.properties +++ b/src/java/us/kbase/templates/module_perl_impl.vm.properties @@ -3,7 +3,7 @@ package ${module_name}::${module_name}Impl; use strict; use Bio::KBase::Exceptions; # Use Semantic Versioning (2.0.0-rc.1) -# http://semver.org +# http://semver.org our $VERSION = '0.0.1'; our $GIT_URL = ''; our $GIT_COMMIT_HASH = ''; @@ -28,22 +28,20 @@ use Bio::SeqIO; use Data::Dumper; #END_HEADER -sub new -{ - my($class, @args) = @_; - my $self = { - }; +sub new { + my ( $class, @args ) = @_; + my $self = {}; bless $self, $class; #BEGIN_CONSTRUCTOR - - my $config_file = $ENV{ KB_DEPLOYMENT_CONFIG }; - my $cfg = Config::IniFiles->new(-file=>$config_file); - my $scratch = $cfg->val('${module_name}', 'scratch'); - my $callbackURL = $ENV{ SDK_CALLBACK_URL }; - - $self->{scratch} = $scratch; - $self->{callbackURL} = $callbackURL; - + + my $config_file = $ENV{ KB_DEPLOYMENT_CONFIG }; + my $cfg = Config::IniFiles->new(-file=>$config_file); + my $scratch = $cfg->val('${module_name}', 'scratch'); + my $callbackURL = $ENV{ SDK_CALLBACK_URL }; + + $self->{scratch} = $scratch; + $self->{callbackURL} = $callbackURL; + #END_CONSTRUCTOR if ($self->can('_init_instance')) @@ -65,126 +63,139 @@ sub new The actual function is declared using 'funcdef' to specify the name and input/return arguments to the function. For all typical KBase -Apps that run in the Narrative, your function should have the +Apps that run in the Narrative, your function should have the 'authentication required' modifier. =back =cut -sub run_${module_name} -{ +sub run_${module_name} { my $self = shift; - my($params) = @_; + my ( $params ) = @_; my @_bad_arguments; - (ref($params) eq 'HASH') or push(@_bad_arguments, "Invalid type for argument \"params\" (value was \"$params\")"); + ( ref($params) eq 'HASH' ) or push( @_bad_arguments, "Invalid type for argument \"params\" (value was \"$params\")" ); if (@_bad_arguments) { - my $msg = "Invalid arguments passed to run_${module_name}:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'run_${module_name}'); + my $msg = "Invalid arguments passed to run_${module_name}:\n" + . join( "", map { "\t$_\n" } @_bad_arguments ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'run_${module_name}' + ); } my $ctx = $${module_name}::${module_name}Server::CallContext; - my($output); + my $output; #BEGIN run_${module_name} - + # Print statements to stdout/stderr are captured and available as the App log print("Starting run_${module_name} method. Parameters:\n"); print(Dumper($params) . "\n"); - + # Step 1 - Parse/examine the parameters and catch any errors # It is important to check that parameters exist and are defined, and that nice error # messages are returned to users. Parameter values go through basic validation when # defined in a Narrative App, but advanced users or other SDK developers can call # this function directly, so validation is still important. - - if (!exists $params->{'workspace_name'}) { + + if ( !exists $params->{ 'workspace_name' } ) { die "Parameter workspace_name is not set in input arguments"; } - my $workspace_name=$params->{'workspace_name'}; - - if (!exists $params->{'assembly_input_ref'}) { + my $workspace_name = $params->{ 'workspace_name' }; + + if ( !exists $params->{ 'assembly_input_ref' } ) { die "Parameter assembly_input_ref is not set in input arguments"; } - my $assy_ref=$params->{'assembly_input_ref'}; - - if (!exists $params->{'min_length'}) { + my $assy_ref = $params->{ 'assembly_input_ref' }; + + if ( !exists $params->{ 'min_length' } ) { die "Parameter min_length is not set in input arguments"; } - my $min_length = $params->{'min_length'}; - if ($min_length < 0) { - die "min_length parameter cannot be negative (".$min_length.")"; + my $min_length = $params->{ 'min_length' }; + if ( $min_length < 0 ) { + die "min_length parameter cannot be negative (" . $min_length . ")"; } - + # Step 2 - Download the input data as a Fasta file # We can use the AssemblyUtils module to download a FASTA file from our Assembly data # object. The return object gives us the path to the file that was created. - + print("Downloading assembly data as FASTA file.\n"); - my $assycli = installed_clients::AssemblyUtilClient->new($self->{callbackURL}); - my $fileobj = $assycli->get_assembly_as_fasta({ref => $assy_ref}); - + my $assycli = installed_clients::AssemblyUtilClient->new( $self->{ callbackURL } ); + my $fileobj = $assycli->get_assembly_as_fasta( { ref => $assy_ref } ); + # Step 3 - Actually perform the filter operation, saving the good contigs to a new # fasta file. - - my $sio_in = Bio::SeqIO->new(-file => $fileobj->{path}); - my $outfile = $self->{scratch} . "/" . "filtered.fasta"; - my $sio_out = Bio::SeqIO->new(-file => ">$outfile", -format=> "fasta"); + + my $sio_in = Bio::SeqIO->new( -file => $fileobj->{ path } ); + my $outfile = $self->{ scratch } . "/" . "filtered.fasta"; + my $sio_out = Bio::SeqIO->new( -file => ">$outfile", -format => "fasta" ); my $total = 0; my $remaining = 0; - while (my $seq = $sio_in->next_seq) { + while ( my $seq = $sio_in->next_seq ) { $total++; - if ($seq->length >= $min_length) { + if ( $seq->length >= $min_length ) { $remaining++; - $sio_out->write_seq($seq); + $sio_out->write_seq( $seq ); } } my $result_text = "Filtered assembly to " . $remaining . " contigs out of " . $total; print($result_text . "\n"); - + # Step 4 - Save the new Assembly back to the system - my $newref = $assycli->save_assembly_from_fasta({assembly_name => $fileobj->{assembly_name}, - workspace_name => $workspace_name, - file => {path => $outfile}}); + my $newref = $assycli->save_assembly_from_fasta( { + assembly_name => $fileobj->{ assembly_name }, + workspace_name => $workspace_name, + file => { + path => $outfile + } + } ); + # Step 5 - Build a report and return - my $repcli = installed_clients::KBaseReportClient->new($self->{callbackURL}); - my $report = $repcli->create( - {workspace_name => $workspace_name, - report => {text_message => $result_text, - objects_created => [{description => "Filtered contigs", - ref => $newref} - ] - } - }); - + my $repcli = installed_clients::KBaseReportClient->new( $self->{ callbackURL } ); + my $report = $repcli->create( { + workspace_name => $workspace_name, + report => { + text_message => $result_text, + objects_created => [ { + description => "Filtered contigs", + ref => $newref + } ] + } + } ); + # Step 6 - construct the output to send back - - my $output = {assembly_output => $newref, - n_initial_contigs => $total, - n_contigs_remaining => $remaining, - n_contigs_removed => $total - $remaining, - report_name => $report->{name}, - report_ref => $report->{ref}}; + + my $output = { + assembly_output => $newref, + n_initial_contigs => $total, + n_contigs_remaining => $remaining, + n_contigs_removed => $total - $remaining, + report_name => $report->{ name }, + report_ref => $report->{ ref } + }; print("returning: ".Dumper($output)."\n"); - + #END run_${module_name} my @_bad_returns; - (ref($output) eq 'HASH') or push(@_bad_returns, "Invalid type for return variable \"output\" (value was \"$output\")"); - if (@_bad_returns) { - my $msg = "Invalid returns passed to run_${module_name}:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'run_${module_name}'); + ( ref($output) eq 'HASH' ) or push( @_bad_returns, "Invalid type for return variable \"output\" (value was \"$output\")" ); + if ( @_bad_returns ) { + my $msg = "Invalid returns passed to run_${module_name}:\n" + . join( "", map { "\t$_\n" } @_bad_returns ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'run_${module_name}' + ); } - return($output); + return( $output ); } - -=head2 status +=head2 status $return = $obj->status() @@ -215,12 +226,17 @@ Return the module status. This is a structure including Semantic Versioning numb =cut sub status { - my($return); + my $return; #BEGIN_STATUS - $return = {"state" => "OK", "message" => "", "version" => $VERSION, - "git_url" => $GIT_URL, "git_commit_hash" => $GIT_COMMIT_HASH}; + $return = { + "state" => "OK", + "message" => "", + "version" => $VERSION, + "git_url" => $GIT_URL, + "git_commit_hash" => $GIT_COMMIT_HASH + }; #END_STATUS - return($return); + return ( $return ); } 1; @@ -232,30 +248,36 @@ use Config::IniFiles; #END_HEADER #BEGIN_CONSTRUCTOR my $config_file = $ENV{ KB_DEPLOYMENT_CONFIG }; - my $cfg = Config::IniFiles->new(-file=>$config_file); - my $scratch = $cfg->val('${module_name}', 'scratch'); + my $cfg = Config::IniFiles->new( -file => $config_file ); + my $scratch = $cfg->val( '${module_name}', 'scratch' ); my $callbackURL = $ENV{ SDK_CALLBACK_URL }; - $self->{scratch} = $scratch; - $self->{callbackURL} = $callbackURL; + $self->{ scratch } = $scratch; + $self->{ callbackURL } = $callbackURL; #END_CONSTRUCTOR #BEGIN run_${module_name} - my $repcli = installed_clients::KBaseReportClient->new($self->{callbackURL}); + my $repcli = installed_clients::KBaseReportClient->new( $self->{ callbackURL } ); my $report = $repcli->create({ - workspace_name => $params->{workspace_name}, + workspace_name => $params->{ workspace_name }, report => { - text_message => $params->{parameter_1}, + text_message => $params->{parameter_1}, objects_created => [] } }); my $output = { - report_name => $report->{name}, - report_ref => $report->{ref} + report_name => $report->{ name }, + report_ref => $report->{ ref } }; #END run_${module_name} #BEGIN_STATUS - $return = {"state" => "OK", "message" => "", "version" => $VERSION, "git_url" => $GIT_URL, "git_commit_hash" => $GIT_COMMIT_HASH}; + $return = { + "state" => "OK", + "message" => "", + "version" => $VERSION, + "git_url" => $GIT_URL, + "git_commit_hash" => $GIT_COMMIT_HASH + }; #END_STATUS -#end \ No newline at end of file +#end diff --git a/src/java/us/kbase/templates/module_test_perl_client.vm.properties b/src/java/us/kbase/templates/module_test_perl_client.vm.properties index b4369ea1..9a3ca283 100644 --- a/src/java/us/kbase/templates/module_test_perl_client.vm.properties +++ b/src/java/us/kbase/templates/module_test_perl_client.vm.properties @@ -11,24 +11,33 @@ use installed_clients::AssemblyUtilClient; use ${module_name}::${module_name}Impl; local $| = 1; -my $token = $ENV{'KB_AUTH_TOKEN'}; -my $config_file = $ENV{'KB_DEPLOYMENT_CONFIG'}; -my $config = new Config::Simple($config_file)->get_block('${module_name}'); -my $ws_url = $config->{"workspace-url"}; -my $ws_name = undef; +my $token = $ENV{ 'KB_AUTH_TOKEN' }; +my $config_file = $ENV{ 'KB_DEPLOYMENT_CONFIG' }; +my $callback_url = $ENV{ 'SDK_CALLBACK_URL' }; + +my $config = new Config::Simple( $config_file )->get_block( '${module_name}' ); +my $ws_url = $config->{ "workspace-url" }; +my $ws_name = undef; my $ws_client = new installed_clients::WorkspaceClient($ws_url,token => $token); -my $scratch = $config->{scratch}; -my $callback_url = $ENV{'SDK_CALLBACK_URL'}; -my $auth_token = Bio::KBase::AuthToken->new(token => $token, ignore_authrc => 1, auth_svc=>$config->{'auth-service-url'}); -my $ctx = LocalCallContext->new($token, $auth_token->user_id); + +my $scratch = $config->{ scratch }; + +my $auth_token = Bio::KBase::AuthToken->new( + token => $token, + ignore_authrc => 1, + auth_svc => $config->{ 'auth-service-url' } +); + +my $ctx = LocalCallContext->new( $token, $auth_token->user_id ); $${module_name}::${module_name}Server::CallContext = $ctx; + my $impl = new ${module_name}::${module_name}Impl(); sub get_ws_name { - if (!defined($ws_name)) { - my $suffix = int(time * 1000); - $ws_name = 'test_${module_name}_' . $suffix; - $ws_client->create_workspace({workspace => $ws_name}); + if ( !defined( $ws_name ) ) { + my $suffix = int( time * 1000 ); + $ws_name = 'test_${module_name}_' . $suffix; + $ws_client->create_workspace( { workspace => $ws_name } ); } return $ws_name; } @@ -38,13 +47,16 @@ sub load_fasta { my $filename = shift; my $object_name = shift; my $filecontents = shift; - open (my $fh, '>', $filename) or die "Could not open file '$filename' ${dollar_sign}!"; + open ( my $fh, '>', $filename ) + or die "Could not open file '$filename' ${dollar_sign}!"; print $fh $filecontents; close $fh; - my $assycli = installed_clients::AssemblyUtilClient->new($callback_url); - return $assycli->save_assembly_from_fasta({assembly_name => $object_name, - workspace_name => get_ws_name(), - file => {path => $filename}}); + my $assycli = installed_clients::AssemblyUtilClient->new( $callback_url ); + return $assycli->save_assembly_from_fasta( { + assembly_name => $object_name, + workspace_name => get_ws_name(), + file => { path => $filename } + } ); } #end @@ -58,13 +70,15 @@ eval { "agctt\n" . ">seq3\n" . "agcttttcatgg"; - - my $ref = load_fasta($scratch . "/test1.fasta", "TestAssembly", $fastaContent); + + my $ref = load_fasta( $scratch . "/test1.fasta", "TestAssembly", $fastaContent ); # Second, call the implementation - my $ret = $impl->run_${module_name}({workspace_name => get_ws_name(), - assembly_input_ref => $ref, - min_length => 10}); + my $ret = $impl->run_${module_name}( { + workspace_name => get_ws_name(), + assembly_input_ref => $ref, + min_length => 10, + } ); # validate the returned data ok($ret->{n_initial_contigs} eq 3, "number of initial contigs"); @@ -72,31 +86,31 @@ eval { ok($ret->{n_contigs_remaining} eq 2, "number of remaining contigs"); $@ = ''; - eval { + eval { $impl->run_${module_name}({workspace_name => get_ws_name(), assembly_input_ref=>"fake", min_length => 10}); }; like($@, qr#separators / in object reference fake#); - - eval { + + eval { $impl->run_${module_name}({workspace_name => get_ws_name(), assembly_input_ref => "fake", min_length => -10}); }; like($@, qr/min_length parameter cannot be negative/); - + eval { $impl->run_${module_name}({workspace_name => get_ws_name(), assembly_input_ref => "fake"}); }; like($@, qr/Parameter min_length is not set in input arguments/); - + done_testing(6); #else # Prepare test data using the appropriate uploader for that data (see the KBase function # catalog for help, https://narrative.kbase.us/#catalog/functions) - + # Run your method by # my $ret = $impl->your_method(parameters...); # @@ -128,37 +142,49 @@ if (defined($err)) { } { + package LocalCallContext; use strict; sub new { - my($class,$token,$user) = @_; + my ( $class, $token, $user ) = @_; my $self = { - token => $token, + token => $token, user_id => $user }; return bless $self, $class; } + sub user_id { - my($self) = @_; - return $self->{user_id}; + my ( $self ) = @_; + return $self->{ user_id }; } + sub token { - my($self) = @_; - return $self->{token}; + my ( $self ) = @_; + return $self->{ token }; } + sub provenance { - my($self) = @_; - return [{'service' => '${module_name}', 'method' => 'please_never_use_it_in_production', 'method_params' => []}]; + my ( $self ) = @_; + return [ { + 'service' => '${module_name}', + 'method' => 'please_never_use_it_in_production', + 'method_params' => [], + } + ]; } + sub authenticated { return 1; } + sub log_debug { - my($self,$msg) = @_; - print STDERR $msg."\n"; + my ( $self, $msg ) = @_; + print STDERR $msg . "\n"; } + sub log_info { - my($self,$msg) = @_; - print STDERR $msg."\n"; + my ( $self, $msg ) = @_; + print STDERR $msg . "\n"; } } diff --git a/src/java/us/kbase/templates/perl_client.vm.properties b/src/java/us/kbase/templates/perl_client.vm.properties index 2fb032ca..262192cb 100644 --- a/src/java/us/kbase/templates/perl_client.vm.properties +++ b/src/java/us/kbase/templates/perl_client.vm.properties @@ -38,74 +38,72 @@ ${module.module_doc} =cut -sub new -{ - my($class, $url, @args) = @_; - +sub new { + my ( $class, $url, @args ) = @_; + #if( $default_service_url ) - if (!defined($url)) - { - $url = '${default_service_url}'; + if ( !defined( $url ) ) { + $url = '${default_service_url}'; } #end my $self = { - client => ${client_package_name}::RpcClient->new, - url => $url, - headers => [], + client => ${client_package_name}::RpcClient->new, + url => $url, + headers => [], }; + #if( $any_async || $async_version ) my %arg_hash = @args; - $self->{async_job_check_time} = 0.1; - if (exists $arg_hash{"async_job_check_time_ms"}) { - $self->{async_job_check_time} = $arg_hash{"async_job_check_time_ms"} / 1000.0; + $self->{ async_job_check_time } = 0.1; + if ( exists $arg_hash{ async_job_check_time_ms } ) { + $self->{ async_job_check_time } = $arg_hash{ async_job_check_time_ms } / 1000.0; } - $self->{async_job_check_time_scale_percent} = 150; - if (exists $arg_hash{"async_job_check_time_scale_percent"}) { - $self->{async_job_check_time_scale_percent} = $arg_hash{"async_job_check_time_scale_percent"}; + + $self->{ async_job_check_time_scale_percent } = 150; + if ( exists $arg_hash{ async_job_check_time_scale_percent } ) { + $self->{ async_job_check_time_scale_percent } = $arg_hash{ async_job_check_time_scale_percent }; } - $self->{async_job_check_max_time} = 300; # 5 minutes - if (exists $arg_hash{"async_job_check_max_time_ms"}) { - $self->{async_job_check_max_time} = $arg_hash{"async_job_check_max_time_ms"} / 1000.0; + + $self->{ async_job_check_max_time } = 300; # 5 minutes + if ( exists $arg_hash{ async_job_check_max_time_ms } ) { + $self->{ async_job_check_max_time } = $arg_hash{ async_job_check_max_time_ms } / 1000.0; } + my $service_version = #if($service_ver)'$service_ver'#{else}undef#{end}; - if (exists $arg_hash{"service_version"}) { - $service_version = $arg_hash{"service_version"}; + if ( exists $arg_hash{ "service_version" } ) { + $service_version = $arg_hash{ "service_version" }; } $self->{service_version} = $service_version; #end - chomp($self->{hostname} = `hostname`); - $self->{hostname} ||= 'unknown-host'; + chomp( $self->{ hostname } = `hostname` ); + $self->{ hostname } ||= 'unknown-host'; # # Set up for propagating KBRPC_TAG and KBRPC_METADATA environment variables through # to invoked services. If these values are not set, we create a new tag # and a metadata field with basic information about the invoking script. # - if ($ENV${empty_escaper}{KBRPC_TAG}) - { - $self->{kbrpc_tag} = $ENV${empty_escaper}{KBRPC_TAG}; + if ( $ENV${empty_escaper}{ KBRPC_TAG } ) { + $self->{ kbrpc_tag } = $ENV${empty_escaper}{ KBRPC_TAG }; } - else - { - my ($t, $us) = &$get_time(); - $us = sprintf("%06d", $us); - my $ts = strftime("%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t); - $self->{kbrpc_tag} = "C:$0:$self->{hostname}:$$:$ts"; + else { + my ( $t, $us ) = &$get_time(); + $us = sprintf( "%06d", $us ); + my $ts = strftime( "%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t ); + $self->{ kbrpc_tag } = "C:$0:$self->{hostname}:$$:$ts"; } - push(@{$self->{headers}}, 'Kbrpc-Tag', $self->{kbrpc_tag}); + push @{ $self->{ headers } }, 'Kbrpc-Tag', $self->{ kbrpc_tag }; - if ($ENV${empty_escaper}{KBRPC_METADATA}) - { - $self->{kbrpc_metadata} = $ENV${empty_escaper}{KBRPC_METADATA}; - push(@{$self->{headers}}, 'Kbrpc-Metadata', $self->{kbrpc_metadata}); + if ( $ENV${empty_escaper}{ KBRPC_METADATA } ) { + $self->{ kbrpc_metadata } = $ENV${empty_escaper}{ KBRPC_METADATA }; + push @{ $self->{ headers } }, 'Kbrpc-Metadata', $self->{ kbrpc_metadata }; } - if ($ENV${empty_escaper}{KBRPC_ERROR_DEST}) - { - $self->{kbrpc_error_dest} = $ENV${empty_escaper}{KBRPC_ERROR_DEST}; - push(@{$self->{headers}}, 'Kbrpc-Errordest', $self->{kbrpc_error_dest}); + if ( $ENV${empty_escaper}{ KBRPC_ERROR_DEST } ) { + $self->{ kbrpc_error_dest } = $ENV${empty_escaper}{ KBRPC_ERROR_DEST }; + push @{ $self->{ headers } }, 'Kbrpc-Errordest', $self->{ kbrpc_error_dest }; } #if( $authenticated ) @@ -115,26 +113,26 @@ sub new # We create an auth token, passing through the arguments that we were (hopefully) given. { - my %arg_hash2 = @args; - if (exists $arg_hash2{"token"}) { - $self->{token} = $arg_hash2{"token"}; - } elsif (exists $arg_hash2{"user_id"}) { - my $token = Bio::KBase::AuthToken->new(@args); - if (!$token->error_message) { - $self->{token} = $token->token; - } - } - - if (exists $self->{token}) - { - $self->{client}->{token} = $self->{token}; - } + my %arg_hash2 = @args; + if ( exists $arg_hash2{ token } ) { + $self->{ token } = $arg_hash2{ token }; + } + elsif ( exists $arg_hash2{ user_id } ) { + my $token = Bio::KBase::AuthToken->new( @args ); + if ( !$token->error_message ) { + $self->{ token } = $token->token; + } + } + + if (exists $self->{ token } ) { + $self->{ client }{ token } = $self->{ token }; + } } #end - my $ua = $self->{client}->ua; - my $timeout = $ENV${empty_escaper}{CDMI_TIMEOUT} || (30 * 60); - $ua->timeout($timeout); + my $ua = $self->{ client }->ua; + my $timeout = $ENV${empty_escaper}{ CDMI_TIMEOUT } || ( 30 * 60 ); + $ua->timeout( $timeout ); bless $self, $class; # $self->_validate_version(); return $self; @@ -143,7 +141,7 @@ sub new #foreach( $module in $modules ) #if( $any_async || $async_version) sub _check_job { - my($self, @args) = @_; + my ( $self, @args ) = @_; # Authentication: ${method.authentication} if ((my $n = @args) != 1) { Bio::KBase::Exceptions::ArgumentValidationError->throw(error => @@ -155,27 +153,38 @@ sub _check_job { (!ref($job_id)) or push(@_bad_arguments, "Invalid type for argument 0 \"job_id\" (it should be a string)"); if (@_bad_arguments) { my $msg = "Invalid arguments passed to _check_job:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => '_check_job'); - } - } - my $result = $self->{client}->call($self->{url}, $self->{headers}, { - method => "${module.module_name}._check_job", - params => \@args}); - if ($result) { - if ($result->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $result->error_message, - code => $result->content->{error}->{code}, - method_name => '_check_job', - data => $result->content->{error}->{error} # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O - ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => '_check_job' + ); + } + } + my $result = $self->{ client }->call( + $self->{ url }, + $self->{ headers }, + { + method => "${module.module_name}._check_job", + params => \@args + } + ); + if ( $result ) { + if ( $result->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $result->error_message, + code => $result->content->{ error }{ code }, + method_name => '_check_job', + # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + data => $result->content->{ error }{ error }, + ); } else { return $result->result->[0]; } } else { - Bio::KBase::Exceptions::HTTP->throw(error => "Error invoking method _check_job", - status_line => $self->{client}->status_line, - method_name => '_check_job'); + Bio::KBase::Exceptions::HTTP->throw( + error => "Error invoking method _check_job", + status_line => $self->{client}->status_line, + method_name => '_check_job' + ); } } @@ -227,29 +236,30 @@ ${method.doc} =cut #if( $method.async || $async_version) -sub ${method.name} -{ - my($self, @args) = @_; - my $job_id = $self->_${method.name}_submit(@args); - my $async_job_check_time = $self->{async_job_check_time}; +sub ${method.name} { + my ( $self, @args ) = @_; + my $job_id = $self->_${method.name}_submit( @args ); + my $async_job_check_time = $self->{ async_job_check_time }; while (1) { Time::HiRes::sleep($async_job_check_time); - $async_job_check_time *= $self->{async_job_check_time_scale_percent} / 100.0; - if ($async_job_check_time > $self->{async_job_check_max_time}) { - $async_job_check_time = $self->{async_job_check_max_time}; + $async_job_check_time *= $self->{ async_job_check_time_scale_percent } / 100.0; + if ( $async_job_check_time > $self->{ async_job_check_max_time } ) { + $async_job_check_time = $self->{ async_job_check_max_time }; } - my $job_state_ref = $self->_check_job($job_id); - if ($job_state_ref->{"finished"} != 0) { - if (!exists $job_state_ref->{"result"}) { - $job_state_ref->{"result"} = []; + my $job_state_ref = $self->_check_job( $job_id ); + if ( $job_state_ref->{ "finished" } != 0 ) { + if ( !exists $job_state_ref->{ "result" } ) { + $job_state_ref->{ "result" } = []; } - return wantarray ? @{$job_state_ref->{"result"}} : $job_state_ref->{"result"}->[0]; + return wantarray + ? @{ $job_state_ref->{ "result" } } + : $job_state_ref->{ "result" }[ 0 ]; } } } sub _${method.name}_submit { - my($self, @args) = @_; + my ( $self, @args ) = @_; # Authentication: ${method.authentication} if ((my $n = @args) != ${method.arg_count}) { Bio::KBase::Exceptions::ArgumentValidationError->throw(error => @@ -270,132 +280,174 @@ sub _${method.name}_submit { } #end my $context = undef; - if ($self->{service_version}) { - $context = {'service_ver' => $self->{service_version}}; - } - my $result = $self->{client}->call($self->{url}, $self->{headers}, { - method => "${module.module_name}._${method.name}_submit", - params => \@args, context => $context}); - if ($result) { - if ($result->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $result->error_message, - code => $result->content->{error}->{code}, - method_name => '_${method.name}_submit', - data => $result->content->{error}->{error} # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + if ( $self->{ service_version } ) { + $context = { 'service_ver' => $self->{ service_version } }; + } + my $result = $self->{ client }->call( + $self->{ url }, + $self->{ headers }, + { + method => "${module.module_name}._${method.name}_submit", + params => \@args, + context => $context + } + ); + + if ( $result ) { + if ( $result->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $result->error_message, + code => $result->content->{ error }{ code }, + method_name => '_${method.name}_submit', + # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + data => $result->content->{ error }{ error }, ); } else { return $result->result->[0]; # job_id } } else { - Bio::KBase::Exceptions::HTTP->throw(error => "Error invoking method _${method.name}_submit", - status_line => $self->{client}->status_line, - method_name => '_${method.name}_submit'); + Bio::KBase::Exceptions::HTTP->throw( + error => "Error invoking method _${method.name}_submit", + status_line => $self->{ client }->status_line, + method_name => '_${method.name}_submit' + ); } } #else ## of if-any-async -sub ${method.name} -{ - my($self, @args) = @_; +sub ${method.name} { + my ( $self, @args ) = @_; # Authentication: ${method.authentication} - if ((my $n = @args) != ${method.arg_count}) - { - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => - "Invalid argument count for function ${method.name} (received $n, expecting ${method.arg_count})"); + if ( ( my $n = @args ) != ${method.arg_count} ) { + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => "Invalid argument count for function ${method.name} (received $n, expecting ${method.arg_count})" + ); } #if( $method.arg_count > 0 ) { - my(${method.arg_vars}) = @args; + my( ${method.arg_vars} ) = @args; - my @_bad_arguments; + my @_bad_arguments; #foreach( $param in $method.params ) - (${param.validator}) or push(@_bad_arguments, "Invalid type for argument ${param.index} \"${param.name}\" (value was \"${param.perl_var}\")"); + ( ${param.validator} ) or push( @_bad_arguments, "Invalid type for argument ${param.index} \"${param.name}\" (value was \"${param.perl_var}\")" ); #end - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to ${method.name}:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => '${method.name}'); - } + if ( @_bad_arguments ) { + my $msg = "Invalid arguments passed to ${method.name}:\n" . join("", map { "\t$_\n" } @_bad_arguments); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => '${method.name}' + ); + } } #end #if( $dynserv_ver ) - my $service_state = $self->{client}->call($self->{url}, $self->{headers}, { - method => "ServiceWizard.get_service_status", - params => [{module_name=>"${module.module_name}", version=>$self->{service_version}}]}); - if ($service_state->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $service_state->error_message, - code => $service_state->content->{error}->{code}, - method_name => 'ServiceWizard.get_service_status', - data => $service_state->content->{error}->{error} - ); + my $service_state = $self->{ client }->call( + $self->{ url }, + $self->{ headers }, + { + method => "ServiceWizard.get_service_status", + params => + [ + { + module_name => "${module.module_name}", + version => $self->{ service_version } + } + ] + } + ); + if ( $service_state->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $service_state->error_message, + code => $service_state->content->{ error }{ code }, + method_name => 'ServiceWizard.get_service_status', + data => $service_state->content->{ error }{ error } + ); } my $url = $service_state->result->[0]->{url}; #else - my $url = $self->{url}; + my $url = $self->{ url }; #end - my $result = $self->{client}->call($url, $self->{headers}, { - method => "${module.module_name}.${method.name}", - params => \@args, - }); - if ($result) { - if ($result->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $result->error_message, - code => $result->content->{error}->{code}, - method_name => '${method.name}', - data => $result->content->{error}->{error} # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O - ); - } else { + my $result = $self->{ client }->call( + $url, + $self->{ headers }, + { + method => "${module.module_name}.${method.name}", + params => \@args, + } + ); + if ( $result ) { + if ( $result->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $result->error_message, + code => $result->content->{ error }{ code }, + method_name => '${method.name}', + # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + data => $result->content->{ error }{ error }, + ); + } else { #if( $method.ret_count > 0 ) - return wantarray ? @{$result->result} : $result->result->[0]; + return wantarray ? @{ $result->result } : $result->result->[ 0 ]; #else - return; + return; #end - } + } } else { - Bio::KBase::Exceptions::HTTP->throw(error => "Error invoking method ${method.name}", - status_line => $self->{client}->status_line, - method_name => '${method.name}', - ); + Bio::KBase::Exceptions::HTTP->throw( + error => "Error invoking method ${method.name}", + status_line => $self->{ client }->status_line, + method_name => '${method.name}', + ); } } + #end ## of if-any-async -#end ## of foreach-method +#end ## of foreach-method #if( !$status_in_kidl ) #if( $async_version ) -sub status -{ - my($self, @args) = @_; +sub status { + my ( $self, @args ) = @_; my $job_id = undef; - if ((my $n = @args) != 0) { - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => - "Invalid argument count for function status (received $n, expecting 0)"); + if ( ( my $n = @args ) != 0 ) { + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => "Invalid argument count for function status (received $n, expecting 0)"); } my $context = undef; - if ($self->{service_version}) { - $context = {'service_ver' => $self->{service_version}}; - } - my $result = $self->{client}->call($self->{url}, $self->{headers}, { - method => "${module.module_name}._status_submit", - params => \@args, context => $context}); - if ($result) { - if ($result->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $result->error_message, - code => $result->content->{error}->{code}, - method_name => '_status_submit', - data => $result->content->{error}->{error} # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + if ( $self->{ service_version } ) { + $context = { 'service_ver' => $self->{ service_version } }; + } + my $result = $self->{ client }->call( + $self->{ url }, + $self->{ headers }, + { + method => "${module.module_name}._status_submit", + params => \@args, + context => $context + } + ); + + if ( $result ) { + if ( $result->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $result->error_message, + code => $result->content->{ error }{ code }, + method_name => '_status_submit', + # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + data => $result->content->{ error }{ error }, ); } else { $job_id = $result->result->[0]; } } else { - Bio::KBase::Exceptions::HTTP->throw(error => "Error invoking method _status_submit", - status_line => $self->{client}->status_line, - method_name => '_status_submit'); + Bio::KBase::Exceptions::HTTP->throw( + error => "Error invoking method _status_submit", + status_line => $self->{ client }->status_line, + method_name => '_status_submit', + ); } my $async_job_check_time = $self->{async_job_check_time}; while (1) { @@ -415,47 +467,67 @@ sub status } #else ## of if-status-async -sub status -{ - my($self, @args) = @_; +sub status { + my ( $self, @args ) = @_; + if ((my $n = @args) != 0) { - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => - "Invalid argument count for function status (received $n, expecting 0)"); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => "Invalid argument count for function status (received $n, expecting 0)"); } #if( $dynserv_ver ) - my $service_state = $self->{client}->call($self->{url}, $self->{headers}, { - method => "ServiceWizard.get_service_status", - params => [{module_name=>"${module.module_name}", version=>$self->{service_version}}]}); - if ($service_state->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $service_state->error_message, - code => $service_state->content->{error}->{code}, - method_name => 'ServiceWizard.get_service_status', - data => $service_state->content->{error}->{error} - ); + + my $service_state = $self->{ client }->call( + $self->{ url }, + $self->{ headers }, + { + method => "ServiceWizard.get_service_status", + params => + [ + { + module_name => "${module.module_name}", + version => $self->{ service_version } + } + ] + } + ); + if ( $service_state->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $service_state->error_message, + code => $service_state->content->{ error }{ code }, + method_name => 'ServiceWizard.get_service_status', + data => $service_state->content->{ error }{ error }, + ); } - my $url = $service_state->result->[0]->{url}; + my $url = $service_state->result->[ 0 ]{ url }; #else - my $url = $self->{url}; + my $url = $self->{ url }; #end - my $result = $self->{client}->call($url, $self->{headers}, { - method => "${module.module_name}.status", - params => \@args, - }); - if ($result) { - if ($result->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $result->error_message, - code => $result->content->{error}->{code}, - method_name => 'status', - data => $result->content->{error}->{error} # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + my $result = $self->{ client }->call( + $url, + $self->{ headers }, + { + method => "${module.module_name}.status", + params => \@args, + } + ); + if ( $result ) { + if ( $result->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $result->error_message, + code => $result->content->{ error }{ code }, + method_name => 'status', + # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + data => $result->content->{ error }{ error }, ); } else { - return wantarray ? @{$result->result} : $result->result->[0]; + return wantarray ? @{ $result->result } : $result->result->[0]; } } else { - Bio::KBase::Exceptions::HTTP->throw(error => "Error invoking method status", - status_line => $self->{client}->status_line, - method_name => 'status', - ); + Bio::KBase::Exceptions::HTTP->throw( + error => "Error invoking method status", + status_line => $self->{ client }->status_line, + method_name => 'status', + ); } } #end ## of if-status-async @@ -464,25 +536,29 @@ sub status sub version { - my ($self) = @_; - my $result = $self->{client}->call($self->{url}, $self->{headers}, { - method => "${last_module.module_name}.version", - params => [], - }); - if ($result) { - if ($result->is_error) { + my ( $self ) = @_; + my $result = $self->{client}->call( + $self->{ url }, + $self->{ headers }, + { + method => "${last_module.module_name}.version", + params => [], + } + ); + if ( $result ) { + if ( $result->is_error ) { Bio::KBase::Exceptions::JSONRPC->throw( - error => $result->error_message, - code => $result->content->{code}, + error => $result->error_message, + code => $result->content->{ code }, method_name => '${last_method.name}', ); } else { - return wantarray ? @{$result->result} : $result->result->[0]; + return wantarray ? @{ $result->result } : $result->result->[ 0 ]; } } else { Bio::KBase::Exceptions::HTTP->throw( - error => "Error invoking method ${last_method.name}", - status_line => $self->{client}->status_line, + error => "Error invoking method ${last_method.name}", + status_line => $self->{ client }->status_line, method_name => '${last_method.name}', ); } @@ -492,26 +568,30 @@ sub _validate_version { my ($self) = @_; my $svr_version = $self->version(); my $client_version = $VERSION; - my ($cMajor, $cMinor) = split(/\./, $client_version); - my ($sMajor, $sMinor) = split(/\./, $svr_version); - if ($sMajor != $cMajor) { + my ( $cMajor, $cMinor ) = split(/\./, $client_version); + my ( $sMajor, $sMinor ) = split(/\./, $svr_version); + + if ( $sMajor != $cMajor ) { Bio::KBase::Exceptions::ClientServerIncompatible->throw( - error => "Major version numbers differ.", - server_version => $svr_version, - client_version => $client_version + error => "Major version numbers differ.", + server_version => $svr_version, + client_version => $client_version ); } - if ($sMinor < $cMinor) { + + if ( $sMinor < $cMinor ) { Bio::KBase::Exceptions::ClientServerIncompatible->throw( - error => "Client minor version greater than Server minor version.", - server_version => $svr_version, - client_version => $client_version + error => "Client minor version greater than server minor version.", + server_version => $svr_version, + client_version => $client_version ); } - if ($sMinor > $cMinor) { + + if ( $sMinor > $cMinor ) { warn "New client version available for ${client_package_name}\n"; } - if ($sMajor == 0) { + + if ( $sMajor == 0 ) { warn "${client_package_name} version is $svr_version. API subject to change.\n"; } } @@ -580,61 +660,58 @@ sub call { #end { - if ($uri =~ /\?/) { - $result = $self->_get($uri); - } - else { - Carp::croak "not hashref." unless (ref $obj eq 'HASH'); - $result = $self->_post($uri, $headers, $obj); - } -#if( $enable_client_retry ) + if ($uri =~ /\?/) { + $result = $self->_get($uri); + } + else { + Carp::croak "not hashref." unless (ref $obj eq 'HASH'); + $result = $self->_post($uri, $headers, $obj); + } + #if( $enable_client_retry ) + + # + # Bail early on success. + # + if ($result->is_success) { + if ($n_retries) { + print STDERR strftime("%F %T", localtime), ": Request succeeded after $n_retries retries\n"; + } + last; + } + $n_retries++; + + # + # Failure. See if we need to retry and loop, or bail with + # a permanent failure. + # - # - # Bail early on success. - # - if ($result->is_success) - { - if ($n_retries) - { - print STDERR strftime("%F %T", localtime), ": Request succeeded after $n_retries retries\n"; - } - last; - } - $n_retries++; - - # - # Failure. See if we need to retry and loop, or bail with - # a permanent failure. - # - my $code = $result->code; - my $msg = $result->message; - my $want_retry = 0; - if ($codes_to_retry${empty_escaper}{$code}) - { - $want_retry = 1; - } - elsif ($code eq 500 && defined( $result->header('client-warning') ) - && $result->header('client-warning') eq 'Internal response') - { - # - # Handle errors that were not thrown by the web - # server but rather picked up by the client library. - # - # If we got a client timeout or connection refused, let us retry. - # - - if ($msg =~ /timeout|connection refused/i) - { - $want_retry = 1; - } - - } - + my $msg = $result->message; + my $want_retry = 0; + if ($codes_to_retry${empty_escaper}{$code}) + { + $want_retry = 1; + } + elsif ($code eq 500 && defined( $result->header('client-warning') ) + && $result->header('client-warning') eq 'Internal response') + { + # + # Handle errors that were not thrown by the web + # server but rather picked up by the client library. + # + # If we got a client timeout or connection refused, let us retry. + # + + if ($msg =~ /timeout|connection refused/i) { + $want_retry = 1; + } + + } + if (!$want_retry || @retries == 0) { - last; + last; } - + # # otherwise, sleep & loop. # @@ -670,13 +747,13 @@ sub call { sub _post { - my ($self, $uri, $headers, $obj) = @_; + my ( $self, $uri, $headers, $obj ) = @_; my $json = $self->json; - $obj->{version} ||= $self->{version} || '1.1'; + $obj->{ version } ||= $self->{ version } || '1.1'; - if ($obj->{version} eq '1.0') { - delete $obj->{version}; + if ( $obj->{ version } eq '1.0' ) { + delete $obj->{ version }; if (exists $obj->{id}) { $self->id($obj->{id}) if ($obj->{id}); # if undef, it is notification. } @@ -685,9 +762,9 @@ sub _post { } } else { - # $obj->{id} = $self->id if (defined $self->id); - # Assign a random number to the id if one hasn't been set - $obj->{id} = (defined $self->id) ? $self->id : substr(rand(),2); + # $obj->{ id } = $self->id if defined $self->id; + # Assign a random number to the id if one hasn't been set + $obj->{ id } = (defined $self->id) ? $self->id : substr( rand(), 2 ); } my $content = $json->encode($obj); @@ -697,11 +774,9 @@ sub _post { Content_Type => $self->{content_type}, Content => $content, Accept => 'application/json', - @$headers, - ($self->{token} ? (Authorization => $self->{token}) : ()), + @$headers, + ($self->{token} ? (Authorization => $self->{token}) : ()), ); } - - 1; diff --git a/src/java/us/kbase/templates/perl_impl.vm.properties b/src/java/us/kbase/templates/perl_impl.vm.properties index 9ff476db..aab85914 100644 --- a/src/java/us/kbase/templates/perl_impl.vm.properties +++ b/src/java/us/kbase/templates/perl_impl.vm.properties @@ -1,8 +1,9 @@ package ${module.impl_package_name}; + use strict; use Bio::KBase::Exceptions; # Use Semantic Versioning (2.0.0-rc.1) -# http://semver.org +# http://semver.org our $VERSION = '${module.semantic_version}'; our $GIT_URL = '${module.git_url}'; our $GIT_COMMIT_HASH = '${module.git_commit_hash}'; @@ -20,18 +21,16 @@ ${module.module_doc} #BEGIN_HEADER ${module.module_header}#END_HEADER -sub new -{ - my($class, @args) = @_; - my $self = { - }; +sub new { + my ( $class, @args ) = @_; + my $self = {}; bless $self, $class; + #BEGIN_CONSTRUCTOR ${module.module_constructor} #END_CONSTRUCTOR - if ($self->can('_init_instance')) - { - $self->_init_instance(); + if ( $self->can( '_init_instance' ) ) { + $self->_init_instance(); } return $self; } @@ -83,26 +82,28 @@ ${method.doc} =cut -sub ${method.name} -{ +sub ${method.name} { my $self = shift; #if( $method.arg_count > 0 ) - my(${method.arg_vars}) = @_; + my( ${method.arg_vars} ) = @_; my @_bad_arguments; #foreach( $param in $method.params ) - (${param.validator}) or push(@_bad_arguments, "Invalid type for argument \"${param.name}\" (value was \"${param.perl_var}\")"); + ( ${param.validator} ) or push( @_bad_arguments, "Invalid type for argument \"${param.name}\" (value was \"${param.perl_var}\")" ); #end - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to ${method.name}:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => '${method.name}'); + if ( @_bad_arguments ) { + my $msg = "Invalid arguments passed to ${method.name}:\n" + . join("", map { "\t$_\n" } @_bad_arguments); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => '${method.name}' + ); } #end my $ctx = $${server_package_name}::CallContext; #if( $method.ret_count > 0 ) - my(${method.ret_vars}); + my ( ${method.ret_vars} ); #end #BEGIN ${method.name} ${method.user_code} #${empty_escaper}END ${method.name} @@ -111,13 +112,16 @@ ${method.user_code} #${empty_escaper}END ${method.name} #foreach( $return in $method.returns ) (${return.validator}) or push(@_bad_returns, "Invalid type for return variable \"${return.name}\" (value was \"${return.perl_var}\")"); #end - if (@_bad_returns) { - my $msg = "Invalid returns passed to ${method.name}:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => '${method.name}'); + if ( @_bad_returns ) { + my $msg = "Invalid returns passed to ${method.name}:\n" + . join( "", map { "\t$_\n" } @_bad_returns ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => '${method.name}' + ); } #end - return(${method.ret_vars}); + return ( ${method.ret_vars} ); } @@ -125,7 +129,7 @@ ${method.user_code} #${empty_escaper}END ${method.name} #if( !$status_in_kidl ) -=head2 status +=head2 status $return = $obj->status() @@ -156,15 +160,21 @@ Return the module status. This is a structure including Semantic Versioning numb =cut sub status { - my($return); + my $return; #BEGIN_STATUS #if( ${module.module_status} ) -${module.module_status}#else - $return = {"state" => "OK", "message" => "", "version" => $VERSION, - "git_url" => $GIT_URL, "git_commit_hash" => $GIT_COMMIT_HASH}; +${module.module_status} +#else + $return = { + "state" => "OK", + "message" => "", + "version" => $VERSION, + "git_url" => $GIT_URL, + "git_commit_hash" => $GIT_COMMIT_HASH + }; #end #END_STATUS - return($return); + return ( $return ); } #end diff --git a/src/java/us/kbase/templates/perl_psgi.vm.properties b/src/java/us/kbase/templates/perl_psgi.vm.properties index 557d2e2c..d3d268a3 100644 --- a/src/java/us/kbase/templates/perl_psgi.vm.properties +++ b/src/java/us/kbase/templates/perl_psgi.vm.properties @@ -7,7 +7,6 @@ use Plack::Middleware::CrossOrigin; #if( $service_options.authenticated ) use UserAuth; - my $user_auth = UserAuth->new(); #end @@ -17,18 +16,23 @@ my @dispatch; #foreach( $module in $modules ) { my $obj = ${module.impl_package_name}->new; - push(@dispatch, '${module.module_name}' => $obj); + push( @dispatch, '${module.module_name}' => $obj ); } #end -my $server = ${server_package_name}->new(instance_dispatch => { @dispatch }, +my $server = ${server_package_name}->new( + instance_dispatch => { @dispatch }, + allow_get => 0, #if( $service_options.authenticated ) - user_auth => $user_auth, + user_auth => $user_auth, #end - allow_get => 0, - ); +); my $handler = sub { $server->handle_input(@_) }; -$handler = Plack::Middleware::CrossOrigin->wrap( $handler, origins => "*", headers => "*"); +$handler = Plack::Middleware::CrossOrigin->wrap( + $handler, + origins => "*", + headers => "*" +); diff --git a/src/java/us/kbase/templates/perl_server.vm.properties b/src/java/us/kbase/templates/perl_server.vm.properties index 1457a62e..666c03a5 100644 --- a/src/java/us/kbase/templates/perl_server.vm.properties +++ b/src/java/us/kbase/templates/perl_server.vm.properties @@ -1,6 +1,6 @@ package ${server_package_name}; -#set( $ctx_pkg = "${server_package_name}Context" ) +#set( $ctx_pkg = "${server_package_name}Context" ) #set( $stderr_pkg = "${server_package_name}StderrWrapper" ) use Data::Dumper; @@ -21,13 +21,17 @@ use Bio::KBase::AuthToken; extends 'RPC::Any::Server::JSONRPC::PSGI'; -has 'instance_dispatch' => (is => 'ro', isa => 'HashRef'); -has 'user_auth' => (is => 'ro', isa => 'UserAuth'); -has 'valid_methods' => (is => 'ro', isa => 'HashRef', lazy => 1, - builder => '_build_valid_methods'); -has 'loggers' => (is => 'ro', required => 1, builder => '_build_loggers'); -has 'config' => (is => 'ro', required => 1, builder => '_build_config'); -has 'local_headers' => (is => 'ro', isa => 'HashRef'); +has 'instance_dispatch' => ( is => 'ro', isa => 'HashRef' ); +has 'user_auth' => ( is => 'ro', isa => 'UserAuth' ); +has 'valid_methods' => ( + is => 'ro', + isa => 'HashRef', + lazy => 1, + builder => '_build_valid_methods' +); +has 'loggers' => ( is => 'ro', required => 1, builder => '_build_loggers' ); +has 'config' => ( is => 'ro', required => 1, builder => '_build_config' ); +has 'local_headers' => ( is => 'ro', isa => 'HashRef' ); our $CallContext; @@ -38,11 +42,11 @@ our %return_counts = ( #if( ${method.name} == "status" ) #set( $status_in_kidl = true ) #end - '${method.name}' => ${method.ret_count}, + '${method.name}' => ${method.ret_count}, #end #end #if( !$status_in_kidl ) - 'status' => 1, + status => 1, #end ); @@ -50,16 +54,15 @@ our %return_counts = ( our %method_authentication = ( #foreach( $module in $modules ) #foreach( $method in $module.methods ) - '${method.name}' => '${method.authentication}', + '${method.name}' => '${method.authentication}', #end #end ); #end -sub _build_valid_methods -{ - my($self) = @_; - my $methods = { +sub _build_valid_methods { + my ( $self ) = @_; + my $methods = { #foreach( $module in $modules ) #set( $last_module = $module ) #foreach( $method in $module.methods ) @@ -67,139 +70,162 @@ sub _build_valid_methods #end #end #if( !$status_in_kidl ) - 'status' => 1, + status => 1, #end }; return $methods; } -my $DEPLOY = 'KB_DEPLOYMENT_CONFIG'; +my $DEPLOY = 'KB_DEPLOYMENT_CONFIG'; my $SERVICE = 'KB_SERVICE_NAME'; -sub get_config_file -{ - my ($self) = @_; - if(!defined $ENV${empty_escaper}{$DEPLOY}) { +sub get_config_file { + my ( $self ) = @_; + if ( !defined $ENV${empty_escaper}{$DEPLOY} ) { return undef; } return $ENV${empty_escaper}{$DEPLOY}; } -sub get_service_name -{ - my ($self) = @_; - if(!defined $ENV${empty_escaper}{$SERVICE}) { +sub get_service_name { + my ( $self ) = @_; + if ( !defined $ENV${empty_escaper}{$SERVICE} ) { return '${service_name}'; } return $ENV${empty_escaper}{$SERVICE}; } -sub _build_config -{ - my ($self) = @_; - my $sn = $self->get_service_name(); - my $cf = $self->get_config_file(); - if (!($cf)) { +sub _build_config { + my ( $self ) = @_; + my $sn = $self->get_service_name; + my $cf = $self->get_config_file; + if ( !($cf) ) { return {}; } - my $cfg = new Config::Simple($cf); - my $cfgdict = $cfg->get_block($sn); - if (!($cfgdict)) { + my $cfg = new Config::Simple( $cf ); + my $cfgdict = $cfg->get_block( $sn ); + if ( !($cfgdict) ) { return {}; } return $cfgdict; } -sub logcallback -{ - my ($self) = @_; - $self->loggers()->{serverlog}->set_log_file( - $self->{loggers}->{userlog}->get_log_file()); +sub logcallback { + my ( $self ) = @_; + $self->loggers()->{ serverlog }->set_log_file( + $self->{ loggers }{ userlog }->get_log_file() + ); } -sub log -{ - my ($self, $level, $context, $message, $tag) = @_; - my $user = defined($context->user_id()) ? $context->user_id(): undef; - $self->loggers()->{serverlog}->log_message($level, $message, $user, - $context->module(), $context->method(), $context->call_id(), - $context->client_ip(), $tag); +sub log { + my ( $self, $level, $context, $message, $tag ) = @_; + my $user + = defined( $context->user_id() ) + ? $context->user_id() + : undef; + $self->loggers()->{ serverlog }->log_message( + $level, $message, $user, $context->module(), $context->method(), + $context->call_id(), $context->client_ip(), $tag + ); } -sub _build_loggers -{ - my ($self) = @_; - my $submod = $self->get_service_name(); - my $loggers = {}; - my $callback = sub {$self->logcallback();}; - $loggers->{userlog} = Bio::KBase::Log->new( - $submod, {}, {ip_address => 1, authuser => 1, module => 1, - method => 1, call_id => 1, changecallback => $callback, - tag => 1, - config => $self->get_config_file()}); - $loggers->{serverlog} = Bio::KBase::Log->new( - $submod, {}, {ip_address => 1, authuser => 1, module => 1, - method => 1, call_id => 1, - tag => 1, - logfile => $loggers->{userlog}->get_log_file()}); - $loggers->{serverlog}->set_log_level(6); +sub _build_loggers { + my ( $self ) = @_; + my $submod = $self->get_service_name(); + my $loggers = {}; + my $callback = sub { $self->logcallback(); }; + $loggers->{ userlog } = Bio::KBase::Log->new( + $submod, + {}, + { + ip_address => 1, + authuser => 1, + module => 1, + method => 1, + call_id => 1, + changecallback => $callback, + tag => 1, + config => $self->get_config_file() + } + ); + $loggers->{ serverlog } = Bio::KBase::Log->new( + $submod, + {}, + { + ip_address => 1, + authuser => 1, + module => 1, + method => 1, + call_id => 1, + tag => 1, + logfile => $loggers->{ userlog }->get_log_file() + } + ); + $loggers->{ serverlog }->set_log_level( 6 ); return $loggers; } #override of RPC::Any::Server sub handle_error { - my ($self, $error) = @_; - - unless (ref($error) eq 'HASH' || - (blessed $error and $error->isa('RPC::Any::Exception'))) { - $error = RPC::Any::Exception::PerlError->new(message => $error); + my ( $self, $error ) = @_; + + unless ( ref( $error ) eq 'HASH' + || ( blessed $error and $error->isa( 'RPC::Any::Exception' ) ) ) { + $error = RPC::Any::Exception::PerlError->new( message => $error ); } my $output; eval { - my $encoded_error = $self->encode_output_from_exception($error); - $output = $self->produce_output($encoded_error); + my $encoded_error = $self->encode_output_from_exception( $error ); + $output = $self->produce_output( $encoded_error ); }; - + return $output if $output; - + die "$error\n\nAlso, an error was encountered while trying to send" . " this error: $@\n"; } #override of RPC::Any::JSONRPC sub encode_output_from_exception { - my ($self, $exception) = @_; + my ( $self, $exception ) = @_; + my %error_params; - if (ref($exception) eq 'HASH') { - %error_params = %{$exception}; - if(defined($error_params${empty_escaper}{context})) { + + if ( ref $exception eq 'HASH' ) { + %error_params = %$exception; + if ( defined( $error_params${empty_escaper}{ context } ) ) { my @errlines; - $errlines[0] = $error_params${empty_escaper}{message}; - push @errlines, split("\n", $error_params${empty_escaper}{data}); - $self->log($Bio::KBase::Log::ERR, $error_params${empty_escaper}{context}, \@errlines); - delete $error_params${empty_escaper}{context}; + $errlines[0] = $error_params${empty_escaper}{ message }; + push @errlines, split( "\n", $error_params${empty_escaper}{ data } ); + $self->log( + $Bio::KBase::Log::ERR, + $error_params${empty_escaper}{ context }, + \@errlines + ); + delete $error_params${empty_escaper}{ context }; } - } else { + } + else { %error_params = ( message => $exception->message, code => $exception->code, ); } my $json_error; - if ($self->_last_call) { - $json_error = $self->_last_call->return_error(%error_params); + if ( $self->_last_call ) { + $json_error = $self->_last_call->return_error( %error_params ); } # Default to default_version. This happens when we throw an exception # before inbound parsing is complete. else { - $json_error = $self->_default_error(%error_params); + $json_error = $self->_default_error( %error_params ); } - return $self->encode_output_from_object($json_error); + return $self->encode_output_from_object( $json_error ); } sub trim { - my ($str) = @_; - if (!(defined $str)) { + my ( $str ) = @_; + if ( !( defined $str ) ) { return $str; } $str =~ s/^\s+|\s+$//g; @@ -207,264 +233,289 @@ sub trim { } sub getIPAddress { - my ($self) = @_; - my $xFF = trim($self->_plack_req_header("X-Forwarded-For")); - my $realIP = trim($self->_plack_req_header("X-Real-IP")); - my $nh = $self->config->{"dont_trust_x_ip_headers"}; - my $trustXHeaders = !(defined $nh) || $nh ne "true"; - - if ($trustXHeaders) { - if ($xFF) { - my @tmp = split(",", $xFF); - return trim($tmp[0]); + my ( $self ) = @_; + my $xFF = trim( $self->_plack_req_header( "X-Forwarded-For" ) ); + my $realIP = trim( $self->_plack_req_header( "X-Real-IP" ) ); + my $nh = $self->config->{ "dont_trust_x_ip_headers" }; + my $trustXHeaders = !( defined $nh ) || $nh ne "true"; + + if ( $trustXHeaders ) { + if ( $xFF ) { + my @tmp = split ",", $xFF; + return trim( $tmp[ 0 ] ); } - if ($realIP) { + if ( $realIP ) { return $realIP; } } - if (defined($self->_plack_req)) { + if ( defined( $self->_plack_req ) ) { return $self->_plack_req->address; } return "localhost"; } sub call_method { - my ($self, $data, $method_info) = @_; - - my ($module, $method, $modname) = @$method_info${empty_escaper}{qw(module method modname)}; - - my $ctx = ${ctx_pkg}->new($self->{loggers}->{userlog}, - client_ip => $self->getIPAddress()); - $ctx->module($modname); - $ctx->method($method); - $ctx->call_id($self->{_last_call}->{id}); - - my $args = $data->{arguments}; - my $prov_action = {'service' => $modname, 'method' => $method, 'method_params' => $args}; - $ctx->provenance([$prov_action]); + my ( $self, $data, $method_info ) = @_; + + my ( $module, $method, $modname ) = @$method_info${empty_escaper}{ qw( module method modname ) }; + + my $ctx = ${ctx_pkg}->new( + $self->{ loggers }{ userlog }, + client_ip => $self->getIPAddress() + ); + $ctx->module( $modname ); + $ctx->method( $method ); + $ctx->call_id( $self->{ _last_call }{ id } ); + + my $args = $data->{ arguments }; + my $prov_action = { + service => $modname, + method => $method, + method_params => $args + }; + + $ctx->provenance( [ $prov_action ] ); + #if( $authenticated ) { # Service ${service_name} requires authentication. - my $method_auth = $method_authentication${empty_escaper}{$method}; - $ctx->authenticated(0); - if ($method_auth eq 'none') - { - # No authentication required here. Move along. + my $method_auth = $method_authentication${empty_escaper}{ $method }; + $ctx->authenticated( 0 ); + if ( $method_auth eq 'none' ) { + # No authentication required here. Move along. } - else - { - my $token = $self->_plack_req_header("Authorization"); + else { + my $token = $self->_plack_req_header( "Authorization" ); - if (!$token && $method_auth eq 'required') - { - $self->exception('PerlError', "Authentication required for ${last_module.module_name} but no authentication header was passed"); - } + if ( !$token && $method_auth eq 'required' ) { + $self->exception( 'PerlError', + "Authentication required for ${last_module.module_name} but no authentication header was passed" + ); + } - my $auth_token; - if ($self->config->{'auth-service-url'}) - { - $auth_token = Bio::KBase::AuthToken->new(token => $token, ignore_authrc => 1, auth_svc=>$self->config->{'auth-service-url'}); + my $auth_token; + if ( $self->config->{ 'auth-service-url' } ) { + $auth_token = Bio::KBase::AuthToken->new( + token => $token, + ignore_authrc => 1, + auth_svc =>$self->config->{ 'auth-service-url' } + ); } else { - $auth_token = Bio::KBase::AuthToken->new(token => $token, ignore_authrc => 1); + $auth_token = Bio::KBase::AuthToken->new( + token => $token, + ignore_authrc => 1 + ); } + my $valid = $auth_token->validate(); - my $valid = $auth_token->validate(); - # Only throw an exception if authentication was required and it fails - if ($method_auth eq 'required' && !$valid) - { - $self->exception('PerlError', "Token validation failed: " . $auth_token->error_message); - } elsif ($valid) { - $ctx->authenticated(1); - $ctx->user_id($auth_token->user_id); - $ctx->token( $token); - } + # Only throw an exception if authentication was required and it fails + if ( $method_auth eq 'required' && !$valid ) { + $self->exception( 'PerlError', + "Token validation failed: " + . $auth_token->error_message + ); + } + elsif ( $valid ) { + $ctx->authenticated( 1 ); + $ctx->user_id( $auth_token->user_id ); + $ctx->token( $token ); + } } } #else # Service ${service_name} does not require authentication. #end - my $new_isa = $self->get_package_isa($module); + my $new_isa = $self->get_package_isa( $module ); no strict 'refs'; local @{"${module}::ISA"} = @$new_isa; local $CallContext = $ctx; my @result; { - # + # # Process tag and metadata information if present. # - my $tag = $self->_plack_req_header("Kbrpc-Tag"); - if (!$tag) - { + my $tag = $self->_plack_req_header( "Kbrpc-Tag" ); + if ( !$tag ) { my ($t, $us) = &$get_time(); - $us = sprintf("%06d", $us); - my $ts = strftime("%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t); - $tag = "S:$self->{hostname}:$$:$ts"; + $us = sprintf( "%06d", $us ); + my $ts = strftime( "%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t ); + $tag = "S:$self->{hostname}:$$:$ts"; } - local $ENV${empty_escaper}{KBRPC_TAG} = $tag; - my $kb_metadata = $self->_plack_req_header("Kbrpc-Metadata"); - my $kb_errordest = $self->_plack_req_header("Kbrpc-Errordest"); - local $ENV${empty_escaper}{KBRPC_METADATA} = $kb_metadata if $kb_metadata; - local $ENV${empty_escaper}{KBRPC_ERROR_DEST} = $kb_errordest if $kb_errordest; - - my $stderr = ${stderr_pkg}->new($ctx, $get_time); - $ctx->stderr($stderr); - - my $xFF = $self->_plack_req_header("X-Forwarded-For"); - if ($xFF) { - $self->log($Bio::KBase::Log::INFO, $ctx, - "X-Forwarded-For: " . $xFF, $tag); + local $ENV${empty_escaper}{ KBRPC_TAG } = $tag; + my $kb_metadata = $self->_plack_req_header( "Kbrpc-Metadata" ); + my $kb_errordest = $self->_plack_req_header( "Kbrpc-Errordest" ); + local $ENV${empty_escaper}{ KBRPC_METADATA } = $kb_metadata if $kb_metadata; + local $ENV${empty_escaper}{ KBRPC_ERROR_DEST } = $kb_errordest if $kb_errordest; + + my $stderr = ${stderr_pkg}->new( $ctx, $get_time ); + $ctx->stderr( $stderr ); + + my $xFF = $self->_plack_req_header( "X-Forwarded-For" ); + if ( $xFF ) { + $self->log( + $Bio::KBase::Log::INFO, + $ctx, + "X-Forwarded-For: " . $xFF, + $tag + ); } - + my $err; eval { - $self->log($Bio::KBase::Log::INFO, $ctx, "start method", $tag); - local $SIG${empty_escaper}{__WARN__} = sub { - my($msg) = @_; - $stderr->log($msg); + $self->log( $Bio::KBase::Log::INFO, $ctx, "start method", $tag ); + local $SIG${empty_escaper}{ __WARN__ } = sub { + my ( $msg ) = @_; + $stderr->log( $msg ); print STDERR $msg; }; - @result = $module->$method(@{ $data->{arguments} }); - $self->log($Bio::KBase::Log::INFO, $ctx, "end method", $tag); + @result = $module->$method( @{ $data->{ arguments } } ); + $self->log( $Bio::KBase::Log::INFO, $ctx, "end method", $tag ); }; if ($@) { my $err = $@; - $stderr->log($err); - $ctx->stderr(undef); + $stderr->log( $err ); + $ctx->stderr( undef ); undef $stderr; - $self->log($Bio::KBase::Log::INFO, $ctx, "fail method", $tag); + $self->log( $Bio::KBase::Log::INFO, $ctx, "fail method", $tag ); my $nicerr; - if(ref($err) eq "Bio::KBase::Exceptions::KBaseException") { - $nicerr = {code => -32603, # perl error from RPC::Any::Exception - message => $err->error, - data => $err->trace->as_string, - context => $ctx - }; - } else { + + if ( ref( $err ) eq "Bio::KBase::Exceptions::KBaseException" ) { + $nicerr = { + code => -32603, # perl error from RPC::Any::Exception + message => $err->error, + data => $err->trace->as_string, + context => $ctx + }; + } + else { my $str = "$err"; - $str =~ s/Bio::KBase::CDMI::Service::call_method.*//s; # is this still necessary? not sure + # is this still necessary? not sure + $str =~ s/Bio::KBase::CDMI::Service::call_method.*//s; my $msg = $str; - $msg =~ s/ at [^\s]+.pm line \d+.\n$//; - $nicerr = {code => -32603, # perl error from RPC::Any::Exception - message => $msg, - data => $str, - context => $ctx - }; + $msg =~ s/ at [^\s]+.pm line \d+.\n$//; + $nicerr = { + code => -32603, # perl error from RPC::Any::Exception + message => $msg, + data => $str, + context => $ctx + }; } die $nicerr; } - $ctx->stderr(undef); + $ctx->stderr( undef ); undef $stderr; } my $result; - if ($return_counts${empty_escaper}{$method} == 1) - { - $result = [[$result[0]]]; + if ( $return_counts${empty_escaper}{ $method } == 1 ) { + $result = [ [ $result[ 0 ] ] ]; } - else - { + else { $result = \@result; } return $result; } sub _plack_req_header { - my ($self, $header_name) = @_; - if (defined($self->local_headers)) { - return $self->local_headers->{$header_name}; + my ( $self, $header_name ) = @_; + if ( defined( $self->local_headers ) ) { + return $self->local_headers->{ $header_name }; } - return $self->_plack_req->header($header_name); + return $self->_plack_req->header( $header_name ); } -sub get_method -{ - my ($self, $data) = @_; - - my $full_name = $data->{method}; - +sub get_method { + my ( $self, $data ) = @_; + + my $full_name = $data->{ method }; + $full_name =~ /^(\S+)\.([^\.]+)$/; - my ($package, $method) = ($1, $2); - - if (!$package || !$method) { - $self->exception('NoSuchMethod', - "'$full_name' is not a valid method. It must" - . " contain a package name, followed by a period," - . " followed by a method name."); + my ( $package, $method ) = ( $1, $2 ); + + if ( !$package || !$method ) { + $self->exception( 'NoSuchMethod', + "'$full_name' is not a valid method. It must" + . " contain a package name, followed by a period," + . " followed by a method name." + ); } - if (!$self->valid_methods->{$method}) - { - $self->exception('NoSuchMethod', - "'$method' is not a valid method in service ${service_name}."); + if ( !$self->valid_methods->{ $method } ) { + $self->exception( 'NoSuchMethod', + "'$method' is not a valid method in service ${service_name}." + ); } - - my $inst = $self->instance_dispatch->{$package}; + + my $inst = $self->instance_dispatch->{ $package }; my $module; - if ($inst) - { - $module = $inst; + if ( $inst ) { + $module = $inst; } - else - { - $module = $self->get_module($package); - if (!$module) { - $self->exception('NoSuchMethod', - "There is no method package named '$package'."); - } - - Class::MOP::load_class($module); + else { + $module = $self->get_module( $package ); + if ( ! $module ) { + $self->exception( 'NoSuchMethod', + "There is no method package named '$package'." + ); + } + + Class::MOP::load_class( $module ); } - if (!$module->can($method)) { - $self->exception('NoSuchMethod', - "There is no method named '$method' in the" - . " '$package' package."); + if ( !$module->can( $method ) ) { + $self->exception( 'NoSuchMethod', + "There is no method named '$method' in the" . " '$package' package." + ); } - + return { module => $module, method => $method, modname => $package }; } sub handle_input_cli { - my ($self, $input) = @_; + my ( $self, $input ) = @_; my $retval; eval { - #my $input_info = $self->check_input($input); - #my $input_object = $self->decode_input_to_object($input); - $self->parser->json->utf8(utf8::is_utf8($input) ? 0 : 1); - my $input_object = $self->parser->json_to_call($input); - my $data = $self->input_object_to_data($input_object); - #$self->fix_data($data, $input_info); - my $method_info = $self->get_method($data); - my $method_result = $self->call_method($data, $method_info); - my $collapsed_result = $self->collapse_result($method_result); - my $output_object = $self->output_data_to_object($collapsed_result); - $retval = $self->parser->return_to_json($output_object); + #my $input_info = $self->check_input( $input ); + #my $input_object = $self->decode_input_to_object( $input ); + $self->parser->json->utf8( utf8::is_utf8( $input ) ? 0 : 1 ); + my $input_object = $self->parser->json_to_call( $input ); + my $data = $self->input_object_to_data( $input_object ); + + #$self->fix_data( $data, $input_info ); + my $method_info = $self->get_method( $data ); + my $method_result = $self->call_method( $data, $method_info ); + my $collapsed_result = $self->collapse_result( $method_result ); + my $output_object = $self->output_data_to_object( $collapsed_result ); + $retval = $self->parser->return_to_json( $output_object ); }; + return $retval if defined $retval; + # The only way that we can get here is by something throwing an error. - return $self->handle_error_cli($@); + return $self->handle_error_cli( $@ ); } sub handle_error_cli { - my ($self, $error) = @_; + my ( $self, $error ) = @_; my $output; eval { my %error_params; - if (ref($error) eq 'HASH') { - %error_params = %{$error}; - if(defined($error_params${empty_escaper}{context})) { + if ( ref( $error ) eq 'HASH' ) { + %error_params = %$error; + if ( defined ( $error_params${empty_escaper}{ context } ) ) { # my @errlines; # $errlines[0] = $error_params${empty_escaper}{message}; # push @errlines, split("\n", $error_params${empty_escaper}{data}); - # $self->log($Bio::KBase::Log::ERR, $error_params${empty_escaper}{context}, \@errlines); - delete $error_params${empty_escaper}{context}; + # $self->log( $Bio::KBase::Log::ERR, $error_params${empty_escaper}{context}, \@errlines ); + delete $error_params${empty_escaper}{ context }; } - } else { - unless (blessed $error and $error->isa('RPC::Any::Exception')) { - $error = RPC::Any::Exception::PerlError->new(message => $error); + } + else { + unless ( blessed $error and $error->isa( 'RPC::Any::Exception' ) ) { + $error = RPC::Any::Exception::PerlError->new( message => $error ); } %error_params = ( message => $error->message, @@ -472,14 +523,16 @@ sub handle_error_cli { ); } my $json_error; - if ($self->_last_call) { - $json_error = $self->_last_call->return_error(%error_params); + if ( $self->_last_call ) { + $json_error = $self->_last_call->return_error( %error_params ); } else { - $json_error = $self->_default_error(%error_params); + $json_error = $self->_default_error( %error_params ); } - $output = $self->parser->return_to_json($json_error); + $output = $self->parser->return_to_json( $json_error ); }; + return $output if $output; + die "$error\n\nAlso, an error was encountered while trying to send" . " this error: $@\n"; } @@ -503,274 +556,241 @@ is available via $context->client_ip. use base 'Class::Accessor'; -__PACKAGE__->mk_accessors(qw(user_id client_ip authenticated token - module method call_id hostname stderr - provenance)); - -sub new -{ - my($class, $logger, %opts) = @_; - - my $self = { - %opts, +__PACKAGE__->mk_accessors( qw( + user_id client_ip authenticated token + module method call_id hostname stderr + provenance +) ); + + +sub new { + my ( $class, $logger, %opts ) = @_; + + my $self = { %opts, }; + chomp( $self->{ hostname } = `hostname` ); + $self->{ hostname } ||= 'unknown-host'; + $self->{ _logger } = $logger; + $self->{ _debug_levels } = { + 7 => 1, + 8 => 1, + 9 => 1, + DEBUG => 1, + DEBUG2 => 1, + DEBUG3 => 1, }; - chomp($self->{hostname} = `hostname`); - $self->{hostname} ||= 'unknown-host'; - $self->{_logger} = $logger; - $self->{_debug_levels} = {7 => 1, 8 => 1, 9 => 1, - 'DEBUG' => 1, 'DEBUG2' => 1, 'DEBUG3' => 1}; return bless $self, $class; } -sub _get_user -{ - my ($self) = @_; - return defined($self->user_id()) ? $self->user_id(): undef; +sub _get_user { + my ( $self ) = @_; + return defined( $self->user_id() ) ? $self->user_id() : undef; } -sub _log -{ - my ($self, $level, $message) = @_; - $self->{_logger}->log_message($level, $message, $self->_get_user(), - $self->module(), $self->method(), $self->call_id(), - $self->client_ip()); +sub _log { + my ( $self, $level, $message ) = @_; + $self->{ _logger }->log_message( $level, $message, $self->_get_user(), + $self->module(), $self->method(), $self->call_id(), $self->client_ip() ); } -sub log_err -{ - my ($self, $message) = @_; - $self->_log($Bio::KBase::Log::ERR, $message); +sub log_err { + my ( $self, $message ) = @_; + $self->_log( $Bio::KBase::Log::ERR, $message ); } -sub log_info -{ - my ($self, $message) = @_; - $self->_log($Bio::KBase::Log::INFO, $message); +sub log_info { + my ( $self, $message ) = @_; + $self->_log( $Bio::KBase::Log::INFO, $message ); } -sub log_debug -{ - my ($self, $message, $level) = @_; - if(!defined($level)) { +sub log_debug { + my ( $self, $message, $level ) = @_; + if ( !defined( $level ) ) { $level = 1; } - if($self->{_debug_levels}->{$level}) { - } else { - if ($level =~ /\D/ || $level < 1 || $level > 3) { + if ( $self->{ _debug_levels }->{ $level } ) { + } + else { + if ( $level =~ /\D/ || $level < 1 || $level > 3 ) { die "Invalid log level: $level"; } $level += 6; } - $self->_log($level, $message); + $self->_log( $level, $message ); } -sub set_log_level -{ - my ($self, $level) = @_; - $self->{_logger}->set_log_level($level); +sub set_log_level { + my ( $self, $level ) = @_; + $self->{ _logger }->set_log_level( $level ); } -sub get_log_level -{ - my ($self) = @_; - return $self->{_logger}->get_log_level(); +sub get_log_level { + my ( $self ) = @_; + return $self->{ _logger }->get_log_level(); } -sub clear_log_level -{ - my ($self) = @_; - $self->{_logger}->clear_user_log_level(); +sub clear_log_level { + my ( $self ) = @_; + $self->{ _logger }->clear_user_log_level(); } + package ${stderr_pkg}; use strict; use POSIX; use Time::HiRes 'gettimeofday'; -sub new -{ - my($class, $ctx, $get_time) = @_; - my $self = { - get_time => $get_time, - }; - my $dest = $ENV${empty_escaper}{KBRPC_ERROR_DEST} if exists $ENV${empty_escaper}{KBRPC_ERROR_DEST}; - my $tag = $ENV${empty_escaper}{KBRPC_TAG} if exists $ENV${empty_escaper}{KBRPC_TAG}; - my ($t, $us) = gettimeofday(); - $us = sprintf("%06d", $us); - my $ts = strftime("%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t); +sub new { + my ( $class, $ctx, $get_time ) = @_; + my $self = { get_time => $get_time, }; - my $name = join(".", $ctx->module, $ctx->method, $ctx->hostname, $ts); + my $dest = $ENV${empty_escaper}{ KBRPC_ERROR_DEST } + if exists $ENV${empty_escaper}{ KBRPC_ERROR_DEST }; + my $tag = $ENV${empty_escaper}{ KBRPC_TAG } + if exists $ENV${empty_escaper}{ KBRPC_TAG }; + my ( $t, $us ) = gettimeofday(); + $us = sprintf("%06d", $us); + my $ts = strftime("%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t); - if ($dest && $dest =~ m,^/,) - { - # - # File destination - # - my $fh; - - if ($tag) - { - $tag =~ s,/,_,g; - $dest = "$dest/$tag"; - if (! -d $dest) - { - mkdir($dest); - } - } - if (open($fh, ">", "$dest/$name")) - { - $self->{file} = "$dest/$name"; - $self->{dest} = $fh; - } - else - { - warn "Cannot open log file $dest/$name: $${empty_escaper}!"; - } - } - else - { - # - # Log to string. - # - my $stderr; - $self->{dest} = \$stderr; + my $name = join( ".", $ctx->module, $ctx->method, $ctx->hostname, $ts ); + + if ( $dest && $dest =~ m!^/! ) { + # + # File destination + # + my $fh; + if ($tag) { + $tag =~ s,/,_,g; + $dest = "$dest/$tag"; + mkdir($dest) if ! -d $dest; + } + + if ( open($fh, ">", "$dest/$name") ) { + $self->{file} = "$dest/$name"; + $self->{dest} = $fh; + } + else { + warn "Cannot open log file $dest/$name: $${empty_escaper}!"; + } + } + else { + # + # Log to string. + # + my $stderr; + $self->{ dest } = \$stderr; } - + bless $self, $class; - for my $e (sort { $a cmp $b } keys %ENV) - { - $self->log_cmd($e, $ENV${empty_escaper}{$e}); + for my $e ( sort { $a cmp $b } keys %ENV ) { + $self->log_cmd( $e, $ENV${empty_escaper}{ $e } ); } return $self; } -sub redirect -{ - my($self) = @_; - if ($self->{dest}) - { - return("2>", $self->{dest}); - } - else - { - return (); - } +sub redirect { + my ( $self ) = @_; + + return () unless $self->{ dest }; + + return ( "2>", $self->{ dest } ); } -sub redirect_both -{ - my($self) = @_; - if ($self->{dest}) - { - return(">&", $self->{dest}); - } - else - { - return (); - } +sub redirect_both { + my ( $self ) = @_; + + return () unless $self->{ dest }; + + return ( ">&", $self->{ dest } ); } -sub timestamp -{ - my($self) = @_; - my ($t, $us) = $self->{get_time}->(); - $us = sprintf("%06d", $us); - my $ts = strftime("%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t); +sub timestamp { + my ( $self ) = @_; + my ( $t, $us ) = $self->{ get_time }->(); + $us = sprintf( "%06d", $us ); + my $ts = strftime( "%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t ); return $ts; } -sub log -{ - my($self, $str) = @_; - my $d = $self->{dest}; +sub log { + my ( $self, $str ) = @_; + my $d = $self->{ dest }; my $ts = $self->timestamp(); - if (ref($d) eq 'SCALAR') - { - $$d .= "[$ts] " . $str . "\n"; - return 1; + if ( ref( $d ) eq 'SCALAR' ) { + $$d .= "[$ts] " . $str . "\n"; + return 1; } - elsif ($d) - { - print $d "[$ts] " . $str . "\n"; - return 1; + elsif ( $d ) { + print $d "[$ts] " . $str . "\n"; + return 1; } return 0; } -sub log_cmd -{ - my($self, @cmd) = @_; - my $d = $self->{dest}; +sub log_cmd { + my ( $self, @cmd ) = @_; + my $d = $self->{ dest }; my $str; my $ts = $self->timestamp(); - if (ref($cmd[0])) - { - $str = join(" ", @{$cmd[0]}); + if ( ref( $cmd[ 0 ] ) ) { + $str = join( " ", @{ $cmd[ 0 ] } ); } - else - { - $str = join(" ", @cmd); + else { + $str = join( " ", @cmd ); } - if (ref($d) eq 'SCALAR') - { - $$d .= "[$ts] " . $str . "\n"; + if ( ref( $d ) eq 'SCALAR' ) { + $$d .= "[$ts] " . $str . "\n"; } - elsif ($d) - { - print $d "[$ts] " . $str . "\n"; + elsif ( $d ) { + print $d "[$ts] " . $str . "\n"; } - + } -sub dest -{ - my($self) = @_; - return $self->{dest}; +sub dest { + my ( $self ) = @_; + return $self->{ dest }; } -sub text_value -{ - my($self) = @_; - if (ref($self->{dest}) eq 'SCALAR') - { - my $r = $self->{dest}; - return $$r; - } - else - { - return $self->{file}; +sub text_value { + my ( $self ) = @_; + if ( ref( $self->{ dest } ) eq 'SCALAR' ) { + my $r = $self->{ dest }; + return $$r; } + return $self->{ file }; } -unless (caller) { - my($input_file,$output_file,$token) = @ARGV; +unless ( caller ) { + my ( $input_file, $output_file, $token ) = @ARGV; my @dispatch; #foreach( $module in $modules ) { use ${module.impl_package_name}; my $obj = ${module.impl_package_name}->new; - push(@dispatch, '${module.module_name}' => $obj); + push( @dispatch, '${module.module_name}' => $obj ); } #end my %headers = ( - "Authorization" => $token, - "CLI" => "1" + Authorization => $token, + CLI => "1", ); my $server = ${server_package_name}->new( instance_dispatch => { @dispatch }, - allow_get => 0, - local_headers => \%headers); - open(my $fih, '<', $input_file) or die $${empty_escaper}!; + allow_get => 0, + local_headers => \%headers); + + open ( my $fih, '<', $input_file ) or die $${empty_escaper}!; my $input = ''; while (<$fih>) { $input .= $_; } close $fih; - my $output = $server->handle_input_cli($input); - open(my $foh, '>', $output_file) or die "Could not open file '$output_file' $${empty_escaper}!"; + my $output = $server->handle_input_cli( $input ); + open ( my $foh, '>', $output_file ) + or die "Could not open file '$output_file': $${empty_escaper}!"; print $foh $output; close $foh; } diff --git a/test_scripts/perl/lib/TestImpl.pm b/test_scripts/perl/lib/TestImpl.pm index 8efcf851..98b30226 100644 --- a/test_scripts/perl/lib/TestImpl.pm +++ b/test_scripts/perl/lib/TestImpl.pm @@ -1,8 +1,9 @@ package TestImpl; use strict; use Bio::KBase::Exceptions; + # Use Semantic Versioning (2.0.0-rc.1) -# http://semver.org +# http://semver.org our $VERSION = "0.1.0"; =head1 NAME @@ -18,18 +19,16 @@ Basic #BEGIN_HEADER #END_HEADER -sub new -{ - my($class, @args) = @_; - my $self = { - }; +sub new { + my ( $class, @args ) = @_; + my $self = {}; bless $self, $class; + #BEGIN_CONSTRUCTOR #END_CONSTRUCTOR - if ($self->can('_init_instance')) - { - $self->_init_instance(); + if ( $self->can( '_init_instance' ) ) { + $self->_init_instance(); } return $self; } @@ -74,37 +73,45 @@ $return is an int =cut -sub one_simple_param -{ +sub one_simple_param { my $self = shift; - my($val) = @_; + my ( $val ) = @_; my @_bad_arguments; - (!ref($val)) or push(@_bad_arguments, "Invalid type for argument \"val\" (value was \"$val\")"); - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to one_simple_param:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'one_simple_param'); + ( !ref( $val ) ) + or + push( @_bad_arguments, "Invalid type for argument \"val\" (value was \"$val\")" ); + if ( @_bad_arguments ) { + my $msg = "Invalid arguments passed to one_simple_param:\n" + . join( "", map { "\t$_\n" } @_bad_arguments ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'one_simple_param' + ); } my $ctx = $basicsrvServer::CallContext; - my($return); + my ( $return ); + #BEGIN one_simple_param $return = $val; + #END one_simple_param my @_bad_returns; - (!ref($return)) or push(@_bad_returns, "Invalid type for return variable \"return\" (value was \"$return\")"); - if (@_bad_returns) { - my $msg = "Invalid returns passed to one_simple_param:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'one_simple_param'); + ( !ref( $return ) ) + or push( @_bad_returns, + "Invalid type for return variable \"return\" (value was \"$return\")" ); + if ( @_bad_returns ) { + my $msg = "Invalid returns passed to one_simple_param:\n" + . join( "", map { "\t$_\n" } @_bad_returns ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'one_simple_param' + ); } - return($return); + return ( $return ); } - - - =head2 nothing $obj->nothing() @@ -137,19 +144,16 @@ sub one_simple_param =cut -sub nothing -{ +sub nothing { my $self = shift; my $ctx = $basicsrvServer::CallContext; + #BEGIN nothing #END nothing - return(); + return (); } - - - =head2 one_complex_param $return = $obj->one_complex_param($val2) @@ -208,37 +212,44 @@ double_number is a float =cut -sub one_complex_param -{ +sub one_complex_param { my $self = shift; - my($val2) = @_; + my ( $val2 ) = @_; my @_bad_arguments; - (ref($val2) eq 'HASH') or push(@_bad_arguments, "Invalid type for argument \"val2\" (value was \"$val2\")"); - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to one_complex_param:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'one_complex_param'); + ( ref( $val2 ) eq 'HASH' ) + or push( @_bad_arguments, + "Invalid type for argument \"val2\" (value was \"$val2\")" ); + if ( @_bad_arguments ) { + my $msg = "Invalid arguments passed to one_complex_param:\n" + . join( "", map { "\t$_\n" } @_bad_arguments ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'one_complex_param' + ); } my $ctx = $basicsrvServer::CallContext; - my($return); + my ( $return ); + #BEGIN one_complex_param - $return = {}; #$val2; - #END one_complex_param + $return = {}; #$val2; + #END one_complex_param my @_bad_returns; - (ref($return) eq 'HASH') or push(@_bad_returns, "Invalid type for return variable \"return\" (value was \"$return\")"); - if (@_bad_returns) { - my $msg = "Invalid returns passed to one_complex_param:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'one_complex_param'); + ( ref( $return ) eq 'HASH' ) + or push( @_bad_returns, + "Invalid type for return variable \"return\" (value was \"$return\")" ); + if ( @_bad_returns ) { + my $msg = "Invalid returns passed to one_complex_param:\n" + . join( "", map { "\t$_\n" } @_bad_returns ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'one_complex_param' + ); } - return($return); + return ( $return ); } - - - =head2 many_simple_params $return_1, $return_2, $return_3, $return_4 = $obj->many_simple_params($val1, $val2, $val3, $val4) @@ -295,46 +306,66 @@ $return_4 is a reference to a list containing 2 items: =cut -sub many_simple_params -{ +sub many_simple_params { my $self = shift; - my($val1, $val2, $val3, $val4) = @_; + my ( $val1, $val2, $val3, $val4 ) = @_; my @_bad_arguments; - (!ref($val1)) or push(@_bad_arguments, "Invalid type for argument \"val1\" (value was \"$val1\")"); - (!ref($val2)) or push(@_bad_arguments, "Invalid type for argument \"val2\" (value was \"$val2\")"); - (!ref($val3)) or push(@_bad_arguments, "Invalid type for argument \"val3\" (value was \"$val3\")"); - (ref($val4) eq 'ARRAY') or push(@_bad_arguments, "Invalid type for argument \"val4\" (value was \"$val4\")"); - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to many_simple_params:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'many_simple_params'); + ( !ref( $val1 ) ) + or push( @_bad_arguments, + "Invalid type for argument \"val1\" (value was \"$val1\")" ); + ( !ref( $val2 ) ) + or push( @_bad_arguments, + "Invalid type for argument \"val2\" (value was \"$val2\")" ); + ( !ref( $val3 ) ) + or push( @_bad_arguments, + "Invalid type for argument \"val3\" (value was \"$val3\")" ); + ( ref( $val4 ) eq 'ARRAY' ) + or push( @_bad_arguments, + "Invalid type for argument \"val4\" (value was \"$val4\")" ); + if ( @_bad_arguments ) { + my $msg = "Invalid arguments passed to many_simple_params:\n" + . join( "", map { "\t$_\n" } @_bad_arguments ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'many_simple_params' + ); } my $ctx = $basicsrvServer::CallContext; - my($return_1, $return_2, $return_3, $return_4); + my ( $return_1, $return_2, $return_3, $return_4 ); + #BEGIN many_simple_params $return_1 = $val1; $return_2 = $val2; $return_3 = $val3; $return_4 = $val4; + #END many_simple_params my @_bad_returns; - (!ref($return_1)) or push(@_bad_returns, "Invalid type for return variable \"return_1\" (value was \"$return_1\")"); - (!ref($return_2)) or push(@_bad_returns, "Invalid type for return variable \"return_2\" (value was \"$return_2\")"); - (!ref($return_3)) or push(@_bad_returns, "Invalid type for return variable \"return_3\" (value was \"$return_3\")"); - (ref($return_4) eq 'ARRAY') or push(@_bad_returns, "Invalid type for return variable \"return_4\" (value was \"$return_4\")"); - if (@_bad_returns) { - my $msg = "Invalid returns passed to many_simple_params:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'many_simple_params'); + ( !ref( $return_1 ) ) + or push( @_bad_returns, + "Invalid type for return variable \"return_1\" (value was \"$return_1\")" ); + ( !ref( $return_2 ) ) + or push( @_bad_returns, + "Invalid type for return variable \"return_2\" (value was \"$return_2\")" ); + ( !ref( $return_3 ) ) + or push( @_bad_returns, + "Invalid type for return variable \"return_3\" (value was \"$return_3\")" ); + ( ref( $return_4 ) eq 'ARRAY' ) + or push( @_bad_returns, + "Invalid type for return variable \"return_4\" (value was \"$return_4\")" ); + if ( @_bad_returns ) { + my $msg = "Invalid returns passed to many_simple_params:\n" + . join( "", map { "\t$_\n" } @_bad_returns ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'many_simple_params' + ); } - return($return_1, $return_2, $return_3, $return_4); + return ( $return_1, $return_2, $return_3, $return_4 ); } - - - =head2 many_complex_params $return_1, $return_2 = $obj->many_complex_params($simple_val, $complex_val) @@ -397,40 +428,52 @@ complex_struct is a reference to a hash where the following keys are defined: =cut -sub many_complex_params -{ +sub many_complex_params { my $self = shift; - my($simple_val, $complex_val) = @_; + my ( $simple_val, $complex_val ) = @_; my @_bad_arguments; - (ref($simple_val) eq 'HASH') or push(@_bad_arguments, "Invalid type for argument \"simple_val\" (value was \"$simple_val\")"); - (ref($complex_val) eq 'HASH') or push(@_bad_arguments, "Invalid type for argument \"complex_val\" (value was \"$complex_val\")"); - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to many_complex_params:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'many_complex_params'); + ( ref( $simple_val ) eq 'HASH' ) + or push( @_bad_arguments, + "Invalid type for argument \"simple_val\" (value was \"$simple_val\")" ); + ( ref( $complex_val ) eq 'HASH' ) + or push( @_bad_arguments, + "Invalid type for argument \"complex_val\" (value was \"$complex_val\")" ); + if ( @_bad_arguments ) { + my $msg = "Invalid arguments passed to many_complex_params:\n" + . join( "", map { "\t$_\n" } @_bad_arguments ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'many_complex_params' + ); } my $ctx = $basicsrvServer::CallContext; - my($return_1, $return_2); + my ( $return_1, $return_2 ); + #BEGIN many_complex_params $return_1 = $simple_val; $return_2 = $complex_val; + #END many_complex_params my @_bad_returns; - (ref($return_1) eq 'HASH') or push(@_bad_returns, "Invalid type for return variable \"return_1\" (value was \"$return_1\")"); - (ref($return_2) eq 'HASH') or push(@_bad_returns, "Invalid type for return variable \"return_2\" (value was \"$return_2\")"); - if (@_bad_returns) { - my $msg = "Invalid returns passed to many_complex_params:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'many_complex_params'); + ( ref( $return_1 ) eq 'HASH' ) + or push( @_bad_returns, + "Invalid type for return variable \"return_1\" (value was \"$return_1\")" ); + ( ref( $return_2 ) eq 'HASH' ) + or push( @_bad_returns, + "Invalid type for return variable \"return_2\" (value was \"$return_2\")" ); + if ( @_bad_returns ) { + my $msg = "Invalid returns passed to many_complex_params:\n" + . join( "", map { "\t$_\n" } @_bad_returns ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'many_complex_params' + ); } - return($return_1, $return_2); + return ( $return_1, $return_2 ); } - - - =head2 with_auth $return = $obj->with_auth($val1) @@ -467,38 +510,46 @@ $return is an int =cut -sub with_auth -{ +sub with_auth { my $self = shift; - my($val1) = @_; + my ( $val1 ) = @_; my @_bad_arguments; - (!ref($val1)) or push(@_bad_arguments, "Invalid type for argument \"val1\" (value was \"$val1\")"); - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to with_auth:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'with_auth'); + ( !ref( $val1 ) ) + or push( @_bad_arguments, + "Invalid type for argument \"val1\" (value was \"$val1\")" ); + if ( @_bad_arguments ) { + my $msg = "Invalid arguments passed to with_auth:\n" + . join( "", map { "\t$_\n" } @_bad_arguments ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'with_auth' + ); } my $ctx = $basicsrvServer::CallContext; - my($return); + my ( $return ); + #BEGIN with_auth $return = $val1; + #END with_auth my @_bad_returns; - (!ref($return)) or push(@_bad_returns, "Invalid type for return variable \"return\" (value was \"$return\")"); - if (@_bad_returns) { - my $msg = "Invalid returns passed to with_auth:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'with_auth'); + ( !ref( $return ) ) + or push( @_bad_returns, + "Invalid type for return variable \"return\" (value was \"$return\")" ); + if ( @_bad_returns ) { + my $msg = "Invalid returns passed to with_auth:\n" + . join( "", map { "\t$_\n" } @_bad_returns ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'with_auth' + ); } - return($return); + return ( $return ); } - - - -=head2 version +=head2 version $return = $obj->version() @@ -666,3 +717,4 @@ large_prop4 has a value which is a reference to a hash where the key is a string =cut 1; + diff --git a/test_scripts/perl/run-client.pl b/test_scripts/perl/run-client.pl index 08304d91..46a9d6e9 100644 --- a/test_scripts/perl/run-client.pl +++ b/test_scripts/perl/run-client.pl @@ -1,9 +1,6 @@ # # Automatic test rig for KBase perl client generation. # -# -# -# use strict; use warnings; @@ -11,18 +8,17 @@ use Data::Dumper; use Getopt::Long; -my $DESCRIPTION = -" +my $DESCRIPTION = " USAGE perl test-client.pl [options] - + DESCRIPTION Conducts an automatic test of a perl client against the specified endpoint using methods and parameters indicated in the tests config file. - + --endpoint [url] endpoint of the server to test --token [token] token for testing authenticated calls - --asyncchecktime [ms] time client waits every cycle of checking state of async methods + --asyncchecktime [ms] time client waits every cycle of checking state of async methods --package [package] package with client module --method [method] method to run --input [filename] input file with parameters in JSON @@ -30,7 +26,7 @@ --error [filename] output file with text error message -h, --help display this help message, ignore all arguments "; - + # first parse options to get the testconfig file my $verbose; my $endpoint; @@ -45,104 +41,109 @@ my $help; -my $opt = GetOptions ( - "verbose|v" => \$verbose, - "endpoint=s" => \$endpoint, - "token=s" => \$token, - "asyncchecktime=i" => \$async_job_check_time_ms, - "help|h" => \$help, - "package=s" => \$client_module, - "method=s" => \$method, - "input=s" => \$input_filename, - "output=s" => \$output_filename, - "error=s" => \$error_filename, - ); - -if ($help) { - print $DESCRIPTION."\n"; +my $opt = GetOptions( + "verbose|v" => \$verbose, + "endpoint=s" => \$endpoint, + "token=s" => \$token, + "asyncchecktime=i" => \$async_job_check_time_ms, + "help|h" => \$help, + "package=s" => \$client_module, + "method=s" => \$method, + "input=s" => \$input_filename, + "output=s" => \$output_filename, + "error=s" => \$error_filename, +); + +if ( $help ) { + print $DESCRIPTION. "\n"; exit 0; } - # endpoint must be defined -if (!$endpoint) { - fail("endpoint parameter must be defined"); +if ( !$endpoint ) { + fail( "endpoint parameter must be defined" ); exit 1; } # package must be defined -if (!$client_module) { - fail("package parameter must be defined"); +if ( !$client_module ) { + fail( "package parameter must be defined" ); exit 1; } # method must be defined -if (!$method) { - fail("method parameter must be defined"); +if ( !$method ) { + fail( "method parameter must be defined" ); exit 1; } # input must be defined -if (!$input_filename) { - fail("input parameter must be defined"); +if ( !$input_filename ) { + fail( "input parameter must be defined" ); exit 1; } # output must be defined -if (!$output_filename) { - fail("output parameter must be defined"); +if ( !$output_filename ) { + fail( "output parameter must be defined" ); exit 1; } # error must be defined -if (!$error_filename) { - fail("error parameter must be defined"); +if ( !$error_filename ) { + fail( "error parameter must be defined" ); exit 1; } #parse the tests -open(my $fh, "<", $input_filename); -my $input_string=''; -while (my $line = <$fh>) { +open( my $fh, "<", $input_filename ); +my $input_string = ''; +while ( my $line = <$fh> ) { chomp $line; $input_string .= $line; } -close($fh); -my $params = JSON->new->decode($input_string); +close( $fh ); +my $params = JSON->new->decode( $input_string ); my $json = JSON->new->canonical; -eval("use $client_module;"); +eval( "use $client_module;" ); + #instantiate an authenticated client and a nonauthenticated client my $client; -if($token) { - if ($async_job_check_time_ms) { - $client=$client_module->new($endpoint, token=>$token, async_job_check_time_ms=>$async_job_check_time_ms); - } else { - $client=$client_module->new($endpoint, token=>$token); +if ( $token ) { + if ( $async_job_check_time_ms ) { + $client = $client_module->new( + $endpoint, + token => $token, + async_job_check_time_ms => $async_job_check_time_ms + ); + } + else { + $client = $client_module->new( $endpoint, token => $token ); } -} else { - $client = $client_module->new($endpoint,ignore_kbase_config=>1); +} +else { + $client = $client_module->new( $endpoint, ignore_kbase_config => 1 ); } # loop over each test case and run it against the server. We create a new client instance # for each test - + my @result; no strict "refs"; -eval { - @result = $client->$method(@{$params}); -}; -if($@) { +eval { @result = $client->$method( @{ $params } ); }; +if ( $@ ) { print "caught: $@"; my $returnedErrorMsg = "$@"; my $OUTFILE; open $OUTFILE, '>>', $error_filename; print { $OUTFILE } $returnedErrorMsg; close $OUTFILE; -} else { - my $serialized_result = $json->encode(\@result); +} +else { + my $serialized_result = $json->encode( \@result ); my $OUTFILE; open $OUTFILE, '>>', $output_filename; print { $OUTFILE } $serialized_result; @@ -150,6 +151,3 @@ } - - - diff --git a/test_scripts/perl/test-client.pl b/test_scripts/perl/test-client.pl index 8d6769b4..db3caa3e 100644 --- a/test_scripts/perl/test-client.pl +++ b/test_scripts/perl/test-client.pl @@ -1,9 +1,6 @@ # # Automatic test rig for KBase perl client generation. # -# -# -# use strict; use warnings; @@ -12,22 +9,21 @@ use Test::More; use Getopt::Long; -my $DESCRIPTION = -" +my $DESCRIPTION = " USAGE perl test-client.pl [options] - + DESCRIPTION Conducts an automatic test of a perl client against the specified endpoint using methods and parameters indicated in the tests config file. - + --tests [filename] config file for tests --endpoint [url] endpoint of the server to test --token [token] token for testing authenticated calls - --asyncchecktime [ms] time client waits every cycle of checking state of async methods + --asyncchecktime [ms] time client waits every cycle of checking state of async methods -h, --help display this help message, ignore all arguments "; - + # first parse options to get the testconfig file my $tests_filename; my $verbose; @@ -38,202 +34,224 @@ my $help; -my $opt = GetOptions ( - "tests=s" => \$tests_filename, - "verbose|v" => \$verbose, - "endpoint=s" => \$endpoint, - "token=s" => \$token, - "asyncchecktime=i" => \$async_job_check_time_ms, - "help|h" => \$help, - ); +my $opt = GetOptions( + "tests=s" => \$tests_filename, + "verbose|v" => \$verbose, + "endpoint=s" => \$endpoint, + "token=s" => \$token, + "asyncchecktime=i" => \$async_job_check_time_ms, + "help|h" => \$help, +); -if ($help) { - print $DESCRIPTION."\n"; +if ( $help ) { + print $DESCRIPTION. "\n"; exit 0; } - # endpoint must be defined -if (!$endpoint) { - fail("endpoint parameter must be defined"); +if ( !$endpoint ) { + fail( "endpoint parameter must be defined" ); done_testing(); exit 1; } # tests must be defined -if (!$tests_filename) { - fail("tests parameter must be defined"); +if ( !$tests_filename ) { + fail( "tests parameter must be defined" ); done_testing(); exit 1; } #parse the tests -open(my $fh, "<", $tests_filename); -my $tests_string=''; -while (my $line = <$fh>) { +open( my $fh, "<", $tests_filename ); +my $tests_string = ''; +while ( my $line = <$fh> ) { chomp $line; $tests_string .= $line; } -close($fh); -my $tests_json = JSON->new->decode($tests_string); +close( $fh ); +my $tests_json = JSON->new->decode( $tests_string ); -my $tests = $tests_json->{tests}; -my $client_module = $tests_json->{package}; +my $tests = $tests_json->{ tests }; +my $client_module = $tests_json->{ package }; -if (!$tests) { - fail("tests array not defined in test config file"); +if ( !$tests ) { + fail( "tests array not defined in test config file" ); done_testing(); exit 1; } -if (!$client_module) { - fail("client module not defined in test config file"); +if ( !$client_module ) { + fail( "client module not defined in test config file" ); done_testing(); exit 1; } # make sure we can import the module my $json = JSON->new->canonical; -use_ok($client_module); +use_ok( $client_module ); #instantiate an authenticated client and a nonauthenticated client -my $nonauthenticated_client = $client_module->new($endpoint,ignore_kbase_config=>1); -ok(defined($nonauthenticated_client),"instantiating nonauthenticated client"); +my $nonauthenticated_client = $client_module->new( $endpoint, ignore_kbase_config => 1 ); +ok( defined( $nonauthenticated_client ), "instantiating nonauthenticated client" ); my $authenticated_client; -if($token) { - if ($async_job_check_time_ms) { - $authenticated_client=$client_module->new($endpoint, token=>$token, async_job_check_time_ms=>$async_job_check_time_ms); - } else { - $authenticated_client=$client_module->new($endpoint, token=>$token); +if ( $token ) { + if ( $async_job_check_time_ms ) { + $authenticated_client = $client_module->new( + $endpoint, + token => $token, + async_job_check_time_ms => $async_job_check_time_ms + ); + } + else { + $authenticated_client = $client_module->new( $endpoint, token => $token ); } - ok(defined($authenticated_client),"instantiating authenticated client"); + ok( defined( $authenticated_client ), "instantiating authenticated client" ); } # loop over each test case and run it against the server. We create a new client instance # for each test -foreach my $test (@{$tests}) { +foreach my $test ( @{ $tests } ) { my $client; - if ($test->{'auth'}) { - if ($authenticated_client) { + if ( $test->{ 'auth' } ) { + if ( $authenticated_client ) { $client = $authenticated_client; - } else { - fail("authenticated call declared, but no user and password set"); + } + else { + fail( "authenticated call declared, but no user and password set" ); done_testing(); exit 1; } - - } else { + + } + else { $client = $nonauthenticated_client; } - ok(defined($client),"instantiating client"); - - my $method = $test->{method}; - my $params = $test->{params}; - my $outcome = $test->{outcome}; + ok( defined( $client ), "instantiating client" ); + + my $method = $test->{ method }; + my $params = $test->{ params }; + my $outcome = $test->{ outcome }; my $useScalarContext; - if ($test->{context} && $test->{context} eq 'scalar') { - $useScalarContext=1; + if ( $test->{ context } && $test->{ context } eq 'scalar' ) { + $useScalarContext = 1; } - - ok($client->can($method), ' ----------------- method "'.$method.'" exists ------------------------'); - if ($client->can($method)) { - my (@result, $scalarResult, $failed); + + ok( $client->can( $method ), + ' ----------------- method "' . $method . '" exists ------------------------' ); + if ( $client->can( $method ) ) { + my ( @result, $scalarResult, $failed ); { no strict "refs"; eval { - if ($useScalarContext) { - $scalarResult = $client->$method(@{$params}); - } else { - @result = $client->$method(@{$params}); + if ( $useScalarContext ) { + $scalarResult = $client->$method( @{ $params } ); + } + else { + @result = $client->$method( @{ $params } ); } }; - if($@) { + if ( $@ ) { $failed = 1; - if ($outcome->{status} eq 'fail') { - pass('expected failure, and yes it failed'); - } else { - fail('did not expect to fail, but it did'); - print STDERR "Failing test of '$method', expected to pass but error thrown:\n"; - print STDERR $@->{message}."\n"; - if(defined($@->{status_line})) {print STDERR $@->{status_line}."\n" }; + if ( $outcome->{ status } eq 'fail' ) { + pass( 'expected failure, and yes it failed' ); + } + else { + fail( 'did not expect to fail, but it did' ); + print STDERR + "Failing test of '$method', expected to pass but error thrown:\n"; + print STDERR $@->{ message } . "\n"; + if ( defined( $@->{ status_line } ) ) { + print STDERR $@->{ status_line } . "\n"; + } } - + # check that the error message contains the right message - if ($outcome->{error}) { - my $returnedErrorMsg = $@->{message}; - foreach my $e (@{$outcome->{error}}) { - if(index($returnedErrorMsg, $e) != -1){ - pass("returned error message contains $e") - } else { - fail("returned error message did not contain $e"); - print STDERR "Failing test of '$method', expected error to contain message, but it didn't:\n"; + if ( $outcome->{ error } ) { + my $returnedErrorMsg = $@->{ message }; + foreach my $e ( @{ $outcome->{ error } } ) { + if ( index( $returnedErrorMsg, $e ) != -1 ) { + pass( "returned error message contains $e" ); + } + else { + fail( "returned error message did not contain $e" ); + print STDERR + "Failing test of '$method', expected error to contain message, but it didn't:\n"; print STDERR " looking for: '$e'\n"; print STDERR " got: '$returnedErrorMsg'\n"; } } } - + # could do more checks here for different failure modes - + } } - if ($outcome->{status} eq 'pass') { - pass('expected to run successfully, and it did'); - my ($serialized_params, $serialized_result); - if ($useScalarContext) { - $serialized_params = $json->encode([@{$params}[0]]); - $serialized_result = $json->encode([$scalarResult]); - } else { - $serialized_params = $json->encode($params); - $serialized_result = $json->encode(\@result); + if ( $outcome->{ status } eq 'pass' ) { + pass( 'expected to run successfully, and it did' ); + my ( $serialized_params, $serialized_result ); + if ( $useScalarContext ) { + $serialized_params = $json->encode( [ @{ $params }[ 0 ] ] ); + $serialized_result = $json->encode( [ $scalarResult ] ); + } + else { + $serialized_params = $json->encode( $params ); + $serialized_result = $json->encode( \@result ); } - - ok($serialized_params eq $serialized_result,"response matches input parameters"); - - if ($serialized_params ne $serialized_result) { - print STDERR "Failing test of '$method', expected input to match output, but they don't match:\n"; - print STDERR " in: ".$serialized_params."\n"; - print STDERR " out: ".$serialized_result."\n"; + + ok( + $serialized_params eq $serialized_result, + "response matches input parameters" + ); + + if ( $serialized_params ne $serialized_result ) { + print STDERR + "Failing test of '$method', expected input to match output, but they don't match:\n"; + print STDERR " in: " . $serialized_params . "\n"; + print STDERR " out: " . $serialized_result . "\n"; print STDERR "\n"; } - } elsif ($outcome->{status} eq 'nomatch') { - pass('expected to run successfully, and it did'); - my ($serialized_params, $serialized_result); - if ($useScalarContext) { - ok($scalarResult,"recieved a response in scalar context"); - $serialized_params = $json->encode([@{$params}[0]]); - $serialized_result = $json->encode([$scalarResult]); - } else { - ok(@result,"recieved a response in array context"); - $serialized_params = $json->encode($params); - $serialized_result = $json->encode(\@result); + } + elsif ( $outcome->{ status } eq 'nomatch' ) { + pass( 'expected to run successfully, and it did' ); + my ( $serialized_params, $serialized_result ); + if ( $useScalarContext ) { + ok( $scalarResult, "recieved a response in scalar context" ); + $serialized_params = $json->encode( [ @{ $params }[ 0 ] ] ); + $serialized_result = $json->encode( [ $scalarResult ] ); + } + else { + ok( @result, "recieved a response in array context" ); + $serialized_params = $json->encode( $params ); + $serialized_result = $json->encode( \@result ); } - - ok($serialized_params ne $serialized_result,"response does NOT match input parameters"); - - if ($serialized_params eq $serialized_result) { - print STDERR "Failing test of '$method', expected input to NOT match output, but they did match:\n"; - print STDERR " in/out: ".$serialized_params."\n"; + + ok( + $serialized_params ne $serialized_result, + "response does NOT match input parameters" + ); + + if ( $serialized_params eq $serialized_result ) { + print STDERR + "Failing test of '$method', expected input to NOT match output, but they did match:\n"; + print STDERR " in/out: " . $serialized_params . "\n"; print STDERR "\n"; } - } elsif ($outcome->{status} eq 'fail') { - if (!$failed) { - fail('expected to fail, but it ran successfully'); + } + elsif ( $outcome->{ status } eq 'fail' ) { + if ( !$failed ) { + fail( 'expected to fail, but it ran successfully' ); } - } else { - fail('expected outcome set to "'.$outcome->{status}.'", but that is not recognized. Outcome can only be: pass | fail | nomatch'); + } + else { + fail( 'expected outcome set to "' + . $outcome->{ status } + . '", but that is not recognized. Outcome can only be: pass | fail | nomatch' + ); } } } - - - - done_testing(); - - - - From ad6b08988e2246dc64d0b59e1c6bf2ec3eabe3e1 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Fri, 18 Sep 2020 08:12:00 -0700 Subject: [PATCH 24/47] Perl changes: - add use strict/use warnings to all code - minor reformatting and whitespace changes from perltidy - remove $PATH from the $PERL5LIB env var: perl will not find any perl libraries in $PATH, which is for executables - use "say" instead of "print" in the demo perl app - fix class initiation to use Class::Name->new instead of new Class::Name - match module_perl_impl template to results from running make --- lib/Bio/KBase/DeploymentConfig.pm | 3 +- lib/Bio/KBase/Exceptions.pm | 5 + lib/Bio/KBase/Log.pm | 2 +- .../mobu/initializer/ModuleInitializer.java | 2 +- .../kbase/scripts/test/Test8.perl.properties | 3 + .../templates/module_dockerfile.vm.properties | 9 +- .../templates/module_makefile.vm.properties | 6 +- .../templates/module_perl_impl.vm.properties | 110 ++++++-------- .../module_test_perl_client.vm.properties | 13 +- .../kbase/templates/perl_client.vm.properties | 141 +++++++++++------- .../kbase/templates/perl_impl.vm.properties | 38 ++--- .../kbase/templates/perl_psgi.vm.properties | 16 +- .../kbase/templates/perl_server.vm.properties | 6 +- 13 files changed, 186 insertions(+), 168 deletions(-) diff --git a/lib/Bio/KBase/DeploymentConfig.pm b/lib/Bio/KBase/DeploymentConfig.pm index 8faa7a31..8e9d0794 100644 --- a/lib/Bio/KBase/DeploymentConfig.pm +++ b/lib/Bio/KBase/DeploymentConfig.pm @@ -1,6 +1,8 @@ package Bio::KBase::DeploymentConfig; use strict; +use warnings; + use base 'Class::Accessor'; use Config::Simple; @@ -103,4 +105,3 @@ Return the name of the service currently configured. =cut 1; - diff --git a/lib/Bio/KBase/Exceptions.pm b/lib/Bio/KBase/Exceptions.pm index d09411fe..37a9a5ef 100644 --- a/lib/Bio/KBase/Exceptions.pm +++ b/lib/Bio/KBase/Exceptions.pm @@ -1,5 +1,8 @@ package Bio::KBase::Exceptions; +use strict; +use warnings; + use Exception::Class ( Bio::KBase::Exceptions::KBaseException => { description => 'KBase exception', @@ -29,6 +32,7 @@ Bio::KBase::Exceptions::KBaseException->Trace( 1 ); package Bio::KBase::Exceptions::HTTP; use strict; +use warnings; sub full_message { my ( $self ) = @_; @@ -42,6 +46,7 @@ sub full_message { package Bio::KBase::Exceptions::JSONRPC; use strict; +use warnings; sub full_message { my ( $self ) = @_; diff --git a/lib/Bio/KBase/Log.pm b/lib/Bio/KBase/Log.pm index 14f20b81..68b41eda 100644 --- a/lib/Bio/KBase/Log.pm +++ b/lib/Bio/KBase/Log.pm @@ -431,7 +431,7 @@ sub _syslog { sub _log { my ( $self, $ident, $message ) = @_; my $time = POSIX::strftime( "%Y-%m-%d %H:%M:%S", localtime ); - my $msg = join( " ", $time, hostname(), $ident . ": " ); + my $msg = join " ", $time, hostname(), $ident . ": "; open LOG, ">>" . $self->get_log_file() || warn "Could not print log message $msg to " . $self->get_log_file() . "\n"; if ( ref( $message ) eq 'ARRAY' ) { diff --git a/src/java/us/kbase/mobu/initializer/ModuleInitializer.java b/src/java/us/kbase/mobu/initializer/ModuleInitializer.java index f8bb566a..4ae97072 100644 --- a/src/java/us/kbase/mobu/initializer/ModuleInitializer.java +++ b/src/java/us/kbase/mobu/initializer/ModuleInitializer.java @@ -25,7 +25,7 @@ public class ModuleInitializer { private boolean verbose; private File dir; - private static String[] subdirs = {"data", + private static String[] subdirs = { "data", "scripts", "lib", "test", diff --git a/src/java/us/kbase/scripts/test/Test8.perl.properties b/src/java/us/kbase/scripts/test/Test8.perl.properties index 68e90e6b..e9db58b0 100644 --- a/src/java/us/kbase/scripts/test/Test8.perl.properties +++ b/src/java/us/kbase/scripts/test/Test8.perl.properties @@ -1,5 +1,8 @@ package StoringImpl; + use strict; +use warnings; + use Bio::KBase::Exceptions; # Use Semantic Versioning (2.0.0-rc.1) # http://semver.org diff --git a/src/java/us/kbase/templates/module_dockerfile.vm.properties b/src/java/us/kbase/templates/module_dockerfile.vm.properties index ece8ea3c..2d41c79a 100644 --- a/src/java/us/kbase/templates/module_dockerfile.vm.properties +++ b/src/java/us/kbase/templates/module_dockerfile.vm.properties @@ -3,7 +3,6 @@ FROM kbase/sdkbase2:python #else FROM kbase/sdkbase2:latest #end -MAINTAINER #if($username)${username}#{else}KBase Developer#{end} # ----------------------------------------- # In this section, you can install any system dependencies required @@ -34,15 +33,13 @@ RUN R -q -e 'if(!require(raster)) install.packages("raster", repos="http://cran. RUN apt-get -y install r-cran-evaluate r-cran-codetools r-cran-testthat #end -# ----------------------------------------- - COPY ./ /kb/module -RUN mkdir -p /kb/module/work -RUN chmod -R a+rw /kb/module WORKDIR /kb/module -RUN make all +RUN mkdir -p /kb/module/work && \ + chmod -R a+rw /kb/module && \ + make all ENTRYPOINT [ "./scripts/entrypoint.sh" ] diff --git a/src/java/us/kbase/templates/module_makefile.vm.properties b/src/java/us/kbase/templates/module_makefile.vm.properties index e0ddc179..7b96c7dc 100644 --- a/src/java/us/kbase/templates/module_makefile.vm.properties +++ b/src/java/us/kbase/templates/module_makefile.vm.properties @@ -63,7 +63,7 @@ build-executable-script: #if($language == "perl") echo '#!/bin/bash' > $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME) echo 'script_dir=$$(dirname "$$(readlink -f "$$0")")' >> $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME) - echo 'export PERL5LIB=$$script_dir/../$(LIB_DIR):$$PATH:$$PERL5LIB' >> $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME) + echo 'export PERL5LIB=$$script_dir/../$(LIB_DIR):$$PERL5LIB' >> $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME) echo 'perl $$script_dir/../$(LIB_DIR)/$(SERVICE_CAPS)/$(SERVICE_CAPS)Server.pm $$1 $$2 $$3' >> $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME) #end #if($language == "python") @@ -89,7 +89,7 @@ build-startup-script: echo 'script_dir=$$(dirname "$$(readlink -f "$$0")")' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME) #if($language == "perl") echo 'export KB_DEPLOYMENT_CONFIG=$$script_dir/../deploy.cfg' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME) - echo 'export PERL5LIB=$$script_dir/../$(LIB_DIR):$$PATH:$$PERL5LIB' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME) + echo 'export PERL5LIB=$$script_dir/../$(LIB_DIR):$$PERL5LIB' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME) echo 'plackup $$script_dir/../$(LIB_DIR)/$(SERVICE_CAPS).psgi' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME) #end #if($language == "python") @@ -117,7 +117,7 @@ build-test-script: echo 'rm -rf $(WORK_DIR)/*' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) echo 'echo "...done removing temp files."' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) #if($language == "perl") - echo 'export PERL5LIB=$$script_dir/../$(LIB_DIR):$$PATH:$$PERL5LIB' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) + echo 'export PERL5LIB=$$script_dir/../$(LIB_DIR):$$PERL5LIB' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) echo 'cd $$script_dir/../$(TEST_DIR)' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) echo "perl -e 'opendir my ${esc.print("\")}${esc.print("$")}${esc.print("$")}dh, \".\"; my @l = grep { /\\\\.pl${esc.print("\")}${esc.print("$")}${esc.print("$")}/ } readdir ${esc.print("\")}${esc.print("$")}${esc.print("$")}dh; my ${esc.print("\")}${esc.print("$")}${esc.print("$")}ret=0; foreach my ${esc.print("\")}${esc.print("$")}${esc.print("$")}s (@l) { print(\"Running \".${esc.print("\")}${esc.print("$")}${esc.print("$")}s.\"\\\\n\"); system(\"perl\", ${esc.print("\")}${esc.print("$")}${esc.print("$")}s) && do{${esc.print("\")}${esc.print("$")}${esc.print("$")}ret=2; next;}; } exit(${esc.print("\")}${esc.print("$")}${esc.print("$")}ret);'" >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) #end diff --git a/src/java/us/kbase/templates/module_perl_impl.vm.properties b/src/java/us/kbase/templates/module_perl_impl.vm.properties index 09a06d81..6b60ce55 100644 --- a/src/java/us/kbase/templates/module_perl_impl.vm.properties +++ b/src/java/us/kbase/templates/module_perl_impl.vm.properties @@ -1,6 +1,8 @@ -#if ($example) package ${module_name}::${module_name}Impl; + use strict; +use warnings; + use Bio::KBase::Exceptions; # Use Semantic Versioning (2.0.0-rc.1) # http://semver.org @@ -20,22 +22,26 @@ This sample module contains one small method that filters contigs. =cut #BEGIN_HEADER +use feature qw( say ); use Bio::KBase::AuthToken; -use installed_clients::AssemblyUtilClient; use installed_clients::KBaseReportClient; use Config::IniFiles; +#if ($example) +use installed_clients::AssemblyUtilClient; use Bio::SeqIO; use Data::Dumper; +#end #END_HEADER sub new { my ( $class, @args ) = @_; my $self = {}; bless $self, $class; + #BEGIN_CONSTRUCTOR my $config_file = $ENV{ KB_DEPLOYMENT_CONFIG }; - my $cfg = Config::IniFiles->new(-file=>$config_file); + my $cfg = Config::IniFiles->new( -file => $config_file ); my $scratch = $cfg->val('${module_name}', 'scratch'); my $callbackURL = $ENV{ SDK_CALLBACK_URL }; @@ -44,8 +50,7 @@ sub new { #END_CONSTRUCTOR - if ($self->can('_init_instance')) - { + if ( $self->can( '_init_instance' ) ) { $self->_init_instance(); } return $self; @@ -54,7 +59,6 @@ sub new { =head1 METHODS - =head2 run_${module_name} $output = $obj->run_${module_name}($params) @@ -74,11 +78,13 @@ sub run_${module_name} { my $self = shift; my ( $params ) = @_; - my @_bad_arguments; - ( ref($params) eq 'HASH' ) or push( @_bad_arguments, "Invalid type for argument \"params\" (value was \"$params\")" ); - if (@_bad_arguments) { + my @bad_arguments; + ( ref( $params ) eq 'HASH' ) or push @_bad_arguments, + "Invalid type for argument \"params\" " + . "(value was \"$params\")"; + if ( @bad_arguments ) { my $msg = "Invalid arguments passed to run_${module_name}:\n" - . join( "", map { "\t$_\n" } @_bad_arguments ); + . join "", map { "\t$_\n" } @bad_arguments; Bio::KBase::Exceptions::ArgumentValidationError->throw( error => $msg, method_name => 'run_${module_name}' @@ -86,12 +92,11 @@ sub run_${module_name} { } my $ctx = $${module_name}::${module_name}Server::CallContext; - my $output; + my ( $output ); #BEGIN run_${module_name} - +#if ($example) # Print statements to stdout/stderr are captured and available as the App log - print("Starting run_${module_name} method. Parameters:\n"); - print(Dumper($params) . "\n"); + say "Starting run_${module_name} method. Parameters: " . Dumper $params; # Step 1 - Parse/examine the parameters and catch any errors # It is important to check that parameters exist and are defined, and that nice error @@ -121,7 +126,7 @@ sub run_${module_name} { # We can use the AssemblyUtils module to download a FASTA file from our Assembly data # object. The return object gives us the path to the file that was created. - print("Downloading assembly data as FASTA file.\n"); + say "Downloading assembly data as FASTA file."; my $assycli = installed_clients::AssemblyUtilClient->new( $self->{ callbackURL } ); my $fileobj = $assycli->get_assembly_as_fasta( { ref => $assy_ref } ); @@ -141,7 +146,7 @@ sub run_${module_name} { } } my $result_text = "Filtered assembly to " . $remaining . " contigs out of " . $total; - print($result_text . "\n"); + say $result_text; # Step 4 - Save the new Assembly back to the system my $newref = $assycli->save_assembly_from_fasta( { @@ -168,7 +173,7 @@ sub run_${module_name} { # Step 6 - construct the output to send back - my $output = { + $output = { assembly_output => $newref, n_initial_contigs => $total, n_contigs_remaining => $remaining, @@ -177,20 +182,36 @@ sub run_${module_name} { report_ref => $report->{ ref } }; - print("returning: ".Dumper($output)."\n"); + say "returning: " . Dumper $output; +#else + my $repcli = installed_clients::KBaseReportClient->new( $self->{ callbackURL } ); + my $report = $repcli->create( { + workspace_name => $params->{ workspace_name }, + report => { + text_message => $params->{ parameter_1 }, + objects_created => [] + } + } ); + my $output = { + report_name => $report->{ name }, + report_ref => $report->{ ref } + }; +#end #END run_${module_name} - my @_bad_returns; - ( ref($output) eq 'HASH' ) or push( @_bad_returns, "Invalid type for return variable \"output\" (value was \"$output\")" ); - if ( @_bad_returns ) { + my @bad_returns; + ( ref($output) eq 'HASH' ) or push @bad_returns, + "Invalid type for return variable \"output\" " + . "(value was \"$output\")"; + if ( @bad_returns ) { my $msg = "Invalid returns passed to run_${module_name}:\n" - . join( "", map { "\t$_\n" } @_bad_returns ); + . join "", map { "\t$_\n" } @bad_returns; Bio::KBase::Exceptions::ArgumentValidationError->throw( error => $msg, method_name => 'run_${module_name}' ); } - return( $output ); + return ( $output ); } @@ -199,7 +220,7 @@ sub run_${module_name} { $return = $obj->status() -=over 4 +=over =item Parameter and return types @@ -240,44 +261,3 @@ sub status { } 1; -#else -#BEGIN_HEADER -use Bio::KBase::AuthToken; -use installed_clients::KBaseReportClient; -use Config::IniFiles; -#END_HEADER -#BEGIN_CONSTRUCTOR - my $config_file = $ENV{ KB_DEPLOYMENT_CONFIG }; - my $cfg = Config::IniFiles->new( -file => $config_file ); - my $scratch = $cfg->val( '${module_name}', 'scratch' ); - my $callbackURL = $ENV{ SDK_CALLBACK_URL }; - - $self->{ scratch } = $scratch; - $self->{ callbackURL } = $callbackURL; -#END_CONSTRUCTOR -#BEGIN run_${module_name} - my $repcli = installed_clients::KBaseReportClient->new( $self->{ callbackURL } ); - my $report = $repcli->create({ - workspace_name => $params->{ workspace_name }, - report => { - text_message => $params->{parameter_1}, - objects_created => [] - } - }); - - my $output = { - report_name => $report->{ name }, - report_ref => $report->{ ref } - }; -#END run_${module_name} - -#BEGIN_STATUS - $return = { - "state" => "OK", - "message" => "", - "version" => $VERSION, - "git_url" => $GIT_URL, - "git_commit_hash" => $GIT_COMMIT_HASH - }; -#END_STATUS -#end diff --git a/src/java/us/kbase/templates/module_test_perl_client.vm.properties b/src/java/us/kbase/templates/module_test_perl_client.vm.properties index 9a3ca283..af884926 100644 --- a/src/java/us/kbase/templates/module_test_perl_client.vm.properties +++ b/src/java/us/kbase/templates/module_test_perl_client.vm.properties @@ -1,4 +1,6 @@ use strict; +use warnings; + use Data::Dumper; use Test::More; use Config::Simple; @@ -15,10 +17,10 @@ my $token = $ENV{ 'KB_AUTH_TOKEN' }; my $config_file = $ENV{ 'KB_DEPLOYMENT_CONFIG' }; my $callback_url = $ENV{ 'SDK_CALLBACK_URL' }; -my $config = new Config::Simple( $config_file )->get_block( '${module_name}' ); +my $config = Config::Simple->new( $config_file )->get_block( '${module_name}' ); my $ws_url = $config->{ "workspace-url" }; my $ws_name = undef; -my $ws_client = new installed_clients::WorkspaceClient($ws_url,token => $token); +my $ws_client = installed_clients::WorkspaceClient->new($ws_url,token => $token); my $scratch = $config->{ scratch }; @@ -31,7 +33,7 @@ my $auth_token = Bio::KBase::AuthToken->new( my $ctx = LocalCallContext->new( $token, $auth_token->user_id ); $${module_name}::${module_name}Server::CallContext = $ctx; -my $impl = new ${module_name}::${module_name}Impl(); +my $impl = ${module_name}::${module_name}Impl->new(); sub get_ws_name { if ( !defined( $ws_name ) ) { @@ -145,6 +147,8 @@ if (defined($err)) { package LocalCallContext; use strict; + use warnings; + sub new { my ( $class, $token, $user ) = @_; my $self = { @@ -187,4 +191,7 @@ if (defined($err)) { my ( $self, $msg ) = @_; print STDERR $msg . "\n"; } + + 1; + } diff --git a/src/java/us/kbase/templates/perl_client.vm.properties b/src/java/us/kbase/templates/perl_client.vm.properties index 262192cb..da92dd5a 100644 --- a/src/java/us/kbase/templates/perl_client.vm.properties +++ b/src/java/us/kbase/templates/perl_client.vm.properties @@ -1,8 +1,10 @@ package ${client_package_name}; +use strict; +use warnings; + use JSON::RPC::Client; use POSIX; -use strict; use Data::Dumper; use URI; use Bio::KBase::Exceptions; @@ -124,7 +126,7 @@ sub new { } } - if (exists $self->{ token } ) { + if ( exists $self->{ token } ) { $self->{ client }{ token } = $self->{ token }; } } @@ -134,31 +136,37 @@ sub new { my $timeout = $ENV${empty_escaper}{ CDMI_TIMEOUT } || ( 30 * 60 ); $ua->timeout( $timeout ); bless $self, $class; - # $self->_validate_version(); return $self; } #foreach( $module in $modules ) #if( $any_async || $async_version) +# Authentication: ${method.authentication} sub _check_job { my ( $self, @args ) = @_; -# Authentication: ${method.authentication} - if ((my $n = @args) != 1) { - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => - "Invalid argument count for function _check_job (received $n, expecting 1)"); + + if ( ( my $n = @args ) != 1 ) { + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => "Invalid argument count for function _check_job " + . "(received $n, expecting 1)" + ); } - { - my($job_id) = @args; - my @_bad_arguments; - (!ref($job_id)) or push(@_bad_arguments, "Invalid type for argument 0 \"job_id\" (it should be a string)"); - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to _check_job:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw( - error => $msg, - method_name => '_check_job' - ); - } + + my ( $job_id ) = @args; + + my @bad_arguments; + ( ! ref( $job_id ) ) or push @bad_arguments, + "Invalid type for argument 0 \"job_id\" (it should be a string)"; + + if ( @bad_arguments ) { + my $msg = "Invalid arguments passed to _check_job:\n" + . join "", map { "\t$_\n" } @bad_arguments; + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => '_check_job' + ); } + my $result = $self->{ client }->call( $self->{ url }, $self->{ headers }, @@ -241,7 +249,7 @@ sub ${method.name} { my $job_id = $self->_${method.name}_submit( @args ); my $async_job_check_time = $self->{ async_job_check_time }; while (1) { - Time::HiRes::sleep($async_job_check_time); + Time::HiRes::sleep( $async_job_check_time ); $async_job_check_time *= $self->{ async_job_check_time_scale_percent } / 100.0; if ( $async_job_check_time > $self->{ async_job_check_max_time } ) { $async_job_check_time = $self->{ async_job_check_max_time }; @@ -258,24 +266,31 @@ sub ${method.name} { } } +# Authentication: ${method.authentication} sub _${method.name}_submit { my ( $self, @args ) = @_; -# Authentication: ${method.authentication} - if ((my $n = @args) != ${method.arg_count}) { - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => - "Invalid argument count for function _${method.name}_submit (received $n, expecting ${method.arg_count})"); + if ( ( my $n = @args ) != ${method.arg_count} ) { + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => "Invalid argument count for function _${method.name}_submit " + . "(received $n, expecting ${method.arg_count})" + ); } #if( $method.arg_count > 0 ) { - my(${method.arg_vars}) = @args; - my @_bad_arguments; + my ( ${method.arg_vars} ) = @args; + my @bad_arguments; #foreach( $param in $method.params ) - (${param.validator}) or push(@_bad_arguments, "Invalid type for argument ${param.index} \"${param.name}\" (value was \"${param.perl_var}\")"); + ( ${param.validator} ) or push @bad_arguments, + "Invalid type for argument ${param.index} \"${param.name}\" " + . "(value was \"${param.perl_var}\")"; #end - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to _${method.name}_submit:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => '_${method.name}_submit'); + if ( @bad_arguments ) { + my $msg = "Invalid arguments passed to _${method.name}_submit:\n" + . join "", map { "\t$_\n" } @bad_arguments; + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => '_${method.name}_submit' + ); } } #end @@ -303,7 +318,7 @@ sub _${method.name}_submit { data => $result->content->{ error }{ error }, ); } else { - return $result->result->[0]; # job_id + return $result->result->[ 0 ]; # job_id } } else { Bio::KBase::Exceptions::HTTP->throw( @@ -315,26 +330,29 @@ sub _${method.name}_submit { } #else ## of if-any-async +# Authentication: ${method.authentication} sub ${method.name} { my ( $self, @args ) = @_; -# Authentication: ${method.authentication} - if ( ( my $n = @args ) != ${method.arg_count} ) { Bio::KBase::Exceptions::ArgumentValidationError->throw( - error => "Invalid argument count for function ${method.name} (received $n, expecting ${method.arg_count})" + error => "Invalid argument count for function ${method.name} " + . "(received $n, expecting ${method.arg_count})" ); } #if( $method.arg_count > 0 ) { - my( ${method.arg_vars} ) = @args; + my ( ${method.arg_vars} ) = @args; - my @_bad_arguments; + my @bad_arguments; #foreach( $param in $method.params ) - ( ${param.validator} ) or push( @_bad_arguments, "Invalid type for argument ${param.index} \"${param.name}\" (value was \"${param.perl_var}\")" ); + ( ${param.validator} ) or push @bad_arguments, + "Invalid type for argument ${param.index} \"${param.name}\" " + . "(value was \"${param.perl_var}\")"; #end - if ( @_bad_arguments ) { - my $msg = "Invalid arguments passed to ${method.name}:\n" . join("", map { "\t$_\n" } @_bad_arguments); + if ( @bad_arguments ) { + my $msg = "Invalid arguments passed to ${method.name}:\n" + . join "", map { "\t$_\n" } @bad_arguments; Bio::KBase::Exceptions::ArgumentValidationError->throw( error => $msg, method_name => '${method.name}' @@ -414,7 +432,8 @@ sub status { my $job_id = undef; if ( ( my $n = @args ) != 0 ) { Bio::KBase::Exceptions::ArgumentValidationError->throw( - error => "Invalid argument count for function status (received $n, expecting 0)"); + error => "Invalid argument count for function status " + . "(received $n, expecting 0)"); } my $context = undef; if ( $self->{ service_version } ) { @@ -440,7 +459,7 @@ sub status { data => $result->content->{ error }{ error }, ); } else { - $job_id = $result->result->[0]; + $job_id = $result->result->[ 0 ]; } } else { Bio::KBase::Exceptions::HTTP->throw( @@ -449,31 +468,37 @@ sub status { method_name => '_status_submit', ); } - my $async_job_check_time = $self->{async_job_check_time}; - while (1) { - Time::HiRes::sleep($async_job_check_time); - $async_job_check_time *= $self->{async_job_check_time_scale_percent} / 100.0; - if ($async_job_check_time > $self->{async_job_check_max_time}) { - $async_job_check_time = $self->{async_job_check_max_time}; - } - my $job_state_ref = $self->_check_job($job_id); - if ($job_state_ref->{"finished"} != 0) { - if (!exists $job_state_ref->{"result"}) { - $job_state_ref->{"result"} = []; + my $async_job_check_time = $self->{ async_job_check_time }; + while ( 1 ) { + Time::HiRes::sleep( $async_job_check_time ); + $async_job_check_time *= $self->{ async_job_check_time_scale_percent } / 100.0; + if ( $async_job_check_time > $self->{ async_job_check_max_time } ) { + $async_job_check_time = $self->{ async_job_check_max_time }; + } + my $job_state_ref = $self->_check_job( $job_id ); + if ( $job_state_ref->{ "finished" } != 0 ) { + if ( !exists $job_state_ref->{ "result" } ) { + $job_state_ref->{ "result" } = []; } - return wantarray ? @{$job_state_ref->{"result"}} : $job_state_ref->{"result"}->[0]; + return wantarray + ? @{ $job_state_ref->{ "result" } } + : $job_state_ref->{ "result" }->[ 0 ]; } } + } #else ## of if-status-async sub status { my ( $self, @args ) = @_; - if ((my $n = @args) != 0) { + if ( ( my $n = @args ) != 0 ) { Bio::KBase::Exceptions::ArgumentValidationError->throw( - error => "Invalid argument count for function status (received $n, expecting 0)"); + error => "Invalid argument count for function status " + . "(received $n, expecting 0)" + ); } + #if( $dynserv_ver ) my $service_state = $self->{ client }->call( @@ -537,7 +562,7 @@ sub status { sub version { my ( $self ) = @_; - my $result = $self->{client}->call( + my $result = $self->{ client }->call( $self->{ url }, $self->{ headers }, { @@ -652,8 +677,8 @@ sub call { #if( $enable_client_retry ) - my @retries = (1, 2, 5, 10, 20, 60, 60, 60, 60, 60, 60); - my %codes_to_retry = map { $_ => 1 } qw(110 408 502 503 504 200) ; + my @retries = ( 1, 2, 5, 10, 20, 60, 60, 60, 60, 60, 60 ); + my %codes_to_retry = map { $_ => 1 } qw( 110 408 502 503 504 200 ) ; my $n_retries; while (1) diff --git a/src/java/us/kbase/templates/perl_impl.vm.properties b/src/java/us/kbase/templates/perl_impl.vm.properties index aab85914..58b8503b 100644 --- a/src/java/us/kbase/templates/perl_impl.vm.properties +++ b/src/java/us/kbase/templates/perl_impl.vm.properties @@ -1,6 +1,8 @@ package ${module.impl_package_name}; use strict; +use warnings; + use Bio::KBase::Exceptions; # Use Semantic Versioning (2.0.0-rc.1) # http://semver.org @@ -43,12 +45,11 @@ ${module.module_constructor} #END_CONSTRUCTOR #set( $status_in_kidl = true ) #end - =head2 ${method.name} #if( "$method.ret_vars" != "" )${method.ret_vars} = #{end}$obj->${method.name}(${method.arg_vars}) -=over 4 +=over =item Parameter and return types @@ -73,7 +74,6 @@ ${docline} =end text - =item Description ${method.doc} @@ -85,17 +85,19 @@ ${method.doc} sub ${method.name} { my $self = shift; #if( $method.arg_count > 0 ) - my( ${method.arg_vars} ) = @_; + my ( ${method.arg_vars} ) = @_; - my @_bad_arguments; + my @bad_arguments; #foreach( $param in $method.params ) - ( ${param.validator} ) or push( @_bad_arguments, "Invalid type for argument \"${param.name}\" (value was \"${param.perl_var}\")" ); + ( ${param.validator} ) or push @bad_arguments, + "Invalid type for argument \"${param.name}\" " + . "(value was \"${param.perl_var}\")"; #end - if ( @_bad_arguments ) { + if ( @bad_arguments ) { my $msg = "Invalid arguments passed to ${method.name}:\n" - . join("", map { "\t$_\n" } @_bad_arguments); + . join "", map { "\t$_\n" } @bad_arguments; Bio::KBase::Exceptions::ArgumentValidationError->throw( - error => $msg, + error => $msg, method_name => '${method.name}' ); } @@ -108,13 +110,15 @@ sub ${method.name} { #BEGIN ${method.name} ${method.user_code} #${empty_escaper}END ${method.name} #if( $method.ret_count > 0 ) - my @_bad_returns; + my @bad_returns; #foreach( $return in $method.returns ) - (${return.validator}) or push(@_bad_returns, "Invalid type for return variable \"${return.name}\" (value was \"${return.perl_var}\")"); + ( ${return.validator} ) or push @bad_returns, + "Invalid type for return variable \"${return.name}\" " + . "(value was \"${return.perl_var}\")"; #end - if ( @_bad_returns ) { + if ( @bad_returns ) { my $msg = "Invalid returns passed to ${method.name}:\n" - . join( "", map { "\t$_\n" } @_bad_returns ); + . join "", map { "\t$_\n" } @bad_returns; Bio::KBase::Exceptions::ArgumentValidationError->throw( error => $msg, method_name => '${method.name}' @@ -133,7 +137,7 @@ ${method.user_code} #${empty_escaper}END ${method.name} $return = $obj->status() -=over 4 +=over =item Parameter and return types @@ -182,20 +186,17 @@ ${module.module_status} #foreach( $type in $module.types ) - =head2 ${type.name} -=over 4 +=over #if( $type.comment && "$type.comment" != "" ) - =item Description ${type.comment} #end - =item Definition =begin html @@ -216,7 +217,6 @@ ${type.english} #end - =cut 1; diff --git a/src/java/us/kbase/templates/perl_psgi.vm.properties b/src/java/us/kbase/templates/perl_psgi.vm.properties index d3d268a3..6039ac6b 100644 --- a/src/java/us/kbase/templates/perl_psgi.vm.properties +++ b/src/java/us/kbase/templates/perl_psgi.vm.properties @@ -1,3 +1,6 @@ +use strict; +use warnings; + #foreach( $module in $modules ) use ${module.impl_package_name}; #end @@ -10,19 +13,14 @@ use UserAuth; my $user_auth = UserAuth->new(); #end - -my @dispatch; - +my %dispatch = ( #foreach( $module in $modules ) -{ - my $obj = ${module.impl_package_name}->new; - push( @dispatch, '${module.module_name}' => $obj ); -} + '${module.module_name}' => ${module.impl_package_name}->new, #end - +); my $server = ${server_package_name}->new( - instance_dispatch => { @dispatch }, + instance_dispatch => \%dispatch, allow_get => 0, #if( $service_options.authenticated ) user_auth => $user_auth, diff --git a/src/java/us/kbase/templates/perl_server.vm.properties b/src/java/us/kbase/templates/perl_server.vm.properties index 666c03a5..edc7d178 100644 --- a/src/java/us/kbase/templates/perl_server.vm.properties +++ b/src/java/us/kbase/templates/perl_server.vm.properties @@ -102,7 +102,7 @@ sub _build_config { if ( !($cf) ) { return {}; } - my $cfg = new Config::Simple( $cf ); + my $cfg = Config::Simple->new( $cf ); my $cfgdict = $cfg->get_block( $sn ); if ( !($cfgdict) ) { return {}; @@ -540,6 +540,7 @@ sub handle_error_cli { package ${ctx_pkg}; use strict; +use warnings; =head1 NAME @@ -637,6 +638,7 @@ sub clear_log_level { package ${stderr_pkg}; use strict; +use warnings; use POSIX; use Time::HiRes 'gettimeofday'; @@ -652,7 +654,7 @@ sub new { $us = sprintf("%06d", $us); my $ts = strftime("%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t); - my $name = join( ".", $ctx->module, $ctx->method, $ctx->hostname, $ts ); + my $name = join ".", $ctx->module, $ctx->method, $ctx->hostname, $ts; if ( $dest && $dest =~ m!^/! ) { # From 51a73c78ea41d571b58c5370232102f61f38c6a8 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Mon, 21 Sep 2020 10:53:29 -0700 Subject: [PATCH 25/47] Add maintainer as a LABEL to the dockerfile --- src/java/us/kbase/templates/module_dockerfile.vm.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/us/kbase/templates/module_dockerfile.vm.properties b/src/java/us/kbase/templates/module_dockerfile.vm.properties index 2d41c79a..4e0b1299 100644 --- a/src/java/us/kbase/templates/module_dockerfile.vm.properties +++ b/src/java/us/kbase/templates/module_dockerfile.vm.properties @@ -3,7 +3,7 @@ FROM kbase/sdkbase2:python #else FROM kbase/sdkbase2:latest #end - +LABEL maintainer="#if($user_name)${user_name}#{else}KBase Developer#{end}" # ----------------------------------------- # In this section, you can install any system dependencies required # to run your App. For instance, you could place an apt-get update or From 1c53e97b95064b3514bdc4dc2c2f8165e09ed963 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Mon, 28 Sep 2020 08:00:07 -0700 Subject: [PATCH 26/47] Add cpanfile for perl dependencies Add cpan installation step to Dockerfile Add compilation tests to perl base repo Update perl test running code to use the perl test harness Update sample repo perl tests to be more reasonable --- .../mobu/initializer/ModuleInitializer.java | 4 +- .../templates/module_cpanfile.vm.properties | 11 ++ .../templates/module_dockerfile.vm.properties | 16 +- .../templates/module_makefile.vm.properties | 6 +- .../module_test_perl_client.vm.properties | 182 +++++++++++------- .../module_test_perl_compile.vm.properties | 29 +++ 6 files changed, 166 insertions(+), 82 deletions(-) create mode 100644 src/java/us/kbase/templates/module_cpanfile.vm.properties create mode 100644 src/java/us/kbase/templates/module_test_perl_compile.vm.properties diff --git a/src/java/us/kbase/mobu/initializer/ModuleInitializer.java b/src/java/us/kbase/mobu/initializer/ModuleInitializer.java index 4ae97072..fc71556d 100644 --- a/src/java/us/kbase/mobu/initializer/ModuleInitializer.java +++ b/src/java/us/kbase/mobu/initializer/ModuleInitializer.java @@ -153,7 +153,9 @@ public void initialize(boolean example) throws Exception { templateFiles.put("module_tox", Paths.get(moduleDir, "tox.ini")); break; case "perl": - templateFiles.put("module_test_perl_client", Paths.get(moduleDir, "test", this.moduleName + "_server_test.pl")); + templateFiles.put("module_cpanfile", Paths.get(moduleDir, "cpanfile")); + templateFiles.put("module_test_perl_client", Paths.get(moduleDir, "test", this.moduleName + "_impl.t")); + templateFiles.put("module_test_perl_compile", Paths.get(moduleDir, "test", "00_compile.t")); break; case "r": templateFiles.put("module_test_r_client", Paths.get(moduleDir, "test", this.moduleName + "_server_test.r")); diff --git a/src/java/us/kbase/templates/module_cpanfile.vm.properties b/src/java/us/kbase/templates/module_cpanfile.vm.properties new file mode 100644 index 00000000..7e8a45b6 --- /dev/null +++ b/src/java/us/kbase/templates/module_cpanfile.vm.properties @@ -0,0 +1,11 @@ +requires 'Config::IniFiles', '>= 3.0000002'; +requires 'Config::Any', '>= 0.32'; +requires 'Data::Dumper::Concise', '>= 2.023'; +requires 'Test::Most', '>= 0.35'; +requires 'Test::Compile', '>= 2.3.1'; +requires 'Ref::Util::XS', '>= 0.117'; +requires 'Ref::Util', '>= 0.204'; +#if( $example ) +# download a sequence IO module +requires 'Bio::SeqIO'; +#end diff --git a/src/java/us/kbase/templates/module_dockerfile.vm.properties b/src/java/us/kbase/templates/module_dockerfile.vm.properties index 4e0b1299..5d40e766 100644 --- a/src/java/us/kbase/templates/module_dockerfile.vm.properties +++ b/src/java/us/kbase/templates/module_dockerfile.vm.properties @@ -17,14 +17,6 @@ LABEL maintainer="#if($user_name)${user_name}#{else}KBase Developer#{end}" RUN cd /kb/deployment/lib/jars \ && wget https://downloads.sourceforge.net/project/jfasta/releases/jfasta-2.2.0/jfasta-2.2.0-jar-with-dependencies.jar #end -#if( $language == "perl") -# download an inifile reader -RUN cpanm -i Config::IniFiles -#if( $example ) -# download a sequence IO module -RUN cpanm -i Bio::SeqIO -#end -#end #if( $language == 'r' ) ENV R_LIBS=/kb/deployment/lib RUN R -q -e 'if(!require(jsonlite)) install.packages("jsonlite", repos="http://cran.us.r-project.org")' @@ -32,6 +24,14 @@ RUN R -q -e 'if(!require(httr)) install.packages("httr", repos="http://cran.us.r RUN R -q -e 'if(!require(raster)) install.packages("raster", repos="http://cran.us.r-project.org")' RUN apt-get -y install r-cran-evaluate r-cran-codetools r-cran-testthat #end +#if( $language == "perl") +COPY ./cpanfile /kb/module/cpanfile +WORKDIR /kb/module +# install cpan dependencies from the cpanfile, +# then tidy up the installation leftovers +RUN cpanm --installdeps . && \ + cd ~ && rm -rf .cpanm +#end COPY ./ /kb/module diff --git a/src/java/us/kbase/templates/module_makefile.vm.properties b/src/java/us/kbase/templates/module_makefile.vm.properties index 7b96c7dc..578dc21e 100644 --- a/src/java/us/kbase/templates/module_makefile.vm.properties +++ b/src/java/us/kbase/templates/module_makefile.vm.properties @@ -115,11 +115,11 @@ build-test-script: echo 'export KB_AUTH_TOKEN=`cat /kb/module/work/token`' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) echo 'echo "Removing temp files..."' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) echo 'rm -rf $(WORK_DIR)/*' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) - echo 'echo "...done removing temp files."' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) + echo 'echo "Finished removing temp files."' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) #if($language == "perl") echo 'export PERL5LIB=$$script_dir/../$(LIB_DIR):$$PERL5LIB' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) - echo 'cd $$script_dir/../$(TEST_DIR)' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) - echo "perl -e 'opendir my ${esc.print("\")}${esc.print("$")}${esc.print("$")}dh, \".\"; my @l = grep { /\\\\.pl${esc.print("\")}${esc.print("$")}${esc.print("$")}/ } readdir ${esc.print("\")}${esc.print("$")}${esc.print("$")}dh; my ${esc.print("\")}${esc.print("$")}${esc.print("$")}ret=0; foreach my ${esc.print("\")}${esc.print("$")}${esc.print("$")}s (@l) { print(\"Running \".${esc.print("\")}${esc.print("$")}${esc.print("$")}s.\"\\\\n\"); system(\"perl\", ${esc.print("\")}${esc.print("$")}${esc.print("$")}s) && do{${esc.print("\")}${esc.print("$")}${esc.print("$")}ret=2; next;}; } exit(${esc.print("\")}${esc.print("$")}${esc.print("$")}ret);'" >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) + echo 'cd $$script_dir/..' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) + echo 'prove -lvrm $$script_dir' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) #end #if($language == "python") echo 'export PYTHONPATH=$$script_dir/../$(LIB_DIR):$$PATH:$$PYTHONPATH' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) diff --git a/src/java/us/kbase/templates/module_test_perl_client.vm.properties b/src/java/us/kbase/templates/module_test_perl_client.vm.properties index af884926..adb9c837 100644 --- a/src/java/us/kbase/templates/module_test_perl_client.vm.properties +++ b/src/java/us/kbase/templates/module_test_perl_client.vm.properties @@ -1,8 +1,8 @@ use strict; use warnings; +use feature qw( say ); -use Data::Dumper; -use Test::More; +use Test::Most; use Config::Simple; use Time::HiRes qw(time); use Bio::KBase::AuthToken; @@ -10,7 +10,8 @@ use installed_clients::WorkspaceClient; #if ($example) use installed_clients::AssemblyUtilClient; #end -use ${module_name}::${module_name}Impl; + +require_ok "${module_name}::${module_name}Impl"; local $| = 1; my $token = $ENV{ 'KB_AUTH_TOKEN' }; @@ -18,11 +19,9 @@ my $config_file = $ENV{ 'KB_DEPLOYMENT_CONFIG' }; my $callback_url = $ENV{ 'SDK_CALLBACK_URL' }; my $config = Config::Simple->new( $config_file )->get_block( '${module_name}' ); -my $ws_url = $config->{ "workspace-url" }; -my $ws_name = undef; -my $ws_client = installed_clients::WorkspaceClient->new($ws_url,token => $token); +my ( $ws_name, $ws_client ); -my $scratch = $config->{ scratch }; +my $scratch = $config->{ scratch }; my $auth_token = Bio::KBase::AuthToken->new( token => $token, @@ -33,10 +32,17 @@ my $auth_token = Bio::KBase::AuthToken->new( my $ctx = LocalCallContext->new( $token, $auth_token->user_id ); $${module_name}::${module_name}Server::CallContext = $ctx; -my $impl = ${module_name}::${module_name}Impl->new(); +sub get_ws_client { + + my $ws_url = $config->{ "workspace-url" }; + $ws_client //= installed_clients::WorkspaceClient->new( $ws_url, token => $token ); + return $ws_client; +} sub get_ws_name { - if ( !defined( $ws_name ) ) { + + $ws_client = get_ws_client(); + unless ( $ws_name ) { my $suffix = int( time * 1000 ); $ws_name = 'test_${module_name}_' . $suffix; $ws_client->create_workspace( { workspace => $ws_name } ); @@ -44,14 +50,25 @@ sub get_ws_name { return $ws_name; } +my $impl; + +subtest 'creating the new ${module_name}Impl object' => sub { + + lives_ok { + $impl = ${module_name}::${module_name}Impl->new(); + } '${module_name}Impl object can be created'; + + isa_ok $impl, '${module_name}::${module_name}Impl'; + +}; + #if($example) sub load_fasta { - my $filename = shift; - my $object_name = shift; - my $filecontents = shift; - open ( my $fh, '>', $filename ) - or die "Could not open file '$filename' ${dollar_sign}!"; - print $fh $filecontents; + my ( $filename, $object_name, $contents ) = @_; + + open my $fh, '>', $filename + or die "Could not open file '$filename': ${dollar_sign}!"; + print $fh $contents; close $fh; my $assycli = installed_clients::AssemblyUtilClient->new( $callback_url ); return $assycli->save_assembly_from_fasta( { @@ -60,12 +77,10 @@ sub load_fasta { file => { path => $filename } } ); } -#end -eval { +subtest 'successful run_${module_name} run' => sub { -#if($example) - # First load a test FASTA file as an KBase Assembly + # load a test FASTA file as an KBase Assembly my $fastaContent = ">seq1 something something asdf\n" . "agcttttcat\n" . ">seq2\n" . @@ -75,74 +90,101 @@ eval { my $ref = load_fasta( $scratch . "/test1.fasta", "TestAssembly", $fastaContent ); - # Second, call the implementation - my $ret = $impl->run_${module_name}( { + # call the implementation + my $result = $impl->run_${module_name}( { workspace_name => get_ws_name(), assembly_input_ref => $ref, min_length => 10, } ); # validate the returned data - ok($ret->{n_initial_contigs} eq 3, "number of initial contigs"); - ok($ret->{n_contigs_removed} eq 1, "number of removed contigs"); - ok($ret->{n_contigs_remaining} eq 2, "number of remaining contigs"); - - $@ = ''; - eval { - $impl->run_${module_name}({workspace_name => get_ws_name(), - assembly_input_ref=>"fake", - min_length => 10}); - }; - like($@, qr#separators / in object reference fake#); + cmp_ok + $result->{ n_initial_contigs }, '==', 3, + 'correct number of initial contigs'; + cmp_ok + $result->{ n_contigs_removed }, '==', 1, + 'correct number of removed contigs'; + cmp_ok + $result->{ n_contigs_remaining }, '==', 2, + 'correct number of remaining contigs'; - eval { - $impl->run_${module_name}({workspace_name => get_ws_name(), - assembly_input_ref => "fake", - min_length => -10}); - }; - like($@, qr/min_length parameter cannot be negative/); +}; - eval { - $impl->run_${module_name}({workspace_name => get_ws_name(), - assembly_input_ref => "fake"}); - }; - like($@, qr/Parameter min_length is not set in input arguments/); +subtest 'run_${module_name} errors' => sub { + + throws_ok { + $impl->run_${module_name}( { + workspace_name => get_ws_name(), + assembly_input_ref =>"fake", + min_length => 10, + } ); + } qr#separators / in object reference fake#, + 'Appropriate error thrown for fake ref object'; + + throws_ok { + $impl->run_${module_name}( { + workspace_name => get_ws_name(), + assembly_input_ref => "fake", + min_length => -10, + } ); + } qr/min_length parameter cannot be negative/, + 'correct error message for min length param'; + + throws_ok { + $impl->run_${module_name}( { + workspace_name => get_ws_name(), + assembly_input_ref => "fake" + } ); + } qr/Parameter min_length is not set in input arguments/, + 'missing min_length param'; - done_testing(6); +}; #else + +subtest 'testing run_${module_name}' => sub { + # Prepare test data using the appropriate uploader for that data (see the KBase function # catalog for help, https://narrative.kbase.us/#catalog/functions) - - # Run your method by - # my $ret = $impl->your_method(parameters...); + # Run your method using + # my $result = $impl->your_method( $parameters... ); # - # Check returned data with - # ok(ret->{...} eq , "tested item") or other Test::More methods - my $ret = $impl->run_${module_name}({workspace_name => get_ws_name(), - parameter_1 => "Hello world"}); -#end + # Check returned data with methods from Test::Most; e.g. + # cmp_deeply + # $result, + # $expected, + # "$impl->your_method returns the expected data with params ..."; + + my $result = $impl->run_${module_name}( { + workspace_name => get_ws_name(), + parameter_1 => "Hello world", + } ); + + my $expect = { + report_name => re('report_[0-9a-f-]+'), + report_ref => re('\d+\/\d+\/\d+'), + }; + + cmp_deeply + $result, + $expect, + 'the expected report was returned' + or diag explain { + got => $result, + expected => $expect, + }; }; -my $err = undef; -if ($@) { - $err = $@; -} -eval { - if (defined($ws_name)) { - $ws_client->delete_workspace({workspace => $ws_name}); - print("Test workspace was deleted\n"); - } -}; -if (defined($err)) { - use Scalar::Util 'blessed'; - if(blessed $err && $err->isa("Bio::KBase::Exceptions::KBaseException")) { - die "Error while running tests. Remote error:\n" . $err->{data} . - "Client-side error:\n" . $err; - } else { - die $err; - } + +#end + +if ( $ws_name ) { + lives_ok { + $ws_client->delete_workspace( { workspace => $ws_name } ); + } 'Test workspace successfully deleted'; } +done_testing(); + { package LocalCallContext; diff --git a/src/java/us/kbase/templates/module_test_perl_compile.vm.properties b/src/java/us/kbase/templates/module_test_perl_compile.vm.properties new file mode 100644 index 00000000..a171b5e5 --- /dev/null +++ b/src/java/us/kbase/templates/module_test_perl_compile.vm.properties @@ -0,0 +1,29 @@ +use Test::Most; +use Test::Compile; + +my $base_dir = '/kb/module/'; +my $tester = Test::Compile->new(); + +# check all .pm and .pl files compile +$tester->all_files_ok( $base_dir ); + +# also check .t and .psgi files +my @all_ext_files = files_with_ext( $tester, qr/\.(t|psgi)$/, $base_dir ); + +for ( @all_ext_files ) { + ok $tester->pl_file_compiles( $_ ), $_ . ' compiles'; +} + +$tester->done_testing(); + +sub files_with_ext { + my ( $tester, $regex, @dirs ) = @_; + + @dirs = @dirs ? @dirs : ( $base_dir ); + + my @files; + for my $file ( $tester->_find_files( @dirs ) ) { + push @files, $file if $file =~ $regex; + } + return @files; +} From 71fc4d7b15be99e8cda8b7b43a101077eda5488c Mon Sep 17 00:00:00 2001 From: bio-boris Date: Fri, 9 Oct 2020 15:55:47 -0500 Subject: [PATCH 27/47] Update entrypoint * Possible side effects are having to use `sudo su` to to delete the files that appear on your machine --- entrypoint | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entrypoint b/entrypoint index afa74c85..7275132b 100755 --- a/entrypoint +++ b/entrypoint @@ -4,7 +4,7 @@ if [ "z$1" = "zshell" ] ; then elif [ "z$1" = "zsetup" ] ; then cat << EOF G=\$(docker run -i -v /var/run/docker.sock:/var/run/docker.sock --entrypoint ls kbase/kb-sdk -l /var/run/docker.sock|awk '{print \$4}'); -alias kb-sdk='docker run -it --rm -v \$HOME:\$HOME -u \$(id -u) -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e KBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$G kbase/kb-sdk' +alias kb-sdk='docker run -it --rm -v \$HOME:\$HOME -u 0 -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e KBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$G kbase/kb-sdk' EOF elif [ "z$1" = "zgenscript" ] ; then cat << EOF @@ -15,7 +15,7 @@ if [ ! -e \$HOME/.kbsdk.cache ] ; then docker run -i -v /var/run/docker.sock:/var/run/docker.sock --entrypoint ls kbase/kb-sdk -l /var/run/docker.sock|awk '{print \$4}' > \$HOME/.kbsdk.cache fi -exec docker run -it --rm -v \$HOME:\$HOME -u \$(id -u) -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e KBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$(cat \$HOME/.kbsdk.cache) kbase/kb-sdk \$@ +exec docker run -it --rm -v \$HOME:\$HOME -u 0 -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e KBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$(cat \$HOME/.kbsdk.cache) kbase/kb-sdk \$@ EOF elif [ "z$1" = "zprune" ] ; then echo "Used during build to shrink image. Not needed by the user." From b9a313af6762d50775229744022939cb9fffbb54 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Wed, 14 Oct 2020 10:27:08 -0500 Subject: [PATCH 28/47] Update RELEASE_NOTES.txt --- RELEASE_NOTES.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 3ba0bac8..24c028d3 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -6,11 +6,17 @@ platform. Modules include all code, specification files, and documentation needed to define and run a set of methods in the KBase Narrative interface. -VERSION: 1.2.4 (Released 09/2020) + +VERSION: 1.2.5 (Unreleased) +--------------------------------- +- `kb-sdk test` no longer can see the docker socket due to a docker permissions issue on the docker socket. By default, the user running in the kb-sdk container now runs as the root user + +VERSION: 1.2.4 (Unreleased) --------------------------------- CHANGES: - `kb-sdk test` no longer dies on its first run; if the user has the KBASE_TEST_TOKEN environment variable set with their developer token, tests can be run without having to edit the test.cfg file. + VERSION: 1.2.3 (Released 06/2020) ------------------------------------ NEW FEATURES: From 60d99393d57797426ddf30235665d2bf6a31c851 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Wed, 14 Oct 2020 10:28:45 -0500 Subject: [PATCH 29/47] Update ModuleBuilder.java --- src/java/us/kbase/mobu/ModuleBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/us/kbase/mobu/ModuleBuilder.java b/src/java/us/kbase/mobu/ModuleBuilder.java index 52ba366c..3b1433f4 100644 --- a/src/java/us/kbase/mobu/ModuleBuilder.java +++ b/src/java/us/kbase/mobu/ModuleBuilder.java @@ -49,7 +49,7 @@ public class ModuleBuilder { public static final String GLOBAL_SDK_HOME_ENV_VAR = "KB_SDK_HOME"; public static final String DEFAULT_METHOD_STORE_URL = "https://appdev.kbase.us/services/narrative_method_store/rpc"; - public static final String VERSION = "1.2.4"; + public static final String VERSION = "1.2.5"; public static void main(String[] args) throws Exception { From 2eacd994045167634abc9794ba4dcf4faffe0626 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 22 Oct 2020 11:18:53 -0700 Subject: [PATCH 30/47] Removing "-t" flag to allow github actions to run kb-sdk --- RELEASE_NOTES.txt | 4 ++++ entrypoint | 4 ++-- src/java/us/kbase/mobu/ModuleBuilder.java | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 56594dfb..bdcb0c2d 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -6,6 +6,10 @@ platform. Modules include all code, specification files, and documentation needed to define and run a set of methods in the KBase Narrative interface. +VERSION: 1.2.6 (Released 10/2020) +--------------------------------- +CHANGES: +- "-t" flag removed from entrypoint commands to enable kb-sdk to be run by github actions. VERSION: 1.2.5 (Unreleased) --------------------------------- diff --git a/entrypoint b/entrypoint index 7275132b..ddeca8fc 100755 --- a/entrypoint +++ b/entrypoint @@ -4,7 +4,7 @@ if [ "z$1" = "zshell" ] ; then elif [ "z$1" = "zsetup" ] ; then cat << EOF G=\$(docker run -i -v /var/run/docker.sock:/var/run/docker.sock --entrypoint ls kbase/kb-sdk -l /var/run/docker.sock|awk '{print \$4}'); -alias kb-sdk='docker run -it --rm -v \$HOME:\$HOME -u 0 -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e KBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$G kbase/kb-sdk' +alias kb-sdk='docker run -i --rm -v \$HOME:\$HOME -u 0 -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e KBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$G kbase/kb-sdk' EOF elif [ "z$1" = "zgenscript" ] ; then cat << EOF @@ -15,7 +15,7 @@ if [ ! -e \$HOME/.kbsdk.cache ] ; then docker run -i -v /var/run/docker.sock:/var/run/docker.sock --entrypoint ls kbase/kb-sdk -l /var/run/docker.sock|awk '{print \$4}' > \$HOME/.kbsdk.cache fi -exec docker run -it --rm -v \$HOME:\$HOME -u 0 -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e KBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$(cat \$HOME/.kbsdk.cache) kbase/kb-sdk \$@ +exec docker run -i --rm -v \$HOME:\$HOME -u 0 -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e KBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$(cat \$HOME/.kbsdk.cache) kbase/kb-sdk \$@ EOF elif [ "z$1" = "zprune" ] ; then echo "Used during build to shrink image. Not needed by the user." diff --git a/src/java/us/kbase/mobu/ModuleBuilder.java b/src/java/us/kbase/mobu/ModuleBuilder.java index 07779f81..cef7db21 100644 --- a/src/java/us/kbase/mobu/ModuleBuilder.java +++ b/src/java/us/kbase/mobu/ModuleBuilder.java @@ -49,8 +49,8 @@ public class ModuleBuilder { public static final String GLOBAL_SDK_HOME_ENV_VAR = "KB_SDK_HOME"; public static final String DEFAULT_METHOD_STORE_URL = "https://appdev.kbase.us/services/narrative_method_store/rpc"; - public static final String VERSION = "1.2.5"; - + public static final String VERSION = "1.2.6"; + public static void main(String[] args) throws Exception { // set up the basic CLI argument parser with global -h and --help commands From 401b3d7786b5154962d3c8159ef187c4eea5bdb0 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Wed, 22 Jun 2022 00:44:21 -0500 Subject: [PATCH 31/47] Update module_test_python_client.vm.properties --- .../kbase/templates/module_test_python_client.vm.properties | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/java/us/kbase/templates/module_test_python_client.vm.properties b/src/java/us/kbase/templates/module_test_python_client.vm.properties index 4952aac4..0445aa9f 100644 --- a/src/java/us/kbase/templates/module_test_python_client.vm.properties +++ b/src/java/us/kbase/templates/module_test_python_client.vm.properties @@ -6,7 +6,11 @@ from configparser import ConfigParser from ${module_name}.${module_name}Impl import ${module_name} from ${module_name}.${module_name}Server import MethodContext -from ${module_name}.authclient import KBaseAuth as _KBaseAuth + +try: + from ${module_name}.authclient import KBaseAuth as _KBaseAuth +except ImportError: + from installed_clients.authclient import KBaseAuth as _KBaseAuth #if ($example) from installed_clients.AssemblyUtilClient import AssemblyUtil From 29d1ee561ff2b34aa61d4939dea85e2f99bb32eb Mon Sep 17 00:00:00 2001 From: bio-boris Date: Wed, 22 Jun 2022 01:33:49 -0500 Subject: [PATCH 32/47] Update python_server.vm.properties --- src/java/us/kbase/templates/python_server.vm.properties | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/java/us/kbase/templates/python_server.vm.properties b/src/java/us/kbase/templates/python_server.vm.properties index 990f1aee..dbf6241c 100644 --- a/src/java/us/kbase/templates/python_server.vm.properties +++ b/src/java/us/kbase/templates/python_server.vm.properties @@ -18,7 +18,10 @@ from jsonrpcbase import ServerError as JSONServerError from biokbase import log #if( $authenticated ) -from ${modules[0].pypackage}authclient import KBaseAuth as _KBaseAuth +try: + from ${modules[0].pypackage}authclient import KBaseAuth as _KBaseAuth +except ImportError: + from installed_clients.authclient import KBaseAuth as _KBaseAuth #end try: From 6eecbf5901aa121f5e61c3df63cc0371db55731f Mon Sep 17 00:00:00 2001 From: bio-boris Date: Thu, 16 Feb 2023 16:17:04 -0600 Subject: [PATCH 33/47] Create manual-build.yml --- .github/workflows/manual-build.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/manual-build.yml diff --git a/.github/workflows/manual-build.yml b/.github/workflows/manual-build.yml new file mode 100644 index 00000000..944f9035 --- /dev/null +++ b/.github/workflows/manual-build.yml @@ -0,0 +1,11 @@ +--- +name: Manual Build & Push +on: + workflow_dispatch: +jobs: + build-push: + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}-develop' + tags: br-${{ github.ref_name }} + secrets: inherit From e9dd9503c152334d5c21298b795981a070d87f27 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Thu, 16 Feb 2023 16:17:29 -0600 Subject: [PATCH 34/47] Create pr_build.yml --- .github/workflows/pr_build.yml | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/pr_build.yml diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml new file mode 100644 index 00000000..0fa1c464 --- /dev/null +++ b/.github/workflows/pr_build.yml @@ -0,0 +1,43 @@ +--- +name: Pull Request Build, Tag, & Push +on: + pull_request: + branches: + - develop + - main + - master + types: + - opened + - reopened + - synchronize + - closed +jobs: + build-develop-open: + if: github.base_ref == 'develop' && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_build.yml@main + secrets: inherit + build-develop-merge: + if: github.base_ref == 'develop' && github.event.pull_request.merged == true + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}-develop' + tags: pr-${{ github.event.number }},latest + secrets: inherit + build-main-open: + if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: pr-${{ github.event.number }} + secrets: inherit + build-main-merge: + if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == true + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: pr-${{ github.event.number }},latest-rc + secrets: inherit + trivy-scans: + if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main + secrets: inherit From 03c6a09b2465d885f0b19d8e3866c2addbb94000 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Thu, 16 Feb 2023 16:17:53 -0600 Subject: [PATCH 35/47] Create release-main.yml --- .github/workflows/release-main.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/release-main.yml diff --git a/.github/workflows/release-main.yml b/.github/workflows/release-main.yml new file mode 100644 index 00000000..a2546781 --- /dev/null +++ b/.github/workflows/release-main.yml @@ -0,0 +1,25 @@ +--- +name: Release - Build & Push Image +on: + release: + branches: + - main + - master + types: [ published ] +jobs: + check-source-branch: + uses: kbase/.github/.github/workflows/reusable_validate-branch.yml@main + with: + build_branch: '${{ github.event.release.target_commitish }}' + validate-release-tag: + needs: check-source-branch + uses: kbase/.github/.github/workflows/reusable_validate-release-tag.yml@main + with: + release_tag: '${{ github.event.release.tag_name }}' + build-push: + needs: validate-release-tag + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: '${{ github.event.release.tag_name }},latest' + secrets: inherit From 8b2d1e650eea1df44ee7a8259c84f524c4ffabf9 Mon Sep 17 00:00:00 2001 From: bio-boris Date: Thu, 16 Feb 2023 16:30:04 -0600 Subject: [PATCH 36/47] Update kb_sdk-actions.yml --- .github/workflows/kb_sdk-actions.yml | 372 +++++++++++++-------------- 1 file changed, 186 insertions(+), 186 deletions(-) diff --git a/.github/workflows/kb_sdk-actions.yml b/.github/workflows/kb_sdk-actions.yml index 346ea3ff..999c09c4 100644 --- a/.github/workflows/kb_sdk-actions.yml +++ b/.github/workflows/kb_sdk-actions.yml @@ -1,187 +1,187 @@ -name: Basic kb-sdk tests - -on: - [push, pull_request] - -jobs: - - test_kb-sdk_builds: - if: "!contains(github.event.head_commit.message, 'skip ci')" - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] - language: [perl, python, java] - test_type: [base, example] - auth: [token, env_var] - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - - name: Build with Ant - run: | - make - - - name: Add bin to $PATH - run: | - echo "::add-path::$GITHUB_WORKSPACE/bin" - - - name: checking kb-sdk functions - env: - KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} - LANGUAGE_TOKEN: ${{ matrix.language }} - run: | - env - kb-sdk help - kb-sdk version - - - name: init base repo - if: matrix.test_type == 'base' - env: - LANGUAGE_TOKEN: ${{ matrix.language }} - run: | - kb-sdk init -l $LANGUAGE_TOKEN -u user SampleApp - - - name: init example repo - if: matrix.test_type == 'example' - env: - LANGUAGE_TOKEN: ${{ matrix.language }} - run: | - kb-sdk init -l $LANGUAGE_TOKEN -u user --example SampleApp - - - name: test ${{ matrix.test_type }} repo, using test.cfg for auth - if: matrix.auth == 'token' - run: | - cd SampleApp - kb-sdk test || true - - - name: test ${{ matrix.test_type }} repo, using test.cfg for auth - if: matrix.auth == 'token' - env: - KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} - run: | - cd SampleApp - sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg - kb-sdk test - - - name: test ${{ matrix.test_type }} repo, using env var for auth - if: matrix.auth == 'env_var' - env: - KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} - run: | - cd SampleApp - kb-sdk test - - - name: make resulting app available as artefact in case of failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: kbaseapp-${{ matrix.language }}-${{ matrix.test_type }}-${{ matrix.os }} - path: SampleApp - - - test_existing_repos: - if: "!contains(github.event.head_commit.message, 'skip ci')" - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] - app: [WsLargeDataIO, KBaseReport] - auth: [token, env_var] - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - - name: Build with Ant - run: | - make - - - name: Add bin to $PATH - run: | - echo "::add-path::$GITHUB_WORKSPACE/bin" - - - name: checking basic kb-sdk functions - run: | - env - kb-sdk help - kb-sdk version - - - name: Checkout existing kbase module ${{ matrix.app }} - uses: actions/checkout@v2 - with: - repository: kbaseapps/${{ matrix.app }} - path: kbase_app - - - name: run repo tests, using test.cfg for auth - if: matrix.auth == 'token' - run: | - cd kbase_app - kb-sdk test || true - - - name: run repo tests, using test.cfg for auth - if: matrix.auth == 'token' - env: - KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} - run: | - cd kbase_app - sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg - kb-sdk test - - - name: run repo tests, using env var for auth - if: matrix.auth == 'env_var' - env: - KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} - run: | - cd kbase_app - kb-sdk test - - - name: make resulting app available as artefact in case of failure - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: kbaseapp-${{ matrix.app }}-${{ matrix.os }} - path: kbase_app - - dockerhub_upload: - runs-on: ubuntu-latest - if: | - (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master') && - github.event_name == 'push' && - !contains(github.event.head_commit.message, 'skip docker') && - !contains(github.event.head_commit.message, 'skip ci') - needs: - - test_existing_repos - - test_kb-sdk_builds - - steps: - - name: checkout git repo - uses: actions/checkout@v2 - - - name: set current branch name as TAG_NAME for docker image - shell: bash - run: echo "::set-env name=TAG_NAME::${GITHUB_REF#refs/heads/}" - - - name: build and push to dockerhub - uses: opspresso/action-docker@master - with: - args: --docker - env: - USERNAME: ${{ secrets.DOCKER_USERNAME }} - PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - DOCKERFILE: "Dockerfile" - IMAGE_NAME: "kbase/kb-sdk" +# name: Basic kb-sdk tests + +# on: +# [push, pull_request] + +# jobs: + +# test_kb-sdk_builds: +# if: "!contains(github.event.head_commit.message, 'skip ci')" +# runs-on: ${{ matrix.os }} +# strategy: +# fail-fast: false +# matrix: +# os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] +# language: [perl, python, java] +# test_type: [base, example] +# auth: [token, env_var] + +# steps: +# - name: Checkout +# uses: actions/checkout@v2 + +# - name: Set up JDK 1.8 +# uses: actions/setup-java@v1 +# with: +# java-version: 1.8 + +# - name: Build with Ant +# run: | +# make + +# - name: Add bin to $PATH +# run: | +# echo "::add-path::$GITHUB_WORKSPACE/bin" + +# - name: checking kb-sdk functions +# env: +# KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} +# LANGUAGE_TOKEN: ${{ matrix.language }} +# run: | +# env +# kb-sdk help +# kb-sdk version + +# - name: init base repo +# if: matrix.test_type == 'base' +# env: +# LANGUAGE_TOKEN: ${{ matrix.language }} +# run: | +# kb-sdk init -l $LANGUAGE_TOKEN -u user SampleApp + +# - name: init example repo +# if: matrix.test_type == 'example' +# env: +# LANGUAGE_TOKEN: ${{ matrix.language }} +# run: | +# kb-sdk init -l $LANGUAGE_TOKEN -u user --example SampleApp + +# - name: test ${{ matrix.test_type }} repo, using test.cfg for auth +# if: matrix.auth == 'token' +# run: | +# cd SampleApp +# kb-sdk test || true + +# - name: test ${{ matrix.test_type }} repo, using test.cfg for auth +# if: matrix.auth == 'token' +# env: +# KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} +# run: | +# cd SampleApp +# sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg +# kb-sdk test + +# - name: test ${{ matrix.test_type }} repo, using env var for auth +# if: matrix.auth == 'env_var' +# env: +# KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} +# run: | +# cd SampleApp +# kb-sdk test + +# - name: make resulting app available as artefact in case of failure +# if: ${{ failure() }} +# uses: actions/upload-artifact@v2 +# with: +# name: kbaseapp-${{ matrix.language }}-${{ matrix.test_type }}-${{ matrix.os }} +# path: SampleApp + + +# test_existing_repos: +# if: "!contains(github.event.head_commit.message, 'skip ci')" +# runs-on: ${{ matrix.os }} +# strategy: +# fail-fast: false +# matrix: +# os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] +# app: [WsLargeDataIO, KBaseReport] +# auth: [token, env_var] + +# steps: +# - name: Checkout +# uses: actions/checkout@v2 + +# - name: Set up JDK 1.8 +# uses: actions/setup-java@v1 +# with: +# java-version: 1.8 + +# - name: Build with Ant +# run: | +# make + +# - name: Add bin to $PATH +# run: | +# echo "::add-path::$GITHUB_WORKSPACE/bin" + +# - name: checking basic kb-sdk functions +# run: | +# env +# kb-sdk help +# kb-sdk version + +# - name: Checkout existing kbase module ${{ matrix.app }} +# uses: actions/checkout@v2 +# with: +# repository: kbaseapps/${{ matrix.app }} +# path: kbase_app + +# - name: run repo tests, using test.cfg for auth +# if: matrix.auth == 'token' +# run: | +# cd kbase_app +# kb-sdk test || true + +# - name: run repo tests, using test.cfg for auth +# if: matrix.auth == 'token' +# env: +# KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} +# run: | +# cd kbase_app +# sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg +# kb-sdk test + +# - name: run repo tests, using env var for auth +# if: matrix.auth == 'env_var' +# env: +# KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} +# run: | +# cd kbase_app +# kb-sdk test + +# - name: make resulting app available as artefact in case of failure +# if: ${{ failure() }} +# uses: actions/upload-artifact@v2 +# with: +# name: kbaseapp-${{ matrix.app }}-${{ matrix.os }} +# path: kbase_app + +# dockerhub_upload: +# runs-on: ubuntu-latest +# if: | +# (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master') && +# github.event_name == 'push' && +# !contains(github.event.head_commit.message, 'skip docker') && +# !contains(github.event.head_commit.message, 'skip ci') +# needs: +# - test_existing_repos +# - test_kb-sdk_builds + +# steps: +# - name: checkout git repo +# uses: actions/checkout@v2 + +# - name: set current branch name as TAG_NAME for docker image +# shell: bash +# run: echo "::set-env name=TAG_NAME::${GITHUB_REF#refs/heads/}" + +# - name: build and push to dockerhub +# uses: opspresso/action-docker@master +# with: +# args: --docker +# env: +# USERNAME: ${{ secrets.DOCKER_USERNAME }} +# PASSWORD: ${{ secrets.DOCKER_PASSWORD }} +# DOCKERFILE: "Dockerfile" +# IMAGE_NAME: "kbase/kb-sdk" From dad228a584648b7d14e8098fe8a456d589f2c700 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 7 Mar 2023 09:02:27 -0800 Subject: [PATCH 37/47] Fixing kb_sdk tests, slimming down dockerfile --- .github/workflows/kb_sdk-actions.yml | 187 ---------------------- .github/workflows/test_functionality.yml | 195 +++++++++++++++++++++++ Dockerfile | 5 +- 3 files changed, 196 insertions(+), 191 deletions(-) delete mode 100644 .github/workflows/kb_sdk-actions.yml create mode 100644 .github/workflows/test_functionality.yml diff --git a/.github/workflows/kb_sdk-actions.yml b/.github/workflows/kb_sdk-actions.yml deleted file mode 100644 index 999c09c4..00000000 --- a/.github/workflows/kb_sdk-actions.yml +++ /dev/null @@ -1,187 +0,0 @@ -# name: Basic kb-sdk tests - -# on: -# [push, pull_request] - -# jobs: - -# test_kb-sdk_builds: -# if: "!contains(github.event.head_commit.message, 'skip ci')" -# runs-on: ${{ matrix.os }} -# strategy: -# fail-fast: false -# matrix: -# os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] -# language: [perl, python, java] -# test_type: [base, example] -# auth: [token, env_var] - -# steps: -# - name: Checkout -# uses: actions/checkout@v2 - -# - name: Set up JDK 1.8 -# uses: actions/setup-java@v1 -# with: -# java-version: 1.8 - -# - name: Build with Ant -# run: | -# make - -# - name: Add bin to $PATH -# run: | -# echo "::add-path::$GITHUB_WORKSPACE/bin" - -# - name: checking kb-sdk functions -# env: -# KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} -# LANGUAGE_TOKEN: ${{ matrix.language }} -# run: | -# env -# kb-sdk help -# kb-sdk version - -# - name: init base repo -# if: matrix.test_type == 'base' -# env: -# LANGUAGE_TOKEN: ${{ matrix.language }} -# run: | -# kb-sdk init -l $LANGUAGE_TOKEN -u user SampleApp - -# - name: init example repo -# if: matrix.test_type == 'example' -# env: -# LANGUAGE_TOKEN: ${{ matrix.language }} -# run: | -# kb-sdk init -l $LANGUAGE_TOKEN -u user --example SampleApp - -# - name: test ${{ matrix.test_type }} repo, using test.cfg for auth -# if: matrix.auth == 'token' -# run: | -# cd SampleApp -# kb-sdk test || true - -# - name: test ${{ matrix.test_type }} repo, using test.cfg for auth -# if: matrix.auth == 'token' -# env: -# KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} -# run: | -# cd SampleApp -# sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg -# kb-sdk test - -# - name: test ${{ matrix.test_type }} repo, using env var for auth -# if: matrix.auth == 'env_var' -# env: -# KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} -# run: | -# cd SampleApp -# kb-sdk test - -# - name: make resulting app available as artefact in case of failure -# if: ${{ failure() }} -# uses: actions/upload-artifact@v2 -# with: -# name: kbaseapp-${{ matrix.language }}-${{ matrix.test_type }}-${{ matrix.os }} -# path: SampleApp - - -# test_existing_repos: -# if: "!contains(github.event.head_commit.message, 'skip ci')" -# runs-on: ${{ matrix.os }} -# strategy: -# fail-fast: false -# matrix: -# os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] -# app: [WsLargeDataIO, KBaseReport] -# auth: [token, env_var] - -# steps: -# - name: Checkout -# uses: actions/checkout@v2 - -# - name: Set up JDK 1.8 -# uses: actions/setup-java@v1 -# with: -# java-version: 1.8 - -# - name: Build with Ant -# run: | -# make - -# - name: Add bin to $PATH -# run: | -# echo "::add-path::$GITHUB_WORKSPACE/bin" - -# - name: checking basic kb-sdk functions -# run: | -# env -# kb-sdk help -# kb-sdk version - -# - name: Checkout existing kbase module ${{ matrix.app }} -# uses: actions/checkout@v2 -# with: -# repository: kbaseapps/${{ matrix.app }} -# path: kbase_app - -# - name: run repo tests, using test.cfg for auth -# if: matrix.auth == 'token' -# run: | -# cd kbase_app -# kb-sdk test || true - -# - name: run repo tests, using test.cfg for auth -# if: matrix.auth == 'token' -# env: -# KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} -# run: | -# cd kbase_app -# sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg -# kb-sdk test - -# - name: run repo tests, using env var for auth -# if: matrix.auth == 'env_var' -# env: -# KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} -# run: | -# cd kbase_app -# kb-sdk test - -# - name: make resulting app available as artefact in case of failure -# if: ${{ failure() }} -# uses: actions/upload-artifact@v2 -# with: -# name: kbaseapp-${{ matrix.app }}-${{ matrix.os }} -# path: kbase_app - -# dockerhub_upload: -# runs-on: ubuntu-latest -# if: | -# (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master') && -# github.event_name == 'push' && -# !contains(github.event.head_commit.message, 'skip docker') && -# !contains(github.event.head_commit.message, 'skip ci') -# needs: -# - test_existing_repos -# - test_kb-sdk_builds - -# steps: -# - name: checkout git repo -# uses: actions/checkout@v2 - -# - name: set current branch name as TAG_NAME for docker image -# shell: bash -# run: echo "::set-env name=TAG_NAME::${GITHUB_REF#refs/heads/}" - -# - name: build and push to dockerhub -# uses: opspresso/action-docker@master -# with: -# args: --docker -# env: -# USERNAME: ${{ secrets.DOCKER_USERNAME }} -# PASSWORD: ${{ secrets.DOCKER_PASSWORD }} -# DOCKERFILE: "Dockerfile" -# IMAGE_NAME: "kbase/kb-sdk" - diff --git a/.github/workflows/test_functionality.yml b/.github/workflows/test_functionality.yml new file mode 100644 index 00000000..33ebf71a --- /dev/null +++ b/.github/workflows/test_functionality.yml @@ -0,0 +1,195 @@ +name: Basic kb-sdk functionality tests + +on: + pull_request: + branches: + - develop + - main + - master + types: + - opened + - reopened + - synchronize + - closed + +jobs: + + test_kb-sdk_builds: + if: "!contains(github.event.head_commit.message, 'skip ci')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] + language: [perl, python, java] + test_type: [base, example] + auth: [token, env_var] + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: Build with Ant + run: | + make + + - name: Add bin to $PATH + run: | + echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH + + - name: checking kb-sdk functions + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + env + kb-sdk help + kb-sdk version + + - name: init base repo + if: matrix.test_type == 'base' + env: + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + kb-sdk init -l $LANGUAGE_TOKEN -u user SampleApp + + - name: init example repo + if: matrix.test_type == 'example' + env: + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + kb-sdk init -l $LANGUAGE_TOKEN -u user --example SampleApp + + - name: test ${{ matrix.test_type }} repo, using test.cfg for auth + if: matrix.auth == 'token' + run: | + cd SampleApp + kb-sdk test || true + + - name: test ${{ matrix.test_type }} repo, using test.cfg for auth + if: matrix.auth == 'token' + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + run: | + cd SampleApp + sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg + kb-sdk test + + - name: test ${{ matrix.test_type }} repo, using env var for auth + if: matrix.auth == 'env_var' + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + run: | + cd SampleApp + kb-sdk test + + - name: make resulting app available as artefact in case of failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: kbaseapp-${{ matrix.language }}-${{ matrix.test_type }}-${{ matrix.os }} + path: SampleApp + + + test_existing_repos: + if: "!contains(github.event.head_commit.message, 'skip ci')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] + app: [WsLargeDataIO, KBaseReport] + auth: [token, env_var] + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: Build with Ant + run: | + make + + - name: Add bin to $PATH + run: | + echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH + + - name: checking basic kb-sdk functions + run: | + env + kb-sdk help + kb-sdk version + + - name: Checkout existing kbase module ${{ matrix.app }} + uses: actions/checkout@v2 + with: + repository: kbaseapps/${{ matrix.app }} + path: kbase_app + + - name: run repo tests, using test.cfg for auth + if: matrix.auth == 'token' + run: | + cd kbase_app + kb-sdk test || true + + - name: run repo tests, using test.cfg for auth + if: matrix.auth == 'token' + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + run: | + cd kbase_app + sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg + kb-sdk test + + - name: run repo tests, using env var for auth + if: matrix.auth == 'env_var' + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + run: | + cd kbase_app + kb-sdk test + + - name: make resulting app available as artefact in case of failure + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: kbaseapp-${{ matrix.app }}-${{ matrix.os }} + path: kbase_app + + dockerhub_upload: + runs-on: ubuntu-latest + if: | + (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master') && + github.event_name == 'push' && + !contains(github.event.head_commit.message, 'skip docker') && + !contains(github.event.head_commit.message, 'skip ci') + needs: + - test_existing_repos + - test_kb-sdk_builds + + steps: + - name: checkout git repo + uses: actions/checkout@v2 + + - name: set current branch name as TAG_NAME for docker image + shell: bash + run: echo "TAG_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + + - name: build and push to dockerhub + uses: opspresso/action-docker@master + with: + args: --docker + env: + USERNAME: ${{ secrets.DOCKER_USERNAME }} + PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKERFILE: "Dockerfile" + IMAGE_NAME: "kbase/kb-sdk" diff --git a/Dockerfile b/Dockerfile index 46ac1bd2..e151a6c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,11 +4,8 @@ MAINTAINER Shane Canon # Update apt and install jdk and docker engine to get docker clients # Docker installation instructions from https://docs.docker.com/engine/install/ubuntu/ RUN apt-get -y update && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install tzdata && \ - apt-get -y install openjdk-8-jdk make git ant curl gnupg-agent && \ - apt-get -y install apt-transport-https ca-certificates software-properties-common && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install tzdata openjdk-8-jdk make git ant curl gnupg-agent apt-transport-https ca-certificates software-properties-common && \ update-java-alternatives -s java-1.8.0-openjdk-amd64 - RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ add-apt-repository \ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ From 830e4af51815656020e21fe785a401769680ff17 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 7 Mar 2023 09:07:57 -0800 Subject: [PATCH 38/47] removing travis file, updating test workflow to remove push to github --- .github/workflows/test_functionality.yml | 47 +++++------------------- .travis.yml | 45 ----------------------- 2 files changed, 9 insertions(+), 83 deletions(-) delete mode 100644 .travis.yml diff --git a/.github/workflows/test_functionality.yml b/.github/workflows/test_functionality.yml index 33ebf71a..0e0246d3 100644 --- a/.github/workflows/test_functionality.yml +++ b/.github/workflows/test_functionality.yml @@ -20,17 +20,17 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] + os: [ubuntu-16.04, ubuntu-18.04, ubuntu-latest] language: [perl, python, java] test_type: [base, example] auth: [token, env_var] steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: 1.8 @@ -90,7 +90,7 @@ jobs: - name: make resulting app available as artefact in case of failure if: ${{ failure() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: kbaseapp-${{ matrix.language }}-${{ matrix.test_type }}-${{ matrix.os }} path: SampleApp @@ -102,16 +102,16 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-16.04, ubuntu-18.04, ubuntu-20.04] + os: [ubuntu-16.04, ubuntu-18.04, ubuntu-latest] app: [WsLargeDataIO, KBaseReport] auth: [token, env_var] steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: 1.8 @@ -130,7 +130,7 @@ jobs: kb-sdk version - name: Checkout existing kbase module ${{ matrix.app }} - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: kbaseapps/${{ matrix.app }} path: kbase_app @@ -160,36 +160,7 @@ jobs: - name: make resulting app available as artefact in case of failure if: ${{ failure() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: kbaseapp-${{ matrix.app }}-${{ matrix.os }} path: kbase_app - - dockerhub_upload: - runs-on: ubuntu-latest - if: | - (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master') && - github.event_name == 'push' && - !contains(github.event.head_commit.message, 'skip docker') && - !contains(github.event.head_commit.message, 'skip ci') - needs: - - test_existing_repos - - test_kb-sdk_builds - - steps: - - name: checkout git repo - uses: actions/checkout@v2 - - - name: set current branch name as TAG_NAME for docker image - shell: bash - run: echo "TAG_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - - name: build and push to dockerhub - uses: opspresso/action-docker@master - with: - args: --docker - env: - USERNAME: ${{ secrets.DOCKER_USERNAME }} - PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - DOCKERFILE: "Dockerfile" - IMAGE_NAME: "kbase/kb-sdk" diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4087b9a8..00000000 --- a/.travis.yml +++ /dev/null @@ -1,45 +0,0 @@ -language: java - -python: - - "2.7" - -jdk: - - openjdk8 - -sudo: required - -services: - - docker - -notifications: -# email: -# recipients: -# - username@email.dom -# on_success: never # default: change -# on_failure: always # default: always - -env: - global: -# Please see https://docs.travis-ci.com/user/encryption-keys/ for information -# about adding encryption keys which can be used to store auth tokens used in running apps against -# the KBase infrastructure -# secure: "blahblahblah" - -before_install: - - env - -install: - - pushd .. - - git clone https://github.com/kbase/jars - - popd - -script: - - make - - export PATH=$(pwd)/bin:$PATH - - source src/sh/sdk-completion.sh - - kb-sdk help - - cd .. - - kb-sdk init --example -l python -u sychan MyContigFilter - - cd MyContigFilter - - make - - kb-sdk validate From c1b938c2912e8f413a0a7439e55ea3bfeaab95f0 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 7 Mar 2023 09:12:33 -0800 Subject: [PATCH 39/47] adding missing distro tag to java action --- .github/workflows/test_functionality.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_functionality.yml b/.github/workflows/test_functionality.yml index 0e0246d3..6e6b1769 100644 --- a/.github/workflows/test_functionality.yml +++ b/.github/workflows/test_functionality.yml @@ -32,6 +32,7 @@ jobs: - name: Set up JDK 1.8 uses: actions/setup-java@v3 with: + distribution: temurin java-version: 1.8 - name: Build with Ant @@ -113,6 +114,7 @@ jobs: - name: Set up JDK 1.8 uses: actions/setup-java@v3 with: + distribution: temurin java-version: 1.8 - name: Build with Ant From 13caa0462c288fdddbcfac9582294e26d749d0ed Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 7 Mar 2023 09:20:48 -0800 Subject: [PATCH 40/47] fix java action again --- .github/workflows/test_functionality.yml | 4 ++-- Dockerfile | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_functionality.yml b/.github/workflows/test_functionality.yml index 6e6b1769..16ae279b 100644 --- a/.github/workflows/test_functionality.yml +++ b/.github/workflows/test_functionality.yml @@ -33,7 +33,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: temurin - java-version: 1.8 + java-version: 8.0.362+9 - name: Build with Ant run: | @@ -115,7 +115,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: temurin - java-version: 1.8 + java-version: 8.0.362+9 - name: Build with Ant run: | diff --git a/Dockerfile b/Dockerfile index e151a6c0..e56188cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ FROM ubuntu:20.04 -MAINTAINER Shane Canon +LABEL MAINTAINER KBase developers # Update apt and install jdk and docker engine to get docker clients # Docker installation instructions from https://docs.docker.com/engine/install/ubuntu/ RUN apt-get -y update && \ DEBIAN_FRONTEND=noninteractive apt-get -y install tzdata openjdk-8-jdk make git ant curl gnupg-agent apt-transport-https ca-certificates software-properties-common && \ - update-java-alternatives -s java-1.8.0-openjdk-amd64 -RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + update-java-alternatives -s java-1.8.0-openjdk-amd64 && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ add-apt-repository \ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) \ From 3acd45d39658d4b9564f61a12e36073ba3be74b9 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Fri, 10 Mar 2023 07:25:01 -0800 Subject: [PATCH 41/47] update to using two most recent ubuntus --- .github/workflows/test_functionality.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_functionality.yml b/.github/workflows/test_functionality.yml index 16ae279b..90d20ce2 100644 --- a/.github/workflows/test_functionality.yml +++ b/.github/workflows/test_functionality.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-16.04, ubuntu-18.04, ubuntu-latest] + os: [ubuntu-20.04, ubuntu-latest] language: [perl, python, java] test_type: [base, example] auth: [token, env_var] @@ -72,7 +72,7 @@ jobs: cd SampleApp kb-sdk test || true - - name: test ${{ matrix.test_type }} repo, using test.cfg for auth + - name: test ${{ matrix.test_type }} repo, using test.cfg for auth, add env var to test.cfg if: matrix.auth == 'token' env: KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} From 28f15f5826af59341742b4cffbad4d9c1c16e3f1 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Fri, 10 Mar 2023 07:28:42 -0800 Subject: [PATCH 42/47] disable generating the example module as the software dep is no longer available --- .github/workflows/test_functionality.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_functionality.yml b/.github/workflows/test_functionality.yml index 90d20ce2..a42a7867 100644 --- a/.github/workflows/test_functionality.yml +++ b/.github/workflows/test_functionality.yml @@ -22,7 +22,7 @@ jobs: matrix: os: [ubuntu-20.04, ubuntu-latest] language: [perl, python, java] - test_type: [base, example] + test_type: [base] #, example] auth: [token, env_var] steps: @@ -103,7 +103,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-16.04, ubuntu-18.04, ubuntu-latest] + os: [ubuntu-20.04, ubuntu-latest] app: [WsLargeDataIO, KBaseReport] auth: [token, env_var] From edd91461d646729a52f80b0658e9fa75bdd9ae72 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 15 Mar 2023 08:54:03 -0700 Subject: [PATCH 43/47] Disable testing of WSLargeDataIO as it relies on a Java repository that no longer exists --- .../{test_functionality.yml => basic_sdk_functionality.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{test_functionality.yml => basic_sdk_functionality.yml} (100%) diff --git a/.github/workflows/test_functionality.yml b/.github/workflows/basic_sdk_functionality.yml similarity index 100% rename from .github/workflows/test_functionality.yml rename to .github/workflows/basic_sdk_functionality.yml From 8f14d8069b2d177cd2ac797b34174239d11d8347 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 15 Mar 2023 09:16:23 -0700 Subject: [PATCH 44/47] disable WsLargeDataIO tests --- .github/workflows/basic_sdk_functionality.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/basic_sdk_functionality.yml b/.github/workflows/basic_sdk_functionality.yml index a42a7867..be671d47 100644 --- a/.github/workflows/basic_sdk_functionality.yml +++ b/.github/workflows/basic_sdk_functionality.yml @@ -104,7 +104,9 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04, ubuntu-latest] - app: [WsLargeDataIO, KBaseReport] + # TODO: add a Java app. WsLargeDataIO does not currently build as it relies on a Java + # repo that no longer exists. + app: [KBaseReport] # , WsLargeDataIO] auth: [token, env_var] steps: From 1a1e51e79e58890482ea383876bc0c4a0d5a78a8 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 15 Mar 2023 10:11:19 -0700 Subject: [PATCH 45/47] Renaming workflow file, adding comments about what tests are doing --- ..._functionality.yml => basic_sdk_functionality_tests.yml} | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) rename .github/workflows/{basic_sdk_functionality.yml => basic_sdk_functionality_tests.yml} (93%) diff --git a/.github/workflows/basic_sdk_functionality.yml b/.github/workflows/basic_sdk_functionality_tests.yml similarity index 93% rename from .github/workflows/basic_sdk_functionality.yml rename to .github/workflows/basic_sdk_functionality_tests.yml index be671d47..b6797f02 100644 --- a/.github/workflows/basic_sdk_functionality.yml +++ b/.github/workflows/basic_sdk_functionality_tests.yml @@ -13,7 +13,6 @@ on: - closed jobs: - test_kb-sdk_builds: if: "!contains(github.event.head_commit.message, 'skip ci')" runs-on: ${{ matrix.os }} @@ -29,6 +28,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 + # build kb-sdk from source and test basic functions - name: Set up JDK 1.8 uses: actions/setup-java@v3 with: @@ -52,6 +52,9 @@ jobs: kb-sdk help kb-sdk version + # the following steps create a KBase app using the SDK + # and runs some checks to ensure the app generated + # is functional and that the SDK can test it. - name: init base repo if: matrix.test_type == 'base' env: @@ -133,6 +136,7 @@ jobs: kb-sdk help kb-sdk version + # the following steps check out and run the tests of an existing KBase app - name: Checkout existing kbase module ${{ matrix.app }} uses: actions/checkout@v3 with: From 89e620e2aaeb3b76b23450ab0c8e966863220cec Mon Sep 17 00:00:00 2001 From: bio-boris Date: Wed, 15 Mar 2023 12:51:06 -0500 Subject: [PATCH 46/47] Update python_server.vm.properties --- src/java/us/kbase/templates/python_server.vm.properties | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/java/us/kbase/templates/python_server.vm.properties b/src/java/us/kbase/templates/python_server.vm.properties index dbf6241c..c5d7d177 100644 --- a/src/java/us/kbase/templates/python_server.vm.properties +++ b/src/java/us/kbase/templates/python_server.vm.properties @@ -134,7 +134,11 @@ class JSONRPCServiceCustom(JSONRPCService): newerr = JSONServerError() newerr.trace = traceback.format_exc() if len(e.args) == 1: - newerr.data = repr(e.args[0]) + # repr adds single quotes around string arguments which is not what we want. + if type(e.args[0]) == str: + newerr.data = e.args[0] + else: + newerr.data = repr(e.args[0]) else: newerr.data = repr(e.args) raise newerr From 0ab2a97b2b0376713bbe8580b4909042e6be250f Mon Sep 17 00:00:00 2001 From: Jason S Fillman Date: Thu, 26 May 2022 17:31:20 -0700 Subject: [PATCH 47/47] Updating docker install to enable `docker buildx` --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e56188cf..a860b3ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN apt-get -y update && \ $(lsb_release -cs) \ stable" && \ apt-get -y update && \ - apt-get -y install docker-ce docker-ce-cli containerd.io + apt-get -y install docker-ce docker-ce-cli containerd.io docker-compose-plugin # Add kb-sdk src and fix CallbackServer interface ADD . /src