diff --git a/.github/workflows/basic_sdk_functionality_tests.yml b/.github/workflows/basic_sdk_functionality_tests.yml new file mode 100644 index 00000000..b6797f02 --- /dev/null +++ b/.github/workflows/basic_sdk_functionality_tests.yml @@ -0,0 +1,174 @@ +name: Basic kb-sdk functionality tests + +on: + pull_request: + branches: + - develop + - main + - master + types: + - opened + - reopened + - synchronize + - closed + +jobs: + test_kb-sdk_builds: + if: "!contains(github.event.head_commit.message, 'skip ci')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, ubuntu-latest] + language: [perl, python, java] + test_type: [base] #, example] + auth: [token, env_var] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + # build kb-sdk from source and test basic functions + - name: Set up JDK 1.8 + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 8.0.362+9 + + - name: Build with Ant + run: | + make + + - name: Add bin to $PATH + run: | + echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH + + - name: checking kb-sdk functions + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + env + kb-sdk help + kb-sdk version + + # the following steps create a KBase app using the SDK + # and runs some checks to ensure the app generated + # is functional and that the SDK can test it. + - name: init base repo + if: matrix.test_type == 'base' + env: + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + kb-sdk init -l $LANGUAGE_TOKEN -u user SampleApp + + - name: init example repo + if: matrix.test_type == 'example' + env: + LANGUAGE_TOKEN: ${{ matrix.language }} + run: | + kb-sdk init -l $LANGUAGE_TOKEN -u user --example SampleApp + + - name: test ${{ matrix.test_type }} repo, using test.cfg for auth + if: matrix.auth == 'token' + run: | + cd SampleApp + kb-sdk test || true + + - name: test ${{ matrix.test_type }} repo, using test.cfg for auth, add env var to test.cfg + if: matrix.auth == 'token' + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + run: | + cd SampleApp + sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg + kb-sdk test + + - name: test ${{ matrix.test_type }} repo, using env var for auth + if: matrix.auth == 'env_var' + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + run: | + cd SampleApp + kb-sdk test + + - name: make resulting app available as artefact in case of failure + if: ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: kbaseapp-${{ matrix.language }}-${{ matrix.test_type }}-${{ matrix.os }} + path: SampleApp + + + test_existing_repos: + if: "!contains(github.event.head_commit.message, 'skip ci')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, ubuntu-latest] + # TODO: add a Java app. WsLargeDataIO does not currently build as it relies on a Java + # repo that no longer exists. + app: [KBaseReport] # , WsLargeDataIO] + auth: [token, env_var] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up JDK 1.8 + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 8.0.362+9 + + - name: Build with Ant + run: | + make + + - name: Add bin to $PATH + run: | + echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH + + - name: checking basic kb-sdk functions + run: | + env + kb-sdk help + kb-sdk version + + # the following steps check out and run the tests of an existing KBase app + - name: Checkout existing kbase module ${{ matrix.app }} + uses: actions/checkout@v3 + with: + repository: kbaseapps/${{ matrix.app }} + path: kbase_app + + - name: run repo tests, using test.cfg for auth + if: matrix.auth == 'token' + run: | + cd kbase_app + kb-sdk test || true + + - name: run repo tests, using test.cfg for auth + if: matrix.auth == 'token' + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + run: | + cd kbase_app + sed -i "s/test_token=/test_token=$KBASE_TEST_TOKEN/" test_local/test.cfg + kb-sdk test + + - name: run repo tests, using env var for auth + if: matrix.auth == 'env_var' + env: + KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }} + run: | + cd kbase_app + kb-sdk test + + - name: make resulting app available as artefact in case of failure + if: ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: kbaseapp-${{ matrix.app }}-${{ matrix.os }} + path: kbase_app diff --git a/.github/workflows/manual-build.yml b/.github/workflows/manual-build.yml new file mode 100644 index 00000000..944f9035 --- /dev/null +++ b/.github/workflows/manual-build.yml @@ -0,0 +1,11 @@ +--- +name: Manual Build & Push +on: + workflow_dispatch: +jobs: + build-push: + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}-develop' + tags: br-${{ github.ref_name }} + secrets: inherit diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml new file mode 100644 index 00000000..0fa1c464 --- /dev/null +++ b/.github/workflows/pr_build.yml @@ -0,0 +1,43 @@ +--- +name: Pull Request Build, Tag, & Push +on: + pull_request: + branches: + - develop + - main + - master + types: + - opened + - reopened + - synchronize + - closed +jobs: + build-develop-open: + if: github.base_ref == 'develop' && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_build.yml@main + secrets: inherit + build-develop-merge: + if: github.base_ref == 'develop' && github.event.pull_request.merged == true + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}-develop' + tags: pr-${{ github.event.number }},latest + secrets: inherit + build-main-open: + if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: pr-${{ github.event.number }} + secrets: inherit + build-main-merge: + if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == true + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: pr-${{ github.event.number }},latest-rc + secrets: inherit + trivy-scans: + if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main + secrets: inherit diff --git a/.github/workflows/release-main.yml b/.github/workflows/release-main.yml new file mode 100644 index 00000000..a2546781 --- /dev/null +++ b/.github/workflows/release-main.yml @@ -0,0 +1,25 @@ +--- +name: Release - Build & Push Image +on: + release: + branches: + - main + - master + types: [ published ] +jobs: + check-source-branch: + uses: kbase/.github/.github/workflows/reusable_validate-branch.yml@main + with: + build_branch: '${{ github.event.release.target_commitish }}' + validate-release-tag: + needs: check-source-branch + uses: kbase/.github/.github/workflows/reusable_validate-release-tag.yml@main + with: + release_tag: '${{ github.event.release.tag_name }}' + build-push: + needs: validate-release-tag + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: '${{ github.event.release.tag_name }},latest' + secrets: inherit diff --git a/.gitignore b/.gitignore index 1cf1ed55..be10695a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,6 @@ /temp /dist /eclipse-classes/ -/sdkbase/ssl -/sdkbase/cluster.ini /classes/ /temp_* /ASimpleModule* @@ -13,3 +11,4 @@ /test_scripts/test.cfg *.pyc env-*/ +submodules/auth diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index fd30344f..00000000 --- a/.travis.yml +++ /dev/null @@ -1,45 +0,0 @@ -language: java - -python: - - "2.7" - -sudo: required - -services: - - docker - -notifications: -# email: -# recipients: -# - username@email.dom -# on_success: never # default: change -# on_failure: always # default: always - -env: - global: -# Please see https://docs.travis-ci.com/user/encryption-keys/ for information -# about adding encryption keys which can be used to store auth tokens used in running apps against -# the KBase infrastructure -# secure: "blahblahblah" - -before_install: - - env - -install: - - pushd .. - - git clone https://github.com/kbase/jars - - popd - -script: - - make - - make sdkbase - - export PATH=$(pwd)/bin:$PATH - - source src/sh/sdk-completion.sh - - kb-sdk help - - cd .. - - kb-sdk init --example -l python -u sychan MyContigFilter - - cd MyContigFilter - - make - - kb-sdk validate - - diff --git a/Dockerfile b/Dockerfile index 63eb7f27..a860b3ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,27 @@ -FROM ubuntu:14.04 -MAINTAINER Shane Canon +FROM ubuntu:20.04 +LABEL MAINTAINER KBase developers # Update apt and install jdk and docker engine to get docker clients +# Docker installation instructions from https://docs.docker.com/engine/install/ubuntu/ RUN apt-get -y update && \ - apt-get -y install openjdk-7-jdk make git ant && \ - apt-get -y install apt-transport-https ca-certificates && \ - apt-key adv \ - --keyserver hkp://ha.pool.sks-keyservers.net:80 \ - --recv-keys 58118E89F3A912897C070ADBF76221572C52609D && \ - echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" > /etc/apt/sources.list.d/docker.list && \ - apt-get -y update && apt-get -y install docker-engine=1.11.2-0~trusty + DEBIAN_FRONTEND=noninteractive apt-get -y install tzdata openjdk-8-jdk make git ant curl gnupg-agent apt-transport-https ca-certificates software-properties-common && \ + update-java-alternatives -s java-1.8.0-openjdk-amd64 && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" && \ + apt-get -y update && \ + apt-get -y install docker-ce docker-ce-cli containerd.io docker-compose-plugin +# Add kb-sdk src and fix CallbackServer interface ADD . /src -# Add kb_sdk src and fix CallbackServer interface RUN \ - cd /src && \ - sed -i 's/en0/eth0/' src/java/us/kbase/common/executionengine/CallbackServer.java && \ - make && \ - /src/entrypoint prune && rm -rf /src/.git + cd /src && \ + sed -i 's/en0/eth0/' src/java/us/kbase/common/executionengine/CallbackServer.java && \ + make && \ + /src/entrypoint prune && rm -rf /src/.git ENV PATH=$PATH:/src/bin diff --git a/Makefile b/Makefile index 2ecd55e3..773931f0 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ KBASE_COMMON_JAR = kbase/common/kbase-common-0.0.23.jar QUOTE = '\'' # make sure our make test works -.PHONY : test test-python sdkbase +.PHONY : test test-python default: compile @@ -59,7 +59,7 @@ bin: jars-submodule-init echo '#!/bin/bash' > bin/kb-sdk echo 'DIR=$(DIR)' >> bin/kb-sdk echo 'KBASE_JARS_DIR=$$DIR/submodules/jars/lib/jars' >> bin/kb-sdk - @# Next command processes links in JAR_DEPS_BIN file and has 5 parts (one on each line): + @# Next command processes links in JAR_DEPS_BIN file and has 5 parts (one on each line): @# (1) removing comments @# (2) trimming each line (picking first word actually) @# (3) skipping empty lines @@ -103,11 +103,6 @@ deploy-scripts: fi; $(ANT) deploy_bin -DBIN_TARGET=$(TARGET)/bin -DBIN_LIB_TARGET=$(TARGET)/lib -DKBASE_COMMON_JAR=$(KBASE_COMMON_JAR) -sdkbase: - # docker rmi -f kbase/deplbase:latest - cd sdkbase && ./makeconfig - docker build --no-cache -t kbase/kbase:sdkbase2.latest sdkbase - test: submodule-init @echo "Running unit tests" make test-python diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index ff1e4f9b..bdcb0c2d 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,11 +1,41 @@ OVERVIEW ----------------------------------------- -The KBase SDK is a set of tools for developing new modules -that can be dynamically registered and run on the KBase -platform. Modules include all code, specification files, and -documentation needed to define and run a set of methods +The KBase SDK is a set of tools for developing new modules +that can be dynamically registered and run on the KBase +platform. Modules include all code, specification files, and +documentation needed to define and run a set of methods in the KBase Narrative interface. +VERSION: 1.2.6 (Released 10/2020) +--------------------------------- +CHANGES: +- "-t" flag removed from entrypoint commands to enable kb-sdk to be run by github actions. + +VERSION: 1.2.5 (Unreleased) +--------------------------------- +- `kb-sdk test` no longer can see the docker socket due to a docker permissions issue on the docker socket. By default, the user running in the kb-sdk container now runs as the root user + +VERSION: 1.2.4 (Unreleased) +--------------------------------- +CHANGES: +- `kb-sdk test` no longer dies on its first run; if the user has the KBASE_TEST_TOKEN environment variable set with their developer token, tests can be run without having to edit the test.cfg file. + + +VERSION: 1.2.3 (Released 06/2020) +------------------------------------ +NEW FEATURES: +- updated kb-sdk Dockerfile to use Java 8, the most recent Docker engine, and Ubuntu 20.04. + +VERSION: 1.2.2 (Released 06/2020) +------------------------------------ +NEW FEATURES: +- kb-sdk version reports the build time and date of the instance +- kb-sdk can use the environment variable KBASE_TEST_TOKEN if a test token is not present in the test.cfg file +- GitHub Actions now used to run basic tests on initialising and testing new base and sample repos. + +BUG FIXES: +- Corrected a class name in the generated tests for the Java SDK module. + VERSION: 1.2.1 (Released 12/04/2018) ------------------------------------ CHANGES: @@ -85,7 +115,7 @@ VERSION: 1.0.0 (Released 11/19/2015) ------------------------------------------ NEW FEATURES: - R language is now supported for generation of clients and server stubs. -- Test sub-command is fully implemented for all 4 languages (perl, +- Test sub-command is fully implemented for all 4 languages (perl, python, java and R). - Version sub-command was added diff --git a/build.xml b/build.xml index abbaa391..6b82a7fa 100644 --- a/build.xml +++ b/build.xml @@ -49,7 +49,7 @@ - + @@ -102,7 +102,7 @@ - + @@ -116,7 +116,8 @@ ### PLEASE DO NOT CHANGE THIS FILE MANUALLY! ### giturl=${git.url} branch=${git.branch} -commit=${git.commit} +commit=${git.commit} +build_timestamp=${DSTAMP}-${TSTAMP} Compiling with jar directory: ${jardir} @@ -205,7 +206,7 @@ echo $pid > $DIR/pid.txt Successfully built: ${ee.dir}/${ee.start.script} - + #!/bin/bash DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) kill $(<"$DIR/pid.txt") diff --git a/doc/building_sdk.md b/doc/building_sdk.md index 04a17f8b..cc6d1866 100644 --- a/doc/building_sdk.md +++ b/doc/building_sdk.md @@ -1,9 +1,9 @@ -#### Building SDK +## Building the SDK System Dependencies: - Mac OS X 10.8+ or Linux. kb-sdk does not run natively in Windows, but see [here](doc/FAQ.md#windows) for more details. - - Java JRE 7 or 8 (9 is currently incompatible) http://www.oracle.com/technetwork/java/javase/downloads/index.html + - Java JRE 8: http://www.oracle.com/technetwork/java/javase/downloads/index.html (9 is currently incompatible; the SDK will run on Java 7, but using the more modern Java 8 is recommended) - (Mac only) Xcode https://developer.apple.com/xcode - git https://git-scm.com - Docker https://www.docker.com (for local testing) @@ -11,24 +11,20 @@ System Dependencies: Get the SDK: git clone https://github.com/kbase/kb_sdk + git clone https://github.com/kbase/jars Pull dependencies and configure the SDK: cd kb_sdk make bin -Download the local KBase SDK base Docker image: - - docker pull kbase/sdkbase2:latest - Add the kb-sdk tool to your PATH and enable command completion. From the kb_sdk directory: # for bash export PATH=$(pwd)/bin:$PATH source src/sh/sdk-completion.sh - -#### Build from source +### Build from source Additional System Dependencies: @@ -39,4 +35,3 @@ Follow basic instructions above. Instead of running `make bin` you can run `mak cd kb_sdk make - diff --git a/doc/codebase_anatomy.md b/doc/codebase_anatomy.md index 91ea0e9e..3ef5aa2c 100644 --- a/doc/codebase_anatomy.md +++ b/doc/codebase_anatomy.md @@ -16,7 +16,6 @@ This document describes the the file structure of the `kb_sdk` codebase. * `Makefile` - commands for compiling, building docker images, and initializing submodules * `Pipfile` and `Pipfile.lock` - python dependencies for pipenv * `reports/` - files generated for JaCoCo test coverage reports -* `sdkbase/` - Docker files for the image used inside actual SDK apps * `src/` - the main source code for this project; see below * `submodules/` and `submodules_hacks` - jars and other dependencies * `test_scripts/` - test helpers in perl, python, and js diff --git a/entrypoint b/entrypoint index e12d17f4..ddeca8fc 100755 --- a/entrypoint +++ b/entrypoint @@ -1,11 +1,10 @@ #!/bin/sh - if [ "z$1" = "zshell" ] ; then exec $DSHELL -l elif [ "z$1" = "zsetup" ] ; then cat << EOF G=\$(docker run -i -v /var/run/docker.sock:/var/run/docker.sock --entrypoint ls kbase/kb-sdk -l /var/run/docker.sock|awk '{print \$4}'); -alias kb-sdk='docker run -it --rm -v \$HOME:\$HOME -u \$(id -u) -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL --group-add \$G kbase/kb-sdk' +alias kb-sdk='docker run -i --rm -v \$HOME:\$HOME -u 0 -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e KBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$G kbase/kb-sdk' EOF elif [ "z$1" = "zgenscript" ] ; then cat << EOF @@ -16,11 +15,8 @@ if [ ! -e \$HOME/.kbsdk.cache ] ; then docker run -i -v /var/run/docker.sock:/var/run/docker.sock --entrypoint ls kbase/kb-sdk -l /var/run/docker.sock|awk '{print \$4}' > \$HOME/.kbsdk.cache fi -exec docker run -it --rm -v \$HOME:\$HOME -u \$(id -u) -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL --group-add \$(cat \$HOME/.kbsdk.cache) kbase/kb-sdk \$@ +exec docker run -i --rm -v \$HOME:\$HOME -u 0 -w \$(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=\$USER -e DSHELL=\$SHELL -e KBASE_TEST_TOKEN=\$KBASE_TEST_TOKEN --group-add \$(cat \$HOME/.kbsdk.cache) kbase/kb-sdk \$@ EOF -elif [ "z$1" = "zsdkbase" ] || [ "z$1" = "zpull-base-image" ] ; then - echo "Pulling and tagging the base image" - docker pull kbase/kbase:sdkbase2.latest elif [ "z$1" = "zprune" ] ; then echo "Used during build to shrink image. Not needed by the user." for f in $(find /src/submodules/jars/lib/jars -type f -name '*.jar') ; do diff --git a/lib/Bio/KBase/DeploymentConfig.pm b/lib/Bio/KBase/DeploymentConfig.pm index 4bb54573..8e9d0794 100644 --- a/lib/Bio/KBase/DeploymentConfig.pm +++ b/lib/Bio/KBase/DeploymentConfig.pm @@ -1,10 +1,12 @@ package Bio::KBase::DeploymentConfig; use strict; +use warnings; + use base 'Class::Accessor'; use Config::Simple; -__PACKAGE__->mk_accessors(qw(service_name settings)); +__PACKAGE__->mk_accessors( qw( service_name settings ) ); =head1 NAME @@ -13,9 +15,9 @@ Bio::KBase::DeploymentConfig =head1 DESCRIPTION The C class wraps the access to a KBase deployment.cfg -file. It tests for the existence of the KB_DEPLOYMENT_CONFIG and -KB_SERVICE_NAME environment variables; if both are present, the -configuration parameters for the given service will be loaded +file. It tests for the existence of the KB_DEPLOYMENT_CONFIG and +KB_SERVICE_NAME environment variables; if both are present, the +configuration parameters for the given service will be loaded from that config file. If they are not present, the module supports fallback to defaults as defined by the module. @@ -46,42 +48,36 @@ A hash reference containing the default values for the service parameters. =back =cut - -sub new -{ - my($class, $service_name, $defaults) = @_; - - if ((my $n = $ENV{KB_SERVICE_NAME}) ne "") - { - $service_name = $n; + +sub new { + my ( $class, $service_name, $defaults ) = @_; + + if ( ( my $n = $ENV{ KB_SERVICE_NAME } ) ne "" ) { + $service_name = $n; } my $settings = {}; - if (ref($defaults)) - { - %$settings = %$defaults; + if ( ref( $defaults ) ) { + %$settings = %$defaults; } - my $cfg_file = $ENV{KB_DEPLOYMENT_CONFIG}; - if (-e $cfg_file) - { - my $cfg = Config::Simple->new(); - $cfg->read($cfg_file); - - my %cfg = $cfg->vars; - - for my $k (keys %cfg) - { - if ($k =~ /^$service_name\.(.*)/) - { - $settings->{$1} = $cfg{$k}; - } - } + my $cfg_file = $ENV{ KB_DEPLOYMENT_CONFIG }; + if ( -e $cfg_file ) { + my $cfg = Config::Simple->new(); + $cfg->read( $cfg_file ); + + my %config = $cfg->vars; + + for my $k ( keys %config ) { + if ( $k =~ /^$service_name\.(.*)/ ) { + $settings->{ $1 } = $config{ $k }; + } + } } my $self = { - settings => $settings, - service_name => $service_name, + settings => $settings, + service_name => $service_name, }; return bless $self, $class; @@ -91,14 +87,13 @@ sub new Retrieve a setting from the configuration. - my $value = $obj->setting("key-name"); + my $value = $obj->setting( "key-name" ); =cut -sub setting -{ - my($self, $key) = @_; - return $self->{settings}->{$key}; +sub setting { + my ( $self, $key ) = @_; + return $self->{ settings }->{ $key }; } =item C diff --git a/lib/Bio/KBase/Exceptions.pm b/lib/Bio/KBase/Exceptions.pm index b7fdf876..37a9a5ef 100644 --- a/lib/Bio/KBase/Exceptions.pm +++ b/lib/Bio/KBase/Exceptions.pm @@ -1,50 +1,57 @@ package Bio::KBase::Exceptions; -use Exception::Class - ( - Bio::KBase::Exceptions::KBaseException => { - description => 'KBase exception', - fields => ['method_name'], - }, - - Bio::KBase::Exceptions::JSONRPC => { - description => 'JSONRPC error', - fields => ['code', 'data'], - isa => 'Bio::KBase::Exceptions::KBaseException', - }, - - Bio::KBase::Exceptions::HTTP => { - description => 'HTTP error', - fields => ['status_line'], - isa => 'Bio::KBase::Exceptions::KBaseException', - }, - - Bio::KBase::Exceptions::ArgumentValidationError => { - description => 'argument validation error', - fields => ['method_name'], - isa => 'Bio::KBase::Exceptions::KBaseException', - }, - - ); - -Bio::KBase::Exceptions::KBaseException->Trace(1); +use strict; +use warnings; + +use Exception::Class ( + Bio::KBase::Exceptions::KBaseException => { + description => 'KBase exception', + fields => [ 'method_name' ], + }, + + Bio::KBase::Exceptions::JSONRPC => { + description => 'JSONRPC error', + fields => [ 'code', 'data' ], + isa => 'Bio::KBase::Exceptions::KBaseException', + }, + + Bio::KBase::Exceptions::HTTP => { + description => 'HTTP error', + fields => [ 'status_line' ], + isa => 'Bio::KBase::Exceptions::KBaseException', + }, + + Bio::KBase::Exceptions::ArgumentValidationError => { + description => 'argument validation error', + fields => [ 'method_name' ], + isa => 'Bio::KBase::Exceptions::KBaseException', + }, +); + +Bio::KBase::Exceptions::KBaseException->Trace( 1 ); package Bio::KBase::Exceptions::HTTP; use strict; - -sub full_message -{ - my($self) = @_; - return $self->message . "\nHTTP status: " . $self->status_line . "\nFunction invoked: " . $self->method_name; +use warnings; + +sub full_message { + my ( $self ) = @_; + return + $self->message + . "\nHTTP status: " + . $self->status_line + . "\nFunction invoked: " + . $self->method_name; } package Bio::KBase::Exceptions::JSONRPC; use strict; +use warnings; -sub full_message -{ - my($self) = @_; - return sprintf("JSONRPC error:\n%s\nJSONRPC error code: %s\nJSONRPC error data:%s\n", $self->message, $self->code, $self->data); +sub full_message { + my ( $self ) = @_; + return sprintf( "JSONRPC error:\n%s\nJSONRPC error code: %s\nJSONRPC error data:%s\n", + $self->message, $self->code, $self->data ); } 1; diff --git a/lib/Bio/KBase/Log.pm b/lib/Bio/KBase/Log.pm index ede10352..68b41eda 100644 --- a/lib/Bio/KBase/Log.pm +++ b/lib/Bio/KBase/Log.pm @@ -14,51 +14,52 @@ use Time::HiRes; use Sys::Syslog qw( :DEFAULT setlogsock); # $ENV{'MLOG_CONFIG_FILE'} should point to an INI-formatted file, or an empty string, or should not exist. -our $MLOG_ENV_FILE = "MLOG_CONFIG_FILE"; -my $_GLOBAL = "global"; -our $MLOG_LOG_LEVEL = "mlog_log_level"; -our $MLOG_API_URL = "mlog_api_url"; -our $MLOG_LOG_FILE = "mlog_log_file"; - -our $DEFAULT_LOG_LEVEL = 6; -our $MSG_FACILITY = 'local1'; -our $EMERG_FACILITY = 'local0'; - -our $EMERG = 0; -our $ALERT = 1; -our $CRIT = 2; -our $ERR = 3; +our $MLOG_ENV_FILE = "MLOG_CONFIG_FILE"; +my $_GLOBAL = "global"; +our $MLOG_LOG_LEVEL = "mlog_log_level"; +our $MLOG_API_URL = "mlog_api_url"; +our $MLOG_LOG_FILE = "mlog_log_file"; + +our $DEFAULT_LOG_LEVEL = 6; +our $MSG_FACILITY = 'local1'; +our $EMERG_FACILITY = 'local0'; + +our $EMERG = 0; +our $ALERT = 1; +our $CRIT = 2; +our $ERR = 3; our $WARNING = 4; -our $NOTICE = 5; -our $INFO = 6; -our $DEBUG = 7; -our $DEBUG2 = 8; -our $DEBUG3 = 9; - -my $_MLOG_TEXT_TO_LEVEL = { 'EMERG' => $EMERG, - 'ALERT' => $ALERT, - 'CRIT' => $CRIT, - 'ERR' => $ERR, - 'WARNING' => $WARNING, - 'NOTICE' => $NOTICE, - 'INFO' => $INFO, - 'DEBUG' => $DEBUG, - 'DEBUG2' => $DEBUG2, - 'DEBUG3' => $DEBUG3, - }; - -my @_MLOG_TO_SYSLOG = (0, 1, 2, 3, 4, 5, 6, 7, 7, 7); +our $NOTICE = 5; +our $INFO = 6; +our $DEBUG = 7; +our $DEBUG2 = 8; +our $DEBUG3 = 9; + +my $_MLOG_TEXT_TO_LEVEL = { + 'EMERG' => $EMERG, + 'ALERT' => $ALERT, + 'CRIT' => $CRIT, + 'ERR' => $ERR, + 'WARNING' => $WARNING, + 'NOTICE' => $NOTICE, + 'INFO' => $INFO, + 'DEBUG' => $DEBUG, + 'DEBUG2' => $DEBUG2, + 'DEBUG3' => $DEBUG3, +}; + +my @_MLOG_TO_SYSLOG = ( 0, 1, 2, 3, 4, 5, 6, 7, 7, 7 ); my $_MLOG_LEVEL_TO_TEXT = {}; -foreach my $k (keys %{$_MLOG_TEXT_TO_LEVEL}) { - $_MLOG_LEVEL_TO_TEXT->{$_MLOG_TEXT_TO_LEVEL->{$k}} = $k; +foreach my $k ( keys %{ $_MLOG_TEXT_TO_LEVEL } ) { + $_MLOG_LEVEL_TO_TEXT->{ $_MLOG_TEXT_TO_LEVEL->{ $k } } = $k; } -our $LOG_LEVEL_MIN = min(keys %{$_MLOG_LEVEL_TO_TEXT}); -our $LOG_LEVEL_MAX = max(keys %{$_MLOG_LEVEL_TO_TEXT}); +our $LOG_LEVEL_MIN = min( keys %{ $_MLOG_LEVEL_TO_TEXT } ); +our $LOG_LEVEL_MAX = max( keys %{ $_MLOG_LEVEL_TO_TEXT } ); -our $USER = $ENV{'USER'}; -our $PARENT_FILE = abs_path($0); +our $USER = $ENV{ 'USER' }; +our $PARENT_FILE = abs_path( $0 ); =pod @@ -174,275 +175,303 @@ clear_user_log_level() : Removes the user-defined log level. =cut sub _get_option { - my ($opts, $key) = @_; - if(!(defined $opts)) { + my ( $opts, $key ) = @_; + if ( !( defined $opts ) ) { return undef; } - return defined $opts->{$key} ? $opts->{$key} : undef; + return defined $opts->{ $key } ? $opts->{ $key } : undef; } sub new { - my ($class, $sub, $lc, $options) = @_; - unless(defined $sub) { + my ( $class, $sub, $lc, $options ) = @_; + unless ( defined $sub ) { die "ERROR: You must define a subsystem when calling init_log()\n"; } my $self = {}; bless $self, $class; - $self->{authuser} = _get_option($options, 'authuser'); - $self->{module} = _get_option($options, 'module'); - $self->{method} = _get_option($options, 'method'); - $self->{call_id} = _get_option($options, 'call_id'); - $self->{ip_address} = _get_option($options, 'ip_address'); - $self->{tag} = _get_option($options, 'tag'); - $self->{_callback} = defined $options->{"changecallback"} ? - $options->{"changecallback"} : sub {}; - $self->{_subsystem} = $sub; - $self->{_mlog_config_file} = _get_option($options, 'config'); - if(!(defined $self->{_mlog_config_file})) { - $self->{_mlog_config_file} = defined $ENV{$MLOG_ENV_FILE} ? - $ENV{$MLOG_ENV_FILE} : undef; + $self->{ authuser } = _get_option( $options, 'authuser' ); + $self->{ module } = _get_option( $options, 'module' ); + $self->{ method } = _get_option( $options, 'method' ); + $self->{ call_id } = _get_option( $options, 'call_id' ); + $self->{ ip_address } = _get_option( $options, 'ip_address' ); + $self->{ tag } = _get_option( $options, 'tag' ); + $self->{ _callback } + = defined $options->{ "changecallback" } + ? $options->{ "changecallback" } + : sub { }; + $self->{ _subsystem } = $sub; + $self->{ _mlog_config_file } = _get_option( $options, 'config' ); + + if ( !( defined $self->{ _mlog_config_file } ) ) { + $self->{ _mlog_config_file } + = defined $ENV{ $MLOG_ENV_FILE } ? $ENV{ $MLOG_ENV_FILE } : undef; } - $self->{_user_log_level} = -1; - $self->{_config_log_level} = -1; - $self->{_user_log_file} = _get_option($options, 'logfile'); - $self->{_config_log_file} = undef; - $self->{_api_log_level} = -1; - $self->{_msgs_since_config_update} = 0; - $self->{_time_at_config_update} = ""; - $self->{msg_count} = 0; - $self->{_recheck_api_msg} = 100; - $self->{_recheck_api_time} = 300; # 5 mins - if(defined $lc) { - $self->{_log_constraints} = $lc; + $self->{ _user_log_level } = -1; + $self->{ _config_log_level } = -1; + $self->{ _user_log_file } = _get_option( $options, 'logfile' ); + $self->{ _config_log_file } = undef; + $self->{ _api_log_level } = -1; + $self->{ _msgs_since_config_update } = 0; + $self->{ _time_at_config_update } = ""; + $self->{ msg_count } = 0; + $self->{ _recheck_api_msg } = 100; + $self->{ _recheck_api_time } = 300; # 5 mins + + if ( defined $lc ) { + $self->{ _log_constraints } = $lc; } - $self->{_init} = 1; + $self->{ _init } = 1; $self->update_config(); - $self->{_init} = undef; + $self->{ _init } = undef; return $self; } sub _get_time_since_start { - my ($self) = @_; - my $now = time; - my $seconds_duration = $now - $self->{_time_at_config_update}; + my ( $self ) = @_; + my $now = time; + my $seconds_duration = $now - $self->{ _time_at_config_update }; return $seconds_duration; } sub get_log_level { - my ($self) = @_; - if($self->{_user_log_level} != -1) { - return $self->{_user_log_level}; - } elsif($self->{_config_log_level} != -1) { - return $self->{_config_log_level}; - } elsif($self->{_api_log_level} != -1) { - return $self->{_api_log_level}; - } else { + my ( $self ) = @_; + if ( $self->{ _user_log_level } != -1 ) { + return $self->{ _user_log_level }; + } + elsif ( $self->{ _config_log_level } != -1 ) { + return $self->{ _config_log_level }; + } + elsif ( $self->{ _api_log_level } != -1 ) { + return $self->{ _api_log_level }; + } + else { return $DEFAULT_LOG_LEVEL; } } sub update_config { - my ($self) = @_; + my ( $self ) = @_; my $loglevel = $self->get_log_level(); - my $logfile = $self->get_log_file(); - - $self->{_api_log_level} = -1; - $self->{_msgs_since_config_update} = 0; - $self->{_time_at_config_update} = time; - + my $logfile = $self->get_log_file(); + + $self->{ _api_log_level } = -1; + $self->{ _msgs_since_config_update } = 0; + $self->{ _time_at_config_update } = time; + # Retrieving config variables. my $api_url = ""; - if(defined $self->{_mlog_config_file} && -e $self->{_mlog_config_file} && - -s $self->{_mlog_config_file} > 0) { - my $cfg = new Config::Simple($self->{_mlog_config_file}); - my $cfgitems = $cfg->get_block($_GLOBAL); - my $subitems = $cfg->get_block($self->{_subsystem}); - foreach my $k (keys %{$subitems}) { - $cfgitems->{$k} = $subitems->{$k}; + if ( defined $self->{ _mlog_config_file } + && -e $self->{ _mlog_config_file } + && -s $self->{ _mlog_config_file } > 0 ) { + my $cfg = new Config::Simple( $self->{ _mlog_config_file } ); + my $cfgitems = $cfg->get_block( $_GLOBAL ); + my $subitems = $cfg->get_block( $self->{ _subsystem } ); + foreach my $k ( keys %{ $subitems } ) { + $cfgitems->{ $k } = $subitems->{ $k }; } - if(defined $cfgitems->{$MLOG_LOG_LEVEL}) { - if($cfgitems->{$MLOG_LOG_LEVEL} !~ /^\d+$/) { - warn "Cannot parse log level " . $cfgitems->{$MLOG_LOG_LEVEL} . - " from file " . $self->{_mlog_config_file} . - " to int. Keeping current log level."; - } else { - $self->{_config_log_level} = $cfgitems->{$MLOG_LOG_LEVEL}; + if ( defined $cfgitems->{ $MLOG_LOG_LEVEL } ) { + if ( $cfgitems->{ $MLOG_LOG_LEVEL } !~ /^\d+$/ ) { + warn "Cannot parse log level " + . $cfgitems->{ $MLOG_LOG_LEVEL } + . " from file " + . $self->{ _mlog_config_file } + . " to int. Keeping current log level."; + } + else { + $self->{ _config_log_level } = $cfgitems->{ $MLOG_LOG_LEVEL }; } } - if(defined $cfgitems->{$MLOG_API_URL}) { - $api_url = $cfgitems->{$MLOG_API_URL}; + if ( defined $cfgitems->{ $MLOG_API_URL } ) { + $api_url = $cfgitems->{ $MLOG_API_URL }; } - if(defined $cfgitems->{$MLOG_LOG_FILE}) { - $self->{_config_log_file} = $cfgitems->{$MLOG_LOG_FILE} + if ( defined $cfgitems->{ $MLOG_LOG_FILE } ) { + $self->{ _config_log_file } = $cfgitems->{ $MLOG_LOG_FILE }; } - } elsif (defined $self->{_mlog_config_file}) { - warn "Cannot read config file " . $self->{_mlog_config_file}; + } + elsif ( defined $self->{ _mlog_config_file } ) { + warn "Cannot read config file " . $self->{ _mlog_config_file }; } - unless($api_url eq "") { - my $subsystem_api_url = $api_url."/".$self->{_subsystem}; - my $json = get($subsystem_api_url); - if(defined $json) { - my $decoded_json = decode_json($json); + unless ( $api_url eq "" ) { + my $subsystem_api_url = $api_url . "/" . $self->{ _subsystem }; + my $json = get( $subsystem_api_url ); + if ( defined $json ) { + my $decoded_json = decode_json( $json ); my $max_matching_level = -1; - foreach my $constraint_set (@{$decoded_json->{'log_levels'}}) { - my $level = $constraint_set->{'level'}; - my $constraints = $constraint_set->{'constraints'}; - if($level <= $max_matching_level) { + foreach my $constraint_set ( @{ $decoded_json->{ 'log_levels' } } ) { + my $level = $constraint_set->{ 'level' }; + my $constraints = $constraint_set->{ 'constraints' }; + if ( $level <= $max_matching_level ) { next; } my $matches = 1; - foreach my $constraint (keys %{$constraints}) { - if(! exists $self->{_log_constraints}->{$constraint}) { + foreach my $constraint ( keys %{ $constraints } ) { + if ( !exists $self->{ _log_constraints }->{ $constraint } ) { $matches = 0; - } elsif($self->{_log_constraints}->{$constraint} ne $constraints->{$constraint}) { + } + elsif ( $self->{ _log_constraints }->{ $constraint } ne + $constraints->{ $constraint } ) + { $matches = 0; } } - if($matches == 1) { + if ( $matches == 1 ) { $max_matching_level = $level; } } - $self->{_api_log_level} = $max_matching_level; - } else { - warn "Could not retrieve Log subsystem from control API at: $subsystem_api_url"; + $self->{ _api_log_level } = $max_matching_level; + } + else { + warn + "Could not retrieve Log subsystem from control API at: $subsystem_api_url"; } } } sub _resolve_log_level { - my ($self, $level) = @_; - if(defined $_MLOG_TEXT_TO_LEVEL->{$level}) { - $level = $_MLOG_TEXT_TO_LEVEL->{$level}; - } elsif (!(defined $_MLOG_LEVEL_TO_TEXT->{$level})) { + my ( $self, $level ) = @_; + if ( defined $_MLOG_TEXT_TO_LEVEL->{ $level } ) { + $level = $_MLOG_TEXT_TO_LEVEL->{ $level }; + } + elsif ( !( defined $_MLOG_LEVEL_TO_TEXT->{ $level } ) ) { die "Illegal log level"; } return $level; } sub set_log_level { - my ($self, $level) = @_; - $self->{_user_log_level} = $self->_resolve_log_level($level); - $self->{_callback}(); + my ( $self, $level ) = @_; + $self->{ _user_log_level } = $self->_resolve_log_level( $level ); + $self->{ _callback }(); } sub get_log_file { - my ($self) = @_; - if ($self->{_user_log_file}) { - return $self->{_user_log_file}; + my ( $self ) = @_; + if ( $self->{ _user_log_file } ) { + return $self->{ _user_log_file }; } - if ($self->{_config_log_file}) { - return $self->{_config_log_file}; + if ( $self->{ _config_log_file } ) { + return $self->{ _config_log_file }; } return undef; } sub set_log_file { - my ($self, $filename) = @_; - $self->{_user_log_file} = $filename; - $self->{_callback}(); + my ( $self, $filename ) = @_; + $self->{ _user_log_file } = $filename; + $self->{ _callback }(); } sub set_log_msg_check_count { - my ($self, $count) = @_; - if($count !~ /^\d+$/ || $count < 0) { - die "Format for calling set_log_msg_check_count is set_log_msg_check_count(integer count)\n"; + my ( $self, $count ) = @_; + if ( $count !~ /^\d+$/ || $count < 0 ) { + die + "Format for calling set_log_msg_check_count is set_log_msg_check_count(integer count)\n"; } - $self->{_recheck_api_msg} = $count; + $self->{ _recheck_api_msg } = $count; } sub set_log_msg_check_interval { - my ($self, $interval) = @_; - if($interval !~ /^\d+$/ || $interval < 0) { - die "Format for calling set_log_msg_check_interval is set_log_msg_check_interval(integer seconds)\n"; + my ( $self, $interval ) = @_; + if ( $interval !~ /^\d+$/ || $interval < 0 ) { + die + "Format for calling set_log_msg_check_interval is set_log_msg_check_interval(integer seconds)\n"; } - $self->{_recheck_api_time} = $interval; + $self->{ _recheck_api_time } = $interval; } sub clear_user_log_level { - my ($self) = @_; - $self->{_user_log_level} = -1; + my ( $self ) = @_; + $self->{ _user_log_level } = -1; } sub _get_ident { - my ($self, $level, $authuser, $module, $method, $call_id, $ip_address, $tag) = @_; - my @infos = ($self->{_subsystem}, $_MLOG_LEVEL_TO_TEXT->{$level}, - Time::HiRes::time(), $USER, $PARENT_FILE, $$); - if ($self->{ip_address}) { + my ( $self, $level, $authuser, $module, $method, $call_id, $ip_address, $tag ) = @_; + my @infos = ( + $self->{ _subsystem }, + $_MLOG_LEVEL_TO_TEXT->{ $level }, + Time::HiRes::time(), $USER, $PARENT_FILE, $$ + ); + if ( $self->{ ip_address } ) { push @infos, $ip_address || '-'; } - if ($self->{authuser}) { + if ( $self->{ authuser } ) { push @infos, $authuser || '-'; } - if ($self->{module}) { + if ( $self->{ module } ) { push @infos, $module || '-'; } - if ($self->{method}) { + if ( $self->{ method } ) { push @infos, $method || '-'; } - if ($self->{call_id}) { + if ( $self->{ call_id } ) { push @infos, $call_id || '-'; } - if ($self->{tag}) { + if ( $self->{ tag } ) { push @infos, $tag || '-'; } - return "[" . join("] [", @infos). "]"; + return "[" . join( "] [", @infos ) . "]"; } sub _syslog { - my ($self, $facility, $level, $ident, $message) = @_; - openlog($ident, "", $facility); - if(ref($message) eq 'ARRAY') { - foreach my $m (@{$message}) { - syslog($_MLOG_TO_SYSLOG[$level], "$m"); + my ( $self, $facility, $level, $ident, $message ) = @_; + openlog( $ident, "", $facility ); + if ( ref( $message ) eq 'ARRAY' ) { + foreach my $m ( @{ $message } ) { + syslog( $_MLOG_TO_SYSLOG[ $level ], "$m" ); } - } else { - syslog($_MLOG_TO_SYSLOG[$level], "$message"); + } + else { + syslog( $_MLOG_TO_SYSLOG[ $level ], "$message" ); } closelog(); } sub _log { - my ($self, $ident, $message) = @_; - my $time = POSIX::strftime("%Y-%m-%d %H:%M:%S", localtime); - my $msg = join(" ", $time, hostname(), $ident . ": "); - open LOG, ">>" . $self->get_log_file() || - warn "Could not print log message $msg to " . $self->get_log_file() . "\n"; - if(ref($message) eq 'ARRAY') { - foreach my $m (@{$message}) { + my ( $self, $ident, $message ) = @_; + my $time = POSIX::strftime( "%Y-%m-%d %H:%M:%S", localtime ); + my $msg = join " ", $time, hostname(), $ident . ": "; + open LOG, ">>" . $self->get_log_file() + || warn "Could not print log message $msg to " . $self->get_log_file() . "\n"; + if ( ref( $message ) eq 'ARRAY' ) { + foreach my $m ( @{ $message } ) { print LOG $msg . "$m\n"; } - } else { + } + else { print LOG $msg . "$message\n"; } close LOG; } sub log_message { - my ($self, $level, $message, $authuser, $module, $method, $call_id, - $ip_address, $tag) = @_; - $level = $self->_resolve_log_level($level); + my ( + $self, $level, $message, $authuser, $module, + $method, $call_id, $ip_address, $tag + ) = @_; + $level = $self->_resolve_log_level( $level ); - ++$self->{msg_count}; - ++$self->{_msgs_since_config_update}; + ++$self->{ msg_count }; + ++$self->{ _msgs_since_config_update }; - if($self->{_msgs_since_config_update} >= $self->{_recheck_api_msg} || - $self->_get_time_since_start() >= $self->{_recheck_api_time}) { + if ( $self->{ _msgs_since_config_update } >= $self->{ _recheck_api_msg } + || $self->_get_time_since_start() >= $self->{ _recheck_api_time } ) { $self->update_config(); } - my $ident = $self->_get_ident($level, $authuser, $module, $method, $call_id, - $ip_address, $tag); + my $ident = $self->_get_ident( $level, $authuser, $module, $method, $call_id, + $ip_address, $tag ); + # If this message is an emergency, send a copy to the emergency facility first. - if($level == 0) { - $self->_syslog($EMERG_FACILITY, $level, $ident, $message); + if ( $level == 0 ) { + $self->_syslog( $EMERG_FACILITY, $level, $ident, $message ); } - if($level <= $self->get_log_level()) { - $self->_syslog($MSG_FACILITY, $level, $ident, $message); - if($self->get_log_file()) { - $self->_log($ident, $message); + if ( $level <= $self->get_log_level() ) { + $self->_syslog( $MSG_FACILITY, $level, $ident, $message ); + if ( $self->get_log_file() ) { + $self->_log( $ident, $message ); } } } diff --git a/sdkbase/Dockerfile b/sdkbase/Dockerfile deleted file mode 100644 index a4045629..00000000 --- a/sdkbase/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -FROM kbase/deplbase:latest - -COPY ./sdkbase.build.tag /tmp/ - -# Update certs -RUN apt-get update -RUN apt-get install ca-certificates - -# Fix Python SSL warnings for python < 2.7.9 (system python on Trusty is 2.7.6) -# https://github.com/pypa/pip/issues/4098 -RUN pip install pip==8.1.2 -RUN pip install --disable-pip-version-check requests requests_toolbelt pyopenssl --upgrade - -#install coverage tool -RUN pip install coverage - -RUN \ - . /kb/dev_container/user-env.sh && \ - cd /kb/dev_container/modules && \ - rm -rf auth && \ - git clone -b auth2 https://github.com/kbase/auth && \ - cd /kb/dev_container/modules/auth && \ - make && make deploy - -COPY ./lets_encript/lets-encrypt-x3-cross-signed.der /tmp/ -RUN keytool -import -keystore /usr/lib/jvm/java-7-oracle/jre/lib/security/cacerts \ - -storepass changeit -noprompt -trustcacerts -alias letsencryptauthorityx31 \ - -file /tmp/lets-encrypt-x3-cross-signed.der -RUN rm /tmp/lets-encrypt-x3-cross-signed.der - -# Update kb_sdk at build time -RUN \ - . /kb/dev_container/user-env.sh && \ - rm /kb/runtime/java && \ - ln -s /usr/lib/jvm/java-7-oracle /kb/runtime/java && \ - cd /kb/dev_container/modules && \ - rm -rf jars && \ - git clone https://github.com/kbase/jars && \ - rm -rf kb_sdk && \ - git clone https://github.com/kbase/kb_sdk -b develop && \ - cd /kb/dev_container/modules/jars && \ - make && make deploy && \ - cd /kb/dev_container/modules/kb_sdk && \ - make && make deploy diff --git a/sdkbase/create_certs b/sdkbase/create_certs deleted file mode 100755 index c78e39d3..00000000 --- a/sdkbase/create_certs +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -[ -e ./ssl ] || mkdir ./ssl - -[ -z $PUBLIC_ADDRESS ] && PUBLIC_ADDRESS="kbase.us" -export PUBLIC_ADDRESS - -# Create certs for development deploy -if [ -e ./ssl/proxy.key ] ; then - echo "Skipping proxy cert" -else - openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout ./ssl/proxy.key -out ./ssl/proxy.crt -config openssl.cnf -batch -fi - -if [ -e ./ssl/narrative.key ] ; then - echo "Skipping narrative cert" -else - cp ./ssl/proxy.key ./ssl/narrative.key - cp ./ssl/proxy.crt ./ssl/narrative.crt -fi diff --git a/sdkbase/lets_encript/lets-encript-readme.txt b/sdkbase/lets_encript/lets-encript-readme.txt deleted file mode 100644 index c0f188be..00000000 --- a/sdkbase/lets_encript/lets-encript-readme.txt +++ /dev/null @@ -1 +0,0 @@ -Certificate was loaded from https://letsencrypt.org/certs/lets-encrypt-x3-cross-signed.der on 2016-12-19. \ No newline at end of file diff --git a/sdkbase/lets_encript/lets-encrypt-x3-cross-signed.der b/sdkbase/lets_encript/lets-encrypt-x3-cross-signed.der deleted file mode 100644 index e08466c5..00000000 Binary files a/sdkbase/lets_encript/lets-encrypt-x3-cross-signed.der and /dev/null differ diff --git a/sdkbase/makeconfig b/sdkbase/makeconfig deleted file mode 100755 index 85ac92d9..00000000 --- a/sdkbase/makeconfig +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -docker run --rm --entrypoint cat kbase/deplbase:latest /kb/deployment/deployment.cfg > cluster.ini -./create_certs diff --git a/sdkbase/openssl.cnf b/sdkbase/openssl.cnf deleted file mode 100644 index 2f072a40..00000000 --- a/sdkbase/openssl.cnf +++ /dev/null @@ -1,355 +0,0 @@ -# -# OpenSSL example configuration file. -# This is mostly being used for generation of certificate requests. -# - -# This definition stops the following lines choking if HOME isn't -# defined. -HOME = . -RANDFILE = $ENV::HOME/.rnd - -# Extra OBJECT IDENTIFIER info: -#oid_file = $ENV::HOME/.oid -oid_section = new_oids - -# To use this configuration file with the "-extfile" option of the -# "openssl x509" utility, name here the section containing the -# X.509v3 extensions to use: -# extensions = -# (Alternatively, use a configuration file that has only -# X.509v3 extensions in its main [= default] section.) - -[ new_oids ] - -# We can add new OIDs in here for use by 'ca', 'req' and 'ts'. -# Add a simple OID like this: -# testoid1=1.2.3.4 -# Or use config file substitution like this: -# testoid2=${testoid1}.5.6 - -# Policies used by the TSA examples. -tsa_policy1 = 1.2.3.4.1 -tsa_policy2 = 1.2.3.4.5.6 -tsa_policy3 = 1.2.3.4.5.7 - -#################################################################### -[ ca ] -default_ca = CA_default # The default ca section - -#################################################################### -[ CA_default ] - -dir = ./demoCA # Where everything is kept -certs = $dir/certs # Where the issued certs are kept -crl_dir = $dir/crl # Where the issued crl are kept -database = $dir/index.txt # database index file. -#unique_subject = no # Set to 'no' to allow creation of - # several ctificates with same subject. -new_certs_dir = $dir/newcerts # default place for new certs. - -certificate = $dir/cacert.pem # The CA certificate -serial = $dir/serial # The current serial number -crlnumber = $dir/crlnumber # the current crl number - # must be commented out to leave a V1 CRL -crl = $dir/crl.pem # The current CRL -private_key = $dir/private/cakey.pem# The private key -RANDFILE = $dir/private/.rand # private random number file - -x509_extensions = usr_cert # The extentions to add to the cert - -# Comment out the following two lines for the "traditional" -# (and highly broken) format. -name_opt = ca_default # Subject Name options -cert_opt = ca_default # Certificate field options - -# Extension copying option: use with caution. -# copy_extensions = copy - -# Extensions to add to a CRL. Note: Netscape communicator chokes on V2 CRLs -# so this is commented out by default to leave a V1 CRL. -# crlnumber must also be commented out to leave a V1 CRL. -# crl_extensions = crl_ext - -default_days = 365 # how long to certify for -default_crl_days= 30 # how long before next CRL -default_md = default # use public key default MD -preserve = no # keep passed DN ordering - -# A few difference way of specifying how similar the request should look -# For type CA, the listed attributes must be the same, and the optional -# and supplied fields are just that :-) -policy = policy_match - -# For the CA policy -[ policy_match ] -countryName = match -stateOrProvinceName = match -organizationName = match -organizationalUnitName = optional -commonName = supplied -emailAddress = optional - -# For the 'anything' policy -# At this point in time, you must list all acceptable 'object' -# types. -[ policy_anything ] -countryName = optional -stateOrProvinceName = optional -localityName = optional -organizationName = optional -organizationalUnitName = optional -commonName = supplied -emailAddress = optional - -#################################################################### -[ req ] -default_bits = 1024 -default_keyfile = privkey.pem -distinguished_name = req_distinguished_name -attributes = req_attributes -x509_extensions = v3_ca # The extentions to add to the self signed cert - -# Passwords for private keys if not present they will be prompted for -# input_password = secret -# output_password = secret - -# This sets a mask for permitted string types. There are several options. -# default: PrintableString, T61String, BMPString. -# pkix : PrintableString, BMPString (PKIX recommendation before 2004) -# utf8only: only UTF8Strings (PKIX recommendation after 2004). -# nombstr : PrintableString, T61String (no BMPStrings or UTF8Strings). -# MASK:XXXX a literal mask value. -# WARNING: ancient versions of Netscape crash on BMPStrings or UTF8Strings. -string_mask = utf8only - -req_extensions = v3_req # The extensions to add to a certificate request - -[ req_distinguished_name ] -countryName = Country Name (2 letter code) -countryName_default = US -countryName_min = 2 -countryName_max = 2 - -stateOrProvinceName = State or Province Name (full name) -stateOrProvinceName_default = California - -localityName = Locality Name (eg, city) - -0.organizationName = Organization Name (eg, company) -0.organizationName_default = KBase - -# we can do this but it is not needed normally :-) -#1.organizationName = Second Organization Name (eg, company) -#1.organizationName_default = World Wide Web Pty Ltd - -organizationalUnitName = Organizational Unit Name (eg, section) -#organizationalUnitName_default = - -commonName = Common Name (e.g. server FQDN or YOUR name) -commonName_max = 64 -commonName_default = ${ENV::PUBLIC_ADDRESS} - -emailAddress = Email Address -emailAddress_max = 64 - -# SET-ex3 = SET extension number 3 - -[ req_attributes ] -challengePassword = A challenge password -challengePassword_min = 4 -challengePassword_max = 20 - -unstructuredName = An optional company name - -[ usr_cert ] - -# These extensions are added when 'ca' signs a request. - -# This goes against PKIX guidelines but some CAs do it and some software -# requires this to avoid interpreting an end user certificate as a CA. - -basicConstraints=CA:FALSE - -# Here are some examples of the usage of nsCertType. If it is omitted -# the certificate can be used for anything *except* object signing. - -# This is OK for an SSL server. -# nsCertType = server - -# For an object signing certificate this would be used. -# nsCertType = objsign - -# For normal client use this is typical -# nsCertType = client, email - -# and for everything including object signing: -# nsCertType = client, email, objsign - -# This is typical in keyUsage for a client certificate. -# keyUsage = nonRepudiation, digitalSignature, keyEncipherment - -# This will be displayed in Netscape's comment listbox. -nsComment = "OpenSSL Generated Certificate" - -# PKIX recommendations harmless if included in all certificates. -subjectKeyIdentifier=hash -authorityKeyIdentifier=keyid,issuer - -# This stuff is for subjectAltName and issuerAltname. -# Import the email address. -# subjectAltName=email:copy -# An alternative to produce certificates that aren't -# deprecated according to PKIX. -# subjectAltName=email:move - -# Copy subject details -# issuerAltName=issuer:copy - -#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem -#nsBaseUrl -#nsRevocationUrl -#nsRenewalUrl -#nsCaPolicyUrl -#nsSslServerName - -# This is required for TSA certificates. -# extendedKeyUsage = critical,timeStamping - -[ v3_req ] - -# Extensions to add to a certificate request - -basicConstraints = CA:TRUE -keyUsage = digitalSignature, keyEncipherment -subjectAltName = @alt_names - -[alt_names] -DNS.1 = $ENV::PUBLIC_ADDRESS - -[ v3_ca ] - - -# Extensions for a typical CA - - -# PKIX recommendation. - -subjectKeyIdentifier=hash - -authorityKeyIdentifier=keyid:always,issuer - -# This is what PKIX recommends but some broken software chokes on critical -# extensions. -#basicConstraints = critical,CA:true -# So we do this instead. -basicConstraints = CA:true - -# Key usage: this is typical for a CA certificate. However since it will -# prevent it being used as an test self-signed certificate it is best -# left out by default. -# keyUsage = cRLSign, keyCertSign - -# Some might want this also -# nsCertType = sslCA, emailCA - -# Include email address in subject alt name: another PKIX recommendation -# subjectAltName=email:copy -# Copy issuer details -# issuerAltName=issuer:copy - -# DER hex encoding of an extension: beware experts only! -# obj=DER:02:03 -# Where 'obj' is a standard or added object -# You can even override a supported extension: -# basicConstraints= critical, DER:30:03:01:01:FF - -[ crl_ext ] - -# CRL extensions. -# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL. - -# issuerAltName=issuer:copy -authorityKeyIdentifier=keyid:always - -[ proxy_cert_ext ] -# These extensions should be added when creating a proxy certificate - -# This goes against PKIX guidelines but some CAs do it and some software -# requires this to avoid interpreting an end user certificate as a CA. - -basicConstraints=CA:FALSE - -# Here are some examples of the usage of nsCertType. If it is omitted -# the certificate can be used for anything *except* object signing. - -# This is OK for an SSL server. -# nsCertType = server - -# For an object signing certificate this would be used. -# nsCertType = objsign - -# For normal client use this is typical -# nsCertType = client, email - -# and for everything including object signing: -# nsCertType = client, email, objsign - -# This is typical in keyUsage for a client certificate. -# keyUsage = nonRepudiation, digitalSignature, keyEncipherment - -# This will be displayed in Netscape's comment listbox. -nsComment = "OpenSSL Generated Certificate" - -# PKIX recommendations harmless if included in all certificates. -subjectKeyIdentifier=hash -authorityKeyIdentifier=keyid,issuer - -# This stuff is for subjectAltName and issuerAltname. -# Import the email address. -# subjectAltName=email:copy -# An alternative to produce certificates that aren't -# deprecated according to PKIX. -# subjectAltName=email:move - -# Copy subject details -# issuerAltName=issuer:copy - -#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem -#nsBaseUrl -#nsRevocationUrl -#nsRenewalUrl -#nsCaPolicyUrl -#nsSslServerName - -# This really needs to be in place for it to be a proxy certificate. -proxyCertInfo=critical,language:id-ppl-anyLanguage,pathlen:3,policy:foo - -#################################################################### -[ tsa ] - -default_tsa = tsa_config1 # the default TSA section - -[ tsa_config1 ] - -# These are used by the TSA reply generation only. -dir = ./demoCA # TSA root directory -serial = $dir/tsaserial # The current serial number (mandatory) -crypto_device = builtin # OpenSSL engine to use for signing -signer_cert = $dir/tsacert.pem # The TSA signing certificate - # (optional) -certs = $dir/cacert.pem # Certificate chain to include in reply - # (optional) -signer_key = $dir/private/tsakey.pem # The TSA private key (optional) - -default_policy = tsa_policy1 # Policy if request did not specify it - # (optional) -other_policies = tsa_policy2, tsa_policy3 # acceptable policies (optional) -digests = md5, sha1 # Acceptable message digests (mandatory) -accuracy = secs:1, millisecs:500, microsecs:100 # (optional) -clock_precision_digits = 0 # number of digits after dot. (optional) -ordering = yes # Is ordering defined for timestamps? - # (optional, default: no) -tsa_name = yes # Must the TSA name be included in the reply? - # (optional, default: no) -ess_cert_id_chain = no # Must the ESS cert id chain be included? - # (optional, default: no) diff --git a/sdkbase/sdkbase.build.tag b/sdkbase/sdkbase.build.tag deleted file mode 100644 index c65fbcf7..00000000 --- a/sdkbase/sdkbase.build.tag +++ /dev/null @@ -1 +0,0 @@ -4/7/2017 diff --git a/src/java/us/kbase/mobu/ModuleBuilder.java b/src/java/us/kbase/mobu/ModuleBuilder.java index d8d618d0..cef7db21 100644 --- a/src/java/us/kbase/mobu/ModuleBuilder.java +++ b/src/java/us/kbase/mobu/ModuleBuilder.java @@ -31,7 +31,7 @@ import us.kbase.mobu.validator.ModuleValidator; public class ModuleBuilder { - + private static final String defaultParentPackage = "us.kbase"; private static final String MODULE_BUILDER_SH_NAME = "kb-sdk"; @@ -45,36 +45,34 @@ public class ModuleBuilder { private static final String INSTALL_COMMAND = "install"; private static final String RUN_COMMAND = "run"; //private static final String SUBMIT_COMMAND = "submit"; - + public static final String GLOBAL_SDK_HOME_ENV_VAR = "KB_SDK_HOME"; public static final String DEFAULT_METHOD_STORE_URL = "https://appdev.kbase.us/services/narrative_method_store/rpc"; - - public static final String VERSION = "1.2.1"; - - + + public static final String VERSION = "1.2.6"; + public static void main(String[] args) throws Exception { - - // setup the basic CLI argument parser with global -h and --help commands - GlobalArgs gArgs = new GlobalArgs(); - JCommander jc = new JCommander(gArgs); - jc.setProgramName(MODULE_BUILDER_SH_NAME); - - - // add the 'init' command - InitCommandArgs initArgs = new InitCommandArgs(); - jc.addCommand(INIT_COMMAND, initArgs); - - // add the 'validate' command - ValidateCommandArgs validateArgs = new ValidateCommandArgs(); - jc.addCommand(VALIDATE_COMMAND, validateArgs); - - // add the 'compile' command - CompileCommandArgs compileArgs = new CompileCommandArgs(); - jc.addCommand(COMPILE_COMMAND, compileArgs); - - // add the 'help' command - HelpCommandArgs help = new HelpCommandArgs(); - jc.addCommand(HELP_COMMAND, help); + + // set up the basic CLI argument parser with global -h and --help commands + GlobalArgs gArgs = new GlobalArgs(); + JCommander jc = new JCommander(gArgs); + jc.setProgramName(MODULE_BUILDER_SH_NAME); + + // add the 'init' command + InitCommandArgs initArgs = new InitCommandArgs(); + jc.addCommand(INIT_COMMAND, initArgs); + + // add the 'validate' command + ValidateCommandArgs validateArgs = new ValidateCommandArgs(); + jc.addCommand(VALIDATE_COMMAND, validateArgs); + + // add the 'compile' command + CompileCommandArgs compileArgs = new CompileCommandArgs(); + jc.addCommand(COMPILE_COMMAND, compileArgs); + + // add the 'help' command + HelpCommandArgs help = new HelpCommandArgs(); + jc.addCommand(HELP_COMMAND, help); // add the 'test' command TestCommandArgs testArgs = new TestCommandArgs(); @@ -91,84 +89,87 @@ public static void main(String[] args) throws Exception { // add the 'install' command InstallCommandArgs installArgs = new InstallCommandArgs(); jc.addCommand(INSTALL_COMMAND, installArgs); - + // add the 'run' command RunCommandArgs runArgs = new RunCommandArgs(); jc.addCommand(RUN_COMMAND, runArgs); - // parse the arguments and gracefully catch any errors - try { - jc.parse(args); - } catch (RuntimeException e) { - showError("Command Line Argument Error", e.getMessage()); - System.exit(1); - } - - // if the help flag is set, then show some help and exit - if(gArgs.help) { - showBriefHelp(jc, System.out); - return; - } - - // no command entered, just print brief info and exit - if(jc.getParsedCommand()==null) { - showBriefHelp(jc, System.out); - return; - } - - // if we get here, we have a valid command, so process it and do stuff - int returnCode = 0; - if(jc.getParsedCommand().equals(HELP_COMMAND)) { - showCommandUsage(jc,help,System.out); - - } else if(jc.getParsedCommand().equals(INIT_COMMAND)) { - returnCode = runInitCommand(initArgs,jc); - } else if(jc.getParsedCommand().equals(VALIDATE_COMMAND)) { - returnCode = runValidateCommand(validateArgs,jc); - } else if(jc.getParsedCommand().equals(COMPILE_COMMAND)) { - returnCode = runCompileCommand(compileArgs,jc); + // print name and version + printVersion(); + + // parse the arguments and gracefully catch any errors + try { + jc.parse(args); + } catch (RuntimeException e) { + showError("Command Line Argument Error", e.getMessage()); + System.exit(1); + } + + // if the help flag is set, then show some help and exit + if(gArgs.help) { + showBriefHelp(jc, System.out); + return; + } + + // no command entered, just print brief info and exit + if(jc.getParsedCommand()==null) { + showBriefHelp(jc, System.out); + return; + } + + // if we get here, we have a valid command, so process it and do stuff + int returnCode = 0; + if(jc.getParsedCommand().equals(HELP_COMMAND)) { + showCommandUsage(jc,help,System.out); + + } else if(jc.getParsedCommand().equals(INIT_COMMAND)) { + returnCode = runInitCommand(initArgs,jc); + } else if(jc.getParsedCommand().equals(VALIDATE_COMMAND)) { + returnCode = runValidateCommand(validateArgs,jc); + } else if(jc.getParsedCommand().equals(COMPILE_COMMAND)) { + returnCode = runCompileCommand(compileArgs,jc); } else if(jc.getParsedCommand().equals(TEST_COMMAND)) { returnCode = runTestCommand(testArgs,jc); - } else if (jc.getParsedCommand().equals(VERSION_COMMAND)) { - returnCode = runVersionCommand(versionArgs, jc); - } else if (jc.getParsedCommand().equals(RENAME_COMMAND)) { - returnCode = runRenameCommand(renameArgs, jc); - } else if (jc.getParsedCommand().equals(INSTALL_COMMAND)) { - returnCode = runInstallCommand(installArgs, jc); - } else if (jc.getParsedCommand().equals(RUN_COMMAND)) { - returnCode = runRunCommand(runArgs, jc); - } - - if(returnCode!=0) { - System.exit(returnCode); - } + } else if (jc.getParsedCommand().equals(VERSION_COMMAND)) { + returnCode = runVersionCommand(versionArgs, jc); + } else if (jc.getParsedCommand().equals(RENAME_COMMAND)) { + returnCode = runRenameCommand(renameArgs, jc); + } else if (jc.getParsedCommand().equals(INSTALL_COMMAND)) { + returnCode = runInstallCommand(installArgs, jc); + } else if (jc.getParsedCommand().equals(RUN_COMMAND)) { + returnCode = runRunCommand(runArgs, jc); + } + + if(returnCode!=0) { + System.exit(returnCode); + } } - + private static int runValidateCommand(ValidateCommandArgs validateArgs, JCommander jc) { - // initialize - if(validateArgs.modules==null) { - validateArgs.modules = new ArrayList(); - } - if(validateArgs.modules.size()==0) { - validateArgs.modules.add("."); - } - try { - ModuleValidator mv = new ModuleValidator(validateArgs.modules,validateArgs.verbose, - validateArgs.methodStoreUrl, validateArgs.allowSyncMethods); - return mv.validateAll(); - } catch (Exception e) { + // initialize + if(validateArgs.modules==null) { + validateArgs.modules = new ArrayList(); + } + if(validateArgs.modules.size()==0) { + validateArgs.modules.add("."); + } + try { + ModuleValidator mv = new ModuleValidator(validateArgs.modules,validateArgs.verbose, + validateArgs.methodStoreUrl, validateArgs.allowSyncMethods); + return mv.validateAll(); + } catch (Exception e) { if (validateArgs.verbose) e.printStackTrace(); showError("Error while validating module", e.getMessage()); return 1; } - } + } /** * Runs the module initialization command - this creates a new module in the relative directory name given. * There's only a couple of possible arguments here in the initArgs: * userName (required) - the user's Github user name, used to set up some optional fields - * moduleNames (required) - this catchall represents the module's name. Any whitespace (e.g. token breaks) + * moduleNames (required) - this catchall represents the module's name. Any whitespace (e.g. token breaks) * are replaced with underscores. So if a user runs: * kb-sdk init my new module * they get a module called "my_new_module" in a directory of the same name. @@ -176,80 +177,80 @@ private static int runValidateCommand(ValidateCommandArgs validateArgs, JCommand * @param jc * @return */ - private static int runInitCommand(InitCommandArgs initArgs, JCommander jc) { - // Figure out module name. - // Join together spaced out names with underscores if necessary. - if (initArgs.moduleNames == null || initArgs.moduleNames.size() == 0) { - ModuleBuilder.showError("Init Error", "A module name is required."); - return 1; - } - String moduleName = StringUtils.join(initArgs.moduleNames, "_"); - - // Get username if available - String userName = null; - if (initArgs.userName != null) - userName = initArgs.userName; - - // Get chosen language - String language = ModuleInitializer.DEFAULT_LANGUAGE; - if (initArgs.language != null) { - language = initArgs.language; - } - try { - ModuleInitializer initer = new ModuleInitializer(moduleName, userName, language, initArgs.verbose); - initer.initialize(initArgs.example); - } - catch (Exception e) { + private static int runInitCommand(InitCommandArgs initArgs, JCommander jc) { + // Figure out module name. + // Join together spaced out names with underscores if necessary. + if (initArgs.moduleNames == null || initArgs.moduleNames.size() == 0) { + ModuleBuilder.showError("Init Error", "A module name is required."); + return 1; + } + String moduleName = StringUtils.join(initArgs.moduleNames, "_"); + + // Get username if available + String userName = null; + if (initArgs.userName != null) + userName = initArgs.userName; + + // Get chosen language + String language = ModuleInitializer.DEFAULT_LANGUAGE; + if (initArgs.language != null) { + language = initArgs.language; + } + try { + ModuleInitializer initer = new ModuleInitializer(moduleName, userName, language, initArgs.verbose); + initer.initialize(initArgs.example); + } + catch (Exception e) { if (initArgs.verbose) e.printStackTrace(); - showError("Error while initializing module", e.getMessage()); - return 1; - } - return 0; - } - - public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { - printVersion(); - // Step 1: convert list of args to a Files (this must be defined because it is required) - File specFile = null; - try { - if(a.specFileNames.size()>1) { - // right now we only support compiling one file at a time + showError("Error while initializing module", e.getMessage()); + return 1; + } + return 0; + } + + public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { + printVersion(); + // Step 1: convert list of args to a Files (this must be defined because it is required) + File specFile = null; + try { + if(a.specFileNames.size()>1) { + // right now we only support compiling one file at a time ModuleBuilder.showError("Compile Error","Too many input KIDL spec files provided", - "Currently there is only support for compiling one module at a time, which \n"+ - "is required to register data types and KIDL specs with the Workspace. This \n"+ - "code may be extended in the future to allow multiple modules to be compiled \n"+ - "together from separate files into the same client/server."); + "Currently there is only support for compiling one module at a time, which \n"+ + "is required to register data types and KIDL specs with the Workspace. This \n"+ + "code may be extended in the future to allow multiple modules to be compiled \n"+ + "together from separate files into the same client/server."); return 1; - } - List specFiles = convertToFiles(a.specFileNames); - specFile = specFiles.get(0); - } catch (IOException | RuntimeException e) { + } + List specFiles = convertToFiles(a.specFileNames); + specFile = specFiles.get(0); + } catch (IOException | RuntimeException e) { showError("Error accessing input KIDL spec file", e.getMessage()); if (a.verbose) { e.printStackTrace(); } return 1; - } - + } + if (a.clAsyncVer != null && a.dynservVer != null) { showError("Bad arguments", "clasyncver and dynserver cannot both be specified"); return 1; } - - // Step 2: check or create the output directory - File outDir = a.outDir == null ? new File(".") : new File(a.outDir); + + // Step 2: check or create the output directory + File outDir = a.outDir == null ? new File(".") : new File(a.outDir); try { - outDir = outDir.getCanonicalFile(); - } catch (IOException e) { + outDir = outDir.getCanonicalFile(); + } catch (IOException e) { showError("Error accessing output directory", e.getMessage()); return 1; - } + } if (!outDir.exists()) outDir.mkdirs(); - - // Step 3: validate the URL option if it is defined + + // Step 3: validate the URL option if it is defined if (a.url != null) { try { new URL(a.url); @@ -258,7 +259,7 @@ public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { return 1; } } - + // Step 4: info collection for status method File moduleDir = specFile.getAbsoluteFile().getParentFile(); String semanticVersion = null; @@ -290,13 +291,13 @@ public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { } } try { - RunCompileCommand.generate(specFile, a.url, a.jsClientSide, a.jsClientName, a.perlClientSide, - a.perlClientName, a.perlServerSide, a.perlServerName, a.perlImplName, - a.perlPsgiName, a.perlEnableRetries, a.pyClientSide, a.pyClientName, - a.pyServerSide, a.pyServerName, a.pyImplName, a.javaClientSide, - a.javaServerSide, a.javaPackageParent, a.javaSrcDir, a.javaLibDir, - a.javaBuildXml, a.javaGwtPackage, a.rClientSide, a.rClientName, - a.rServerSide, a.rServerName, a.rImplName, outDir, a.jsonSchema, + RunCompileCommand.generate(specFile, a.url, a.jsClientSide, a.jsClientName, a.perlClientSide, + a.perlClientName, a.perlServerSide, a.perlServerName, a.perlImplName, + a.perlPsgiName, a.perlEnableRetries, a.pyClientSide, a.pyClientName, + a.pyServerSide, a.pyServerName, a.pyImplName, a.javaClientSide, + a.javaServerSide, a.javaPackageParent, a.javaSrcDir, a.javaLibDir, + a.javaBuildXml, a.javaGwtPackage, a.rClientSide, a.rClientName, + a.rServerSide, a.rServerName, a.rImplName, outDir, a.jsonSchema, a.makefile, a.clAsyncVer, a.dynservVer, a.html, semanticVersion, gitUrl, gitCommitHash); } catch (Throwable e) { @@ -310,30 +311,30 @@ public static int runCompileCommand(CompileCommandArgs a, JCommander jc) { return 0; } - private static String getCmdOutput(File workDir, String... cmd) throws Exception { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - ProcessHelper.cmd(cmd).exec(workDir, null, pw); - return sw.toString().trim(); - } - + private static String getCmdOutput(File workDir, String... cmd) throws Exception { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + ProcessHelper.cmd(cmd).exec(workDir, null, pw); + return sw.toString().trim(); + } + static List convertToFiles(List filenames) throws IOException { - List files = new ArrayList(filenames.size()); - for(String filename: filenames) { - File f = new File(filename); - if(!f.exists()) { - throw new IOException("KIDL file \""+filename+"\" does not exist"); - } + List files = new ArrayList(filenames.size()); + for(String filename: filenames) { + File f = new File(filename); + if(!f.exists()) { + throw new IOException("KIDL file \""+filename+"\" does not exist"); + } files.add(f); - } - return files; + } + return files; } - + public static class GlobalArgs { - @Parameter(names = {"-h","--help"}, help = true, description="Display help and full usage information.") - boolean help; + @Parameter(names = {"-h","--help"}, help = true, description="Display help and full usage information.") + boolean help; } - + /** * Runs the module test command - this runs tests in local docker container. * @param testArgs @@ -360,24 +361,32 @@ private static int runTestCommand(TestCommandArgs testArgs, JCommander jc) { } private static void printVersion() { - String gitCommit = getGitCommit(); - System.out.println("KBase SDK version " + VERSION + (gitCommit == null ? "" : (" (commit " + gitCommit + ")"))); + String gitCommit = getGitProp("commit"); + String timestamp = getGitProp("build_timestamp"); + System.out.println("KBase SDK version " + VERSION + + (timestamp == null ? "" : ("_" + timestamp)) + + (gitCommit == null ? "" : (" (commit " + gitCommit + ")"))); } - public static String getGitCommit() { - String gitCommit = null; + + public static String getGitProp(String propertyToGet) { + String propertyValue = null; + Properties gitProps = new Properties(); try { - Properties gitProps = new Properties(); InputStream is = ModuleBuilder.class.getResourceAsStream("git.properties"); gitProps.load(is); is.close(); - gitCommit = gitProps.getProperty("commit"); - } catch (Exception ignore) {} - return gitCommit; + propertyValue = gitProps.getProperty(propertyToGet); + } catch (Exception e) { + showError("Error while retrieving version information from git.properties file: ", e.getMessage()); + System.exit(1); + } + return propertyValue; } - + + private static int runVersionCommand(VersionCommandArgs testArgs, JCommander jc) { - printVersion(); + // version was printed at start up, so just exit return 0; } @@ -398,14 +407,14 @@ private static int runRenameCommand(RenameCommandArgs renameArgs, JCommander jc) private static int runInstallCommand(InstallCommandArgs installArgs, JCommander jc) { if (installArgs.moduleName == null || installArgs.moduleName.size() != 1) { - ModuleBuilder.showError("Command Line Argument Error", + ModuleBuilder.showError("Command Line Argument Error", "One and only one module name should be provided"); return 1; } try { return new ClientInstaller().install(installArgs.lang, installArgs.async, - installArgs.core || installArgs.sync, installArgs.dynamic, - installArgs.tagVer, installArgs.verbose, installArgs.moduleName.get(0), + installArgs.core || installArgs.sync, installArgs.dynamic, + installArgs.tagVer, installArgs.verbose, installArgs.moduleName.get(0), null, installArgs.clientName); } catch (Exception e) { if (installArgs.verbose) @@ -417,17 +426,17 @@ private static int runInstallCommand(InstallCommandArgs installArgs, JCommander private static int runRunCommand(RunCommandArgs runArgs, JCommander jc) { if (runArgs.methodName == null || runArgs.methodName.size() != 1) { - ModuleBuilder.showError("Command Line Argument Error", + ModuleBuilder.showError("Command Line Argument Error", "One and only one method name should be provided"); return 1; } try { if (runArgs.inputFile == null && runArgs.inputJson == null && !runArgs.stdin) throw new IllegalStateException("No one input method is used " + - "(should be one of '-i', '-j' or '-s')"); - return new ModuleRunner(runArgs.sdkHome).run(runArgs.methodName.get(0), - runArgs.inputFile, runArgs.stdin, runArgs.inputJson, runArgs.output, - runArgs.tagVer, runArgs.verbose, runArgs.keepTempFiles, runArgs.provRefs, + "(should be one of '-i', '-j' or '-s')"); + return new ModuleRunner(runArgs.sdkHome).run(runArgs.methodName.get(0), + runArgs.inputFile, runArgs.stdin, runArgs.inputJson, runArgs.output, + runArgs.tagVer, runArgs.verbose, runArgs.keepTempFiles, runArgs.provRefs, runArgs.mountPoints); } catch (Exception e) { if (runArgs.verbose) @@ -439,150 +448,150 @@ private static int runRunCommand(RunCommandArgs runArgs, JCommander jc) { @Parameters(commandDescription = "Validate a module or modules.") private static class ValidateCommandArgs { - @Parameter(names={"-v","--verbose"}, description="Show verbose output") + @Parameter(names={"-v","--verbose"}, description="Show verbose output") boolean verbose = false; - - @Parameter(names={"-m", "--method_store"}, description="Narrative Method Store URL " + - "(default is " + DEFAULT_METHOD_STORE_URL + ")") - String methodStoreUrl = DEFAULT_METHOD_STORE_URL; - - @Parameter(names={"-a","--allow_sync_method"}, description="Allow synchonous methods " + - "(advanced option, default is false)") + + @Parameter(names={"-m", "--method_store"}, description="Narrative Method Store URL " + + "(default is " + DEFAULT_METHOD_STORE_URL + ")") + String methodStoreUrl = DEFAULT_METHOD_STORE_URL; + + @Parameter(names={"-a","--allow_sync_method"}, description="Allow synchonous methods " + + "(advanced option, default is false)") boolean allowSyncMethods = false; - - @Parameter(description="[path to the module directories]") + + @Parameter(description="[path to the module directories]") List modules; } - + @Parameters(commandDescription = "Initialize a module in the current directory.") private static class InitCommandArgs { - @Parameter(names={"-v","--verbose"}, description="Show verbose output") - boolean verbose = false; - - @Parameter(required=true, names={"-u","--user"}, - description="(Required) provide a username to serve as the owner of this module") - String userName; - - @Parameter(names={"-e","--example"}, - description="Include a fully featured example in your module. " + - "This generates an example set of code and configurations that can " + - "be used to demonstrate a very simple example.") - boolean example = false; - - @Parameter(names={"-l","--language"}, description="Choose a language for developing " + - " code in your module. You can currently choose from Python, Perl, and Java " + - "(default=Python)") - String language = ModuleInitializer.DEFAULT_LANGUAGE; - - @Parameter(required=true, description="") - List moduleNames; + @Parameter(names={"-v","--verbose"}, description="Show verbose output") + boolean verbose = false; + + @Parameter(required=true, names={"-u","--user"}, + description="(Required) provide a username to serve as the owner of this module") + String userName; + + @Parameter(names={"-e","--example"}, + description="Include a fully featured example in your module. " + + "This generates an example set of code and configurations that can " + + "be used to demonstrate a very simple example.") + boolean example = false; + + @Parameter(names={"-l","--language"}, description="Choose a language for developing " + + " code in your module. You can currently choose from Python, Perl, and Java " + + "(default=Python)") + String language = ModuleInitializer.DEFAULT_LANGUAGE; + + @Parameter(required=true, description="") + List moduleNames; } - - + + @Parameters(commandDescription = "Get help and usage information.") private static class HelpCommandArgs { - @Parameter(description="Show usage for this command") + @Parameter(description="Show usage for this command") List command; - @Parameter(names={"-a","--all"}, description="Show usage for all commands") + @Parameter(names={"-a","--all"}, description="Show usage for all commands") boolean showAll = false; } - + @Parameters(commandDescription = "Compile a KIDL file into client and server code") public static class CompileCommandArgs { - - @Parameter(names="--out",description="Set the output folder name (default is the current directory)") - //, metaVar="") + + @Parameter(names="--out",description="Set the output folder name (default is the current directory)") + //, metaVar="") String outDir = null; - - @Parameter(names="--js", description="Generate a JavaScript client with a standard default name") + + @Parameter(names="--js", description="Generate a JavaScript client with a standard default name") boolean jsClientSide = false; - @Parameter(names="--jsclname", description="Generate a JavaScript client with the name provided by this option" + - "(overrides --js option)")//, metaVar = "") + @Parameter(names="--jsclname", description="Generate a JavaScript client with the name provided by this option" + + "(overrides --js option)")//, metaVar = "") String jsClientName = null; - @Parameter(names="--pl", description="Generate a Perl client with the default name") + @Parameter(names="--pl", description="Generate a Perl client with the default name") boolean perlClientSide = false; - @Parameter(names="--plclname", description="Generate a Perl client with the name provided optionally " + - "prefixed by subdirectories separated by :: as in the standard perl module syntax (e.g. "+ - "Bio::KBase::MyModule::Client, overrides the --pl option)")//, metaVar = "") + @Parameter(names="--plclname", description="Generate a Perl client with the name provided optionally " + + "prefixed by subdirectories separated by :: as in the standard perl module syntax (e.g. "+ + "Bio::KBase::MyModule::Client, overrides the --pl option)")//, metaVar = "") String perlClientName = null; - @Parameter(names="--plsrv", description="Generate a Perl server with a " + - "standard default name. If set, Perl clients will automatically be generated too.") + @Parameter(names="--plsrv", description="Generate a Perl server with a " + + "standard default name. If set, Perl clients will automatically be generated too.") boolean perlServerSide = false; - @Parameter(names="--plsrvname", description="Generate a Perl server with with the " + - "name provided, optionally prefixed by subdirectories separated by :: as "+ - "in the standard perl module syntax (e.g. Bio::KBase::MyModule::Server, "+ - "overrides the --plserv option). If set, Perl clients will be generated too.") - //, metaVar = "") + @Parameter(names="--plsrvname", description="Generate a Perl server with with the " + + "name provided, optionally prefixed by subdirectories separated by :: as "+ + "in the standard perl module syntax (e.g. Bio::KBase::MyModule::Server, "+ + "overrides the --plserv option). If set, Perl clients will be generated too.") + //, metaVar = "") String perlServerName = null; - @Parameter(names="--plimplname", description="Generate a Perl server implementation with the " + - "name provided, optionally prefixed by subdirectories separated by :: as "+ - "in the standard Perl module syntax (e.g. Bio::KBase::MyModule::Impl). "+ - "If set, Perl server and client code will be generated too.")//, metaVar = "") + @Parameter(names="--plimplname", description="Generate a Perl server implementation with the " + + "name provided, optionally prefixed by subdirectories separated by :: as "+ + "in the standard Perl module syntax (e.g. Bio::KBase::MyModule::Impl). "+ + "If set, Perl server and client code will be generated too.")//, metaVar = "") String perlImplName = null; - @Parameter(names="--plpsginame", description="Generate a perl PSGI file with the name provided.")//, metaVar = "") + @Parameter(names="--plpsginame", description="Generate a perl PSGI file with the name provided.")//, metaVar = "") String perlPsgiName = null; - @Parameter(names="--plenableretries", description="When set, generated Perl client code will enable "+ - "reconnection retries with the server") + @Parameter(names="--plenableretries", description="When set, generated Perl client code will enable "+ + "reconnection retries with the server") boolean perlEnableRetries = false; - @Parameter(names="--py", description="Generate a Python client with a standard default name") + @Parameter(names="--py", description="Generate a Python client with a standard default name") boolean pyClientSide = false; - @Parameter(names="--pyclname", description="Generate a Python client with with the " + - "name provided, optionally prefixed by subdirectories separated by '.' as "+ - "in the standard Python module syntax (e.g. biokbase.mymodule.client,"+ - "overrides the --py option).")//, metaVar = "") + @Parameter(names="--pyclname", description="Generate a Python client with with the " + + "name provided, optionally prefixed by subdirectories separated by '.' as "+ + "in the standard Python module syntax (e.g. biokbase.mymodule.client,"+ + "overrides the --py option).")//, metaVar = "") String pyClientName = null; - @Parameter(names="--pysrv", description="Generate a Python server with a " + - "standard default name. If set, Python clients will automatically be generated too.") + @Parameter(names="--pysrv", description="Generate a Python server with a " + + "standard default name. If set, Python clients will automatically be generated too.") boolean pyServerSide = false; - @Parameter(names="--pysrvname", description="Generate a Python server with the " + - "name provided, optionally prefixed by subdirectories separated by '.' as "+ - "in the standard Python module syntax (e.g. biokbase.mymodule.server,"+ - "overrides the --pysrv option).")//, metaVar = "") + @Parameter(names="--pysrvname", description="Generate a Python server with the " + + "name provided, optionally prefixed by subdirectories separated by '.' as "+ + "in the standard Python module syntax (e.g. biokbase.mymodule.server,"+ + "overrides the --pysrv option).")//, metaVar = "") String pyServerName = null; - @Parameter(names="--pyimplname", description="Generate a Python server implementation with the " + - "name provided, optionally prefixed by subdirectories separated by '.' as "+ - "in the standard Python module syntax (e.g. biokbase.mymodule.impl)." + - " If set, Python server and client code will be generated too.")//, metaVar = "") + @Parameter(names="--pyimplname", description="Generate a Python server implementation with the " + + "name provided, optionally prefixed by subdirectories separated by '.' as "+ + "in the standard Python module syntax (e.g. biokbase.mymodule.impl)." + + " If set, Python server and client code will be generated too.")//, metaVar = "") String pyImplName = null; - @Parameter(names="--java", description="Generate Java client code in the directory set by --javasrc") + @Parameter(names="--java", description="Generate Java client code in the directory set by --javasrc") boolean javaClientSide = false; - @Parameter(names="--javasrc",description="Set the output folder for generated Java code")//, metaVar = - //"") + @Parameter(names="--javasrc",description="Set the output folder for generated Java code")//, metaVar = + //"") String javaSrcDir = "src"; - @Parameter(names="--javalib",description="Set the output folder for jar files (if defined then --java will be " + - "treated as true automatically)")//, metaVar = - //"") + @Parameter(names="--javalib",description="Set the output folder for jar files (if defined then --java will be " + + "treated as true automatically)")//, metaVar = + //"") String javaLibDir = null; - @Parameter(names="--url", description="Set the default url for the service in generated client code")//, metaVar = "") + @Parameter(names="--url", description="Set the default url for the service in generated client code")//, metaVar = "") String url = null; - @Parameter(names="--javapackage",description="Set the Java package for generated code (module subpackages are " + - "created in this package), default value is " + defaultParentPackage)//, - //metaVar = "") + @Parameter(names="--javapackage",description="Set the Java package for generated code (module subpackages are " + + "created in this package), default value is " + defaultParentPackage)//, + //metaVar = "") String javaPackageParent = defaultParentPackage; - @Parameter(names="--javasrv", description="Generate Java server code in the directory set by --javasrc") + @Parameter(names="--javasrv", description="Generate Java server code in the directory set by --javasrc") boolean javaServerSide = false; - @Parameter(names="--javagwt",description="Generate a GWT client Java package (useful if you need " + - "copies of generated classes for GWT clients)")//, metaVar="") + @Parameter(names="--javagwt",description="Generate a GWT client Java package (useful if you need " + + "copies of generated classes for GWT clients)")//, metaVar="") String javaGwtPackage = null; @Parameter(names="--r", description="DEPRECATED Generate a R client with a standard default name") @@ -610,20 +619,20 @@ public static class CompileCommandArgs { " If set, R server and client code will be generated too.") String rImplName = null; - @Parameter(names="--jsonschema",description="Generate JSON schema documents for the types in the output folder specified.")//, metaVar="") + @Parameter(names="--jsonschema",description="Generate JSON schema documents for the types in the output folder specified.")//, metaVar="") String jsonSchema = null; - - @Parameter(names="--javabuildxml",description="Will generate build.xml template for Ant") + + @Parameter(names="--javabuildxml",description="Will generate build.xml template for Ant") boolean javaBuildXml; - - @Parameter(names="--makefile",description="Will generate makefile templates for servers and/or java client") + + @Parameter(names="--makefile",description="Will generate makefile templates for servers and/or java client") boolean makefile = false; @Parameter(names="--clasyncver",description="Will set in client code version of service for asyncronous calls " + - "(it could be git commit hash of version registered in catalog or one of version tags: dev/beta/release)") + "(it could be git commit hash of version registered in catalog or one of version tags: dev/beta/release)") String clAsyncVer = null; - + @Parameter(names="--dynservver", description="Clients will be built " + "for use with KBase dynamic services (e.g. with URL lookup " + "via the Service Wizard) with the specified version " + @@ -631,11 +640,11 @@ public static class CompileCommandArgs { "clasyncver may not be specified if " + "dynservver is specified.") String dynservVer = null; - + @Parameter(names="--html", description="Generate HTML version(s) of " + "the input spec file(s)") boolean html = false; - + @Parameter(names={"-v", "--verbose"}, description="Print full stack " + "trace on a compile failure") boolean verbose = false; @@ -643,16 +652,16 @@ public static class CompileCommandArgs { @Parameter(required=true, description="") List specFileNames; } - + @Parameters(commandDescription = "Test a module with local Docker.") private static class TestCommandArgs { @Parameter(names={"-m", "--method_store"}, description="Narrative Method Store URL used in validation " + "(default is " + DEFAULT_METHOD_STORE_URL + ")") String methodStoreUrl = DEFAULT_METHOD_STORE_URL; - + @Parameter(names={"-s", "--skip_validation"}, description="Will skip validation step (default is false)") boolean skipValidation = false; - + @Parameter(names={"-a","--allow_sync_method"}, description="Allow synchonous methods " + "(advanced option, part of validation settings, default value is false)") boolean allowSyncMethods = false; @@ -660,11 +669,11 @@ private static class TestCommandArgs { @Parameter(names={"-v","--verbose"}, description="Print more details including error stack traces") boolean verbose = false; } - + @Parameters(commandDescription = "Print current version of kb-sdk.") private static class VersionCommandArgs { } - + @Parameters(commandDescription = "Rename a module name.") private static class RenameCommandArgs { @Parameter(names={"-v","--verbose"}, description="Print more details including error stack traces") @@ -679,16 +688,16 @@ private static class InstallCommandArgs { @Parameter(names={"-l", "--language"}, description="Language of generated client code " + "(default is one defined in kbase.yml)") String lang; - + @Parameter(names={"-a", "--async"}, description="Force generation of asynchronous calls " + - "(default is chosen based on information registered in catalog)") + "(default is chosen based on information registered in catalog)") boolean async = false; @Parameter(names={"-s", "--sync"}, description="Depricated flag, means the same as -c (--core)") boolean sync = false; @Parameter(names={"-c", "--core"}, description="Force generation of calls to core service" + - "(WARNING: please use it only for core services not registered in catalog)") + "(WARNING: please use it only for core services not registered in catalog)") boolean core = false; @Parameter(names={"-d", "--dynamic"}, description="Force generation of dynamic service calls" + @@ -701,7 +710,7 @@ private static class InstallCommandArgs { @Parameter(names={"-v","--verbose"}, description="Print more details including error stack traces") boolean verbose = false; - + @Parameter(names={"-n","--clientname"}, description="Optional parameter defining custom client name " + "(default is module name)") String clientName = null; @@ -713,19 +722,19 @@ private static class InstallCommandArgs { @Parameters(commandDescription = "Run a method of registered KBase module locally.") private static class RunCommandArgs { @Parameter(names={"-i", "--input"}, description="Input JSON file " + - "(optional, if not set -s or -j should be used)") + "(optional, if not set -s or -j should be used)") File inputFile = null; @Parameter(names={"-s","--stdin"}, description="Flag for reading input data from STDIN " + - "(default is false, used if neither -i or -j is set)") + "(default is false, used if neither -i or -j is set)") boolean stdin = false; @Parameter(names={"-j","--json"}, description="Input JSON string " + - "(optional, if not set -s or -i should be used)") + "(optional, if not set -s or -i should be used)") String inputJson = null; @Parameter(names={"-o", "--output"}, description="Output JSON file " + - "(optional, if not set output will be printed to STDOUT)") + "(optional, if not set output will be printed to STDOUT)") File output = null; @Parameter(names={"-t","--tag-or-ver"}, description="Tag or version " + @@ -733,95 +742,95 @@ private static class RunCommandArgs { String tagVer = null; @Parameter(names={"-v","--verbose"}, description="Print more details including error " + - "stack traces") + "stack traces") boolean verbose = false; @Parameter(names={"-k","--keep-tmp"}, description="Keep temporary files/folders at the " + - "end (default value is false)") + "end (default value is false)") boolean keepTempFiles = false; - + @Parameter(names={"-h","--sdk-home"}, description="Home folder of kb-sdk where sdk.cfg " + - "file and run_local folder are expected to be found or created if absent " + + "file and run_local folder are expected to be found or created if absent " + "(default path is loaded from 'KB_SDK_HOME' system environment variable)") String sdkHome = null; @Parameter(names={"-r","--prov-refs"}, description="Optional comma-separated list of " + - "references workspace objects that will be refered from provenance") + "references workspace objects that will be refered from provenance") String provRefs = null; @Parameter(names={"-m","--mount-points"}, description="Optional comma-separated list of " + - "mount point pairs for docker container (this parameter contains of local folder" + - " and inner folder separated by ':', local folder points to place in local file " + - "system, inner folder appears inside docker container, if inner path is not " + - "absolute it's treated as relative to /kb/module/work folder; if some mount " + - "point path doesn't have ':' and inner part then it appears as " + - "/kb/module/work/tmp inside docker)") + "mount point pairs for docker container (this parameter contains of local folder" + + " and inner folder separated by ':', local folder points to place in local file " + + "system, inner folder appears inside docker container, if inner path is not " + + "absolute it's treated as relative to /kb/module/work folder; if some mount " + + "point path doesn't have ':' and inner part then it appears as " + + "/kb/module/work/tmp inside docker)") String mountPoints = null; @Parameter(required=true, description="") + "(with SDK module prefix followed by '.')>") List methodName; } private static void showBriefHelp(JCommander jc, PrintStream out) { - Map commands = jc.getCommands(); - out.println(""); - out.println(MODULE_BUILDER_SH_NAME + " - a developer tool for building and validating KBase modules"); - out.println(""); - out.println("usage: "+MODULE_BUILDER_SH_NAME+" [options]"); - out.println(""); - out.println(" The available commands are:"); - for (Map.Entry command : commands.entrySet()) { - out.println(" " + command.getKey() +" - "+jc.getCommandDescription(command.getKey())); - } - out.println(""); - out.println(" For help on a specific command, see \"kb-sdk help \"."); - out.println(" For full usage information, see \"kb-sdk help -a\"."); - out.println(""); + Map commands = jc.getCommands(); + out.println(""); + out.println(MODULE_BUILDER_SH_NAME + " - a developer tool for building and validating KBase modules"); + out.println(""); + out.println("usage: "+MODULE_BUILDER_SH_NAME+" [options]"); + out.println(""); + out.println(" The available commands are:"); + for (Map.Entry command : commands.entrySet()) { + out.println(" " + command.getKey() +" - "+jc.getCommandDescription(command.getKey())); + } + out.println(""); + out.println(" For help on a specific command, see \"kb-sdk help \"."); + out.println(" For full usage information, see \"kb-sdk help -a\"."); + out.println(""); }; - + private static void showCommandUsage(JCommander jc, HelpCommandArgs helpArgs, PrintStream out) { - if(helpArgs.showAll) { - showFullUsage(jc, out); - } else if(helpArgs.command !=null && helpArgs.command.size()>0) { - String indent = ""; - StringBuilder usage = new StringBuilder(); - for(String cmd:helpArgs.command) { - if(jc.getCommands().containsKey(cmd)) { - usage.append(MODULE_BUILDER_SH_NAME + " "+cmd+"\n"); - jc.usage(cmd,usage,indent+" "); - usage.append("\n"); - } else { - out.println("Command \""+cmd+"\" is not a valid command. To view available commands, run:"); - out.println(" "+MODULE_BUILDER_SH_NAME+" "+HELP_COMMAND); - return; - } - } - out.print(usage.toString()); - } else { - showBriefHelp(jc, out); - } + if(helpArgs.showAll) { + showFullUsage(jc, out); + } else if(helpArgs.command !=null && helpArgs.command.size()>0) { + String indent = ""; + StringBuilder usage = new StringBuilder(); + for(String cmd:helpArgs.command) { + if(jc.getCommands().containsKey(cmd)) { + usage.append(MODULE_BUILDER_SH_NAME + " "+cmd+"\n"); + jc.usage(cmd,usage,indent+" "); + usage.append("\n"); + } else { + out.println("Command \""+cmd+"\" is not a valid command. To view available commands, run:"); + out.println(" "+MODULE_BUILDER_SH_NAME+" "+HELP_COMMAND); + return; + } + } + out.print(usage.toString()); + } else { + showBriefHelp(jc, out); + } }; - + private static void showFullUsage(JCommander jc, PrintStream out) { - String indent = ""; - StringBuilder usage = new StringBuilder(); - jc.usage(usage,indent); - out.print(usage.toString()); + String indent = ""; + StringBuilder usage = new StringBuilder(); + jc.usage(usage,indent); + out.print(usage.toString()); }; - - + + private static void showError(String error, String message, String extraHelp) { - System.err.println(error + ": " + message+"\n"); - if(!extraHelp.isEmpty()) - System.err.println(extraHelp+"\n"); - System.err.println("For more help and usage information, run:"); - System.err.println(" "+MODULE_BUILDER_SH_NAME+" "+HELP_COMMAND); - System.err.println(""); + System.err.println(error + ": " + message+"\n"); + if(!extraHelp.isEmpty()) + System.err.println(extraHelp+"\n"); + System.err.println("For more help and usage information, run:"); + System.err.println(" "+MODULE_BUILDER_SH_NAME+" "+HELP_COMMAND); + System.err.println(""); } - + private static void showError(String error, String message) { - showError(error,message,""); + showError(error,message,""); } } diff --git a/src/java/us/kbase/mobu/compiler/PrevCodeParser.java b/src/java/us/kbase/mobu/compiler/PrevCodeParser.java index f59a7a49..b4a4ed33 100644 --- a/src/java/us/kbase/mobu/compiler/PrevCodeParser.java +++ b/src/java/us/kbase/mobu/compiler/PrevCodeParser.java @@ -25,7 +25,7 @@ public class PrevCodeParser { public static HashMap parsePrevCode(File implFile, String commentPrefix, List funcs, boolean withClassHeader) throws IOException, ParseException { - commentPrefix = Pattern.quote(commentPrefix); + commentPrefix = Pattern.quote(commentPrefix) + "[ ]?"; Pattern PAT_HEADER = Pattern.compile(".*" + commentPrefix + "BEGIN_HEADER\n(.*\n)?[ \t]*" + commentPrefix + "END_HEADER\n.*", Pattern.DOTALL); Pattern PAT_CLASS_HEADER = Pattern.compile(".*" + commentPrefix + diff --git a/src/java/us/kbase/mobu/compiler/report/CompilationReporter.java b/src/java/us/kbase/mobu/compiler/report/CompilationReporter.java index afef7a0e..535031ed 100644 --- a/src/java/us/kbase/mobu/compiler/report/CompilationReporter.java +++ b/src/java/us/kbase/mobu/compiler/report/CompilationReporter.java @@ -34,12 +34,12 @@ public class CompilationReporter { public static void prepareCompileReport(File codeDir, List services, - boolean perlServerSide, String perlImplName, boolean pyServerSide, + boolean perlServerSide, String perlImplName, boolean pyServerSide, String pyImplName, boolean rServerSide, String rImplName, - boolean javaServerSide, String javaPackageParent, String javaSrcPath, + boolean javaServerSide, String javaPackageParent, String javaSrcPath, JavaData javaParsingData, List specFiles, File reportFile) throws Exception { String sdkVersion = ModuleBuilder.VERSION; - String sdkGitCommit = ModuleBuilder.getGitCommit(); + String sdkGitCommit = ModuleBuilder.getGitProp("commit"); String moduleName = null; KbModule module = null; for (KbService srv : services) @@ -74,11 +74,11 @@ public static void prepareCompileReport(File codeDir, List services, FileSaver javaSrcDir = new DiskFileSaver(javaSrc); for (JavaModule jmodule : javaParsingData.getModules()) { if (jmodule.getOriginal().getModuleName().equals(moduleName)) { - String moduleDir = JavaTypeGenerator.sub(javaPackageParent, + String moduleDir = JavaTypeGenerator.sub(javaPackageParent, jmodule.getModulePackage()).replace('.', '/'); - String serverClassName = TextUtils.capitalize(jmodule.getModuleName()) + + String serverClassName = TextUtils.capitalize(jmodule.getModuleName()) + "Server"; - implFile = javaSrcDir.getAsFileOrNull(moduleDir + "/" + + implFile = javaSrcDir.getAsFileOrNull(moduleDir + "/" + serverClassName + ".java"); } } @@ -102,7 +102,7 @@ public static void prepareCompileReport(File codeDir, List services, public static Report createReport(List specFiles, String sdkVersion, String sdkGitCommit, String moduleName, - KbModule module, String implFilePath, String implCommentPrefix, + KbModule module, String implFilePath, String implCommentPrefix, String implText) throws Exception, IOException { Map funcPositions = new LinkedHashMap(); String commentPrefix = Pattern.quote(implCommentPrefix); @@ -111,8 +111,8 @@ public static Report createReport(List specFiles, if (comp instanceof KbFuncdef) { KbFuncdef func = (KbFuncdef)comp; String funcName = func.getName(); - Pattern p = Pattern.compile(MessageFormat.format(".*" + commentPrefix + - "BEGIN {0}\n(.*\n)?[ \t]*" + commentPrefix + "END {0}\n.*", + Pattern p = Pattern.compile(MessageFormat.format(".*" + commentPrefix + + "BEGIN {0}\n(.*\n)?[ \t]*" + commentPrefix + "END {0}\n.*", funcName), Pattern.DOTALL); FunctionPlace place = checkMatch(funcPositions, p, implText); if (place != null) diff --git a/src/java/us/kbase/mobu/initializer/ModuleInitializer.java b/src/java/us/kbase/mobu/initializer/ModuleInitializer.java index 43141980..fc71556d 100644 --- a/src/java/us/kbase/mobu/initializer/ModuleInitializer.java +++ b/src/java/us/kbase/mobu/initializer/ModuleInitializer.java @@ -17,111 +17,111 @@ import us.kbase.templates.TemplateFormatter; public class ModuleInitializer { - public static final String DEFAULT_LANGUAGE = "python"; + public static final String DEFAULT_LANGUAGE = "python"; - private String moduleName; - private String userName; - private String language; - private boolean verbose; - private File dir; - - private static String[] subdirs = {"data", - "scripts", - "lib", - "test", - "ui", - "ui/narrative", - "ui/narrative/methods"}; - - public ModuleInitializer(String moduleName, String userName, boolean verbose) { - this(moduleName, userName, DEFAULT_LANGUAGE, verbose); - } - - public ModuleInitializer(String moduleName, String userName, String language, boolean verbose) { - this(moduleName, userName, language, verbose, null); - } - - public ModuleInitializer(String moduleName, String userName, String language, + private String moduleName; + private String userName; + private String language; + private boolean verbose; + private File dir; + + private static String[] subdirs = { "data", + "scripts", + "lib", + "test", + "ui", + "ui/narrative", + "ui/narrative/methods"}; + + public ModuleInitializer(String moduleName, String userName, boolean verbose) { + this(moduleName, userName, DEFAULT_LANGUAGE, verbose); + } + + public ModuleInitializer(String moduleName, String userName, String language, boolean verbose) { + this(moduleName, userName, language, verbose, null); + } + + public ModuleInitializer(String moduleName, String userName, String language, boolean verbose, File dir) { - this.moduleName = moduleName; - this.userName = userName; - this.language = language == null ? DEFAULT_LANGUAGE : language; - this.verbose = verbose; - this.dir = dir; - } - - /** - * - * @param example - * @throws IOException - */ - public void initialize(boolean example) throws Exception { - if (this.moduleName == null) { - throw new RuntimeException("Unable to create a null directory!"); - } - String moduleDir = dir == null ? this.moduleName : + this.moduleName = moduleName; + this.userName = userName; + this.language = language == null ? DEFAULT_LANGUAGE : language; + this.verbose = verbose; + this.dir = dir; + } + + /** + * + * @param example + * @throws IOException + */ + public void initialize(boolean example) throws Exception { + if (this.moduleName == null) { + throw new RuntimeException("Unable to create a null directory!"); + } + String moduleDir = dir == null ? this.moduleName : new File(dir, this.moduleName).getPath(); - this.language = qualifyLanguage(this.language); - - if (this.verbose) { - String msg = "Initializing module \"" + this.moduleName + "\""; - if (example) - msg += " with example methods"; - System.out.println(msg); - } + this.language = qualifyLanguage(this.language); + + if (this.verbose) { + String msg = "Initializing module \"" + this.moduleName + "\""; + if (example) + msg += " with example methods"; + System.out.println(msg); + } - List subdirList = new ArrayList(Arrays.asList(subdirs)); - if (example && this.language.equals("r")) { - subdirList.add("ui/narrative/methods/count_contigs_in_set"); - subdirList.add("ui/narrative/methods/count_contigs_in_set/img"); - } - else { - subdirList.add("ui/narrative/methods/run_" + this.moduleName); - subdirList.add("ui/narrative/methods/run_" + this.moduleName + "/img"); - } - - // 1. build dir with moduleName - initDirectory(Paths.get(moduleDir), true); - - // 2. build skeleton subdirs - for (String dir : subdirList) { - initDirectory(Paths.get(moduleDir, dir), true); - } + List subdirList = new ArrayList(Arrays.asList(subdirs)); + if (example && this.language.equals("r")) { + subdirList.add("ui/narrative/methods/count_contigs_in_set"); + subdirList.add("ui/narrative/methods/count_contigs_in_set/img"); + } + else { + subdirList.add("ui/narrative/methods/run_" + this.moduleName); + subdirList.add("ui/narrative/methods/run_" + this.moduleName + "/img"); + } + + // 1. build dir with moduleName + initDirectory(Paths.get(moduleDir), true); + + // 2. build skeleton subdirs + for (String dir : subdirList) { + initDirectory(Paths.get(moduleDir, dir), true); + } + + /* + * 3. Fill in templated files and write them + * + * Set up the context - the set of variables used to flesh out the templates */ + String specFile = Paths.get(this.moduleName + ".spec").toString(); - /* - * 3. Fill in templated files and write them - * - * Set up the context - the set of variables used to flesh out the templates */ - String specFile = Paths.get(this.moduleName + ".spec").toString(); - - Map moduleContext = new HashMap(); - moduleContext.put("module_name", this.moduleName); - moduleContext.put("user_name", this.userName); - moduleContext.put("spec_file", specFile); - moduleContext.put("language", this.language); - moduleContext.put("module_root_path", Paths.get(moduleDir).toAbsolutePath()); - moduleContext.put("example", example); - moduleContext.put("dollar_sign", "$"); + Map moduleContext = new HashMap(); + moduleContext.put("module_name", this.moduleName); + moduleContext.put("user_name", this.userName); + moduleContext.put("spec_file", specFile); + moduleContext.put("language", this.language); + moduleContext.put("module_root_path", Paths.get(moduleDir).toAbsolutePath()); + moduleContext.put("example", example); + moduleContext.put("dollar_sign", "$"); moduleContext.put("os_name", System.getProperty("os.name")); - Map templateFiles = new HashMap(); - templateFiles.put("module_typespec", Paths.get(moduleDir, specFile)); - templateFiles.put("module_travis", Paths.get(moduleDir, ".travis.yml")); - templateFiles.put("module_dockerfile", Paths.get(moduleDir, "Dockerfile")); - templateFiles.put("module_readme", Paths.get(moduleDir, "README.md")); - templateFiles.put("module_release_notes", Paths.get(moduleDir, "RELEASE_NOTES.md")); - templateFiles.put("module_makefile", Paths.get(moduleDir, "Makefile")); - templateFiles.put("module_deploy_cfg", Paths.get(moduleDir, "deploy.cfg")); - templateFiles.put("module_license", Paths.get(moduleDir, "LICENSE")); - templateFiles.put("module_docker_entrypoint", Paths.get(moduleDir, "scripts", "entrypoint.sh")); + Map templateFiles = new HashMap(); + templateFiles.put("module_typespec", Paths.get(moduleDir, specFile)); + templateFiles.put("module_travis", Paths.get(moduleDir, ".travis.yml")); + templateFiles.put("module_dockerfile", Paths.get(moduleDir, "Dockerfile")); + templateFiles.put("module_readme", Paths.get(moduleDir, "README.md")); + templateFiles.put("module_release_notes", Paths.get(moduleDir, "RELEASE_NOTES.md")); + templateFiles.put("module_makefile", Paths.get(moduleDir, "Makefile")); + templateFiles.put("module_deploy_cfg", Paths.get(moduleDir, "deploy.cfg")); + templateFiles.put("module_license", Paths.get(moduleDir, "LICENSE")); + templateFiles.put("module_docker_entrypoint", Paths.get(moduleDir, "scripts", "entrypoint.sh")); templateFiles.put("module_prepare_deploy_cfg", Paths.get(moduleDir, "scripts", "prepare_deploy_cfg.py")); templateFiles.put("module_run_async", Paths.get(moduleDir, "scripts", "run_async.sh")); - templateFiles.put("module_readme_lib", Paths.get(moduleDir, "lib", "README.md")); - templateFiles.put("module_readme_ui", Paths.get(moduleDir, "ui", "README.md")); - templateFiles.put("module_readme_test", Paths.get(moduleDir, "test", "README.md")); - templateFiles.put("module_readme_data", Paths.get(moduleDir, "data", "README.md")); - templateFiles.put("module_config_yaml", Paths.get(moduleDir, "kbase.yml")); + templateFiles.put("module_readme_lib", Paths.get(moduleDir, "lib", "README.md")); + templateFiles.put("module_readme_ui", Paths.get(moduleDir, "ui", "README.md")); + templateFiles.put("module_readme_test", Paths.get(moduleDir, "test", "README.md")); + templateFiles.put("module_readme_data", Paths.get(moduleDir, "data", "README.md")); + templateFiles.put("module_config_yaml", Paths.get(moduleDir, "kbase.yml")); templateFiles.put("module_gitignore", Paths.get(moduleDir, ".gitignore")); templateFiles.put("module_dockerignore", Paths.get(moduleDir, ".dockerignore")); templateFiles.put("module_readme_test_local", Paths.get(moduleDir, "test_local", "readme.txt")); @@ -129,13 +129,13 @@ public void initialize(boolean example) throws Exception { templateFiles.put("module_run_tests", Paths.get(moduleDir, "test_local", "run_tests.sh")); templateFiles.put("module_run_bash", Paths.get(moduleDir, "test_local", "run_bash.sh")); templateFiles.put("module_run_docker", Paths.get(moduleDir, "test_local", "run_docker.sh")); - - switch (language) { - case "java": + + switch (language) { + case "java": templateFiles.put("module_build_xml", Paths.get(moduleDir, "build.xml")); - templateFiles.put("module_web_xml", Paths.get(moduleDir, "scripts", "web.xml")); + templateFiles.put("module_web_xml", Paths.get(moduleDir, "scripts", "web.xml")); templateFiles.put("module_jetty_xml", Paths.get(moduleDir, "scripts", "jetty.xml")); - fillTemplate(moduleContext, "module_typespec", templateFiles.remove("module_typespec")); + fillTemplate(moduleContext, "module_typespec", templateFiles.remove("module_typespec")); String javaPackageParent = "."; // Special value meaning top level package. moduleContext.put("java_package_parent", javaPackageParent); JavaData data = JavaTypeGenerator.parseSpec(new File(moduleDir, specFile)); @@ -148,40 +148,39 @@ public void initialize(boolean example) throws Exception { File testJavaFile = new File(testSrcDir, modulePackage.replace('.', '/') + "/test/" + javaModuleName + "ServerTest.java"); fillTemplate(moduleContext, "module_test_java_client", testJavaFile.toPath()); break; - case "python": + case "python": templateFiles.put("module_test_python_client", Paths.get(moduleDir, "test", this.moduleName + "_server_test.py")); templateFiles.put("module_tox", Paths.get(moduleDir, "tox.ini")); break; case "perl": - templateFiles.put("module_test_perl_client", Paths.get(moduleDir, "test", this.moduleName + "_server_test.pl")); + templateFiles.put("module_cpanfile", Paths.get(moduleDir, "cpanfile")); + templateFiles.put("module_test_perl_client", Paths.get(moduleDir, "test", this.moduleName + "_impl.t")); + templateFiles.put("module_test_perl_compile", Paths.get(moduleDir, "test", "00_compile.t")); break; case "r": templateFiles.put("module_test_r_client", Paths.get(moduleDir, "test", this.moduleName + "_server_test.r")); break; - } - - if (example && this.language.equals("r")) { - templateFiles.put("module_method_spec_json", Paths.get(moduleDir, "ui", "narrative", "methods", "count_contigs_in_set", "spec.json")); - templateFiles.put("module_method_spec_yaml", Paths.get(moduleDir, "ui", "narrative", "methods", "count_contigs_in_set", "display.yaml")); - } else { + } + + if (example && this.language.equals("r")) { + templateFiles.put("module_method_spec_json", Paths.get(moduleDir, "ui", "narrative", "methods", "count_contigs_in_set", "spec.json")); + templateFiles.put("module_method_spec_yaml", Paths.get(moduleDir, "ui", "narrative", "methods", "count_contigs_in_set", "display.yaml")); + } else { templateFiles.put("module_method_spec_json", Paths.get(moduleDir, "ui", "narrative", "methods", "run_"+this.moduleName, "spec.json")); templateFiles.put("module_method_spec_yaml", Paths.get(moduleDir, "ui", "narrative", "methods", "run_"+this.moduleName, "display.yaml")); } switch(this.language) { - // Perl just needs an impl file and a start server script + // start_server scripts are generated by the Makefile case "perl": - //templateFiles.put("module_start_perl_server", Paths.get(moduleDir, "scripts", "start_server.sh")); - // start_server script is now made in Makefile + // Perl needs an impl file templateFiles.put("module_perl_impl", Paths.get(moduleDir, "lib", this.moduleName, this.moduleName + "Impl.pm")); break; - // Python needs some empty __init__.py files and the impl file (Done, see TemplateBasedGenerator.initPyhtonPackages) case "python": + // python requires impl and __init__.py files initDirectory(Paths.get(moduleDir, "lib", this.moduleName), false); initFile(Paths.get(moduleDir, "lib", this.moduleName, "__init__.py"), false); templateFiles.put("module_python_impl", Paths.get(moduleDir, "lib", this.moduleName, this.moduleName + "Impl.py")); - //templateFiles.put("module_start_python_server", Paths.get(moduleDir, "scripts", "start_server.sh")); - // start_server script is now made in Makefile break; case "r": templateFiles.put("module_r_impl", Paths.get(moduleDir, "lib", this.moduleName, this.moduleName + "Impl.r")); @@ -194,131 +193,129 @@ public void initialize(boolean example) throws Exception { File serverJavaFile = new File(srcDir, modulePackage.replace('.', '/') + "/" + javaModuleName + "Server.java"); fillTemplate(moduleContext, "module_java_impl", serverJavaFile.toPath()); JavaTypeGenerator.processSpec(new File(moduleDir, specFile), srcDir, javaPackageParent, true, null, null, null); - //templateFiles.put("module_start_java_server", Paths.get(moduleDir, "scripts", "start_server.sh")); - // start_server script is now made in Makefile break; default: break; } - for (String templateName : templateFiles.keySet()) { - fillTemplate(moduleContext, templateName, templateFiles.get(templateName)); - } - - if (example) { - // Generated examples require some other SDK dependencies + for (String templateName : templateFiles.keySet()) { + fillTemplate(moduleContext, templateName, templateFiles.get(templateName)); + } + + if (example) { + // Generated examples require some other SDK dependencies new ClientInstaller(new File(moduleDir), false).install( this.language, - true, // async clients - false, // core or sync clients - false, // dynamic client - null, //tagVer + true, // async clients + false, // core or sync clients + false, // dynamic client + null, // tagVer this.verbose, "AssemblyUtil", null, - null // clientName + null // clientName ); - } - // Let's install fresh workspace client in any cases (we need it at least in tests): - new ClientInstaller(new File(moduleDir), false).install( + } + // Let's install fresh workspace client in any cases (we need it at least in tests): + new ClientInstaller(new File(moduleDir), false).install( this.language, - false, // async clients - true, // core or sync clients - false, // dynamic client - null, //tagVer + false, // async clients + true, // core or sync clients + false, // dynamic client + null, // tagVer this.verbose, "https://raw.githubusercontent.com/kbase/workspace_deluxe/master/workspace.spec", null, - null // clientName + null // clientName ); new ClientInstaller(new File(moduleDir), false).install( this.language, - true, // async clients - false, // core or sync clients - false, // dynamic client - null, //tagVer + true, // async clients + false, // core or sync clients + false, // dynamic client + null, // tagVer this.verbose, "KBaseReport", null, - null // clientName + null // clientName ); - System.out.println("Done! Your module is available in the " + moduleDir + " directory."); - System.out.println("Compile and run the example methods with the following inputs:"); - System.out.println(" cd " + moduleDir); - System.out.println(" make (required after making changes to " + new File(specFile).getName() + ")"); - System.out.println(" kb-sdk test (will require setting test user account credentials in test_local/test.cfg)"); - System.out.println(); - } - - /** - * - * @param dirPath - * @throws IOException - */ - private void initDirectory(Path dirPath, boolean failOnExist) throws IOException { - if (this.verbose) System.out.println("Making directory \"" + dirPath.toString() + "\""); - File newDir = dirPath.toFile(); - if (!newDir.exists()) { - newDir.mkdirs(); - } - else if (failOnExist) { - throw new IOException("Error while creating module - " + dirPath + " already exists!"); - } - } - - private void initFile(Path filePath, boolean failOnExist) throws IOException { - if (this.verbose) System.out.println("Building empty file \"" + filePath.toString() + "\""); - boolean done = filePath.toFile().createNewFile(); - if (!done && failOnExist) - throw new IOException("Unable to create file \"" + filePath.toString() + "\" - file already exists!"); - } - /** - * - * @param context - * @param templateName - * @param outfile - * @throws IOException - */ - private void fillTemplate(Map context, String templateName, Path outfilePath) throws IOException { - if (this.verbose) System.out.println("Building file \"" + outfilePath.toString() + "\""); - initDirectory(outfilePath.getParent(), false); - TemplateFormatter.formatTemplate(templateName, context, outfilePath.toFile()); - } - - /** - * Takes a language string and returns a "qualified" form. E.g. "perl", "Perl", "pl", ".pl", should all - * return "perl", "Python", "python", ".py", and "py" should all return Python, etc. - * - * Right now, we support Perl, Python and Java for implementation languages - * @param language - * @return - */ - public static String qualifyLanguage(String language) { - String lang = language.toLowerCase(); - - String[] perlNames = {"perl", ".pl", "pl"}; - if (Arrays.asList(perlNames).contains(lang)) - return "perl"; - - String[] pythonNames = {"python", ".py", "py"}; - if (Arrays.asList(pythonNames).contains(lang)) - return "python"; - - String[] javaNames = {"java", ".java"}; - if (Arrays.asList(javaNames).contains(lang)) - return "java"; - - String[] rNames = {"r", ".r"}; - if (Arrays.asList(rNames).contains(lang)) { - System.out.println( - "************************************************************************\n" + - "WARNING: R support is deprecated and will be removed in a future release\n" + - "************************************************************************"); - return "r"; - } - - // If we get here, then we don't recognize it! throw a runtime exception - throw new RuntimeException("Unrecognized language: " + language + "\n\tWe currently only support Python, Perl, and Java."); - } -} \ No newline at end of file + System.out.println("Done! Your module is available in the " + moduleDir + " directory."); + System.out.println("Compile and run the example methods with the following inputs:"); + System.out.println(" cd " + moduleDir); + System.out.println(" make (required after making changes to " + new File(specFile).getName() + ")"); + System.out.println(" kb-sdk test (will require setting test user account credentials in test_local/test.cfg)"); + System.out.println(); + } + + /** + * + * @param dirPath + * @throws IOException + */ + private void initDirectory(Path dirPath, boolean failOnExist) throws IOException { + if (this.verbose) System.out.println("Making directory \"" + dirPath.toString() + "\""); + File newDir = dirPath.toFile(); + if (!newDir.exists()) { + newDir.mkdirs(); + } + else if (failOnExist) { + throw new IOException("Error while creating module - " + dirPath + " already exists!"); + } + } + + private void initFile(Path filePath, boolean failOnExist) throws IOException { + if (this.verbose) System.out.println("Building empty file \"" + filePath.toString() + "\""); + boolean done = filePath.toFile().createNewFile(); + if (!done && failOnExist) + throw new IOException("Unable to create file \"" + filePath.toString() + "\" - file already exists!"); + } + /** + * + * @param context + * @param templateName + * @param outfile + * @throws IOException + */ + private void fillTemplate(Map context, String templateName, Path outfilePath) throws IOException { + if (this.verbose) System.out.println("Building file \"" + outfilePath.toString() + "\""); + initDirectory(outfilePath.getParent(), false); + TemplateFormatter.formatTemplate(templateName, context, outfilePath.toFile()); + } + + /** + * Takes a language string and returns a "qualified" form. E.g. "perl", "Perl", "pl", ".pl", should all + * return "perl", "Python", "python", ".py", and "py" should all return Python, etc. + * + * Right now, we support Perl, Python and Java for implementation languages + * @param language + * @return + */ + public static String qualifyLanguage(String language) { + String lang = language.toLowerCase(); + + String[] perlNames = {"perl", ".pl", "pl"}; + if (Arrays.asList(perlNames).contains(lang)) + return "perl"; + + String[] pythonNames = {"python", ".py", "py"}; + if (Arrays.asList(pythonNames).contains(lang)) + return "python"; + + String[] javaNames = {"java", ".java"}; + if (Arrays.asList(javaNames).contains(lang)) + return "java"; + + String[] rNames = {"r", ".r"}; + if (Arrays.asList(rNames).contains(lang)) { + System.out.println( + "************************************************************************\n" + + "WARNING: R support is deprecated and will be removed in a future release\n" + + "************************************************************************"); + return "r"; + } + + // If we get here, then we don't recognize it! throw a runtime exception + throw new RuntimeException("Unrecognized language: " + language + "\n\tWe currently only support Python, Perl, and Java."); + } +} diff --git a/src/java/us/kbase/mobu/installer/ClientInstaller.java b/src/java/us/kbase/mobu/installer/ClientInstaller.java index bdb30951..6533e223 100644 --- a/src/java/us/kbase/mobu/installer/ClientInstaller.java +++ b/src/java/us/kbase/mobu/installer/ClientInstaller.java @@ -47,7 +47,7 @@ public class ClientInstaller { private final String language; private final Properties sdkConfig; private final URL catalogUrl; - + public ClientInstaller() throws Exception { this(null, true); } @@ -84,13 +84,13 @@ public ClientInstaller(File dir, boolean showWarnings) throws Exception { catalogUrl = new URL(catalogUrlText); } - public int install(String lang, boolean async, boolean core, boolean dynamic, String tagVer, + public int install(String lang, boolean async, boolean core, boolean dynamic, String tagVer, boolean verbose, String moduleName, String libDirName) throws Exception { return install(lang, async, core, dynamic, tagVer, verbose, moduleName, libDirName, null); } - - public int install(String lang, boolean async, boolean core, boolean dynamic, String tagVer, - boolean verbose, String moduleName, String libDirName, String clientName) + + public int install(String lang, boolean async, boolean core, boolean dynamic, String tagVer, + boolean verbose, String moduleName, String libDirName, String clientName) throws Exception { if (core && (dynamic || async)) { throw new IllegalStateException("It's not allowed to set 'core' mode and one " + @@ -142,7 +142,7 @@ public String loadIncludedSpec(String specFileName) throws Exception { }; } else { throw new IllegalStateException("Path " + moduleName + " is not recognized as " + - "existing local file or URL"); + "existing local file or URL"); } moduleName = null; } else { @@ -159,11 +159,11 @@ public String loadIncludedSpec(String specFileName) throws Exception { } if (modVer.getCompilationReport() == null) throw new IllegalStateException("Compilation report is not found for this " + - "version of [" + moduleName + "] module."); + "version of [" + moduleName + "] module."); List specFiles = modVer.getCompilationReport().getSpecFiles(); if (specFiles == null) throw new IllegalStateException("Compilation report returned from catalog is " + - "out of date. [" + moduleName + "] module should be reregistered again."); + "out of date. [" + moduleName + "] module should be reregistered again."); final List mainSpec = new ArrayList(); final Map deps = new LinkedHashMap(); for (SpecFile spec : specFiles) { @@ -183,22 +183,22 @@ public String loadIncludedSpec(String specFileName) throws Exception { }; } if (async && dynamic) { - compile(lang, async, false, tagVer, verbose, moduleName, libDirName, fp, + compile(lang, async, false, tagVer, verbose, moduleName, libDirName, fp, semanticVersion, gitUrl, gitCommitHash, clientName, filePath); if (clientName == null) { clientName = moduleName; } - compile(lang, false, dynamic, tagVer, verbose, moduleName, libDirName, fp, + compile(lang, false, dynamic, tagVer, verbose, moduleName, libDirName, fp, semanticVersion, gitUrl, gitCommitHash, clientName + "Service", filePath); } else { - compile(lang, async, dynamic, tagVer, verbose, moduleName, libDirName, fp, + compile(lang, async, dynamic, tagVer, verbose, moduleName, libDirName, fp, semanticVersion, gitUrl, gitCommitHash, clientName, filePath); } return 0; } - private void compile(String lang, boolean async, boolean dynamic, String tagVer, - boolean verbose, String moduleName, String libDirName, FileProvider fp, + private void compile(String lang, boolean async, boolean dynamic, String tagVer, + boolean verbose, String moduleName, String libDirName, FileProvider fp, String semanticVersion, String gitUrl, String gitCommitHash, String clientName, String filePath) throws Exception { String url = null; @@ -216,7 +216,7 @@ private void compile(String lang, boolean async, boolean dynamic, String tagVer, final FileProvider fp2 = fp; IncludeProvider ip = new IncludeProvider() { @Override - public Map parseInclude(String includeLine) + public Map parseInclude(String includeLine) throws KidlParseException { String specPath = includeLine.trim(); if (specPath.startsWith("#include")) @@ -285,8 +285,8 @@ public Map parseInclude(String includeLine) String javaPackageParent = "."; String customClientClassName = TextUtils.capitalize(clientName) + "Client"; URL urlEndpoint = url == null ? null : new URL(url); - JavaTypeGenerator.processSpec(services, javaSrcDir, javaPackageParent, false, null, - null, urlEndpoint, null, null, clientAsyncVer, dynservVer, semanticVersion, + JavaTypeGenerator.processSpec(services, javaSrcDir, javaPackageParent, false, null, + null, urlEndpoint, null, null, clientAsyncVer, dynservVer, semanticVersion, gitUrl, gitCommitHash, null, customClientClassName); } else { String perlClientName = null; @@ -302,9 +302,9 @@ public Map parseInclude(String includeLine) if (isJS) jsClientName = "installed_clients/" + clientName + "Client"; FileSaver output = new DiskFileSaver(libDir); - TemplateBasedGenerator.generate(services, url, isJS, jsClientName, isPerl, + TemplateBasedGenerator.generate(services, url, isJS, jsClientName, isPerl, perlClientName, false, null, null, null, isPython, pyClientName, false, null, - null, isR, rClientName, false, null, null, false, ip, output, null, null, + null, isR, rClientName, false, null, null, false, ip, output, null, null, async, clientAsyncVer, dynservVer, semanticVersion, gitUrl, gitCommitHash); } // Now let's add record about this client to dependencies.json file @@ -313,8 +313,8 @@ public Map parseInclude(String includeLine) FileSaver depsDir = new DiskFileSaver(moduleDir); addDependency(moduleName, isSdk, versionTag, filePath, depsDir); } - - public static void addDependency(String moduleName, boolean isSdk, String versionTag, + + public static void addDependency(String moduleName, boolean isSdk, String versionTag, String filePath, FileSaver depsDir) throws Exception { Map depMap = new TreeMap(); ObjectMapper mapper = new ObjectMapper(); @@ -322,13 +322,13 @@ public static void addDependency(String moduleName, boolean isSdk, String versio File depsFile = depsDir.getAsFileOrNull("dependencies.json"); if (depsFile != null && depsFile.exists()) { try { - List deps = mapper.readValue(depsFile, + List deps = mapper.readValue(depsFile, new TypeReference>() {}); for (Dependency dep : deps) { depMap.put(dep.moduleName.toLowerCase(), dep); } } catch (Exception ex) { - throw new IllegalStateException("Error parsing depedencies file [" + depsFile + + throw new IllegalStateException("Error parsing dependencies file [" + depsFile + "]: " + ex.getMessage(), ex); } } @@ -343,7 +343,7 @@ public static void addDependency(String moduleName, boolean isSdk, String versio mapper.writeValue(depsWr, deps); } } - + private static boolean isUrl(String url) { try { new URL(url); @@ -352,7 +352,7 @@ private static boolean isUrl(String url) { return false; } } - + private static boolean isLocalFile(String path) { try { File f = new File(path); @@ -362,7 +362,7 @@ private static boolean isLocalFile(String path) { return false; } } - + private static interface FileProvider { public String loadMainSpec() throws Exception; public String loadIncludedSpec(String specFileName) throws Exception; diff --git a/src/java/us/kbase/mobu/tester/ConfigLoader.java b/src/java/us/kbase/mobu/tester/ConfigLoader.java index fe947478..b9d6a71d 100644 --- a/src/java/us/kbase/mobu/tester/ConfigLoader.java +++ b/src/java/us/kbase/mobu/tester/ConfigLoader.java @@ -29,7 +29,7 @@ public class ConfigLoader { private final String catalogUrl; private final Map secureCfgParams; - public ConfigLoader(Properties props, boolean testMode, + public ConfigLoader(Properties props, boolean testMode, String configPathInfo, boolean tryHomeCfg) throws Exception { if (configPathInfo == null) { configPathInfo = "test_local/test.cfg"; @@ -38,6 +38,11 @@ public ConfigLoader(Properties props, boolean testMode, if (authUrl == null) throw new IllegalStateException("Error: 'auth_service_url' parameter is not set in " + configPathInfo); + endPoint = props.getProperty("kbase_endpoint"); + if (endPoint == null) + throw new IllegalStateException("Error: KBase services end-point is not set in " + + configPathInfo); + String testPrefix = testMode ? "test_" : ""; String user = props.getProperty(testPrefix + "user"); if (user != null && user.trim().isEmpty()) { @@ -48,9 +53,18 @@ public ConfigLoader(Properties props, boolean testMode, if (tokenString != null && tokenString.trim().isEmpty()) { tokenString = null; } + if (tokenString == null) { + System.out.println("No token found in test.cfg file; checking environment"); + String test_token = System.getenv("KBASE_TEST_TOKEN"); + if (test_token != null && !test_token.trim().isEmpty()) { + tokenString = test_token; + System.out.println("Using token from KBASE_TEST_TOKEN env var"); + } + } + if (user == null && tokenString == null) { throw new IllegalStateException("Error: KBase account credentials are not set in " + - configPathInfo); + configPathInfo + " or the KBASE_TEST_TOKEN environment variable"); } authAllowInsecure = props.getProperty("auth_service_url_allow_insecure"); ConfigurableAuthService auth = new ConfigurableAuthService( @@ -60,16 +74,12 @@ public ConfigLoader(Properties props, boolean testMode, token = auth.validateToken(tokenString); } else { if (password == null || password.trim().isEmpty()) { - System.out.println("You haven't preset your password in " +configPathInfo + + System.out.println("You haven't preset your password in " +configPathInfo + " file. Please enter it now."); password = new String(System.console().readPassword("Password: ")); } token = auth.login(user.trim(), password.trim()).getToken(); } - endPoint = props.getProperty("kbase_endpoint"); - if (endPoint == null) - throw new IllegalStateException("Error: KBase services end-point is not set in " + - configPathInfo); jobSrvUrl = getConfigUrl(props, "job_service_url", endPoint, "userandjobstate"); wsUrl = getConfigUrl(props, "workspace_url", endPoint, "ws"); shockUrl = getConfigUrl(props, "shock_url", endPoint, "shock-api"); @@ -86,51 +96,51 @@ public ConfigLoader(Properties props, boolean testMode, } } } - + public String getAuthUrl() { return authUrl; } - + public String getAuthAllowInsecure() { return authAllowInsecure; } - + public AuthToken getToken() { return token; } - + public String getEndPoint() { return endPoint; } - + public String getCatalogUrl() { return catalogUrl; } - + public String getHandleUrl() { return handleUrl; } - + public String getJobSrvUrl() { return jobSrvUrl; } - + public String getNjswUrl() { return njswUrl; } - + public String getShockUrl() { return shockUrl; } - + public String getSrvWizUrl() { return srvWizUrl; } - + public String getWsUrl() { return wsUrl; } - + public void generateConfigProperties(File configPropsFile) throws Exception { PrintWriter pw = new PrintWriter(configPropsFile); try { @@ -143,7 +153,7 @@ public void generateConfigProperties(File configPropsFile) throws Exception { pw.println("srv_wiz_url = " + srvWizUrl); pw.println("njsw_url = " + njswUrl); pw.println("auth_service_url = " + authUrl); - pw.println("auth_service_url_allow_insecure = " + + pw.println("auth_service_url_allow_insecure = " + (authAllowInsecure == null ? "false" : authAllowInsecure)); for (String param : secureCfgParams.keySet()) { pw.println(param + " = " + secureCfgParams.get(param)); @@ -152,7 +162,7 @@ public void generateConfigProperties(File configPropsFile) throws Exception { pw.close(); } } - + public CallbackServerConfig buildCallbackServerConfig( URL callbackUrl, Path workDir, LineLogger logger) throws Exception { return new CallbackServerConfigBuilder( @@ -160,8 +170,8 @@ public CallbackServerConfig buildCallbackServerConfig( new URL(handleUrl), new URL(srvWizUrl), new URL(njswUrl), new URL(authUrl), authAllowInsecure, new URL(catalogUrl), callbackUrl, workDir, null, logger).build(); } - - private static String getConfigUrl(Properties props, String key, String endPoint, + + private static String getConfigUrl(Properties props, String key, String endPoint, String defaultUrlSuffix) { String ret = props.getProperty(key); return ret == null ? (endPoint + "/" + defaultUrlSuffix) : ret; diff --git a/src/java/us/kbase/mobu/tester/ModuleTester.java b/src/java/us/kbase/mobu/tester/ModuleTester.java index 7600b8d0..289a7693 100644 --- a/src/java/us/kbase/mobu/tester/ModuleTester.java +++ b/src/java/us/kbase/mobu/tester/ModuleTester.java @@ -53,7 +53,7 @@ public class ModuleTester { public ModuleTester() throws Exception { this(null); } - + public ModuleTester(File dir) throws Exception { moduleDir = dir == null ? DirUtils.findModuleDir() : DirUtils.findModuleDir(dir); String kbaseYml = TextUtils.readFileText(new File(moduleDir, "kbase.yml")); @@ -70,7 +70,7 @@ public ModuleTester(File dir) throws Exception { ModuleInitializer.qualifyLanguage((String) kbaseYmlConfig.get("service-language")); moduleContext.put("os_name", System.getProperty("os.name")); } - + private static void checkIgnoreLine(File f, String line) throws IOException { List lines = new ArrayList(); if (f.exists()) @@ -82,18 +82,18 @@ private static void checkIgnoreLine(File f, String line) throws IOException { FileUtils.writeLines(f, lines); } } - + public int runTests(String methodStoreUrl, boolean skipValidation, boolean allowSyncMethods) throws Exception { if (skipValidation) { System.out.println("Validation step is skipped"); } else { - ModuleValidator mv = new ModuleValidator(Arrays.asList(moduleDir.getCanonicalPath()), + ModuleValidator mv = new ModuleValidator(Arrays.asList(moduleDir.getCanonicalPath()), false, methodStoreUrl, allowSyncMethods); int returnCode = mv.validateAll(); if (returnCode!=0) { System.out.println("You can skip validation step using -s (or --skip_validation)" + - " flag"); + " flag"); System.exit(returnCode); } } @@ -114,7 +114,7 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow if (kbaseYmlConfig.get("data-version") != null) { File refDataDir = new File(tlDir, "refdata"); if (!refDataDir.exists()) { - TemplateFormatter.formatTemplate("module_run_tests", moduleContext, + TemplateFormatter.formatTemplate("module_run_tests", moduleContext, runTestsSh); refDataDir.mkdir(); } @@ -127,9 +127,6 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow TemplateFormatter.formatTemplate("module_run_docker", moduleContext, runDockerSh); if (!testCfg.exists()) { TemplateFormatter.formatTemplate("module_test_cfg", moduleContext, testCfg); - System.out.println("Set KBase account credentials in test_local/test.cfg and then " + - "test again"); - return 1; } Properties props = new Properties(); InputStream is = new FileInputStream(testCfg); @@ -138,10 +135,8 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow } finally { is.close(); } - + ConfigLoader cfgLoader = new ConfigLoader(props, true, "test_local/test.cfg", true); - - File workDir = new File(tlDir, "workdir"); workDir.mkdir(); File tokenFile = new File(workDir, "token"); @@ -175,7 +170,7 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow String callbackNetworksText = props.getProperty("callback_networks"); if (callbackNetworksText != null) { callbackNetworks = callbackNetworksText.trim().split("\\s*,\\s*"); - System.out.println("Custom network instarface list is defined: " + + System.out.println("Custom network instarface list is defined: " + Arrays.asList(callbackNetworks)); } URL callbackUrl = CallbackServer.getCallbackUrl(callbackPort, callbackNetworks); @@ -185,7 +180,7 @@ public int runTests(String methodStoreUrl, boolean skipValidation, boolean allow JsonServerSyslog.setStaticUseSyslog(false); JsonServerSyslog.setStaticMlogFile("callback.log"); } - CallbackServerConfig cfg = cfgLoader.buildCallbackServerConfig(callbackUrl, + CallbackServerConfig cfg = cfgLoader.buildCallbackServerConfig(callbackUrl, tlDir.toPath(), new LineLogger() { @Override public void logNextLine(String line, boolean isError) { @@ -217,7 +212,7 @@ public void logNextLine(String line, boolean isError) { } else { if (callbackNetworks != null && callbackNetworks.length > 0) { throw new IllegalStateException("No proper callback IP was found, " + - "please check callback_networks parameter in test.cfg"); + "please check callback_networks parameter in test.cfg"); } System.out.println("WARNING: No callback URL was received " + "by the job runner. Local callbacks are disabled."); @@ -226,9 +221,13 @@ public void logNextLine(String line, boolean isError) { try { System.out.println(); ProcessHelper.cmd("chmod", "+x", runTestsSh.getCanonicalPath()).exec(tlDir); - int exitCode = ProcessHelper.cmd("bash", DirUtils.getFilePath(runTestsSh), - callbackUrl == null ? "http://fakecallbackurl" : - callbackUrl.toExternalForm()).exec(tlDir).getExitCode(); + int exitCode = ProcessHelper.cmd( + "bash", + DirUtils.getFilePath(runTestsSh), + callbackUrl == null + ? "http://fakecallbackurl" + : callbackUrl.toExternalForm() + ).exec(tlDir).getExitCode(); return exitCode; } finally { if (jettyServer != null) { @@ -251,7 +250,7 @@ public static boolean buildNewDockerImageWithCleanup(File moduleDir, File tlDir, ProcessHelper.cmd("bash", runDockerPath, "rm", "-v", "-f", cntId).exec(tlDir); } } - String oldImageId = findImageIdByName(tlDir, imageName, runDockerSh); + String oldImageId = findImageIdByName(tlDir, imageName, runDockerSh); System.out.println(); System.out.println("Build Docker image"); boolean ok = buildImage(moduleDir, imageName, runDockerSh); @@ -267,7 +266,7 @@ public static boolean buildNewDockerImageWithCleanup(File moduleDir, File tlDir, } return true; } - + public static String findImageIdByName(File tlDir, String imageName, File runDockerSh) throws Exception { List lines; @@ -295,7 +294,7 @@ public static String[] splitByWhiteSpaces(String line) { String[] parts = line.split("\\s+"); return parts; } - + private static List exec(File workDir, String... cmd) throws Exception { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); @@ -312,13 +311,13 @@ private static List exec(File workDir, String... cmd) throws Exception { br.close(); return ret; } - - public static boolean buildImage(File repoDir, String targetImageName, + + public static boolean buildImage(File repoDir, String targetImageName, File runDockerSh) throws Exception { String scriptPath = DirUtils.getFilePath(runDockerSh); String repoPath = DirUtils.getFilePath(repoDir); - Process p = Runtime.getRuntime().exec(new String[] {"bash", - scriptPath, "build", "--rm", "-t", + Process p = Runtime.getRuntime().exec(new String[] {"bash", + scriptPath, "build", "--rm", "-t", targetImageName, repoPath}); List workers = new ArrayList(); InputStream[] inputStreams = new InputStream[] {p.getInputStream(), p.getErrorStream()}; @@ -358,7 +357,7 @@ public void run() { } catch (Exception e) { e.printStackTrace(); throw new IllegalStateException("Error reading data from executed " + - "container", e); + "container", e); } } }); @@ -374,7 +373,7 @@ public void run() { if (cntIdToDelete[0] != null) { System.out.println("Cleaning up building container: " + cntIdToDelete[0]); Thread.sleep(1000); - ProcessHelper.cmd("bash", scriptPath, + ProcessHelper.cmd("bash", scriptPath, "rm", "-v", "-f", cntIdToDelete[0]).exec(repoDir); } } catch (Exception ex) { diff --git a/src/java/us/kbase/mobu/tester/test/ModuleTesterTest.java b/src/java/us/kbase/mobu/tester/test/ModuleTesterTest.java index ae1f2488..3cbd4ee9 100644 --- a/src/java/us/kbase/mobu/tester/test/ModuleTesterTest.java +++ b/src/java/us/kbase/mobu/tester/test/ModuleTesterTest.java @@ -21,44 +21,44 @@ public class ModuleTesterTest { - private static final String SIMPLE_MODULE_NAME = "ASimpleModule_for_unit_testing"; - private static final boolean cleanupAfterTests = true; - - private static List createdModuleNames = new ArrayList(); - private static AuthToken token; - + private static final String SIMPLE_MODULE_NAME = "ASimpleModule_for_unit_testing"; + private static final boolean cleanupAfterTests = true; + + private static List createdModuleNames = new ArrayList(); + private static AuthToken token; + @BeforeClass public static void beforeClass() throws Exception { token = TestConfigHelper.getToken(); } - @AfterClass - public static void tearDownModule() throws Exception { - if (cleanupAfterTests) - for (String moduleName : createdModuleNames) - try { - deleteDir(moduleName); - } catch (Exception ex) { - System.err.println("Error cleaning up module [" + - moduleName + "]: " + ex.getMessage()); - } - } - - @After - public void afterText() { - System.out.println(); - } - - private static void deleteDir(String moduleName) throws Exception { - File module = new File(moduleName); - if (module.exists() && module.isDirectory()) - FileUtils.deleteDirectory(module); - } - + @AfterClass + public static void tearDownModule() throws Exception { + if (cleanupAfterTests) + for (String moduleName : createdModuleNames) + try { + deleteDir(moduleName); + } catch (Exception ex) { + System.err.println("Error cleaning up module [" + + moduleName + "]: " + ex.getMessage()); + } + } + + @After + public void afterText() { + System.out.println(); + } + + private static void deleteDir(String moduleName) throws Exception { + File module = new File(moduleName); + if (module.exists() && module.isDirectory()) + FileUtils.deleteDirectory(module); + } + private void init(String lang, String moduleName) throws Exception { deleteDir(moduleName); createdModuleNames.add(moduleName); - ModuleInitializer initer = new ModuleInitializer(moduleName, token.getUserName(), + ModuleInitializer initer = new ModuleInitializer(moduleName, token.getUserName(), lang, false); initer.initialize(true); } @@ -68,13 +68,13 @@ private int runTestsInDocker(String moduleName) throws Exception { return runTestsInDocker(moduleDir, token); } - public static int runTestsInDocker(File moduleDir, AuthToken token) throws Exception { - return runTestsInDocker(moduleDir, token, false); - } - + public static int runTestsInDocker(File moduleDir, AuthToken token) throws Exception { + return runTestsInDocker(moduleDir, token, false); + } + public static int runTestsInDocker( final File moduleDir, - final AuthToken token, + final AuthToken token, final boolean skipValidation) throws Exception { DockerClientServerTester.correctDockerfile(moduleDir); @@ -83,7 +83,7 @@ public static int runTestsInDocker( "test_token=" + token.getToken() + "\n" + "kbase_endpoint=" + TestConfigHelper.getKBaseEndpoint() + "\n" + "auth_service_url=" + TestConfigHelper.getAuthServiceUrl() + "\n" + - "auth_service_url_allow_insecure=" + + "auth_service_url_allow_insecure=" + TestConfigHelper.getAuthServiceUrlInsecure() + "\n"; FileUtils.writeStringToFile(testCfgFile, testCfgText); int exitCode = new ModuleTester(moduleDir).runTests(ModuleBuilder.DEFAULT_METHOD_STORE_URL, @@ -92,84 +92,84 @@ public static int runTestsInDocker( return exitCode; } - @Test - public void testPerlModuleExample() throws Exception { - System.out.println("Test [testPerlModuleExample]"); - String lang = "perl"; - String moduleName = SIMPLE_MODULE_NAME + "Perl"; - init(lang, moduleName); - int exitCode = runTestsInDocker(moduleName); - Assert.assertEquals(0, exitCode); - } - - @Test - public void testPerlModuleError() throws Exception { - System.out.println("Test [testPerlModuleError]"); - String lang = "perl"; - String moduleName = SIMPLE_MODULE_NAME + "PerlError"; - init(lang, moduleName); - File implFile = new File(moduleName + "/lib/" + moduleName + "/" + moduleName + "Impl.pm"); - String implText = FileUtils.readFileToString(implFile); - implText = implText.replace(" #BEGIN filter_contigs", - " #BEGIN filter_contigs\n" + - " die \"Special error\";"); - FileUtils.writeStringToFile(implFile, implText); - int exitCode = runTestsInDocker(moduleName); - Assert.assertEquals(2, exitCode); - } - - @Test - public void testPythonModuleExample() throws Exception { - System.out.println("Test [testPythonModuleExample]"); - String lang = "python"; - String moduleName = SIMPLE_MODULE_NAME + "Python"; - init(lang, moduleName); - int exitCode = runTestsInDocker(moduleName); + @Test + public void testPerlModuleExample() throws Exception { + System.out.println("Test [testPerlModuleExample]"); + String lang = "perl"; + String moduleName = SIMPLE_MODULE_NAME + "Perl"; + init(lang, moduleName); + int exitCode = runTestsInDocker(moduleName); Assert.assertEquals(0, exitCode); - } + } - @Test - public void testPythonModuleError() throws Exception { + @Test + public void testPerlModuleError() throws Exception { + System.out.println("Test [testPerlModuleError]"); + String lang = "perl"; + String moduleName = SIMPLE_MODULE_NAME + "PerlError"; + init(lang, moduleName); + File implFile = new File(moduleName + "/lib/" + moduleName + "/" + moduleName + "Impl.pm"); + String implText = FileUtils.readFileToString(implFile); + implText = implText.replace(" #BEGIN filter_contigs", + " #BEGIN filter_contigs\n" + + " die \"Special error\";"); + FileUtils.writeStringToFile(implFile, implText); + int exitCode = runTestsInDocker(moduleName); + Assert.assertEquals(2, exitCode); + } + + @Test + public void testPythonModuleExample() throws Exception { + System.out.println("Test [testPythonModuleExample]"); + String lang = "python"; + String moduleName = SIMPLE_MODULE_NAME + "Python"; + init(lang, moduleName); + int exitCode = runTestsInDocker(moduleName); + Assert.assertEquals(0, exitCode); + } + + @Test + public void testPythonModuleError() throws Exception { System.out.println("Test [testPythonModuleError]"); - String lang = "python"; + String lang = "python"; String moduleName = SIMPLE_MODULE_NAME + "PythonError"; init(lang, moduleName); File implFile = new File(moduleName + "/lib/" + moduleName + "/" + moduleName + "Impl.py"); String implText = FileUtils.readFileToString(implFile); - implText = implText.replace(" #BEGIN filter_contigs", + implText = implText.replace(" #BEGIN filter_contigs", " #BEGIN filter_contigs\n" + " raise ValueError('Special error')"); FileUtils.writeStringToFile(implFile, implText); int exitCode = runTestsInDocker(moduleName); Assert.assertEquals(2, exitCode); - } + } - @Test - public void testJavaModuleExample() throws Exception { + @Test + public void testJavaModuleExample() throws Exception { System.out.println("Test [testJavaModuleExample]"); - String lang = "java"; + String lang = "java"; String moduleName = SIMPLE_MODULE_NAME + "Java"; init(lang, moduleName); int exitCode = runTestsInDocker(moduleName); Assert.assertEquals(0, exitCode); - } - - @Test - public void testJavaModuleError() throws Exception { - System.out.println("Test [testJavaModuleError]"); - String lang = "java"; - String moduleName = SIMPLE_MODULE_NAME + "JavaError"; - init(lang, moduleName); + } + + @Test + public void testJavaModuleError() throws Exception { + System.out.println("Test [testJavaModuleError]"); + String lang = "java"; + String moduleName = SIMPLE_MODULE_NAME + "JavaError"; + init(lang, moduleName); File implFile = new File(moduleName + "/lib/src/" + - "asimplemoduleforunittestingjavaerror/ASimpleModuleForUnitTestingJavaErrorServer.java"); + "asimplemoduleforunittestingjavaerror/ASimpleModuleForUnitTestingJavaErrorServer.java"); String implText = FileUtils.readFileToString(implFile); - implText = implText.replace(" //BEGIN filter_contigs", + implText = implText.replace(" //BEGIN filter_contigs", " //BEGIN filter_contigs\n" + " if (true) throw new IllegalStateException(\"Special error\");"); FileUtils.writeStringToFile(implFile, implText); - int exitCode = runTestsInDocker(moduleName); - Assert.assertEquals(2, exitCode); - } + int exitCode = runTestsInDocker(moduleName); + Assert.assertEquals(2, exitCode); + } @Test public void testRModuleError() throws Exception { @@ -179,14 +179,14 @@ public void testRModuleError() throws Exception { init(lang, moduleName); File implFile = new File(moduleName + "/lib/" + moduleName + "/" + moduleName + "Impl.r"); String implText = FileUtils.readFileToString(implFile); - implText = implText.replace(" #BEGIN count_contigs", + implText = implText.replace(" #BEGIN count_contigs", " #BEGIN count_contigs\n" + " stop(\"Special error\")"); FileUtils.writeStringToFile(implFile, implText); int exitCode = runTestsInDocker(moduleName); Assert.assertEquals(2, exitCode); } - + @Test public void testSelfCalls() throws Exception { System.out.println("Test [testSelfCalls]"); @@ -214,17 +214,17 @@ public void testSelfCalls() throws Exception { " returnVal = input * input\n" + " #END calc_square\n"; File moduleDir = new File(moduleName); - File implFile = new File(moduleDir, "lib/" + moduleName + "/" + + File implFile = new File(moduleDir, "lib/" + moduleName + "/" + moduleName + "Impl.py"); ModuleInitializer initer = new ModuleInitializer(moduleName, token.getUserName(), lang, false); initer.initialize(false); File specFile = new File(moduleDir, moduleName + ".spec"); - String specText = FileUtils.readFileToString(specFile).replace("};", + String specText = FileUtils.readFileToString(specFile).replace("};", "funcdef run_local(int input) returns (int) authentication required;\n" + "funcdef calc_square(int input) returns (int) authentication required;\n" + "};"); File testFile = new File(moduleDir, "test/" + moduleName + "_server_test.py"); - String testCode = FileUtils.readFileToString(testFile).replace(" def test_your_method(self):", + String testCode = FileUtils.readFileToString(testFile).replace(" def test_your_method(self):", " def test_your_method(self):\n" + " self.assertEqual(25, self.getImpl().run_local(self.getContext(), 5)[0])" ); @@ -234,4 +234,4 @@ public void testSelfCalls() throws Exception { int exitCode = runTestsInDocker(moduleDir, token, true); Assert.assertEquals(0, exitCode); } -} \ No newline at end of file +} diff --git a/src/java/us/kbase/mobu/util/ProcessHelper.java b/src/java/us/kbase/mobu/util/ProcessHelper.java index c28e75f5..0f102886 100644 --- a/src/java/us/kbase/mobu/util/ProcessHelper.java +++ b/src/java/us/kbase/mobu/util/ProcessHelper.java @@ -61,7 +61,7 @@ public static ProcessHelper exec(CommandHolder cmd, File workDir, BufferedReader } public static ProcessHelper exec(CommandHolder cmd, File workDir, BufferedReader input, boolean saveOutput, boolean saveErrors, boolean waitFor) throws IOException { - return new ProcessHelper(cmd, workDir, saveOutput ? OutType.StringOut : OutType.SystemOut, + return new ProcessHelper(cmd, workDir, saveOutput ? OutType.StringOut : OutType.SystemOut, saveErrors ? OutType.StringErr : OutType.SystemErr, input, null, null, waitFor); } @@ -75,8 +75,9 @@ public ProcessHelper(CommandHolder cmd, File workDir, OutType outType, OutType e outPw = output; if (error != null) errPw = error; - process = cmd.cmdLine != null ? Runtime.getRuntime().exec(cmd.cmdLine, null, workDir) : - Runtime.getRuntime().exec(cmd.cmdParts, null, workDir); + process = cmd.cmdLine != null + ? Runtime.getRuntime().exec(cmd.cmdLine, null, workDir) + : Runtime.getRuntime().exec(cmd.cmdParts, null, workDir); Thread outTh = readInNewThread(process.getInputStream(), outType); Thread errTh = readInNewThread(process.getErrorStream(), errType); if (input != null) { @@ -108,9 +109,9 @@ public ProcessHelper(CommandHolder cmd, File workDir, OutType outType, OutType e } public Process getProcess() { - return process; - } - + return process; + } + private Thread readInNewThread(final InputStream is, final OutType outType) { Thread ret = new Thread(new Runnable() { public void run() { @@ -189,10 +190,10 @@ public CommandHolder add(String... newCmdParts) { } public CommandHolder dontWaitFor() { - waitFor = false; - return this; + waitFor = false; + return this; } - + public ProcessHelper exec(File workDir) throws IOException { return ProcessHelper.exec(this, workDir, waitFor); } diff --git a/src/java/us/kbase/mobu/validator/ModuleValidator.java b/src/java/us/kbase/mobu/validator/ModuleValidator.java index 9363d4b0..2f1a3d75 100644 --- a/src/java/us/kbase/mobu/validator/ModuleValidator.java +++ b/src/java/us/kbase/mobu/validator/ModuleValidator.java @@ -39,44 +39,44 @@ public class ModuleValidator { - - - private static final String KBASE_YML_FILE = "kbase.yml"; - - - protected List modulePaths; - protected boolean verbose; - protected String methodStoreUrl; - protected boolean allowSyncMethods; - - public ModuleValidator(List modulePaths, boolean verbose, - String defaultMethodStoreUrl, boolean allowSyncMethods) throws Exception { - this.modulePaths = modulePaths; - this.verbose = verbose; - if (modulePaths.size() == 1) { - File module = new File(modulePaths.get(0)); - File testCfg = new File(new File(module, "test_local"), "test.cfg"); - if (testCfg.exists()) { - Properties props = new Properties(); - InputStream is = new FileInputStream(testCfg); - try { - props.load(is); - } finally { - is.close(); - } - String endPoint = props.getProperty("kbase_endpoint"); - if (endPoint != null) - this.methodStoreUrl = endPoint + "/narrative_method_store/rpc"; - } - } - if (this.methodStoreUrl == null) { - this.methodStoreUrl = defaultMethodStoreUrl; - System.out.println("WARNING! 'kbase_endpoint' property was not found in " + - "/test_local/test.cfg so validation is done against NMS in appdev"); - } - this.allowSyncMethods = allowSyncMethods; - } - + + + private static final String KBASE_YML_FILE = "kbase.yml"; + + + protected List modulePaths; + protected boolean verbose; + protected String methodStoreUrl; + protected boolean allowSyncMethods; + + public ModuleValidator(List modulePaths, boolean verbose, + String defaultMethodStoreUrl, boolean allowSyncMethods) throws Exception { + this.modulePaths = modulePaths; + this.verbose = verbose; + if (modulePaths.size() == 1) { + File module = new File(modulePaths.get(0)); + File testCfg = new File(new File(module, "test_local"), "test.cfg"); + if (testCfg.exists()) { + Properties props = new Properties(); + InputStream is = new FileInputStream(testCfg); + try { + props.load(is); + } finally { + is.close(); + } + String endPoint = props.getProperty("kbase_endpoint"); + if (endPoint != null) + this.methodStoreUrl = endPoint + "/narrative_method_store/rpc"; + } + } + if (this.methodStoreUrl == null) { + this.methodStoreUrl = defaultMethodStoreUrl; + System.out.println("WARNING! 'kbase_endpoint' property was not found in " + + "/test_local/test.cfg so validation is done against NMS in appdev"); + } + this.allowSyncMethods = allowSyncMethods; + } + private static boolean isModuleDir(File dir) { return new File(dir, "Dockerfile").exists() && new File(dir, "Makefile").exists() && @@ -86,52 +86,52 @@ private static boolean isModuleDir(File dir) { new File(dir, "test").exists() && new File(dir, "ui").exists(); } - - public int validateAll() { - - int errors = 0; - - for(String modulePathString : modulePaths) { - File module = new File(modulePathString); - System.out.println("\nValidating module in ("+module+")"); - - if(!module.exists()) { - System.err.println(" **ERROR** - the module does not exist"); - errors++; continue; - } - if(!module.isDirectory()) { - System.err.println(" **ERROR** - the module location is not a directory."); - errors++; continue; - } - - try { - if(verbose) System.out.println(" - canonical path = "+module.getCanonicalPath()+""); - File dir = module.getCanonicalFile(); - while (!isModuleDir(dir)) { - dir = dir.getParentFile(); - if (dir == null) - throw new IllegalStateException(" **ERROR** - cannot find folder with module structure"); - } - module = dir; - } catch (IOException e) { - System.err.println(" **ERROR** - unable to extract module canonical path:"); - System.err.println(" "+e.getMessage()); - } - - - // 1) Validate the configuration file - try { - int status = validateKBaseYmlConfig(module); - if(status!=0) { - errors++; continue; - } - } catch (Exception e) { - System.err.println(" **ERROR** - configuration file validation failed:"); - System.err.println(" "+e.getMessage()); - errors++; continue; - } - - KbModule parsedKidl = null; + + public int validateAll() { + + int errors = 0; + + for(String modulePathString : modulePaths) { + File module = new File(modulePathString); + System.out.println("\nValidating module in ("+module+")"); + + if(!module.exists()) { + System.err.println(" **ERROR** - the module does not exist"); + errors++; continue; + } + if(!module.isDirectory()) { + System.err.println(" **ERROR** - the module location is not a directory."); + errors++; continue; + } + + try { + if(verbose) System.out.println(" - canonical path = "+module.getCanonicalPath()+""); + File dir = module.getCanonicalFile(); + while (!isModuleDir(dir)) { + dir = dir.getParentFile(); + if (dir == null) + throw new IllegalStateException(" **ERROR** - cannot find folder with module structure"); + } + module = dir; + } catch (IOException e) { + System.err.println(" **ERROR** - unable to extract module canonical path:"); + System.err.println(" "+e.getMessage()); + } + + + // 1) Validate the configuration file + try { + int status = validateKBaseYmlConfig(module); + if(status!=0) { + errors++; continue; + } + } catch (Exception e) { + System.err.println(" **ERROR** - configuration file validation failed:"); + System.err.println(" "+e.getMessage()); + errors++; continue; + } + + KbModule parsedKidl = null; try { Map config = parseKBaseYaml(new File(module, KBASE_YML_FILE)); String moduleName = (String)config.get("module-name"); @@ -153,78 +153,78 @@ public int validateAll() { errors++; continue; } - // 2) Validate UI components - - // 2a) Validate Narrative Methods - - File uiNarrativeMethodsDir = new File(new File(new File(module, "ui"), "narrative"), "methods"); - if (uiNarrativeMethodsDir.exists()) { - for (File methodDir : uiNarrativeMethodsDir.listFiles()) { - if (methodDir.isDirectory()) { - System.out.println("\nValidating method in ("+methodDir+")"); - try { - int status = validateMethodSpec(methodDir, parsedKidl, this.allowSyncMethods); - if (status != 0) { - errors++; - continue; - } - } catch (Exception e) { - System.err.println(" **ERROR** - method-spec validation failed:"); - System.err.println(" "+e.getMessage()); - errors++; continue; - } - } - } - } - - } - - - - - if(errors>0) { - if(modulePaths.size()==1) { - System.out.println("\n\nThis module contains errors.\n"); - } else { - System.out.println("\n\nErrors detected in "+errors +" of "+modulePaths.size()+" modules.\n"); - } - return 1; - } - if(modulePaths.size()==1) { - System.out.println("\n\nCongrats- this module is valid.\n"); - } else { - System.out.println("\n\nCongrats- all "+modulePaths.size()+" modules are valid.\n"); - } - return 0; - } - - protected int validateKBaseYmlConfig(File module) throws IOException { - File kbaseYmlFile = new File(module.getCanonicalPath()+File.separator+KBASE_YML_FILE); - if(verbose) System.out.println(" - configuration file = "+kbaseYmlFile); - - if(!kbaseYmlFile.exists()) { - System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file does not exist in module directory."); - return 1; - } - if(kbaseYmlFile.isDirectory()) { - System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file location is a directory, not a file."); - return 1; - } - - try { - parseKBaseYaml(kbaseYmlFile); - if(verbose) System.out.println(" - configuration file is valid YAML"); - } catch(Exception e) { - System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file location is invalid:"); - System.err.println(" "+e.getMessage()); - return 1; - - } - - - - return 0; - } + // 2) Validate UI components + + // 2a) Validate Narrative Methods + + File uiNarrativeMethodsDir = new File(new File(new File(module, "ui"), "narrative"), "methods"); + if (uiNarrativeMethodsDir.exists()) { + for (File methodDir : uiNarrativeMethodsDir.listFiles()) { + if (methodDir.isDirectory()) { + System.out.println("\nValidating method in ("+methodDir+")"); + try { + int status = validateMethodSpec(methodDir, parsedKidl, this.allowSyncMethods); + if (status != 0) { + errors++; + continue; + } + } catch (Exception e) { + System.err.println(" **ERROR** - method-spec validation failed:"); + System.err.println(" "+e.getMessage()); + errors++; continue; + } + } + } + } + + } + + + + + if(errors>0) { + if(modulePaths.size()==1) { + System.out.println("\n\nThis module contains errors.\n"); + } else { + System.out.println("\n\nErrors detected in "+errors +" of "+modulePaths.size()+" modules.\n"); + } + return 1; + } + if(modulePaths.size()==1) { + System.out.println("\n\nCongrats- this module is valid.\n"); + } else { + System.out.println("\n\nCongrats- all "+modulePaths.size()+" modules are valid.\n"); + } + return 0; + } + + protected int validateKBaseYmlConfig(File module) throws IOException { + File kbaseYmlFile = new File(module.getCanonicalPath()+File.separator+KBASE_YML_FILE); + if(verbose) System.out.println(" - configuration file = "+kbaseYmlFile); + + if(!kbaseYmlFile.exists()) { + System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file does not exist in module directory."); + return 1; + } + if(kbaseYmlFile.isDirectory()) { + System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file location is a directory, not a file."); + return 1; + } + + try { + parseKBaseYaml(kbaseYmlFile); + if(verbose) System.out.println(" - configuration file is valid YAML"); + } catch(Exception e) { + System.err.println(" **ERROR** - "+KBASE_YML_FILE+" configuration file location is invalid:"); + System.err.println(" "+e.getMessage()); + return 1; + + } + + + + return 0; + } @SuppressWarnings("unchecked") public Map parseKBaseYaml(File kbaseYmlFile) throws IOException { @@ -232,14 +232,14 @@ public Map parseKBaseYaml(File kbaseYmlFile) throws IOException { String kbaseYml = TextUtils.readFileText(kbaseYmlFile); return (Map) yaml.load(kbaseYml); } - - - protected int validateMethodSpec(File methodDir, KbModule parsedKidl, - boolean allowSyncMethods) throws IOException { - NarrativeMethodStoreClient nms = new NarrativeMethodStoreClient(new URL(methodStoreUrl)); - nms.setAllSSLCertificatesTrusted(true); - nms.setIsInsecureHttpConnectionAllowed(true); - String spec = FileUtils.readFileToString(new File(methodDir, "spec.json")); + + + protected int validateMethodSpec(File methodDir, KbModule parsedKidl, + boolean allowSyncMethods) throws IOException { + NarrativeMethodStoreClient nms = new NarrativeMethodStoreClient(new URL(methodStoreUrl)); + nms.setAllSSLCertificatesTrusted(true); + nms.setIsInsecureHttpConnectionAllowed(true); + String spec = FileUtils.readFileToString(new File(methodDir, "spec.json")); String display = FileUtils.readFileToString(new File(methodDir, "display.yaml")); Map extraFiles = new LinkedHashMap(); for (File f : methodDir.listFiles()) { @@ -272,11 +272,11 @@ protected int validateMethodSpec(File methodDir, KbModule parsedKidl, System.err.println(" "+e.getMessage()); return 1; } - } + } - public static void validateMethodSpecMapping(String specText, KbModule parsedKidl, - boolean allowSyncMethods) throws IOException { - JsonNode spec = new ObjectMapper().readTree(specText); + public static void validateMethodSpecMapping(String specText, KbModule parsedKidl, + boolean allowSyncMethods) throws IOException { + JsonNode spec = new ObjectMapper().readTree(specText); JsonNode behaviorNode = get("/", spec, "behavior"); if (behaviorNode.get("none") != null) return; // Don't pay attention at viewer methods (since they don't use back-end) @@ -336,7 +336,7 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid } } if (func == null) { - throw new IllegalStateException(" **ERROR** - unknown method \"" + + throw new IllegalStateException(" **ERROR** - unknown method \"" + methodName + "\" defined within path " + "[behavior/service-mapping/method] in spec.json"); } @@ -361,9 +361,9 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid targetArgPos = targetArgPosNode.asInt(); if (targetArgPos >= func.getParameters().size()) { throw new IllegalStateException(" **ERROR** - value " + targetArgPos + " within " + - "path [" + path + "/target_argument_position] in spec.json is out of " + - "bounds (number of arguments defined for function \"" + methodName + "\" " + - "is " + func.getParameters().size() + ")"); + "path [" + path + "/target_argument_position] in spec.json is out of " + + "bounds (number of arguments defined for function \"" + methodName + "\" " + + "is " + func.getParameters().size() + ")"); } argsUsed.add(targetArgPos); KbType argType = func.getParameters().get(targetArgPos).getType(); @@ -378,7 +378,7 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid argType instanceof KbTuple) { throw new IllegalStateException(" **ERROR** - value " + targetProp + " within " + "path [" + path + "/target_property] in spec.json can't be applied to " + - "type " + argType.getClass().getSimpleName() + " (defined for argument " + + "type " + argType.getClass().getSimpleName() + " (defined for argument " + targetArgPos + ")"); } if (argType instanceof KbStruct) { @@ -392,8 +392,8 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid } if (!found) { System.err.println(" **WARNINGS** - value \"" + targetProp + "\" within " + - "path [" + path + "/target_property] in spec.json doesn't match " + - "any field of structure defined as argument type" + + "path [" + path + "/target_property] in spec.json doesn't match " + + "any field of structure defined as argument type" + (struct.getName() != null ? (" (" + struct.getName() + ")") : "")); } } @@ -411,22 +411,22 @@ public static void validateMethodSpecMapping(String specText, KbModule parsedKid } if (func.getParameters().size() != argsUsed.size()) { throw new IllegalStateException(" **ERROR** - not all arguments are set for function " + - "\"" + func.getName() + "\", list of defined arguments is: " + argsUsed); + "\"" + func.getName() + "\", list of defined arguments is: " + argsUsed); } if (inputParamIdToType.size() != paramsUsed.size()) { Set paramsNotUsed = new TreeSet(inputParamIdToType.keySet()); paramsNotUsed.removeAll(paramsUsed); - System.err.println(" **WARNINGS** - some of input parameters are not used: " + + System.err.println(" **WARNINGS** - some of input parameters are not used: " + paramsNotUsed); } - } - - private static JsonNode get(String nodePath, JsonNode node, String childName) { - JsonNode ret = node.get(childName); - if (ret == null) - throw new IllegalStateException(" **ERROR** - can't find sub-node [" + childName + - "] within path [" + nodePath + "] in spec.json"); - return ret; - } + } + + private static JsonNode get(String nodePath, JsonNode node, String childName) { + JsonNode ret = node.get(childName); + if (ret == null) + throw new IllegalStateException(" **ERROR** - can't find sub-node [" + childName + + "] within path [" + nodePath + "] in spec.json"); + return ret; + } } diff --git a/src/java/us/kbase/scripts/test/Test8.perl.properties b/src/java/us/kbase/scripts/test/Test8.perl.properties index 31565d25..e9db58b0 100644 --- a/src/java/us/kbase/scripts/test/Test8.perl.properties +++ b/src/java/us/kbase/scripts/test/Test8.perl.properties @@ -1,8 +1,11 @@ package StoringImpl; + use strict; +use warnings; + use Bio::KBase::Exceptions; # Use Semantic Versioning (2.0.0-rc.1) -# http://semver.org +# http://semver.org our $VERSION = "0.1.0"; =head1 NAME @@ -226,7 +229,7 @@ sub m2second -=head2 version +=head2 version $return = $obj->version() diff --git a/src/java/us/kbase/templates/module_cpanfile.vm.properties b/src/java/us/kbase/templates/module_cpanfile.vm.properties new file mode 100644 index 00000000..7e8a45b6 --- /dev/null +++ b/src/java/us/kbase/templates/module_cpanfile.vm.properties @@ -0,0 +1,11 @@ +requires 'Config::IniFiles', '>= 3.0000002'; +requires 'Config::Any', '>= 0.32'; +requires 'Data::Dumper::Concise', '>= 2.023'; +requires 'Test::Most', '>= 0.35'; +requires 'Test::Compile', '>= 2.3.1'; +requires 'Ref::Util::XS', '>= 0.117'; +requires 'Ref::Util', '>= 0.204'; +#if( $example ) +# download a sequence IO module +requires 'Bio::SeqIO'; +#end diff --git a/src/java/us/kbase/templates/module_dockerfile.vm.properties b/src/java/us/kbase/templates/module_dockerfile.vm.properties index ece8ea3c..5d40e766 100644 --- a/src/java/us/kbase/templates/module_dockerfile.vm.properties +++ b/src/java/us/kbase/templates/module_dockerfile.vm.properties @@ -3,8 +3,7 @@ FROM kbase/sdkbase2:python #else FROM kbase/sdkbase2:latest #end -MAINTAINER #if($username)${username}#{else}KBase Developer#{end} - +LABEL maintainer="#if($user_name)${user_name}#{else}KBase Developer#{end}" # ----------------------------------------- # In this section, you can install any system dependencies required # to run your App. For instance, you could place an apt-get update or @@ -18,14 +17,6 @@ MAINTAINER #if($username)${username}#{else}KBase Developer#{end} RUN cd /kb/deployment/lib/jars \ && wget https://downloads.sourceforge.net/project/jfasta/releases/jfasta-2.2.0/jfasta-2.2.0-jar-with-dependencies.jar #end -#if( $language == "perl") -# download an inifile reader -RUN cpanm -i Config::IniFiles -#if( $example ) -# download a sequence IO module -RUN cpanm -i Bio::SeqIO -#end -#end #if( $language == 'r' ) ENV R_LIBS=/kb/deployment/lib RUN R -q -e 'if(!require(jsonlite)) install.packages("jsonlite", repos="http://cran.us.r-project.org")' @@ -33,16 +24,22 @@ RUN R -q -e 'if(!require(httr)) install.packages("httr", repos="http://cran.us.r RUN R -q -e 'if(!require(raster)) install.packages("raster", repos="http://cran.us.r-project.org")' RUN apt-get -y install r-cran-evaluate r-cran-codetools r-cran-testthat #end - -# ----------------------------------------- +#if( $language == "perl") +COPY ./cpanfile /kb/module/cpanfile +WORKDIR /kb/module +# install cpan dependencies from the cpanfile, +# then tidy up the installation leftovers +RUN cpanm --installdeps . && \ + cd ~ && rm -rf .cpanm +#end COPY ./ /kb/module -RUN mkdir -p /kb/module/work -RUN chmod -R a+rw /kb/module WORKDIR /kb/module -RUN make all +RUN mkdir -p /kb/module/work && \ + chmod -R a+rw /kb/module && \ + make all ENTRYPOINT [ "./scripts/entrypoint.sh" ] diff --git a/src/java/us/kbase/templates/module_java_impl.vm.properties b/src/java/us/kbase/templates/module_java_impl.vm.properties index 97ee23b5..ee13035f 100644 --- a/src/java/us/kbase/templates/module_java_impl.vm.properties +++ b/src/java/us/kbase/templates/module_java_impl.vm.properties @@ -22,7 +22,7 @@ import assemblyutil.GetAssemblyParams; import assemblyutil.SaveAssemblyParams; import kbasereport.CreateParams; import kbasereport.KBaseReportClient; -import kbasereport.Report; +import kbasereport.SimpleReport; import kbasereport.ReportInfo; import kbasereport.WorkspaceObject; import net.sf.jfasta.FASTAElement; @@ -73,11 +73,11 @@ public class ${java_module_name}Server extends JsonServerServlet { public FilterContigsResults filterContigs(FilterContigsParams params, AuthToken authPart, RpcContext jsonRpcContext) throws Exception { FilterContigsResults returnVal = null; //BEGIN run_${module_name} - + // Print statements to stdout/stderr are captured and available as the App log System.out.println("Starting filter contigs. Parameters:"); System.out.println(params); - + /* Step 1 - Parse/examine the parameters and catch any errors * It is important to check that parameters exist and are defined, and that nice error * messages are returned to users. Parameter values go through basic validation when @@ -103,7 +103,7 @@ public class ${java_module_name}Server extends JsonServerServlet { throw new IllegalArgumentException("min_length parameter cannot be negative (" + minLength + ")"); } - + /* Step 2 - Download the input data as a Fasta file * We can use the AssemblyUtils module to download a FASTA file from our Assembly data * object. The return object gives us the path to the file that was created. @@ -142,26 +142,26 @@ public class ${java_module_name}Server extends JsonServerServlet { final String resultText = String.format("Filtered assembly to %s contigs out of %s", remaining, total); System.out.println(resultText); - + // Step 4 - Save the new Assembly back to the system - + final String newAssyRef = assyUtil.saveAssemblyFromFasta(new SaveAssemblyParams() .withAssemblyName(fileobj.getAssemblyName()) .withWorkspaceName(workspaceName) .withFile(new FastaAssemblyFile().withPath(out.toString()))); - + // Step 5 - Build a Report and return - + final KBaseReportClient kbr = new KBaseReportClient(callbackURL, authPart); // see note above about bad practice kbr.setIsInsecureHttpConnectionAllowed(true); final ReportInfo report = kbr.create(new CreateParams().withWorkspaceName(workspaceName) - .withReport(new Report().withTextMessage(resultText) + .withReport(new SimpleReport().withTextMessage(resultText) .withObjectsCreated(Arrays.asList(new WorkspaceObject() .withDescription("Filtered contigs") .withRef(newAssyRef))))); // Step 6: contruct the output to send back - + returnVal = new ReportResults() .withReportName(report.getName()) .withReportRef(report.getRef()); @@ -195,7 +195,7 @@ import java.net.MalformedURLException; import kbasereport.CreateParams; import kbasereport.KBaseReportClient; -import kbasereport.Report; +import kbasereport.SimpleReport; import kbasereport.ReportInfo; import kbasereport.WorkspaceObject; import ${java_package}.ReportResults; @@ -221,7 +221,7 @@ import ${java_package}.ReportResults; final String parameter_1 = params.get("parameter_1").asInstance(); final ReportInfo report = kbr.create(new CreateParams() .withWorkspaceName(ws_name) - .withReport(new Report() + .withReport(new SimpleReport() .withTextMessage(parameter_1) .withObjectsCreated(new java.util.ArrayList()))); @@ -230,4 +230,4 @@ import ${java_package}.ReportResults; .withReportRef(report.getRef()); //END run_${module_name} -#end \ No newline at end of file +#end diff --git a/src/java/us/kbase/templates/module_makefile.vm.properties b/src/java/us/kbase/templates/module_makefile.vm.properties index 19e103ce..578dc21e 100644 --- a/src/java/us/kbase/templates/module_makefile.vm.properties +++ b/src/java/us/kbase/templates/module_makefile.vm.properties @@ -63,7 +63,7 @@ build-executable-script: #if($language == "perl") echo '#!/bin/bash' > $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME) echo 'script_dir=$$(dirname "$$(readlink -f "$$0")")' >> $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME) - echo 'export PERL5LIB=$$script_dir/../$(LIB_DIR):$$PATH:$$PERL5LIB' >> $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME) + echo 'export PERL5LIB=$$script_dir/../$(LIB_DIR):$$PERL5LIB' >> $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME) echo 'perl $$script_dir/../$(LIB_DIR)/$(SERVICE_CAPS)/$(SERVICE_CAPS)Server.pm $$1 $$2 $$3' >> $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME) #end #if($language == "python") @@ -89,7 +89,7 @@ build-startup-script: echo 'script_dir=$$(dirname "$$(readlink -f "$$0")")' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME) #if($language == "perl") echo 'export KB_DEPLOYMENT_CONFIG=$$script_dir/../deploy.cfg' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME) - echo 'export PERL5LIB=$$script_dir/../$(LIB_DIR):$$PATH:$$PERL5LIB' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME) + echo 'export PERL5LIB=$$script_dir/../$(LIB_DIR):$$PERL5LIB' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME) echo 'plackup $$script_dir/../$(LIB_DIR)/$(SERVICE_CAPS).psgi' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME) #end #if($language == "python") @@ -115,16 +115,16 @@ build-test-script: echo 'export KB_AUTH_TOKEN=`cat /kb/module/work/token`' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) echo 'echo "Removing temp files..."' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) echo 'rm -rf $(WORK_DIR)/*' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) - echo 'echo "...done removing temp files."' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) + echo 'echo "Finished removing temp files."' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) #if($language == "perl") - echo 'export PERL5LIB=$$script_dir/../$(LIB_DIR):$$PATH:$$PERL5LIB' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) - echo 'cd $$script_dir/../$(TEST_DIR)' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) - echo "perl -e 'opendir my ${esc.print("\")}${esc.print("$")}${esc.print("$")}dh, \".\"; my @l = grep { /\\\\.pl${esc.print("\")}${esc.print("$")}${esc.print("$")}/ } readdir ${esc.print("\")}${esc.print("$")}${esc.print("$")}dh; my ${esc.print("\")}${esc.print("$")}${esc.print("$")}ret=0; foreach my ${esc.print("\")}${esc.print("$")}${esc.print("$")}s (@l) { print(\"Running \".${esc.print("\")}${esc.print("$")}${esc.print("$")}s.\"\\\\n\"); system(\"perl\", ${esc.print("\")}${esc.print("$")}${esc.print("$")}s) && do{${esc.print("\")}${esc.print("$")}${esc.print("$")}ret=2; next;}; } exit(${esc.print("\")}${esc.print("$")}${esc.print("$")}ret);'" >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) + echo 'export PERL5LIB=$$script_dir/../$(LIB_DIR):$$PERL5LIB' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) + echo 'cd $$script_dir/..' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) + echo 'prove -lvrm $$script_dir' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) #end #if($language == "python") echo 'export PYTHONPATH=$$script_dir/../$(LIB_DIR):$$PATH:$$PYTHONPATH' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) echo 'cd $$script_dir/../$(TEST_DIR)' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) - echo 'python -m nose --with-coverage --cover-package=$(SERVICE_CAPS) --cover-html --cover-html-dir=/kb/module/work/test_coverage --nocapture --nologcapture .' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) + echo 'python -m nose --with-coverage --cover-package=$(SERVICE_CAPS) --cover-html --cover-html-dir=/kb/module/work/test_coverage --cover-xml --cover-xml-file=/kb/module/work/test_coverage/coverage.xml --nocapture --nologcapture .' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) #end #if($language == "java") echo 'export JAVA_HOME=$(JAVA_HOME)' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME) diff --git a/src/java/us/kbase/templates/module_perl_impl.vm.properties b/src/java/us/kbase/templates/module_perl_impl.vm.properties index ffbf4e07..6b60ce55 100644 --- a/src/java/us/kbase/templates/module_perl_impl.vm.properties +++ b/src/java/us/kbase/templates/module_perl_impl.vm.properties @@ -1,9 +1,11 @@ -#if ($example) package ${module_name}::${module_name}Impl; + use strict; +use warnings; + use Bio::KBase::Exceptions; # Use Semantic Versioning (2.0.0-rc.1) -# http://semver.org +# http://semver.org our $VERSION = '0.0.1'; our $GIT_URL = ''; our $GIT_COMMIT_HASH = ''; @@ -20,34 +22,35 @@ This sample module contains one small method that filters contigs. =cut #BEGIN_HEADER +use feature qw( say ); use Bio::KBase::AuthToken; -use installed_clients::AssemblyUtilClient; use installed_clients::KBaseReportClient; use Config::IniFiles; +#if ($example) +use installed_clients::AssemblyUtilClient; use Bio::SeqIO; use Data::Dumper; +#end #END_HEADER -sub new -{ - my($class, @args) = @_; - my $self = { - }; +sub new { + my ( $class, @args ) = @_; + my $self = {}; bless $self, $class; + #BEGIN_CONSTRUCTOR - - my $config_file = $ENV{ KB_DEPLOYMENT_CONFIG }; - my $cfg = Config::IniFiles->new(-file=>$config_file); - my $scratch = $cfg->val('${module_name}', 'scratch'); - my $callbackURL = $ENV{ SDK_CALLBACK_URL }; - - $self->{scratch} = $scratch; - $self->{callbackURL} = $callbackURL; - + + my $config_file = $ENV{ KB_DEPLOYMENT_CONFIG }; + my $cfg = Config::IniFiles->new( -file => $config_file ); + my $scratch = $cfg->val('${module_name}', 'scratch'); + my $callbackURL = $ENV{ SDK_CALLBACK_URL }; + + $self->{scratch} = $scratch; + $self->{callbackURL} = $callbackURL; + #END_CONSTRUCTOR - if ($self->can('_init_instance')) - { + if ( $self->can( '_init_instance' ) ) { $self->_init_instance(); } return $self; @@ -56,7 +59,6 @@ sub new =head1 METHODS - =head2 run_${module_name} $output = $obj->run_${module_name}($params) @@ -65,130 +67,160 @@ sub new The actual function is declared using 'funcdef' to specify the name and input/return arguments to the function. For all typical KBase -Apps that run in the Narrative, your function should have the +Apps that run in the Narrative, your function should have the 'authentication required' modifier. =back =cut -sub run_${module_name} -{ +sub run_${module_name} { my $self = shift; - my($params) = @_; - - my @_bad_arguments; - (ref($params) eq 'HASH') or push(@_bad_arguments, "Invalid type for argument \"params\" (value was \"$params\")"); - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to run_${module_name}:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'run_${module_name}'); + my ( $params ) = @_; + + my @bad_arguments; + ( ref( $params ) eq 'HASH' ) or push @_bad_arguments, + "Invalid type for argument \"params\" " + . "(value was \"$params\")"; + if ( @bad_arguments ) { + my $msg = "Invalid arguments passed to run_${module_name}:\n" + . join "", map { "\t$_\n" } @bad_arguments; + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'run_${module_name}' + ); } my $ctx = $${module_name}::${module_name}Server::CallContext; - my($output); + my ( $output ); #BEGIN run_${module_name} - +#if ($example) # Print statements to stdout/stderr are captured and available as the App log - print("Starting run_${module_name} method. Parameters:\n"); - print(Dumper($params) . "\n"); - + say "Starting run_${module_name} method. Parameters: " . Dumper $params; + # Step 1 - Parse/examine the parameters and catch any errors # It is important to check that parameters exist and are defined, and that nice error # messages are returned to users. Parameter values go through basic validation when # defined in a Narrative App, but advanced users or other SDK developers can call # this function directly, so validation is still important. - - if (!exists $params->{'workspace_name'}) { + + if ( !exists $params->{ 'workspace_name' } ) { die "Parameter workspace_name is not set in input arguments"; } - my $workspace_name=$params->{'workspace_name'}; - - if (!exists $params->{'assembly_input_ref'}) { + my $workspace_name = $params->{ 'workspace_name' }; + + if ( !exists $params->{ 'assembly_input_ref' } ) { die "Parameter assembly_input_ref is not set in input arguments"; } - my $assy_ref=$params->{'assembly_input_ref'}; - - if (!exists $params->{'min_length'}) { + my $assy_ref = $params->{ 'assembly_input_ref' }; + + if ( !exists $params->{ 'min_length' } ) { die "Parameter min_length is not set in input arguments"; } - my $min_length = $params->{'min_length'}; - if ($min_length < 0) { - die "min_length parameter cannot be negative (".$min_length.")"; + my $min_length = $params->{ 'min_length' }; + if ( $min_length < 0 ) { + die "min_length parameter cannot be negative (" . $min_length . ")"; } - + # Step 2 - Download the input data as a Fasta file # We can use the AssemblyUtils module to download a FASTA file from our Assembly data # object. The return object gives us the path to the file that was created. - - print("Downloading assembly data as FASTA file.\n"); - my $assycli = installed_clients::AssemblyUtilClient->new($self->{callbackURL}); - my $fileobj = $assycli->get_assembly_as_fasta({ref => $assy_ref}); - + + say "Downloading assembly data as FASTA file."; + my $assycli = installed_clients::AssemblyUtilClient->new( $self->{ callbackURL } ); + my $fileobj = $assycli->get_assembly_as_fasta( { ref => $assy_ref } ); + # Step 3 - Actually perform the filter operation, saving the good contigs to a new # fasta file. - - my $sio_in = Bio::SeqIO->new(-file => $fileobj->{path}); - my $outfile = $self->{scratch} . "/" . "filtered.fasta"; - my $sio_out = Bio::SeqIO->new(-file => ">$outfile", -format=> "fasta"); + + my $sio_in = Bio::SeqIO->new( -file => $fileobj->{ path } ); + my $outfile = $self->{ scratch } . "/" . "filtered.fasta"; + my $sio_out = Bio::SeqIO->new( -file => ">$outfile", -format => "fasta" ); my $total = 0; my $remaining = 0; - while (my $seq = $sio_in->next_seq) { + while ( my $seq = $sio_in->next_seq ) { $total++; - if ($seq->length >= $min_length) { + if ( $seq->length >= $min_length ) { $remaining++; - $sio_out->write_seq($seq); + $sio_out->write_seq( $seq ); } } my $result_text = "Filtered assembly to " . $remaining . " contigs out of " . $total; - print($result_text . "\n"); - + say $result_text; + # Step 4 - Save the new Assembly back to the system - my $newref = $assycli->save_assembly_from_fasta({assembly_name => $fileobj->{assembly_name}, - workspace_name => $workspace_name, - file => {path => $outfile}}); + my $newref = $assycli->save_assembly_from_fasta( { + assembly_name => $fileobj->{ assembly_name }, + workspace_name => $workspace_name, + file => { + path => $outfile + } + } ); + # Step 5 - Build a report and return - my $repcli = installed_clients::KBaseReportClient->new($self->{callbackURL}); - my $report = $repcli->create( - {workspace_name => $workspace_name, - report => {text_message => $result_text, - objects_created => [{description => "Filtered contigs", - ref => $newref} - ] - } - }); - + my $repcli = installed_clients::KBaseReportClient->new( $self->{ callbackURL } ); + my $report = $repcli->create( { + workspace_name => $workspace_name, + report => { + text_message => $result_text, + objects_created => [ { + description => "Filtered contigs", + ref => $newref + } ] + } + } ); + # Step 6 - construct the output to send back - - my $output = {assembly_output => $newref, - n_initial_contigs => $total, - n_contigs_remaining => $remaining, - n_contigs_removed => $total - $remaining, - report_name => $report->{name}, - report_ref => $report->{ref}}; - - print("returning: ".Dumper($output)."\n"); - + + $output = { + assembly_output => $newref, + n_initial_contigs => $total, + n_contigs_remaining => $remaining, + n_contigs_removed => $total - $remaining, + report_name => $report->{ name }, + report_ref => $report->{ ref } + }; + + say "returning: " . Dumper $output; +#else + my $repcli = installed_clients::KBaseReportClient->new( $self->{ callbackURL } ); + my $report = $repcli->create( { + workspace_name => $params->{ workspace_name }, + report => { + text_message => $params->{ parameter_1 }, + objects_created => [] + } + } ); + + my $output = { + report_name => $report->{ name }, + report_ref => $report->{ ref } + }; +#end #END run_${module_name} - my @_bad_returns; - (ref($output) eq 'HASH') or push(@_bad_returns, "Invalid type for return variable \"output\" (value was \"$output\")"); - if (@_bad_returns) { - my $msg = "Invalid returns passed to run_${module_name}:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'run_${module_name}'); + my @bad_returns; + ( ref($output) eq 'HASH' ) or push @bad_returns, + "Invalid type for return variable \"output\" " + . "(value was \"$output\")"; + if ( @bad_returns ) { + my $msg = "Invalid returns passed to run_${module_name}:\n" + . join "", map { "\t$_\n" } @bad_returns; + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'run_${module_name}' + ); } - return($output); + return ( $output ); } - -=head2 status +=head2 status $return = $obj->status() -=over 4 +=over =item Parameter and return types @@ -215,47 +247,17 @@ Return the module status. This is a structure including Semantic Versioning numb =cut sub status { - my($return); + my $return; #BEGIN_STATUS - $return = {"state" => "OK", "message" => "", "version" => $VERSION, - "git_url" => $GIT_URL, "git_commit_hash" => $GIT_COMMIT_HASH}; + $return = { + "state" => "OK", + "message" => "", + "version" => $VERSION, + "git_url" => $GIT_URL, + "git_commit_hash" => $GIT_COMMIT_HASH + }; #END_STATUS - return($return); + return ( $return ); } 1; -#else -#BEGIN_HEADER -use Bio::KBase::AuthToken; -use installed_clients::KBaseReportClient; -use Config::IniFiles; -#END_HEADER -#BEGIN_CONSTRUCTOR - my $config_file = $ENV{ KB_DEPLOYMENT_CONFIG }; - my $cfg = Config::IniFiles->new(-file=>$config_file); - my $scratch = $cfg->val('${module_name}', 'scratch'); - my $callbackURL = $ENV{ SDK_CALLBACK_URL }; - - $self->{scratch} = $scratch; - $self->{callbackURL} = $callbackURL; -#END_CONSTRUCTOR -#BEGIN run_${module_name} - my $repcli = installed_clients::KBaseReportClient->new($self->{callbackURL}); - my $report = $repcli->create({ - workspace_name => $params->{workspace_name}, - report => { - text_message => $params->{parameter_1}, - objects_created => [] - } - }); - - my $output = { - report_name => $report->{name}, - report_ref => $report->{ref} - }; -#END run_${module_name} - -#BEGIN_STATUS - $return = {"state" => "OK", "message" => "", "version" => $VERSION, "git_url" => $GIT_URL, "git_commit_hash" => $GIT_COMMIT_HASH}; -#END_STATUS -#end \ No newline at end of file diff --git a/src/java/us/kbase/templates/module_test_java_client.vm.properties b/src/java/us/kbase/templates/module_test_java_client.vm.properties index 51bdce43..0dbc2fd8 100644 --- a/src/java/us/kbase/templates/module_test_java_client.vm.properties +++ b/src/java/us/kbase/templates/module_test_java_client.vm.properties @@ -46,7 +46,7 @@ public class ${java_module_name}ServerTest { private static ${java_module_name}Server impl = null; private static Path scratch; private static URL callbackURL; - + @BeforeClass public static void init() throws Exception { // Config loading @@ -72,7 +72,7 @@ public class ${java_module_name}ServerTest { .getAbsolutePath()); impl = new ${java_module_name}Server(); } - + private static String getWsName() throws Exception { if (wsName == null) { long suffix = System.currentTimeMillis(); @@ -81,13 +81,13 @@ public class ${java_module_name}ServerTest { } return wsName; } - + private static RpcContext getContext() { return new RpcContext().withProvenance(Arrays.asList(new ProvenanceAction() .withService("${module_name}").withMethod("please_never_use_it_in_production") .withMethodParams(new ArrayList()))); } - + @AfterClass public static void cleanup() { if (wsName != null) { @@ -99,7 +99,7 @@ public class ${java_module_name}ServerTest { } } } - + #if ($example) private String loadFASTA( final Path filename, @@ -116,9 +116,9 @@ public class ${java_module_name}ServerTest { .withAssemblyName(objectName) .withWorkspaceName(getWsName()) .withFile(new FastaAssemblyFile().withPath(filename.toString()))); - + } - + @Test public void test_run${java_module_name}_ok() throws Exception { // First load a test FASTA file as an KBase Assembly @@ -128,9 +128,9 @@ public class ${java_module_name}ServerTest { "agctt\n" + ">seq3\n" + "agcttttcatgg"; - + final String ref = loadFASTA(scratch.resolve("test1.fasta"), "TestAssembly", fastaContent); - + // second, call the implementation Map params = new HashMap<>(); params.put("workspace_name", new UObject(getWsName())); @@ -139,7 +139,7 @@ public class ${java_module_name}ServerTest { impl.run${java_module_name}(params, token, getContext()); } - + @Test public void test_run${java_module_name}_err1() throws Exception { try { @@ -153,7 +153,7 @@ public class ${java_module_name}ServerTest { ex.getMessage()); } } - + @Test public void test_run${java_module_name}_err2() throws Exception { try { @@ -167,7 +167,7 @@ public class ${java_module_name}ServerTest { Assert.assertEquals("min_length parameter cannot be negative (-10)", ex.getMessage()); } } - + @Test public void test_run${java_module_name}_err3() throws Exception { try { @@ -178,8 +178,12 @@ public class ${java_module_name}ServerTest { impl.run${java_module_name}(params, token, getContext()); Assert.fail("Error is expected above"); } catch (ServerException ex) { - Assert.assertEquals("Error on ObjectSpecification #1: Illegal number of separators " + - "/ in object reference fake", ex.getMessage()); + // ex.getMessage() returns the error string quoted + // so just test that the message is the same + Assert.assertTrue( ex.getMessage().contains( + "Error on ObjectSpecification #1: Illegal number of separators " + + "/ in object reference fake" + ) ); } } #else @@ -200,4 +204,4 @@ public class ${java_module_name}ServerTest { final ReportResults ret = impl.run${java_module_name}(params, token, getContext()); } #end -} \ No newline at end of file +} diff --git a/src/java/us/kbase/templates/module_test_perl_client.vm.properties b/src/java/us/kbase/templates/module_test_perl_client.vm.properties index b4369ea1..adb9c837 100644 --- a/src/java/us/kbase/templates/module_test_perl_client.vm.properties +++ b/src/java/us/kbase/templates/module_test_perl_client.vm.properties @@ -1,6 +1,8 @@ use strict; -use Data::Dumper; -use Test::More; +use warnings; +use feature qw( say ); + +use Test::Most; use Config::Simple; use Time::HiRes qw(time); use Bio::KBase::AuthToken; @@ -8,157 +10,230 @@ use installed_clients::WorkspaceClient; #if ($example) use installed_clients::AssemblyUtilClient; #end -use ${module_name}::${module_name}Impl; + +require_ok "${module_name}::${module_name}Impl"; local $| = 1; -my $token = $ENV{'KB_AUTH_TOKEN'}; -my $config_file = $ENV{'KB_DEPLOYMENT_CONFIG'}; -my $config = new Config::Simple($config_file)->get_block('${module_name}'); -my $ws_url = $config->{"workspace-url"}; -my $ws_name = undef; -my $ws_client = new installed_clients::WorkspaceClient($ws_url,token => $token); -my $scratch = $config->{scratch}; -my $callback_url = $ENV{'SDK_CALLBACK_URL'}; -my $auth_token = Bio::KBase::AuthToken->new(token => $token, ignore_authrc => 1, auth_svc=>$config->{'auth-service-url'}); -my $ctx = LocalCallContext->new($token, $auth_token->user_id); +my $token = $ENV{ 'KB_AUTH_TOKEN' }; +my $config_file = $ENV{ 'KB_DEPLOYMENT_CONFIG' }; +my $callback_url = $ENV{ 'SDK_CALLBACK_URL' }; + +my $config = Config::Simple->new( $config_file )->get_block( '${module_name}' ); +my ( $ws_name, $ws_client ); + +my $scratch = $config->{ scratch }; + +my $auth_token = Bio::KBase::AuthToken->new( + token => $token, + ignore_authrc => 1, + auth_svc => $config->{ 'auth-service-url' } +); + +my $ctx = LocalCallContext->new( $token, $auth_token->user_id ); $${module_name}::${module_name}Server::CallContext = $ctx; -my $impl = new ${module_name}::${module_name}Impl(); + +sub get_ws_client { + + my $ws_url = $config->{ "workspace-url" }; + $ws_client //= installed_clients::WorkspaceClient->new( $ws_url, token => $token ); + return $ws_client; +} sub get_ws_name { - if (!defined($ws_name)) { - my $suffix = int(time * 1000); - $ws_name = 'test_${module_name}_' . $suffix; - $ws_client->create_workspace({workspace => $ws_name}); + + $ws_client = get_ws_client(); + unless ( $ws_name ) { + my $suffix = int( time * 1000 ); + $ws_name = 'test_${module_name}_' . $suffix; + $ws_client->create_workspace( { workspace => $ws_name } ); } return $ws_name; } +my $impl; + +subtest 'creating the new ${module_name}Impl object' => sub { + + lives_ok { + $impl = ${module_name}::${module_name}Impl->new(); + } '${module_name}Impl object can be created'; + + isa_ok $impl, '${module_name}::${module_name}Impl'; + +}; + #if($example) sub load_fasta { - my $filename = shift; - my $object_name = shift; - my $filecontents = shift; - open (my $fh, '>', $filename) or die "Could not open file '$filename' ${dollar_sign}!"; - print $fh $filecontents; + my ( $filename, $object_name, $contents ) = @_; + + open my $fh, '>', $filename + or die "Could not open file '$filename': ${dollar_sign}!"; + print $fh $contents; close $fh; - my $assycli = installed_clients::AssemblyUtilClient->new($callback_url); - return $assycli->save_assembly_from_fasta({assembly_name => $object_name, - workspace_name => get_ws_name(), - file => {path => $filename}}); + my $assycli = installed_clients::AssemblyUtilClient->new( $callback_url ); + return $assycli->save_assembly_from_fasta( { + assembly_name => $object_name, + workspace_name => get_ws_name(), + file => { path => $filename } + } ); } -#end -eval { +subtest 'successful run_${module_name} run' => sub { -#if($example) - # First load a test FASTA file as an KBase Assembly + # load a test FASTA file as an KBase Assembly my $fastaContent = ">seq1 something something asdf\n" . "agcttttcat\n" . ">seq2\n" . "agctt\n" . ">seq3\n" . "agcttttcatgg"; - - my $ref = load_fasta($scratch . "/test1.fasta", "TestAssembly", $fastaContent); - # Second, call the implementation - my $ret = $impl->run_${module_name}({workspace_name => get_ws_name(), - assembly_input_ref => $ref, - min_length => 10}); + my $ref = load_fasta( $scratch . "/test1.fasta", "TestAssembly", $fastaContent ); + + # call the implementation + my $result = $impl->run_${module_name}( { + workspace_name => get_ws_name(), + assembly_input_ref => $ref, + min_length => 10, + } ); # validate the returned data - ok($ret->{n_initial_contigs} eq 3, "number of initial contigs"); - ok($ret->{n_contigs_removed} eq 1, "number of removed contigs"); - ok($ret->{n_contigs_remaining} eq 2, "number of remaining contigs"); - - $@ = ''; - eval { - $impl->run_${module_name}({workspace_name => get_ws_name(), - assembly_input_ref=>"fake", - min_length => 10}); - }; - like($@, qr#separators / in object reference fake#); - - eval { - $impl->run_${module_name}({workspace_name => get_ws_name(), - assembly_input_ref => "fake", - min_length => -10}); - }; - like($@, qr/min_length parameter cannot be negative/); - - eval { - $impl->run_${module_name}({workspace_name => get_ws_name(), - assembly_input_ref => "fake"}); - }; - like($@, qr/Parameter min_length is not set in input arguments/); - - done_testing(6); + cmp_ok + $result->{ n_initial_contigs }, '==', 3, + 'correct number of initial contigs'; + cmp_ok + $result->{ n_contigs_removed }, '==', 1, + 'correct number of removed contigs'; + cmp_ok + $result->{ n_contigs_remaining }, '==', 2, + 'correct number of remaining contigs'; + +}; + +subtest 'run_${module_name} errors' => sub { + + throws_ok { + $impl->run_${module_name}( { + workspace_name => get_ws_name(), + assembly_input_ref =>"fake", + min_length => 10, + } ); + } qr#separators / in object reference fake#, + 'Appropriate error thrown for fake ref object'; + + throws_ok { + $impl->run_${module_name}( { + workspace_name => get_ws_name(), + assembly_input_ref => "fake", + min_length => -10, + } ); + } qr/min_length parameter cannot be negative/, + 'correct error message for min length param'; + + throws_ok { + $impl->run_${module_name}( { + workspace_name => get_ws_name(), + assembly_input_ref => "fake" + } ); + } qr/Parameter min_length is not set in input arguments/, + 'missing min_length param'; + +}; #else + +subtest 'testing run_${module_name}' => sub { + # Prepare test data using the appropriate uploader for that data (see the KBase function # catalog for help, https://narrative.kbase.us/#catalog/functions) - - # Run your method by - # my $ret = $impl->your_method(parameters...); + # Run your method using + # my $result = $impl->your_method( $parameters... ); # - # Check returned data with - # ok(ret->{...} eq , "tested item") or other Test::More methods - my $ret = $impl->run_${module_name}({workspace_name => get_ws_name(), - parameter_1 => "Hello world"}); -#end + # Check returned data with methods from Test::Most; e.g. + # cmp_deeply + # $result, + # $expected, + # "$impl->your_method returns the expected data with params ..."; + + my $result = $impl->run_${module_name}( { + workspace_name => get_ws_name(), + parameter_1 => "Hello world", + } ); + + my $expect = { + report_name => re('report_[0-9a-f-]+'), + report_ref => re('\d+\/\d+\/\d+'), + }; + + cmp_deeply + $result, + $expect, + 'the expected report was returned' + or diag explain { + got => $result, + expected => $expect, + }; }; -my $err = undef; -if ($@) { - $err = $@; -} -eval { - if (defined($ws_name)) { - $ws_client->delete_workspace({workspace => $ws_name}); - print("Test workspace was deleted\n"); - } -}; -if (defined($err)) { - use Scalar::Util 'blessed'; - if(blessed $err && $err->isa("Bio::KBase::Exceptions::KBaseException")) { - die "Error while running tests. Remote error:\n" . $err->{data} . - "Client-side error:\n" . $err; - } else { - die $err; - } + +#end + +if ( $ws_name ) { + lives_ok { + $ws_client->delete_workspace( { workspace => $ws_name } ); + } 'Test workspace successfully deleted'; } +done_testing(); + { + package LocalCallContext; use strict; + use warnings; + sub new { - my($class,$token,$user) = @_; + my ( $class, $token, $user ) = @_; my $self = { - token => $token, + token => $token, user_id => $user }; return bless $self, $class; } + sub user_id { - my($self) = @_; - return $self->{user_id}; + my ( $self ) = @_; + return $self->{ user_id }; } + sub token { - my($self) = @_; - return $self->{token}; + my ( $self ) = @_; + return $self->{ token }; } + sub provenance { - my($self) = @_; - return [{'service' => '${module_name}', 'method' => 'please_never_use_it_in_production', 'method_params' => []}]; + my ( $self ) = @_; + return [ { + 'service' => '${module_name}', + 'method' => 'please_never_use_it_in_production', + 'method_params' => [], + } + ]; } + sub authenticated { return 1; } + sub log_debug { - my($self,$msg) = @_; - print STDERR $msg."\n"; + my ( $self, $msg ) = @_; + print STDERR $msg . "\n"; } + sub log_info { - my($self,$msg) = @_; - print STDERR $msg."\n"; + my ( $self, $msg ) = @_; + print STDERR $msg . "\n"; } + + 1; + } diff --git a/src/java/us/kbase/templates/module_test_perl_compile.vm.properties b/src/java/us/kbase/templates/module_test_perl_compile.vm.properties new file mode 100644 index 00000000..a171b5e5 --- /dev/null +++ b/src/java/us/kbase/templates/module_test_perl_compile.vm.properties @@ -0,0 +1,29 @@ +use Test::Most; +use Test::Compile; + +my $base_dir = '/kb/module/'; +my $tester = Test::Compile->new(); + +# check all .pm and .pl files compile +$tester->all_files_ok( $base_dir ); + +# also check .t and .psgi files +my @all_ext_files = files_with_ext( $tester, qr/\.(t|psgi)$/, $base_dir ); + +for ( @all_ext_files ) { + ok $tester->pl_file_compiles( $_ ), $_ . ' compiles'; +} + +$tester->done_testing(); + +sub files_with_ext { + my ( $tester, $regex, @dirs ) = @_; + + @dirs = @dirs ? @dirs : ( $base_dir ); + + my @files; + for my $file ( $tester->_find_files( @dirs ) ) { + push @files, $file if $file =~ $regex; + } + return @files; +} diff --git a/src/java/us/kbase/templates/module_test_python_client.vm.properties b/src/java/us/kbase/templates/module_test_python_client.vm.properties index 4952aac4..0445aa9f 100644 --- a/src/java/us/kbase/templates/module_test_python_client.vm.properties +++ b/src/java/us/kbase/templates/module_test_python_client.vm.properties @@ -6,7 +6,11 @@ from configparser import ConfigParser from ${module_name}.${module_name}Impl import ${module_name} from ${module_name}.${module_name}Server import MethodContext -from ${module_name}.authclient import KBaseAuth as _KBaseAuth + +try: + from ${module_name}.authclient import KBaseAuth as _KBaseAuth +except ImportError: + from installed_clients.authclient import KBaseAuth as _KBaseAuth #if ($example) from installed_clients.AssemblyUtilClient import AssemblyUtil diff --git a/src/java/us/kbase/templates/module_travis.vm.properties b/src/java/us/kbase/templates/module_travis.vm.properties index a7c1b256..e51dcae5 100644 --- a/src/java/us/kbase/templates/module_travis.vm.properties +++ b/src/java/us/kbase/templates/module_travis.vm.properties @@ -20,7 +20,6 @@ install: - git clone https://github.com/kbase/kb_sdk - cd kb_sdk - make bin - - make sdkbase - export PATH=$(pwd)/bin:$PATH - source src/sh/sdk-completion.sh - popd @@ -28,4 +27,4 @@ install: script: - kb-sdk validate -after_script: \ No newline at end of file +after_script: diff --git a/src/java/us/kbase/templates/perl_client.vm.properties b/src/java/us/kbase/templates/perl_client.vm.properties index 2fb032ca..da92dd5a 100644 --- a/src/java/us/kbase/templates/perl_client.vm.properties +++ b/src/java/us/kbase/templates/perl_client.vm.properties @@ -1,8 +1,10 @@ package ${client_package_name}; +use strict; +use warnings; + use JSON::RPC::Client; use POSIX; -use strict; use Data::Dumper; use URI; use Bio::KBase::Exceptions; @@ -38,74 +40,72 @@ ${module.module_doc} =cut -sub new -{ - my($class, $url, @args) = @_; - +sub new { + my ( $class, $url, @args ) = @_; + #if( $default_service_url ) - if (!defined($url)) - { - $url = '${default_service_url}'; + if ( !defined( $url ) ) { + $url = '${default_service_url}'; } #end my $self = { - client => ${client_package_name}::RpcClient->new, - url => $url, - headers => [], + client => ${client_package_name}::RpcClient->new, + url => $url, + headers => [], }; + #if( $any_async || $async_version ) my %arg_hash = @args; - $self->{async_job_check_time} = 0.1; - if (exists $arg_hash{"async_job_check_time_ms"}) { - $self->{async_job_check_time} = $arg_hash{"async_job_check_time_ms"} / 1000.0; + $self->{ async_job_check_time } = 0.1; + if ( exists $arg_hash{ async_job_check_time_ms } ) { + $self->{ async_job_check_time } = $arg_hash{ async_job_check_time_ms } / 1000.0; } - $self->{async_job_check_time_scale_percent} = 150; - if (exists $arg_hash{"async_job_check_time_scale_percent"}) { - $self->{async_job_check_time_scale_percent} = $arg_hash{"async_job_check_time_scale_percent"}; + + $self->{ async_job_check_time_scale_percent } = 150; + if ( exists $arg_hash{ async_job_check_time_scale_percent } ) { + $self->{ async_job_check_time_scale_percent } = $arg_hash{ async_job_check_time_scale_percent }; } - $self->{async_job_check_max_time} = 300; # 5 minutes - if (exists $arg_hash{"async_job_check_max_time_ms"}) { - $self->{async_job_check_max_time} = $arg_hash{"async_job_check_max_time_ms"} / 1000.0; + + $self->{ async_job_check_max_time } = 300; # 5 minutes + if ( exists $arg_hash{ async_job_check_max_time_ms } ) { + $self->{ async_job_check_max_time } = $arg_hash{ async_job_check_max_time_ms } / 1000.0; } + my $service_version = #if($service_ver)'$service_ver'#{else}undef#{end}; - if (exists $arg_hash{"service_version"}) { - $service_version = $arg_hash{"service_version"}; + if ( exists $arg_hash{ "service_version" } ) { + $service_version = $arg_hash{ "service_version" }; } $self->{service_version} = $service_version; #end - chomp($self->{hostname} = `hostname`); - $self->{hostname} ||= 'unknown-host'; + chomp( $self->{ hostname } = `hostname` ); + $self->{ hostname } ||= 'unknown-host'; # # Set up for propagating KBRPC_TAG and KBRPC_METADATA environment variables through # to invoked services. If these values are not set, we create a new tag # and a metadata field with basic information about the invoking script. # - if ($ENV${empty_escaper}{KBRPC_TAG}) - { - $self->{kbrpc_tag} = $ENV${empty_escaper}{KBRPC_TAG}; + if ( $ENV${empty_escaper}{ KBRPC_TAG } ) { + $self->{ kbrpc_tag } = $ENV${empty_escaper}{ KBRPC_TAG }; } - else - { - my ($t, $us) = &$get_time(); - $us = sprintf("%06d", $us); - my $ts = strftime("%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t); - $self->{kbrpc_tag} = "C:$0:$self->{hostname}:$$:$ts"; + else { + my ( $t, $us ) = &$get_time(); + $us = sprintf( "%06d", $us ); + my $ts = strftime( "%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t ); + $self->{ kbrpc_tag } = "C:$0:$self->{hostname}:$$:$ts"; } - push(@{$self->{headers}}, 'Kbrpc-Tag', $self->{kbrpc_tag}); + push @{ $self->{ headers } }, 'Kbrpc-Tag', $self->{ kbrpc_tag }; - if ($ENV${empty_escaper}{KBRPC_METADATA}) - { - $self->{kbrpc_metadata} = $ENV${empty_escaper}{KBRPC_METADATA}; - push(@{$self->{headers}}, 'Kbrpc-Metadata', $self->{kbrpc_metadata}); + if ( $ENV${empty_escaper}{ KBRPC_METADATA } ) { + $self->{ kbrpc_metadata } = $ENV${empty_escaper}{ KBRPC_METADATA }; + push @{ $self->{ headers } }, 'Kbrpc-Metadata', $self->{ kbrpc_metadata }; } - if ($ENV${empty_escaper}{KBRPC_ERROR_DEST}) - { - $self->{kbrpc_error_dest} = $ENV${empty_escaper}{KBRPC_ERROR_DEST}; - push(@{$self->{headers}}, 'Kbrpc-Errordest', $self->{kbrpc_error_dest}); + if ( $ENV${empty_escaper}{ KBRPC_ERROR_DEST } ) { + $self->{ kbrpc_error_dest } = $ENV${empty_escaper}{ KBRPC_ERROR_DEST }; + push @{ $self->{ headers } }, 'Kbrpc-Errordest', $self->{ kbrpc_error_dest }; } #if( $authenticated ) @@ -115,67 +115,84 @@ sub new # We create an auth token, passing through the arguments that we were (hopefully) given. { - my %arg_hash2 = @args; - if (exists $arg_hash2{"token"}) { - $self->{token} = $arg_hash2{"token"}; - } elsif (exists $arg_hash2{"user_id"}) { - my $token = Bio::KBase::AuthToken->new(@args); - if (!$token->error_message) { - $self->{token} = $token->token; - } - } - - if (exists $self->{token}) - { - $self->{client}->{token} = $self->{token}; - } + my %arg_hash2 = @args; + if ( exists $arg_hash2{ token } ) { + $self->{ token } = $arg_hash2{ token }; + } + elsif ( exists $arg_hash2{ user_id } ) { + my $token = Bio::KBase::AuthToken->new( @args ); + if ( !$token->error_message ) { + $self->{ token } = $token->token; + } + } + + if ( exists $self->{ token } ) { + $self->{ client }{ token } = $self->{ token }; + } } #end - my $ua = $self->{client}->ua; - my $timeout = $ENV${empty_escaper}{CDMI_TIMEOUT} || (30 * 60); - $ua->timeout($timeout); + my $ua = $self->{ client }->ua; + my $timeout = $ENV${empty_escaper}{ CDMI_TIMEOUT } || ( 30 * 60 ); + $ua->timeout( $timeout ); bless $self, $class; - # $self->_validate_version(); return $self; } #foreach( $module in $modules ) #if( $any_async || $async_version) -sub _check_job { - my($self, @args) = @_; # Authentication: ${method.authentication} - if ((my $n = @args) != 1) { - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => - "Invalid argument count for function _check_job (received $n, expecting 1)"); +sub _check_job { + my ( $self, @args ) = @_; + + if ( ( my $n = @args ) != 1 ) { + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => "Invalid argument count for function _check_job " + . "(received $n, expecting 1)" + ); } - { - my($job_id) = @args; - my @_bad_arguments; - (!ref($job_id)) or push(@_bad_arguments, "Invalid type for argument 0 \"job_id\" (it should be a string)"); - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to _check_job:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => '_check_job'); - } - } - my $result = $self->{client}->call($self->{url}, $self->{headers}, { - method => "${module.module_name}._check_job", - params => \@args}); - if ($result) { - if ($result->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $result->error_message, - code => $result->content->{error}->{code}, - method_name => '_check_job', - data => $result->content->{error}->{error} # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O - ); + + my ( $job_id ) = @args; + + my @bad_arguments; + ( ! ref( $job_id ) ) or push @bad_arguments, + "Invalid type for argument 0 \"job_id\" (it should be a string)"; + + if ( @bad_arguments ) { + my $msg = "Invalid arguments passed to _check_job:\n" + . join "", map { "\t$_\n" } @bad_arguments; + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => '_check_job' + ); + } + + my $result = $self->{ client }->call( + $self->{ url }, + $self->{ headers }, + { + method => "${module.module_name}._check_job", + params => \@args + } + ); + if ( $result ) { + if ( $result->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $result->error_message, + code => $result->content->{ error }{ code }, + method_name => '_check_job', + # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + data => $result->content->{ error }{ error }, + ); } else { return $result->result->[0]; } } else { - Bio::KBase::Exceptions::HTTP->throw(error => "Error invoking method _check_job", - status_line => $self->{client}->status_line, - method_name => '_check_job'); + Bio::KBase::Exceptions::HTTP->throw( + error => "Error invoking method _check_job", + status_line => $self->{client}->status_line, + method_name => '_check_job' + ); } } @@ -227,235 +244,315 @@ ${method.doc} =cut #if( $method.async || $async_version) -sub ${method.name} -{ - my($self, @args) = @_; - my $job_id = $self->_${method.name}_submit(@args); - my $async_job_check_time = $self->{async_job_check_time}; +sub ${method.name} { + my ( $self, @args ) = @_; + my $job_id = $self->_${method.name}_submit( @args ); + my $async_job_check_time = $self->{ async_job_check_time }; while (1) { - Time::HiRes::sleep($async_job_check_time); - $async_job_check_time *= $self->{async_job_check_time_scale_percent} / 100.0; - if ($async_job_check_time > $self->{async_job_check_max_time}) { - $async_job_check_time = $self->{async_job_check_max_time}; - } - my $job_state_ref = $self->_check_job($job_id); - if ($job_state_ref->{"finished"} != 0) { - if (!exists $job_state_ref->{"result"}) { - $job_state_ref->{"result"} = []; + Time::HiRes::sleep( $async_job_check_time ); + $async_job_check_time *= $self->{ async_job_check_time_scale_percent } / 100.0; + if ( $async_job_check_time > $self->{ async_job_check_max_time } ) { + $async_job_check_time = $self->{ async_job_check_max_time }; + } + my $job_state_ref = $self->_check_job( $job_id ); + if ( $job_state_ref->{ "finished" } != 0 ) { + if ( !exists $job_state_ref->{ "result" } ) { + $job_state_ref->{ "result" } = []; } - return wantarray ? @{$job_state_ref->{"result"}} : $job_state_ref->{"result"}->[0]; + return wantarray + ? @{ $job_state_ref->{ "result" } } + : $job_state_ref->{ "result" }[ 0 ]; } } } -sub _${method.name}_submit { - my($self, @args) = @_; # Authentication: ${method.authentication} - if ((my $n = @args) != ${method.arg_count}) { - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => - "Invalid argument count for function _${method.name}_submit (received $n, expecting ${method.arg_count})"); +sub _${method.name}_submit { + my ( $self, @args ) = @_; + if ( ( my $n = @args ) != ${method.arg_count} ) { + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => "Invalid argument count for function _${method.name}_submit " + . "(received $n, expecting ${method.arg_count})" + ); } #if( $method.arg_count > 0 ) { - my(${method.arg_vars}) = @args; - my @_bad_arguments; + my ( ${method.arg_vars} ) = @args; + my @bad_arguments; #foreach( $param in $method.params ) - (${param.validator}) or push(@_bad_arguments, "Invalid type for argument ${param.index} \"${param.name}\" (value was \"${param.perl_var}\")"); + ( ${param.validator} ) or push @bad_arguments, + "Invalid type for argument ${param.index} \"${param.name}\" " + . "(value was \"${param.perl_var}\")"; #end - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to _${method.name}_submit:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => '_${method.name}_submit'); + if ( @bad_arguments ) { + my $msg = "Invalid arguments passed to _${method.name}_submit:\n" + . join "", map { "\t$_\n" } @bad_arguments; + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => '_${method.name}_submit' + ); } } #end my $context = undef; - if ($self->{service_version}) { - $context = {'service_ver' => $self->{service_version}}; - } - my $result = $self->{client}->call($self->{url}, $self->{headers}, { - method => "${module.module_name}._${method.name}_submit", - params => \@args, context => $context}); - if ($result) { - if ($result->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $result->error_message, - code => $result->content->{error}->{code}, - method_name => '_${method.name}_submit', - data => $result->content->{error}->{error} # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + if ( $self->{ service_version } ) { + $context = { 'service_ver' => $self->{ service_version } }; + } + my $result = $self->{ client }->call( + $self->{ url }, + $self->{ headers }, + { + method => "${module.module_name}._${method.name}_submit", + params => \@args, + context => $context + } + ); + + if ( $result ) { + if ( $result->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $result->error_message, + code => $result->content->{ error }{ code }, + method_name => '_${method.name}_submit', + # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + data => $result->content->{ error }{ error }, ); } else { - return $result->result->[0]; # job_id + return $result->result->[ 0 ]; # job_id } } else { - Bio::KBase::Exceptions::HTTP->throw(error => "Error invoking method _${method.name}_submit", - status_line => $self->{client}->status_line, - method_name => '_${method.name}_submit'); + Bio::KBase::Exceptions::HTTP->throw( + error => "Error invoking method _${method.name}_submit", + status_line => $self->{ client }->status_line, + method_name => '_${method.name}_submit' + ); } } #else ## of if-any-async -sub ${method.name} -{ - my($self, @args) = @_; - # Authentication: ${method.authentication} +sub ${method.name} { + my ( $self, @args ) = @_; - if ((my $n = @args) != ${method.arg_count}) - { - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => - "Invalid argument count for function ${method.name} (received $n, expecting ${method.arg_count})"); + if ( ( my $n = @args ) != ${method.arg_count} ) { + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => "Invalid argument count for function ${method.name} " + . "(received $n, expecting ${method.arg_count})" + ); } #if( $method.arg_count > 0 ) { - my(${method.arg_vars}) = @args; + my ( ${method.arg_vars} ) = @args; - my @_bad_arguments; + my @bad_arguments; #foreach( $param in $method.params ) - (${param.validator}) or push(@_bad_arguments, "Invalid type for argument ${param.index} \"${param.name}\" (value was \"${param.perl_var}\")"); + ( ${param.validator} ) or push @bad_arguments, + "Invalid type for argument ${param.index} \"${param.name}\" " + . "(value was \"${param.perl_var}\")"; #end - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to ${method.name}:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => '${method.name}'); - } + if ( @bad_arguments ) { + my $msg = "Invalid arguments passed to ${method.name}:\n" + . join "", map { "\t$_\n" } @bad_arguments; + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => '${method.name}' + ); + } } #end #if( $dynserv_ver ) - my $service_state = $self->{client}->call($self->{url}, $self->{headers}, { - method => "ServiceWizard.get_service_status", - params => [{module_name=>"${module.module_name}", version=>$self->{service_version}}]}); - if ($service_state->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $service_state->error_message, - code => $service_state->content->{error}->{code}, - method_name => 'ServiceWizard.get_service_status', - data => $service_state->content->{error}->{error} - ); + my $service_state = $self->{ client }->call( + $self->{ url }, + $self->{ headers }, + { + method => "ServiceWizard.get_service_status", + params => + [ + { + module_name => "${module.module_name}", + version => $self->{ service_version } + } + ] + } + ); + if ( $service_state->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $service_state->error_message, + code => $service_state->content->{ error }{ code }, + method_name => 'ServiceWizard.get_service_status', + data => $service_state->content->{ error }{ error } + ); } my $url = $service_state->result->[0]->{url}; #else - my $url = $self->{url}; + my $url = $self->{ url }; #end - my $result = $self->{client}->call($url, $self->{headers}, { - method => "${module.module_name}.${method.name}", - params => \@args, - }); - if ($result) { - if ($result->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $result->error_message, - code => $result->content->{error}->{code}, - method_name => '${method.name}', - data => $result->content->{error}->{error} # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O - ); - } else { + my $result = $self->{ client }->call( + $url, + $self->{ headers }, + { + method => "${module.module_name}.${method.name}", + params => \@args, + } + ); + if ( $result ) { + if ( $result->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $result->error_message, + code => $result->content->{ error }{ code }, + method_name => '${method.name}', + # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + data => $result->content->{ error }{ error }, + ); + } else { #if( $method.ret_count > 0 ) - return wantarray ? @{$result->result} : $result->result->[0]; + return wantarray ? @{ $result->result } : $result->result->[ 0 ]; #else - return; + return; #end - } + } } else { - Bio::KBase::Exceptions::HTTP->throw(error => "Error invoking method ${method.name}", - status_line => $self->{client}->status_line, - method_name => '${method.name}', - ); + Bio::KBase::Exceptions::HTTP->throw( + error => "Error invoking method ${method.name}", + status_line => $self->{ client }->status_line, + method_name => '${method.name}', + ); } } + #end ## of if-any-async -#end ## of foreach-method +#end ## of foreach-method #if( !$status_in_kidl ) #if( $async_version ) -sub status -{ - my($self, @args) = @_; +sub status { + my ( $self, @args ) = @_; my $job_id = undef; - if ((my $n = @args) != 0) { - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => - "Invalid argument count for function status (received $n, expecting 0)"); + if ( ( my $n = @args ) != 0 ) { + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => "Invalid argument count for function status " + . "(received $n, expecting 0)"); } my $context = undef; - if ($self->{service_version}) { - $context = {'service_ver' => $self->{service_version}}; - } - my $result = $self->{client}->call($self->{url}, $self->{headers}, { - method => "${module.module_name}._status_submit", - params => \@args, context => $context}); - if ($result) { - if ($result->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $result->error_message, - code => $result->content->{error}->{code}, - method_name => '_status_submit', - data => $result->content->{error}->{error} # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + if ( $self->{ service_version } ) { + $context = { 'service_ver' => $self->{ service_version } }; + } + my $result = $self->{ client }->call( + $self->{ url }, + $self->{ headers }, + { + method => "${module.module_name}._status_submit", + params => \@args, + context => $context + } + ); + + if ( $result ) { + if ( $result->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $result->error_message, + code => $result->content->{ error }{ code }, + method_name => '_status_submit', + # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + data => $result->content->{ error }{ error }, ); } else { - $job_id = $result->result->[0]; + $job_id = $result->result->[ 0 ]; } } else { - Bio::KBase::Exceptions::HTTP->throw(error => "Error invoking method _status_submit", - status_line => $self->{client}->status_line, - method_name => '_status_submit'); + Bio::KBase::Exceptions::HTTP->throw( + error => "Error invoking method _status_submit", + status_line => $self->{ client }->status_line, + method_name => '_status_submit', + ); } - my $async_job_check_time = $self->{async_job_check_time}; - while (1) { - Time::HiRes::sleep($async_job_check_time); - $async_job_check_time *= $self->{async_job_check_time_scale_percent} / 100.0; - if ($async_job_check_time > $self->{async_job_check_max_time}) { - $async_job_check_time = $self->{async_job_check_max_time}; - } - my $job_state_ref = $self->_check_job($job_id); - if ($job_state_ref->{"finished"} != 0) { - if (!exists $job_state_ref->{"result"}) { - $job_state_ref->{"result"} = []; + my $async_job_check_time = $self->{ async_job_check_time }; + while ( 1 ) { + Time::HiRes::sleep( $async_job_check_time ); + $async_job_check_time *= $self->{ async_job_check_time_scale_percent } / 100.0; + if ( $async_job_check_time > $self->{ async_job_check_max_time } ) { + $async_job_check_time = $self->{ async_job_check_max_time }; + } + my $job_state_ref = $self->_check_job( $job_id ); + if ( $job_state_ref->{ "finished" } != 0 ) { + if ( !exists $job_state_ref->{ "result" } ) { + $job_state_ref->{ "result" } = []; } - return wantarray ? @{$job_state_ref->{"result"}} : $job_state_ref->{"result"}->[0]; + return wantarray + ? @{ $job_state_ref->{ "result" } } + : $job_state_ref->{ "result" }->[ 0 ]; } } + } #else ## of if-status-async -sub status -{ - my($self, @args) = @_; - if ((my $n = @args) != 0) { - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => - "Invalid argument count for function status (received $n, expecting 0)"); +sub status { + my ( $self, @args ) = @_; + + if ( ( my $n = @args ) != 0 ) { + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => "Invalid argument count for function status " + . "(received $n, expecting 0)" + ); } + #if( $dynserv_ver ) - my $service_state = $self->{client}->call($self->{url}, $self->{headers}, { - method => "ServiceWizard.get_service_status", - params => [{module_name=>"${module.module_name}", version=>$self->{service_version}}]}); - if ($service_state->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $service_state->error_message, - code => $service_state->content->{error}->{code}, - method_name => 'ServiceWizard.get_service_status', - data => $service_state->content->{error}->{error} - ); + + my $service_state = $self->{ client }->call( + $self->{ url }, + $self->{ headers }, + { + method => "ServiceWizard.get_service_status", + params => + [ + { + module_name => "${module.module_name}", + version => $self->{ service_version } + } + ] + } + ); + if ( $service_state->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $service_state->error_message, + code => $service_state->content->{ error }{ code }, + method_name => 'ServiceWizard.get_service_status', + data => $service_state->content->{ error }{ error }, + ); } - my $url = $service_state->result->[0]->{url}; + my $url = $service_state->result->[ 0 ]{ url }; #else - my $url = $self->{url}; + my $url = $self->{ url }; #end - my $result = $self->{client}->call($url, $self->{headers}, { - method => "${module.module_name}.status", - params => \@args, - }); - if ($result) { - if ($result->is_error) { - Bio::KBase::Exceptions::JSONRPC->throw(error => $result->error_message, - code => $result->content->{error}->{code}, - method_name => 'status', - data => $result->content->{error}->{error} # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + my $result = $self->{ client }->call( + $url, + $self->{ headers }, + { + method => "${module.module_name}.status", + params => \@args, + } + ); + if ( $result ) { + if ( $result->is_error ) { + Bio::KBase::Exceptions::JSONRPC->throw( + error => $result->error_message, + code => $result->content->{ error }{ code }, + method_name => 'status', + # JSON::RPC::ReturnObject only supports JSONRPC 1.1 or 1.O + data => $result->content->{ error }{ error }, ); } else { - return wantarray ? @{$result->result} : $result->result->[0]; + return wantarray ? @{ $result->result } : $result->result->[0]; } } else { - Bio::KBase::Exceptions::HTTP->throw(error => "Error invoking method status", - status_line => $self->{client}->status_line, - method_name => 'status', - ); + Bio::KBase::Exceptions::HTTP->throw( + error => "Error invoking method status", + status_line => $self->{ client }->status_line, + method_name => 'status', + ); } } #end ## of if-status-async @@ -464,25 +561,29 @@ sub status sub version { - my ($self) = @_; - my $result = $self->{client}->call($self->{url}, $self->{headers}, { - method => "${last_module.module_name}.version", - params => [], - }); - if ($result) { - if ($result->is_error) { + my ( $self ) = @_; + my $result = $self->{ client }->call( + $self->{ url }, + $self->{ headers }, + { + method => "${last_module.module_name}.version", + params => [], + } + ); + if ( $result ) { + if ( $result->is_error ) { Bio::KBase::Exceptions::JSONRPC->throw( - error => $result->error_message, - code => $result->content->{code}, + error => $result->error_message, + code => $result->content->{ code }, method_name => '${last_method.name}', ); } else { - return wantarray ? @{$result->result} : $result->result->[0]; + return wantarray ? @{ $result->result } : $result->result->[ 0 ]; } } else { Bio::KBase::Exceptions::HTTP->throw( - error => "Error invoking method ${last_method.name}", - status_line => $self->{client}->status_line, + error => "Error invoking method ${last_method.name}", + status_line => $self->{ client }->status_line, method_name => '${last_method.name}', ); } @@ -492,26 +593,30 @@ sub _validate_version { my ($self) = @_; my $svr_version = $self->version(); my $client_version = $VERSION; - my ($cMajor, $cMinor) = split(/\./, $client_version); - my ($sMajor, $sMinor) = split(/\./, $svr_version); - if ($sMajor != $cMajor) { + my ( $cMajor, $cMinor ) = split(/\./, $client_version); + my ( $sMajor, $sMinor ) = split(/\./, $svr_version); + + if ( $sMajor != $cMajor ) { Bio::KBase::Exceptions::ClientServerIncompatible->throw( - error => "Major version numbers differ.", - server_version => $svr_version, - client_version => $client_version + error => "Major version numbers differ.", + server_version => $svr_version, + client_version => $client_version ); } - if ($sMinor < $cMinor) { + + if ( $sMinor < $cMinor ) { Bio::KBase::Exceptions::ClientServerIncompatible->throw( - error => "Client minor version greater than Server minor version.", - server_version => $svr_version, - client_version => $client_version + error => "Client minor version greater than server minor version.", + server_version => $svr_version, + client_version => $client_version ); } - if ($sMinor > $cMinor) { + + if ( $sMinor > $cMinor ) { warn "New client version available for ${client_package_name}\n"; } - if ($sMajor == 0) { + + if ( $sMajor == 0 ) { warn "${client_package_name} version is $svr_version. API subject to change.\n"; } } @@ -572,69 +677,66 @@ sub call { #if( $enable_client_retry ) - my @retries = (1, 2, 5, 10, 20, 60, 60, 60, 60, 60, 60); - my %codes_to_retry = map { $_ => 1 } qw(110 408 502 503 504 200) ; + my @retries = ( 1, 2, 5, 10, 20, 60, 60, 60, 60, 60, 60 ); + my %codes_to_retry = map { $_ => 1 } qw( 110 408 502 503 504 200 ) ; my $n_retries; while (1) #end { - if ($uri =~ /\?/) { - $result = $self->_get($uri); - } - else { - Carp::croak "not hashref." unless (ref $obj eq 'HASH'); - $result = $self->_post($uri, $headers, $obj); - } -#if( $enable_client_retry ) + if ($uri =~ /\?/) { + $result = $self->_get($uri); + } + else { + Carp::croak "not hashref." unless (ref $obj eq 'HASH'); + $result = $self->_post($uri, $headers, $obj); + } + #if( $enable_client_retry ) + + # + # Bail early on success. + # + if ($result->is_success) { + if ($n_retries) { + print STDERR strftime("%F %T", localtime), ": Request succeeded after $n_retries retries\n"; + } + last; + } + $n_retries++; + + # + # Failure. See if we need to retry and loop, or bail with + # a permanent failure. + # - # - # Bail early on success. - # - if ($result->is_success) - { - if ($n_retries) - { - print STDERR strftime("%F %T", localtime), ": Request succeeded after $n_retries retries\n"; - } - last; - } - $n_retries++; - - # - # Failure. See if we need to retry and loop, or bail with - # a permanent failure. - # - my $code = $result->code; - my $msg = $result->message; - my $want_retry = 0; - if ($codes_to_retry${empty_escaper}{$code}) - { - $want_retry = 1; - } - elsif ($code eq 500 && defined( $result->header('client-warning') ) - && $result->header('client-warning') eq 'Internal response') - { - # - # Handle errors that were not thrown by the web - # server but rather picked up by the client library. - # - # If we got a client timeout or connection refused, let us retry. - # - - if ($msg =~ /timeout|connection refused/i) - { - $want_retry = 1; - } - - } - + my $msg = $result->message; + my $want_retry = 0; + if ($codes_to_retry${empty_escaper}{$code}) + { + $want_retry = 1; + } + elsif ($code eq 500 && defined( $result->header('client-warning') ) + && $result->header('client-warning') eq 'Internal response') + { + # + # Handle errors that were not thrown by the web + # server but rather picked up by the client library. + # + # If we got a client timeout or connection refused, let us retry. + # + + if ($msg =~ /timeout|connection refused/i) { + $want_retry = 1; + } + + } + if (!$want_retry || @retries == 0) { - last; + last; } - + # # otherwise, sleep & loop. # @@ -670,13 +772,13 @@ sub call { sub _post { - my ($self, $uri, $headers, $obj) = @_; + my ( $self, $uri, $headers, $obj ) = @_; my $json = $self->json; - $obj->{version} ||= $self->{version} || '1.1'; + $obj->{ version } ||= $self->{ version } || '1.1'; - if ($obj->{version} eq '1.0') { - delete $obj->{version}; + if ( $obj->{ version } eq '1.0' ) { + delete $obj->{ version }; if (exists $obj->{id}) { $self->id($obj->{id}) if ($obj->{id}); # if undef, it is notification. } @@ -685,9 +787,9 @@ sub _post { } } else { - # $obj->{id} = $self->id if (defined $self->id); - # Assign a random number to the id if one hasn't been set - $obj->{id} = (defined $self->id) ? $self->id : substr(rand(),2); + # $obj->{ id } = $self->id if defined $self->id; + # Assign a random number to the id if one hasn't been set + $obj->{ id } = (defined $self->id) ? $self->id : substr( rand(), 2 ); } my $content = $json->encode($obj); @@ -697,11 +799,9 @@ sub _post { Content_Type => $self->{content_type}, Content => $content, Accept => 'application/json', - @$headers, - ($self->{token} ? (Authorization => $self->{token}) : ()), + @$headers, + ($self->{token} ? (Authorization => $self->{token}) : ()), ); } - - 1; diff --git a/src/java/us/kbase/templates/perl_impl.vm.properties b/src/java/us/kbase/templates/perl_impl.vm.properties index 9ff476db..58b8503b 100644 --- a/src/java/us/kbase/templates/perl_impl.vm.properties +++ b/src/java/us/kbase/templates/perl_impl.vm.properties @@ -1,8 +1,11 @@ package ${module.impl_package_name}; + use strict; +use warnings; + use Bio::KBase::Exceptions; # Use Semantic Versioning (2.0.0-rc.1) -# http://semver.org +# http://semver.org our $VERSION = '${module.semantic_version}'; our $GIT_URL = '${module.git_url}'; our $GIT_COMMIT_HASH = '${module.git_commit_hash}'; @@ -20,18 +23,16 @@ ${module.module_doc} #BEGIN_HEADER ${module.module_header}#END_HEADER -sub new -{ - my($class, @args) = @_; - my $self = { - }; +sub new { + my ( $class, @args ) = @_; + my $self = {}; bless $self, $class; + #BEGIN_CONSTRUCTOR ${module.module_constructor} #END_CONSTRUCTOR - if ($self->can('_init_instance')) - { - $self->_init_instance(); + if ( $self->can( '_init_instance' ) ) { + $self->_init_instance(); } return $self; } @@ -44,12 +45,11 @@ ${module.module_constructor} #END_CONSTRUCTOR #set( $status_in_kidl = true ) #end - =head2 ${method.name} #if( "$method.ret_vars" != "" )${method.ret_vars} = #{end}$obj->${method.name}(${method.arg_vars}) -=over 4 +=over =item Parameter and return types @@ -74,7 +74,6 @@ ${docline} =end text - =item Description ${method.doc} @@ -83,41 +82,50 @@ ${method.doc} =cut -sub ${method.name} -{ +sub ${method.name} { my $self = shift; #if( $method.arg_count > 0 ) - my(${method.arg_vars}) = @_; + my ( ${method.arg_vars} ) = @_; - my @_bad_arguments; + my @bad_arguments; #foreach( $param in $method.params ) - (${param.validator}) or push(@_bad_arguments, "Invalid type for argument \"${param.name}\" (value was \"${param.perl_var}\")"); + ( ${param.validator} ) or push @bad_arguments, + "Invalid type for argument \"${param.name}\" " + . "(value was \"${param.perl_var}\")"; #end - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to ${method.name}:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => '${method.name}'); + if ( @bad_arguments ) { + my $msg = "Invalid arguments passed to ${method.name}:\n" + . join "", map { "\t$_\n" } @bad_arguments; + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => '${method.name}' + ); } #end my $ctx = $${server_package_name}::CallContext; #if( $method.ret_count > 0 ) - my(${method.ret_vars}); + my ( ${method.ret_vars} ); #end #BEGIN ${method.name} ${method.user_code} #${empty_escaper}END ${method.name} #if( $method.ret_count > 0 ) - my @_bad_returns; + my @bad_returns; #foreach( $return in $method.returns ) - (${return.validator}) or push(@_bad_returns, "Invalid type for return variable \"${return.name}\" (value was \"${return.perl_var}\")"); + ( ${return.validator} ) or push @bad_returns, + "Invalid type for return variable \"${return.name}\" " + . "(value was \"${return.perl_var}\")"; #end - if (@_bad_returns) { - my $msg = "Invalid returns passed to ${method.name}:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => '${method.name}'); + if ( @bad_returns ) { + my $msg = "Invalid returns passed to ${method.name}:\n" + . join "", map { "\t$_\n" } @bad_returns; + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => '${method.name}' + ); } #end - return(${method.ret_vars}); + return ( ${method.ret_vars} ); } @@ -125,11 +133,11 @@ ${method.user_code} #${empty_escaper}END ${method.name} #if( !$status_in_kidl ) -=head2 status +=head2 status $return = $obj->status() -=over 4 +=over =item Parameter and return types @@ -156,15 +164,21 @@ Return the module status. This is a structure including Semantic Versioning numb =cut sub status { - my($return); + my $return; #BEGIN_STATUS #if( ${module.module_status} ) -${module.module_status}#else - $return = {"state" => "OK", "message" => "", "version" => $VERSION, - "git_url" => $GIT_URL, "git_commit_hash" => $GIT_COMMIT_HASH}; +${module.module_status} +#else + $return = { + "state" => "OK", + "message" => "", + "version" => $VERSION, + "git_url" => $GIT_URL, + "git_commit_hash" => $GIT_COMMIT_HASH + }; #end #END_STATUS - return($return); + return ( $return ); } #end @@ -172,20 +186,17 @@ ${module.module_status}#else #foreach( $type in $module.types ) - =head2 ${type.name} -=over 4 +=over #if( $type.comment && "$type.comment" != "" ) - =item Description ${type.comment} #end - =item Definition =begin html @@ -206,7 +217,6 @@ ${type.english} #end - =cut 1; diff --git a/src/java/us/kbase/templates/perl_psgi.vm.properties b/src/java/us/kbase/templates/perl_psgi.vm.properties index 557d2e2c..6039ac6b 100644 --- a/src/java/us/kbase/templates/perl_psgi.vm.properties +++ b/src/java/us/kbase/templates/perl_psgi.vm.properties @@ -1,3 +1,6 @@ +use strict; +use warnings; + #foreach( $module in $modules ) use ${module.impl_package_name}; #end @@ -7,28 +10,27 @@ use Plack::Middleware::CrossOrigin; #if( $service_options.authenticated ) use UserAuth; - my $user_auth = UserAuth->new(); #end - -my @dispatch; - +my %dispatch = ( #foreach( $module in $modules ) -{ - my $obj = ${module.impl_package_name}->new; - push(@dispatch, '${module.module_name}' => $obj); -} + '${module.module_name}' => ${module.impl_package_name}->new, #end +); - -my $server = ${server_package_name}->new(instance_dispatch => { @dispatch }, +my $server = ${server_package_name}->new( + instance_dispatch => \%dispatch, + allow_get => 0, #if( $service_options.authenticated ) - user_auth => $user_auth, + user_auth => $user_auth, #end - allow_get => 0, - ); +); my $handler = sub { $server->handle_input(@_) }; -$handler = Plack::Middleware::CrossOrigin->wrap( $handler, origins => "*", headers => "*"); +$handler = Plack::Middleware::CrossOrigin->wrap( + $handler, + origins => "*", + headers => "*" +); diff --git a/src/java/us/kbase/templates/perl_server.vm.properties b/src/java/us/kbase/templates/perl_server.vm.properties index 1457a62e..edc7d178 100644 --- a/src/java/us/kbase/templates/perl_server.vm.properties +++ b/src/java/us/kbase/templates/perl_server.vm.properties @@ -1,6 +1,6 @@ package ${server_package_name}; -#set( $ctx_pkg = "${server_package_name}Context" ) +#set( $ctx_pkg = "${server_package_name}Context" ) #set( $stderr_pkg = "${server_package_name}StderrWrapper" ) use Data::Dumper; @@ -21,13 +21,17 @@ use Bio::KBase::AuthToken; extends 'RPC::Any::Server::JSONRPC::PSGI'; -has 'instance_dispatch' => (is => 'ro', isa => 'HashRef'); -has 'user_auth' => (is => 'ro', isa => 'UserAuth'); -has 'valid_methods' => (is => 'ro', isa => 'HashRef', lazy => 1, - builder => '_build_valid_methods'); -has 'loggers' => (is => 'ro', required => 1, builder => '_build_loggers'); -has 'config' => (is => 'ro', required => 1, builder => '_build_config'); -has 'local_headers' => (is => 'ro', isa => 'HashRef'); +has 'instance_dispatch' => ( is => 'ro', isa => 'HashRef' ); +has 'user_auth' => ( is => 'ro', isa => 'UserAuth' ); +has 'valid_methods' => ( + is => 'ro', + isa => 'HashRef', + lazy => 1, + builder => '_build_valid_methods' +); +has 'loggers' => ( is => 'ro', required => 1, builder => '_build_loggers' ); +has 'config' => ( is => 'ro', required => 1, builder => '_build_config' ); +has 'local_headers' => ( is => 'ro', isa => 'HashRef' ); our $CallContext; @@ -38,11 +42,11 @@ our %return_counts = ( #if( ${method.name} == "status" ) #set( $status_in_kidl = true ) #end - '${method.name}' => ${method.ret_count}, + '${method.name}' => ${method.ret_count}, #end #end #if( !$status_in_kidl ) - 'status' => 1, + status => 1, #end ); @@ -50,16 +54,15 @@ our %return_counts = ( our %method_authentication = ( #foreach( $module in $modules ) #foreach( $method in $module.methods ) - '${method.name}' => '${method.authentication}', + '${method.name}' => '${method.authentication}', #end #end ); #end -sub _build_valid_methods -{ - my($self) = @_; - my $methods = { +sub _build_valid_methods { + my ( $self ) = @_; + my $methods = { #foreach( $module in $modules ) #set( $last_module = $module ) #foreach( $method in $module.methods ) @@ -67,139 +70,162 @@ sub _build_valid_methods #end #end #if( !$status_in_kidl ) - 'status' => 1, + status => 1, #end }; return $methods; } -my $DEPLOY = 'KB_DEPLOYMENT_CONFIG'; +my $DEPLOY = 'KB_DEPLOYMENT_CONFIG'; my $SERVICE = 'KB_SERVICE_NAME'; -sub get_config_file -{ - my ($self) = @_; - if(!defined $ENV${empty_escaper}{$DEPLOY}) { +sub get_config_file { + my ( $self ) = @_; + if ( !defined $ENV${empty_escaper}{$DEPLOY} ) { return undef; } return $ENV${empty_escaper}{$DEPLOY}; } -sub get_service_name -{ - my ($self) = @_; - if(!defined $ENV${empty_escaper}{$SERVICE}) { +sub get_service_name { + my ( $self ) = @_; + if ( !defined $ENV${empty_escaper}{$SERVICE} ) { return '${service_name}'; } return $ENV${empty_escaper}{$SERVICE}; } -sub _build_config -{ - my ($self) = @_; - my $sn = $self->get_service_name(); - my $cf = $self->get_config_file(); - if (!($cf)) { +sub _build_config { + my ( $self ) = @_; + my $sn = $self->get_service_name; + my $cf = $self->get_config_file; + if ( !($cf) ) { return {}; } - my $cfg = new Config::Simple($cf); - my $cfgdict = $cfg->get_block($sn); - if (!($cfgdict)) { + my $cfg = Config::Simple->new( $cf ); + my $cfgdict = $cfg->get_block( $sn ); + if ( !($cfgdict) ) { return {}; } return $cfgdict; } -sub logcallback -{ - my ($self) = @_; - $self->loggers()->{serverlog}->set_log_file( - $self->{loggers}->{userlog}->get_log_file()); +sub logcallback { + my ( $self ) = @_; + $self->loggers()->{ serverlog }->set_log_file( + $self->{ loggers }{ userlog }->get_log_file() + ); } -sub log -{ - my ($self, $level, $context, $message, $tag) = @_; - my $user = defined($context->user_id()) ? $context->user_id(): undef; - $self->loggers()->{serverlog}->log_message($level, $message, $user, - $context->module(), $context->method(), $context->call_id(), - $context->client_ip(), $tag); +sub log { + my ( $self, $level, $context, $message, $tag ) = @_; + my $user + = defined( $context->user_id() ) + ? $context->user_id() + : undef; + $self->loggers()->{ serverlog }->log_message( + $level, $message, $user, $context->module(), $context->method(), + $context->call_id(), $context->client_ip(), $tag + ); } -sub _build_loggers -{ - my ($self) = @_; - my $submod = $self->get_service_name(); - my $loggers = {}; - my $callback = sub {$self->logcallback();}; - $loggers->{userlog} = Bio::KBase::Log->new( - $submod, {}, {ip_address => 1, authuser => 1, module => 1, - method => 1, call_id => 1, changecallback => $callback, - tag => 1, - config => $self->get_config_file()}); - $loggers->{serverlog} = Bio::KBase::Log->new( - $submod, {}, {ip_address => 1, authuser => 1, module => 1, - method => 1, call_id => 1, - tag => 1, - logfile => $loggers->{userlog}->get_log_file()}); - $loggers->{serverlog}->set_log_level(6); +sub _build_loggers { + my ( $self ) = @_; + my $submod = $self->get_service_name(); + my $loggers = {}; + my $callback = sub { $self->logcallback(); }; + $loggers->{ userlog } = Bio::KBase::Log->new( + $submod, + {}, + { + ip_address => 1, + authuser => 1, + module => 1, + method => 1, + call_id => 1, + changecallback => $callback, + tag => 1, + config => $self->get_config_file() + } + ); + $loggers->{ serverlog } = Bio::KBase::Log->new( + $submod, + {}, + { + ip_address => 1, + authuser => 1, + module => 1, + method => 1, + call_id => 1, + tag => 1, + logfile => $loggers->{ userlog }->get_log_file() + } + ); + $loggers->{ serverlog }->set_log_level( 6 ); return $loggers; } #override of RPC::Any::Server sub handle_error { - my ($self, $error) = @_; - - unless (ref($error) eq 'HASH' || - (blessed $error and $error->isa('RPC::Any::Exception'))) { - $error = RPC::Any::Exception::PerlError->new(message => $error); + my ( $self, $error ) = @_; + + unless ( ref( $error ) eq 'HASH' + || ( blessed $error and $error->isa( 'RPC::Any::Exception' ) ) ) { + $error = RPC::Any::Exception::PerlError->new( message => $error ); } my $output; eval { - my $encoded_error = $self->encode_output_from_exception($error); - $output = $self->produce_output($encoded_error); + my $encoded_error = $self->encode_output_from_exception( $error ); + $output = $self->produce_output( $encoded_error ); }; - + return $output if $output; - + die "$error\n\nAlso, an error was encountered while trying to send" . " this error: $@\n"; } #override of RPC::Any::JSONRPC sub encode_output_from_exception { - my ($self, $exception) = @_; + my ( $self, $exception ) = @_; + my %error_params; - if (ref($exception) eq 'HASH') { - %error_params = %{$exception}; - if(defined($error_params${empty_escaper}{context})) { + + if ( ref $exception eq 'HASH' ) { + %error_params = %$exception; + if ( defined( $error_params${empty_escaper}{ context } ) ) { my @errlines; - $errlines[0] = $error_params${empty_escaper}{message}; - push @errlines, split("\n", $error_params${empty_escaper}{data}); - $self->log($Bio::KBase::Log::ERR, $error_params${empty_escaper}{context}, \@errlines); - delete $error_params${empty_escaper}{context}; + $errlines[0] = $error_params${empty_escaper}{ message }; + push @errlines, split( "\n", $error_params${empty_escaper}{ data } ); + $self->log( + $Bio::KBase::Log::ERR, + $error_params${empty_escaper}{ context }, + \@errlines + ); + delete $error_params${empty_escaper}{ context }; } - } else { + } + else { %error_params = ( message => $exception->message, code => $exception->code, ); } my $json_error; - if ($self->_last_call) { - $json_error = $self->_last_call->return_error(%error_params); + if ( $self->_last_call ) { + $json_error = $self->_last_call->return_error( %error_params ); } # Default to default_version. This happens when we throw an exception # before inbound parsing is complete. else { - $json_error = $self->_default_error(%error_params); + $json_error = $self->_default_error( %error_params ); } - return $self->encode_output_from_object($json_error); + return $self->encode_output_from_object( $json_error ); } sub trim { - my ($str) = @_; - if (!(defined $str)) { + my ( $str ) = @_; + if ( !( defined $str ) ) { return $str; } $str =~ s/^\s+|\s+$//g; @@ -207,264 +233,289 @@ sub trim { } sub getIPAddress { - my ($self) = @_; - my $xFF = trim($self->_plack_req_header("X-Forwarded-For")); - my $realIP = trim($self->_plack_req_header("X-Real-IP")); - my $nh = $self->config->{"dont_trust_x_ip_headers"}; - my $trustXHeaders = !(defined $nh) || $nh ne "true"; - - if ($trustXHeaders) { - if ($xFF) { - my @tmp = split(",", $xFF); - return trim($tmp[0]); + my ( $self ) = @_; + my $xFF = trim( $self->_plack_req_header( "X-Forwarded-For" ) ); + my $realIP = trim( $self->_plack_req_header( "X-Real-IP" ) ); + my $nh = $self->config->{ "dont_trust_x_ip_headers" }; + my $trustXHeaders = !( defined $nh ) || $nh ne "true"; + + if ( $trustXHeaders ) { + if ( $xFF ) { + my @tmp = split ",", $xFF; + return trim( $tmp[ 0 ] ); } - if ($realIP) { + if ( $realIP ) { return $realIP; } } - if (defined($self->_plack_req)) { + if ( defined( $self->_plack_req ) ) { return $self->_plack_req->address; } return "localhost"; } sub call_method { - my ($self, $data, $method_info) = @_; - - my ($module, $method, $modname) = @$method_info${empty_escaper}{qw(module method modname)}; - - my $ctx = ${ctx_pkg}->new($self->{loggers}->{userlog}, - client_ip => $self->getIPAddress()); - $ctx->module($modname); - $ctx->method($method); - $ctx->call_id($self->{_last_call}->{id}); - - my $args = $data->{arguments}; - my $prov_action = {'service' => $modname, 'method' => $method, 'method_params' => $args}; - $ctx->provenance([$prov_action]); + my ( $self, $data, $method_info ) = @_; + + my ( $module, $method, $modname ) = @$method_info${empty_escaper}{ qw( module method modname ) }; + + my $ctx = ${ctx_pkg}->new( + $self->{ loggers }{ userlog }, + client_ip => $self->getIPAddress() + ); + $ctx->module( $modname ); + $ctx->method( $method ); + $ctx->call_id( $self->{ _last_call }{ id } ); + + my $args = $data->{ arguments }; + my $prov_action = { + service => $modname, + method => $method, + method_params => $args + }; + + $ctx->provenance( [ $prov_action ] ); + #if( $authenticated ) { # Service ${service_name} requires authentication. - my $method_auth = $method_authentication${empty_escaper}{$method}; - $ctx->authenticated(0); - if ($method_auth eq 'none') - { - # No authentication required here. Move along. + my $method_auth = $method_authentication${empty_escaper}{ $method }; + $ctx->authenticated( 0 ); + if ( $method_auth eq 'none' ) { + # No authentication required here. Move along. } - else - { - my $token = $self->_plack_req_header("Authorization"); + else { + my $token = $self->_plack_req_header( "Authorization" ); - if (!$token && $method_auth eq 'required') - { - $self->exception('PerlError', "Authentication required for ${last_module.module_name} but no authentication header was passed"); - } + if ( !$token && $method_auth eq 'required' ) { + $self->exception( 'PerlError', + "Authentication required for ${last_module.module_name} but no authentication header was passed" + ); + } - my $auth_token; - if ($self->config->{'auth-service-url'}) - { - $auth_token = Bio::KBase::AuthToken->new(token => $token, ignore_authrc => 1, auth_svc=>$self->config->{'auth-service-url'}); + my $auth_token; + if ( $self->config->{ 'auth-service-url' } ) { + $auth_token = Bio::KBase::AuthToken->new( + token => $token, + ignore_authrc => 1, + auth_svc =>$self->config->{ 'auth-service-url' } + ); } else { - $auth_token = Bio::KBase::AuthToken->new(token => $token, ignore_authrc => 1); + $auth_token = Bio::KBase::AuthToken->new( + token => $token, + ignore_authrc => 1 + ); } + my $valid = $auth_token->validate(); - my $valid = $auth_token->validate(); - # Only throw an exception if authentication was required and it fails - if ($method_auth eq 'required' && !$valid) - { - $self->exception('PerlError', "Token validation failed: " . $auth_token->error_message); - } elsif ($valid) { - $ctx->authenticated(1); - $ctx->user_id($auth_token->user_id); - $ctx->token( $token); - } + # Only throw an exception if authentication was required and it fails + if ( $method_auth eq 'required' && !$valid ) { + $self->exception( 'PerlError', + "Token validation failed: " + . $auth_token->error_message + ); + } + elsif ( $valid ) { + $ctx->authenticated( 1 ); + $ctx->user_id( $auth_token->user_id ); + $ctx->token( $token ); + } } } #else # Service ${service_name} does not require authentication. #end - my $new_isa = $self->get_package_isa($module); + my $new_isa = $self->get_package_isa( $module ); no strict 'refs'; local @{"${module}::ISA"} = @$new_isa; local $CallContext = $ctx; my @result; { - # + # # Process tag and metadata information if present. # - my $tag = $self->_plack_req_header("Kbrpc-Tag"); - if (!$tag) - { + my $tag = $self->_plack_req_header( "Kbrpc-Tag" ); + if ( !$tag ) { my ($t, $us) = &$get_time(); - $us = sprintf("%06d", $us); - my $ts = strftime("%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t); - $tag = "S:$self->{hostname}:$$:$ts"; + $us = sprintf( "%06d", $us ); + my $ts = strftime( "%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t ); + $tag = "S:$self->{hostname}:$$:$ts"; } - local $ENV${empty_escaper}{KBRPC_TAG} = $tag; - my $kb_metadata = $self->_plack_req_header("Kbrpc-Metadata"); - my $kb_errordest = $self->_plack_req_header("Kbrpc-Errordest"); - local $ENV${empty_escaper}{KBRPC_METADATA} = $kb_metadata if $kb_metadata; - local $ENV${empty_escaper}{KBRPC_ERROR_DEST} = $kb_errordest if $kb_errordest; - - my $stderr = ${stderr_pkg}->new($ctx, $get_time); - $ctx->stderr($stderr); - - my $xFF = $self->_plack_req_header("X-Forwarded-For"); - if ($xFF) { - $self->log($Bio::KBase::Log::INFO, $ctx, - "X-Forwarded-For: " . $xFF, $tag); + local $ENV${empty_escaper}{ KBRPC_TAG } = $tag; + my $kb_metadata = $self->_plack_req_header( "Kbrpc-Metadata" ); + my $kb_errordest = $self->_plack_req_header( "Kbrpc-Errordest" ); + local $ENV${empty_escaper}{ KBRPC_METADATA } = $kb_metadata if $kb_metadata; + local $ENV${empty_escaper}{ KBRPC_ERROR_DEST } = $kb_errordest if $kb_errordest; + + my $stderr = ${stderr_pkg}->new( $ctx, $get_time ); + $ctx->stderr( $stderr ); + + my $xFF = $self->_plack_req_header( "X-Forwarded-For" ); + if ( $xFF ) { + $self->log( + $Bio::KBase::Log::INFO, + $ctx, + "X-Forwarded-For: " . $xFF, + $tag + ); } - + my $err; eval { - $self->log($Bio::KBase::Log::INFO, $ctx, "start method", $tag); - local $SIG${empty_escaper}{__WARN__} = sub { - my($msg) = @_; - $stderr->log($msg); + $self->log( $Bio::KBase::Log::INFO, $ctx, "start method", $tag ); + local $SIG${empty_escaper}{ __WARN__ } = sub { + my ( $msg ) = @_; + $stderr->log( $msg ); print STDERR $msg; }; - @result = $module->$method(@{ $data->{arguments} }); - $self->log($Bio::KBase::Log::INFO, $ctx, "end method", $tag); + @result = $module->$method( @{ $data->{ arguments } } ); + $self->log( $Bio::KBase::Log::INFO, $ctx, "end method", $tag ); }; if ($@) { my $err = $@; - $stderr->log($err); - $ctx->stderr(undef); + $stderr->log( $err ); + $ctx->stderr( undef ); undef $stderr; - $self->log($Bio::KBase::Log::INFO, $ctx, "fail method", $tag); + $self->log( $Bio::KBase::Log::INFO, $ctx, "fail method", $tag ); my $nicerr; - if(ref($err) eq "Bio::KBase::Exceptions::KBaseException") { - $nicerr = {code => -32603, # perl error from RPC::Any::Exception - message => $err->error, - data => $err->trace->as_string, - context => $ctx - }; - } else { + + if ( ref( $err ) eq "Bio::KBase::Exceptions::KBaseException" ) { + $nicerr = { + code => -32603, # perl error from RPC::Any::Exception + message => $err->error, + data => $err->trace->as_string, + context => $ctx + }; + } + else { my $str = "$err"; - $str =~ s/Bio::KBase::CDMI::Service::call_method.*//s; # is this still necessary? not sure + # is this still necessary? not sure + $str =~ s/Bio::KBase::CDMI::Service::call_method.*//s; my $msg = $str; - $msg =~ s/ at [^\s]+.pm line \d+.\n$//; - $nicerr = {code => -32603, # perl error from RPC::Any::Exception - message => $msg, - data => $str, - context => $ctx - }; + $msg =~ s/ at [^\s]+.pm line \d+.\n$//; + $nicerr = { + code => -32603, # perl error from RPC::Any::Exception + message => $msg, + data => $str, + context => $ctx + }; } die $nicerr; } - $ctx->stderr(undef); + $ctx->stderr( undef ); undef $stderr; } my $result; - if ($return_counts${empty_escaper}{$method} == 1) - { - $result = [[$result[0]]]; + if ( $return_counts${empty_escaper}{ $method } == 1 ) { + $result = [ [ $result[ 0 ] ] ]; } - else - { + else { $result = \@result; } return $result; } sub _plack_req_header { - my ($self, $header_name) = @_; - if (defined($self->local_headers)) { - return $self->local_headers->{$header_name}; + my ( $self, $header_name ) = @_; + if ( defined( $self->local_headers ) ) { + return $self->local_headers->{ $header_name }; } - return $self->_plack_req->header($header_name); + return $self->_plack_req->header( $header_name ); } -sub get_method -{ - my ($self, $data) = @_; - - my $full_name = $data->{method}; - +sub get_method { + my ( $self, $data ) = @_; + + my $full_name = $data->{ method }; + $full_name =~ /^(\S+)\.([^\.]+)$/; - my ($package, $method) = ($1, $2); - - if (!$package || !$method) { - $self->exception('NoSuchMethod', - "'$full_name' is not a valid method. It must" - . " contain a package name, followed by a period," - . " followed by a method name."); + my ( $package, $method ) = ( $1, $2 ); + + if ( !$package || !$method ) { + $self->exception( 'NoSuchMethod', + "'$full_name' is not a valid method. It must" + . " contain a package name, followed by a period," + . " followed by a method name." + ); } - if (!$self->valid_methods->{$method}) - { - $self->exception('NoSuchMethod', - "'$method' is not a valid method in service ${service_name}."); + if ( !$self->valid_methods->{ $method } ) { + $self->exception( 'NoSuchMethod', + "'$method' is not a valid method in service ${service_name}." + ); } - - my $inst = $self->instance_dispatch->{$package}; + + my $inst = $self->instance_dispatch->{ $package }; my $module; - if ($inst) - { - $module = $inst; + if ( $inst ) { + $module = $inst; } - else - { - $module = $self->get_module($package); - if (!$module) { - $self->exception('NoSuchMethod', - "There is no method package named '$package'."); - } - - Class::MOP::load_class($module); + else { + $module = $self->get_module( $package ); + if ( ! $module ) { + $self->exception( 'NoSuchMethod', + "There is no method package named '$package'." + ); + } + + Class::MOP::load_class( $module ); } - if (!$module->can($method)) { - $self->exception('NoSuchMethod', - "There is no method named '$method' in the" - . " '$package' package."); + if ( !$module->can( $method ) ) { + $self->exception( 'NoSuchMethod', + "There is no method named '$method' in the" . " '$package' package." + ); } - + return { module => $module, method => $method, modname => $package }; } sub handle_input_cli { - my ($self, $input) = @_; + my ( $self, $input ) = @_; my $retval; eval { - #my $input_info = $self->check_input($input); - #my $input_object = $self->decode_input_to_object($input); - $self->parser->json->utf8(utf8::is_utf8($input) ? 0 : 1); - my $input_object = $self->parser->json_to_call($input); - my $data = $self->input_object_to_data($input_object); - #$self->fix_data($data, $input_info); - my $method_info = $self->get_method($data); - my $method_result = $self->call_method($data, $method_info); - my $collapsed_result = $self->collapse_result($method_result); - my $output_object = $self->output_data_to_object($collapsed_result); - $retval = $self->parser->return_to_json($output_object); + #my $input_info = $self->check_input( $input ); + #my $input_object = $self->decode_input_to_object( $input ); + $self->parser->json->utf8( utf8::is_utf8( $input ) ? 0 : 1 ); + my $input_object = $self->parser->json_to_call( $input ); + my $data = $self->input_object_to_data( $input_object ); + + #$self->fix_data( $data, $input_info ); + my $method_info = $self->get_method( $data ); + my $method_result = $self->call_method( $data, $method_info ); + my $collapsed_result = $self->collapse_result( $method_result ); + my $output_object = $self->output_data_to_object( $collapsed_result ); + $retval = $self->parser->return_to_json( $output_object ); }; + return $retval if defined $retval; + # The only way that we can get here is by something throwing an error. - return $self->handle_error_cli($@); + return $self->handle_error_cli( $@ ); } sub handle_error_cli { - my ($self, $error) = @_; + my ( $self, $error ) = @_; my $output; eval { my %error_params; - if (ref($error) eq 'HASH') { - %error_params = %{$error}; - if(defined($error_params${empty_escaper}{context})) { + if ( ref( $error ) eq 'HASH' ) { + %error_params = %$error; + if ( defined ( $error_params${empty_escaper}{ context } ) ) { # my @errlines; # $errlines[0] = $error_params${empty_escaper}{message}; # push @errlines, split("\n", $error_params${empty_escaper}{data}); - # $self->log($Bio::KBase::Log::ERR, $error_params${empty_escaper}{context}, \@errlines); - delete $error_params${empty_escaper}{context}; + # $self->log( $Bio::KBase::Log::ERR, $error_params${empty_escaper}{context}, \@errlines ); + delete $error_params${empty_escaper}{ context }; } - } else { - unless (blessed $error and $error->isa('RPC::Any::Exception')) { - $error = RPC::Any::Exception::PerlError->new(message => $error); + } + else { + unless ( blessed $error and $error->isa( 'RPC::Any::Exception' ) ) { + $error = RPC::Any::Exception::PerlError->new( message => $error ); } %error_params = ( message => $error->message, @@ -472,14 +523,16 @@ sub handle_error_cli { ); } my $json_error; - if ($self->_last_call) { - $json_error = $self->_last_call->return_error(%error_params); + if ( $self->_last_call ) { + $json_error = $self->_last_call->return_error( %error_params ); } else { - $json_error = $self->_default_error(%error_params); + $json_error = $self->_default_error( %error_params ); } - $output = $self->parser->return_to_json($json_error); + $output = $self->parser->return_to_json( $json_error ); }; + return $output if $output; + die "$error\n\nAlso, an error was encountered while trying to send" . " this error: $@\n"; } @@ -487,6 +540,7 @@ sub handle_error_cli { package ${ctx_pkg}; use strict; +use warnings; =head1 NAME @@ -503,274 +557,242 @@ is available via $context->client_ip. use base 'Class::Accessor'; -__PACKAGE__->mk_accessors(qw(user_id client_ip authenticated token - module method call_id hostname stderr - provenance)); - -sub new -{ - my($class, $logger, %opts) = @_; - - my $self = { - %opts, +__PACKAGE__->mk_accessors( qw( + user_id client_ip authenticated token + module method call_id hostname stderr + provenance +) ); + + +sub new { + my ( $class, $logger, %opts ) = @_; + + my $self = { %opts, }; + chomp( $self->{ hostname } = `hostname` ); + $self->{ hostname } ||= 'unknown-host'; + $self->{ _logger } = $logger; + $self->{ _debug_levels } = { + 7 => 1, + 8 => 1, + 9 => 1, + DEBUG => 1, + DEBUG2 => 1, + DEBUG3 => 1, }; - chomp($self->{hostname} = `hostname`); - $self->{hostname} ||= 'unknown-host'; - $self->{_logger} = $logger; - $self->{_debug_levels} = {7 => 1, 8 => 1, 9 => 1, - 'DEBUG' => 1, 'DEBUG2' => 1, 'DEBUG3' => 1}; return bless $self, $class; } -sub _get_user -{ - my ($self) = @_; - return defined($self->user_id()) ? $self->user_id(): undef; +sub _get_user { + my ( $self ) = @_; + return defined( $self->user_id() ) ? $self->user_id() : undef; } -sub _log -{ - my ($self, $level, $message) = @_; - $self->{_logger}->log_message($level, $message, $self->_get_user(), - $self->module(), $self->method(), $self->call_id(), - $self->client_ip()); +sub _log { + my ( $self, $level, $message ) = @_; + $self->{ _logger }->log_message( $level, $message, $self->_get_user(), + $self->module(), $self->method(), $self->call_id(), $self->client_ip() ); } -sub log_err -{ - my ($self, $message) = @_; - $self->_log($Bio::KBase::Log::ERR, $message); +sub log_err { + my ( $self, $message ) = @_; + $self->_log( $Bio::KBase::Log::ERR, $message ); } -sub log_info -{ - my ($self, $message) = @_; - $self->_log($Bio::KBase::Log::INFO, $message); +sub log_info { + my ( $self, $message ) = @_; + $self->_log( $Bio::KBase::Log::INFO, $message ); } -sub log_debug -{ - my ($self, $message, $level) = @_; - if(!defined($level)) { +sub log_debug { + my ( $self, $message, $level ) = @_; + if ( !defined( $level ) ) { $level = 1; } - if($self->{_debug_levels}->{$level}) { - } else { - if ($level =~ /\D/ || $level < 1 || $level > 3) { + if ( $self->{ _debug_levels }->{ $level } ) { + } + else { + if ( $level =~ /\D/ || $level < 1 || $level > 3 ) { die "Invalid log level: $level"; } $level += 6; } - $self->_log($level, $message); + $self->_log( $level, $message ); } -sub set_log_level -{ - my ($self, $level) = @_; - $self->{_logger}->set_log_level($level); +sub set_log_level { + my ( $self, $level ) = @_; + $self->{ _logger }->set_log_level( $level ); } -sub get_log_level -{ - my ($self) = @_; - return $self->{_logger}->get_log_level(); +sub get_log_level { + my ( $self ) = @_; + return $self->{ _logger }->get_log_level(); } -sub clear_log_level -{ - my ($self) = @_; - $self->{_logger}->clear_user_log_level(); +sub clear_log_level { + my ( $self ) = @_; + $self->{ _logger }->clear_user_log_level(); } + package ${stderr_pkg}; use strict; +use warnings; use POSIX; use Time::HiRes 'gettimeofday'; -sub new -{ - my($class, $ctx, $get_time) = @_; - my $self = { - get_time => $get_time, - }; - my $dest = $ENV${empty_escaper}{KBRPC_ERROR_DEST} if exists $ENV${empty_escaper}{KBRPC_ERROR_DEST}; - my $tag = $ENV${empty_escaper}{KBRPC_TAG} if exists $ENV${empty_escaper}{KBRPC_TAG}; - my ($t, $us) = gettimeofday(); - $us = sprintf("%06d", $us); - my $ts = strftime("%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t); +sub new { + my ( $class, $ctx, $get_time ) = @_; + my $self = { get_time => $get_time, }; - my $name = join(".", $ctx->module, $ctx->method, $ctx->hostname, $ts); + my $dest = $ENV${empty_escaper}{ KBRPC_ERROR_DEST } + if exists $ENV${empty_escaper}{ KBRPC_ERROR_DEST }; + my $tag = $ENV${empty_escaper}{ KBRPC_TAG } + if exists $ENV${empty_escaper}{ KBRPC_TAG }; + my ( $t, $us ) = gettimeofday(); + $us = sprintf("%06d", $us); + my $ts = strftime("%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t); - if ($dest && $dest =~ m,^/,) - { - # - # File destination - # - my $fh; - - if ($tag) - { - $tag =~ s,/,_,g; - $dest = "$dest/$tag"; - if (! -d $dest) - { - mkdir($dest); - } - } - if (open($fh, ">", "$dest/$name")) - { - $self->{file} = "$dest/$name"; - $self->{dest} = $fh; - } - else - { - warn "Cannot open log file $dest/$name: $${empty_escaper}!"; - } - } - else - { - # - # Log to string. - # - my $stderr; - $self->{dest} = \$stderr; + my $name = join ".", $ctx->module, $ctx->method, $ctx->hostname, $ts; + + if ( $dest && $dest =~ m!^/! ) { + # + # File destination + # + my $fh; + if ($tag) { + $tag =~ s,/,_,g; + $dest = "$dest/$tag"; + mkdir($dest) if ! -d $dest; + } + + if ( open($fh, ">", "$dest/$name") ) { + $self->{file} = "$dest/$name"; + $self->{dest} = $fh; + } + else { + warn "Cannot open log file $dest/$name: $${empty_escaper}!"; + } + } + else { + # + # Log to string. + # + my $stderr; + $self->{ dest } = \$stderr; } - + bless $self, $class; - for my $e (sort { $a cmp $b } keys %ENV) - { - $self->log_cmd($e, $ENV${empty_escaper}{$e}); + for my $e ( sort { $a cmp $b } keys %ENV ) { + $self->log_cmd( $e, $ENV${empty_escaper}{ $e } ); } return $self; } -sub redirect -{ - my($self) = @_; - if ($self->{dest}) - { - return("2>", $self->{dest}); - } - else - { - return (); - } +sub redirect { + my ( $self ) = @_; + + return () unless $self->{ dest }; + + return ( "2>", $self->{ dest } ); } -sub redirect_both -{ - my($self) = @_; - if ($self->{dest}) - { - return(">&", $self->{dest}); - } - else - { - return (); - } +sub redirect_both { + my ( $self ) = @_; + + return () unless $self->{ dest }; + + return ( ">&", $self->{ dest } ); } -sub timestamp -{ - my($self) = @_; - my ($t, $us) = $self->{get_time}->(); - $us = sprintf("%06d", $us); - my $ts = strftime("%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t); +sub timestamp { + my ( $self ) = @_; + my ( $t, $us ) = $self->{ get_time }->(); + $us = sprintf( "%06d", $us ); + my $ts = strftime( "%Y-%m-%dT%H:%M:%S.${us}Z", gmtime $t ); return $ts; } -sub log -{ - my($self, $str) = @_; - my $d = $self->{dest}; +sub log { + my ( $self, $str ) = @_; + my $d = $self->{ dest }; my $ts = $self->timestamp(); - if (ref($d) eq 'SCALAR') - { - $$d .= "[$ts] " . $str . "\n"; - return 1; + if ( ref( $d ) eq 'SCALAR' ) { + $$d .= "[$ts] " . $str . "\n"; + return 1; } - elsif ($d) - { - print $d "[$ts] " . $str . "\n"; - return 1; + elsif ( $d ) { + print $d "[$ts] " . $str . "\n"; + return 1; } return 0; } -sub log_cmd -{ - my($self, @cmd) = @_; - my $d = $self->{dest}; +sub log_cmd { + my ( $self, @cmd ) = @_; + my $d = $self->{ dest }; my $str; my $ts = $self->timestamp(); - if (ref($cmd[0])) - { - $str = join(" ", @{$cmd[0]}); + if ( ref( $cmd[ 0 ] ) ) { + $str = join( " ", @{ $cmd[ 0 ] } ); } - else - { - $str = join(" ", @cmd); + else { + $str = join( " ", @cmd ); } - if (ref($d) eq 'SCALAR') - { - $$d .= "[$ts] " . $str . "\n"; + if ( ref( $d ) eq 'SCALAR' ) { + $$d .= "[$ts] " . $str . "\n"; } - elsif ($d) - { - print $d "[$ts] " . $str . "\n"; + elsif ( $d ) { + print $d "[$ts] " . $str . "\n"; } - + } -sub dest -{ - my($self) = @_; - return $self->{dest}; +sub dest { + my ( $self ) = @_; + return $self->{ dest }; } -sub text_value -{ - my($self) = @_; - if (ref($self->{dest}) eq 'SCALAR') - { - my $r = $self->{dest}; - return $$r; - } - else - { - return $self->{file}; +sub text_value { + my ( $self ) = @_; + if ( ref( $self->{ dest } ) eq 'SCALAR' ) { + my $r = $self->{ dest }; + return $$r; } + return $self->{ file }; } -unless (caller) { - my($input_file,$output_file,$token) = @ARGV; +unless ( caller ) { + my ( $input_file, $output_file, $token ) = @ARGV; my @dispatch; #foreach( $module in $modules ) { use ${module.impl_package_name}; my $obj = ${module.impl_package_name}->new; - push(@dispatch, '${module.module_name}' => $obj); + push( @dispatch, '${module.module_name}' => $obj ); } #end my %headers = ( - "Authorization" => $token, - "CLI" => "1" + Authorization => $token, + CLI => "1", ); my $server = ${server_package_name}->new( instance_dispatch => { @dispatch }, - allow_get => 0, - local_headers => \%headers); - open(my $fih, '<', $input_file) or die $${empty_escaper}!; + allow_get => 0, + local_headers => \%headers); + + open ( my $fih, '<', $input_file ) or die $${empty_escaper}!; my $input = ''; while (<$fih>) { $input .= $_; } close $fih; - my $output = $server->handle_input_cli($input); - open(my $foh, '>', $output_file) or die "Could not open file '$output_file' $${empty_escaper}!"; + my $output = $server->handle_input_cli( $input ); + open ( my $foh, '>', $output_file ) + or die "Could not open file '$output_file': $${empty_escaper}!"; print $foh $output; close $foh; } diff --git a/src/java/us/kbase/templates/python_impl.vm.properties b/src/java/us/kbase/templates/python_impl.vm.properties index ea22c9f7..ac5a3c5b 100644 --- a/src/java/us/kbase/templates/python_impl.vm.properties +++ b/src/java/us/kbase/templates/python_impl.vm.properties @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -#BEGIN_HEADER -${module.py_module_header}#END_HEADER +# BEGIN_HEADER +${module.py_module_header}# END_HEADER #set( $specToPy = {} ) #set( $void = $specToPy.put("list", "list") ) #set( $void = $specToPy.put("tuple", "list") ) @@ -31,14 +31,14 @@ class ${module.module_name}: GIT_URL = "${module.git_url}" GIT_COMMIT_HASH = "${module.git_commit_hash}" - #BEGIN_CLASS_HEADER -${module.py_module_class_header} #END_CLASS_HEADER + # BEGIN_CLASS_HEADER +${module.py_module_class_header} # END_CLASS_HEADER # config contains contents of config file in a hash or None if it couldn't # be found def __init__(self, config): - #BEGIN_CONSTRUCTOR -${module.py_module_constructor} #END_CONSTRUCTOR + # BEGIN_CONSTRUCTOR +${module.py_module_constructor} # END_CONSTRUCTOR pass #set( $status_in_kidl = false ) @@ -74,8 +74,8 @@ ${module.py_module_constructor} #END_CONSTRUCTOR #if( $method.ret_count > 0 ) # return variables are: $display.join($retlist, ", ") #end - #BEGIN ${method.name} -${method.py_user_code} #[[#END]]# ${method.name} + # BEGIN ${method.name} +${method.py_user_code} #[[# END]]# ${method.name} #if( $method.ret_count > 0 ) # At some point might do deeper type checking... @@ -83,8 +83,9 @@ ${method.py_user_code} #[[#END]]# ${method.name} #set( $spectype = $ret.baretype ) #set( $type = $specToPy.get($spectype) ) if not isinstance(${ret.name}, ${type}): - raise ValueError('Method ${method.name} return value ' + - '${ret.name} is not type ${type} as required.') + raise ValueError('Method ${method.name} ' + + 'return value ${ret.name} ' + + 'is not type ${type} as required.') #end # return the results return [$display.join($retlist, ', ')] @@ -93,8 +94,9 @@ ${method.py_user_code} #[[#END]]# ${method.name} #end #end #if( !$status_in_kidl ) + def status(self, ctx): - #BEGIN_STATUS + # BEGIN_STATUS #if( ${module.py_module_status} ) ${module.py_module_status}#else returnVal = {'state': "OK", @@ -103,6 +105,6 @@ ${module.py_module_status}#else 'git_url': self.GIT_URL, 'git_commit_hash': self.GIT_COMMIT_HASH} #end - #END_STATUS + # END_STATUS return [returnVal] #end diff --git a/src/java/us/kbase/templates/python_server.vm.properties b/src/java/us/kbase/templates/python_server.vm.properties index 136f476a..c5d7d177 100644 --- a/src/java/us/kbase/templates/python_server.vm.properties +++ b/src/java/us/kbase/templates/python_server.vm.properties @@ -18,7 +18,10 @@ from jsonrpcbase import ServerError as JSONServerError from biokbase import log #if( $authenticated ) -from ${modules[0].pypackage}authclient import KBaseAuth as _KBaseAuth +try: + from ${modules[0].pypackage}authclient import KBaseAuth as _KBaseAuth +except ImportError: + from installed_clients.authclient import KBaseAuth as _KBaseAuth #end try: @@ -51,6 +54,7 @@ def get_config(): retconfig[nameval[0]] = nameval[1] return retconfig + config = get_config() #set( $specToPy = {} ) @@ -130,7 +134,11 @@ class JSONRPCServiceCustom(JSONRPCService): newerr = JSONServerError() newerr.trace = traceback.format_exc() if len(e.args) == 1: - newerr.data = repr(e.args[0]) + # repr adds single quotes around string arguments which is not what we want. + if type(e.args[0]) == str: + newerr.data = e.args[0] + else: + newerr.data = repr(e.args[0]) else: newerr.data = repr(e.args) raise newerr @@ -146,12 +154,6 @@ class JSONRPCServiceCustom(JSONRPCService): debugging purposes. """ rdata = jsondata - # we already deserialize the json string earlier in the server code, no - # need to do it again -# try: -# rdata = json.loads(jsondata) -# except ValueError: -# raise ParseError # set some default values for error handling request = self._get_default_vals() @@ -526,6 +528,7 @@ class Application(object): 60) return "%s%+02d:%02d" % (dtnow.isoformat(), hh, mm) + application = Application() # This is the uwsgi application dictionary. On startup uwsgi will look @@ -639,6 +642,7 @@ def process_async_cli(input_file_path, output_file_path, token): f.write(json.dumps(resp, cls=JSONObjectEncoder)) return exit_code + if __name__ == "__main__": if (len(sys.argv) >= 3 and len(sys.argv) <= 4 and os.path.isfile(sys.argv[1])): @@ -668,7 +672,3 @@ if __name__ == "__main__": assert False, "unhandled option" start_server(host=host, port=port) -# print("Listening on port %s" % port) -# httpd = make_server( host, port, application) -# -# httpd.serve_forever() diff --git a/test_scripts/perl/lib/TestImpl.pm b/test_scripts/perl/lib/TestImpl.pm index 8efcf851..98b30226 100644 --- a/test_scripts/perl/lib/TestImpl.pm +++ b/test_scripts/perl/lib/TestImpl.pm @@ -1,8 +1,9 @@ package TestImpl; use strict; use Bio::KBase::Exceptions; + # Use Semantic Versioning (2.0.0-rc.1) -# http://semver.org +# http://semver.org our $VERSION = "0.1.0"; =head1 NAME @@ -18,18 +19,16 @@ Basic #BEGIN_HEADER #END_HEADER -sub new -{ - my($class, @args) = @_; - my $self = { - }; +sub new { + my ( $class, @args ) = @_; + my $self = {}; bless $self, $class; + #BEGIN_CONSTRUCTOR #END_CONSTRUCTOR - if ($self->can('_init_instance')) - { - $self->_init_instance(); + if ( $self->can( '_init_instance' ) ) { + $self->_init_instance(); } return $self; } @@ -74,37 +73,45 @@ $return is an int =cut -sub one_simple_param -{ +sub one_simple_param { my $self = shift; - my($val) = @_; + my ( $val ) = @_; my @_bad_arguments; - (!ref($val)) or push(@_bad_arguments, "Invalid type for argument \"val\" (value was \"$val\")"); - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to one_simple_param:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'one_simple_param'); + ( !ref( $val ) ) + or + push( @_bad_arguments, "Invalid type for argument \"val\" (value was \"$val\")" ); + if ( @_bad_arguments ) { + my $msg = "Invalid arguments passed to one_simple_param:\n" + . join( "", map { "\t$_\n" } @_bad_arguments ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'one_simple_param' + ); } my $ctx = $basicsrvServer::CallContext; - my($return); + my ( $return ); + #BEGIN one_simple_param $return = $val; + #END one_simple_param my @_bad_returns; - (!ref($return)) or push(@_bad_returns, "Invalid type for return variable \"return\" (value was \"$return\")"); - if (@_bad_returns) { - my $msg = "Invalid returns passed to one_simple_param:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'one_simple_param'); + ( !ref( $return ) ) + or push( @_bad_returns, + "Invalid type for return variable \"return\" (value was \"$return\")" ); + if ( @_bad_returns ) { + my $msg = "Invalid returns passed to one_simple_param:\n" + . join( "", map { "\t$_\n" } @_bad_returns ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'one_simple_param' + ); } - return($return); + return ( $return ); } - - - =head2 nothing $obj->nothing() @@ -137,19 +144,16 @@ sub one_simple_param =cut -sub nothing -{ +sub nothing { my $self = shift; my $ctx = $basicsrvServer::CallContext; + #BEGIN nothing #END nothing - return(); + return (); } - - - =head2 one_complex_param $return = $obj->one_complex_param($val2) @@ -208,37 +212,44 @@ double_number is a float =cut -sub one_complex_param -{ +sub one_complex_param { my $self = shift; - my($val2) = @_; + my ( $val2 ) = @_; my @_bad_arguments; - (ref($val2) eq 'HASH') or push(@_bad_arguments, "Invalid type for argument \"val2\" (value was \"$val2\")"); - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to one_complex_param:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'one_complex_param'); + ( ref( $val2 ) eq 'HASH' ) + or push( @_bad_arguments, + "Invalid type for argument \"val2\" (value was \"$val2\")" ); + if ( @_bad_arguments ) { + my $msg = "Invalid arguments passed to one_complex_param:\n" + . join( "", map { "\t$_\n" } @_bad_arguments ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'one_complex_param' + ); } my $ctx = $basicsrvServer::CallContext; - my($return); + my ( $return ); + #BEGIN one_complex_param - $return = {}; #$val2; - #END one_complex_param + $return = {}; #$val2; + #END one_complex_param my @_bad_returns; - (ref($return) eq 'HASH') or push(@_bad_returns, "Invalid type for return variable \"return\" (value was \"$return\")"); - if (@_bad_returns) { - my $msg = "Invalid returns passed to one_complex_param:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'one_complex_param'); + ( ref( $return ) eq 'HASH' ) + or push( @_bad_returns, + "Invalid type for return variable \"return\" (value was \"$return\")" ); + if ( @_bad_returns ) { + my $msg = "Invalid returns passed to one_complex_param:\n" + . join( "", map { "\t$_\n" } @_bad_returns ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'one_complex_param' + ); } - return($return); + return ( $return ); } - - - =head2 many_simple_params $return_1, $return_2, $return_3, $return_4 = $obj->many_simple_params($val1, $val2, $val3, $val4) @@ -295,46 +306,66 @@ $return_4 is a reference to a list containing 2 items: =cut -sub many_simple_params -{ +sub many_simple_params { my $self = shift; - my($val1, $val2, $val3, $val4) = @_; + my ( $val1, $val2, $val3, $val4 ) = @_; my @_bad_arguments; - (!ref($val1)) or push(@_bad_arguments, "Invalid type for argument \"val1\" (value was \"$val1\")"); - (!ref($val2)) or push(@_bad_arguments, "Invalid type for argument \"val2\" (value was \"$val2\")"); - (!ref($val3)) or push(@_bad_arguments, "Invalid type for argument \"val3\" (value was \"$val3\")"); - (ref($val4) eq 'ARRAY') or push(@_bad_arguments, "Invalid type for argument \"val4\" (value was \"$val4\")"); - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to many_simple_params:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'many_simple_params'); + ( !ref( $val1 ) ) + or push( @_bad_arguments, + "Invalid type for argument \"val1\" (value was \"$val1\")" ); + ( !ref( $val2 ) ) + or push( @_bad_arguments, + "Invalid type for argument \"val2\" (value was \"$val2\")" ); + ( !ref( $val3 ) ) + or push( @_bad_arguments, + "Invalid type for argument \"val3\" (value was \"$val3\")" ); + ( ref( $val4 ) eq 'ARRAY' ) + or push( @_bad_arguments, + "Invalid type for argument \"val4\" (value was \"$val4\")" ); + if ( @_bad_arguments ) { + my $msg = "Invalid arguments passed to many_simple_params:\n" + . join( "", map { "\t$_\n" } @_bad_arguments ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'many_simple_params' + ); } my $ctx = $basicsrvServer::CallContext; - my($return_1, $return_2, $return_3, $return_4); + my ( $return_1, $return_2, $return_3, $return_4 ); + #BEGIN many_simple_params $return_1 = $val1; $return_2 = $val2; $return_3 = $val3; $return_4 = $val4; + #END many_simple_params my @_bad_returns; - (!ref($return_1)) or push(@_bad_returns, "Invalid type for return variable \"return_1\" (value was \"$return_1\")"); - (!ref($return_2)) or push(@_bad_returns, "Invalid type for return variable \"return_2\" (value was \"$return_2\")"); - (!ref($return_3)) or push(@_bad_returns, "Invalid type for return variable \"return_3\" (value was \"$return_3\")"); - (ref($return_4) eq 'ARRAY') or push(@_bad_returns, "Invalid type for return variable \"return_4\" (value was \"$return_4\")"); - if (@_bad_returns) { - my $msg = "Invalid returns passed to many_simple_params:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'many_simple_params'); + ( !ref( $return_1 ) ) + or push( @_bad_returns, + "Invalid type for return variable \"return_1\" (value was \"$return_1\")" ); + ( !ref( $return_2 ) ) + or push( @_bad_returns, + "Invalid type for return variable \"return_2\" (value was \"$return_2\")" ); + ( !ref( $return_3 ) ) + or push( @_bad_returns, + "Invalid type for return variable \"return_3\" (value was \"$return_3\")" ); + ( ref( $return_4 ) eq 'ARRAY' ) + or push( @_bad_returns, + "Invalid type for return variable \"return_4\" (value was \"$return_4\")" ); + if ( @_bad_returns ) { + my $msg = "Invalid returns passed to many_simple_params:\n" + . join( "", map { "\t$_\n" } @_bad_returns ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'many_simple_params' + ); } - return($return_1, $return_2, $return_3, $return_4); + return ( $return_1, $return_2, $return_3, $return_4 ); } - - - =head2 many_complex_params $return_1, $return_2 = $obj->many_complex_params($simple_val, $complex_val) @@ -397,40 +428,52 @@ complex_struct is a reference to a hash where the following keys are defined: =cut -sub many_complex_params -{ +sub many_complex_params { my $self = shift; - my($simple_val, $complex_val) = @_; + my ( $simple_val, $complex_val ) = @_; my @_bad_arguments; - (ref($simple_val) eq 'HASH') or push(@_bad_arguments, "Invalid type for argument \"simple_val\" (value was \"$simple_val\")"); - (ref($complex_val) eq 'HASH') or push(@_bad_arguments, "Invalid type for argument \"complex_val\" (value was \"$complex_val\")"); - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to many_complex_params:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'many_complex_params'); + ( ref( $simple_val ) eq 'HASH' ) + or push( @_bad_arguments, + "Invalid type for argument \"simple_val\" (value was \"$simple_val\")" ); + ( ref( $complex_val ) eq 'HASH' ) + or push( @_bad_arguments, + "Invalid type for argument \"complex_val\" (value was \"$complex_val\")" ); + if ( @_bad_arguments ) { + my $msg = "Invalid arguments passed to many_complex_params:\n" + . join( "", map { "\t$_\n" } @_bad_arguments ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'many_complex_params' + ); } my $ctx = $basicsrvServer::CallContext; - my($return_1, $return_2); + my ( $return_1, $return_2 ); + #BEGIN many_complex_params $return_1 = $simple_val; $return_2 = $complex_val; + #END many_complex_params my @_bad_returns; - (ref($return_1) eq 'HASH') or push(@_bad_returns, "Invalid type for return variable \"return_1\" (value was \"$return_1\")"); - (ref($return_2) eq 'HASH') or push(@_bad_returns, "Invalid type for return variable \"return_2\" (value was \"$return_2\")"); - if (@_bad_returns) { - my $msg = "Invalid returns passed to many_complex_params:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'many_complex_params'); + ( ref( $return_1 ) eq 'HASH' ) + or push( @_bad_returns, + "Invalid type for return variable \"return_1\" (value was \"$return_1\")" ); + ( ref( $return_2 ) eq 'HASH' ) + or push( @_bad_returns, + "Invalid type for return variable \"return_2\" (value was \"$return_2\")" ); + if ( @_bad_returns ) { + my $msg = "Invalid returns passed to many_complex_params:\n" + . join( "", map { "\t$_\n" } @_bad_returns ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'many_complex_params' + ); } - return($return_1, $return_2); + return ( $return_1, $return_2 ); } - - - =head2 with_auth $return = $obj->with_auth($val1) @@ -467,38 +510,46 @@ $return is an int =cut -sub with_auth -{ +sub with_auth { my $self = shift; - my($val1) = @_; + my ( $val1 ) = @_; my @_bad_arguments; - (!ref($val1)) or push(@_bad_arguments, "Invalid type for argument \"val1\" (value was \"$val1\")"); - if (@_bad_arguments) { - my $msg = "Invalid arguments passed to with_auth:\n" . join("", map { "\t$_\n" } @_bad_arguments); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'with_auth'); + ( !ref( $val1 ) ) + or push( @_bad_arguments, + "Invalid type for argument \"val1\" (value was \"$val1\")" ); + if ( @_bad_arguments ) { + my $msg = "Invalid arguments passed to with_auth:\n" + . join( "", map { "\t$_\n" } @_bad_arguments ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'with_auth' + ); } my $ctx = $basicsrvServer::CallContext; - my($return); + my ( $return ); + #BEGIN with_auth $return = $val1; + #END with_auth my @_bad_returns; - (!ref($return)) or push(@_bad_returns, "Invalid type for return variable \"return\" (value was \"$return\")"); - if (@_bad_returns) { - my $msg = "Invalid returns passed to with_auth:\n" . join("", map { "\t$_\n" } @_bad_returns); - Bio::KBase::Exceptions::ArgumentValidationError->throw(error => $msg, - method_name => 'with_auth'); + ( !ref( $return ) ) + or push( @_bad_returns, + "Invalid type for return variable \"return\" (value was \"$return\")" ); + if ( @_bad_returns ) { + my $msg = "Invalid returns passed to with_auth:\n" + . join( "", map { "\t$_\n" } @_bad_returns ); + Bio::KBase::Exceptions::ArgumentValidationError->throw( + error => $msg, + method_name => 'with_auth' + ); } - return($return); + return ( $return ); } - - - -=head2 version +=head2 version $return = $obj->version() @@ -666,3 +717,4 @@ large_prop4 has a value which is a reference to a hash where the key is a string =cut 1; + diff --git a/test_scripts/perl/run-client.pl b/test_scripts/perl/run-client.pl index 08304d91..46a9d6e9 100644 --- a/test_scripts/perl/run-client.pl +++ b/test_scripts/perl/run-client.pl @@ -1,9 +1,6 @@ # # Automatic test rig for KBase perl client generation. # -# -# -# use strict; use warnings; @@ -11,18 +8,17 @@ use Data::Dumper; use Getopt::Long; -my $DESCRIPTION = -" +my $DESCRIPTION = " USAGE perl test-client.pl [options] - + DESCRIPTION Conducts an automatic test of a perl client against the specified endpoint using methods and parameters indicated in the tests config file. - + --endpoint [url] endpoint of the server to test --token [token] token for testing authenticated calls - --asyncchecktime [ms] time client waits every cycle of checking state of async methods + --asyncchecktime [ms] time client waits every cycle of checking state of async methods --package [package] package with client module --method [method] method to run --input [filename] input file with parameters in JSON @@ -30,7 +26,7 @@ --error [filename] output file with text error message -h, --help display this help message, ignore all arguments "; - + # first parse options to get the testconfig file my $verbose; my $endpoint; @@ -45,104 +41,109 @@ my $help; -my $opt = GetOptions ( - "verbose|v" => \$verbose, - "endpoint=s" => \$endpoint, - "token=s" => \$token, - "asyncchecktime=i" => \$async_job_check_time_ms, - "help|h" => \$help, - "package=s" => \$client_module, - "method=s" => \$method, - "input=s" => \$input_filename, - "output=s" => \$output_filename, - "error=s" => \$error_filename, - ); - -if ($help) { - print $DESCRIPTION."\n"; +my $opt = GetOptions( + "verbose|v" => \$verbose, + "endpoint=s" => \$endpoint, + "token=s" => \$token, + "asyncchecktime=i" => \$async_job_check_time_ms, + "help|h" => \$help, + "package=s" => \$client_module, + "method=s" => \$method, + "input=s" => \$input_filename, + "output=s" => \$output_filename, + "error=s" => \$error_filename, +); + +if ( $help ) { + print $DESCRIPTION. "\n"; exit 0; } - # endpoint must be defined -if (!$endpoint) { - fail("endpoint parameter must be defined"); +if ( !$endpoint ) { + fail( "endpoint parameter must be defined" ); exit 1; } # package must be defined -if (!$client_module) { - fail("package parameter must be defined"); +if ( !$client_module ) { + fail( "package parameter must be defined" ); exit 1; } # method must be defined -if (!$method) { - fail("method parameter must be defined"); +if ( !$method ) { + fail( "method parameter must be defined" ); exit 1; } # input must be defined -if (!$input_filename) { - fail("input parameter must be defined"); +if ( !$input_filename ) { + fail( "input parameter must be defined" ); exit 1; } # output must be defined -if (!$output_filename) { - fail("output parameter must be defined"); +if ( !$output_filename ) { + fail( "output parameter must be defined" ); exit 1; } # error must be defined -if (!$error_filename) { - fail("error parameter must be defined"); +if ( !$error_filename ) { + fail( "error parameter must be defined" ); exit 1; } #parse the tests -open(my $fh, "<", $input_filename); -my $input_string=''; -while (my $line = <$fh>) { +open( my $fh, "<", $input_filename ); +my $input_string = ''; +while ( my $line = <$fh> ) { chomp $line; $input_string .= $line; } -close($fh); -my $params = JSON->new->decode($input_string); +close( $fh ); +my $params = JSON->new->decode( $input_string ); my $json = JSON->new->canonical; -eval("use $client_module;"); +eval( "use $client_module;" ); + #instantiate an authenticated client and a nonauthenticated client my $client; -if($token) { - if ($async_job_check_time_ms) { - $client=$client_module->new($endpoint, token=>$token, async_job_check_time_ms=>$async_job_check_time_ms); - } else { - $client=$client_module->new($endpoint, token=>$token); +if ( $token ) { + if ( $async_job_check_time_ms ) { + $client = $client_module->new( + $endpoint, + token => $token, + async_job_check_time_ms => $async_job_check_time_ms + ); + } + else { + $client = $client_module->new( $endpoint, token => $token ); } -} else { - $client = $client_module->new($endpoint,ignore_kbase_config=>1); +} +else { + $client = $client_module->new( $endpoint, ignore_kbase_config => 1 ); } # loop over each test case and run it against the server. We create a new client instance # for each test - + my @result; no strict "refs"; -eval { - @result = $client->$method(@{$params}); -}; -if($@) { +eval { @result = $client->$method( @{ $params } ); }; +if ( $@ ) { print "caught: $@"; my $returnedErrorMsg = "$@"; my $OUTFILE; open $OUTFILE, '>>', $error_filename; print { $OUTFILE } $returnedErrorMsg; close $OUTFILE; -} else { - my $serialized_result = $json->encode(\@result); +} +else { + my $serialized_result = $json->encode( \@result ); my $OUTFILE; open $OUTFILE, '>>', $output_filename; print { $OUTFILE } $serialized_result; @@ -150,6 +151,3 @@ } - - - diff --git a/test_scripts/perl/test-client.pl b/test_scripts/perl/test-client.pl index 8d6769b4..db3caa3e 100644 --- a/test_scripts/perl/test-client.pl +++ b/test_scripts/perl/test-client.pl @@ -1,9 +1,6 @@ # # Automatic test rig for KBase perl client generation. # -# -# -# use strict; use warnings; @@ -12,22 +9,21 @@ use Test::More; use Getopt::Long; -my $DESCRIPTION = -" +my $DESCRIPTION = " USAGE perl test-client.pl [options] - + DESCRIPTION Conducts an automatic test of a perl client against the specified endpoint using methods and parameters indicated in the tests config file. - + --tests [filename] config file for tests --endpoint [url] endpoint of the server to test --token [token] token for testing authenticated calls - --asyncchecktime [ms] time client waits every cycle of checking state of async methods + --asyncchecktime [ms] time client waits every cycle of checking state of async methods -h, --help display this help message, ignore all arguments "; - + # first parse options to get the testconfig file my $tests_filename; my $verbose; @@ -38,202 +34,224 @@ my $help; -my $opt = GetOptions ( - "tests=s" => \$tests_filename, - "verbose|v" => \$verbose, - "endpoint=s" => \$endpoint, - "token=s" => \$token, - "asyncchecktime=i" => \$async_job_check_time_ms, - "help|h" => \$help, - ); +my $opt = GetOptions( + "tests=s" => \$tests_filename, + "verbose|v" => \$verbose, + "endpoint=s" => \$endpoint, + "token=s" => \$token, + "asyncchecktime=i" => \$async_job_check_time_ms, + "help|h" => \$help, +); -if ($help) { - print $DESCRIPTION."\n"; +if ( $help ) { + print $DESCRIPTION. "\n"; exit 0; } - # endpoint must be defined -if (!$endpoint) { - fail("endpoint parameter must be defined"); +if ( !$endpoint ) { + fail( "endpoint parameter must be defined" ); done_testing(); exit 1; } # tests must be defined -if (!$tests_filename) { - fail("tests parameter must be defined"); +if ( !$tests_filename ) { + fail( "tests parameter must be defined" ); done_testing(); exit 1; } #parse the tests -open(my $fh, "<", $tests_filename); -my $tests_string=''; -while (my $line = <$fh>) { +open( my $fh, "<", $tests_filename ); +my $tests_string = ''; +while ( my $line = <$fh> ) { chomp $line; $tests_string .= $line; } -close($fh); -my $tests_json = JSON->new->decode($tests_string); +close( $fh ); +my $tests_json = JSON->new->decode( $tests_string ); -my $tests = $tests_json->{tests}; -my $client_module = $tests_json->{package}; +my $tests = $tests_json->{ tests }; +my $client_module = $tests_json->{ package }; -if (!$tests) { - fail("tests array not defined in test config file"); +if ( !$tests ) { + fail( "tests array not defined in test config file" ); done_testing(); exit 1; } -if (!$client_module) { - fail("client module not defined in test config file"); +if ( !$client_module ) { + fail( "client module not defined in test config file" ); done_testing(); exit 1; } # make sure we can import the module my $json = JSON->new->canonical; -use_ok($client_module); +use_ok( $client_module ); #instantiate an authenticated client and a nonauthenticated client -my $nonauthenticated_client = $client_module->new($endpoint,ignore_kbase_config=>1); -ok(defined($nonauthenticated_client),"instantiating nonauthenticated client"); +my $nonauthenticated_client = $client_module->new( $endpoint, ignore_kbase_config => 1 ); +ok( defined( $nonauthenticated_client ), "instantiating nonauthenticated client" ); my $authenticated_client; -if($token) { - if ($async_job_check_time_ms) { - $authenticated_client=$client_module->new($endpoint, token=>$token, async_job_check_time_ms=>$async_job_check_time_ms); - } else { - $authenticated_client=$client_module->new($endpoint, token=>$token); +if ( $token ) { + if ( $async_job_check_time_ms ) { + $authenticated_client = $client_module->new( + $endpoint, + token => $token, + async_job_check_time_ms => $async_job_check_time_ms + ); + } + else { + $authenticated_client = $client_module->new( $endpoint, token => $token ); } - ok(defined($authenticated_client),"instantiating authenticated client"); + ok( defined( $authenticated_client ), "instantiating authenticated client" ); } # loop over each test case and run it against the server. We create a new client instance # for each test -foreach my $test (@{$tests}) { +foreach my $test ( @{ $tests } ) { my $client; - if ($test->{'auth'}) { - if ($authenticated_client) { + if ( $test->{ 'auth' } ) { + if ( $authenticated_client ) { $client = $authenticated_client; - } else { - fail("authenticated call declared, but no user and password set"); + } + else { + fail( "authenticated call declared, but no user and password set" ); done_testing(); exit 1; } - - } else { + + } + else { $client = $nonauthenticated_client; } - ok(defined($client),"instantiating client"); - - my $method = $test->{method}; - my $params = $test->{params}; - my $outcome = $test->{outcome}; + ok( defined( $client ), "instantiating client" ); + + my $method = $test->{ method }; + my $params = $test->{ params }; + my $outcome = $test->{ outcome }; my $useScalarContext; - if ($test->{context} && $test->{context} eq 'scalar') { - $useScalarContext=1; + if ( $test->{ context } && $test->{ context } eq 'scalar' ) { + $useScalarContext = 1; } - - ok($client->can($method), ' ----------------- method "'.$method.'" exists ------------------------'); - if ($client->can($method)) { - my (@result, $scalarResult, $failed); + + ok( $client->can( $method ), + ' ----------------- method "' . $method . '" exists ------------------------' ); + if ( $client->can( $method ) ) { + my ( @result, $scalarResult, $failed ); { no strict "refs"; eval { - if ($useScalarContext) { - $scalarResult = $client->$method(@{$params}); - } else { - @result = $client->$method(@{$params}); + if ( $useScalarContext ) { + $scalarResult = $client->$method( @{ $params } ); + } + else { + @result = $client->$method( @{ $params } ); } }; - if($@) { + if ( $@ ) { $failed = 1; - if ($outcome->{status} eq 'fail') { - pass('expected failure, and yes it failed'); - } else { - fail('did not expect to fail, but it did'); - print STDERR "Failing test of '$method', expected to pass but error thrown:\n"; - print STDERR $@->{message}."\n"; - if(defined($@->{status_line})) {print STDERR $@->{status_line}."\n" }; + if ( $outcome->{ status } eq 'fail' ) { + pass( 'expected failure, and yes it failed' ); + } + else { + fail( 'did not expect to fail, but it did' ); + print STDERR + "Failing test of '$method', expected to pass but error thrown:\n"; + print STDERR $@->{ message } . "\n"; + if ( defined( $@->{ status_line } ) ) { + print STDERR $@->{ status_line } . "\n"; + } } - + # check that the error message contains the right message - if ($outcome->{error}) { - my $returnedErrorMsg = $@->{message}; - foreach my $e (@{$outcome->{error}}) { - if(index($returnedErrorMsg, $e) != -1){ - pass("returned error message contains $e") - } else { - fail("returned error message did not contain $e"); - print STDERR "Failing test of '$method', expected error to contain message, but it didn't:\n"; + if ( $outcome->{ error } ) { + my $returnedErrorMsg = $@->{ message }; + foreach my $e ( @{ $outcome->{ error } } ) { + if ( index( $returnedErrorMsg, $e ) != -1 ) { + pass( "returned error message contains $e" ); + } + else { + fail( "returned error message did not contain $e" ); + print STDERR + "Failing test of '$method', expected error to contain message, but it didn't:\n"; print STDERR " looking for: '$e'\n"; print STDERR " got: '$returnedErrorMsg'\n"; } } } - + # could do more checks here for different failure modes - + } } - if ($outcome->{status} eq 'pass') { - pass('expected to run successfully, and it did'); - my ($serialized_params, $serialized_result); - if ($useScalarContext) { - $serialized_params = $json->encode([@{$params}[0]]); - $serialized_result = $json->encode([$scalarResult]); - } else { - $serialized_params = $json->encode($params); - $serialized_result = $json->encode(\@result); + if ( $outcome->{ status } eq 'pass' ) { + pass( 'expected to run successfully, and it did' ); + my ( $serialized_params, $serialized_result ); + if ( $useScalarContext ) { + $serialized_params = $json->encode( [ @{ $params }[ 0 ] ] ); + $serialized_result = $json->encode( [ $scalarResult ] ); + } + else { + $serialized_params = $json->encode( $params ); + $serialized_result = $json->encode( \@result ); } - - ok($serialized_params eq $serialized_result,"response matches input parameters"); - - if ($serialized_params ne $serialized_result) { - print STDERR "Failing test of '$method', expected input to match output, but they don't match:\n"; - print STDERR " in: ".$serialized_params."\n"; - print STDERR " out: ".$serialized_result."\n"; + + ok( + $serialized_params eq $serialized_result, + "response matches input parameters" + ); + + if ( $serialized_params ne $serialized_result ) { + print STDERR + "Failing test of '$method', expected input to match output, but they don't match:\n"; + print STDERR " in: " . $serialized_params . "\n"; + print STDERR " out: " . $serialized_result . "\n"; print STDERR "\n"; } - } elsif ($outcome->{status} eq 'nomatch') { - pass('expected to run successfully, and it did'); - my ($serialized_params, $serialized_result); - if ($useScalarContext) { - ok($scalarResult,"recieved a response in scalar context"); - $serialized_params = $json->encode([@{$params}[0]]); - $serialized_result = $json->encode([$scalarResult]); - } else { - ok(@result,"recieved a response in array context"); - $serialized_params = $json->encode($params); - $serialized_result = $json->encode(\@result); + } + elsif ( $outcome->{ status } eq 'nomatch' ) { + pass( 'expected to run successfully, and it did' ); + my ( $serialized_params, $serialized_result ); + if ( $useScalarContext ) { + ok( $scalarResult, "recieved a response in scalar context" ); + $serialized_params = $json->encode( [ @{ $params }[ 0 ] ] ); + $serialized_result = $json->encode( [ $scalarResult ] ); + } + else { + ok( @result, "recieved a response in array context" ); + $serialized_params = $json->encode( $params ); + $serialized_result = $json->encode( \@result ); } - - ok($serialized_params ne $serialized_result,"response does NOT match input parameters"); - - if ($serialized_params eq $serialized_result) { - print STDERR "Failing test of '$method', expected input to NOT match output, but they did match:\n"; - print STDERR " in/out: ".$serialized_params."\n"; + + ok( + $serialized_params ne $serialized_result, + "response does NOT match input parameters" + ); + + if ( $serialized_params eq $serialized_result ) { + print STDERR + "Failing test of '$method', expected input to NOT match output, but they did match:\n"; + print STDERR " in/out: " . $serialized_params . "\n"; print STDERR "\n"; } - } elsif ($outcome->{status} eq 'fail') { - if (!$failed) { - fail('expected to fail, but it ran successfully'); + } + elsif ( $outcome->{ status } eq 'fail' ) { + if ( !$failed ) { + fail( 'expected to fail, but it ran successfully' ); } - } else { - fail('expected outcome set to "'.$outcome->{status}.'", but that is not recognized. Outcome can only be: pass | fail | nomatch'); + } + else { + fail( 'expected outcome set to "' + . $outcome->{ status } + . '", but that is not recognized. Outcome can only be: pass | fail | nomatch' + ); } } } - - - - done_testing(); - - - -