From 0d0d2b4c72110ae710c74ac1d995811d091e8e9d Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sat, 27 Mar 2021 17:09:33 +0100 Subject: [PATCH 1/2] openms: update test data generation - for CTDConverter 1.4 - remove tmp - use jmchiltons 'fork' of pepnovo to get models for tests --- tools/openms/generate-foo.sh | 2 +- tools/openms/generate.sh | 2 +- tools/openms/test-data.sh | 37 ++++++++++++++++++------------------ 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/tools/openms/generate-foo.sh b/tools/openms/generate-foo.sh index 608a04e46..797e1f958 100644 --- a/tools/openms/generate-foo.sh +++ b/tools/openms/generate-foo.sh @@ -87,7 +87,7 @@ function get_tests2 { # echo $ctdtmp # >&2 cat $ctdtmp testtmp=$(mktemp) - python3 $CTDCONVERTER/convert.py galaxy -i $ctdtmp -o $testtmp -s tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p hardcoded_params.json --tool-version $VERSION --test-only --test-unsniffable csv tsv txt dta dta2d edta mrm splib > /dev/null + CTDConverter galaxy -i $ctdtmp -o $testtmp -s tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p hardcoded_params.json --tool-version $VERSION --test-only --test-unsniffable csv tsv txt dta dta2d edta mrm splib > /dev/null cat $testtmp | grep -v ' convert.out 2> convert.err +CTDConverter galaxy -i ctd/*ctd -o ./ -s tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p hardcoded_params.json --test-macros macros_autotest.xml --test-macros-prefix autotest_ --test-macros macros_test.xml --test-macros-prefix manutest_ --tool-version $VERSION --tool-profile $PROFILE --bump-file bump.json > convert.out 2> convert.err if [[ "$?" -ne "0" ]]; then >&2 echo 'CTD -> XML conversion failed'; >&2 echo -e "stderr:\n$(cat convert.err)"; fi conda deactivate diff --git a/tools/openms/test-data.sh b/tools/openms/test-data.sh index 38bacf9b7..8b822efdb 100755 --- a/tools/openms/test-data.sh +++ b/tools/openms/test-data.sh @@ -45,7 +45,7 @@ eval "$(conda shell.bash hook)" echo "Clone OpenMS $VERSION sources" if [[ ! -d $OPENMSGIT ]]; then - git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT + git clone --depth 1 -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT cd $OPENMSGIT git submodule init git submodule update @@ -84,11 +84,10 @@ fi ############################################################################### echo "Clone CTDConverter" if [[ ! -d $CTDCONVERTER ]]; then - #git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter - git clone -b topic/fix-selects https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER + git clone --depth 1 -b master https://github.com/WorkflowConversion/CTDConverter.git $CTDCONVERTER else cd $CTDCONVERTER - git pull origin topic/fix-selects + git pull origin master cd - fi @@ -105,25 +104,27 @@ cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/ cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/ cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/ if [[ ! -f test-data/MetaboliteSpectralDB.mzML ]]; then - wget -nc https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML - mv MetaboliteSpectralDB.mzML test-data/ + if [[ ! -f $tmp/MetaboliteSpectralDB.mzML ]]; then + wget -nc https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML -O $tmp/MetaboliteSpectralDB.mzML + fi + cp $tmp/MetaboliteSpectralDB.mzML test-data/ fi -ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt -ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv +ln -fs TOFCalibration_ref_masses.tsv test-data/TOFCalibration_ref_masses.txt +ln -fs TOFCalibration_const.tsv test-data/TOFCalibration_const.csv if [ ! -d test-data/pepnovo_models/ ]; then - mkdir -p /tmp/pepnovo - wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip - unzip PepNovo.20120423.zip -d /tmp/pepnovo/ - mv /tmp/pepnovo/Models test-data/pepnovo_models/ - rm PepNovo.20120423.zip - rm -rf /tmp/pepnovo + if [ ! -d $tmp/pepnovo/ ]; then + git clone --depth 1 -b master https://github.com/jmchilton/pepnovo.git $tmp/pepnovo/ + fi + cp -r $tmp/pepnovo/Models test-data/pepnovo_models/ fi ############################################################################### ## generate ctd files using the binaries in the conda package ############################################################################### echo "Create CTD files" conda activate $OPENMSENV +python3 -m pip install --no-deps --no-cache-dir --force-reinstall $CTDCONVERTER + rm -rf ctd mkdir -p ctd @@ -262,7 +263,7 @@ conda deactivate ## remove broken symlinks in test-data find test-data/ -xtype l -delete -# if [ ! -z "$created" ]; then -# echo "Removing temporary directory" -# rm -rf "$tmp" -# fi +if [ ! -z "$created" ]; then + echo "Removing temporary directory" + rm -rf "$tmp" +fi \ No newline at end of file From 8dbcc9eef57fd915a72febff989a3fd2c66831be Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sat, 27 Mar 2021 19:59:15 +0100 Subject: [PATCH 2/2] sync with IUC --- .github/workflows/ci.yaml | 6 +++--- .github/workflows/pr.yaml | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0cedd2cac..7b29d4a47 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,7 +23,7 @@ jobs: chunk-list: ${{ steps.discover.outputs.chunk-list }} strategy: matrix: - python-version: [3.7] + python-version: ['3.7'] steps: - name: Add reaction if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }} @@ -84,7 +84,7 @@ jobs: fail-fast: false matrix: chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }} - python-version: [3.7] + python-version: ['3.7'] services: postgres: image: postgres:11 @@ -148,7 +148,7 @@ jobs: needs: [setup, test] strategy: matrix: - python-version: [3.7] + python-version: ['3.7'] # This job runs on Linux runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 877379bd6..6730f843a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,7 +23,7 @@ jobs: chunk-list: ${{ steps.discover.outputs.chunk-list }} strategy: matrix: - python-version: [3.7] + python-version: ['3.7'] steps: - name: Print github context properties run: | @@ -89,7 +89,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7] + python-version: ['3.7'] steps: # checkout the repository # and use it as the current working directory @@ -122,7 +122,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7] + python-version: ['3.7'] steps: # checkout the repository to master # and use it as the current working directory @@ -147,10 +147,11 @@ jobs: name: Lint R scripts needs: setup if: needs.setup.outputs.repository-list != '' - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: - r-version: [4.0.1] + os: [ubuntu-20.04] + r-version: ['4.0.1'] steps: # checkout the repository to master # and use it as the current working directory @@ -164,7 +165,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.R_LIBS_USER }} - key: r_cache_${{ matrix.r-version }} + key: r_cache_${{ matrix.os }}_${{ matrix.r-version }} - name: Install non-R lintr dependencies run: sudo apt-get install libcurl4-openssl-dev - name: Install lintr @@ -209,7 +210,7 @@ jobs: fail-fast: false matrix: chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }} - python-version: [3.7] + python-version: ['3.7'] services: postgres: image: postgres:11 @@ -279,7 +280,7 @@ jobs: needs: [setup, test] strategy: matrix: - python-version: [3.7] + python-version: ['3.7'] # This job runs on Linux runs-on: ubuntu-latest steps: @@ -317,7 +318,7 @@ jobs: needs: [setup, lint, flake8, lintr, combine_outputs] strategy: matrix: - python-version: [3.7] + python-version: ['3.7'] runs-on: ubuntu-latest if: github.ref == 'refs/heads/master' && github.repository_owner == 'galaxyproteomics' steps: