Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix CI - by fixing OpenMS test-data generation #579

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Fix CI - by fixing OpenMS test-data generation #579

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0d0d2b4
openms: update test data generation
bernt-matthias Mar 27, 2021
8dbcc9e
sync with IUC
bernt-matthias Mar 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
chunk-list: ${{ steps.discover.outputs.chunk-list }}
strategy:
matrix:
python-version: [3.7]
python-version: ['3.7']
steps:
- name: Add reaction
if: ${{ github.event.client_payload.slash_command.command == 'run-all-tool-tests' }}
Expand Down Expand Up @@ -84,7 +84,7 @@ jobs:
fail-fast: false
matrix:
chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }}
python-version: [3.7]
python-version: ['3.7']
services:
postgres:
image: postgres:11
Expand Down Expand Up @@ -148,7 +148,7 @@ jobs:
needs: [setup, test]
strategy:
matrix:
python-version: [3.7]
python-version: ['3.7']
# This job runs on Linux
runs-on: ubuntu-latest
steps:
Expand Down
19 changes: 10 additions & 9 deletions .github/workflows/pr.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
chunk-list: ${{ steps.discover.outputs.chunk-list }}
strategy:
matrix:
python-version: [3.7]
python-version: ['3.7']
steps:
- name: Print github context properties
run: |
Expand Down Expand Up @@ -89,7 +89,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: [3.7]
python-version: ['3.7']
steps:
# checkout the repository
# and use it as the current working directory
Expand Down Expand Up @@ -122,7 +122,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: [3.7]
python-version: ['3.7']
steps:
# checkout the repository to master
# and use it as the current working directory
Expand All @@ -147,10 +147,11 @@ jobs:
name: Lint R scripts
needs: setup
if: needs.setup.outputs.repository-list != ''
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
strategy:
matrix:
r-version: [4.0.1]
os: [ubuntu-20.04]
r-version: ['4.0.1']
steps:
# checkout the repository to master
# and use it as the current working directory
Expand All @@ -164,7 +165,7 @@ jobs:
uses: actions/cache@v2
with:
path: ${{ env.R_LIBS_USER }}
key: r_cache_${{ matrix.r-version }}
key: r_cache_${{ matrix.os }}_${{ matrix.r-version }}
- name: Install non-R lintr dependencies
run: sudo apt-get install libcurl4-openssl-dev
- name: Install lintr
Expand Down Expand Up @@ -209,7 +210,7 @@ jobs:
fail-fast: false
matrix:
chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }}
python-version: [3.7]
python-version: ['3.7']
services:
postgres:
image: postgres:11
Expand Down Expand Up @@ -279,7 +280,7 @@ jobs:
needs: [setup, test]
strategy:
matrix:
python-version: [3.7]
python-version: ['3.7']
# This job runs on Linux
runs-on: ubuntu-latest
steps:
Expand Down Expand Up @@ -317,7 +318,7 @@ jobs:
needs: [setup, lint, flake8, lintr, combine_outputs]
strategy:
matrix:
python-version: [3.7]
python-version: ['3.7']
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/master' && github.repository_owner == 'galaxyproteomics'
steps:
Expand Down
2 changes: 1 addition & 1 deletion tools/openms/generate-foo.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ function get_tests2 {
# echo $ctdtmp
# >&2 cat $ctdtmp
testtmp=$(mktemp)
python3 $CTDCONVERTER/convert.py galaxy -i $ctdtmp -o $testtmp -s tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p hardcoded_params.json --tool-version $VERSION --test-only --test-unsniffable csv tsv txt dta dta2d edta mrm splib > /dev/null
CTDConverter galaxy -i $ctdtmp -o $testtmp -s tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p hardcoded_params.json --tool-version $VERSION --test-only --test-unsniffable csv tsv txt dta dta2d edta mrm splib > /dev/null
cat $testtmp | grep -v '<output.*file=""' # | grep -v 'CHEMISTRY/'
rm $ctdtmp $testtmp

Expand Down
2 changes: 1 addition & 1 deletion tools/openms/generate.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ fi
find . -maxdepth 0 -name "[A-Z]*xml" -delete
source $(dirname $(which conda))/../etc/profile.d/conda.sh
conda activate OpenMS$VERSION-env
python $CTDCONVERTER/convert.py galaxy -i ctd/*ctd -o ./ -s tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p hardcoded_params.json --test-macros macros_autotest.xml --test-macros-prefix autotest_ --test-macros macros_test.xml --test-macros-prefix manutest_ --tool-version $VERSION --tool-profile $PROFILE --bump-file bump.json > convert.out 2> convert.err
CTDConverter galaxy -i ctd/*ctd -o ./ -s tools_blacklist.txt -f "$FILETYPES" -m macros.xml -t tool.conf -p hardcoded_params.json --test-macros macros_autotest.xml --test-macros-prefix autotest_ --test-macros macros_test.xml --test-macros-prefix manutest_ --tool-version $VERSION --tool-profile $PROFILE --bump-file bump.json > convert.out 2> convert.err
if [[ "$?" -ne "0" ]]; then >&2 echo 'CTD -> XML conversion failed'; >&2 echo -e "stderr:\n$(cat convert.err)"; fi
conda deactivate

Expand Down
37 changes: 19 additions & 18 deletions tools/openms/test-data.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ eval "$(conda shell.bash hook)"

echo "Clone OpenMS $VERSION sources"
if [[ ! -d $OPENMSGIT ]]; then
git clone -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT
git clone --depth 1 -b release/$VERSION.0 https://github.com/OpenMS/OpenMS.git $OPENMSGIT
cd $OPENMSGIT
git submodule init
git submodule update
Expand Down Expand Up @@ -84,11 +84,10 @@ fi
###############################################################################
echo "Clone CTDConverter"
if [[ ! -d $CTDCONVERTER ]]; then
#git clone https://github.com/WorkflowConversion/CTDConverter.git CTDConverter
git clone -b topic/fix-selects https://github.com/bernt-matthias/CTDConverter.git $CTDCONVERTER
git clone --depth 1 -b master https://github.com/WorkflowConversion/CTDConverter.git $CTDCONVERTER
else
cd $CTDCONVERTER
git pull origin topic/fix-selects
git pull origin master
cd -
fi

Expand All @@ -105,25 +104,27 @@ cp -r $OPENMSGIT/share/OpenMS/MAPPING/ test-data/
cp -r $OPENMSGIT/share/OpenMS/CHEMISTRY test-data/
cp -r $OPENMSGIT/share/OpenMS/examples/ test-data/
if [[ ! -f test-data/MetaboliteSpectralDB.mzML ]]; then
wget -nc https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML
mv MetaboliteSpectralDB.mzML test-data/
if [[ ! -f $tmp/MetaboliteSpectralDB.mzML ]]; then
wget -nc https://abibuilder.informatik.uni-tuebingen.de/archive/openms/Tutorials/Data/latest/Example_Data/Metabolomics/databases/MetaboliteSpectralDB.mzML -O $tmp/MetaboliteSpectralDB.mzML
fi
cp $tmp/MetaboliteSpectralDB.mzML test-data/
fi
ln -fs TOFCalibration_ref_masses test-data/TOFCalibration_ref_masses.txt
ln -fs TOFCalibration_const test-data/TOFCalibration_const.csv
ln -fs TOFCalibration_ref_masses.tsv test-data/TOFCalibration_ref_masses.txt
ln -fs TOFCalibration_const.tsv test-data/TOFCalibration_const.csv

if [ ! -d test-data/pepnovo_models/ ]; then
mkdir -p /tmp/pepnovo
wget -nc http://proteomics.ucsd.edu/Software/PepNovo/PepNovo.20120423.zip
unzip PepNovo.20120423.zip -d /tmp/pepnovo/
mv /tmp/pepnovo/Models test-data/pepnovo_models/
rm PepNovo.20120423.zip
rm -rf /tmp/pepnovo
if [ ! -d $tmp/pepnovo/ ]; then
git clone --depth 1 -b master https://github.com/jmchilton/pepnovo.git $tmp/pepnovo/
fi
cp -r $tmp/pepnovo/Models test-data/pepnovo_models/
fi
###############################################################################
## generate ctd files using the binaries in the conda package
###############################################################################
echo "Create CTD files"
conda activate $OPENMSENV
python3 -m pip install --no-deps --no-cache-dir --force-reinstall $CTDCONVERTER

rm -rf ctd
mkdir -p ctd

Expand Down Expand Up @@ -262,7 +263,7 @@ conda deactivate
## remove broken symlinks in test-data
find test-data/ -xtype l -delete

# if [ ! -z "$created" ]; then
# echo "Removing temporary directory"
# rm -rf "$tmp"
# fi
if [ ! -z "$created" ]; then
echo "Removing temporary directory"
rm -rf "$tmp"
fi