From fda16b81efa096a6272012b2196480af29dd18ef Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 13 Jun 2023 15:23:54 -0400 Subject: [PATCH 1/6] Remove extra uncached builds --- .gitlab-ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0dc89f9c7d3..1fcc9d74e03 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -112,10 +112,6 @@ build-job: - DOCKER_TAG=ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA} - make include/vg_git_version.hpp - cat include/vg_git_version.hpp - # Build but don't push, just fill the cache - - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target run -f Dockerfile . - # Run the tests - - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target test -f Dockerfile . # Connect so we can upload our images - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" # Note that A LOCAL CACHE CAN ONLY HOLD ONE TAG/TARGET AT A TIME! From 3b124e22035abf2848f97083808d6219178026f7 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 13 Jun 2023 16:25:10 -0400 Subject: [PATCH 2/6] Fix and test Giraffe presets --- src/subcommand/giraffe_main.cpp | 8 ++++---- src/subcommand/options.hpp | 6 +++--- test/t/50_vg_giraffe.t | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/subcommand/giraffe_main.cpp b/src/subcommand/giraffe_main.cpp index 59ff648b9c7..42879e23452 100644 --- a/src/subcommand/giraffe_main.cpp +++ b/src/subcommand/giraffe_main.cpp @@ -516,11 +516,11 @@ int main_giraffe(int argc, char** argv) { .add_entry("max-multimaps", 1) .add_entry("max-extensions", 400) .add_entry("max-alignments", 8) - .add_entry("cluster-score", 50) - .add_entry("pad-cluster-score", 0) + .add_entry("cluster-score", 50) + .add_entry("pad-cluster-score", 0) .add_entry("cluster-coverage", 0.2) - .add_entry("extension-set", 20) - .add_entry("extension-score", 1); + .add_entry("extension-set", 20) + .add_entry("extension-score", 1); // And a default preset that doesn't. presets["default"]; // And a chaining preset (TODO: make into PacBio and Nanopore) diff --git a/src/subcommand/options.hpp b/src/subcommand/options.hpp index 611cbb4b2a0..0c311db4cce 100644 --- a/src/subcommand/options.hpp +++ b/src/subcommand/options.hpp @@ -517,7 +517,7 @@ struct ArgSpec : public BaseArgSpec { this->set_value(as_typed->value); } } else { - throw std::runtime_error("Could not cast valuation"); + throw std::runtime_error("Could not cast valuation for " + this->option + " from " + typeid(&entry).name() + " to " + typeid(Valuation*).name()); } } @@ -529,7 +529,7 @@ struct ArgSpec : public BaseArgSpec { // Apply the value this->set_value(as_typed->value); } else { - throw std::runtime_error("Could not cast valuation"); + throw std::runtime_error("Could not cast valuation for " + this->option + " from " + typeid(&entry).name() + " to " + typeid(Valuation*).name()); } } @@ -541,7 +541,7 @@ struct ArgSpec : public BaseArgSpec { // Put our value in there. as_typed->value = this->get_value(); } else { - throw std::runtime_error("Could not cast valuation"); + throw std::runtime_error("Could not cast valuation for " + this->option + " from " + typeid(&entry).name() + " to " + typeid(Valuation*).name()); } } diff --git a/test/t/50_vg_giraffe.t b/test/t/50_vg_giraffe.t index bb190d81247..500f199568b 100644 --- a/test/t/50_vg_giraffe.t +++ b/test/t/50_vg_giraffe.t @@ -5,7 +5,7 @@ BASH_TAP_ROOT=../deps/bash-tap PATH=../bin:$PATH # for vg -plan tests 47 +plan tests 49 vg construct -a -r small/x.fa -v small/x.vcf.gz >x.vg vg index -x x.xg x.vg @@ -40,6 +40,12 @@ is "$(vg view -aj mapped1.gam | grep 'time_used' | wc -l)" "1" "Mapping logs run is "$(vg view -aj mapped1.gam | jq '.score')" "73" "Mapping produces the correct score" +vg giraffe -Z x.giraffe.gbz -f reads/small.middle.ref.fq -b fast >/dev/null +is "${?}" "0" "a read can be mapped with the fast preset" + +vg giraffe -Z x.giraffe.gbz -f reads/small.middle.ref.fq -b default >/dev/null +is "${?}" "0" "a read can be mapped with the default preset" + vg giraffe -Z x.giraffe.gbz -f reads/small.middle.ref.fq --full-l-bonus 0 > mapped-nobonus.gam is "$(vg view -aj mapped-nobonus.gam | jq '.score')" "63" "Mapping without a full length bonus produces the correct score" rm -f mapped-nobonus.gam From 62ff298aa0ecfb413c1ad4c041e3f045a3943915 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 13 Jun 2023 16:52:03 -0400 Subject: [PATCH 3/6] Use a fresh directory every time for the docs build checkout --- doc/publish-docs.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/publish-docs.sh b/doc/publish-docs.sh index 661f9c53160..2fcc07102b1 100755 --- a/doc/publish-docs.sh +++ b/doc/publish-docs.sh @@ -33,8 +33,13 @@ make docs # Get ready to deploy the docs # Make a scratch directory *outside* our normal git repo -SCRATCH_DIR="$(pwd)/../tmp" -mkdir -p "${SCRATCH_DIR}" +SCRATCH_DIR="$(mktemp -d)" +# And clean it up when we stop +function cleanup { + rm -Rf ${SCRATCH_DIR} +} +trap cleanup EXIT + # Set up our SSH key From 6aa5a5910d5344631c6684bdf7d3869a6c320c75 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 13 Jun 2023 18:35:10 -0400 Subject: [PATCH 4/6] Restart Docker daemon th pick up new registry --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1fcc9d74e03..91ed998b7d0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,8 @@ before_script: echo "{\"registry-mirrors\": [\"${DOCKER_HUB_MIRROR}\"], \"insecure-registries\": [\"${DOCKER_HUB_MIRROR##*://}\"]}" | sudo tee /etc/docker/daemon.json fi fi + # Restart or start the Docker daemon + - stopdocker || true - startdocker || true # Get buildx - mkdir -p ~/.docker/cli-plugins/ ; curl -L https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64 > ~/.docker/cli-plugins/docker-buildx ; chmod u+x ~/.docker/cli-plugins/docker-buildx From b237ae7d6ebb2420706d6013b28236a510465ee6 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Wed, 14 Jun 2023 10:07:01 -0400 Subject: [PATCH 5/6] Increase timeouts that are too low for new runners --- vgci/vgci.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vgci/vgci.py b/vgci/vgci.py index 2fa02210e77..12e96b98587 100755 --- a/vgci/vgci.py +++ b/vgci/vgci.py @@ -1252,7 +1252,7 @@ def test_sim_mhc_snp1kg(self): score_baseline_graph='primary', sample='HG00096', acc_threshold=0.02, auc_threshold=0.02) - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_sim_mhc_cactus(self): """ Mapping test for MHC cactus graph """ log.info("Test start at {}".format(datetime.now())) @@ -1261,7 +1261,7 @@ def test_sim_mhc_cactus(self): mappers = ['map', 'mpmap'], source_path_names=['GI568335986', 'GI568335994'], acc_threshold=0.02, auc_threshold=0.04) - @timeout_decorator.timeout(4800) + @timeout_decorator.timeout(9600) def test_sim_chr21_snp1kg(self): log.info("Test start at {}".format(datetime.now())) self._test_mapeval(300000, 'CHR21', 'snp1kg', From 7ac81b3b4dc447716368a74b5d857f5abb593904 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Wed, 14 Jun 2023 14:37:25 -0400 Subject: [PATCH 6/6] Double remaining test timeouts --- vgci/vgci.py | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/vgci/vgci.py b/vgci/vgci.py index 12e96b98587..e80a3cd17a7 100755 --- a/vgci/vgci.py +++ b/vgci/vgci.py @@ -1207,7 +1207,7 @@ def _test_calleval(self, region, baseline_graph, sample, vg_path, gam_path, bam_ self._verify_calleval(tag=tag, threshold=f1_threshold) #@skip("skipping test to keep runtime down") - @timeout_decorator.timeout(8000) + @timeout_decorator.timeout(16000) def test_call_chr21_snp1kg(self): """ calling comparison between call, genotype and freebayes on an alignment extracted @@ -1230,7 +1230,7 @@ def test_call_chr21_snp1kg(self): 0.035) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(600) + @timeout_decorator.timeout(1200) def test_sim_brca1_snp1kg(self): """ Mapping and calling bakeoff F1 test for BRCA1 primary graph """ # Using 100k simulated reads from snp1kg BRCA1, realign against all @@ -1243,7 +1243,7 @@ def test_sim_brca1_snp1kg(self): score_baseline_graph='primary', sample='HG00096', acc_threshold=0.02, auc_threshold=0.02) - @timeout_decorator.timeout(1200) + @timeout_decorator.timeout(2400) def test_sim_mhc_snp1kg(self): """ Mapping and calling bakeoff F1 test for BRCA1 primary graph """ log.info("Test start at {}".format(datetime.now())) @@ -1272,7 +1272,7 @@ def test_sim_chr21_snp1kg(self): acc_threshold=0.0075, auc_threshold=0.075, mappers = ['map', 'mpmap'], sim_opts='-l 150 -p 570 -v 150 -e 0.01 -i 0.002') - @timeout_decorator.timeout(4400) + @timeout_decorator.timeout(8800) def test_sim_chr21_snp1kg_trained(self): self._test_mapeval(100000, 'CHR21', 'snp1kg', ['primary', 'snp1kg'], @@ -1287,7 +1287,7 @@ def test_sim_chr21_snp1kg_trained(self): sim_fastq='ftp://ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/data/NA12878/NIST_NA12878_HG001_HiSeq_300x/131219_D00360_005_BH814YADXX/Project_RM8398/Sample_U5a/U5a_AGTCAA_L002_R1_007.fastq.gz') @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(1200) + @timeout_decorator.timeout(2400) def test_sim_brca2_snp1kg_mpmap(self): """ multipath mapper test, which is a smaller version of above. we catch all errors so vgci doesn't report failures. vg is run only in single ended with multipath on @@ -1301,7 +1301,7 @@ def test_sim_brca2_snp1kg_mpmap(self): acc_threshold=0.02, auc_threshold=0.02) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(2400) + @timeout_decorator.timeout(4800) def test_sim_chr21_snp1kg_mpmap(self): """ multipath mapper test, which is a smaller version of above. we catch all errors so vgci doesn't report failures. vg is run only in single ended with multipath on @@ -1315,7 +1315,7 @@ def test_sim_chr21_snp1kg_mpmap(self): sim_opts='-d 0.01 -p 1000 -v 75.0 -S 5 -I', sim_fastq=self._input('platinum_NA12878_MHC.fq.gz')) - @timeout_decorator.timeout(3000) + @timeout_decorator.timeout(6000) def test_sim_mhc_snp1kg_mpmap(self): """ multipath mapper test, which is a smaller version of above. we catch all errors so vgci doesn't report failures. vg is run only in single ended with multipath on @@ -1330,7 +1330,7 @@ def test_sim_mhc_snp1kg_mpmap(self): sim_fastq=self._input('platinum_NA12878_MHC.fq.gz'), more_mpmap_opts=['-u 8 -M 1']) - @timeout_decorator.timeout(4000) + @timeout_decorator.timeout(8000) def test_sim_yeast_cactus(self): """ Yeast test based on the cactus graphs. Reads are simulated from the SK1 path of the full graph. The other graphs are made from this graph using vg mod: @@ -1351,19 +1351,19 @@ def test_sim_yeast_cactus(self): acc_threshold=0.02, auc_threshold=0.02, sim_opts='-p 500 -v 50 -S 4 -i 0.002') - @timeout_decorator.timeout(400) + @timeout_decorator.timeout(800) def test_map_brca1_primary(self): """ Mapping and calling bakeoff F1 test for BRCA1 primary graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('BRCA1', 'primary', True) - @timeout_decorator.timeout(400) + @timeout_decorator.timeout(800) def test_map_brca1_snp1kg(self): """ Mapping and calling bakeoff F1 test for BRCA1 snp1kg graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('BRCA1', 'snp1kg', True) - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_brca1_snp1kg_mpmap(self): """ Mapping and calling bakeoff F1 test for BRCA1 snp1kg graph on mpmap. The filter_opts are the defaults minus the identity filter because mpmap doesn't @@ -1372,86 +1372,86 @@ def test_map_brca1_snp1kg_mpmap(self): self._test_bakeoff('BRCA1', 'snp1kg', True, mapper='mpmap', tag_ext='-mpmap', misc_opts='--filter_opts \"-q 15 -m 1 -D 999 -s 1000\"') - @timeout_decorator.timeout(400) + @timeout_decorator.timeout(800) def test_map_brca1_cactus(self): """ Mapping and calling bakeoff F1 test for BRCA1 cactus graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('BRCA1', 'cactus', True) - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_full_brca2_primary(self): """ Indexing, mapping and calling bakeoff F1 test for BRCA2 primary graph """ log.info("Test start at {}".format(datetime.now())) self.f1_threshold = 0.01 self._test_bakeoff('BRCA2', 'primary', False) - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_full_brca2_snp1kg(self): """ Indexing, mapping and calling bakeoff F1 test for BRCA2 snp1kg graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('BRCA2', 'snp1kg', False) - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_full_brca2_cactus(self): """ Indexing, mapping and calling bakeoff F1 test for BRCA2 cactus graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('BRCA2', 'cactus', False) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_sma_primary(self): """ Indexing, mapping and calling bakeoff F1 test for SMA primary graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('SMA', 'primary', True) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_sma_snp1kg(self): """ Indexing, mapping and calling bakeoff F1 test for SMA snp1kg graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('SMA', 'snp1kg', True) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_sma_cactus(self): """ Indexing, mapping and calling bakeoff F1 test for SMA cactus graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('SMA', 'cactus', True) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_lrc_kir_primary(self): """ Indexing, mapping and calling bakeoff F1 test for LRC-KIR primary graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('LRC-KIR', 'primary', True) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_lrc_kir_snp1kg(self): """ Indexing, mapping and calling bakeoff F1 test for LRC-KIR snp1kg graph """ self._test_bakeoff('LRC-KIR', 'snp1kg', True) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_lrc_kir_cactus(self): """ Indexing, mapping and calling bakeoff F1 test for LRC-KIR cactus graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('LRC-KIR', 'cactus', True) - @timeout_decorator.timeout(1200) + @timeout_decorator.timeout(2400) def test_map_mhc_primary(self): """ Indexing, mapping and calling bakeoff F1 test for MHC primary graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('MHC', 'primary', True) - @timeout_decorator.timeout(2000) + @timeout_decorator.timeout(4000) def test_map_mhc_snp1kg(self): """ Indexing, mapping and calling bakeoff F1 test for MHC snp1kg graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('MHC', 'snp1kg', True) @skip("skipping test to keep runtime down (baseline missing as well)") - @timeout_decorator.timeout(1200) + @timeout_decorator.timeout(2400) def test_map_mhc_cactus(self): """ Indexing, mapping and calling bakeoff F1 test for MHC cactus graph """ log.info("Test start at {}".format(datetime.now()))