diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0dc89f9c7d3..91ed998b7d0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,8 @@ before_script: echo "{\"registry-mirrors\": [\"${DOCKER_HUB_MIRROR}\"], \"insecure-registries\": [\"${DOCKER_HUB_MIRROR##*://}\"]}" | sudo tee /etc/docker/daemon.json fi fi + # Restart or start the Docker daemon + - stopdocker || true - startdocker || true # Get buildx - mkdir -p ~/.docker/cli-plugins/ ; curl -L https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64 > ~/.docker/cli-plugins/docker-buildx ; chmod u+x ~/.docker/cli-plugins/docker-buildx @@ -112,10 +114,6 @@ build-job: - DOCKER_TAG=ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA} - make include/vg_git_version.hpp - cat include/vg_git_version.hpp - # Build but don't push, just fill the cache - - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target run -f Dockerfile . - # Run the tests - - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target test -f Dockerfile . # Connect so we can upload our images - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" # Note that A LOCAL CACHE CAN ONLY HOLD ONE TAG/TARGET AT A TIME! diff --git a/doc/publish-docs.sh b/doc/publish-docs.sh index 661f9c53160..2fcc07102b1 100755 --- a/doc/publish-docs.sh +++ b/doc/publish-docs.sh @@ -33,8 +33,13 @@ make docs # Get ready to deploy the docs # Make a scratch directory *outside* our normal git repo -SCRATCH_DIR="$(pwd)/../tmp" -mkdir -p "${SCRATCH_DIR}" +SCRATCH_DIR="$(mktemp -d)" +# And clean it up when we stop +function cleanup { + rm -Rf ${SCRATCH_DIR} +} +trap cleanup EXIT + # Set up our SSH key diff --git a/src/subcommand/giraffe_main.cpp b/src/subcommand/giraffe_main.cpp index 59ff648b9c7..42879e23452 100644 --- a/src/subcommand/giraffe_main.cpp +++ b/src/subcommand/giraffe_main.cpp @@ -516,11 +516,11 @@ int main_giraffe(int argc, char** argv) { .add_entry("max-multimaps", 1) .add_entry("max-extensions", 400) .add_entry("max-alignments", 8) - .add_entry("cluster-score", 50) - .add_entry("pad-cluster-score", 0) + .add_entry("cluster-score", 50) + .add_entry("pad-cluster-score", 0) .add_entry("cluster-coverage", 0.2) - .add_entry("extension-set", 20) - .add_entry("extension-score", 1); + .add_entry("extension-set", 20) + .add_entry("extension-score", 1); // And a default preset that doesn't. presets["default"]; // And a chaining preset (TODO: make into PacBio and Nanopore) diff --git a/src/subcommand/options.hpp b/src/subcommand/options.hpp index 611cbb4b2a0..0c311db4cce 100644 --- a/src/subcommand/options.hpp +++ b/src/subcommand/options.hpp @@ -517,7 +517,7 @@ struct ArgSpec : public BaseArgSpec { this->set_value(as_typed->value); } } else { - throw std::runtime_error("Could not cast valuation"); + throw std::runtime_error("Could not cast valuation for " + this->option + " from " + typeid(&entry).name() + " to " + typeid(Valuation*).name()); } } @@ -529,7 +529,7 @@ struct ArgSpec : public BaseArgSpec { // Apply the value this->set_value(as_typed->value); } else { - throw std::runtime_error("Could not cast valuation"); + throw std::runtime_error("Could not cast valuation for " + this->option + " from " + typeid(&entry).name() + " to " + typeid(Valuation*).name()); } } @@ -541,7 +541,7 @@ struct ArgSpec : public BaseArgSpec { // Put our value in there. as_typed->value = this->get_value(); } else { - throw std::runtime_error("Could not cast valuation"); + throw std::runtime_error("Could not cast valuation for " + this->option + " from " + typeid(&entry).name() + " to " + typeid(Valuation*).name()); } } diff --git a/test/t/50_vg_giraffe.t b/test/t/50_vg_giraffe.t index bb190d81247..500f199568b 100644 --- a/test/t/50_vg_giraffe.t +++ b/test/t/50_vg_giraffe.t @@ -5,7 +5,7 @@ BASH_TAP_ROOT=../deps/bash-tap PATH=../bin:$PATH # for vg -plan tests 47 +plan tests 49 vg construct -a -r small/x.fa -v small/x.vcf.gz >x.vg vg index -x x.xg x.vg @@ -40,6 +40,12 @@ is "$(vg view -aj mapped1.gam | grep 'time_used' | wc -l)" "1" "Mapping logs run is "$(vg view -aj mapped1.gam | jq '.score')" "73" "Mapping produces the correct score" +vg giraffe -Z x.giraffe.gbz -f reads/small.middle.ref.fq -b fast >/dev/null +is "${?}" "0" "a read can be mapped with the fast preset" + +vg giraffe -Z x.giraffe.gbz -f reads/small.middle.ref.fq -b default >/dev/null +is "${?}" "0" "a read can be mapped with the default preset" + vg giraffe -Z x.giraffe.gbz -f reads/small.middle.ref.fq --full-l-bonus 0 > mapped-nobonus.gam is "$(vg view -aj mapped-nobonus.gam | jq '.score')" "63" "Mapping without a full length bonus produces the correct score" rm -f mapped-nobonus.gam diff --git a/vgci/vgci.py b/vgci/vgci.py index 2fa02210e77..e80a3cd17a7 100755 --- a/vgci/vgci.py +++ b/vgci/vgci.py @@ -1207,7 +1207,7 @@ def _test_calleval(self, region, baseline_graph, sample, vg_path, gam_path, bam_ self._verify_calleval(tag=tag, threshold=f1_threshold) #@skip("skipping test to keep runtime down") - @timeout_decorator.timeout(8000) + @timeout_decorator.timeout(16000) def test_call_chr21_snp1kg(self): """ calling comparison between call, genotype and freebayes on an alignment extracted @@ -1230,7 +1230,7 @@ def test_call_chr21_snp1kg(self): 0.035) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(600) + @timeout_decorator.timeout(1200) def test_sim_brca1_snp1kg(self): """ Mapping and calling bakeoff F1 test for BRCA1 primary graph """ # Using 100k simulated reads from snp1kg BRCA1, realign against all @@ -1243,7 +1243,7 @@ def test_sim_brca1_snp1kg(self): score_baseline_graph='primary', sample='HG00096', acc_threshold=0.02, auc_threshold=0.02) - @timeout_decorator.timeout(1200) + @timeout_decorator.timeout(2400) def test_sim_mhc_snp1kg(self): """ Mapping and calling bakeoff F1 test for BRCA1 primary graph """ log.info("Test start at {}".format(datetime.now())) @@ -1252,7 +1252,7 @@ def test_sim_mhc_snp1kg(self): score_baseline_graph='primary', sample='HG00096', acc_threshold=0.02, auc_threshold=0.02) - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_sim_mhc_cactus(self): """ Mapping test for MHC cactus graph """ log.info("Test start at {}".format(datetime.now())) @@ -1261,7 +1261,7 @@ def test_sim_mhc_cactus(self): mappers = ['map', 'mpmap'], source_path_names=['GI568335986', 'GI568335994'], acc_threshold=0.02, auc_threshold=0.04) - @timeout_decorator.timeout(4800) + @timeout_decorator.timeout(9600) def test_sim_chr21_snp1kg(self): log.info("Test start at {}".format(datetime.now())) self._test_mapeval(300000, 'CHR21', 'snp1kg', @@ -1272,7 +1272,7 @@ def test_sim_chr21_snp1kg(self): acc_threshold=0.0075, auc_threshold=0.075, mappers = ['map', 'mpmap'], sim_opts='-l 150 -p 570 -v 150 -e 0.01 -i 0.002') - @timeout_decorator.timeout(4400) + @timeout_decorator.timeout(8800) def test_sim_chr21_snp1kg_trained(self): self._test_mapeval(100000, 'CHR21', 'snp1kg', ['primary', 'snp1kg'], @@ -1287,7 +1287,7 @@ def test_sim_chr21_snp1kg_trained(self): sim_fastq='ftp://ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/data/NA12878/NIST_NA12878_HG001_HiSeq_300x/131219_D00360_005_BH814YADXX/Project_RM8398/Sample_U5a/U5a_AGTCAA_L002_R1_007.fastq.gz') @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(1200) + @timeout_decorator.timeout(2400) def test_sim_brca2_snp1kg_mpmap(self): """ multipath mapper test, which is a smaller version of above. we catch all errors so vgci doesn't report failures. vg is run only in single ended with multipath on @@ -1301,7 +1301,7 @@ def test_sim_brca2_snp1kg_mpmap(self): acc_threshold=0.02, auc_threshold=0.02) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(2400) + @timeout_decorator.timeout(4800) def test_sim_chr21_snp1kg_mpmap(self): """ multipath mapper test, which is a smaller version of above. we catch all errors so vgci doesn't report failures. vg is run only in single ended with multipath on @@ -1315,7 +1315,7 @@ def test_sim_chr21_snp1kg_mpmap(self): sim_opts='-d 0.01 -p 1000 -v 75.0 -S 5 -I', sim_fastq=self._input('platinum_NA12878_MHC.fq.gz')) - @timeout_decorator.timeout(3000) + @timeout_decorator.timeout(6000) def test_sim_mhc_snp1kg_mpmap(self): """ multipath mapper test, which is a smaller version of above. we catch all errors so vgci doesn't report failures. vg is run only in single ended with multipath on @@ -1330,7 +1330,7 @@ def test_sim_mhc_snp1kg_mpmap(self): sim_fastq=self._input('platinum_NA12878_MHC.fq.gz'), more_mpmap_opts=['-u 8 -M 1']) - @timeout_decorator.timeout(4000) + @timeout_decorator.timeout(8000) def test_sim_yeast_cactus(self): """ Yeast test based on the cactus graphs. Reads are simulated from the SK1 path of the full graph. The other graphs are made from this graph using vg mod: @@ -1351,19 +1351,19 @@ def test_sim_yeast_cactus(self): acc_threshold=0.02, auc_threshold=0.02, sim_opts='-p 500 -v 50 -S 4 -i 0.002') - @timeout_decorator.timeout(400) + @timeout_decorator.timeout(800) def test_map_brca1_primary(self): """ Mapping and calling bakeoff F1 test for BRCA1 primary graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('BRCA1', 'primary', True) - @timeout_decorator.timeout(400) + @timeout_decorator.timeout(800) def test_map_brca1_snp1kg(self): """ Mapping and calling bakeoff F1 test for BRCA1 snp1kg graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('BRCA1', 'snp1kg', True) - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_brca1_snp1kg_mpmap(self): """ Mapping and calling bakeoff F1 test for BRCA1 snp1kg graph on mpmap. The filter_opts are the defaults minus the identity filter because mpmap doesn't @@ -1372,86 +1372,86 @@ def test_map_brca1_snp1kg_mpmap(self): self._test_bakeoff('BRCA1', 'snp1kg', True, mapper='mpmap', tag_ext='-mpmap', misc_opts='--filter_opts \"-q 15 -m 1 -D 999 -s 1000\"') - @timeout_decorator.timeout(400) + @timeout_decorator.timeout(800) def test_map_brca1_cactus(self): """ Mapping and calling bakeoff F1 test for BRCA1 cactus graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('BRCA1', 'cactus', True) - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_full_brca2_primary(self): """ Indexing, mapping and calling bakeoff F1 test for BRCA2 primary graph """ log.info("Test start at {}".format(datetime.now())) self.f1_threshold = 0.01 self._test_bakeoff('BRCA2', 'primary', False) - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_full_brca2_snp1kg(self): """ Indexing, mapping and calling bakeoff F1 test for BRCA2 snp1kg graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('BRCA2', 'snp1kg', False) - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_full_brca2_cactus(self): """ Indexing, mapping and calling bakeoff F1 test for BRCA2 cactus graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('BRCA2', 'cactus', False) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_sma_primary(self): """ Indexing, mapping and calling bakeoff F1 test for SMA primary graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('SMA', 'primary', True) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_sma_snp1kg(self): """ Indexing, mapping and calling bakeoff F1 test for SMA snp1kg graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('SMA', 'snp1kg', True) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_sma_cactus(self): """ Indexing, mapping and calling bakeoff F1 test for SMA cactus graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('SMA', 'cactus', True) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_lrc_kir_primary(self): """ Indexing, mapping and calling bakeoff F1 test for LRC-KIR primary graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('LRC-KIR', 'primary', True) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_lrc_kir_snp1kg(self): """ Indexing, mapping and calling bakeoff F1 test for LRC-KIR snp1kg graph """ self._test_bakeoff('LRC-KIR', 'snp1kg', True) @skip("skipping test to keep runtime down") - @timeout_decorator.timeout(900) + @timeout_decorator.timeout(1800) def test_map_lrc_kir_cactus(self): """ Indexing, mapping and calling bakeoff F1 test for LRC-KIR cactus graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('LRC-KIR', 'cactus', True) - @timeout_decorator.timeout(1200) + @timeout_decorator.timeout(2400) def test_map_mhc_primary(self): """ Indexing, mapping and calling bakeoff F1 test for MHC primary graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('MHC', 'primary', True) - @timeout_decorator.timeout(2000) + @timeout_decorator.timeout(4000) def test_map_mhc_snp1kg(self): """ Indexing, mapping and calling bakeoff F1 test for MHC snp1kg graph """ log.info("Test start at {}".format(datetime.now())) self._test_bakeoff('MHC', 'snp1kg', True) @skip("skipping test to keep runtime down (baseline missing as well)") - @timeout_decorator.timeout(1200) + @timeout_decorator.timeout(2400) def test_map_mhc_cactus(self): """ Indexing, mapping and calling bakeoff F1 test for MHC cactus graph """ log.info("Test start at {}".format(datetime.now()))