Merge pull request #3990 from vgteam/fix-giraffe-presets

Fix giraffe presets

.gitlab-ci.yml

@@ -24,6 +24,8 @@ before_script:
         echo "{\"registry-mirrors\": [\"${DOCKER_HUB_MIRROR}\"], \"insecure-registries\": [\"${DOCKER_HUB_MIRROR##*://}\"]}" | sudo tee /etc/docker/daemon.json
       fi
     fi
+  # Restart or start the Docker daemon
+  - stopdocker || true
   - startdocker || true
   # Get buildx
   - mkdir -p ~/.docker/cli-plugins/ ; curl -L https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64 >  ~/.docker/cli-plugins/docker-buildx ; chmod u+x ~/.docker/cli-plugins/docker-buildx
@@ -112,10 +114,6 @@ build-job:
     - DOCKER_TAG=ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}
     - make include/vg_git_version.hpp
     - cat include/vg_git_version.hpp
-    # Build but don't push, just fill the cache
-    - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target run -f Dockerfile .
-    # Run the tests
-    - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target test -f Dockerfile .
     # Connect so we can upload our images
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
     # Note that A LOCAL CACHE CAN ONLY HOLD ONE TAG/TARGET AT A TIME!

doc/publish-docs.sh

@@ -33,8 +33,13 @@ make docs
 # Get ready to deploy the docs
 
 # Make a scratch directory *outside* our normal git repo
-SCRATCH_DIR="$(pwd)/../tmp"
-mkdir -p "${SCRATCH_DIR}"
+SCRATCH_DIR="$(mktemp -d)"
+# And clean it up when we stop
+function cleanup {
+  rm -Rf ${SCRATCH_DIR}
+}
+trap cleanup EXIT
+
 
 
 # Set up our SSH key

src/subcommand/giraffe_main.cpp

@@ -516,11 +516,11 @@ int main_giraffe(int argc, char** argv) {
         .add_entry<size_t>("max-multimaps", 1)
         .add_entry<size_t>("max-extensions", 400)
         .add_entry<size_t>("max-alignments", 8)
-        .add_entry<size_t>("cluster-score", 50)
-        .add_entry<size_t>("pad-cluster-score", 0)
+        .add_entry<double>("cluster-score", 50)
+        .add_entry<double>("pad-cluster-score", 0)
         .add_entry<double>("cluster-coverage", 0.2)
-        .add_entry<size_t>("extension-set", 20)
-        .add_entry<size_t>("extension-score", 1);
+        .add_entry<double>("extension-set", 20)
+        .add_entry<int>("extension-score", 1);
     // And a default preset that doesn't.
     presets["default"];
     // And a chaining preset (TODO: make into PacBio and Nanopore)

src/subcommand/options.hpp

@@ -517,7 +517,7 @@ struct ArgSpec : public BaseArgSpec<Receiver> {
                 this->set_value(as_typed->value);
             }
         } else {
-            throw std::runtime_error("Could not cast valuation");
+            throw std::runtime_error("Could not cast valuation for " + this->option + " from " + typeid(&entry).name() + " to " + typeid(Valuation<T>*).name());
         }
     }
 
@@ -529,7 +529,7 @@ struct ArgSpec : public BaseArgSpec<Receiver> {
             // Apply the value
             this->set_value(as_typed->value);
         } else {
-            throw std::runtime_error("Could not cast valuation");
+            throw std::runtime_error("Could not cast valuation for " + this->option + " from " + typeid(&entry).name() + " to " + typeid(Valuation<T>*).name());
         }
     }
 
@@ -541,7 +541,7 @@ struct ArgSpec : public BaseArgSpec<Receiver> {
             // Put our value in there.
             as_typed->value = this->get_value();
         } else {
-            throw std::runtime_error("Could not cast valuation");
+            throw std::runtime_error("Could not cast valuation for " + this->option + " from " + typeid(&entry).name() + " to " + typeid(Valuation<T>*).name());
         }
     }
 

test/t/50_vg_giraffe.t

@@ -5,7 +5,7 @@ BASH_TAP_ROOT=../deps/bash-tap
 
 PATH=../bin:$PATH # for vg
 
-plan tests 47
+plan tests 49
 
 vg construct -a -r small/x.fa -v small/x.vcf.gz >x.vg
 vg index -x x.xg x.vg
@@ -40,6 +40,12 @@ is "$(vg view -aj mapped1.gam | grep 'time_used' | wc -l)" "1" "Mapping logs run
 
 is "$(vg view -aj mapped1.gam | jq '.score')" "73" "Mapping produces the correct score"
 
+vg giraffe -Z x.giraffe.gbz -f reads/small.middle.ref.fq -b fast >/dev/null
+is "${?}" "0" "a read can be mapped with the fast preset"
+
+vg giraffe -Z x.giraffe.gbz -f reads/small.middle.ref.fq -b default >/dev/null
+is "${?}" "0" "a read can be mapped with the default preset"
+
 vg giraffe -Z x.giraffe.gbz -f reads/small.middle.ref.fq --full-l-bonus 0 > mapped-nobonus.gam
 is "$(vg view -aj  mapped-nobonus.gam | jq '.score')" "63" "Mapping without a full length bonus produces the correct score"
 rm -f mapped-nobonus.gam

vgci/vgci.py

@@ -1207,7 +1207,7 @@ def _test_calleval(self, region, baseline_graph, sample, vg_path, gam_path, bam_
             self._verify_calleval(tag=tag, threshold=f1_threshold)
 
     #@skip("skipping test to keep runtime down")
-    @timeout_decorator.timeout(8000)            
+    @timeout_decorator.timeout(16000)            
     def test_call_chr21_snp1kg(self):
         """
         calling comparison between call, genotype and freebayes on an alignment extracted
@@ -1230,7 +1230,7 @@ def test_call_chr21_snp1kg(self):
                             0.035)
 
     @skip("skipping test to keep runtime down")
-    @timeout_decorator.timeout(600)
+    @timeout_decorator.timeout(1200)
     def test_sim_brca1_snp1kg(self):
         """ Mapping and calling bakeoff F1 test for BRCA1 primary graph """
         # Using 100k simulated reads from snp1kg BRCA1, realign against all
@@ -1243,7 +1243,7 @@ def test_sim_brca1_snp1kg(self):
                            score_baseline_graph='primary',
                            sample='HG00096', acc_threshold=0.02, auc_threshold=0.02)
 
-    @timeout_decorator.timeout(1200)                       
+    @timeout_decorator.timeout(2400)                       
     def test_sim_mhc_snp1kg(self):
         """ Mapping and calling bakeoff F1 test for BRCA1 primary graph """
         log.info("Test start at {}".format(datetime.now()))
@@ -1252,7 +1252,7 @@ def test_sim_mhc_snp1kg(self):
                            score_baseline_graph='primary',
                            sample='HG00096', acc_threshold=0.02, auc_threshold=0.02)
 
-    @timeout_decorator.timeout(900)
+    @timeout_decorator.timeout(1800)
     def test_sim_mhc_cactus(self):
         """ Mapping test for MHC cactus graph """
         log.info("Test start at {}".format(datetime.now()))        
@@ -1261,7 +1261,7 @@ def test_sim_mhc_cactus(self):
                            mappers = ['map', 'mpmap'],
                            source_path_names=['GI568335986', 'GI568335994'], acc_threshold=0.02, auc_threshold=0.04)
 
-    @timeout_decorator.timeout(4800)        
+    @timeout_decorator.timeout(9600)
     def test_sim_chr21_snp1kg(self):
         log.info("Test start at {}".format(datetime.now()))
         self._test_mapeval(300000, 'CHR21', 'snp1kg',
@@ -1272,7 +1272,7 @@ def test_sim_chr21_snp1kg(self):
                            acc_threshold=0.0075, auc_threshold=0.075, mappers = ['map', 'mpmap'],
                            sim_opts='-l 150 -p 570 -v 150 -e 0.01 -i 0.002')
 
-    @timeout_decorator.timeout(4400)        
+    @timeout_decorator.timeout(8800)        
     def test_sim_chr21_snp1kg_trained(self):
         self._test_mapeval(100000, 'CHR21', 'snp1kg',
                            ['primary', 'snp1kg'],
@@ -1287,7 +1287,7 @@ def test_sim_chr21_snp1kg_trained(self):
                            sim_fastq='ftp://ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/data/NA12878/NIST_NA12878_HG001_HiSeq_300x/131219_D00360_005_BH814YADXX/Project_RM8398/Sample_U5a/U5a_AGTCAA_L002_R1_007.fastq.gz')
 
     @skip("skipping test to keep runtime down")        
-    @timeout_decorator.timeout(1200)
+    @timeout_decorator.timeout(2400)
     def test_sim_brca2_snp1kg_mpmap(self):
         """ multipath mapper test, which is a smaller version of above.  we catch all errors
         so vgci doesn't report failures.  vg is run only in single ended with multipath on
@@ -1301,7 +1301,7 @@ def test_sim_brca2_snp1kg_mpmap(self):
                            acc_threshold=0.02, auc_threshold=0.02)
 
     @skip("skipping test to keep runtime down")        
-    @timeout_decorator.timeout(2400)
+    @timeout_decorator.timeout(4800)
     def test_sim_chr21_snp1kg_mpmap(self):
         """ multipath mapper test, which is a smaller version of above.  we catch all errors
         so vgci doesn't report failures.  vg is run only in single ended with multipath on
@@ -1315,7 +1315,7 @@ def test_sim_chr21_snp1kg_mpmap(self):
                            sim_opts='-d 0.01 -p 1000 -v 75.0 -S 5 -I',
                            sim_fastq=self._input('platinum_NA12878_MHC.fq.gz'))
 
-    @timeout_decorator.timeout(3000)
+    @timeout_decorator.timeout(6000)
     def test_sim_mhc_snp1kg_mpmap(self):
         """ multipath mapper test, which is a smaller version of above.  we catch all errors
         so vgci doesn't report failures.  vg is run only in single ended with multipath on
@@ -1330,7 +1330,7 @@ def test_sim_mhc_snp1kg_mpmap(self):
                            sim_fastq=self._input('platinum_NA12878_MHC.fq.gz'),
                            more_mpmap_opts=['-u 8 -M 1'])
 
-    @timeout_decorator.timeout(4000)
+    @timeout_decorator.timeout(8000)
     def test_sim_yeast_cactus(self):
         """ Yeast test based on the cactus graphs.  Reads are simulated from the SK1 path
         of the full graph.  The other graphs are made from this graph using vg mod:
@@ -1351,19 +1351,19 @@ def test_sim_yeast_cactus(self):
                            acc_threshold=0.02, auc_threshold=0.02,
                            sim_opts='-p 500 -v 50 -S 4 -i 0.002')
 
-    @timeout_decorator.timeout(400)
+    @timeout_decorator.timeout(800)
     def test_map_brca1_primary(self):
         """ Mapping and calling bakeoff F1 test for BRCA1 primary graph """
         log.info("Test start at {}".format(datetime.now()))
         self._test_bakeoff('BRCA1', 'primary', True)
 
-    @timeout_decorator.timeout(400)        
+    @timeout_decorator.timeout(800)        
     def test_map_brca1_snp1kg(self):
         """ Mapping and calling bakeoff F1 test for BRCA1 snp1kg graph """
         log.info("Test start at {}".format(datetime.now()))
         self._test_bakeoff('BRCA1', 'snp1kg', True)
 
-    @timeout_decorator.timeout(900)
+    @timeout_decorator.timeout(1800)
     def test_map_brca1_snp1kg_mpmap(self):
         """ Mapping and calling bakeoff F1 test for BRCA1 snp1kg graph on mpmap.  
         The filter_opts are the defaults minus the identity filter because mpmap doesn't 
@@ -1372,86 +1372,86 @@ def test_map_brca1_snp1kg_mpmap(self):
         self._test_bakeoff('BRCA1', 'snp1kg', True, mapper='mpmap', tag_ext='-mpmap',
                            misc_opts='--filter_opts \"-q 15 -m 1 -D 999 -s 1000\"')
 
-    @timeout_decorator.timeout(400)        
+    @timeout_decorator.timeout(800)        
     def test_map_brca1_cactus(self):
         """ Mapping and calling bakeoff F1 test for BRCA1 cactus graph """
         log.info("Test start at {}".format(datetime.now()))
         self._test_bakeoff('BRCA1', 'cactus', True)
 
-    @timeout_decorator.timeout(900)        
+    @timeout_decorator.timeout(1800)        
     def test_full_brca2_primary(self):
         """ Indexing, mapping and calling bakeoff F1 test for BRCA2 primary graph """
         log.info("Test start at {}".format(datetime.now()))
         self.f1_threshold = 0.01
         self._test_bakeoff('BRCA2', 'primary', False)
 
-    @timeout_decorator.timeout(900)        
+    @timeout_decorator.timeout(1800)        
     def test_full_brca2_snp1kg(self):
         """ Indexing, mapping and calling bakeoff F1 test for BRCA2 snp1kg graph """
         log.info("Test start at {}".format(datetime.now()))
         self._test_bakeoff('BRCA2', 'snp1kg', False)
 
-    @timeout_decorator.timeout(900)        
+    @timeout_decorator.timeout(1800)        
     def test_full_brca2_cactus(self):
         """ Indexing, mapping and calling bakeoff F1 test for BRCA2 cactus graph """
         log.info("Test start at {}".format(datetime.now()))
         self._test_bakeoff('BRCA2', 'cactus', False)
 
     @skip("skipping test to keep runtime down")
-    @timeout_decorator.timeout(900)        
+    @timeout_decorator.timeout(1800)        
     def test_map_sma_primary(self):
         """ Indexing, mapping and calling bakeoff F1 test for SMA primary graph """
         log.info("Test start at {}".format(datetime.now()))
         self._test_bakeoff('SMA', 'primary', True)
 
     @skip("skipping test to keep runtime down")        
-    @timeout_decorator.timeout(900)        
+    @timeout_decorator.timeout(1800)        
     def test_map_sma_snp1kg(self):
         """ Indexing, mapping and calling bakeoff F1 test for SMA snp1kg graph """
         log.info("Test start at {}".format(datetime.now()))
         self._test_bakeoff('SMA', 'snp1kg', True)
 
     @skip("skipping test to keep runtime down")        
-    @timeout_decorator.timeout(900)        
+    @timeout_decorator.timeout(1800)        
     def test_map_sma_cactus(self):
         """ Indexing, mapping and calling bakeoff F1 test for SMA cactus graph """
         log.info("Test start at {}".format(datetime.now()))
         self._test_bakeoff('SMA', 'cactus', True)
 
     @skip("skipping test to keep runtime down")         
-    @timeout_decorator.timeout(900)        
+    @timeout_decorator.timeout(1800)        
     def test_map_lrc_kir_primary(self):
         """ Indexing, mapping and calling bakeoff F1 test for LRC-KIR primary graph """
         log.info("Test start at {}".format(datetime.now()))
         self._test_bakeoff('LRC-KIR', 'primary', True)
 
     @skip("skipping test to keep runtime down")         
-    @timeout_decorator.timeout(900)        
+    @timeout_decorator.timeout(1800)        
     def test_map_lrc_kir_snp1kg(self):
         """ Indexing, mapping and calling bakeoff F1 test for LRC-KIR snp1kg graph """
         self._test_bakeoff('LRC-KIR', 'snp1kg', True)
 
     @skip("skipping test to keep runtime down")         
-    @timeout_decorator.timeout(900)        
+    @timeout_decorator.timeout(1800)        
     def test_map_lrc_kir_cactus(self):
         """ Indexing, mapping and calling bakeoff F1 test for LRC-KIR cactus graph """
         log.info("Test start at {}".format(datetime.now()))
         self._test_bakeoff('LRC-KIR', 'cactus', True)
 
-    @timeout_decorator.timeout(1200)        
+    @timeout_decorator.timeout(2400)        
     def test_map_mhc_primary(self):
         """ Indexing, mapping and calling bakeoff F1 test for MHC primary graph """
         log.info("Test start at {}".format(datetime.now()))
         self._test_bakeoff('MHC', 'primary', True)
 
-    @timeout_decorator.timeout(2000)
+    @timeout_decorator.timeout(4000)
     def test_map_mhc_snp1kg(self):
         """ Indexing, mapping and calling bakeoff F1 test for MHC snp1kg graph """
         log.info("Test start at {}".format(datetime.now()))
         self._test_bakeoff('MHC', 'snp1kg', True)
 
     @skip("skipping test to keep runtime down (baseline missing as well)")          
-    @timeout_decorator.timeout(1200)        
+    @timeout_decorator.timeout(2400)        
     def test_map_mhc_cactus(self):
         """ Indexing, mapping and calling bakeoff F1 test for MHC cactus graph """
         log.info("Test start at {}".format(datetime.now()))