prevent CI timeout

Dockerfile

@@ -14,7 +14,7 @@ RUN apt-get -qq update && \
      curl wget ca-certificates git-core less netbase \
      g++ cmake autoconf make file valgrind \
      libjemalloc-dev libzip-dev libsnappy-dev libbz2-dev zlib1g-dev liblzma-dev libzstd-dev \
-     python-pyvcf
+     python-pyvcf bcftools pv
 
 # Copy in the local source tree / build context
 ADD . /GLnexus
@@ -32,7 +32,7 @@ FROM ubuntu:19.10
 ENV LC_ALL C.UTF-8
 ENV LANG C.UTF-8
 ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get -qq update && apt-get -qq install -y libjemalloc2 bcftools tabix
+RUN apt-get -qq update && apt-get -qq install -y libjemalloc2 bcftools tabix pv
 
 ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2
 COPY --from=builder /GLnexus/glnexus_cli /usr/local/bin/

src/RocksKeyValue.cc

@@ -53,7 +53,7 @@ size_t calculate_mem_budget(size_t specified_mem_budget) {
     if (specified_mem_budget > 0) {
         ans = std::min(ans, specified_mem_budget);
     }
-    ans = std::max(ans, 4U*size_t(1<<30));
+    ans = std::max(ans, size_t(1<<30));
     return ans;
 }
 
@@ -94,7 +94,7 @@ static Status convertStatus(const rocksdb::Status &s)
 
 // Create RocksDB block cache to be shared among all collections in one database
 std::shared_ptr<rocksdb::Cache> NewBlockCache(OpenMode mode, size_t mem_budget) {
-    assert(mem_budget >= 4U*size_t(1<<30));
+    assert(mem_budget >= size_t(1<<30));
     if (mode != OpenMode::BULK_LOAD) {
         return rocksdb::NewLRUCache(mem_budget * 3 / 4, 8);
     } else {
@@ -159,7 +159,7 @@ void ApplyColumnFamilyOptions(OpenMode mode, size_t prefix_length, size_t mem_bu
         opts.memtable_factory = std::make_shared<rocksdb::VectorRepFactory>();
 
         // Increase memtable size
-        assert(mem_budget >= 4U*size_t(1<<30));
+        assert(mem_budget >= size_t(1<<30));
         opts.write_buffer_size = mem_budget / 6;
         opts.max_write_buffer_number = 4;
         opts.min_write_buffer_number_to_merge = 1;

src/cli_utils.cc

@@ -1455,7 +1455,7 @@ Status genotype(std::shared_ptr<spdlog::logger> logger,
     string sampleset;
     S(data->all_samples_sampleset(sampleset));
 
-    logger->info("genotyping {} sites; sample set = {} threads = {}", sites.size(), sampleset, nr_threads);
+    logger->info("genotyping {} sites; sample set = {} mem_budget = {} threads = {}", sites.size(), sampleset, mem_budget, nr_threads);
     S(svc->genotype_sites(genotyper_cfg, sampleset, sites, output_filename));
     logger->info("genotyping complete!");
 

test/getting_started.sh

@@ -1,21 +1,24 @@
 #!/bin/bash
 # Test glnexus_cli as suggested by the Getting Started guide on the wiki:
 # https://github.com/dnanexus-rnd/GLnexus/wiki/Getting-Started
-set -e -o pipefail
+set -euo pipefail
 
 echo "--- testing Getting Started examples"
 
+free -h
+
 HERE=$(dirname "$0")
 
 if [ ! -f /tmp/dv_1000G_ALDH2_gvcf.tar ]; then
-    wget -O /tmp/dv_1000G_ALDH2_gvcf.tar https://raw.githubusercontent.com/wiki/dnanexus-rnd/GLnexus/data/dv_1000G_ALDH2_gvcf.tar
+    wget -nv -O /tmp/dv_1000G_ALDH2_gvcf.tar https://raw.githubusercontent.com/wiki/dnanexus-rnd/GLnexus/data/dv_1000G_ALDH2_gvcf.tar
 fi
 
 rm -rf /tmp/dv_1000G_ALDH2_gvcf/ GLnexus.DB/
 tar xf /tmp/dv_1000G_ALDH2_gvcf.tar -C /tmp
 
+export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so  # needed to keep memory usage under control in Travis CI
 echo -e "chr12\t111760000\t111820000" > /tmp/dv_1000G_ALDH2_gvcf/ALDH2.bed
-time $HERE/../glnexus_cli --config DeepVariant --squeeze --bed /tmp/dv_1000G_ALDH2_gvcf/ALDH2.bed \
-    /tmp/dv_1000G_ALDH2_gvcf/*.g.vcf.gz > /tmp/dv_1000G_ALDH2_gvcf/dv_1000G_ALDH2.bcf
+time $HERE/../glnexus_cli --config DeepVariant --squeeze --bed /tmp/dv_1000G_ALDH2_gvcf/ALDH2.bed --mem-gbytes 1 \
+    /tmp/dv_1000G_ALDH2_gvcf/*.g.vcf.gz | bcftools view | pv -Wni 10 -s 84968191 > /tmp/dv_1000G_ALDH2_gvcf/dv_1000G_ALDH2.vcf
 
 echo "--- Getting Started examples OK"