Merge pull request #1620 from merenlab/release-candidate

anvi'o v7, "hope"

AUTHORS.txt

@@ -4,11 +4,13 @@ Developers and contributors at the University of Chicago
 A. Murat Eren (a.murat.eren at gmail dot com)
 Evan Kiefl (kiefl.evan at gmail dot com)
 Iva Veseli (iveseli at uchicago.edu)
+Samuel Miller (samuelmiller at uchicago.edu)
 Matthew Schechter (mschechter at uchicago dot edu)
 Isaac Fink (iafink at uchicago dot edu)
+Matthew Lawrence Klein (mtt.l.kln at gmail dot com)
 Andrea Watson (andreawatson at uchicago dot edu)
-Emily Fogarty (efogarty at uchicago dot edu)
 Florian Trigodet (trigodet dot florian at gmail dot com)
+Emily Fogarty (efogarty at uchicago dot edu)
 
 ===============================================================================
 Developers and contributors at large
@@ -27,10 +29,10 @@ Gökmen Görgen              Radity, Inc                            Switzerland
 Doğan Can Kilment          Makroser, Inc.                         Turkey
 Srinidhi Varadharajan      University of Oslo                     Norway
 Amy Willis                 University of Washington               United States
+Mike Lee                   NASA Ames Research Center              United States
 Xabier Vázquez-Campos      University of New South Wales          Australia
 John Eppley                University of Hawaiʻi at Mānoa         United States
 S. Çağlar Onur             Google, Inc.                           United States
-Mike Lee                   NASA Ames Research Center              United States
 Dan Utter                  Harvard University                     United States
 Tom Delmont                Genoscope                              France
 Finlay Maguire             Dalhousie University                   Canada
@@ -52,7 +54,7 @@ Anvi'o is an open-source, community-driven project and the purpose of this file
 to mention names who contributed to its codebase. The file divides individuals into
 two main groups: (1) those who are currently employed at the University of Chicago
 and work on anvi'o as a part of their job description (hence they are ready to
-respond to technical questions or work on feature requests), and (2) those who
+respond to technical questions or consider feature requests), and (2) those who
 contribute (or have contributed) to the platform and help (or have helped) its
 progress from elsewhere. The order of names in this file follows the logic
 implemented at GitHub's contributors page at the following URL, which ranks

Dockerfile

@@ -2,16 +2,17 @@
 # to clean up your docker (although please note that it will remove all existing containers
 # and cached states):
 #
-# docker rmi --force $(docker images -a --filter=dangling=true -q)
-# docker rm --force $(docker ps --filter=status=exited --filter=status=created -q)
-# docker system prune --force -a
+#     docker rmi --force $(docker images -a --filter=dangling=true -q)
+#     docker rm --force $(docker ps --filter=status=exited --filter=status=created -q)
+#     docker system prune --force -a
 #
 # after that, you can start the build with the following:
 #
-# docker build -t meren/anvio:test-build .
+#     BUILDKIT_PROGRESS=plain docker build -t meren/anvio:$ANVIO_VERSION .
+#
 
-FROM continuumio/miniconda3:4.8.2
-ENV ANVIO_VERSION "6.2_master"
+FROM continuumio/miniconda3:4.9.2
+ENV ANVIO_VERSION "7"
 
 RUN conda config --env --add channels bioconda
 RUN conda config --env --add channels conda-forge
@@ -28,9 +29,9 @@ ENV CONDA_DEFAULT_ENV anvioenv
 ENV CONDA_PREFIX /opt/conda/envs/anvioenv
 
 RUN echo "conda activate anvioenv" >> ~/.bashrc
-SHELL ["/bin/bash", "-c"]
 
 RUN conda install -y conda-build
+RUN conda install -y conda-verify
 
 COPY conda-recipe /tmp/conda-recipe
 
@@ -81,3 +82,7 @@ RUN Rscript -e 'install.packages(c("optparse"), repos="https://cran.rstudio.com"
 RUN echo "export PS1=\"\[\e[0m\e[47m\e[1;30m\] :: anvi'o v$ANVIO_VERSION :: \[\e[0m\e[0m \[\e[1;34m\]\]\w\[\e[m\] \[\e[1;32m\]>>>\[\e[m\] \[\e[0m\]\"" >> /root/.bashrc
 
 CMD /bin/bash -l
+
+# To test the build, you can run this:
+#
+# docker run --rm -it -v `pwd`:`pwd` -w `pwd` -p 8080:8080 meren/anvio:test-build

anvio/__init__.py

@@ -15,8 +15,9 @@
 # unless you want to explode `bottle`:
 import pkg_resources
 
-anvio_version = '6.2-master'
-anvio_codename = 'esther'
+anvio_version = '7'
+anvio_codename = 'hope' # after Hope E. Hopps, https://sivb.org/awards/student-awards/hope-e-hopps-award.html
+                        # see the release notes for details: https://github.com/merenlab/anvio/releases/tag/v7
 
 anvio_version_for_help_docs = "main" if anvio_version.endswith('master') else anvio_version
 
@@ -3098,15 +3099,15 @@ def get_version_tuples():
 
 
 def print_version():
-    run.info("Anvi'o version", "%s (v%s)" % (__codename__, __version__), mc='green')
-    run.info("Profile DB version", __profile__version__)
-    run.info("Contigs DB version", __contigs__version__)
-    run.info("Pan DB version", __pan__version__)
-    run.info("Genome data storage version", __genomes_storage_version__)
-    run.info("Auxiliary data storage version", __auxiliary_data_version__)
-    run.info("Structure DB version", __structure__version__)
-    run.info("Kegg Modules DB version", __kegg_modules_version__)
-    run.info("tRNA-seq DB version", __trnaseq__version__)
+    run.info("Anvi'o", "%s (v%s)" % (__codename__, __version__), mc='green', nl_after=1)
+    run.info("Profile database", __profile__version__)
+    run.info("Contigs database", __contigs__version__)
+    run.info("Pan database", __pan__version__)
+    run.info("Genome data storage", __genomes_storage_version__)
+    run.info("Auxiliary data storage", __auxiliary_data_version__)
+    run.info("Structure database", __structure__version__)
+    run.info("Metabolic modules database", __kegg_modules_version__)
+    run.info("tRNA-seq database", __trnaseq__version__, nl_after=1)
 
 
 __version__, \

anvio/argparse.py

@@ -48,14 +48,14 @@ def get_anvio_epilogue(self):
 
         if os.path.exists(os.path.join(os.path.dirname(docs.__file__), f"programs/{self.prog}.md")):
             if atty:
-                epilog += f'''\n🍺 {attr('bold')}More on `{self.prog}`:{attr('reset')}\n\n   {fg('blue') + program_help + attr('reset')}'''
+                epilog += f'''\n🍺 {attr('bold')}More on `{self.prog}`:{attr('reset')}\n\n   {fg('cyan') + program_help + attr('reset')}'''
             else:
                 epilog += f'''\n🍺 More on `{self.prog}`:\n\n   {program_help}'''
         else:
             epilog = ""
 
         if atty:
-            epilog += f'''\n\n🍻 {attr('bold')}All anvi'o programs and artifacts:{attr('reset')}\n\n   {fg('blue') + general_help + attr('reset')}'''
+            epilog += f'''\n\n🍻 {attr('bold')}All anvi'o programs and artifacts:{attr('reset')}\n\n   {fg('cyan') + general_help + attr('reset')}'''
         else:
             epilog += f'''\n\n🍻 All anvi'o programs and artifacts:\n\n   {general_help}'''
 

anvio/data/interactive/charts.html

@@ -191,7 +191,7 @@ <h3 class="modal-title" style="color: black;"><b>Search items</b></h3>
 
 
                   <span class="settings-header">Genes</span>
-                  <p class="settings-section-info">Change colors of genes based on their functional annotations.</p>
+                  <p class="settings-section-info">Change colors of genes based on their functional annotations. You can also manually color genes temporarily. First, righ-click on a gene. It will add the gene into the list below with a default color. Then, you can use the color picker next to it to set a color manually. To see the effect of changing colors, you need to press the 'Redraw Gene Arrows' button.</p>
                   <button type="button" class="btn btn-primary btn-xs" onclick="resetFunctionColors()" class="btn btn-outline-primary">Reset Function colors</button>
                   <button type="button" class="btn btn-primary btn-xs" onclick="redrawArrows()" class="btn btn-outline-primary">Redraw Gene Arrows</button>
                   <button type="button" class="btn btn-primary btn-xs" onclick="toggleUnmarkedGenes()" class="btn btn-outline-primary">Toggle Unmarked Gene Pickers</button>

anvio/migrations/profile/v31_to_v32.py

@@ -57,7 +57,8 @@ def migrate(db_path):
     progress.end()
 
     if is_full_profile:
-        run.info_single("Your profile db is now %s." % next_version, nl_after=1, nl_before=1, mc='green')
+        run.info_single("Your profile db is now %s. We just added a bunch of new variables to the `self` table "
+                        "of your database. All good now." % next_version, nl_after=1, nl_before=1, mc='green')
     else:
         run.info_single("Your profile db is now version %s. But essentially nothing really happened to your "
                         "database since it was a blank profile (which is OK, move along)." \

anvio/migrations/profile/v34_to_v35.py

@@ -52,9 +52,11 @@ def migrate(db_path):
     profile_db.disconnect()
 
     progress.end()
-    run.info_single("The profile database is now %s. This upgrade redefined the stored format of indels to "
-                    "provide a more robust working framework. Unfortunately, if you had indels, you don't anymore." \
-                    % (next_version), nl_after=1, nl_before=1, mc='green')
+    run.info_single("The profile database is now %s. This upgrade redefined the stored format of INDELS to "
+                    "provide a more robust working framework. If you are upgrading this database from `v6`, "
+                    "you don't have anything to worry about. But if you were using the active branch of anvi'o, "
+                    "then you lost your INDELs now and you would need to re-profile your BAM files if you want "
+                    "them back :)" % (next_version), nl_after=1, nl_before=1, mc='green')
 
 
 if __name__ == '__main__':

anvio/scgdomainclassifier.py

@@ -159,7 +159,7 @@ def init(self):
 
         self.progress.update("Learning about the number of contigs databases in each domain subdirectory")
         for domain in self.SCG_domains:
-            self.contigs_dbs[domain] = glob.glob(os.path.join(D(domain), '*')) 
+            self.contigs_dbs[domain] = glob.glob(os.path.join(D(domain), '*.db'))
 
             if len(self.contigs_dbs[domain]) == 0:
                 self.progress.end()

anvio/taxonomyops/__init__.py

@@ -1218,9 +1218,10 @@ def populate_contigs_database(self):
                         raise ConfigError(f"Bad news. The database search operation failed somewhere :( It is very hard for anvi'o "
                                           f"to know what happened, but this is what we heard last: '{error_text}'. If this error message "
                                           f"makes sense ot you, great. Otherwise a LIKELY reason for this is that you have a BLAST version "
-                                          f"installed on your system that is incompatible with anvi'o :/ If you see `2.6.0` or higher "
+                                          f"installed on your system that is incompatible with anvi'o :/ If you see `2.10.1` or higher "
                                           f"version numbers when you type `blastn -version` in your terminal, it may be wortwhile to "
-                                          f"get in touch with anvi'o developers.")
+                                          f"get in touch with anvi'o developers. Note for developers: if the user has a newer version of "
+                                          f"blast, this may be due to changing database extensions generated by `makeblastdb`.")
 
             try:
                 search_output += output_queue.get()

anvio/taxonomyops/trna.py

@@ -37,6 +37,9 @@
 
 HASH = lambda d: str(hashlib.sha224(''.join([str(d[level]) for level in constants.levels_of_taxonomy]).encode('utf-8')).hexdigest()[0:8])
 
+# these things seem to change between BLAST versions. makeblastdb for BLAST 2.10.1 seems to
+# generate the following ones.
+nucl_database_extensions = ['.ndb', '.nhr', '.nin', '.not', '.nsq', '.ntf', '.nto']
 
 class TRNATaxonomyContext(AccessionIdToTaxonomy):
     """The purpose of this base class is ot define file paths and constants for trna taxonomy ops."""
@@ -148,7 +151,7 @@ def sanity_check(self):
             # PopulateContigsDatabaseWithTRNATaxonomy
             ###########################################################
             if self.__class__.__name__ in ['PopulateContigsDatabaseWithTRNATaxonomy']:
-                for prefix in ['.nhr', '.nin', '.nsq']:
+                for prefix in nucl_database_extensions:
                     missing_anticodon_databases = [anticodon for anticodon in self.ctx.anticodons if not os.path.exists(self.ctx.anticodons[anticodon]['db'] + '.nhr')]
                     if len(missing_anticodon_databases):
                         raise ConfigError("OK. It is very likley that if you run `anvi-setup-trna-taxonomy` first you will be golden. "
@@ -447,7 +450,7 @@ def create_search_databases(self):
 
         self.progress.new("Creating search databases")
         self.progress.update("Removing any database that still exists in the output directory...")
-        for prefix in ['.nhr', '.nin', '.nsq']:
+        for prefix in nucl_database_extensions:
             [os.remove(database_path) for database_path in [s['db'] + prefix for s in self.ctx.anticodons.values()] if os.path.exists(database_path)]
 
         # compresssing and decompressing FASTA files changes their hash and make them look like
@@ -479,7 +482,7 @@ def create_search_databases(self):
             blast.log_file_path = os.path.join(os.path.dirname(FASTA_file_path_for_anticodon), '%s.log' % anticodon)
             blast.makedb(dbtype='nucl')
 
-            for prefix in ['.nhr', '.nin', '.nsq']:
+            for prefix in nucl_database_extensions:
                 if not os.path.exists(FASTA_file_path_for_anticodon + prefix):
                     raise ConfigError("Something went wrong and BLAST did not create the database file it was supposed to "
                                       "for %s :(" % anticodon)

anvio/tests/sandbox/workflows/metagenomics/config-idba_ud-no-qc.json

@@ -42,7 +42,7 @@
         "SHORT_READ_FILTER_DIR": "01_SHORT_READ_FILTER_idba_ud_no_qc",
         "LOGS_DIR": "00_LOGS_idba_ud_no_qc"
     },
-    "config_version": "1",
+    "config_version": "2",
     "anvi_gen_contigs_database": {
         "--project-name": "{group}"
     },

anvio/tests/sandbox/workflows/metagenomics/config-idba_ud.json

@@ -17,7 +17,7 @@
         "MERGE_DIR": "06_MERGED_idba_ud",
         "LOGS_DIR": "00_LOGS_idba_ud"
     },
-    "config_version": "1",
+    "config_version": "2",
     "anvi_gen_contigs_database": {
         "--project-name": "{group}"
     },

anvio/tests/sandbox/workflows/metagenomics/config-megahit-no-qc-all-against-all.json

@@ -11,7 +11,7 @@
     "samples_txt": "samples.txt",
     "megahit": {
         "--min-contig-len": 1000,
-        "--memory": 1e6,
+        "--memory": 1000000.0,
         "threads": 11,
         "run": true,
         "--min-count": "",
@@ -52,7 +52,7 @@
         "MERGE_DIR": "06_MERGED_megahit_no_qc",
         "LOGS_DIR": "00_LOGS_megahit_no_qc"
     },
-    "config_version": "1",
+    "config_version": "2",
     "anvi_gen_contigs_database": {
         "--project-name": "{group}"
     },

anvio/tests/sandbox/workflows/metagenomics/config-megahit.json

@@ -56,7 +56,7 @@
         "MERGE_DIR": "06_MERGED_megahit",
         "LOGS_DIR": "00_LOGS_megahit"
     },
-    "config_version": "1",
+    "config_version": "2",
     "anvi_gen_contigs_database": {
         "--project-name": "{group}"
     },

anvio/tests/sandbox/workflows/metagenomics/config-metaspades-no-qc-use-scaffolds.json

@@ -21,7 +21,7 @@
         "TAXONOMY_DIR": "07_TAXONOMY_metaspades_no_qc",
         "LOGS_DIR": "00_LOGS_metaspades_no_qc"
     },
-    "config_version": "1",
+    "config_version": "2",
     "anvi_gen_contigs_database": {
         "--project-name": "{group}"
     },

anvio/tests/sandbox/workflows/metagenomics/config-metaspades.json

@@ -21,7 +21,7 @@
         "TAXONOMY_DIR": "07_TAXONOMY_metaspades",
         "LOGS_DIR": "00_LOGS_metaspades"
     },
-    "config_version": "1",
+    "config_version": "2",
     "anvi_gen_contigs_database": {
         "--project-name": "{group}"
     },

anvio/tests/sandbox/workflows/metagenomics/config-references-mode-no-qc-no-gzip-no-groups.json

@@ -19,7 +19,7 @@
         "MERGE_DIR": "06_MERGED_references_mode_no_qc_no_gzip",
         "LOGS_DIR": "00_LOGS_references_mode_no_qc_no_gzip"
     },
-    "config_version": "1",
+    "config_version": "2",
     "anvi_gen_contigs_database": {
         "--project-name": "{group}"
     },

anvio/tests/sandbox/workflows/metagenomics/config-references-mode.json

@@ -23,7 +23,7 @@
         "LOGS_DIR": "00_LOGS_references_mode",
         "SHORT_READ_FILTER_DIR": "01_SHORT_READ_FILTER_references_mode"
     },
-    "config_version": "1",
+    "config_version": "2",
     "anvi_gen_contigs_database": {
         "--project-name": "{group}"
     },

anvio/tests/sandbox/workflows/pangenomics/config.json

@@ -25,7 +25,7 @@
         "PAN_DIR": "03_PAN",
         "LOGS_DIR": "00_LOGS"
     },
-    "config_version": "1",
+    "config_version": "2",
     "anvi_gen_contigs_database": {
         "--project-name": "{group}"
     },

anvio/tests/sandbox/workflows/pangenomics/pan-config-with-phylogeny-using-hmms.json

@@ -36,7 +36,7 @@
         "PAN_DIR": "03_PAN_FIVE_PAN",
         "LOGS_DIR": "00_LOGS_FIVE_PAN"
     },
-    "config_version": "1",
+    "config_version": "2",
     "anvi_gen_contigs_database": {
         "--project-name": "{group}"
     },

anvio/tests/sandbox/workflows/pangenomics/pan-config-with-phylogeny.json

@@ -35,7 +35,7 @@
         "PAN_DIR": "03_PAN_FIVE_PAN",
         "LOGS_DIR": "00_LOGS_FIVE_PAN"
     },
-    "config_version": "1",
+    "config_version": "2",
     "anvi_gen_contigs_database": {
         "--project-name": "{group}"
     },

anvio/tests/sandbox/workflows/phylogenomics/config.json

@@ -31,7 +31,7 @@
         "PHYLO_DIR": "01_PHYLOGENOMICS",
         "LOGS_DIR": "00_LOGS"
     },
-    "config_version": "1",
+    "config_version": "2",
     "anvi_gen_contigs_database": {
         "--project-name": "{group}"
     },

anvio/utils.py

@@ -655,10 +655,11 @@ def store_dict_as_TAB_delimited_file(d, output_path, headers=None, file_obj=None
         header_text = '\t'.join(headers)
 
     if anvio.AS_MARKDOWN:
-        f.write(f"|%s|\n" % header_text.replace('\t', '|'))
+        tab = '\t'
+        f.write(f"|{header_text.replace(tab, '|')}|\n")
         f.write(f"|{':--|' + '|'.join([':--:'] * (len(headers[1:])))}|\n")
     else:
-        f.write('%s\n' % header_text)
+        f.write(f"{header_text}\n")
 
     if not keys_order:
         keys_order = sorted(d.keys())
@@ -4067,10 +4068,11 @@ def check_h5py_module():
         import h5py
         h5py.__version__
     except:
-        raise ConfigError("Please install the Python module `h5py` manually for this migration task to continue. "
-                          "The reason why the standard anvi'o installation did not install module is complicated, "
-                          "and really unimportant. If you run `pip install h5py` in your Python virtual environment "
-                          "for anvi'o, and try running the migration program again things should be alright.")
+        raise ConfigError("There is an issue but it is easy to resolve and everything is fine! To continue, please "
+                          "first install the Python module `h5py` by running `pip install h5py==2.8.0` in your "
+                          "anvi'o environment. The reason why the standard anvi'o package does not include "
+                          "this module is both complicated and really unimportant. Re-running the migration "
+                          "after `h5py` is installed will make things go smootly.")
 
 
 def RepresentsInt(s):

conda-recipe/anvio-minimal/meta.yaml

@@ -1,13 +1,12 @@
-{% set version = "6.2_master" %}
+{% set version = "7" %}
 
 package:
   name: anvio-minimal
   version: {{ version }}
 
 source:
   git_url: https://github.com/merenlab/anvio
-  git_rev: HEAD
-  git_depth: 1 # shallow clone
+  git_tag: release-candidate
 
 build:
   number: 0
@@ -50,7 +49,7 @@ test:
     - anvi-self-test --help
 
 about:
-  home: http://merenlab.org/software/anvio/index.html
+  home: http://merenlab.org/software/anvio/
   license: GPL-3.0-or-later
   license_family: GPL3
   license_file: LICENSE.txt