Dev

.github/workflows/end-to-end-tests.yml

@@ -14,6 +14,7 @@ jobs:
       - name: Cypress run
         uses: cypress-io/github-action@v4
         with:
+          config-file: config/cypress.config.js
           start: npm run test
         env:
             CYPRESS_IGNORE_TESTS: search

.gitignore

@@ -1,6 +1,11 @@
 .*
 !.github
 data/*
+data_test/*
+!data_test/timetree.nwk
+!data_test/xbbtree.nwk
+!data_test/clusters.json
+!data_test/dbstats.json
 !data/*mut_annotations.json
 .idea/
 treetime/*

DBINSTALL.md

@@ -0,0 +1,167 @@
+## Installing `MongoDB Community Version`
+### On Ubuntu
+  * [Full Instructions](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-ubuntu/)
+### On MacOS
+  * [Full Instructions](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-os-x/)
+### On Windows
+  * [Full Instructions](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-windows/)
+________________
+
+## Directories and Files (`config`, `logs`, `data`)
+These are the _default locations_ for some important `mongodb` files. 
+### On Ubuntu
+  * __config file__ : `/etc/mongod.conf`
+  * __log directory__ : `/var/log/mongodb`
+  * __data directotry__ : `/var/lib/mongodb`
+### On MacOS Intel Processor
+  * __config file__ : `/usr/local/etc/mongod.conf`
+  * __log directory__ : `/usr/local/var/log/mongodb`
+  * __data directotry__ : `/usr/local/var/mongodb`
+### On MacOS M1 Processor
+  * __config file__ :   `/opt/homebrew/etc/mongod.conf`
+  * __log directory__ :   `/opt/homebrew/var/log/mongodb`
+  * __data directory__ :   `/opt/homebrew/var/mongodb`
+________________
+
+## To start/stop/restart the `MongoDB  daemon`
+_When running as a service, the default config file will be used_ 
+### On Ubuntu
+  * `$ sudo systemctl start mongod`
+  * `$ sudo systemctl stop mongod`
+  * `$ sudo systemctl restart mongod`
+
+### On MacOS
+  * `$ brew services start mongodb/brew/mongodb-community`
+  * `$ brew services stop mongodb/brew/mongodb-community`
+  * `$ brew services restart mongodb/brew/mongodb-community`
+________________
+
+## To manually start `MongoDB` with a custom config file
+### For MacOS Intel Processors
+* `$ mongod --config /usr/local/etc/mongod.conf --fork`
+### For MacOS M1 processors
+* `$ mongod --config /opt/homebrew/etc/mongod.conf --fork`
+* _NOTE_: This could be useful when needing to run a `test` database with separate `log` files and `data` directory. Or we could use the same `log` files and the same `data` directory and instead just create a new database `covizu_test` within the Mongo application.
+________________
+
+## To open the Command Line Tool for `MongoDB` a.k.a `mongosh`
+* `$ mongosh`
+
+## Database Authorization
+* By default MongoDB does not seem to enable any authorization. Meaning anyone can connect to our database on `port:27017` and CRUD our data.
+* However, it might be different for you. When trying to open the `cli` if you get an error that looks like this
+```
+Current Mongosh Log ID:	63fc56344c05787dbf853d87
+Connecting to:mongodb://127.0.0.1:27017/?directConnection=true&serverSelectionTimeoutMS=2000&appName=mongosh+1.7.1
+MongoNetworkError: connect ECONNREFUSED 127.0.0.1:27017
+```
+then it probably means your `MongoDB` has locked you out. You need to disable authorization. 
+* To enable/disable authorization, open the afore-mentioned `mongod.conf` configuraiton file.
+* Add (remove) the following lines to the bottom of the config file to enable (disable) authorization
+```
+security:
+  authorization: enabled
+```
+#### _NOTE_ : That `whitespace` between `authorization:` and `enabled` might be crucial. 
+* After editing those lines, restart the `MongoDB daemon`. 
+* To confirm that you've succesfully enabled(disabled) authorization, open `mongosh` without any auth token.
+    * `$ mongosh`
+* Switch to the `admin` database 
+    * `$ use admin`
+* View all the databases 
+    * `$ show databases`
+* If `MongoDB` prints out something like this then you've succesfully disabled (unsuccesfully enabled) authentication.
+```
+admin> show databases
+admin     180.00 KiB
+config    108.00 KiB
+local     120.00 KiB
+```
+* If instead, `MongoDB` prints out something like this then you've succesfully enabled (unsuccesfully disabled) authentication.
+```
+admin> show databases
+MongoServerError: command listDatabases requires authentication
+```
+___________________
+
+
+## Creating user accounts
+* On first install, we need to create a `admin` account, and a `covizu` acccount (readonly). To do so, first disable authorization in the `mongod.conf` configuration file
+* Then ensure that your `.env.dev` or `.env.prod` file contains the environment variables for the following
+```
+DATA_FOLDER='data'
+ADMIN_USERNAME='admin'
+ADMIN_PASSWORD='supersecretpassword'
+COVIZU_USERNAME='covizu'
+COVIZU_PASSWORD='supersecretpassword'
+DB_URL='localhost:27017'
+```
+* Before creating the user accounts, first we check for any existing user accounts of the same name as our `ADMIN_USERNAME` or `COVIZU_USERNAME`. Run the following `nodejs` script.
+    * `$ npm run delete-users`
+* To confirm that there are no user accounts with conflicting names open the `mongosh` 
+    * `$ mongosh`
+    * `test> use admin`
+    * `test> show users`
+* After deleting any existing user accounts, run the following script
+  * `$ npm run create-users`
+
+
+* Once your user accounts are created, edit the `config` file to enable authorization and restart the service. 
+
+## Populating the database
+* If you haven't already, obtain the following data files from our main server at https://filogeneti.ca/covizu/data/:
+* `timetree.nwk`
+* `dbstats.json`
+* `clusters.json`
+and save your local copies under `covizu/data/`.
+* To import these data files into our `MongoDB` database run the following script
+    * `$ npm run update-db1`
+* This will create our primary database and import the `JSON`/`text` records into it.
+* Once the import script has finished executing, start the server by executing the following script
+    * `$ npm run start-db1`
+* Navgiate your browser to `localhost:8001` to verify that it works. 
+* If that went well, then run the next script to setup our secondary database
+    * `$ npm run update-db2`
+* After executing the import script, start the server with a connection to the secondary database
+    * `$ npm run start-db2`
+
+## Updating the database
+* To update the database, first obtain the new data files from `https://filogeneti.ca/covizu/data` and replace the files in `covizu/data`
+* If your `node` server is currently using the primary (secondary) database then update the secondary (primary) database
+    * `$ npm run update-db2` (`$ npm run update-db1`) 
+* After updating the secondary (primary) database, close the currently running server which is still serving data from the primary(secondary) database and restart the server with a connection to the secondary (primary) database
+    * `$ npm run start-db2` (`$ npm run start-db1`)
+* On the next batch of new data, update the primary (secondaary) database and restart the node server with a connection to the primary (secondary) database
+
+
+## Some useful `mongosh` shell commands
+* To open `mongo` CLI in non-authenticated mode
+    * `$ mongosh`
+* To open `mongo` CLI as an `admin`
+    * `$ mongosh --username admin --authenticationDatabase admin`
+
+* To view all users
+    * Connect as `admin` ()
+    * `use admin`
+    * `show users`
+
+* To delete an existing user `"myusername"`
+    * Connect as `admin`
+    * `use admin`
+    * `show users` 
+    * `db.dropUser('myusername')`
+
+* To view all collections within a database `"mydatabase"` 
+    * Connect as `admin`
+    * `use mydatabase`
+    * `show collections`
+
+* To drop the database `"mydatabase"`
+    * Connect as `admin`
+    * `use mydatabase`
+    * `db.dropDatabase()`
+
+* To drop the collection `"mycollection"` with the database `"mydatabase"`
+    * Connect as `admin`
+    * `use mydatabase`
+    * `db.mycollection.drop()`

INSTALL.md

@@ -58,6 +58,11 @@ DATA_FOLDER='data_test'
 HTTP_PORT='8001'
 NODE_ENV='DEV'
 DATA_FOLDER='data'
+ADMIN_USERNAME='admin'
+ADMIN_PASSWORD='password'
+COVIZU_USERNAME='covizu'
+COVIZU_PASSWORD='password'
+DB_URL='localhost:27017'
 ```
 #### `.env.prod` (if running a production server with SSL enabled)
 ```
@@ -67,13 +72,20 @@ NODE_ENV='PROD'
 DATA_FOLDER='data'
 PRVTKEY='/path/to/private-key-file.pem'
 CRT='/path/to/certificate-file.crt'
+ADMIN_USERNAME='admin'
+ADMIN_PASSWORD='password'
+COVIZU_USERNAME='covizu'
+COVIZU_PASSWORD='password'
+DB_URL='localhost:27017'
 ```
 
+Install and configure the `MongoDB` database by following these [instructions](DBINSTALL.md)
+
 Finally, run the following commands:
 ```
  cd covizu
  npm install
- npm start
+ npm run start-db1
 ```
 
 Once you launch the local webserver with `npm start`, allow up to a minute for the server to initialize and then navigate your browser to `localhost:8001`.
@@ -86,3 +98,4 @@ and in a new terminal terminal window run
 ```
 npx cypress run
 ```
+

batch.py

@@ -149,13 +149,60 @@ def process_feed(args, callback=None):
     # filter data, align genomes, extract features, sort by lineage
     by_lineage = process_feed(args, cb.callback)
 
-    # reconstruct time-scaled tree relating lineages
-    timetree, residuals = build_timetree(by_lineage, args, cb.callback)
+    # separate XBB and other recombinant lineages
+    aliases = parse_alias(args.alias)
+    designation = {}
+    for prefix, truename in aliases.items():
+        if type(truename) is list:
+            designation.update({prefix: {
+                'type': 'XBB' if prefix == 'XBB' else 'recombinant',
+                'fullname': '/'.join(truename)
+            }})
+        else:
+            designation.update({prefix: {
+                'type': 'XBB' if truename.startswith("XBB") else 'non-recombinant',
+                'fullname': truename
+            }})
+
+    # use results to partition by_lineage database
+    non_recomb = {}
+    xbb = {}
+    other_recomb = {}
+    for lineage, ldata in by_lineage.items():
+        # Put unassigned lineages in non-recombinant category
+        if lineage.lower() == "unassigned":
+            non_recomb.update({lineage: ldata})
+            continue
+
+        prefix = lineage.split('.')[0]
+        category = designation[prefix]['type']
+        if category == 'non-recombinant':
+            non_recomb.update({lineage: ldata})
+        elif category == 'XBB':
+            xbb.update({lineage: ldata})
+        else:
+            other_recomb.update({lineage: ldata})
+
+    if len(xbb) < 2:
+        other_recomb.update(xbb)
+        xbb = None  # no point in building a tree
+
+
+    # reconstruct time-scaled trees 
+    timetree, residuals = build_timetree(non_recomb, args, cb.callback)
     timestamp = datetime.now().isoformat().split('.')[0]
     nwk_file = os.path.join(args.outdir, 'timetree.{}.nwk'.format(timestamp))
     with open(nwk_file, 'w') as handle:
         Phylo.write(timetree, file=handle, format='newick')
 
+    xbb_file = os.path.join(args.outdir, 'xbbtree.{}.nwk'.format(timestamp))
+    with open(xbb_file, 'w') as handle:
+        if xbb is not None:
+            timetree_xbb, residuals_xbb = build_timetree(xbb, args, cb.callback)
+            residuals.update(residuals_xbb)
+            Phylo.write(timetree_xbb, file=handle, format='newick')
+        # else empty file
+
     # clustering analysis of lineages
     result, infection_prediction = make_beadplots(by_lineage, args, cb.callback, t0=cb.t0.timestamp())
     clust_file = os.path.join(args.outdir, 'clusters.{}.json'.format(timestamp))
@@ -172,9 +219,7 @@ def process_feed(args, callback=None):
     # write data stats
     dbstat_file = os.path.join(args.outdir, 'dbstats.{}.json'.format(timestamp))
 
-    alias = parse_alias(args.alias)
-
-    with open(dbstat_file, 'w') as handle:
+    with (open(dbstat_file, 'w') as handle):
         # total number of sequences
         nseqs = 0
         for records in by_lineage.values():
@@ -187,7 +232,14 @@ def process_feed(args, callback=None):
         }
         for lineage, records in by_lineage.items():
             prefix = lineage.split('.')[0]
-            lname = lineage.replace(prefix, alias[prefix]) if lineage.lower() not in ['unclassifiable', 'unassigned'] and not prefix.startswith('X') and alias[prefix] != '' else lineage
+
+            # resolve PANGO prefix aliases 
+            lname = lineage
+            if (lineage.lower() not in ['unclassifiable', 'unassigned'] 
+                    and not prefix.startswith('X') 
+                    and aliases[prefix] != ''):
+                lname = lineage.replace(prefix, aliases[prefix])
+
             samples = unpack_records(records)
             ndiffs = [len(x['diffs']) for x in samples]
             val['lineages'][lineage] = {
@@ -206,6 +258,7 @@ def process_feed(args, callback=None):
     if not args.dry_run:
         server_root = 'filogeneti.ca:/var/www/html/covizu/data'
         subprocess.check_call(['scp', nwk_file, '{}/timetree.nwk'.format(server_root)])
+        subprocess.check_call(['scp', xbb_file, '{}/xbbtree.nwk'.format(server_root)])
         subprocess.check_call(['scp', clust_file, '{}/clusters.json'.format(server_root)])
         subprocess.check_call(['scp', dbstat_file, '{}/dbstats.json'.format(server_root)])
 

config.js → config/config.js

@@ -9,55 +9,52 @@ var $SSL_CREDENTIALS;
 var $NODE_ENV = process.env.NODE_ENV;
 
 // based on the NODE_ENV env var we read a specific .env file
-
-if ($NODE_ENV == 'TEST'){
+if ($NODE_ENV == 'TEST') {
     require('dotenv').config({ path: '.env.test' });
 }
-else if ($NODE_ENV == 'PROD'){
+else if ($NODE_ENV == 'PROD') {
     require('dotenv').config({ path: '.env.prod' });
 }
-else{
+else {
     require('dotenv').config({ path: '.env.dev' });
 }
 
-
-$NODE_ENV = process.env.NODE_ENV;//in case we didnt specify the NODE_ENV on the command line as well as inside the .env file this catches it
+$NODE_ENV = process.env.NODE_ENV;//in case we didnt specify the NODE_ENV on the command line...
 $DATA_FOLDER = process.env.DATA_FOLDER;
 $HTTP_PORT = process.env.HTTP_PORT;
 $HTTPS_PORT = process.env.HTTPS_PORT;
 
-if ($NODE_ENV=='PROD') {
+if ($NODE_ENV == 'PROD') {
     try {
         $SSL_CREDENTIALS = {
-            key: fs.readFileSync(process.env.PRVTKEY), 
+            key: fs.readFileSync(process.env.PRVTKEY),
             cert: fs.readFileSync(process.env.CRT)
         }
-    } 
+    }
     catch (e) {
         console.error("PROD server requires SSL encryption but .env file is missing PRVTKEY or CRT");
         throw new Error(e);
     }
 }
 
-if(!$HTTP_PORT){
+if (!$HTTP_PORT) {
     console.warn(".env is missing HTTP_PORT. Defaulting to 8001")
     $HTTP_PORT = 8001;
 }
 
-if(!$HTTPS_PORT && $NODE_ENV=='PROD'){
+if (!$HTTPS_PORT && $NODE_ENV == 'PROD') {
     console.warn(".env is missing HTTPS_PORT. Defaulting to 8002")
     $HTTPS_PORT = 8002;
 }
-if(!$DATA_FOLDER){
+if (!$DATA_FOLDER) {
     console.warn('.env is missing DATA_FOLDER env variable. Defaulting to data/')
     $DATA_FOLDER = 'data'
 }
-if(!$NODE_ENV){
+if (!$NODE_ENV) {
     console.warn('.env is missing NODE_ENV. Defaulting to DEV')
     $NODE_ENV = 'DEV';
 }
 
-
 module.exports = {
     $HTTP_PORT,
     $HTTPS_PORT,