diff --git a/.github/workflows/end-to-end-tests.yml b/.github/workflows/end-to-end-tests.yml index 8dd74145..716b927b 100644 --- a/.github/workflows/end-to-end-tests.yml +++ b/.github/workflows/end-to-end-tests.yml @@ -14,6 +14,7 @@ jobs: - name: Cypress run uses: cypress-io/github-action@v4 with: + config-file: config/cypress.config.js start: npm run test env: CYPRESS_IGNORE_TESTS: search diff --git a/.gitignore b/.gitignore index 9687d7fd..865131b0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,11 @@ .* !.github data/* +data_test/* +!data_test/timetree.nwk +!data_test/xbbtree.nwk +!data_test/clusters.json +!data_test/dbstats.json !data/*mut_annotations.json .idea/ treetime/* diff --git a/DBINSTALL.md b/DBINSTALL.md new file mode 100644 index 00000000..77a8134d --- /dev/null +++ b/DBINSTALL.md @@ -0,0 +1,167 @@ +## Installing `MongoDB Community Version` +### On Ubuntu + * [Full Instructions](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-ubuntu/) +### On MacOS + * [Full Instructions](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-os-x/) +### On Windows + * [Full Instructions](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-windows/) +________________ + +## Directories and Files (`config`, `logs`, `data`) +These are the _default locations_ for some important `mongodb` files. +### On Ubuntu + * __config file__ : `/etc/mongod.conf` + * __log directory__ : `/var/log/mongodb` + * __data directotry__ : `/var/lib/mongodb` +### On MacOS Intel Processor + * __config file__ : `/usr/local/etc/mongod.conf` + * __log directory__ : `/usr/local/var/log/mongodb` + * __data directotry__ : `/usr/local/var/mongodb` +### On MacOS M1 Processor + * __config file__ : `/opt/homebrew/etc/mongod.conf` + * __log directory__ : `/opt/homebrew/var/log/mongodb` + * __data directory__ : `/opt/homebrew/var/mongodb` +________________ + +## To start/stop/restart the `MongoDB daemon` +_When running as a service, the default config file will be used_ +### On Ubuntu + * `$ sudo systemctl start mongod` + * `$ sudo systemctl stop mongod` + * `$ sudo systemctl restart mongod` + +### On MacOS + * `$ brew services start mongodb/brew/mongodb-community` + * `$ brew services stop mongodb/brew/mongodb-community` + * `$ brew services restart mongodb/brew/mongodb-community` +________________ + +## To manually start `MongoDB` with a custom config file +### For MacOS Intel Processors +* `$ mongod --config /usr/local/etc/mongod.conf --fork` +### For MacOS M1 processors +* `$ mongod --config /opt/homebrew/etc/mongod.conf --fork` +* _NOTE_: This could be useful when needing to run a `test` database with separate `log` files and `data` directory. Or we could use the same `log` files and the same `data` directory and instead just create a new database `covizu_test` within the Mongo application. +________________ + +## To open the Command Line Tool for `MongoDB` a.k.a `mongosh` +* `$ mongosh` + +## Database Authorization +* By default MongoDB does not seem to enable any authorization. Meaning anyone can connect to our database on `port:27017` and CRUD our data. +* However, it might be different for you. When trying to open the `cli` if you get an error that looks like this +``` +Current Mongosh Log ID: 63fc56344c05787dbf853d87 +Connecting to:mongodb://127.0.0.1:27017/?directConnection=true&serverSelectionTimeoutMS=2000&appName=mongosh+1.7.1 +MongoNetworkError: connect ECONNREFUSED 127.0.0.1:27017 +``` +then it probably means your `MongoDB` has locked you out. You need to disable authorization. +* To enable/disable authorization, open the afore-mentioned `mongod.conf` configuraiton file. +* Add (remove) the following lines to the bottom of the config file to enable (disable) authorization +``` +security: + authorization: enabled +``` +#### _NOTE_ : That `whitespace` between `authorization:` and `enabled` might be crucial. +* After editing those lines, restart the `MongoDB daemon`. +* To confirm that you've succesfully enabled(disabled) authorization, open `mongosh` without any auth token. + * `$ mongosh` +* Switch to the `admin` database + * `$ use admin` +* View all the databases + * `$ show databases` +* If `MongoDB` prints out something like this then you've succesfully disabled (unsuccesfully enabled) authentication. +``` +admin> show databases +admin 180.00 KiB +config 108.00 KiB +local 120.00 KiB +``` +* If instead, `MongoDB` prints out something like this then you've succesfully enabled (unsuccesfully disabled) authentication. +``` +admin> show databases +MongoServerError: command listDatabases requires authentication +``` +___________________ + + +## Creating user accounts +* On first install, we need to create a `admin` account, and a `covizu` acccount (readonly). To do so, first disable authorization in the `mongod.conf` configuration file +* Then ensure that your `.env.dev` or `.env.prod` file contains the environment variables for the following +``` +DATA_FOLDER='data' +ADMIN_USERNAME='admin' +ADMIN_PASSWORD='supersecretpassword' +COVIZU_USERNAME='covizu' +COVIZU_PASSWORD='supersecretpassword' +DB_URL='localhost:27017' +``` +* Before creating the user accounts, first we check for any existing user accounts of the same name as our `ADMIN_USERNAME` or `COVIZU_USERNAME`. Run the following `nodejs` script. + * `$ npm run delete-users` +* To confirm that there are no user accounts with conflicting names open the `mongosh` + * `$ mongosh` + * `test> use admin` + * `test> show users` +* After deleting any existing user accounts, run the following script + * `$ npm run create-users` + + +* Once your user accounts are created, edit the `config` file to enable authorization and restart the service. + +## Populating the database +* If you haven't already, obtain the following data files from our main server at https://filogeneti.ca/covizu/data/: +* `timetree.nwk` +* `dbstats.json` +* `clusters.json` +and save your local copies under `covizu/data/`. +* To import these data files into our `MongoDB` database run the following script + * `$ npm run update-db1` +* This will create our primary database and import the `JSON`/`text` records into it. +* Once the import script has finished executing, start the server by executing the following script + * `$ npm run start-db1` +* Navgiate your browser to `localhost:8001` to verify that it works. +* If that went well, then run the next script to setup our secondary database + * `$ npm run update-db2` +* After executing the import script, start the server with a connection to the secondary database + * `$ npm run start-db2` + +## Updating the database +* To update the database, first obtain the new data files from `https://filogeneti.ca/covizu/data` and replace the files in `covizu/data` +* If your `node` server is currently using the primary (secondary) database then update the secondary (primary) database + * `$ npm run update-db2` (`$ npm run update-db1`) +* After updating the secondary (primary) database, close the currently running server which is still serving data from the primary(secondary) database and restart the server with a connection to the secondary (primary) database + * `$ npm run start-db2` (`$ npm run start-db1`) +* On the next batch of new data, update the primary (secondaary) database and restart the node server with a connection to the primary (secondary) database + + +## Some useful `mongosh` shell commands +* To open `mongo` CLI in non-authenticated mode + * `$ mongosh` +* To open `mongo` CLI as an `admin` + * `$ mongosh --username admin --authenticationDatabase admin` + +* To view all users + * Connect as `admin` () + * `use admin` + * `show users` + +* To delete an existing user `"myusername"` + * Connect as `admin` + * `use admin` + * `show users` + * `db.dropUser('myusername')` + +* To view all collections within a database `"mydatabase"` + * Connect as `admin` + * `use mydatabase` + * `show collections` + +* To drop the database `"mydatabase"` + * Connect as `admin` + * `use mydatabase` + * `db.dropDatabase()` + +* To drop the collection `"mycollection"` with the database `"mydatabase"` + * Connect as `admin` + * `use mydatabase` + * `db.mycollection.drop()` diff --git a/INSTALL.md b/INSTALL.md index e23c21c8..5051e301 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -58,6 +58,11 @@ DATA_FOLDER='data_test' HTTP_PORT='8001' NODE_ENV='DEV' DATA_FOLDER='data' +ADMIN_USERNAME='admin' +ADMIN_PASSWORD='password' +COVIZU_USERNAME='covizu' +COVIZU_PASSWORD='password' +DB_URL='localhost:27017' ``` #### `.env.prod` (if running a production server with SSL enabled) ``` @@ -67,13 +72,20 @@ NODE_ENV='PROD' DATA_FOLDER='data' PRVTKEY='/path/to/private-key-file.pem' CRT='/path/to/certificate-file.crt' +ADMIN_USERNAME='admin' +ADMIN_PASSWORD='password' +COVIZU_USERNAME='covizu' +COVIZU_PASSWORD='password' +DB_URL='localhost:27017' ``` +Install and configure the `MongoDB` database by following these [instructions](DBINSTALL.md) + Finally, run the following commands: ``` cd covizu npm install - npm start + npm run start-db1 ``` Once you launch the local webserver with `npm start`, allow up to a minute for the server to initialize and then navigate your browser to `localhost:8001`. @@ -86,3 +98,4 @@ and in a new terminal terminal window run ``` npx cypress run ``` + diff --git a/batch.py b/batch.py index c5d720d3..933edb69 100644 --- a/batch.py +++ b/batch.py @@ -149,13 +149,60 @@ def process_feed(args, callback=None): # filter data, align genomes, extract features, sort by lineage by_lineage = process_feed(args, cb.callback) - # reconstruct time-scaled tree relating lineages - timetree, residuals = build_timetree(by_lineage, args, cb.callback) + # separate XBB and other recombinant lineages + aliases = parse_alias(args.alias) + designation = {} + for prefix, truename in aliases.items(): + if type(truename) is list: + designation.update({prefix: { + 'type': 'XBB' if prefix == 'XBB' else 'recombinant', + 'fullname': '/'.join(truename) + }}) + else: + designation.update({prefix: { + 'type': 'XBB' if truename.startswith("XBB") else 'non-recombinant', + 'fullname': truename + }}) + + # use results to partition by_lineage database + non_recomb = {} + xbb = {} + other_recomb = {} + for lineage, ldata in by_lineage.items(): + # Put unassigned lineages in non-recombinant category + if lineage.lower() == "unassigned": + non_recomb.update({lineage: ldata}) + continue + + prefix = lineage.split('.')[0] + category = designation[prefix]['type'] + if category == 'non-recombinant': + non_recomb.update({lineage: ldata}) + elif category == 'XBB': + xbb.update({lineage: ldata}) + else: + other_recomb.update({lineage: ldata}) + + if len(xbb) < 2: + other_recomb.update(xbb) + xbb = None # no point in building a tree + + + # reconstruct time-scaled trees + timetree, residuals = build_timetree(non_recomb, args, cb.callback) timestamp = datetime.now().isoformat().split('.')[0] nwk_file = os.path.join(args.outdir, 'timetree.{}.nwk'.format(timestamp)) with open(nwk_file, 'w') as handle: Phylo.write(timetree, file=handle, format='newick') + xbb_file = os.path.join(args.outdir, 'xbbtree.{}.nwk'.format(timestamp)) + with open(xbb_file, 'w') as handle: + if xbb is not None: + timetree_xbb, residuals_xbb = build_timetree(xbb, args, cb.callback) + residuals.update(residuals_xbb) + Phylo.write(timetree_xbb, file=handle, format='newick') + # else empty file + # clustering analysis of lineages result, infection_prediction = make_beadplots(by_lineage, args, cb.callback, t0=cb.t0.timestamp()) clust_file = os.path.join(args.outdir, 'clusters.{}.json'.format(timestamp)) @@ -172,9 +219,7 @@ def process_feed(args, callback=None): # write data stats dbstat_file = os.path.join(args.outdir, 'dbstats.{}.json'.format(timestamp)) - alias = parse_alias(args.alias) - - with open(dbstat_file, 'w') as handle: + with (open(dbstat_file, 'w') as handle): # total number of sequences nseqs = 0 for records in by_lineage.values(): @@ -187,7 +232,14 @@ def process_feed(args, callback=None): } for lineage, records in by_lineage.items(): prefix = lineage.split('.')[0] - lname = lineage.replace(prefix, alias[prefix]) if lineage.lower() not in ['unclassifiable', 'unassigned'] and not prefix.startswith('X') and alias[prefix] != '' else lineage + + # resolve PANGO prefix aliases + lname = lineage + if (lineage.lower() not in ['unclassifiable', 'unassigned'] + and not prefix.startswith('X') + and aliases[prefix] != ''): + lname = lineage.replace(prefix, aliases[prefix]) + samples = unpack_records(records) ndiffs = [len(x['diffs']) for x in samples] val['lineages'][lineage] = { @@ -206,6 +258,7 @@ def process_feed(args, callback=None): if not args.dry_run: server_root = 'filogeneti.ca:/var/www/html/covizu/data' subprocess.check_call(['scp', nwk_file, '{}/timetree.nwk'.format(server_root)]) + subprocess.check_call(['scp', xbb_file, '{}/xbbtree.nwk'.format(server_root)]) subprocess.check_call(['scp', clust_file, '{}/clusters.json'.format(server_root)]) subprocess.check_call(['scp', dbstat_file, '{}/dbstats.json'.format(server_root)]) diff --git a/config.js b/config/config.js similarity index 81% rename from config.js rename to config/config.js index 0751caf3..d8144505 100644 --- a/config.js +++ b/config/config.js @@ -9,55 +9,52 @@ var $SSL_CREDENTIALS; var $NODE_ENV = process.env.NODE_ENV; // based on the NODE_ENV env var we read a specific .env file - -if ($NODE_ENV == 'TEST'){ +if ($NODE_ENV == 'TEST') { require('dotenv').config({ path: '.env.test' }); } -else if ($NODE_ENV == 'PROD'){ +else if ($NODE_ENV == 'PROD') { require('dotenv').config({ path: '.env.prod' }); } -else{ +else { require('dotenv').config({ path: '.env.dev' }); } - -$NODE_ENV = process.env.NODE_ENV;//in case we didnt specify the NODE_ENV on the command line as well as inside the .env file this catches it +$NODE_ENV = process.env.NODE_ENV;//in case we didnt specify the NODE_ENV on the command line... $DATA_FOLDER = process.env.DATA_FOLDER; $HTTP_PORT = process.env.HTTP_PORT; $HTTPS_PORT = process.env.HTTPS_PORT; -if ($NODE_ENV=='PROD') { +if ($NODE_ENV == 'PROD') { try { $SSL_CREDENTIALS = { - key: fs.readFileSync(process.env.PRVTKEY), + key: fs.readFileSync(process.env.PRVTKEY), cert: fs.readFileSync(process.env.CRT) } - } + } catch (e) { console.error("PROD server requires SSL encryption but .env file is missing PRVTKEY or CRT"); throw new Error(e); } } -if(!$HTTP_PORT){ +if (!$HTTP_PORT) { console.warn(".env is missing HTTP_PORT. Defaulting to 8001") $HTTP_PORT = 8001; } -if(!$HTTPS_PORT && $NODE_ENV=='PROD'){ +if (!$HTTPS_PORT && $NODE_ENV == 'PROD') { console.warn(".env is missing HTTPS_PORT. Defaulting to 8002") $HTTPS_PORT = 8002; } -if(!$DATA_FOLDER){ +if (!$DATA_FOLDER) { console.warn('.env is missing DATA_FOLDER env variable. Defaulting to data/') $DATA_FOLDER = 'data' } -if(!$NODE_ENV){ +if (!$NODE_ENV) { console.warn('.env is missing NODE_ENV. Defaulting to DEV') $NODE_ENV = 'DEV'; } - module.exports = { $HTTP_PORT, $HTTPS_PORT, diff --git a/cypress.config.js b/config/cypress.config.js similarity index 100% rename from cypress.config.js rename to config/cypress.config.js diff --git a/config/dbconfig.js b/config/dbconfig.js new file mode 100644 index 00000000..f05e0130 --- /dev/null +++ b/config/dbconfig.js @@ -0,0 +1,97 @@ +const path = require('path'); + +const $PROJECT_ROOT = path.resolve(__dirname+"/../"); +const $JSON_DATA_FOLDER = "data" +const $DATABASE__PRIMARY = "covizu_1"; +const $DATABASE__SECONDARY = "covizu_2"; +const $COLLECTION__CLUSTERS = "clusters"; +const $COLLECTION__DBSTATS = "dbstats"; +const $COLLECTION__BEADDATA = "beaddata"; +const $COLLECTION__TIPS = "tips"; +const $COLLECTION__RECOMBINANT_TIPS = "recombinant_tips"; +const $COLLECTION__ACCN_TO_CID = "accn_to_cid"; +const $COLLECTION__LINEAGE_TO_CID = "lineage_to_cid"; +const $COLLECTION__REGION_MAP = "region_map"; +const $COLLECTION__DF_TREE = "df_tree"; +const $COLLECTION__XBB_TREE = "xbb_tree"; +const $COLLECTION__AUTOCOMPLETE_DATA = "autocomplete_data"; +const $COLLECTION__FLAT_DATA = "flat_data"; + +const $JSONFILE__CLUSTERS = "clusters.json"; +const $JSONFILE__DBSTATS = "dbstats.json"; +const $NWKFILE__TREE = "timetree.nwk"; +const $XBB__TREE = "xbbtree.nwk"; + +var $NODE_ENV = process.env.NODE_ENV; + +if ($NODE_ENV == 'TEST'){ + require('dotenv').config({ path: '.env.test' }); +} +else if ($NODE_ENV == 'PROD'){ + require('dotenv').config({ path: '.env.prod' }); +} +else{ + require('dotenv').config({ path: '.env.dev' }); +} + +let $ACTIVE_DATABASE; +if($NODE_ENV != 'TEST'){ + const $DBNUMBER = process.env.DBNUMBER; + if($DBNUMBER == 1) + { + console.log("Setting active database to ", $DATABASE__PRIMARY); + $ACTIVE_DATABASE = $DATABASE__PRIMARY; + } + else if($DBNUMBER == 2) + { + console.log("Setting active database to ", $DATABASE__SECONDARY); + $ACTIVE_DATABASE = $DATABASE__SECONDARY; + } + else + { + $ACTIVE_DATABASE = undefined; + } +} + +const $ADMIN_USERNAME = encodeURIComponent(process.env.ADMIN_USERNAME); +const $ADMIN_PASSWORD = encodeURIComponent(process.env.ADMIN_PASSWORD); +const $COVIZU_USERNAME = encodeURIComponent(process.env.COVIZU_USERNAME); +const $COVIZU_PASSWORD = encodeURIComponent(process.env.COVIZU_PASSWORD); +const $AUTHMECHANISM = "DEFAULT"; +const $DB_URL = process.env.DB_URL; + +// this covizu_admin connection is required to create the database/collections/documents. +const $ADMIN_CONNECTION_URI = `mongodb://${$ADMIN_USERNAME}:${$ADMIN_PASSWORD}@${$DB_URL}/?authMechanism=${$AUTHMECHANISM}`; +// this covizu connection will be used for reading data from the database and delivering to the webserver and the frontend +const $COVIZU_CONNECTION_URI = `mongodb://${$COVIZU_USERNAME}:${$COVIZU_PASSWORD}@${$DB_URL}/?authMechanism=${$AUTHMECHANISM}`; + +module.exports = { + $DATABASE__PRIMARY, + $DATABASE__SECONDARY, + $ADMIN_USERNAME, + $ADMIN_PASSWORD, + $COVIZU_USERNAME, + $COVIZU_PASSWORD, + $DB_URL, + $ADMIN_CONNECTION_URI, + $COVIZU_CONNECTION_URI, + $ACTIVE_DATABASE, + $COLLECTION__CLUSTERS, + $COLLECTION__DBSTATS, + $COLLECTION__BEADDATA, + $COLLECTION__TIPS, + $COLLECTION__RECOMBINANT_TIPS, + $COLLECTION__ACCN_TO_CID, + $COLLECTION__LINEAGE_TO_CID, + $COLLECTION__REGION_MAP, + $COLLECTION__DF_TREE, + $COLLECTION__XBB_TREE, + $COLLECTION__AUTOCOMPLETE_DATA, + $COLLECTION__FLAT_DATA, + $PROJECT_ROOT, + $JSON_DATA_FOLDER, + $JSONFILE__CLUSTERS, + $JSONFILE__DBSTATS, + $NWKFILE__TREE, + $XBB__TREE +} \ No newline at end of file diff --git a/pm2.config.js b/config/pm2.config.js similarity index 100% rename from pm2.config.js rename to config/pm2.config.js diff --git a/covizu/data/ProblematicSites_SARS-CoV2 b/covizu/data/ProblematicSites_SARS-CoV2 index dd969327..a36cee5d 160000 --- a/covizu/data/ProblematicSites_SARS-CoV2 +++ b/covizu/data/ProblematicSites_SARS-CoV2 @@ -1 +1 @@ -Subproject commit dd96932793e293de8433d8fea2c061101997ee36 +Subproject commit a36cee5dc5ce8fabcfd23f73b690874c739c2928 diff --git a/covizu/data/pango-designation b/covizu/data/pango-designation index a5388529..bf7bf4a1 160000 --- a/covizu/data/pango-designation +++ b/covizu/data/pango-designation @@ -1 +1 @@ -Subproject commit a5388529fbcdb389e6a3935f9d8d3d59d4816f6c +Subproject commit bf7bf4a1bcfbc1642291507a766bdaa7341fab50 diff --git a/covizu/utils/seq_utils.py b/covizu/utils/seq_utils.py index c9e1db20..fb9ffe3e 100644 --- a/covizu/utils/seq_utils.py +++ b/covizu/utils/seq_utils.py @@ -101,22 +101,20 @@ def apply_features(diffs, missing, refseq): :param refseq: str, reference genome :return: str, aligned genome """ - result = list(refseq) # strings are not mutable + result = refseq # apply missing intervals for left, right in missing: - for i in range(left, right): - result[i] = 'N' + result = '%s%s%s'%(result[:left], 'N'*(right - left), result[right:]) # apply substitutions and deletions (skip insertions) for dtype, pos, diff in diffs: if dtype == '~': - result[pos] = diff + result = '%s%s%s'%(result[:pos], diff, result[pos + 1:]) elif dtype == '-': - for i in range(pos, pos+diff): - result[i] = '-' + result = '%s%s%s'%(result[:pos], '-'*(diff), result[pos + diff:]) - return ''.join(result) + return result def fromisoformat(dt): diff --git a/css/style.css b/css/style.css index d426e622..0ba2f9d4 100644 --- a/css/style.css +++ b/css/style.css @@ -782,7 +782,7 @@ h2.modal-title { #tree-slider.ui-slider-horizontal { appearance: none; - width: 79%; + width: 195px; /* Setting the slider width to match width of the tree axis */ height: 5px; opacity: 0.7; /* Set transparency (for mouse-over effects on hover) */ transition: opacity .2s; diff --git a/cypress/e2e/beadplot.spec.cy.js b/cypress/e2e/beadplot.spec.cy.js index 5f65b286..5342b21d 100644 --- a/cypress/e2e/beadplot.spec.cy.js +++ b/cypress/e2e/beadplot.spec.cy.js @@ -3,21 +3,19 @@ import clusters from '../../data_test/clusters.json' describe('Beadplot components', () => { it('Edge Labels', ()=> { cy.get('.beadplot-content>svg>g>text').should(($text) => { - expect($text).to.have.length(3) - expect($text.eq(0)).to.contain('unsampled0') - expect($text.eq(1)).to.contain('Scotland/GCVR-16F930') - expect($text.eq(2)).to.contain('Scotland/GCVR-170556') + expect($text).to.have.length(14) + expect($text.eq(0)).to.contain('USA/AZ-CDC-LC0932884') + expect($text.eq(1)).to.contain('USA/AZ-CDC-STM-TNFPTQHEE') + expect($text.eq(2)).to.contain('USA/AZ-CDC-LC0951048') }) }) it('Samples', () => { cy.get('.beadplot-content>svg>g>circle').then(($circ) => { - expect($circ).to.have.length(4) - for (let i = 0; i < 3; i++) { - cy.get($circ.eq(i)).should('have.attr', 'cy', 45) + expect($circ).to.have.length(23) + for (let i = 0; i < 7; i++) { + cy.get($circ.eq(i)).should('have.attr', 'cy', 20) } - cy.get($circ.eq(3)).should('have.attr', 'cy', 70) - }) - + }) }) }) @@ -29,19 +27,15 @@ describe('Tables', () => { }) it('Country details: gentable', () => { cy.get('#tabs').contains('Countries').click() - var tips = {'allregions': {'North America': 13}, 'country': {'Canada': 8, 'USA': 5}} + var tips = {'allregions': {'North America': 23}, 'country': {'USA': 23}} cy.window().then((win) => { win.gentable(tips) }) - cy.get('table:visible>tbody>tr').should('have.length', 2) + cy.get('table:visible>tbody>tr').should('have.length', 1) cy.get('table:visible>tbody>tr').eq(0).contains('North America') - cy.get('table:visible>tbody>tr').eq(0).contains('Canada') - cy.get('table:visible>tbody>tr').eq(0).contains('8') - - cy.get('table:visible>tbody>tr').eq(1).contains('North America') - cy.get('table:visible>tbody>tr').eq(1).contains('USA') - cy.get('table:visible>tbody>tr').eq(1).contains('5') + cy.get('table:visible>tbody>tr').eq(0).contains('USA') + cy.get('table:visible>tbody>tr').eq(0).contains('23') }) }) @@ -60,7 +54,7 @@ describe('Edge slider', () => { // https://stackoverflow.com/questions/64855669/moving-slider-with-cypress - let targetValue = 12 + let targetValue = 5.01 let currentValue = 2 let increment = 0.01 steps = (targetValue - currentValue) / increment + 1 @@ -74,28 +68,23 @@ describe('Edge slider', () => { cy.get('#custom-handle').should('contain', targetValue) }) it('All edges are visible on max slider value ', () => { - cy.get('[stroke="#bbd"]').should('have.length', 2) - }) - it('No edges are visible on slider value of 11.99 ', () => { - cy.get('#left-arrow').click() - cy.get('#custom-handle').should('contain', 11.99) - cy.get('[stroke="#bbd"]').should('not.exist') + cy.get('[stroke="#bbd"]').should('have.length', 12) }) }) describe('Tooltips', () => { it('Horizontal Edge', () => { - cy.get('#Scotland-GCVR-16F930').first().trigger('mouseover') - cy.get('.tooltip').contains('Parent: unsampled0') - cy.get('.tooltip').contains('Europe: 3') - cy.get('.tooltip').contains('Unique collection dates: 3') - cy.get('.tooltip').contains('2020-03-19 / 2020-03-27') + cy.get('#USA-AZ-CDC-LC0983866').first().trigger('mouseover') + cy.get('.tooltip').contains('Parent: USA/AZ-CDC-LC0932884') + cy.get('.tooltip').contains('North America: 2') + cy.get('.tooltip').contains('Unique collection dates: 2') + cy.get('.tooltip').contains('2023-01-02 / 2023-01-07') }) it('Bead', () => { - cy.get('circle:visible').first().trigger('mouseover') + cy.get('#EPI_ISL_1054790').first().trigger('mouseover') cy.get('.tooltip').contains('Parent: unsampled0') - cy.get('.tooltip').contains('Genomic distance: 12') - cy.get('.tooltip').contains('Europe: 1') - cy.get('.tooltip').contains('Collection date: 2020-03-19') + cy.get('.tooltip').contains('Genomic distance: 1.82') + cy.get('.tooltip').contains('North America: 1') + cy.get('.tooltip').contains('Collection date: 2022-11-12') }) }) \ No newline at end of file diff --git a/cypress/e2e/covizu.spec.cy.js b/cypress/e2e/covizu.spec.cy.js index 8851dbad..20ca2b53 100644 --- a/cypress/e2e/covizu.spec.cy.js +++ b/cypress/e2e/covizu.spec.cy.js @@ -89,9 +89,9 @@ describe('Search Interface', () => { }) // this test was failing because (i think) it was searching for a data-point ('HMH') that no longer exists in the current dataset - // I've replaced 'HMH' with 'China' - it("Searching 'China' results in selection and expected point count", () => { - cy.get('#search-input').type('China') + // I've replaced 'HMH' with 'Japan' + it("Searching 'Japan' results in selection and expected point count", () => { + cy.get('#search-input').type('Japan') cy.get('#search-button').click({force:true}) cy.get('.selectionH').should('exist') cy.get('.SelectedCluster').should('exist') @@ -123,20 +123,27 @@ describe('Search Interface', () => { describe('Tooltips', () => { it('Appear on hover over cluster', () => { - cy.get('[id=id-0]').trigger('mouseover'); - cy.get('.tooltip').should('be.visible').should('have.length', 1) + cy.window().then((win)=>{ + win.reset_tree(true); + const tip_id = win.display_id.non_recombinants.last; + cy.get(`[id=id-${tip_id}]`).trigger('mouseover'); + cy.get('.tooltip').should('be.visible').should('have.length', 1) + }); }) it('Cluster tooltips contain relevant and correct information', () => { - cy.get('[id=id-0]').trigger('mouseover'); cy.window().then((win)=>{ - cy.get('.tooltip').contains(`Sampled: ${win.tips[0]['varcount']}`) - cy.get('.tooltip').contains(`Displayed: ${win.tips[0]['sampled_varcount']}`) - cy.wrap(Object.keys(win.tips[0]['allregions'])).each(($el, index) => { - cy.get('.tooltip').contains(`${$el}: ${Object.values(win.tips[0]['allregions'])[index]}`) + win.reset_tree(true); + const tip_id = win.display_id.non_recombinants.last; + const last_index = win.tips.length - 1; + cy.get(`[id=id-${tip_id}]`).trigger('mouseover'); + cy.get('.tooltip').contains(`Sampled: ${win.tips[last_index]['varcount']}`) + cy.get('.tooltip').contains(`Displayed: ${win.tips[last_index]['sampled_varcount']}`) + cy.wrap(Object.keys(win.tips[last_index]['allregions'])).each(($el, index) => { + cy.get('.tooltip').contains(`${$el}: ${Object.values(win.tips[last_index]['allregions'])[index]}`) }) - cy.get('.tooltip').contains(`Mean diffs from root: ${Math.round(100*win.tips[0]['mean_ndiffs'])/100}`) - cy.get('.tooltip').contains(`Deviation from clock: ${win.tips[0]['residual'].toFixed(2)}`) - cy.get('.tooltip').contains(`${win.tips[0]['first_date'].toISOString().slice(0, 10)} / ${win.tips[0]['last_date'].toISOString().slice(0, 10)}`) + cy.get('.tooltip').contains(`Mean diffs from root: ${Math.round(100*win.tips[last_index]['mean_ndiffs'])/100}`) + cy.get('.tooltip').contains(`Deviation from clock: ${win.tips[last_index]['residual'].toFixed(2)}`) + cy.get('.tooltip').contains(`${win.tips[last_index]['first_date'].toISOString().slice(0, 10)} / ${win.tips[last_index]['last_date'].toISOString().slice(0, 10)}`) }) }) it('Appear on hover over bead', () => { @@ -161,9 +168,38 @@ describe('Tooltips', () => { }) }) +describe("Display options", () => { + it("Verify options within the Display dropdown", () => { + cy.visit("http://localhost:8001"); + cy.get("#splash-button").click(); + cy.get("#display-tree").children().should(($options) => { + expect($options).to.have.length(3); + expect($options.eq(0)).to.contain('Non-Recombinants'); + expect($options.eq(1)).to.contain('XBB Lineages'); + expect($options.eq(2)).to.contain('Other Recombinants') + }) + }) + + it("Selecting 'XBB Lineages' Display option", () => { + cy.get("#display-tree").select(1).should('have.value','XBB Lineages') + cy.window().then((win)=> { + win.reset_tree(true); + cy.get(`[id=id-${win.display_id.xbb.last}]`).should('be.visible').trigger('mouseover'); + cy.get(`[id=id-${win.display_id.xbb.first}]`).should('be.visible').trigger('mouseover'); + }) + }) + + it("Selecting 'Other Recombinants' Display option", () => { + cy.get("#display-tree").select(2).should('have.value','Other Recombinants') + cy.window().then((win)=> { + win.reset_tree(true); + cy.get(`[id=id-${win.display_id.other_recombinants.last}]`).should('be.visible').trigger('mouseover'); + cy.get(`[id=id-${win.display_id.other_recombinants.first}]`).should('be.visible').trigger('mouseover'); + }) + }) +}) describe("Colour tree", () => { - it("Verify within
@@ -197,17 +197,22 @@

Search Error

- Árbol del tiempo - +
+ Árbol del tiempo +   NWK   - -
  CSV  
- 🔰 - - + +
  CSV  
+ 🔰 +
+
+ + +
@@ -220,11 +225,6 @@

Search Error

-
-
Recombinantes
-
-
-
No Recombinantes
@@ -232,7 +232,7 @@

Search Error

Diagrama de perlas
-
  NWK  
+
  NWK  
  SVG  
🔰
diff --git a/index-fr.html b/index-fr.html index 873969c7..c81a55f0 100644 --- a/index-fr.html +++ b/index-fr.html @@ -145,8 +145,8 @@

Search Error

- - + +
@@ -197,17 +197,22 @@

Search Error

- Phylogénie temporelle - +
+ Phylogénie temporelle +   NWK   - -
  CSV  
- 🔰 - - + +
  CSV  
+ 🔰 +
+
+ + +
@@ -220,11 +225,6 @@

Search Error

-
-
Les Recombinants
-
-
-
Non-Recombinants
@@ -232,7 +232,7 @@

Search Error

Diagramme en perle
-
  NWK  
+
  NWK  
  SVG  
🔰
diff --git a/index-zh.html b/index-zh.html index 404de1cf..b52c3891 100644 --- a/index-zh.html +++ b/index-zh.html @@ -146,8 +146,8 @@

Search Error

- - + +
@@ -198,17 +198,22 @@

Search Error

- 时间尺度的种系发生树 - +
+ 时间尺度的种系发生树 +   NWK   - -
  CSV  
- 🔰 - - + +
  CSV  
+ 🔰 +
+
+ + +
@@ -221,11 +226,6 @@

Search Error

-
-
重组体
-
-
-
非重组体
@@ -233,7 +233,7 @@

Search Error

珠图
-
  NWK  
+
  NWK  
  SVG  
🔰
diff --git a/index.html b/index.html index f7d190a7..0fff5b33 100644 --- a/index.html +++ b/index.html @@ -146,8 +146,8 @@

Search Error

- - + +
@@ -198,17 +198,22 @@

Search Error

- Time-scaled tree - -   NWK   - -
  CSV  
- 🔰 - - +
+ Time-scaled tree + +   NWK   + +
  CSV  
+ 🔰 +
+
+ + +
@@ -222,11 +227,6 @@

Search Error

-
-
Recombinants
-
-
-
Non-Recombinants
@@ -235,7 +235,7 @@

Search Error

Beadplot
-
  NWK  
+
  NWK  
  SVG  
🔰
diff --git a/js/beadplot.js b/js/beadplot.js index 61b03999..e0918d89 100644 --- a/js/beadplot.js +++ b/js/beadplot.js @@ -57,7 +57,7 @@ function parse_mutation_annotations(mut_annotations) { var mutations = []; // Sorts tips according to cluster_idx - sorted_tips = [...tips, ...recombinant_tips] + sorted_tips = [...tips, ...recombinant_tips, ...df_xbb] sorted_tips.sort(function(a,b) { return a.cluster_idx - b.cluster_idx }); @@ -409,6 +409,11 @@ function expand() { } + if (edgelist.length === 0) + $('#beadplot-nwk').hide(); + else + $('#beadplot-nwk').show(); + // update vertical range for consistent spacing between variants heightB = max_y * 10 + 40; $("#svg-cluster > svg").attr("height", heightB + marginB.top + marginB.bottom); @@ -937,6 +942,7 @@ function region_to_string(my_regions) { */ function gen_mut_table(obj) { + console.log("GEERNATING MUTATION TABLE ", obj) var mutations = []; // Check for a list of samples @@ -1373,6 +1379,7 @@ function serialize_branch(parent, edgelist) { // terminal node, emit string // TODO: add count information (number of samples per variant) coldate = branch[0].x1.toISOString().split('T')[0]; + console.log("coldate=",coldate); return branch[0].child+'|'+coldate+':'+branch[0].dist; // node name : branch length } else { diff --git a/js/covizu.js b/js/covizu.js index ed7c6804..042efe84 100644 --- a/js/covizu.js +++ b/js/covizu.js @@ -197,6 +197,7 @@ $.ajaxSetup({ var dbstats, req; req = $.getJSON("data/dbstats.json", function(data) { dbstats = data; + console.log("dbstats = ", dbstats) dbstats.nlineages = Object.keys(dbstats.lineages).length; }); req.done(function() { @@ -226,23 +227,25 @@ var phenotypes = { } // load time-scaled phylogeny from server -var nwk, df, countries, mut_annotations, region_map; -$.ajax({ - url: "data/timetree.nwk", - success: function(data) { - nwk = data; - } -}); -$.getJSON("data/mut_annotations.json", function(data) { - mut_annotations = data; -}); +var nwk, df, df_xbb, countries, mut_annotations, region_map; + +// $.getJSON("data/mut_annotations.json", function(data) { +// mut_annotations = data; +// console.log("MUTATION ANNOTATION ",data) +// }); var clusters, beaddata, tips, recombinant_tips, accn_to_cid, cindex, lineage_to_cid, lineage; var edgelist = [], points = [], variants = [] -var map_cidx_to_id = [], id_to_cidx = []; +var map_cidx_to_id = [], id_to_cidx = [], display_id = {}; req = $.when( + + $.getJSON("data/mut_annotations.json", function(data) { + mut_annotations = data; + console.log("MUTATION ANNOTATION ",data) + }), + $.getJSON("/api/tips", function(data) { tips = data; tips.forEach(x => { @@ -270,12 +273,22 @@ req = $.when( x.mcoldate = x.coldate ? new Date(x.mcoldate) : undefined }); }), + $.getJSON("/api/xbb", function(data) { + df_xbb = data; + df_xbb.forEach(x => { + x.first_date = x.first_date ? new Date(x.first_date) : undefined + x.last_date = x.last_date ? new Date(x.last_date) : undefined + x.coldate = x.coldate ? new Date(x.coldate) : undefined + x.mcoldate = x.coldate ? new Date(x.mcoldate) : undefined + }); + }), $.getJSON("/api/regionmap", function(data) { region_map = data; }) ); req.done(async function() { + var urlParams = new URLSearchParams(window.location.search); var search = urlParams.get('search') || ''; @@ -290,16 +303,41 @@ req.done(async function() { // Maps id to a cidx const reverse_recombinant_tips = [...recombinant_tips].reverse() - var all_tips = [...tips, ...reverse_recombinant_tips] - for (i in all_tips) { - id_to_cidx[i] = 'cidx-' + all_tips[i].cluster_idx + var i = 0, first, last; + first = i; + for (const index in reverse_recombinant_tips) { + id_to_cidx[i++] = 'cidx-' + reverse_recombinant_tips[index].cluster_idx; } + last = i - 1; + display_id["other_recombinants"] = {"first": first, "last": last}; + + first = i; + var xbb_tips = df_xbb.filter(x=>x.isTip); + for (const index in xbb_tips) { + id_to_cidx[i++] = 'cidx-' + xbb_tips[index].cluster_idx; + } + last = i - 1; + display_id["xbb"] = {"first": first, "last": last}; + + first = i; + for (const index in tips) { + id_to_cidx[i++] = 'cidx-' + tips[index].cluster_idx; + } + last = i - 1; + display_id["non_recombinants"] = {"first": first, "last": last}; // Maps cidx to an id const reverseMapping = o => Object.keys(o).reduce((r, k) => Object.assign(r, { [o[k]]: (r[o[k]] || parseInt(k)) }), {}) map_cidx_to_id = reverseMapping(id_to_cidx) - await redraw_tree(formatDate(curr_date), redraw=false); + switch($("#display-tree").val()) { + case "XBB Lineages": + await redraw_tree(df_xbb, formatDate(curr_date), redraw=false); + break; + case "Other Recombinants": + default: + await redraw_tree(df, formatDate(curr_date), redraw=false); + } //spinner.stop(); var rect = d3.selectAll("#svg-timetree > svg > rect"), @@ -308,6 +346,7 @@ req.done(async function() { // initial display // d3.select(node).dispatch("click"); cindex = node.__data__.cluster_idx; + console.log("NODE = ",node); d3.select(node).attr("class", "clicked"); window.addEventListener("resize", expand, true); @@ -321,6 +360,7 @@ req.done(async function() { await fetch(`/api/lineagetocid`) .then(response => response.json()) .then(data => lineage_to_cid = data) + .then(()=>{console.log("lineage_to_cid",lineage_to_cid)}) $('#search-input').autocomplete({ source: function(req, res) { @@ -392,7 +432,6 @@ req.done(async function() { $('#error_message').text(``); $('#search-button').removeAttr("disabled"); $('#clear_button').removeAttr("disabled"); - reset_tree(); wrap_search(); enable_buttons(); @@ -411,6 +450,7 @@ req.done(async function() { gentable(node.__data__); draw_region_distribution(node.__data__.allregions); gen_details_table(points); // update details table with all samples + console.log("Generating with ", mutations,cindex) gen_mut_table(mutations[cindex]); draw_cluster_box(d3.select(node)); } @@ -445,7 +485,6 @@ req.done(async function() { $('#error_message').text(``); $("#loading").show(); $("#loading_text").text(i18n_text.loading); - await reset_tree(partial_redraw=true); await wrap_search(); enable_buttons(); @@ -522,7 +561,6 @@ req.done(async function() { $('#error_message').text(``); $("#loading").show(); $("#loading_text").text(i18n_text.loading); - await reset_tree(partial_redraw=true); await wrap_search(); enable_buttons(); $("#loading").hide(); @@ -598,19 +636,6 @@ req.done(async function() { expand(); }); - $('#display-option').on('change', function() { - if (!$('#display-option').attr('checked')) { - $('#display-option').attr('checked', 'checked'); - $(".recombinant-tree-content").show() - $(".recombtitle").show() - } - else { - $('#display-option').removeAttr('checked'); - $(".recombinant-tree-content").hide() - $(".recombtitle").hide() - } - }); - $(window).on('resize', set_height); // Sets the scrolling speed when scrolling through the beadplot @@ -644,21 +669,21 @@ req.done(async function() { var cutoff_line = $("#cutoff-line"); var tree_cutoff = $("#tree-cutoff"); - const tree_multiplier = 100000000; - var min = Math.floor((d3.min(df, xValue)-0.05) * tree_multiplier); - var max = Math.ceil(date_to_xaxis(d3.max(df, function(d) {return d.last_date})) * tree_multiplier); + const tree_multiplier = 100000; // Slider value needs to be an integer + var min = (d3.min(df, xValue)-0.05) * tree_multiplier; + var max = date_to_xaxis(d3.max(df, function(d) {return d.last_date})) * tree_multiplier; var start_value = date_to_xaxis(curr_date) * tree_multiplier; $("#tree-slider").slider({ create: function( event, ui ) { - cutoff_date.text(xaxis_to_date($( this ).slider( "value" )/tree_multiplier, tips[0])); + cutoff_date.text(xaxis_to_date($( this ).slider( "value" )/tree_multiplier, df[0], d3.min(df, function(d) {return d.first_date}), d3.max(df, function(d) {return d.last_date}))); var cutoff_pos = handle.position().left; tree_cutoff.css('left', cutoff_pos); }, slide: async function( event, ui ) { move_arrow(); - cutoff_date.text(xaxis_to_date(ui.value/tree_multiplier, tips[0])); + cutoff_date.text(xaxis_to_date(ui.value/tree_multiplier, df[0], d3.min(df, function(d) {return d.first_date}), d3.max(df, function(d) {return d.last_date}))); await handle.change() cutoff_line.css('visibility', 'visible'); @@ -674,7 +699,13 @@ req.done(async function() { $("#loading").show(); $("#loading_text").text(i18n_text.loading); - await redraw_tree(cutoff_date.text()); + switch($("#display-tree").val()) { + case "XBB Lineages": + await redraw_tree(df_xbb, cutoff_date.text()); + break; + default: + await redraw_tree(df, cutoff_date.text()); + } $("#loading").hide(); $("#loading_text").text(``); } @@ -939,8 +970,16 @@ $( "#dialog" ).dialog({ autoOpen: false }); // implement save buttons var blob; function save_timetree() { - blob = new Blob([nwk], {type: "text/plain;charset=utf-8"}); - saveAs(blob, "timetree.nwk"); + var filename = $("#display-tree").val() === "XBB Lineages" ? "xbbtree.nwk" : "timetree.nwk" + + $.ajax({ + url: `data/${filename}`, + success: function(data) { + nwk = data; + blob = new Blob([nwk], {type: "text/plain;charset=utf-8"}); + saveAs(blob, filename); + } + }); } function save_beadplot() { @@ -968,11 +1007,14 @@ function export_svg() { } function export_csv() { + var all_tips = [...tips, ...recombinant_tips, ...df_xbb]; + // write lineage-level information to CSV file for download var csvFile = 'lineage,mean.diffs,clock.residual,num.cases,num.variants,min.coldate,max.coldate,mean.coldate'; var lineage_info = [] - for (tip of tips) { - lineage_info.push([`${tip.thisLabel},${Math.round(100*tip.mean_ndiffs)/100.},${Math.round(100*tip.residual)/100.},${tip.nsamples},${tip.varcount},${formatDate(tip.first_date)},${formatDate(tip.last_date)},${formatDate(tip.mcoldate)}`]); + for (tip of all_tips) { + if (tip.isTip === undefined || tip.isTip) + lineage_info.push([`${tip.thisLabel === undefined ? tip.label1 : tip.thisLabel},${Math.round(100*tip.mean_ndiffs)/100.},${Math.round(100*tip.residual)/100.},${tip.nsamples},${tip.varcount},${formatDate(tip.first_date)},${formatDate(tip.last_date)},${formatDate(tip.mcoldate)}`]); } csvFile = csvFile + "\n" + lineage_info.join("\n"); blob = new Blob([csvFile], {type: "text/csv"}); diff --git a/js/drawtree.js b/js/drawtree.js index 530000b0..62e65ef8 100644 --- a/js/drawtree.js +++ b/js/drawtree.js @@ -9,6 +9,7 @@ var sample_pal, coldate_pal, diverge_pal; // set up plotting scales var xValue = function(d) { return d.x; }, xScale = d3.scaleLinear().range([0, width]), + xMap = function(d) { return xScale(d.x); }, xMap1 = function(d) { return xScale(d.x1); }, // lines xMap2 = function(d) { return xScale(d.x2); }, xWide = function(d) { return xScale(d.x2 - d.x1)}; @@ -29,16 +30,18 @@ var vis = d3.select("div#svg-timetree") //.attr("height", height + margin.top + margin.bottom); //.append("g"); -var vis_recombinant = d3.select("div#svg-recombinants") - .append("svg") - .attr("width", width + margin.left + margin.right + padding.right); - var axis = d3.select("div#svg-timetreeaxis") .append("svg") .attr("width", width + margin.left + margin.right + padding.right) .attr("height", 25) .append("g"); +var timetree_axis, + axis_padding = -0.05, + axis_padding_trees = 0.05, + cluster_box_padding = 2, + numTicks = 3; + let cTooltip = d3.select("#tooltipContainer") .style("opacity", 0); @@ -52,48 +55,49 @@ var null_infections_colour = '#b5b5b5'; */ function draw_cluster_box(rect) { var d = rect.datum(); - var rectWidth = xScale(date_to_xaxis(d.last_date)) - xScale(d.x2); - - if (d.label1[0] == 'X') { - vis_recombinant.append("rect") - .attr('class', "clickedH") - .attr("x", function() { - return xScale(date_to_xaxis(d.first_date)) - 2; - }) - .attr("y", function() { - return d.y*11; - }) - .attr("width", function() { - rectWidth = (xScale(date_to_xaxis(d.last_date)) - xScale(date_to_xaxis(d.first_date))) + 4; - if (rectWidth < minRectWidth) - return minRectWidth + 4 - return rectWidth - }) - .attr("height", 14) - .attr("fill", "white") - .attr("stroke", "grey") - .attr("fill-opacity", 1) - .attr("stroke-width", 2); + var rectWidth = xScale(date_to_xaxis(d.last_date) - d.x); + + switch($("#display-tree").val()) { + case "Non-Recombinants": + case "XBB Lineages": + // draw a box around the cluster rectangle + vis.append("rect") + .attr('class', "clickedH") + .attr("x", xMap(d) - 2) + .attr("y", yMap(d) - 2) + .attr("width", function() { + if (rectWidth < minRectWidth) + return minRectWidth + 4 + return xScale(date_to_xaxis(d.last_date) - d.x) + (cluster_box_padding * 2) + }) + .attr("height", 14) + .attr("fill", "white") + .attr("stroke", "grey") + .attr("fill-opacity", 1) + .attr("stroke-width", 2); + break; + case "Other Recombinants": + vis.append("rect") + .attr('class', "clickedH") + .attr("x", function() { + return xScale(date_to_xaxis(d.first_date)) - cluster_box_padding; + }) + .attr("y", function() { + return d.y*11; + }) + .attr("width", function() { + rectWidth = xScale(date_to_xaxis(d.last_date) - date_to_xaxis(d.first_date)); + if (rectWidth < minRectWidth) + return minRectWidth + (cluster_box_padding * 2) + return rectWidth + (cluster_box_padding * 2) + }) + .attr("height", 14) + .attr("fill", "white") + .attr("stroke", "grey") + .attr("fill-opacity", 1) + .attr("stroke-width", 2); } - else { - // draw a box around the cluster rectangle - vis.append("rect") - .attr('class', "clickedH") - .attr("x", xMap2(d) - 2) - .attr("y", yMap(d) - 2) - .attr("width", function() { - if (rectWidth < minRectWidth) - return minRectWidth + 4 - return xScale(date_to_xaxis(d.last_date)) - xScale(d.x2) + 4 - }) - .attr("height", 14) - .attr("fill", "white") - .attr("stroke", "grey") - .attr("fill-opacity", 1) - .attr("stroke-width", 2); - } - - + // move the box to the background by promoting other objects rect.raise(); d3.select("#svg-timetree") @@ -115,39 +119,54 @@ function drawtree(df, org_df, redraw=true) { $('#tree-vscroll').css('margin-top', document.getElementById("tree-title").clientHeight + document.getElementById("svg-timetreeaxis").clientHeight + $('#inner-hscroll').height() + 5); $('#svg-timetreeaxis').css('padding-bottom', $('#inner-hscroll').height()) - var edgeset = edges(df, rectangular=true); - - // rescale SVG for size of tree - var ntips = df.map(x => x.children.length === 0).reduce((x,y) => x+y); - height = ntips*11 + margin.top + margin.bottom; - vis.attr("height", height); - yScale = d3.scaleLinear().range([height, 10]); // add room for time axis - // adjust d3 scales to data frame if(!redraw) { - xScale.domain([ - d3.min(org_df, xValue)-0.05, - date_to_xaxis(d3.max(org_df, function(d) {return d.last_date})) - ]); + switch($("#display-tree").val()) { + case "Other Recombinants": + xScale.domain([ + axis_padding, + date_to_xaxis(d3.max(recombinant_tips, function(d) {return d.last_date})) + ]); + break; + default: + xScale.domain([ + d3.min(org_df, xValue)-axis_padding_trees, + date_to_xaxis(d3.max(org_df, function(d) {return d.last_date})) + ]); + break; + } } - yScale.domain([ - d3.min(df, yValue)-1, d3.max(df, yValue)+1 - ]); - - // draw lines - vis.selectAll("lines") - .data(edgeset) - .enter().append("line") - .attr("class", "lines") - .attr("x1", xMap1) - .attr("y1", yMap1) - .attr("x2", xMap2) - .attr("y2", yMap2) - .attr("stroke-width", 1.5) - .attr("stroke", "#777"); - - $('#tree-inner-vscroll').css('height', $('.tree-content > svg').height()); + switch($("#display-tree").val()) { + case "Other Recombinants": + break; + default: + // rescale SVG for size of tree + var ntips = df.map(x => x.children.length === 0).reduce((x,y) => x+y); + height = ntips*11 + margin.top + margin.bottom; + vis.attr("height", height); + yScale = d3.scaleLinear().range([height, 10]); // add room for time axis + + yScale.domain([ + d3.min(df, yValue)-1, d3.max(df, yValue)+1 + ]); + + var edgeset = edges(df, rectangular=true); + + // draw lines + vis.selectAll("lines") + .data(edgeset) + .enter().append("line") + .attr("class", "lines") + .attr("x1", xMap1) + .attr("y1", yMap1) + .attr("x2", xMap2) + .attr("y2", yMap2) + .attr("stroke-width", 1.5) + .attr("stroke", "#777"); + + $('#tree-inner-vscroll').css('height', $('.tree-content > svg').height()); + } } @@ -160,9 +179,16 @@ function drawtree(df, org_df, redraw=true) { * @param tip: Object, representing a reference tip in the tree * @returns {string} new date in ISO format (yyyy-mm-dd) */ -function xaxis_to_date(x, tip) { +function xaxis_to_date(x, tip, earliest, latest) { var coldate = new Date(tip.first_date); // collection date of reference tip - coldate = d3.timeDay.offset(coldate, 365.25*(x - tip.x)); + var interval = d3.timeDay.count(earliest, latest)/numTicks + switch($("#display-tree").val()) { + case "Other Recombinants": + coldate = d3.timeDay.offset(coldate, interval*x); + break; + default: + coldate = d3.timeDay.offset(coldate, interval*(x - tip.x)); + } return (coldate.toISOString().split('T')[0]); } @@ -174,8 +200,23 @@ function xaxis_to_date(x, tip) { * @returns {float} x-coordinate value */ function date_to_xaxis(coldate) { - var numDays = d3.timeDay.count(tips[0].first_date, coldate) - return (numDays/365.25) + tips[0].x; + var numDays, earliest, latest, interval, tip_obj; + switch ($("#display-tree").val()) { + case "XBB Lineages": + tip_obj = df_xbb; + break; + case "Non-Recombinants": + tip_obj = df; + break; + case "Other Recombinants": + tip_obj = recombinant_tips; + } + + numDays = d3.timeDay.count(tip_obj[0].first_date, coldate); + earliest = d3.min(tip_obj, function(d) {return d.first_date}); + latest = d3.max(tip_obj, function(d) {return d.last_date}); + interval = d3.timeDay.count(earliest, latest)/numTicks; + return (numDays/interval) + tip_obj[0].x; } @@ -206,27 +247,52 @@ function sort_mutations(mutations) { * Add subtree objects to time-scaled tree. * @param {Array} tips, clusters that have been mapped to tips of tree */ -function draw_clusters(tips, recombinant_tips, redraw=false) { +function draw_clusters(tips, filtered_recombinant_tips, redraw=false) { + + var tickVals = [], minVal, maxVal, interval, tip_obj; + switch ($("#display-tree").val()) { + case "XBB Lineages": + minVal = d3.min(df_xbb, xValue)-axis_padding_trees; + maxVal = date_to_xaxis(d3.max(df_xbb, function(d) {return d.last_date})); + break; + case "Non-Recombinants": + minVal = d3.min(df, xValue)-axis_padding_trees; + maxVal = date_to_xaxis(d3.max(df, function(d) {return d.last_date})); + break; + case "Other Recombinants": + console.log(d3.min(recombinant_tips, function(d) {return d.first_date})) + minVal = axis_padding; + maxVal = date_to_xaxis(d3.max(recombinant_tips, function(d) {return d.last_date})); + break; + } - var tickVals = [], - minVal = d3.min(df, xValue)-0.05, - maxVal = date_to_xaxis(d3.max(df, function(d) {return d.last_date})), - interval = (maxVal - minVal)/3; + interval = (maxVal - minVal)/3; for (var i = 0; i < 3; i++) { tickVals.push(minVal + (interval/2) + (i*interval)); } if(!redraw) { + if (timetree_axis) { + timetree_axis.remove(); + } + // Draws the axis for the time scaled tree - axis.append("g") + timetree_axis = axis.append("g") .attr("class", "treeaxis") .attr("transform", "translate(0,20)") .call(d3.axisTop(xScale) - .ticks(3) + .ticks(numTicks) .tickValues(tickVals) .tickFormat(function(d) { - return xaxis_to_date(d, tips[0]) + switch($("#display-tree").val()) { + case "Other Recombinants": + return xaxis_to_date(d, recombinant_tips[0], d3.min(recombinant_tips, function(d) {return d.first_date}), d3.max(recombinant_tips, function(d) {return d.last_date})) + case "XBB Lineages": + return xaxis_to_date(d, df_xbb[0], d3.min(df_xbb, function(d) {return d.first_date}), d3.max(df_xbb, function(d) {return d.last_date})) + default: + return xaxis_to_date(d, df[0], d3.min(df, function(d) {return d.first_date}), d3.max(df, function(d) {return d.last_date})) + } }) ); } @@ -266,96 +332,106 @@ function draw_clusters(tips, recombinant_tips, redraw=false) { }); } - vis.selectAll("rect") - .data(tips) - .enter() - .lower() - .append("rect") - //.attr("selected", false) - .attr("x", xMap2) - .attr("y", yMap) - .attr("width", function(d) { - var rectWidth = xScale(date_to_xaxis(d.last_date)) - xScale(d.x2); - if (rectWidth < minRectWidth) - return minRectWidth; - return rectWidth; - }) - .attr("height", 10) - .attr("class", "default") - .attr("cidx", function(d) { return "cidx-" + d.cluster_idx; }) - .attr("id", function(d, i) { return "id-" + map_cidx_to_id['cidx-' + d.cluster_idx]; }) - .on('mouseover', mouseover) - .on("mouseout", function() { - d3.select(this) - .attr("txt_hover", null); - - cTooltip.transition() // Hide tooltip - .duration(50) - .style("opacity", 0); - }) - .on("click", async function(d) { - var cluster_info = this; - $('#error_message').text(``); - $("#loading").show(); - $("#loading_text").text(`Loading. Please Wait...`); - await click_cluster(d, cluster_info); - $("#loading").hide(); - $("#loading_text").text(``); - }); + // Add logic for XBB Tree + switch ($("#display-tree").val()) { + case "XBB Lineages": + case "Non-Recombinants": + vis.selectAll("rect") + .data(tips) + .enter() + .lower() + .append("rect") + .attr("x", xMap) + .attr("y", yMap) + .attr("width", function(d) { + var rectWidth = xScale(date_to_xaxis(d.last_date) - d.x) + axis_padding_trees; + if (rectWidth < minRectWidth) + return minRectWidth; + return rectWidth; + }) + .attr("height", 10) + .attr("class", "default") + .attr("cidx", function(d) { return "cidx-" + d.cluster_idx; }) + .attr("id", function(d, i) { return "id-" + map_cidx_to_id['cidx-' + d.cluster_idx]; }) + .on('mouseover', mouseover) + .on("mouseout", function() { + d3.select(this) + .attr("txt_hover", null); + + cTooltip.transition() // Hide tooltip + .duration(50) + .style("opacity", 0); + }) + .on("click", async function(d) { + var cluster_info = this; + $('#error_message').text(``); + $("#loading").show(); + $("#loading_text").text(`Loading. Please Wait...`); + await click_cluster(d, cluster_info); + $("#loading").hide(); + $("#loading_text").text(``); + }); - vis_recombinant - .attr("height", recombinant_tips.length * 11.1) - - vis_recombinant.selectAll("rect") - .data(recombinant_tips) - .enter() - .lower() - .append("rect") - //.attr("selected", false) - .attr("x", function(d) { - return xScale(date_to_xaxis(d.first_date)) - }) - .attr("y", function(d) { - return d.y*11 + 2; - }) - .attr("width", function(d) { - var rectWidth = xScale(date_to_xaxis(d.last_date)) - xScale(date_to_xaxis(d.first_date)); - if (rectWidth < minRectWidth) - return minRectWidth; - return rectWidth; - }) - .attr("height", 10) - .attr("class", "default recombinant") - .attr("cidx", function(d) { return "cidx-" + d.cluster_idx; }) - .attr("id", function(d, i) { return "id-" + map_cidx_to_id['cidx-' + d.cluster_idx]; }) - .on('mouseover', mouseover) - .on("mouseout", function() { - d3.select(this) - .attr("txt_hover", null); - - cTooltip.transition() // Hide tooltip - .duration(50) - .style("opacity", 0); - }) - .on("click", async function(d) { - var cluster_info = this; - $('#error_message').text(``); - $("#loading").show(); - $("#loading_text").text(`Loading. Please Wait...`); - await click_cluster(d, cluster_info); - $("#loading").hide(); - $("#loading_text").text(``); - }); + tip_obj = tips; + + break; + case "Other Recombinants": + vis.selectAll("rect") + .data(filtered_recombinant_tips) + .enter() + .lower() + .append("rect") + .attr("x", function(d) { + return xScale(date_to_xaxis(d.first_date)) + }) + .attr("y", function(d) { + return d.y*11 + 2; + }) + .attr("width", function(d) { + var rectWidth = xScale(date_to_xaxis(d.last_date) - date_to_xaxis(d.first_date)); + if (rectWidth < minRectWidth) + return minRectWidth; + return rectWidth; + }) + .attr("height", 10) + .attr("class", "default recombinant") + .attr("cidx", function(d) { return "cidx-" + d.cluster_idx; }) + .attr("id", function(d, i) { return "id-" + map_cidx_to_id['cidx-' + d.cluster_idx]; }) + .on('mouseover', mouseover) + .on("mouseout", function() { + d3.select(this) + .attr("txt_hover", null); + + cTooltip.transition() // Hide tooltip + .duration(50) + .style("opacity", 0); + }) + .on("click", async function(d) { + var cluster_info = this; + $('#error_message').text(``); + $("#loading").show(); + $("#loading_text").text(`Loading. Please Wait...`); + await click_cluster(d, cluster_info); + $("#loading").hide(); + $("#loading_text").text(``); + }); + + vis + .attr("height", filtered_recombinant_tips.length * 12) // Height of a rect element is 10 + + tip_obj = recombinant_tips; + $('#tree-inner-vscroll').css('height', $('.tree-content > svg').height()); + } // generate colour palettes sample_pal = d3.scaleSequential(d3.interpolatePuBu) - .domain(d3.extent(tips, function(d) { return Math.log10(d.nsamples); })); + .domain(d3.extent(tip_obj, function(d) { return Math.log10(d.nsamples); })); coldate_pal = d3.scaleSequential(d3.interpolateCividis) - .domain(d3.extent(tips, function(d) { return d.last_date; })); + .domain(d3.extent(tip_obj, function(d) { return d.last_date; })); diverge_pal = d3.scaleSequential(d3.interpolatePlasma) - .domain(d3.extent(tips, function(d) { return d.residual; })); + .domain(d3.extent(tip_obj, function(d) { return d.residual; })); infections_pal = d3.scaleSequential(d3.interpolateViridis) - .domain([0, d3.max(tips, function(d) { return d.infections; })]); + .domain([0, d3.max(tip_obj, function(d) { return d.infections; })]); generate_legends(); changeTreeColour(); @@ -364,62 +440,66 @@ function draw_clusters(tips, recombinant_tips, redraw=false) { .selectAll("line") .raise(); - vis.selectAll("text") - .data(tips) - .enter().append("text") - .style("font-size", "10px") - .attr("text-anchor", "start") - .attr("alignment-baseline", "middle") - .attr("cursor", "default") - .attr("id", function(d) { return "cidx-" + d.cluster_idx; }) - .attr("x", function(d) { - var rectWidth = xScale(date_to_xaxis(d.last_date)) - xScale(d.x2); - if (rectWidth < minRectWidth) - return xScale(d.x2) + minRectWidth + 3; - return xScale(d.x2) + rectWidth + 3; - }) - .attr("y", function(d) { - return(yScale(d.y-0.15)); - }) - .text(function(d) { return(d.label1); }) - .on("mouseover", function(d) { - mouseover(d); - }) - .on("mouseout", function(d) { - d3.select("[cidx=cidx-" + d.cluster_idx + "]").dispatch('mouseout'); - }) - .on("click", function(d) { - d3.select("[cidx=cidx-" + d.cluster_idx + "]").dispatch('click'); - }); - - vis_recombinant.selectAll("text") - .data(recombinant_tips) - .enter().append("text") - .style("font-size", "10px") - .attr("text-anchor", "start") - .attr("alignment-baseline", "middle") - .attr("cursor", "default") - .attr("id", function(d) { return "cidx-" + d.cluster_idx; }) - .attr("x", function(d) { - var rectWidth = xScale(date_to_xaxis(d.last_date)) - xScale(date_to_xaxis(d.first_date)); - if (rectWidth < minRectWidth) - return xScale(date_to_xaxis(d.first_date)) + minRectWidth + 3; - return xScale(date_to_xaxis(d.first_date)) + rectWidth + 3; - }) - .attr("y", function(d) { - // return(yScale(d.y-0.15)); - return d.y*11 + 7; - }) - .text(function(d) { return(d.label1); }) - .on("mouseover", function(d) { - mouseover(d); - }) - .on("mouseout", function(d) { - d3.select("[cidx=cidx-" + d.cluster_idx + "]").dispatch('mouseout'); - }) - .on("click", function(d) { - d3.select("[cidx=cidx-" + d.cluster_idx + "]").dispatch('click'); - }); + switch ($("#display-tree").val()) { + case "XBB Lineages": + case "Non-Recombinants": + vis.selectAll("text") + .data(tips) + .enter().append("text") + .style("font-size", "10px") + .attr("text-anchor", "start") + .attr("alignment-baseline", "middle") + .attr("cursor", "default") + .attr("id", function(d) { return "cidx-" + d.cluster_idx; }) + .attr("x", function(d) { + var rectWidth = xScale(date_to_xaxis(d.last_date) - d.x) + axis_padding_trees + 4; + if (rectWidth < minRectWidth) + return xScale(d.x) + minRectWidth + cluster_box_padding; + return xScale(d.x) + rectWidth + cluster_box_padding; + }) + .attr("y", function(d) { + return(yScale(d.y-0.15)); + }) + .text(function(d) { return(d.label1); }) + .on("mouseover", function(d) { + mouseover(d); + }) + .on("mouseout", function(d) { + d3.select("[cidx=cidx-" + d.cluster_idx + "]").dispatch('mouseout'); + }) + .on("click", function(d) { + d3.select("[cidx=cidx-" + d.cluster_idx + "]").dispatch('click'); + }); + break; + default: + vis.selectAll("text") + .data(filtered_recombinant_tips) + .enter().append("text") + .style("font-size", "10px") + .attr("text-anchor", "start") + .attr("alignment-baseline", "middle") + .attr("cursor", "default") + .attr("id", function(d) { return "cidx-" + d.cluster_idx; }) + .attr("x", function(d) { + var rectWidth = xScale(date_to_xaxis(d.last_date) - date_to_xaxis(d.first_date)); + if (rectWidth < minRectWidth) + return xScale(date_to_xaxis(d.first_date)) + minRectWidth + (cluster_box_padding + 1); + return xScale(date_to_xaxis(d.first_date)) + rectWidth + (cluster_box_padding + 1); + }) + .attr("y", function(d) { + return d.y*11 + 7; + }) + .text(function(d) { return(d.label1); }) + .on("mouseover", function(d) { + mouseover(d); + }) + .on("mouseout", function(d) { + d3.select("[cidx=cidx-" + d.cluster_idx + "]").dispatch('mouseout'); + }) + .on("click", function(d) { + d3.select("[cidx=cidx-" + d.cluster_idx + "]").dispatch('click'); + }); + } } @@ -459,39 +539,12 @@ function changeTreeColour() { else return dsc_infections_colour; } else { // Divergence - $("div#svg-diverge-legend").show(); - return(diverge_pal(d.residual)); - } - } - }) - - vis_recombinant.selectAll("rect") - .transition() - .duration(300) - .style("fill", function(d) { - if (d !== undefined) { - let opt = $("#select-tree-colours").val(); - if (opt === "Region") { - $("#div-region-legend").show(); - return(country_pal[d.region]); - } - else if (opt === "No. samples") { - $("div#svg-sample-legend").show(); - return(sample_pal(Math.log10(d.nsamples))); // placeholder values - } - else if (opt === "Collection date") { - $("div#svg-coldate-legend").show(); - return(coldate_pal(d.last_date)); - } - else if (opt === "Infections") { - $("div#svg-infections-legend").show(); - if (d.infections > 0) return infections_pal(d.infections); - else if (d.infections == 0) return null_infections_colour; - else return dsc_infections_colour; - } - else { // Divergence - $("div#svg-diverge-legend").show(); - return(diverge_pal(d.residual)); + // Issue 489 - Do not colour other recombinants + if ($("#display-tree").val() !== "Other Recombinants") { + $("div#svg-diverge-legend").show(); + return(diverge_pal(d.residual)); + } + return '#949391'; } } }) @@ -503,6 +556,72 @@ $("#select-tree-colours").change(function() { }); +async function changeDisplay() { + var curr_date = new Date(); + curr_date.setFullYear(curr_date.getFullYear() - 1); + + var handle = $( "#tree-slider-handle" ); + var cutoff_date = $( "#cutoff-date" ); + var cutoff_line = $("#cutoff-line"); + var tree_cutoff = $("#tree-cutoff"); + + var tips_obj; + switch($("#display-tree").val()) { + case "XBB Lineages": + $('#nwk-button').show(); + tips_obj = df_xbb; + break; + case "Other Recombinants": + $('#nwk-button').hide(); + tips_obj = recombinant_tips; + break; + default: + $('#nwk-button').show(); + tips_obj = df; + } + + const tree_multiplier = 100000; // Slider value needs to be an integer + var min = (d3.min(tips_obj, xValue)-0.05) * tree_multiplier; + var max = date_to_xaxis(d3.max(tips_obj, function(d) {return d.last_date})) * tree_multiplier; + var start_value = date_to_xaxis(curr_date) * tree_multiplier; + + $("#tree-slider").slider("option", "min", min); + $("#tree-slider").slider("option", "max", max); + $("#tree-slider").slider("option", "value", (start_value > min && start_value < max) ? start_value : min); + + $("#cutoff-date").text(xaxis_to_date($("#tree-slider").slider("option", "value")/tree_multiplier, tips_obj[0], d3.min(tips_obj, function(d) {return d.first_date}), d3.max(tips_obj, function(d) {return d.last_date}))); + $("#tree-cutoff").css('left', $("#tree-slider-handle").position().left); + + $("#tree-slider").slider("option", "slide", async function( event, ui ) { + cutoff_date.text(xaxis_to_date(ui.value/tree_multiplier, tips_obj[0], d3.min(tips_obj, function(d) {return d.first_date}), d3.max(tips_obj, function(d) {return d.last_date}))); + await handle.change() + cutoff_line.css('visibility', 'visible'); + var cutoff_pos = handle.position().left; + cutoff_line.css('left', cutoff_pos + 29); + tree_cutoff.css('left', cutoff_pos); + }); + await redraw_tree(tips_obj, formatDate(curr_date), redraw=false); + + // Draw beadplot and update tables + var rect = d3.selectAll("#svg-timetree > svg > rect"), + node = $("#display-tree").val() === "Other Recombinants" ? rect.nodes()[0] : rect.nodes()[rect.size()-1]; + + cindex = node.__data__.cluster_idx; + d3.select(node).attr("class", "clicked"); + draw_cluster_box(d3.select(node)); + + await beadplot(cindex); + gentable(node.__data__); + draw_region_distribution(node.__data__.allregions); + gen_details_table(points); // update details table with all samples + gen_mut_table(mutations[cindex]); +} + +$("#display-tree").change(async function() { + await changeDisplay(); +}); + + // from https://observablehq.com/@d3/color-legend function legend({ color, @@ -814,10 +933,9 @@ async function click_cluster(d, cluster_info) { draw_cluster_box(d3.select(cluster_info)); } -async function redraw_tree(cutoff_date, redraw=true, partial_redraw=false) { +async function redraw_tree(df, cutoff_date, redraw=true, partial_redraw=false) { // deep copy the df and clear all references to children df_copy = structuredClone(df); - var df_copy = df_copy.map(x => { x.children = []; return x; @@ -825,7 +943,7 @@ async function redraw_tree(cutoff_date, redraw=true, partial_redraw=false) { // filter for tips with a collection date after the cutoff var filtered_df = df_copy.filter(x => { - if (formatDate(x.coldate) >= cutoff_date && x.isTip == true && !x.rawLabel.startsWith("X")) return x; + if (formatDate(x.coldate) >= cutoff_date && x.isTip == true) return x; }); if(filtered_df.length > 0) { @@ -914,18 +1032,7 @@ async function redraw_tree(cutoff_date, redraw=true, partial_redraw=false) { x.y = i }) - if (filtered_recomb_tips.length === 0) { - $(".recombinant-tree-content").hide() - $(".recombtitle").hide() - } - else if ($('#display-option').attr('checked')) { - $(".recombinant-tree-content").show() - $(".recombtitle").show() - } - - document.querySelector("#svg-timetree > svg").innerHTML = ''; - document.querySelector("#svg-recombinants > svg").innerHTML = ''; if (partial_redraw) { drawtree(final_df, df_copy, redraw=partial_redraw); draw_clusters(filtered_tips, filtered_recomb_tips, partial_redraw); @@ -961,6 +1068,12 @@ function reset_tree(partial_redraw=false) { $("#cutoff-date").text(min_date); $("#tree-cutoff").css('left', $("#tree-slider-handle").position().left); $("#tree-slider").slider({ disabled: true}); - redraw_tree(min_date, redraw=true, partial_redraw=partial_redraw); + switch($("#display-tree").val()) { + case "XBB Lineages": + redraw_tree(df_xbb, min_date, redraw=true, partial_redraw=partial_redraw); + break; + default: + redraw_tree(df, min_date, redraw=true, partial_redraw=partial_redraw); + } if (partial_redraw) d3.select('#cidx-' + cindex).attr("class", "clicked"); } \ No newline at end of file diff --git a/js/search.js b/js/search.js index 90b8a9d5..ec6209a3 100644 --- a/js/search.js +++ b/js/search.js @@ -170,11 +170,37 @@ async function main_search(all_bead_data, text_query, start_date, end_date) { return a - b; }); + await reset_tree(partial_redraw=true); + d3.selectAll("rect[class='clicked']").attr("class", "default") + // Keeps track of the clicked cluster - var curr_cluster = d3.selectAll(".clicked").nodes()[0].attributes.id.nodeValue; - var selected_cidx = id_to_cidx[closest_match(curr_cluster, hit_id)]; + var selected_cidx, + curr_cluster = d3.selectAll(".clicked").nodes()[0].attributes.id.nodeValue; + + switch($("#display-tree").val()) { + case "Other Recombinants": + selected_cidx = closest_display_match(curr_cluster, hit_id, display_id.other_recombinants.first, display_id.other_recombinants.last); + break; + case "XBB Lineages": + selected_cidx = closest_display_match(curr_cluster, hit_id, display_id.xbb.first, display_id.xbb.last); + break; + default: + selected_cidx = closest_display_match(curr_cluster, hit_id, display_id.non_recombinants.first, display_id.non_recombinants.last); + } - var selections = d3.selectAll("#svg-timetree > svg > rect:not(.clickedH), #svg-recombinants > svg > rect:not(.clickedH)") + if (selected_cidx === null) { + // If null, then there isn't a match in the selected display. Figure out which display has the closest idx + selected_cidx = closest_match(curr_cluster, hit_id); + + var display = await getdata(`/api/display/${id_to_cidx[selected_cidx].substring(5)}`); + await $("#display-tree").val(display[0]) + await changeDisplay(); + await reset_tree(partial_redraw=true); + } + + selected_cidx = id_to_cidx[selected_cidx]; + + var selections = d3.selectAll("#svg-timetree > svg > rect:not(.clickedH)") .filter(function() { return hit_id.includes(parseInt(this.id.substring(3))) }); @@ -236,6 +262,16 @@ async function lineage_search(text_query) { return; } + // Get the lineage to determine if the tree needs to be redrawn + var display = await getdata(`/api/display/${cidx}`); + + if (!($("#display-tree").val() === display[0])) { + await $("#display-tree").val(display[0]) + await changeDisplay(); + } + + await reset_tree(partial_redraw=true); + var cluster = select_cluster("cidx-"+cidx); // Reduces the opacity of all clusters @@ -268,6 +304,16 @@ async function accession_search(text_query) { return; } + // Get the lineage to determine if the tree needs to be redrawn + var display = await getdata(`/api/display/${cidx}`); + + if (!($("#display-tree").val() === display[0])) { + await $("#display-tree").val(display[0]) + await changeDisplay(); + } + + await reset_tree(partial_redraw=true); + var cluster = select_cluster("cidx-"+cidx); d3.select("#svg-timetree") @@ -356,6 +402,33 @@ async function select_next_prev_bead(bead_id_to_accession, curr_bead) { .then(response => response.text()) .then(data => curr_cid = data); + // Get the lineage to determine if the tree needs to be redrawn + var display = await getdata(`/api/display/${curr_cid}`); + + if (!($("#display-tree").val() === display[0])) { + await $("#display-tree").val(display[0]) + await changeDisplay(); + await reset_tree(partial_redraw=true); + + // Change opacity of clusters with results + var hits = search_results.get().hit_ids; + + var selections = d3.selectAll("#svg-timetree > svg > rect:not(.clickedH)") + .filter(function() { + return hits.includes(parseInt(this.id.substring(3))) + }); + + // Reduces the opacity of all clusters + d3.select("#svg-timetree") + .selectAll("rect:not(.clicked):not(.clickedH)") + .attr("class","not_SelectedCluster"); + + // Increases the opacity for only the clusters with hits + for (const node of selections.nodes()){ + d3.select(node).attr("class", "SelectedCluster"); + } + } + var next_cluster = d3.selectAll('rect[cidx="cidx-'+curr_cid+'"]'); d3.selectAll("rect.clickedH").remove(); d3.selectAll(".SelectedCluster.clicked").attr('class', 'SelectedCluster'); @@ -577,38 +650,68 @@ function select_bead_by_accession(accn, reset_clusters_tree = true) { } } -function find_beads_points(beadsdata){ - return beadsdata.map(bead => bead.points).flat() -} - /** * This function searches for the closest cluster with a search hit (Binary search) * @param non_hit_cluster_index: cidx value * @param hit_ids: Sorted list of ids (not cidx) of clusters with a hit for the search query */ +function closest_display_match(non_hit_cluster_index, hit_ids, minRange, maxRange) { + const current_index = map_cidx_to_id[non_hit_cluster_index]; + + var i = 0, j = hit_ids.length, mid = 0, currentValue = null, closestValue = null; + while (i < j) { + mid = Math.floor((i + j)/2); + currentValue = hit_ids[mid]; + + if (hit_ids[mid] === current_index) + return hit_ids[mid]; + + if (minRange <= currentValue && currentValue <= maxRange) { + if (closestValue === null || Math.abs(currentValue - current_index) < Math.abs(closestValue - current_index)) + closestValue = currentValue; + + if (currentValue < current_index) + i = mid + 1; + else + j = mid - 1; + } + else if (currentValue < minRange) + i = mid + 1; + else + j = mid - 1; + } + return closestValue; +} + +/** + * Find the closest id from a sorted list + * @param non_hit_cluster_index + * @param hit_ids + * @returns {*} + */ function closest_match(non_hit_cluster_index, hit_ids) { - const target_index = map_cidx_to_id[non_hit_cluster_index]; + const current_index = map_cidx_to_id[non_hit_cluster_index]; - if (target_index <= hit_ids[0]) + if (current_index <= hit_ids[0]) return hit_ids[0]; - - if (target_index >= hit_ids[hit_ids.length - 1]) + + if (current_index >= hit_ids[hit_ids.length - 1]) return hit_ids[hit_ids.length - 1]; var i = 0, j = hit_ids.length, mid = 0; while (i < j) { mid = Math.floor((i + j)/2); - if (hit_ids[mid] == target_index) + if (hit_ids[mid] == current_index) return hit_ids[mid]; - if (target_index < hit_ids[mid]) { - if (mid > 0 && target_index > hit_ids[mid - 1]) - return getClosest(hit_ids[mid-1], hit_ids[mid], target_index); + if (current_index < hit_ids[mid]) { + if (mid > 0 && current_index > hit_ids[mid - 1]) + return getClosest(hit_ids[mid-1], hit_ids[mid], current_index); j = mid; } else { - if (mid < hit_ids.length - 1 && target_index < hit_ids[mid+1]) - return getClosest(hit_ids[mid], hit_ids[mid+1], target_index) + if (mid < hit_ids.length - 1 && current_index < hit_ids[mid+1]) + return getClosest(hit_ids[mid], hit_ids[mid+1], current_index) i = mid + 1; } } diff --git a/mongodb_scripts/createUsers.js b/mongodb_scripts/createUsers.js new file mode 100644 index 00000000..6bd24a4f --- /dev/null +++ b/mongodb_scripts/createUsers.js @@ -0,0 +1,77 @@ + +/** +A nodeJS script to create the user accounts for the MongoDB database +You must disable authentication for the mongodb server before running this script. +To disable authentication, edit the last two lines of the mongod.conf file +The MongoDB config files are usually located at +MAC INTEL /usr/local/etc/mongod.conf +MAC M1 /opt/homebrew/etc/mongod.conf + +Add/Remove the following two lines at the bottom of the config file to enable/disable authentication + security: + authorization: enabled +Use a # to comment out the line + +After disabling authentication, restart the mongodb service + brew services restart mongodb/brew/mongodb-community + sudo systemctl restart mongodb +*/ +const MongoClient = require('mongodb').MongoClient; + +const { + $ADMIN_USERNAME, + $ADMIN_PASSWORD, + $COVIZU_USERNAME, + $COVIZU_PASSWORD, + $DB_URL, + $DATABASE__PRIMARY, + $DATABASE__SECONDARY +} = require('../config/dbconfig') + +const url = `mongodb://${$DB_URL}`; + +// Create a new user account +async function createUserAccounts() { + const client = new MongoClient(url); + + try { + // Connect to MongoDB server + await client.connect(); + + // Access the admin database + const adminDb = client.db('admin'); + + let result; + + result = await adminDb.command({ + createUser: $ADMIN_USERNAME, + pwd: $ADMIN_PASSWORD, + roles: [ + { role: "root", db: $ADMIN_USERNAME }, + { role: "dbAdmin", db: $DATABASE__PRIMARY }, + { role: "dbAdmin", db: $DATABASE__SECONDARY }, + ] + }); + console.log('Admin user account created successfully:', result); + + + result = await adminDb.command({ + createUser: $COVIZU_USERNAME, + pwd: $COVIZU_PASSWORD, + roles: [ + { role: "read", db: $DATABASE__PRIMARY }, + { role: "read", db: $DATABASE__SECONDARY }, + ] + }); + console.log('User covizu account created successfully:', result); + + } catch (error) { + console.error('Error creating user account:', error); + } finally { + // Close the connection + client.close(); + } +} + +// Call the function to create the user account +createUserAccounts(); diff --git a/mongodb_scripts/deleteUsers.js b/mongodb_scripts/deleteUsers.js new file mode 100644 index 00000000..d3de9dfd --- /dev/null +++ b/mongodb_scripts/deleteUsers.js @@ -0,0 +1,61 @@ +/** +A nodeJS script to delete all exisiting user accounts in the MongoDB database +You must disable authentication for the mongodb server before running this script. +To disable authentication, edit the last two lines of the mongod.conf file +The MongoDB config files are usually located at +MAC INTEL /usr/local/etc/mongod.conf +MAC M1 /opt/homebrew/etc/mongod.conf + +Add/Remove the following two lines at the bottom of the config file to enable/disable authentication + security: + authorization: enabled +Use a # to comment out the line + +After disabling authentication, restart the mongodb service + brew services restart mongodb/brew/mongodb-community + sudo systemctl restart mongodb +*/ + +const MongoClient = require('mongodb').MongoClient; + +const { + $ADMIN_USERNAME, + $COVIZU_USERNAME, + $DB_URL, +} = require('../config/dbconfig') + +const url = `mongodb://${$DB_URL}`; + +// Find all user accounts +async function deleteAllUserAccounts() { + const client = new MongoClient(url); + let existingUsers = []; + // Connect to MongoDB server + await client.connect(); + // Access the admin database + const adminDb = client.db($ADMIN_USERNAME); + + // Get the list of user accounts + const users = await adminDb.command({ usersInfo: 1 }); + + console.log('User accounts:', users.users); + + try { + await adminDb.command({ dropUser: $ADMIN_USERNAME}); + console.log(`Deleted user account ${$ADMIN_USERNAME}.${$ADMIN_USERNAME}`) + } + catch(error){ + console.log(`Failed to delete user account ${$ADMIN_USERNAME}: ${error}`); + } + try + { + await adminDb.command({ dropUser: $COVIZU_USERNAME}); + console.log(`Deleted user account ${$ADMIN_USERNAME}.${$COVIZU_USERNAME}`) + } + catch(error){ + console.log(`Failed to delete user account ${$COVIZU_USERNAME}: ${error}`); + } + + await client.close(); +} +deleteAllUserAccounts(); \ No newline at end of file diff --git a/mongodb_scripts/updateDatabase.js b/mongodb_scripts/updateDatabase.js new file mode 100644 index 00000000..343aad27 --- /dev/null +++ b/mongodb_scripts/updateDatabase.js @@ -0,0 +1,254 @@ +const { MongoClient } = require("mongodb"); +const readline = require("readline"); +const fs = require('fs'); + +const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout +}); + +const { + parse_clusters, + map_clusters_to_tips, + index_accessions, + index_lineage +} = require('../server/parseCluster') + +const { readTree } = require("../server/phylo"); + +// we have to use globalVariables because the parse_clusters updates the global.region_map on the fly. +require("../globalVariables") + +const { + $ADMIN_CONNECTION_URI, + $ACTIVE_DATABASE, + $COLLECTION__CLUSTERS, + $COLLECTION__DBSTATS, + $COLLECTION__BEADDATA, + $COLLECTION__TIPS, + $COLLECTION__RECOMBINANT_TIPS, + $COLLECTION__ACCN_TO_CID, + $COLLECTION__LINEAGE_TO_CID, + $COLLECTION__REGION_MAP, + $COLLECTION__DF_TREE, + $COLLECTION__XBB_TREE, + $COLLECTION__AUTOCOMPLETE_DATA, + $COLLECTION__FLAT_DATA, + $PROJECT_ROOT, + $JSON_DATA_FOLDER, + $JSONFILE__CLUSTERS, + $JSONFILE__DBSTATS, + $NWKFILE__TREE, + $XBB__TREE +} = require('../config/dbconfig') + + +const adminUserConnection = new MongoClient($ADMIN_CONNECTION_URI); +console.log( + `ADMIN_CONNECTION_URI: ${$ADMIN_CONNECTION_URI} + ACTIVE_DATABASE: ${$ACTIVE_DATABASE} + COLLECTION__CLUSTERS: ${$COLLECTION__CLUSTERS} + COLLECTION__BEADDATA: ${$COLLECTION__BEADDATA} + COLLECTION__TIPS: ${$COLLECTION__TIPS} + COLLECTION__RECOMBINANT_TIPS ${$COLLECTION__RECOMBINANT_TIPS} + COLLECTION__ACCN_TO_CID ${$COLLECTION__ACCN_TO_CID} + COLLECTION__LINEAGE_TO_CID ${$COLLECTION__LINEAGE_TO_CID} + COLLECTION__REGION_MAP ${$COLLECTION__REGION_MAP} + COLLECTION__DF_TREE ${$COLLECTION__DF_TREE} + COLLECTION__AUTOCOMPLETE_DATA ${$COLLECTION__AUTOCOMPLETE_DATA} + COLLECTION__FLAT_DATA ${$COLLECTION__FLAT_DATA} + PROJECT_ROOT ${$PROJECT_ROOT} + JSON_DATA_FOLDER ${$JSON_DATA_FOLDER} + JSONFILE__CLUSTERS ${$JSONFILE__CLUSTERS} + JSONFILE__DBSTATS ${$JSONFILE__DBSTATS} + NWKFILE_TREE ${$NWKFILE__TREE} + XBB__TREE ${$XBB__TREE}` +) + +if(!$ACTIVE_DATABASE) +{ + throw new Error("Environment variable DBNUMBER must be 1 or 2"); +} + +async function updateDatabase() { + + console.log('starting connection...'); + adminUserConnection.connect() + .then((res) => { + console.log("Connecting as covizu_admin...") + }) + .catch((error) => { + console.log(`Error while connecting as covizu_admin:${error}`); + throw new Error(error); + }) + .then(() => { + console.log("CONNECTED!"); + }) + .then(async () => { + let db; + + db = adminUserConnection.db($ACTIVE_DATABASE); + + /** delete all collections */ + const existingCollections = await db.listCollections().toArray(); + existingCollections.forEach((e)=>{ + console.log(`Deleting collection ${$ACTIVE_DATABASE}.${e.name}`); + db.collection(e.name).drop(); + }) + + let textfile; + textfile = $PROJECT_ROOT+"/"+$JSON_DATA_FOLDER+"/"+$NWKFILE__TREE; + console.log(`Reading ${textfile}`); + try { + // global.tree = fs.readFileSync(`${$JSON_DATA_FOLDER}/${$NWKFILE__TREE}`, 'utf8'); + global.tree = fs.readFileSync(textfile, 'utf8'); + } + catch (e) { + console.error(`Failed reading ${textfile} : `, e); + return; + } + + textfile = $PROJECT_ROOT+"/"+$JSON_DATA_FOLDER+"/"+$XBB__TREE; + console.log(`Reading ${textfile}`); + try { + global.xbbtree = fs.readFileSync(textfile, 'utf8'); + } + catch (e) { + console.error(`Failed reading ${textfile} : `, e); + return; + } + + textfile = $PROJECT_ROOT+"/"+$JSON_DATA_FOLDER+"/"+$JSONFILE__CLUSTERS; + console.log(`Reading ${textfile}`); + try{ + global.clusters = require(textfile); + } + catch(e) + { + console.error(`Failed reading ${textfile} : `, e); + } + + textfile = $PROJECT_ROOT+"/"+$JSON_DATA_FOLDER+"/"+$JSONFILE__DBSTATS; + console.log(`Reading ${textfile}`); + try{ + global.dbstats = require(textfile); + } + catch(e) + { + console.error(`Failed reading ${textfile} : `, e); + } + + console.log("Preparing beaddata from clusters"); + global.beaddata = parse_clusters(global.clusters); + + console.log("Preparing df from tree"); + global.df = readTree(global.tree); + global.df_xbb = readTree(global.xbbtree); + + console.log("Preparing tips, recombinant_tips from df,clusters") + const { tips, tips_xbb, recombinant_tips } = map_clusters_to_tips(global.df, global.df_xbb, global.clusters); + global.tips = tips; + global.recombinant_tips = recombinant_tips; + + console.log("Preparing accn_to_cid from clusters") + global.accn_to_cid = index_accessions(global.clusters) + + console.log("Preparing lineage_to_cid from clusters"); + global.lineage_to_cid = index_lineage(global.clusters) + + console.log("Preparing autocomplete_data from accn_to_cid and lineage_to_cid"); + global.autocomplete_data = Object.keys(global.accn_to_cid).sort() + .concat(Object.keys(global.lineage_to_cid).sort()) + .map(accn => [global.normalize(accn), accn]); + + console.log("Preparing flat_data from beaddata"); + global.flat_data = global.beaddata.map(bead=>bead.points).flat(); + + + console.log("Writing to database...") + let res; + + res = await db.collection($COLLECTION__CLUSTERS).insertMany(global.clusters); + console.log(`Created ${res.insertedCount} documents in ${$ACTIVE_DATABASE}.${$COLLECTION__CLUSTERS}`); + delete global.clusters; + + global.lineage_stats = Object.entries(global.dbstats['lineages']).map(([key, value]) => ({ _id: key, ...value})); + res = await db.collection($COLLECTION__DBSTATS).insertMany(global.lineage_stats); + console.log(`Created ${res.insertedCount} documents in ${$ACTIVE_DATABASE}.${$COLLECTION__DBSTATS}`); + delete global.dbstats; + delete global.lineage_stats; + + res = await db.collection($COLLECTION__BEADDATA).insertMany(global.beaddata); + console.log(`Created ${res.insertedCount} documents in ${$ACTIVE_DATABASE}.${$COLLECTION__BEADDATA}`); + delete global.beaddata; + + res = await db.collection($COLLECTION__TIPS).insertMany(global.tips); + console.log(`Created ${res.insertedCount} documents in ${$ACTIVE_DATABASE}.${$COLLECTION__TIPS}`); + delete global.tips; + + if (global.recombinant_tips.length > 0) { + res = await db.collection($COLLECTION__RECOMBINANT_TIPS).insertMany(global.recombinant_tips); + console.log(`Created ${res.insertedCount} documents in ${$ACTIVE_DATABASE}.${$COLLECTION__RECOMBINANT_TIPS}`); + delete global.recombinant_tips; + } + + /** MongoDB has a size-limit of ~17MB per record. So accn_to_cid needs to be broken down into an array of objects*/ + res = await db.collection($COLLECTION__ACCN_TO_CID).insertMany(Object.entries(global.accn_to_cid).map(el => { j = {}; j[el[0]] = el[1]; return j })); + console.log(`Created ${res.insertedCount} documents in ${$ACTIVE_DATABASE}.${$COLLECTION__ACCN_TO_CID}`); + delete global.accn_to_cid; + + /** MongoDB has a size-limit of ~17MB per record. So lineage_to_cid needs to be broken down into an array of objects */ + res = await db.collection($COLLECTION__LINEAGE_TO_CID).insertMany(Object.entries(global.lineage_to_cid).map(el => { j = {}; j[el[0]] = el[1]; return j })); + console.log(`Created ${res.insertedCount} documents in ${$ACTIVE_DATABASE}.${$COLLECTION__LINEAGE_TO_CID}`); + delete global.lineage_to_cid; + + res = await db.collection($COLLECTION__REGION_MAP).insertOne(global.region_map); + console.log(`Created ${res.insertedCount} documents in ${$ACTIVE_DATABASE}.${$COLLECTION__REGION_MAP}`); + delete global.region_map; + + res = await db.collection($COLLECTION__DF_TREE).insertMany(global.df); + console.log(`Created ${res.insertedCount} documents in ${$ACTIVE_DATABASE}.${$COLLECTION__DF_TREE}`); + delete global.df; + + res = await db.collection($COLLECTION__XBB_TREE).insertMany(global.df_xbb); + console.log(`Created ${res.insertedCount} documents in ${$ACTIVE_DATABASE}.${$COLLECTION__DF_TREE}`); + delete global.df_xbb; + + res = await db.collection($COLLECTION__AUTOCOMPLETE_DATA).insertMany(global.autocomplete_data.map(subArray => {return {'norm': subArray[0],'accn': subArray[1]};})) + console.log(`Created ${res.insertedCount} documents in ${$ACTIVE_DATABASE}.${$COLLECTION__AUTOCOMPLETE_DATA}`); + delete global.autocomplete_data; + + res = await db.collection($COLLECTION__FLAT_DATA).insertMany(global.flat_data) + console.log(`Created ${res.insertedCount} documents in ${$ACTIVE_DATABASE}.${$COLLECTION__FLAT_DATA}`); + delete global.flat_data; + + + res = await adminUserConnection.close(); + console.log("Sucessfully wrote to database and closed connection!"); + + }) + .catch(error => { + console.log(`ERROR: ${error}`) + adminUserConnection.close().then(() => { + console.log("Failed to write to database. Closing connection."); + }) + }) +} + +const question_warning = "This will delete records within database " + $ACTIVE_DATABASE + ". Are you sure? (Y/n)"; +const userConfirmationCallback = function (answer) { + if (answer == "Y") { + rl.close(); + updateDatabase(); + } + else if (answer == 'n') { + rl.close(); + console.log("EXITING"); + process.exit(); + } + else { + rl.question(question_warning, userConfirmationCallback); + } +} + +rl.question(question_warning, userConfirmationCallback) \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index c0d3aeba..3054e064 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,8 @@ "express": "^4.17.2", "fs": "^0.0.1-security", "http": "^0.0.1-security", - "https": "^1.0.0" + "https": "^1.0.0", + "mongodb": "^5.1.0" }, "devDependencies": { "cypress": "^12.1.0" @@ -96,8 +97,7 @@ "node_modules/@types/node": { "version": "14.18.34", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.34.tgz", - "integrity": "sha512-hcU9AIQVHmPnmjRK+XUUYlILlr9pQrsqSrwov/JK1pnf3GTQowVBhx54FbvM0AU/VXGH4i3+vgXS5EguR7fysA==", - "dev": true + "integrity": "sha512-hcU9AIQVHmPnmjRK+XUUYlILlr9pQrsqSrwov/JK1pnf3GTQowVBhx54FbvM0AU/VXGH4i3+vgXS5EguR7fysA==" }, "node_modules/@types/sinonjs__fake-timers": { "version": "8.1.1", @@ -111,6 +111,20 @@ "integrity": "sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ==", "dev": true }, + "node_modules/@types/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==" + }, + "node_modules/@types/whatwg-url": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.2.tgz", + "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==", + "dependencies": { + "@types/node": "*", + "@types/webidl-conversions": "*" + } + }, "node_modules/@types/yauzl": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.0.tgz", @@ -370,6 +384,14 @@ "concat-map": "0.0.1" } }, + "node_modules/bson": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-5.2.0.tgz", + "integrity": "sha512-HevkSpDbpUfsrHWmWiAsNavANKYIErV2ePXllp1bwq5CDreAaFVj6RVlZpJnxK4WWDCJ/5jMUpaY6G526q3Hjg==", + "engines": { + "node": ">=14.20.1" + } + }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -1407,6 +1429,11 @@ "node": ">=10" } }, + "node_modules/ip": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", + "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -1681,6 +1708,12 @@ "node": ">= 0.6" } }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "optional": true + }, "node_modules/merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -1760,6 +1793,47 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/mongodb": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-5.3.0.tgz", + "integrity": "sha512-Wy/sbahguL8c3TXQWXmuBabiLD+iVmz+tOgQf+FwkCjhUIorqbAxRbbz00g4ZoN4sXIPwpAlTANMaGRjGGTikQ==", + "dependencies": { + "bson": "^5.2.0", + "mongodb-connection-string-url": "^2.6.0", + "socks": "^2.7.1" + }, + "engines": { + "node": ">=14.20.1" + }, + "optionalDependencies": { + "saslprep": "^1.0.3" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.201.0", + "mongodb-client-encryption": ">=2.3.0 <3", + "snappy": "^7.2.2" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + } + } + }, + "node_modules/mongodb-connection-string-url": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.6.0.tgz", + "integrity": "sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==", + "dependencies": { + "@types/whatwg-url": "^8.2.1", + "whatwg-url": "^11.0.0" + } + }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -1959,7 +2033,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", - "dev": true, "engines": { "node": ">=6" } @@ -2070,6 +2143,18 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/saslprep": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", + "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", + "optional": true, + "dependencies": { + "sparse-bitfield": "^3.0.3" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/semver": { "version": "7.3.8", "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", @@ -2186,6 +2271,37 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz", + "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==", + "dependencies": { + "ip": "^2.0.0", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.13.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "optional": true, + "dependencies": { + "memory-pager": "^1.0.2" + } + }, "node_modules/sshpk": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", @@ -2314,6 +2430,17 @@ "node": ">=0.8" } }, + "node_modules/tr46": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "dependencies": { + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/tslib": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", @@ -2427,6 +2554,26 @@ "extsprintf": "^1.2.0" } }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "engines": { + "node": ">=12" + } + }, + "node_modules/whatwg-url": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "dependencies": { + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -2554,8 +2701,7 @@ "@types/node": { "version": "14.18.34", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.34.tgz", - "integrity": "sha512-hcU9AIQVHmPnmjRK+XUUYlILlr9pQrsqSrwov/JK1pnf3GTQowVBhx54FbvM0AU/VXGH4i3+vgXS5EguR7fysA==", - "dev": true + "integrity": "sha512-hcU9AIQVHmPnmjRK+XUUYlILlr9pQrsqSrwov/JK1pnf3GTQowVBhx54FbvM0AU/VXGH4i3+vgXS5EguR7fysA==" }, "@types/sinonjs__fake-timers": { "version": "8.1.1", @@ -2569,6 +2715,20 @@ "integrity": "sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ==", "dev": true }, + "@types/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==" + }, + "@types/whatwg-url": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-8.2.2.tgz", + "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==", + "requires": { + "@types/node": "*", + "@types/webidl-conversions": "*" + } + }, "@types/yauzl": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.0.tgz", @@ -2759,6 +2919,11 @@ "concat-map": "0.0.1" } }, + "bson": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-5.2.0.tgz", + "integrity": "sha512-HevkSpDbpUfsrHWmWiAsNavANKYIErV2ePXllp1bwq5CDreAaFVj6RVlZpJnxK4WWDCJ/5jMUpaY6G526q3Hjg==" + }, "buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -3527,6 +3692,11 @@ "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==", "dev": true }, + "ip": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", + "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==" + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -3727,6 +3897,12 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" }, + "memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "optional": true + }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -3782,6 +3958,26 @@ "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==", "dev": true }, + "mongodb": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-5.3.0.tgz", + "integrity": "sha512-Wy/sbahguL8c3TXQWXmuBabiLD+iVmz+tOgQf+FwkCjhUIorqbAxRbbz00g4ZoN4sXIPwpAlTANMaGRjGGTikQ==", + "requires": { + "bson": "^5.2.0", + "mongodb-connection-string-url": "^2.6.0", + "saslprep": "^1.0.3", + "socks": "^2.7.1" + } + }, + "mongodb-connection-string-url": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-2.6.0.tgz", + "integrity": "sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==", + "requires": { + "@types/whatwg-url": "^8.2.1", + "whatwg-url": "^11.0.0" + } + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -3932,8 +4128,7 @@ "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", - "dev": true + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, "qs": { "version": "6.11.0", @@ -4019,6 +4214,15 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "saslprep": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", + "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", + "optional": true, + "requires": { + "sparse-bitfield": "^3.0.3" + } + }, "semver": { "version": "7.3.8", "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", @@ -4113,6 +4317,29 @@ "is-fullwidth-code-point": "^3.0.0" } }, + "smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==" + }, + "socks": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz", + "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==", + "requires": { + "ip": "^2.0.0", + "smart-buffer": "^4.2.0" + } + }, + "sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "optional": true, + "requires": { + "memory-pager": "^1.0.2" + } + }, "sshpk": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", @@ -4206,6 +4433,14 @@ "punycode": "^2.1.1" } }, + "tr46": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "requires": { + "punycode": "^2.1.1" + } + }, "tslib": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", @@ -4286,6 +4521,20 @@ "extsprintf": "^1.2.0" } }, + "webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==" + }, + "whatwg-url": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "requires": { + "tr46": "^3.0.0", + "webidl-conversions": "^7.0.0" + } + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index e613fab2..7f30e8de 100644 --- a/package.json +++ b/package.json @@ -9,8 +9,14 @@ }, "scripts": { "test": "NODE_ENV='TEST' HTTP_PORT='8001' DATA_FOLDER='data_test' node server.js", - "start": "node server.js", - "dev": "nodemon server.js --ignore 'js/*.js'" + "start": "node --max_old_space_size=8192 server.js", + "start-db1": "DBNUMBER=1 node server.js", + "start-db2": "DBNUMBER=2 node server.js", + "dev": "nodemon server.js --ignore 'js/*.js'", + "delete-users": "node mongodb_scripts/deleteUsers.js", + "create-users": "node mongodb_scripts/createUsers.js", + "update-db1": "DBNUMBER=1 node --max_old_space_size=8192 mongodb_scripts/updateDatabase.js", + "update-db2": "DBNUMBER=2 node --max_old_space_size=8192 mongodb_scripts/updateDatabase.js" }, "repository": { "type": "git", @@ -29,7 +35,8 @@ "express": "^4.17.2", "fs": "^0.0.1-security", "http": "^0.0.1-security", - "https": "^1.0.0" + "https": "^1.0.0", + "mongodb": "^5.1.0" }, "devDependencies": { "cypress": "^12.1.0" diff --git a/server.js b/server.js index 61c7b055..7eed042c 100644 --- a/server.js +++ b/server.js @@ -1,123 +1,140 @@ -const {$DATA_FOLDER,$HTTP_PORT,$HTTPS_PORT,$SSL_CREDENTIALS,$NODE_ENV} = require("./config") +const { $HTTP_PORT, $HTTPS_PORT, $SSL_CREDENTIALS, $NODE_ENV } = require("./config/config") +const { + $COVIZU_CONNECTION_URI, $ACTIVE_DATABASE, $COLLECTION__CLUSTERS +} = require('./config/dbconfig') +require("./globalVariables"); + const compression = require('compression'); const express = require('express'); const app = express(); -const clusters = require(`./${$DATA_FOLDER}/clusters.json`) const { utcDate } = require('./server/utils') -const { parse_clusters, map_clusters_to_tips, index_accessions, index_lineage, get_recombinants, get_region_map } = require('./server/parseCluster') -const { readTree } = require('./server/phylo') -const fs = require('fs'); var http = require('http'); var https = require('https'); app.use(compression()); +app.use(express.static('.')); -try { - var tree = fs.readFileSync(`./${$DATA_FOLDER}/timetree.nwk`, 'utf8'); -} catch(e) { - console.log('Error:', e.stack); +let dbManager; +if ($NODE_ENV == 'TEST') { + const { TestDBManager } = require("./dbmanager_test"); + dbManager = new TestDBManager(); + dbManager.load_globalData(); +} +if ($NODE_ENV == 'DEV' || $NODE_ENV == 'PROD') { + const { DBManager } = require("./dbmanager"); + dbManager = new DBManager(); + dbManager.set_dbUrl($COVIZU_CONNECTION_URI) + dbManager.set_dbName($ACTIVE_DATABASE) + dbManager.start() + .then(() => { + dbManager.load_globalData(); + }) } - -const df = readTree(tree) -const beaddata = parse_clusters(clusters) -const { tips, recombinant_tips } = map_clusters_to_tips(df, clusters) -const accn_to_cid = index_accessions(clusters) -const lineage_to_cid = index_lineage(clusters) -const recombinants = get_recombinants() -const region_map = get_region_map() - -// This is a hack to match anything that could be an acc number prefix -const prefix = /^(E|I|EP|IS|EPI_I|EPI_IS|EPI_ISL_?|EPI_?|ISL_?|[A-Z]\.[1-9]+)$/i; -const MIN_RESULTS = 10; -const normalize = (str) => str.replace(/[^a-z0-9]/gi, '').toLowerCase(); -const data = Object.keys(accn_to_cid).sort().concat(Object.keys(lineage_to_cid).sort()).map(accn => [ - normalize(accn), accn -]); app.get('/api/edgeList/:cindex', (req, res) => { - res.send(beaddata[req.params.cindex].edgelist) + dbManager.get_edgeList(req.params.cindex).then(result=>{ + res.send(result); + }) }); app.get('/api/points/:cindex', (req, res) => { - res.send(beaddata[req.params.cindex].points) + dbManager.get_points(req.params.cindex).then(result=>{ + res.send(result); + }) }); app.get('/api/variants/:cindex', (req, res) => { - res.send(beaddata[req.params.cindex].variants) + dbManager.get_variants(req.params.cindex).then(result=>{ + res.send(result); + }) +}); + +app.get('/api/lineage/:cindex', (req, res) => { + dbManager.get_lineage(req.params.cindex).then(result=>{ + res.send(result); + }) }); app.get('/api/tips', (req, res) => { - res.send(tips) + dbManager.get_tips().then(result=>{ + res.send(result); + }) +}); + +app.get('/api/display/:cindex', (req, res) => { + dbManager.get_lineage(req.params.cindex).then(result => { + dbManager.get_display(result).then(display => { + res.send(display); + }) + }) }); app.get('/api/df', (req, res) => { - res.send(df) + dbManager.get_df().then(result=>{ + res.send(result) + }) +}); + +app.get('/api/xbb', (req, res) => { + dbManager.get_xbb_df().then(result=>{ + res.send(result) + }) }); app.get('/api/regionmap', (req, res) => { - res.send(region_map) + dbManager.get_regionMap().then(result=>{ + res.send(result); + }) + // res.send(dbManager.get_regionMap()); }) -app.get('/api/lineage/:cindex', (req, res) => { - res.send(clusters[req.params.cindex].lineage) -}); - app.get('/api/cid/:accession', (req, res) => { - res.send(accn_to_cid[req.params.accession]) + dbManager.get_accession(req.params.accession).then(result=>{ + res.send(result); + }) }); +// this endpoint serves a massive file ~60MB. should we keep it? app.get('/api/cid', (req, res) => { - res.send(accn_to_cid) + dbManager.get_cid().then(result=>{ + res.send(result); + }) + // res.send(dbManager.get_cid()); }); app.get('/api/lineagetocid', (req, res) => { - res.send(lineage_to_cid) + dbManager.get_lineageToCid().then(result=>{ + res.send(result); + }) + // res.send(dbManager.get_lineageToCid()); }); app.get('/api/recombtips', (req, res) => { - res.send(recombinant_tips) + dbManager.get_recombinantTips().then(result=>{ + res.send(result); + }) + // res.send(dbManager.get_recombinantTips()); }); app.get('/api/searchHits/:query/:start/:end', (req, res) => { - // Flatten the json data to an array with bead data only - let flat_data = beaddata.map(bead => bead.points).flat(); - let start_date = utcDate(req.params.start); - let end_date = utcDate(req.params.end) - - //Find all the beads that are a hit. Convert text_query to lower case and checks to see if there is a match - let search_hits = flat_data.filter(function(bead) { - temp = (bead.accessions.some(accession => (accession.toLowerCase()).includes(req.params.query.toLowerCase())) || - bead.labels.some(label => (label.toLowerCase()).includes(req.params.query.toLowerCase()))) && - (bead.x >= start_date && bead.x <= end_date); - return temp; - }); - - res.send(search_hits) + const start_date = utcDate(req.params.start); + const end_date = utcDate(req.params.end); + const query = req.params.query.toLocaleLowerCase(); + dbManager.get_searchHits(start_date, end_date, query).then(result=>{ + res.send(result); + }) + // res.send(dbManager.get_searchHits(start_date, end_date, query)); }); app.get('/api/getHits/:query', (req, res) => { - function as_label(search_data) { - const [, accn] = search_data; - return accn; - } - const term = req.params.query - - if (!/\d/.test(term)) { - if (prefix.test(term)) { - res.send(data.slice(0, MIN_RESULTS).map(as_label)); - } else { - res.send([]); - } - } else { - const result = data.filter(array => array[0].indexOf(normalize(term)) > -1); - res.send(result.slice(0, MIN_RESULTS).map(as_label)); - } + dbManager.get_hits(term).then(result=>{ + res.send(result); + }) + // res.send(dbManager.get_hits(term)); }); -app.use(express.static('.')); - // For the prod environment, need to create a https server if ($NODE_ENV=='PROD') { var httpsServer = https.createServer($SSL_CREDENTIALS, app); diff --git a/server/parseCluster.js b/server/parseCluster.js index b13d7d39..b4e1f71b 100644 --- a/server/parseCluster.js +++ b/server/parseCluster.js @@ -1,11 +1,10 @@ // regular expression to remove redundant sequence name components const pat = /^hCoV-19\/(.+\/.+)\/20[0-9]{2}$/gi; const { unique, mode, tabulate, merge_tables, utcDate } = require('./utils') -const {$DATA_FOLDER} = require("../config") +const {$DATA_FOLDER} = require("../config/config") const dbstats = require(`../${$DATA_FOLDER}/dbstats.json`) const d3 = require('../js/d3') -var recombinants = [] -var region_map = {} // map country to region +require("../globalVariables") /** * Parse nodes that belong to the same variant. @@ -68,8 +67,8 @@ const parse_variant = (variant, y, cidx, accn, mindate, maxdate) => { for (let i=0; i < country.length; i++) { let this_country = country[i], this_region = region[i]; - if (region_map[this_country] === undefined) { - region_map[this_country] = this_region; + if (global.region_map[this_country] === undefined) { + global.region_map[this_country] = this_region; } } @@ -305,9 +304,86 @@ const parse_clusters = (clusters) => { * @param {Date} coldate * @returns {float} x-coordinate value */ - function date_to_xaxis(reftip, coldate) { - var numDays = d3.timeDay.count(reftip.first_date, coldate) - return (numDays/365.25) + reftip.x; +function date_to_xaxis(tips_obj, coldate) { + var numDays, earliest, latest, interval; + numDays = d3.timeDay.count(tips_obj[0].first_date, coldate); + earliest = d3.min(tips_obj, function(d) {return d.first_date}); + latest = d3.max(tips_obj, function(d) {return d.last_date}); + interval = d3.timeDay.count(earliest, latest)/3; + return (numDays/interval) + tips_obj[0].x; +} + +const map_tips = (cidx, labels, root, tips, tip_labels, cluster) => { + var root_idx = tip_labels.indexOf(root), // row index in data frame + root_xcoord = tips[root_idx].x; // left side of cluster starts at end of tip + + // find most recent sample collection date + var coldates = Array(), + label, variant; + + for (var i=0; i x[0])); + } + coldates.sort(); // in place, ascending order + + var first_date = utcDate(coldates[0]), + last_date = utcDate(coldates[coldates.length-1]); + + // Calculate the mean collection date + let date_diffs = coldates.map(x => (utcDate(x)-first_date) / (1000 * 3600 * 24) + // d3.timeDay.count(first_date, utcDate(x)) + ), + mean_date = Math.round(date_diffs.reduce((a, b) => a + b, 0) / date_diffs.length); + + // let date_diffs = coldates.map(x => d3.timeDay.count(first_date, utcDate(x))), + // mean_date = Math.round(date_diffs.reduce((a, b) => a + b, 0) / date_diffs.length); + + // augment data frame with cluster data + tips[root_idx].cluster_idx = cidx; + tips[root_idx].allregions = cluster.allregions; + tips[root_idx].region = cluster.region; + tips[root_idx].country = cluster.country; + tips[root_idx].searchtext = cluster.searchtext; + tips[root_idx].label1 = cluster["lineage"]; + tips[root_idx].count = coldates.length; + tips[root_idx].varcount = cluster["sampled_variants"]; // Number for sampled variants + tips[root_idx].sampled_varcount = labels.filter(x => x.substring(0,9) !== "unsampled").length; + tips[root_idx].first_date = first_date; + tips[root_idx].last_date = last_date; + tips[root_idx].pdist = cluster.pdist; + tips[root_idx].rdist = cluster.rdist; + + tips[root_idx].coldate = last_date; + tips[root_idx].x1 = root_xcoord - ((last_date - first_date) / 3.154e10); + tips[root_idx].x2 = root_xcoord; + + // map dbstats for lineage to tip + tip_stats = dbstats["lineages"][cluster["lineage"]]; + tips[root_idx].max_ndiffs = tip_stats.max_ndiffs; + tips[root_idx].mean_ndiffs = tip_stats.mean_ndiffs; + tips[root_idx].nsamples = tip_stats.nsamples; + tips[root_idx].mutations = tip_stats.mutations; + tips[root_idx].infections = tip_stats.infections; + + // calculate residual from mean differences and mean collection date - fixes #241 + let times = coldates.map(x => utcDate(x).getTime()), + origin = 18231, // days between 2019-12-01 and UNIX epoch (1970-01-01) + mean_time = times.reduce((x, y)=>x+y) / times.length / 8.64e7 - origin, + rate = 0.0655342, // subs per genome per day + exp_diffs = rate * mean_time; // expected number of differences + tips[root_idx].residual = tip_stats.mean_ndiffs - exp_diffs; // tip_stats.residual; + + Date.prototype.addDays = function(days) { + var date = new Date(this.valueOf()); + date.setDate(date.getDate() + days); + return date; + } + + tips[root_idx].mcoldate = first_date.addDays(mean_date); + + return tips; } @@ -317,7 +393,8 @@ const parse_clusters = (clusters) => { * @param {Array} clusters: data from clusters JSON * @returns {Array} subset of data frame annotated with cluster data */ -const map_clusters_to_tips = (df, clusters) => { +const map_clusters_to_tips = (df, df_xbb, clusters) => { + let recombinants = [], xbb = []; // extract accession numbers from phylogeny data frame var tips = df.filter(x => x.children.length===0), tip_labels = tips.map(x => x.thisLabel), // accessions @@ -329,89 +406,35 @@ const map_clusters_to_tips = (df, clusters) => { continue } - // find variant in cluster that matches a tip label - var labels = Object.keys(cluster["nodes"]), - root = tip_labels.filter(value => value === cluster['lineage'])[0]; - + if (dbstats["lineages"][cluster["lineage"]]["raw_lineage"].startsWith("XBB")) { + xbb.push(cidx); + continue; + } + if (dbstats["lineages"][cluster["lineage"]]["raw_lineage"].startsWith("X")) { recombinants.push(cidx) continue; } + // find variant in cluster that matches a tip label + var labels = Object.keys(cluster["nodes"]), + root = tip_labels.filter(value => value === cluster['lineage'])[0]; + if (root === undefined) { - console.log("Failed to match cluster of index ", cidx, " to a tip in the tree"); - continue; - } - - var root_idx = tip_labels.indexOf(root), // row index in data frame - root_xcoord = tips[root_idx].x; // left side of cluster starts at end of tip - - // find most recent sample collection date - var coldates = Array(), - label, variant; - - for (var i=0; i x[0])); + console.log("Failed to match cluster of index ", cidx, " to a tip in the tree"); + continue; } - coldates.sort(); // in place, ascending order - - var first_date = utcDate(coldates[0]), - last_date = utcDate(coldates[coldates.length-1]); - // console.log(first_date, last_date) - - // Calculate the mean collection date - let date_diffs = coldates.map(x => (utcDate(x)-first_date) / (1000 * 3600 * 24) - // d3.timeDay.count(first_date, utcDate(x)) - ), - mean_date = Math.round(date_diffs.reduce((a, b) => a + b, 0) / date_diffs.length); - // let date_diffs = coldates.map(x => d3.timeDay.count(first_date, utcDate(x))), - // mean_date = Math.round(date_diffs.reduce((a, b) => a + b, 0) / date_diffs.length); - - // augment data frame with cluster data - tips[root_idx].cluster_idx = cidx; - tips[root_idx].allregions = cluster.allregions; - tips[root_idx].region = cluster.region; - tips[root_idx].country = cluster.country; - tips[root_idx].searchtext = cluster.searchtext; - tips[root_idx].label1 = cluster["lineage"]; - tips[root_idx].count = coldates.length; - tips[root_idx].varcount = cluster["sampled_variants"]; // Number for sampled variants - tips[root_idx].sampled_varcount = labels.filter(x => x.substring(0,9) !== "unsampled").length; - tips[root_idx].first_date = first_date; - tips[root_idx].last_date = last_date; - tips[root_idx].pdist = cluster.pdist; - tips[root_idx].rdist = cluster.rdist; - - tips[root_idx].coldate = last_date; - tips[root_idx].x1 = root_xcoord - ((last_date - first_date) / 3.154e10); - tips[root_idx].x2 = root_xcoord; - - // map dbstats for lineage to tip - tip_stats = dbstats["lineages"][cluster["lineage"]]; - tips[root_idx].max_ndiffs = tip_stats.max_ndiffs; - tips[root_idx].mean_ndiffs = tip_stats.mean_ndiffs; - tips[root_idx].nsamples = tip_stats.nsamples; - tips[root_idx].mutations = tip_stats.mutations; - tips[root_idx].infections = tip_stats.infections; + tips = map_tips(cidx, labels, root, tips, tip_labels, cluster) + } - // calculate residual from mean differences and mean collection date - fixes #241 - let times = coldates.map(x => utcDate(x).getTime()), - origin = 18231, // days between 2019-12-01 and UNIX epoch (1970-01-01) - mean_time = times.reduce((x, y)=>x+y) / times.length / 8.64e7 - origin, - rate = 0.0655342, // subs per genome per day - exp_diffs = rate * mean_time; // expected number of differences - tips[root_idx].residual = tip_stats.mean_ndiffs - exp_diffs; // tip_stats.residual; + var mapped_x; - Date.prototype.addDays = function(days) { - var date = new Date(this.valueOf()); - date.setDate(date.getDate() + days); - return date; + for (var tip of tips) { + mapped_x = date_to_xaxis(tips, tip.first_date); + if (mapped_x > tip.x) { + tip.x = mapped_x } - - tips[root_idx].mcoldate = first_date.addDays(mean_date); } var recombinant_tips = [] @@ -439,8 +462,6 @@ const map_clusters_to_tips = (df, clusters) => { var first_date = utcDate(coldates[0]), last_date = utcDate(coldates[coldates.length-1]); - var root_xcoord = date_to_xaxis(tips[0], last_date); // left side of cluster starts at end of tip - // Calculate the mean collection date let date_diffs = coldates.map(x => (utcDate(x)-first_date) / (1000 * 3600 * 24) // d3.timeDay.count(first_date, utcDate(x)) @@ -474,8 +495,9 @@ const map_clusters_to_tips = (df, clusters) => { rdist: cluster.rdist, coldate: last_date, - x1: root_xcoord - ((last_date - first_date) / 3.154e10), - x2: root_xcoord, + x: 0, + x1: 0, + x2: 0, y: cidx, // map dbstats for lineage to tip @@ -497,7 +519,36 @@ const map_clusters_to_tips = (df, clusters) => { recombinant_tips[cidx].y = cidx } - return { tips, recombinant_tips }; + // XBB Tree + var tips_xbb = df_xbb.filter(x => x.children.length===0) + tip_labels = tips_xbb.map(x => x.thisLabel) // accessions + + for (const [_, xbb_cidx] of Object.entries(xbb)) { + cluster = clusters[xbb_cidx]; + if (cluster["nodes"].length === 1) { + continue + } + + // find variant in cluster that matches a tip label + labels = Object.keys(cluster["nodes"]) + root = tip_labels.filter(value => value === cluster['lineage'])[0]; + + if (root === undefined) { + console.log("Failed to match cluster of index ", xbb_cidx , " to a tip in the tree"); + continue; + } + + tips_xbb = map_tips(xbb_cidx, labels, root, tips_xbb, tip_labels, cluster) + } + + for (var tip of tips_xbb) { + mapped_x = date_to_xaxis(tips_xbb, tip.first_date); + if (mapped_x > tip.x) { + tip.x = mapped_x + } + } + + return { tips, tips_xbb, recombinant_tips }; } @@ -534,20 +585,9 @@ function index_lineage(clusters) { } -function get_recombinants() { - return recombinants; -} - -function get_region_map() { - return region_map; -} - - module.exports = { parse_clusters, map_clusters_to_tips, index_accessions, index_lineage, - get_recombinants, - get_region_map }; \ No newline at end of file diff --git a/server/phylo.js b/server/phylo.js index 0e6c40cb..cc0275c4 100644 --- a/server/phylo.js +++ b/server/phylo.js @@ -1,4 +1,4 @@ -const {$DATA_FOLDER} = require("../config") +const {$DATA_FOLDER} = require("../config/config") const dbstats = require(`../${$DATA_FOLDER}/dbstats.json`) /**