Dquest-flask initial version.

.gitignore

@@ -0,0 +1,2 @@
+/dquest-flask/app/templates/backup/
+/dquest-flask/venv/

dquest-flask/README.md

@@ -0,0 +1,127 @@
+# dquest-flask
+dquest implementation using flask
+
+### Install and deploy
+###### Enviroment
+1. python2.7
+2. create env and dependency
+```buildoutcfg
+sudo pip install virtualenv 
+sudo virtualenv venv
+source venv/bin/activate 
+pip install -r requirements.txt
+
+```
+
+###### Bug to fix before deploy
+```buildoutcfg
+Traceback (most recent call last):
+  File "/Users/cl3720/Projects/dquest-flask/run.py", line 1, in <module>
+    from app import app
+  File "/Users/cl3720/Projects/dquest-flask/app/__init__.py", line 9, in <module>
+    cache.init_app (app)
+  File "/Users/cl3720/Projects/dquest-flask/env/lib/python2.7/site-packages/flask_cache/__init__.py", line 156, in init_app
+    from .jinja2ext import CacheExtension, JINJA_CACHE_ATTR_NAME
+  File "/Users/cl3720/Projects/dquest-flask/env/lib/python2.7/site-packages/flask_cache/jinja2ext.py", line 33, in <module>
+    from flask.ext.cache import make_template_fragment_key
+ImportError: No module named ext.cache
+```
+1. find jinja2ext.py
+2. locate line 33 and change to 
+```buildoutcfg
+from flask_cache import make_template_fragment_key
+```
+###### Change configuration
+```
+# vi app/lib/config.py
+CRITERIA_HOST = 'elixr.XXX'
+CRITERIA_DATABASE = 'trial_knowledge_base'
+CRITEIRA_USERNAME = 'XXX'
+CRITERIA_PASSWORD = 'XXX'
+CRITERIA_DRIVER = '{ODBC Driver 17 for SQL Server}' # make sure the driver is installed on the server
+CRITERIA_PORT  = XXX
+
+AACT_HOST = 'aact-db.ctti-clinicaltrials.org'
+AACT_PORT = 5432
+AACT_DATABASE = 'aact'
+AACT_USERNAME = 'XXX'
+AACT_PASSWORD = 'XXX'
+
+CSRF_ENABLED = True
+SECRET_KEY = 'XXX'
+```
+make sure the correct version of ODBC version is available on server
+```buildoutcfg
+odbcinst -j
+``` 
+###### Deploy in Ubuntu with Apache2
+Install and Enable mod_wsgi
+```buildoutcfg
+sudo apt-get install libapache2-mod-wsgi python-dev
+sudo a2enmod wsgi
+cd /var/www 
+git clone https://github.com/stormliucong/dquest-flask.git
+```
+
+Add the following lines of code to the file to configure the virtual host. Be sure to change the ServerName to your domain or cloud server's IP address:
+```buildoutcfg
+# sudo vi  /etc/apache2/sites-available/dquest-flask.conf
+<VirtualHost *:80>
+		ServerName mywebsite.com
+		ServerAdmin [email protected]
+		WSGIScriptAlias /dquest /var/www/dquest-flask/app.wsgi
+		<Directory /var/www/FlaskApp/FlaskApp/>
+			Order allow,deny
+			Allow from all
+		</Directory>
+		ErrorLog ${APACHE_LOG_DIR}/error.log
+		LogLevel warn
+		CustomLog ${APACHE_LOG_DIR}/access.log combined
+</VirtualHost>
+```
+Enable the virtual host and restart apache2
+```buildoutcfg
+sudo a2ensite dquest-flask
+sudo service apache2 restart 
+```
+browser the website
+```buildoutcfg
+http://mywebsite.com/dquest-flask
+```
+Change different algorithms for question ranking by selecting one of the following three.
+1. question_info_entropy using information entropy to rank questions.
+2. question using raw entity_text (without omop mapping) to rank questions.
+3. question_cluster using frequency to rank questions.
+```
+import lib.question_info_entropy as qst
+import lib.question as qst
+import lib.question_cluster as qst
+```
+
+change log file path
+```
+# app/lib/log.py
+hfile_info = logging.FileHandler("app/log/dquest-info.log")
+hfile_error = logging.FileHandler("app/log/dquest-error.log")
+```
+
+check path
+```
+# app/view.py
+@app.route('/')
+def index ():
+```
+## Versioning
+0.0.1
+
+## New features under development
+https://docs.google.com/document/d/1h4PVeiIdWwsHzuxIovAnBbqT4RD-xk59A3ZOaJHKng8/edit
+
+## Publications
+Uder revision
+## Authors
+Cong Liu, Chi Yuan, Alex Butler, Chunhua Weng
+[email protected]
+
+
+

dquest-flask/app.wsgi

@@ -0,0 +1,13 @@
+import sys
+import logging
+import os
+logging.basicConfig(stream=sys.stderr)
+sys.path.insert(0,"/var/www/dquest-flask/")
+PROJECT_DIR = '/var/www/dquest-flask/venv'
+
+activate_this = os.path.join(PROJECT_DIR, 'bin', 'activate_this.py')
+execfile(activate_this, dict(__file__=activate_this))
+sys.path.append(PROJECT_DIR)
+
+
+from app import app as application

dquest-flask/app/__init__.py

@@ -0,0 +1,35 @@
+from flask import Flask
+from lib.extensions import cache
+from DBUtils.PooledDB import PooledDB
+import pyodbc
+import psycopg2
+
+app = Flask(__name__)
+
+# cache.init_app (app)
+
+app.config.from_object ('app.lib.config')
+
+
+# pool for knowledgebase
+general_pool_criteria = PooledDB(creator=pyodbc,
+                                 host=app.config.get('CRITERIA_HOST'),
+                                 port=app.config.get('CRITERIA_PORT'),
+                                 database=app.config.get('CRITERIA_DATABASE'),
+                                 user=app.config.get('CRITEIRA_USERNAME'),
+                                 password=app.config.get('CRITERIA_PASSWORD'),
+                                 driver = app.config.get('CRITERIA_DRIVER'),
+                                 maxconnections=12,
+                                 blocking=False)
+
+# pool for nct details in AACT
+general_pool_aact = PooledDB(creator=psycopg2,
+                             host=app.config.get('AACT_HOST'),
+                             port=app.config.get('AACT_PORT'),
+                             database=app.config.get('AACT_DATABASE'),
+                             user=app.config.get('AACT_USERNAME'),
+                             password=app.config.get('AACT_PASSWORD'),
+                             maxconnections=12,
+                             blocking=False)
+
+from app import views

dquest-flask/app/lib/config.py

@@ -0,0 +1,15 @@
+CRITERIA_HOST = '192.168.1.35'
+CRITERIA_DATABASE = 'trial_knowledge_base'
+CRITEIRA_USERNAME = 'SA'
+CRITERIA_PASSWORD = 'Dquest123$'
+CRITERIA_DRIVER = '{ODBC Driver 17 for SQL Server}' # make sure the driver is installed on the server
+CRITERIA_PORT  = 1433
+
+AACT_HOST = 'aact-db.ctti-clinicaltrials.org'
+AACT_PORT = 5432
+AACT_DATABASE = 'aact'
+AACT_USERNAME = 'hl2659'
+AACT_PASSWORD = '3Z7eqbzgigF6G8v'
+
+CSRF_ENABLED = True
+SECRET_KEY = 'XXX'

dquest-flask/app/lib/ctgov.py

@@ -0,0 +1,127 @@
+'''
+set of function to interact with ClinicalTrials.gov
+'''
+
+from app import app
+from web import download_web_data
+from xml.etree import ElementTree
+from extensions import cache
+import oformat as of
+import urllib
+
+# global ctgov search
+def get_initial_nct (txt):
+    txt = of.format_search_terms (txt)
+    url = 'http://clinicaltrials.gov/search?cond=%s&displayxml=true' % txt
+    return get_initial_nct_from_url (url)
+
+def get_initial_nct_detail (recrs,cond,locn):
+    recrs = of.format_search_terms (recrs)
+    cond = of.format_search_terms (cond)
+    locn = of.format_search_terms (locn)
+    url = 'http://clinicaltrials.gov/search?cond=%s&recrs=%s&locn=%s&displayxml=true' % (cond,recrs,locn)
+    print(url)
+    return get_initial_nct_from_url (url)
+
+def get_initial_nct_patient (cond,locn):
+    cond = of.format_search_terms (cond)
+    locn = of.format_search_terms (locn)
+    # url = 'http://clinicaltrials.gov/search?cond=%s&recrs=recrs&type=intr&locn=%s&displayxml=true' % (cond,locn)
+    url = 'http://clinicaltrials.gov/search?cond=%s&type=intr&locn=%s&displayxml=true' % (cond, locn)
+    print(url)
+    return get_initial_nct_from_url (url)
+
+
+
+# global ctgov search from url
+@cache.memoize(604800)
+def get_initial_nct_from_url (url):
+    # num. of studies available
+    n = get_nct_number('%s&count=0' % url)
+    print(n)
+    if n == 0:
+        return []
+    # get the list of clinical studies
+    xmltree = ElementTree.fromstring (download_web_data('%s&count=%d' % (url, n)))
+    lnct = xmltree.findall ('clinical_study')
+    rnct = []
+    i = 1
+    for ct in lnct:
+        ids = ct.find ('nct_id')
+        if ids is None:
+            continue
+        rnct.append ('%s;%d' % (ids.text,i))
+        i += 1
+    return rnct
+
+
+# get result number
+def get_nct_number (url):
+    xml = download_web_data(url)
+    if xml is None:
+        return 0
+    xmltree = ElementTree.fromstring (xml)
+    nnct = xmltree.get ('count')
+    return int(nnct)
+
+
+# parse clinical trial details
+def parse_xml_nct (ct):
+    ids = ct.find ('nct_id')
+    if ids is None:
+        return ids
+    ids = ids.text
+    rank = ct.find ('order')
+    if rank is not None:
+        rank = rank.text
+    title = ct.find ('title')
+    if title is not None:
+        title = title.text
+    condition = ct.find ('condition_summary')
+    if condition is not None:
+        condition = condition.text
+    return (ids, rank, title, condition)
+
+
+# ctgov search
+def search (txt, npag):
+    txt = of.format_search_terms (txt)
+    url = 'http://clinicaltrials.gov/search?cond=%s&displayxml=true' % txt
+    return retrieve_trials (url, npag)
+
+
+# ctgov advanced search
+def advanced_search(param):
+    url = form_advanced_search_url (param)
+    return retrieve_trials (url, param.get('npag'))
+
+
+# get advanced search url
+def form_advanced_search_url (param):
+    ctg_param = ''
+    for r in sorted(param):
+        k = urllib.quote(r,'')
+        if k == 'qlabel':
+            continue
+        for v in param.getlist(r):
+            ctg_param += "%s=%s&" % (k, urllib.quote(v, ''))
+    return 'http://clinicaltrials.gov/ct2/results?%sdisplayxml=True' % ctg_param
+
+
+# get the list of resulting clinical trials
+def retrieve_trials (url, npag):
+    # num. of studies available
+    n = of.format_nct_number(get_nct_number('%s&count=0' % url))
+    # get the list of clinical studies
+    xml = download_web_data('%s&pg=%s' % (url, npag))
+    if xml is None:
+        return (0, [])
+    xmltree = ElementTree.fromstring (xml)
+    lnct = xmltree.findall ('clinical_study')
+    nct = []
+    for ct in lnct:
+        pct = parse_xml_nct (ct)
+        if pct[0] is not None:
+            cond = of.format_condition (pct[3])
+            nct.append ((pct[0], pct[1], pct[2], cond))
+    return (n, nct)

dquest-flask/app/lib/extensions.py

@@ -0,0 +1,8 @@
+from flask_cache import Cache
+from log import slogger
+import file as ufile
+import sys
+
+log = slogger ('dquest-ext')
+
+cache = Cache(config={'CACHE_TYPE': 'filesystem', 'CACHE_DEFAULT_TIMEOUT': 86400, 'CACHE_THRESHOLD': 500, 'CACHE_DIR': 'app/resources/cache'})