Skip to content

Commit

Permalink
Moved query construction logic to microservice
Browse files Browse the repository at this point in the history
  • Loading branch information
@kelockhart
kelockhart committed Mar 5, 2020
1 parent 245ef47 commit 7fcb215
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 87 deletions.
20 changes: 15 additions & 5 deletions myadsp/tests/test_tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,9 @@ def test_task_process_myads(self):
'frequency': 'weekly',
'type': 'template',
'template': 'authors',
'data': {'data': 'author:Kurtz'}},
'data': {'data': 'author:Kurtz'},
'query': [{'q': 'author:Kurtz entdate:["2020-01-01Z00:00" TO "2020-01-01Z23:59"] pubdate:[2019-00 TO *]',
'sort': 'score desc, bibcode desc'}]},
{'id': 4,
'name': 'Query 4',
'qid': None,
Expand All @@ -103,7 +105,14 @@ def test_task_process_myads(self):
'type': 'template',
'template': 'arxiv',
'data': 'star',
'classes': ['astro-ph']}
'classes': ['astro-ph'],
'query': [{'q': 'bibstem:arxiv (arxiv_class:(astro-ph.*) (star)) '
'entdate:["2020-01-01Z00:00" TO "2020-01-01Z23:59"] pubdate:[2019-00 TO *]',
'sort': 'score desc, bibcode desc'},
{'q': 'bibstem:arxiv (arxiv_class:(astro-ph.*) NOT (star)) '
'entdate:["2020-01-01Z00:00" TO "2020-01-01Z23:59"] pubdate:[2019-00 TO *]',
'sort': 'bibcode desc'}]
}
])
)

Expand Down Expand Up @@ -191,16 +200,17 @@ def test_task_process_myads(self):
)
httpretty.register_uri(
httpretty.GET, self.app.conf['API_SOLR_QUERY_ENDPOINT']+'?q={0}&sort={1}&fl={2}&rows={3}'.
format('author:Kurtz', 'score+desc+bibcode+desc', 'bibcode,title,author_norm', 5),
format('author:Kurtz entdate:["2020-01-01Z00:00" TO "2020-01-01Z23:59"] pubdate:[2019-00 TO *]',
'score+desc,+bibcode+desc', 'bibcode,title,author_norm', 5),
content_type='application/json',
status=200,
body=json.dumps({"responseHeader": {"status": 0,
"QTime": 23,
"params": {"q": "author:Kurtz",
"params": {"q": 'author:Kurtz entdate:["2020-01-01Z00:00" TO "2020-01-01Z23:59"] pubdate:[2019-00 TO *]',
"x-amzn-trace-id": "Root=1-5d769c6c-f96bfa49d348f03d8ecb7464",
"fl": "bibcode,title,author_norm",
"start": "0",
"sort": "score desc",
"sort": "score desc, bibcode desc",
"rows": "5",
"wt": "json"}},
"response": {"numFound": 2712,
Expand Down
41 changes: 31 additions & 10 deletions myadsp/tests/test_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,9 @@ def test_get_query_results(self):
@httpretty.activate
def test_get_template_query_results(self):
# test arxiv query
start = (utils.get_date() - datetime.timedelta(days=25)).date()
end = utils.get_date().date()
start_year = (utils.get_date() - datetime.timedelta(days=180)).year
myADSsetup = {'name': 'Test Query - arxiv',
'qid': 1,
'active': True,
Expand All @@ -162,6 +165,9 @@ def test_get_template_query_results(self):
'template': 'arxiv',
'data': 'AGN',
'classes': ['astro-ph'],
'query': [{'q': 'bibstem:arxiv (arxiv_class:(astro-ph.*) (AGN)) entdate:["{0}Z00:00" TO '
'"{1}Z23:59"] pubdate:[{2}-00 TO *]'.format(start, end, start_year),
'sort': 'score desc, bibcode desc'}],
'fields': 'bibcode,title,author_norm,identifier',
'rows': 2000}

Expand Down Expand Up @@ -220,6 +226,8 @@ def test_get_template_query_results(self):
'type': 'template',
'template': 'citations',
'data': 'author:Kurtz OR author:"Kurtz, M."',
'query': [{'q': 'citations(author:Kurtz OR author:"Kurtz, M.")',
'sort': 'entry_date desc, bibcode desc'}],
'fields': 'bibcode,title,author_norm,identifier',
'rows': 5}

Expand All @@ -228,7 +236,7 @@ def test_get_template_query_results(self):
format(endpoint=self.app._config.get('API_SOLR_QUERY_ENDPOINT'),
query=urllib.quote_plus(
'citations(author:Kurtz OR author:"Kurtz, M.")'),
sort=urllib.quote_plus('date desc, bibcode desc'),
sort=urllib.quote_plus('entry_date desc, bibcode desc'),
fields='bibcode,title,author_norm,identifier',
rows=5),
content_type='application/json',
Expand Down Expand Up @@ -309,7 +317,10 @@ def test_get_template_query_results(self):
'frequency': 'weekly',
'type': 'template',
'template': 'authors',
'data': 'database:astronomy author:Kurtz',
'data': 'author:Kurtz',
'query': [{'q': 'author:Kurtz entdate:["{0}Z00:00" TO "{1}Z23:59"] pubdate:[{2}-00 TO *]'.
format(start, end, start_year),
'sort': 'score desc, bibcode desc'}],
'fields': 'bibcode,title,author_norm,identifier',
'rows': 5}

Expand All @@ -326,7 +337,8 @@ def test_get_template_query_results(self):
body=json.dumps({"responseHeader": {"status": 0,
"QTime": 23,
"params": {
"q": "database:astronomy author:Kurtz",
"q": 'author:Kurtz entdate:["{0}Z00:00" TO "{1}Z23:59"] '
'pubdate:[{2}-00 TO *]'.format(start, end, start_year),
"fl": "bibcode,title,author_norm,identifier,year,bibstem",
"start": "0",
"sort": "score desc, bibcode desc",
Expand All @@ -345,9 +357,9 @@ def test_get_template_query_results(self):

results = utils.get_template_query_results(myADSsetup)
self.assertEqual(results, [{'name': myADSsetup['name'],
'query': 'database:astronomy author:Kurtz entdate:["{0}Z00:00" TO "{1}Z23:59"] pubdate:[{2}-00 TO *]'.format(start, end, start_year),
'query': 'author:Kurtz entdate:["{0}Z00:00" TO "{1}Z23:59"] pubdate:[{2}-00 TO *]'.format(start, end, start_year),
'query_url': 'https://ui.adsabs.harvard.edu/search/q={0}&sort={1}'.
format(urllib.quote_plus('database:astronomy author:Kurtz entdate:["{0}Z00:00" TO "{1}Z23:59"] pubdate:[{2}-00 TO *]'.format(start, end, start_year)),
format(urllib.quote_plus('author:Kurtz entdate:["{0}Z00:00" TO "{1}Z23:59"] pubdate:[{2}-00 TO *]'.format(start, end, start_year)),
urllib.quote_plus("score desc, bibcode desc")),
'results': [{u"bibcode": u"1971JVST....8..324K",
u"title": [
Expand All @@ -367,6 +379,13 @@ def test_get_template_query_results(self):
'template': 'keyword',
'data': 'AGN',
'classes': ['astro-ph', 'physics.space-ph'],
'query': [{'q': 'AGN arxiv_class:(astro-ph.* OR physics.space-ph) entdate:["{0}Z00:00" TO '
'"{1}Z23:59"] pubdate:[{2}-00 TO *]'.format(start, end, start_year),
'sort': 'entry_date desc, bibcode desc'},
{'q': 'trending(AGN arxiv_class:(astro-ph.* OR physics.space-ph))',
'sort': 'score desc, bibcode desc'},
{'q': 'useful(AGN arxiv_class:(astro-ph.* OR physics.space-ph))',
'sort': 'score desc, bibcode desc'}],
'fields': 'bibcode,title,author_norm,identifier',
'rows': 5}

Expand All @@ -382,7 +401,9 @@ def test_get_template_query_results(self):
body=json.dumps({"responseHeader": {"status": 0,
"QTime": 23,
"params": {
"q": "arxiv_class:(astro-ph.* OR physics.space-ph) AGN",
"q": 'AGN arxiv_class:(astro-ph.* OR physics.space-ph) '
'entdate:["{0}Z00:00" TO "{1}Z23:59"] '
'pubdate:[{2}-00 TO *]'.format(start, end, start_year),
"fl": "bibcode,title,author_norm,identifier,year,bibstem",
"start": "0",
"sort": "entry_date desc, bibcode desc",
Expand All @@ -402,7 +423,7 @@ def test_get_template_query_results(self):
httpretty.register_uri(
httpretty.GET, '{endpoint}?q={query}&sort={sort}&fl={fields}&rows={rows}'.
format(endpoint=self.app._config.get('API_SOLR_QUERY_ENDPOINT'),
query=urllib.quote_plus('trending(arxiv_class:(astro-ph.* OR physics.space-ph) AGN)'),
query=urllib.quote_plus('trending(AGN arxiv_class:(astro-ph.* OR physics.space-ph))'),
sort=urllib.quote_plus('score desc, bibcode desc'),
fields='bibcode,title,author_norm,identifier',
rows=5),
Expand All @@ -411,7 +432,7 @@ def test_get_template_query_results(self):
body=json.dumps({"responseHeader": {"status": 0,
"QTime": 23,
"params": {
"q": "trending(arxiv_class:(astro-ph.* OR physics.space-ph) AGN)",
"q": "trending(AGN arxiv_class:(astro-ph.* OR physics.space-ph))",
"fl": "bibcode,title,author_norm,identifier,year,bibstem",
"start": "0",
"sort": "score desc, bibcode desc",
Expand All @@ -431,7 +452,7 @@ def test_get_template_query_results(self):
httpretty.register_uri(
httpretty.GET, '{endpoint}?q={query}&sort={sort}&fl={fields}&rows={rows}'.
format(endpoint=self.app._config.get('API_SOLR_QUERY_ENDPOINT'),
query=urllib.quote_plus('useful(arxiv_class:(astro-ph.* OR physics.space-ph) AGN)'),
query=urllib.quote_plus('useful(AGN arxiv_class:(astro-ph.* OR physics.space-ph))'),
sort=urllib.quote_plus('score desc, bibcode desc'),
fields='bibcode,title,author_norm,identifier',
rows=5),
Expand All @@ -440,7 +461,7 @@ def test_get_template_query_results(self):
body=json.dumps({"responseHeader": {"status": 0,
"QTime": 23,
"params": {
"q": "useful(arxiv_class:(astro-ph.* OR physics.space-ph) AGN)",
"q": "useful(AGN arxiv_class:(astro-ph.* OR physics.space-ph))",
"fl": "bibcode,title,author_norm,identifier,year,bibstem",
"start": "0",
"sort": "score desc, bibcode desc",
Expand Down
Loading

0 comments on commit 7fcb215

Please sign in to comment.