Skip to content

Commit

Permalink
Merge pull request MSUSAzureAccelerators#65 from MSUSAzureAccelerator…
Browse files Browse the repository at this point in the history 
…s/main

update utils.py
  • Loading branch information
@pablomarin
pablomarin authored Nov 6, 2023
2 parents d8f3b29 + 7d3441b commit 0af66f6
Show file tree
Hide file tree
Showing 11 changed files with 1,281 additions and 527 deletions.
26 changes: 15 additions & 11 deletions 01-Load-Data-ACogSearch.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -548,7 +548,7 @@
},
{
"cell_type": "code",
"execution_count": 20,
"execution_count": 15,
"metadata": {
"tags": []
},
Expand All @@ -559,20 +559,24 @@
"text": [
"200\n",
"Status: inProgress\n",
"Items Processed: 3250\n",
"Items Processed: 400\n",
"True\n"
]
}
],
"source": [
"# Optionally, get indexer status to confirm that it's running\n",
"r = requests.get(os.environ['AZURE_SEARCH_ENDPOINT'] + \"/indexers/\" + indexer_name +\n",
" \"/status\", headers=headers, params=params)\n",
"# pprint(json.dumps(r.json(), indent=1))\n",
"print(r.status_code)\n",
"print(\"Status:\",r.json().get('lastResult').get('status'))\n",
"print(\"Items Processed:\",r.json().get('lastResult').get('itemsProcessed'))\n",
"print(r.ok)"
"try:\n",
" r = requests.get(os.environ['AZURE_SEARCH_ENDPOINT'] + \"/indexers/\" + indexer_name +\n",
" \"/status\", headers=headers, params=params)\n",
" # pprint(json.dumps(r.json(), indent=1))\n",
" print(r.status_code)\n",
" print(\"Status:\",r.json().get('lastResult').get('status'))\n",
" print(\"Items Processed:\",r.json().get('lastResult').get('itemsProcessed'))\n",
" print(r.ok)\n",
" \n",
"except Exception as e:\n",
" print(\"Wait a few seconds until the process starts and run this cell again.\")"
]
},
{
Expand Down Expand Up @@ -609,14 +613,14 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"204\n",
"201\n",
"True\n"
]
}
Expand Down
108 changes: 56 additions & 52 deletions 02-LoadCSVOneToMany-ACogSearch.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,69 +148,69 @@
"text/html": [
"<style type=\"text/css\">\n",
"</style>\n",
"<table id=\"T_6ac41\">\n",
"<table id=\"T_87464\">\n",
" <thead>\n",
" <tr>\n",
" <th class=\"blank level0\" >&nbsp;</th>\n",
" <th id=\"T_6ac41_level0_col0\" class=\"col_heading level0 col0\" >cord_uid</th>\n",
" <th id=\"T_6ac41_level0_col1\" class=\"col_heading level0 col1\" >source_x</th>\n",
" <th id=\"T_6ac41_level0_col2\" class=\"col_heading level0 col2\" >title</th>\n",
" <th id=\"T_6ac41_level0_col3\" class=\"col_heading level0 col3\" >abstract</th>\n",
" <th id=\"T_6ac41_level0_col4\" class=\"col_heading level0 col4\" >authors</th>\n",
" <th id=\"T_6ac41_level0_col5\" class=\"col_heading level0 col5\" >url</th>\n",
" <th id=\"T_87464_level0_col0\" class=\"col_heading level0 col0\" >cord_uid</th>\n",
" <th id=\"T_87464_level0_col1\" class=\"col_heading level0 col1\" >source_x</th>\n",
" <th id=\"T_87464_level0_col2\" class=\"col_heading level0 col2\" >title</th>\n",
" <th id=\"T_87464_level0_col3\" class=\"col_heading level0 col3\" >abstract</th>\n",
" <th id=\"T_87464_level0_col4\" class=\"col_heading level0 col4\" >authors</th>\n",
" <th id=\"T_87464_level0_col5\" class=\"col_heading level0 col5\" >url</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th id=\"T_6ac41_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
" <td id=\"T_6ac41_row0_col0\" class=\"data row0 col0\" >ug7v899j</td>\n",
" <td id=\"T_6ac41_row0_col1\" class=\"data row0 col1\" >PMC</td>\n",
" <td id=\"T_6ac41_row0_col2\" class=\"data row0 col2\" >Clinical features of culture-p...</td>\n",
" <td id=\"T_6ac41_row0_col3\" class=\"data row0 col3\" >OBJECTIVE: This retrospective ...</td>\n",
" <td id=\"T_6ac41_row0_col4\" class=\"data row0 col4\" >Madani, Tariq A; Al-Ghamdi, Ai...</td>\n",
" <td id=\"T_6ac41_row0_col5\" class=\"data row0 col5\" ><a href=\"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC35282/\">https://www.ncbi.nlm.nih.gov/pmc/articles/PMC35282/</a></td>\n",
" <th id=\"T_87464_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
" <td id=\"T_87464_row0_col0\" class=\"data row0 col0\" >ug7v899j</td>\n",
" <td id=\"T_87464_row0_col1\" class=\"data row0 col1\" >PMC</td>\n",
" <td id=\"T_87464_row0_col2\" class=\"data row0 col2\" >Clinical features of culture-p...</td>\n",
" <td id=\"T_87464_row0_col3\" class=\"data row0 col3\" >OBJECTIVE: This retrospective ...</td>\n",
" <td id=\"T_87464_row0_col4\" class=\"data row0 col4\" >Madani, Tariq A; Al-Ghamdi, Ai...</td>\n",
" <td id=\"T_87464_row0_col5\" class=\"data row0 col5\" ><a href=\"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC35282/\">https://www.ncbi.nlm.nih.gov/pmc/articles/PMC35282/</a></td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_6ac41_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
" <td id=\"T_6ac41_row1_col0\" class=\"data row1 col0\" >02tnwd4m</td>\n",
" <td id=\"T_6ac41_row1_col1\" class=\"data row1 col1\" >PMC</td>\n",
" <td id=\"T_6ac41_row1_col2\" class=\"data row1 col2\" >Nitric oxide: a pro-inflammato...</td>\n",
" <td id=\"T_6ac41_row1_col3\" class=\"data row1 col3\" >Inflammatory diseases of the r...</td>\n",
" <td id=\"T_6ac41_row1_col4\" class=\"data row1 col4\" >Vliet, Albert van der; Eiseric...</td>\n",
" <td id=\"T_6ac41_row1_col5\" class=\"data row1 col5\" ><a href=\"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59543/\">https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59543/</a></td>\n",
" <th id=\"T_87464_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
" <td id=\"T_87464_row1_col0\" class=\"data row1 col0\" >02tnwd4m</td>\n",
" <td id=\"T_87464_row1_col1\" class=\"data row1 col1\" >PMC</td>\n",
" <td id=\"T_87464_row1_col2\" class=\"data row1 col2\" >Nitric oxide: a pro-inflammato...</td>\n",
" <td id=\"T_87464_row1_col3\" class=\"data row1 col3\" >Inflammatory diseases of the r...</td>\n",
" <td id=\"T_87464_row1_col4\" class=\"data row1 col4\" >Vliet, Albert van der; Eiseric...</td>\n",
" <td id=\"T_87464_row1_col5\" class=\"data row1 col5\" ><a href=\"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59543/\">https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59543/</a></td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_6ac41_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
" <td id=\"T_6ac41_row2_col0\" class=\"data row2 col0\" >ejv2xln0</td>\n",
" <td id=\"T_6ac41_row2_col1\" class=\"data row2 col1\" >PMC</td>\n",
" <td id=\"T_6ac41_row2_col2\" class=\"data row2 col2\" >Surfactant protein-D and pulmo...</td>\n",
" <td id=\"T_6ac41_row2_col3\" class=\"data row2 col3\" >Surfactant protein-D (SP-D) pa...</td>\n",
" <td id=\"T_6ac41_row2_col4\" class=\"data row2 col4\" >Crouch, Erika C...</td>\n",
" <td id=\"T_6ac41_row2_col5\" class=\"data row2 col5\" ><a href=\"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59549/\">https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59549/</a></td>\n",
" <th id=\"T_87464_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
" <td id=\"T_87464_row2_col0\" class=\"data row2 col0\" >ejv2xln0</td>\n",
" <td id=\"T_87464_row2_col1\" class=\"data row2 col1\" >PMC</td>\n",
" <td id=\"T_87464_row2_col2\" class=\"data row2 col2\" >Surfactant protein-D and pulmo...</td>\n",
" <td id=\"T_87464_row2_col3\" class=\"data row2 col3\" >Surfactant protein-D (SP-D) pa...</td>\n",
" <td id=\"T_87464_row2_col4\" class=\"data row2 col4\" >Crouch, Erika C...</td>\n",
" <td id=\"T_87464_row2_col5\" class=\"data row2 col5\" ><a href=\"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59549/\">https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59549/</a></td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_6ac41_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
" <td id=\"T_6ac41_row3_col0\" class=\"data row3 col0\" >2b73a28n</td>\n",
" <td id=\"T_6ac41_row3_col1\" class=\"data row3 col1\" >PMC</td>\n",
" <td id=\"T_6ac41_row3_col2\" class=\"data row3 col2\" >Role of endothelin-1 in lung d...</td>\n",
" <td id=\"T_6ac41_row3_col3\" class=\"data row3 col3\" >Endothelin-1 (ET-1) is a 21 am...</td>\n",
" <td id=\"T_6ac41_row3_col4\" class=\"data row3 col4\" >Fagan, Karen A; McMurtry, Ivan...</td>\n",
" <td id=\"T_6ac41_row3_col5\" class=\"data row3 col5\" ><a href=\"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59574/\">https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59574/</a></td>\n",
" <th id=\"T_87464_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
" <td id=\"T_87464_row3_col0\" class=\"data row3 col0\" >2b73a28n</td>\n",
" <td id=\"T_87464_row3_col1\" class=\"data row3 col1\" >PMC</td>\n",
" <td id=\"T_87464_row3_col2\" class=\"data row3 col2\" >Role of endothelin-1 in lung d...</td>\n",
" <td id=\"T_87464_row3_col3\" class=\"data row3 col3\" >Endothelin-1 (ET-1) is a 21 am...</td>\n",
" <td id=\"T_87464_row3_col4\" class=\"data row3 col4\" >Fagan, Karen A; McMurtry, Ivan...</td>\n",
" <td id=\"T_87464_row3_col5\" class=\"data row3 col5\" ><a href=\"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59574/\">https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59574/</a></td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_6ac41_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
" <td id=\"T_6ac41_row4_col0\" class=\"data row4 col0\" >9785vg6d</td>\n",
" <td id=\"T_6ac41_row4_col1\" class=\"data row4 col1\" >PMC</td>\n",
" <td id=\"T_6ac41_row4_col2\" class=\"data row4 col2\" >Gene expression in epithelial ...</td>\n",
" <td id=\"T_6ac41_row4_col3\" class=\"data row4 col3\" >Respiratory syncytial virus (R...</td>\n",
" <td id=\"T_6ac41_row4_col4\" class=\"data row4 col4\" >Domachowske, Joseph B; Bonvill...</td>\n",
" <td id=\"T_6ac41_row4_col5\" class=\"data row4 col5\" ><a href=\"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59580/\">https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59580/</a></td>\n",
" <th id=\"T_87464_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
" <td id=\"T_87464_row4_col0\" class=\"data row4 col0\" >9785vg6d</td>\n",
" <td id=\"T_87464_row4_col1\" class=\"data row4 col1\" >PMC</td>\n",
" <td id=\"T_87464_row4_col2\" class=\"data row4 col2\" >Gene expression in epithelial ...</td>\n",
" <td id=\"T_87464_row4_col3\" class=\"data row4 col3\" >Respiratory syncytial virus (R...</td>\n",
" <td id=\"T_87464_row4_col4\" class=\"data row4 col4\" >Domachowske, Joseph B; Bonvill...</td>\n",
" <td id=\"T_87464_row4_col5\" class=\"data row4 col5\" ><a href=\"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59580/\">https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59580/</a></td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n"
],
"text/plain": [
"<pandas.io.formats.style.Styler at 0x7f31574398d0>"
"<pandas.io.formats.style.Styler at 0x7f36d4016ad0>"
]
},
"execution_count": 6,
Expand Down Expand Up @@ -443,7 +443,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 10,
"id": "6132c041-7213-410e-a206-1a8c7385128e",
"metadata": {},
"outputs": [
Expand All @@ -460,13 +460,17 @@
],
"source": [
"# Optionally, get indexer status to confirm that it's running\n",
"r = requests.get(os.environ['AZURE_SEARCH_ENDPOINT'] + \"/indexers/\" + indexer_name +\n",
" \"/status\", headers=headers, params=params)\n",
"# pprint(json.dumps(r.json(), indent=1))\n",
"print(r.status_code)\n",
"print(\"Status:\",r.json().get('lastResult').get('status'))\n",
"print(\"Items Processed:\",r.json().get('lastResult').get('itemsProcessed'))\n",
"print(r.ok)"
"try:\n",
" r = requests.get(os.environ['AZURE_SEARCH_ENDPOINT'] + \"/indexers/\" + indexer_name +\n",
" \"/status\", headers=headers, params=params)\n",
" # pprint(json.dumps(r.json(), indent=1))\n",
" print(r.status_code)\n",
" print(\"Status:\",r.json().get('lastResult').get('status'))\n",
" print(\"Items Processed:\",r.json().get('lastResult').get('itemsProcessed'))\n",
" print(r.ok)\n",
" \n",
"except Exception as e:\n",
" print(\"Wait a few seconds until the process starts and run this cell again.\")"
]
},
{
Expand All @@ -487,7 +491,7 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 11,
"id": "ec359823-3b9f-4b7f-af38-c3f2f916d5fa",
"metadata": {},
"outputs": [
Expand Down
Loading

0 comments on commit 0af66f6

Please sign in to comment.