Skip to content

Commit

Permalink
mrc observation data update
Browse files Browse the repository at this point in the history
  • Loading branch information
@biplovbhandari
biplovbhandari committed May 22, 2021
1 parent 611e6f9 commit e1612f7
Show file tree
Hide file tree
Showing 19 changed files with 9,942 additions and 4,444 deletions.
137 changes: 91 additions & 46 deletions mrc_data_download.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 95,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -25,7 +25,22 @@
},
{
"cell_type": "code",
"execution_count": 142,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"start_date = '2012-01-01'\n",
"end_date = '2019-12-31'\n",
"date_tail = 'T00:00:00.000Z'\n",
"request_param = dict(sd=f'{start_date}{date_tail}', ed=f'{end_date}{date_tail}')\n",
"full_dates = pd.date_range(datetime.strptime(start_date, '%Y-%m-%d'),\n",
" datetime.strptime(end_date, '%Y-%m-%d'), freq='d')\\\n",
" .strftime('%Y-%m-%d').tolist()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -83,12 +98,37 @@
" 'name': 'Kratie',\n",
" 'discharge_id': ['17c0ffe74e224b50a9e3142fc404262b', 'ecc50c5540564a27bb9f7f147027bd9f'],\n",
" },\n",
" {\n",
" 'name': 'Kompong_Chen',\n",
" 'stage_id': '53359074bfcf4961866daac9fbee8292',\n",
" },\n",
" {\n",
" 'name': 'Kompong_Cham',\n",
" 'stage_id': 'b5da800e9c8e4346a3b8d9d6daa73e0a',\n",
" },\n",
" {\n",
" 'name': 'Kompong_Thom',\n",
" 'stage_id': ['dae00e365f134b4bbfa6bf4162eb815c', 'aaebd95890ca49b9b5b5a82699159d6d'],\n",
" },\n",
" {\n",
" 'name': 'Siempang',\n",
" 'discharge_id': '3f588063c8bd4f06a477251701ec6ab0',\n",
" 'stage_id': ['6a49fa1f6cb247d39b9e6c405205bfa3', '5794c0c5c5c045639e0efa1540466828'],\n",
" },\n",
" {\n",
" 'name': 'Battambang',\n",
" 'stage_id': ['cba718574b1f4244a4ae2ef58806732d', '8456239a6baf4221a744886c8cb6efd3'],\n",
" },\n",
" {\n",
" 'name': 'Sisophon',\n",
" 'stage_id': 'f08efea6bd684203ab33c5a316e2459a',\n",
" },\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": 143,
"execution_count": 4,
"metadata": {},
"outputs": [
{
Expand All @@ -97,7 +137,7 @@
"2922"
]
},
"execution_count": 143,
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -109,22 +149,7 @@
},
{
"cell_type": "code",
"execution_count": 144,
"metadata": {},
"outputs": [],
"source": [
"start_date = '2013-01-01'\n",
"end_date = '2020-12-31'\n",
"date_tail = 'T00:00:00.000Z'\n",
"request_param = dict(sd=f'{start_date}{date_tail}', ed=f'{end_date}{date_tail}')\n",
"full_dates = pd.date_range(datetime.strptime(start_date, '%Y-%m-%d'),\n",
" datetime.strptime(end_date, '%Y-%m-%d'), freq='d')\\\n",
" .strftime('%Y-%m-%d').tolist()"
]
},
{
"cell_type": "code",
"execution_count": 145,
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -143,7 +168,7 @@
},
{
"cell_type": "code",
"execution_count": 146,
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -152,7 +177,7 @@
},
{
"cell_type": "code",
"execution_count": 147,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -173,7 +198,7 @@
},
{
"cell_type": "code",
"execution_count": 148,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -186,7 +211,7 @@
},
{
"cell_type": "code",
"execution_count": 149,
"execution_count": null,
"metadata": {},
"outputs": [
{
Expand All @@ -209,18 +234,18 @@
"-------------------------------------------\n",
"station: Kompong_Kdei\n",
"discharge: len(dates): 0\n",
"before len(df_stage): 2765\n",
"after len(df_stage): 2765\n",
"before len(df_stage): 2762\n",
"after len(df_stage): 2762\n",
"-------------------------------------------\n",
"station: Prek_Kdam\n",
"discharge: len(dates): 64\n",
"discharge: len(dates): 85\n",
"before len(df_stage): 2922\n",
"after len(df_stage): 2858\n",
"after len(df_stage): 2837\n",
"-------------------------------------------\n",
"station: Kg._Thmar\n",
"discharge: len(dates): 0\n",
"before len(df_stage): 2756\n",
"after len(df_stage): 2756\n",
"before len(df_stage): 2755\n",
"after len(df_stage): 2755\n",
"-------------------------------------------\n",
"station: Koh_Khel\n",
"discharge: len(dates): 0\n",
Expand All @@ -243,12 +268,30 @@
"after len(df_stage): 2922\n",
"-------------------------------------------\n",
"station: Voeun_Sai\n",
"discharge: len(dates): 2769\n",
"before len(df_stage): 2769\n",
"discharge: len(dates): 2768\n",
"before len(df_stage): 2768\n",
"after len(df_stage): 0\n",
"-------------------------------------------\n",
"station: Kratie\n",
"discharge: len(dates): 2922\n"
"discharge: len(dates): 2922\n",
"-------------------------------------------\n",
"station: Kompong_Chen\n",
"discharge: len(dates): 0\n",
"before len(df_stage): 2620\n",
"after len(df_stage): 2620\n",
"-------------------------------------------\n",
"station: Kompong_Cham\n",
"discharge: len(dates): 0\n",
"before len(df_stage): 2922\n",
"after len(df_stage): 2922\n",
"-------------------------------------------\n",
"station: Kompong_Thom\n",
"discharge: len(dates): 0\n",
"before len(df_stage): 366\n",
"after len(df_stage): 366\n",
"-------------------------------------------\n",
"station: Siempang\n",
"discharge: len(dates): 364\n"
]
}
],
Expand All @@ -268,13 +311,14 @@
" if isinstance(discharge_id, list):\n",
" for _id in discharge_id:\n",
" _df = get_data(f'{base_url}/{_id}', request_param, headers, cols[1])\n",
" _df = _df.groupby(_df.datetime.dt.date).mean()\n",
" _df['datetime'] = _df.index\n",
" if not df_discharge.empty and not _df.empty:\n",
" # replace measured with calculated\n",
" df_discharge.loc[df_discharge.datetime.isin(_df.datetime), cols] = _df[cols]\n",
" else:\n",
" df_discharge = _df\n",
" if not _df.empty:\n",
" _df = _df.groupby(_df.datetime.dt.date).mean()\n",
" _df['datetime'] = _df.index\n",
" if not df_discharge.empty:\n",
" # replace measured with calculated\n",
" df_discharge.loc[df_discharge.datetime.isin(_df.datetime), cols] = _df[cols]\n",
" else:\n",
" df_discharge = _df\n",
" else:\n",
" df_discharge = get_data(f'{base_url}/{discharge_id}', request_param, headers, cols[1])\n",
" df_discharge = df_discharge.groupby(df_discharge.datetime.dt.date).mean()\n",
Expand All @@ -300,12 +344,13 @@
" if isinstance(stage_id, list):\n",
" for _id in stage_id:\n",
" _df = get_data(f'{base_url}/{_id}', request_param, headers, cols[1])\n",
" _df = _df.groupby(_df.datetime.dt.date).mean()\n",
" _df['datetime'] = _df.index\n",
" if not df_stage.empty and not _df.empty:\n",
" df_stage.loc[df_stage.datetime.isin(_df.datetime), cols] = _df[cols]\n",
" else:\n",
" df_stage = _df\n",
" if not _df.empty:\n",
" _df = _df.groupby(_df.datetime.dt.date).mean()\n",
" _df['datetime'] = _df.index\n",
" if not df_stage.empty:\n",
" df_stage.loc[df_stage.datetime.isin(_df.datetime), cols] = _df[cols]\n",
" else:\n",
" df_stage = _df\n",
" else:\n",
" df_stage = get_data(f'{base_url}/{stage_id}', request_param, headers, cols[1])\n",
" df_stage = df_stage.groupby(df_stage.datetime.dt.date).mean()\n",
Expand Down
Loading

0 comments on commit e1612f7

Please sign in to comment.