ceds_derived_sectors.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# As of 2024/08/30, this notebook has been superseded by the functions in the file gap_filling/ceds_derived_sectors.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Created on: 2024/08/28\n",
    "\n",
    "Created by: DPM\n",
    "\n",
    "\n",
    "Summary: The purpose of this notebook is to use a combination of CEDS and EDGAR data to derive new sectors to partially replace the pre-2024 other-manufacturing CT sector. For a complete understanding of the methods and equations used to derive the new sectors see the following documents:\n",
    "\n",
    "https://docs.google.com/spreadsheets/d/1Acei92ou6mZsOL2nXEOYNP7WPNH_A7Gds-AQ3GXdQDs/edit?gid=1870391791#gid=1870391791 ('2024 gap equations' Tab)\n",
    "\n",
    "https://docs.google.com/presentation/d/1mHkGVHPjibjqEK2o6fG6Wlp7QnSpHzcmnTdHRliraZU/edit?usp=sharing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import datetime\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from gap_filling.data_handler import DataHandler\n",
    "from gap_filling.edgar_projection import ProjectData\n",
    "from gap_filling.utils import (parse_and_format_data_to_insert, get_all_edgar_data, get_all_ceds_data, get_all_ceds_data_csv)\n",
    "from gap_filling import annexI_food_bev\n",
    "\n",
    "\n",
    "# def process_all(args, new_db):\n",
    "############################\n",
    "# Get the data\n",
    "############################\n",
    "# Init the Data Handler\n",
    "\n",
    "\n",
    "# get connections\n",
    "getedgar_conn = DataHandler()\n",
    "getct_conn = DataHandler()\n",
    "get_ceds_conn = DataHandler()\n",
    "# write_conn = DataHandler()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "edgar_data = get_all_edgar_data(getedgar_conn, get_projected=False)\n",
    "#Convert column names to strings for processing\n",
    "edgar_data.columns = edgar_data.columns.astype(str)\n",
    "\n",
    "\n",
    "#Get CEDS data\n",
    "# ceds_data = get_all_ceds_data(get_ceds_conn)\n",
    "# #Convert column names to strings for processing\n",
    "# ceds_data.columns = ceds_data.columns.astype(str)\n",
    "\n",
    "ceds_data = get_all_ceds_data_csv()\n",
    "\n",
    "#Now ensure all ceds and edgar values are numpy floats\n",
    "for yr in range(2015,2023):\n",
    "    edgar_data[str(yr)] = edgar_data[str(yr)].astype(float)\n",
    "    ceds_data[str(yr)] = ceds_data[str(yr)].astype(float)\n",
    "    \n",
    "\n",
    "# Get the CT data from db just for reference\n",
    "ct_data = getct_conn.load_data(\"climate-trace\", years_to_columns=True)\n",
    "ct_data.columns = ct_data.columns.astype(str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below are individual code snippets to arrive at the ceds-derived data to be plugged into Christy's gap-filling equations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Now create new CEDS sectors:\n",
    "#First iron-and-steel (need to subtract CT steel)\n",
    "add_secs = ['1A2a_Ind-Comb-Iron-steel', '2C1_Iron-steel-alloy-prod']\n",
    "iron_steel_ceds_derived_data = ceds_data[ceds_data[\"Sector\"].isin(add_secs)].groupby([\"ID\", \"Gas\"]).sum().reset_index()\n",
    "iron_steel_ceds_derived_data[\"Sector\"] = \"iron-and-steel\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Need to figure out why length of output dataframe in these steps does not match input dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sector_fractional_contribution(inventory_data, \n",
    "                            add_secs, \n",
    "                            inventory_data_contributing_sec, \n",
    "                            inventory_data_to_take_fraction_of, \n",
    "                            new_sector_name):\n",
    "    \"\"\"Calculates the fractional contribution of one \n",
    "    subsector to the sum of a list of sectors. Then\n",
    "    applies that fractional contribution to a final \n",
    "    sector.\n",
    "\n",
    "    Example: calculating the contribution of Glass to \n",
    "    to sum of Glass + Lime + Cement + other-minerals\n",
    "    and applying that fraction to the combustion from\n",
    "    those same sectors.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    inventory_data: pandas df\n",
    "    Country-level annual emissions by sector.\n",
    "    Must include the sectors to be summed. Could be \n",
    "    edgar, ceds, faostat, climate-trace\n",
    "\n",
    "    add_secs: list of strings\n",
    "    list of sectors to be summed\n",
    "\n",
    "    inventory_data_contributing_sec: pandas df\n",
    "    Should be same inventory as inventory_data\n",
    "    but is only for the sector(s) that you want\n",
    "    to calculate the fractional contribution of.\n",
    "\n",
    "    inventory_data_to_take_fraction_of: pandas df\n",
    "    Can be from same or other inventory as inventory_data.\n",
    "    Contains only one sector of which you would like to \n",
    "    attribute only a fraction to the new sector.\n",
    "\n",
    "    new_sector_name: str\n",
    "    name for the new sector which is the fraction of\n",
    "    inventory_data_to_take_fraction_of contributed by\n",
    "    inventory_data_contributing_sec sector.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    frac_of_other_sector: pandas df\n",
    "    inventory containing the new sector, no others.\n",
    "\n",
    "    \"\"\"\n",
    "    \n",
    "    inventory_data_sum_add_secs = inventory_data[inventory_data[\"Sector\"].isin(add_secs)].groupby([\"ID\", \"Gas\"]).sum().reset_index()\n",
    "    # t_edgar_total_sector_data[\"Sector\"] = \"cement-lime-glass-other\"\n",
    "    print('For testing... all add_secs sectors summed for the USA is:', \\\n",
    "          inventory_data_sum_add_secs.loc[inventory_data_sum_add_secs.ID == \"USA\",['Gas', '2020']])\n",
    "\n",
    "    #Next add Glass-production so we can get its contribution to the total\n",
    "    inventory_frac_subsector = pd.merge(inventory_data_sum_add_secs, inventory_data_contributing_sec, on=['ID', \"Gas\"], suffixes=('_total',''))\n",
    "    for yr in range(2015,2023):\n",
    "        inventory_frac_subsector[f'{yr}'] = inventory_frac_subsector[f'{yr}'] / inventory_frac_subsector[f'{yr}_total']\n",
    "        inventory_frac_subsector.drop(columns=[f'{yr}_total'], inplace=True)\n",
    "    inventory_frac_subsector.fillna(0, inplace=True)\n",
    "    # edgar_frac_glass[\"Sector\"] = \"edgar-fraction-glass\"\n",
    "    inventory_frac_subsector.drop(columns=['Data source'], inplace=True)\n",
    "\n",
    "    print('For testing... the fraction of the specified sector compared to the sum above for the USA is:', \\\n",
    "          inventory_frac_subsector.loc[inventory_frac_subsector.ID == \"USA\",['Gas', '2020']])\n",
    "\n",
    "\n",
    "    #Now multiply edgar_frac_glass by 1.A.2.f to get the contribution of glass to 1.A.2.f\n",
    "    frac_of_other_sector = pd.merge(inventory_frac_subsector, inventory_data_to_take_fraction_of, on=['ID', \"Gas\"], suffixes=('_frac','_total'))\n",
    "    for yr in range(2015,2023):\n",
    "        frac_of_other_sector[f'{yr}'] = frac_of_other_sector[f'{yr}_frac'] * frac_of_other_sector[f'{yr}_total']\n",
    "        frac_of_other_sector.drop(columns=[f'{yr}_total', f'{yr}_frac'], inplace=True)\n",
    "    frac_of_other_sector.fillna(0, inplace=True)\n",
    "    frac_of_other_sector.drop(columns=['Sector_total', 'Sector_frac',\n",
    "                                \"Unit_frac\", \"Unit_total\"], inplace=True)\n",
    "    frac_of_other_sector[\"Sector\"] = new_sector_name\n",
    "    frac_of_other_sector[\"Unit\"] = \"tonnes\"\n",
    "    return frac_of_other_sector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For testing... all add_secs sectors summed for the USA is: year  Gas          2020\n",
      "195   co2  6.648744e+07\n",
      "For testing... the fraction of the specified sector compared to the sum above for the USA is: year  Gas      2020\n",
      "108   co2  0.178229\n",
      "Total lime-production emissions:         Gas          2020\n",
      "11973  ch4  0.000000e+00\n",
      "11973  co2  7.248908e+06\n",
      "11973  n2o  0.000000e+00\n",
      "Total non-metalic-minerals combustion:         Gas          2020\n",
      "11950  ch4  7.960794e+02\n",
      "11950  co2  3.477485e+07\n",
      "11950  n2o  5.615269e+02\n",
      "Contribution of lime to non-metalic-minerals combustion:       Gas          2020\n",
      "107  co2  6.197892e+06\n",
      "\n",
      " If everything was successful, the following must be true:\n",
      " 1) the first number (for a given gas) times the second number (for a given gas) should equal the third number, \n",
      " 2) the final number should equal the product of the second and fourth numbers.\n"
     ]
    }
   ],
   "source": [
    "#Now, need to handle proportions of sectors\n",
    "#First get some of all sectors as we did above:\n",
    "add_secs = ['2.A.3 Glass Production', '2.A.1 Cement production', '2.A.2 Lime production', \\\n",
    "            '2.A.4 Other Process Uses of Carbonates']\n",
    "\n",
    "\n",
    "ceds_lime_comb = sector_fractional_contribution(edgar_data, add_secs, edgar_data[edgar_data['Sector'] =='2.A.2 Lime production'],\n",
    "                        ceds_data[ceds_data['Sector'] == '1A2f_Ind-Comb-Non-metalic-minerals'],\n",
    "                        'lime-combustion')\n",
    "\n",
    "\n",
    "#Now check the work!\n",
    "print(\"Total lime-production emissions: \", ceds_data.loc[(ceds_data['ID'] == \"USA\") & (ceds_data[\"Sector\"] == '2A2_Lime-production'),['Gas', '2020']])\n",
    "print(\"Total non-metalic-minerals combustion: \", ceds_data.loc[(ceds_data['ID'] == \"USA\") & (ceds_data[\"Sector\"] == '1A2f_Ind-Comb-Non-metalic-minerals'),['Gas', '2020']])\n",
    "print(\"Contribution of lime to non-metalic-minerals combustion: \", ceds_lime_comb.loc[ceds_lime_comb.ID==\"USA\",['Gas', '2020']])\n",
    "\n",
    "print(\"\\n If everything was successful, the following must be true:\\n 1) the first number (for a given gas) times the second number (for a given gas) should equal the third number, \\n 2) the final number should equal the product of the second and fourth numbers.\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For testing... all add_secs sectors summed for the USA is: year  Gas          2020\n",
      "195   co2  6.648744e+07\n",
      "For testing... the fraction of the specified sector compared to the sum above for the USA is: year  Gas      2020\n",
      "195   co2  0.027083\n",
      "Total glass-production emissions:  year  Gas         2020\n",
      "6479  co2  1800675.072\n",
      "Total non-metalic-minerals combustion:         Gas          2020\n",
      "11950  ch4  7.960794e+02\n",
      "11950  co2  3.477485e+07\n",
      "11950  n2o  5.615269e+02\n",
      "Contribution of glass to non-metalic-minerals combustion:       Gas           2020\n",
      "192  co2  941804.979466\n",
      "\n",
      " If everything was successful, the following must be true:\n",
      " 1) the first number (for a given gas) times the second number (for a given gas) should equal the third number, \n",
      " 2) the final number should equal the product of the second and fourth numbers.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#Now a slightly more complicated case where we have to employ EDGAR data to derive a fractional contribution of glass to CEDS\n",
    "# 1A2f_Ind-Comb-Non-metalic-minerals\n",
    "add_secs = ['2.A.3 Glass Production', '2.A.1 Cement production', '2.A.2 Lime production', \\\n",
    "            '2.A.4 Other Process Uses of Carbonates']\n",
    "\n",
    "ceds_glass_comb = sector_fractional_contribution(edgar_data, add_secs, edgar_data[edgar_data['Sector'] =='2.A.3 Glass Production'],\n",
    "                        ceds_data[ceds_data['Sector'] == '1A2f_Ind-Comb-Non-metalic-minerals'],\n",
    "                        'glass-combustion')\n",
    "\n",
    "\n",
    "'''I don't think this is needed since the gapfilling code fills in for missing gases\n",
    "\n",
    "#Finally, we have to add 'n2o' and 'ch4' to this dataframe for each country because EDGAR assumes 0 emissions from these gases, while \n",
    "# CEDS does not.\n",
    "def create_dummy_df(country_values, gas, sector, unit=\"tonnes\"):\n",
    "    dummy_df = pd.DataFrame({'ID': country_values})\n",
    "    dummy_df['Gas'] = gas\n",
    "    dummy_df['Unit'] = unit\n",
    "    dummy_df['Sector'] = sector\n",
    "    for c in np.arange(2015,2023).astype(str): dummy_df[c] = 0\n",
    "    return dummy_df\n",
    "\n",
    "n2o_dummy_df = create_dummy_df(np.unique(ceds_glass_comb.ID.values), 'n2o', 'glass-combustion')\n",
    "ch4_dummy_df = create_dummy_df(np.unique(ceds_glass_comb.ID.values), 'ch4', 'glass-combustion')\n",
    "ceds_glass_comb = pd.concat([ceds_glass_comb, n2o_dummy_df, ch4_dummy_df], axis=0).reset_index(drop=True)\n",
    "'''\n",
    "\n",
    "#Now check the work!\n",
    "print(\"Total glass-production emissions: \", edgar_data.loc[(edgar_data['ID'] == \"USA\") & (edgar_data[\"Sector\"] == '2.A.3 Glass Production'),['Gas', '2020']])\n",
    "print(\"Total non-metalic-minerals combustion: \", ceds_data.loc[(ceds_data['ID'] == \"USA\") & (ceds_data[\"Sector\"] == '1A2f_Ind-Comb-Non-metalic-minerals'),['Gas', '2020']])\n",
    "print(\"Contribution of glass to non-metalic-minerals combustion: \", ceds_glass_comb.loc[ceds_glass_comb.ID==\"USA\",['Gas', '2020']])\n",
    "\n",
    "print(\"\\n If everything was successful, the following must be true:\\n 1) the first number (for a given gas) times the second number (for a given gas) should equal the third number, \\n 2) the final number should equal the product of the second and fourth numbers.\")\n",
    "\n",
    "#It seems like everything works!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For testing... all add_secs sectors summed for the USA is: year  Gas          2020\n",
      "195   co2  6.648744e+07\n",
      "For testing... the fraction of the specified sector compared to the sum above for the USA is: year  Gas      2020\n",
      "173   co2  0.183084\n",
      "Total other-mineral-production emissions:  year  Gas         2020\n",
      "6664  co2  12172762.94\n",
      "Total non-metalic-minerals combustion:         Gas          2020\n",
      "11950  ch4  7.960794e+02\n",
      "11950  co2  3.477485e+07\n",
      "11950  n2o  5.615269e+02\n",
      "Contribution of 'other' to non-metalic-minerals combustion:       Gas          2020\n",
      "171  co2  6.366706e+06\n",
      "\n",
      " If everything was successful, the following must be true:\n",
      " 1) the first number (for a given gas) times the second number (for a given gas) should equal the third number, \n",
      " 2) the final number should equal the product of the second and fourth numbers.\n"
     ]
    }
   ],
   "source": [
    "## now need to make misc-mineral-industry-combustion ceds-derived sector to be included into misc CT sector\n",
    "add_secs = ['2.A.3 Glass Production', '2.A.1 Cement production', '2.A.2 Lime production', \\\n",
    "            '2.A.4 Other Process Uses of Carbonates']\n",
    "\n",
    "ceds_mineral_comb = sector_fractional_contribution(edgar_data, add_secs, edgar_data[edgar_data['Sector'] =='2.A.4 Other Process Uses of Carbonates'],\n",
    "                        ceds_data[ceds_data['Sector'] == '1A2f_Ind-Comb-Non-metalic-minerals'],\n",
    "                        'misc-mineral-industry-combustion')\n",
    "\n",
    "#Now check the work!\n",
    "print(\"Total other-mineral-production emissions: \", edgar_data.loc[(edgar_data['ID'] == \"USA\") & (edgar_data[\"Sector\"] == '2.A.4 Other Process Uses of Carbonates'),['Gas', '2020']])\n",
    "print(\"Total non-metalic-minerals combustion: \", ceds_data.loc[(ceds_data['ID'] == \"USA\") & (ceds_data[\"Sector\"] == '1A2f_Ind-Comb-Non-metalic-minerals'),['Gas', '2020']])\n",
    "print(\"Contribution of 'other' to non-metalic-minerals combustion: \", ceds_mineral_comb.loc[ceds_mineral_comb.ID==\"USA\",['Gas', '2020']])\n",
    "\n",
    "print(\"\\n If everything was successful, the following must be true:\\n 1) the first number (for a given gas) times the second number (for a given gas) should equal the third number, \\n 2) the final number should equal the product of the second and fourth numbers.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test that misc_mineral_industry_combustion +cement_combustion + lime_combustion + glass-combustion equals non-metalic-minerals-combustion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For testing... all add_secs sectors summed for the USA is: year  Gas          2020\n",
      "195   co2  6.648744e+07\n",
      "For testing... the fraction of the specified sector compared to the sum above for the USA is: year  Gas      2020\n",
      "156   co2  0.611604\n",
      "Testing complete... separate combustion emissions sum to total CEDS 1A2f combustion emissions.\n"
     ]
    }
   ],
   "source": [
    "#First get cement-combustion for completion:\n",
    "add_secs = ['2.A.3 Glass Production', '2.A.1 Cement production', '2.A.2 Lime production', \\\n",
    "            '2.A.4 Other Process Uses of Carbonates']\n",
    "\n",
    "ceds_cement_comb = sector_fractional_contribution(edgar_data, add_secs, edgar_data[edgar_data['Sector'] =='2.A.1 Cement production'],\n",
    "                        ceds_data[ceds_data['Sector'] == '1A2f_Ind-Comb-Non-metalic-minerals'],\n",
    "                        'cement-combustion')\n",
    "\n",
    "#Combine all four dfs:\n",
    "dfs_to_combine = [ceds_cement_comb, ceds_mineral_comb, ceds_glass_comb, ceds_lime_comb]\n",
    "comb_df = dfs_to_combine[0]\n",
    "suffixes = [\"_cement\", \"_other\", \"_glass\", \"_lime\"]\n",
    "for i in range(1, len(dfs_to_combine)):\n",
    "    comb_df = pd.merge(comb_df, dfs_to_combine[i], on=[\"ID\", \"Gas\"], suffixes=('', suffixes[i]))\n",
    "comb_df.columns = [col + suffixes[0] if col in np.arange(2015,2023).astype(str).tolist() else col\\\n",
    "                   for col in comb_df.columns]\n",
    "\n",
    "#Now sum them\n",
    "for yr in range(2015,2023):\n",
    "    comb_df[str(yr)] = comb_df[f\"{yr}_cement\"] + comb_df[f\"{yr}_glass\"] +\\\n",
    "                    comb_df[f\"{yr}_lime\"] + comb_df[f\"{yr}_other\"]\n",
    "\n",
    "comb_df = comb_df[np.arange(2015,2023).astype(str).tolist() + [\"Gas\"] + [\"ID\"]]\n",
    "\n",
    "#Now merge with 1A2e CEDS data and take difference by year\n",
    "ceds_comb_data = ceds_data[ceds_data[\"Sector\"] == \"1A2f_Ind-Comb-Non-metalic-minerals\"]\n",
    "merged_df = pd.merge(comb_df, ceds_comb_data, on=[\"ID\", \"Gas\"], suffixes=('_summed', \"_total\"))\n",
    "\n",
    "test_to_be_zero = []\n",
    "for yr in range(2015,2023):\n",
    "    test_to_be_zero.append(np.nansum(merged_df[f\"{str(yr)}_summed\"] - merged_df[f\"{str(yr)}_total\"]))\n",
    "\n",
    "if all([x < 1e-5 for x in test_to_be_zero]):\n",
    "    print(\"Testing complete... separate combustion emissions sum to total CEDS 1A2f combustion emissions.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We will be doing something slightly different for food-beverage-tobacco. We have decided to take the Annex I countries that have reported both 2.H.2 and 1.A.2.e emissions (n=6 countries), quantified the country-specific relationship between these two (either mean of the annual ratios, or the slope of annual values) and applied this relationship to CEDS 1.A.2.e emissions for these 6 countries to estimate their 2.H.2 emissions. We will assume 0 emissions for all other countries."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the calculation of the country-specific EFs, see the food_bev_pulp_paper_estimates.ipynb notebook. We will read these as a csv output by functions developed in that notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dpmoore/Code/climate-trace-gapfilling/gap_filling/annexI_food_bev.py:113: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  sector_ceds_df.loc[:, \"Sector\"] = \"2.H.2-food-beverage-and-tobacco-direct\"\n"
     ]
    }
   ],
   "source": [
    "food_bev_direct = annexI_food_bev.main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now we can combine everything into one dataframe for inserting into the db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "ceds_derived_df = pd.concat([ceds_mineral_comb,\n",
    "                             ceds_lime_comb, ceds_glass_comb,\n",
    "                             food_bev_direct])\n",
    "ceds_derived_df['Unit'] = 'tonnes'\n",
    "ceds_derived_df['Data source'] = 'ceds-derived'\n",
    "ceds_derived_df = ceds_derived_df.sort_values(by=['ID', 'Sector','Gas']).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Need to convert columns to integers\n",
    "def convert_column_to_int(col):\n",
    "    try:\n",
    "        return(int(col))\n",
    "    except ValueError:\n",
    "        return col\n",
    "ceds_derived_df.rename(columns=convert_column_to_int, inplace=True)\n",
    "\n",
    "data_to_insert = parse_and_format_data_to_insert(ceds_derived_df)\n",
    "data_to_insert['created_date'] = datetime.datetime.now().isoformat()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "climate-trace-gapfilling",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}