parse_STEAD_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "grFb3giy6RB-"
   },
   "source": [
    "# Parsing STEAD Data\n",
    "\n",
    "STEAD is a 3 component seimograms (each 1 min long) where the three components are vertial (Z), north-south (N), and east-west (E).\n",
    "\n",
    "Since each chunk of the STEAD data has ~200,000 waveform, this notebook will parse the STEAD data to a smaller hdf5 and csv file pair."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "of-Jz2VS2QBN"
   },
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16722,
     "status": "ok",
     "timestamp": 1716749096897,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "AvGqJEuY1_ep",
    "outputId": "12600fa6-530c-4b48-c56c-b917b0c48c5c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/gdrive/\n"
     ]
    }
   ],
   "source": [
    "# this step mounts your Google drive to the current Colab runtime\n",
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive/', force_remount=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 181,
     "status": "ok",
     "timestamp": 1714865905958,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "cZiHq9XFx6Ng",
    "outputId": "bc7f34b4-526f-4dc4-eff9-a37ed6438fe4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 554,
     "status": "ok",
     "timestamp": 1716749099677,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "XNMX2h8FkGD2",
    "outputId": "5c74906a-a6d8-4ca9-a306-9a508e4946e4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/gdrive/.shortcut-targets-by-id/1kkgdA-hKtghS6jCSGo9StdGqoqNNek2i/CSE 547 Project/STEAD Data\n"
     ]
    }
   ],
   "source": [
    "# Move to STEAD folder\n",
    "%cd /content/gdrive/MyDrive/path_to_STEAD_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 380,
     "status": "ok",
     "timestamp": 1714865913275,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "eUp-q-CMCCCX",
    "outputId": "b928a8b6-6ea7-4e80-8dd7-802d39e8db94"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "01a_dataset_basics.ipynb\t  parse_STEAD_data.ipynb  small_subset.hdf5  waveform_chunk2\n",
      "convert_STEAD_to_Seisbench.ipynb  seisbench_data\t  subset.csv\n",
      "noise_data\t\t\t  small_subset.csv\t  subset.hdf5\n"
     ]
    }
   ],
   "source": [
    "# Confirm contents of current directory.\n",
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 6122,
     "status": "ok",
     "timestamp": 1716749106730,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "M3SOgeJPu3aO",
    "outputId": "8ab97835-550c-4359-9864-36fa892bfdb7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: h5py in /usr/local/lib/python3.10/dist-packages (3.9.0)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.10/dist-packages (from h5py) (1.25.2)\n"
     ]
    }
   ],
   "source": [
    "# Install h5py to read hdf5 files.\n",
    "!pip install h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "executionInfo": {
     "elapsed": 578,
     "status": "ok",
     "timestamp": 1716749107305,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "FPk-IMpf_sbZ"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import h5py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hx4SYipodSwU"
   },
   "source": [
    "## Read in Noise & Waveform data\n",
    "\n",
    "The noise file had 235426 entries and the earthquake data (chunk2) had exactly 200,000 observations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 8188,
     "status": "ok",
     "timestamp": 1716749116846,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "KLvMFUmGdXjp",
    "outputId": "cd5b7d21-b9f3-4bd6-b8bf-5ae5db6453d3"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-6-eaaa6a7ca401>:9: DtypeWarning: Columns (26) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  eq_csv = pd.read_csv('./waveform_chunk2/chunk2.csv')\n"
     ]
    }
   ],
   "source": [
    "# Read in the noise data\n",
    "noise_data = h5py.File('./noise_data/chunk1.hdf5', 'r')\n",
    "noise_wave = noise_data.get('data')\n",
    "noise_csv = pd.read_csv('./noise_data/chunk1.csv')\n",
    "\n",
    "# Read in the earthquake data\n",
    "eq_data = h5py.File('./waveform_chunk2/chunk2.hdf5', 'r')\n",
    "eq_wave = eq_data.get('data')\n",
    "eq_csv = pd.read_csv('./waveform_chunk2/chunk2.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hvLj8Un9e5aJ"
   },
   "source": [
    "Steps for making a smaller hdf5 file.\n",
    "1. First create a new hdf5 file.\n",
    "2. Then create the group 'data' using the create_group fn.\n",
    "3. Partition the csv file so we have a managable number of entries.\n",
    "4. For each trace_name in the subset, create a dataset using the create_dataset fn. Examples under dataset documentation for the h5py package."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VILqsCd-gPLu"
   },
   "source": [
    "To keep the ratio of noise:earthquake similar to the original STEAD data, select 3,350 noise : 20,0000 eqarthquake data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1716662840416,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "oCw01UtfE8y0",
    "outputId": "b45cb8ac-69cb-4171-9f9d-2c72b31e7ec9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "235426"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(noise_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 169,
     "status": "ok",
     "timestamp": 1716662848746,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "TD6ljCV_FDJR",
    "outputId": "f6d36596-58a9-401a-d67d-4122bdf11a57"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200000"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(eq_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mMKzbnAefNos"
   },
   "outputs": [],
   "source": [
    "# Select random observations. Use set seed for reproducibility.\n",
    "np.random.seed(123)\n",
    "\n",
    "n_eq = 150000#200000#10000#1000#20000\n",
    "n_noise = 25125#33500#16750#10#3350\n",
    "\n",
    "csv_filename = 'chunk1_34_wnoise.csv'\n",
    "hdf5_filename = 'chunk1_34_wnoise.hdf5'\n",
    "\n",
    "noise_samp = np.sort(np.random.choice(len(noise_csv), n_noise, replace=False))\n",
    "eq_samp = np.sort(np.random.choice(len(eq_csv), n_eq, replace=False))\n",
    "\n",
    "noise_subset = noise_csv.iloc[noise_samp]\n",
    "eq_subset = eq_csv.iloc[eq_samp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pSIsMPw5lCVi"
   },
   "outputs": [],
   "source": [
    "# Combine the noise & earthquake csv data to one dataframe\n",
    "combined = pd.concat([noise_subset, eq_subset], axis=0)\n",
    "\n",
    "# Write the combined csv to a file\n",
    "combined.to_csv(csv_filename, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QGRt0CwChUS2"
   },
   "outputs": [],
   "source": [
    "# Retrieve the trace_name for the selected observations.\n",
    "noise_names = noise_subset['trace_name']\n",
    "eq_names = eq_subset['trace_name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1Nqi2N5qjBTc"
   },
   "outputs": [],
   "source": [
    "# Write to h5py file\n",
    "with h5py.File(hdf5_filename, \"w\") as f:\n",
    "  grp = f.create_group('data')\n",
    "\n",
    "  for name in noise_names:\n",
    "    data = noise_wave.get(name)\n",
    "    value = data[0:]\n",
    "    grp.create_dataset(name = name, data = data)\n",
    "\n",
    "  for name in eq_names:\n",
    "    data = eq_wave.get(name)\n",
    "    vale = data[0:]\n",
    "    grp.create_dataset(name = name, data = data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "s0_m0Xucmk6a"
   },
   "source": [
    "# Original Super Dirty Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 3294,
     "status": "ok",
     "timestamp": 1716699412569,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "17877937658835215580"
     },
     "user_tz": 420
    },
    "id": "_3ENocVqQVHN",
    "outputId": "509b22d9-ec55-4969-ab2a-b13ba9a50648"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-6-79e738f42327>:1: DtypeWarning: Columns (7,11,13,14,24,25,26,30,31) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  test = pd.read_csv('./seisbench_data/chunk1_wnoise_data3/metadata_c1wn.csv')\n"
     ]
    }
   ],
   "source": [
    "test = pd.read_csv('./seisbench_data/chunk1_wnoise_data3/metadata_c1wn.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 6,
     "status": "ok",
     "timestamp": 1716699418435,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "17877937658835215580"
     },
     "user_tz": 420
    },
    "id": "XcJo-MbTQjYb",
    "outputId": "51bad3f5-4692-4376-91d3-ab306e497f9d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "233500"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 391
    },
    "executionInfo": {
     "elapsed": 642,
     "status": "error",
     "timestamp": 1716699429202,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "17877937658835215580"
     },
     "user_tz": 420
    },
    "id": "nKs039LSmC00",
    "outputId": "8108a307-5954-49ca-8c07-337dd5012e69"
   },
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "Unable to open file (bad object header version number)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-f1b638a59415>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5py\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./seisbench_data/chunk1_wnoise_data3/waveforms_c1wn.hdf5'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/h5py/_hl/files.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, mode, driver, libver, userblock_size, swmr, rdcc_nslots, rdcc_nbytes, rdcc_w0, track_order, fs_strategy, fs_persist, fs_threshold, fs_page_size, page_buf_size, min_meta_keep, min_raw_keep, locking, alignment_threshold, alignment_interval, meta_block_size, **kwds)\u001b[0m\n\u001b[1;32m    565\u001b[0m                                  \u001b[0mfs_persist\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfs_persist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfs_threshold\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfs_threshold\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    566\u001b[0m                                  fs_page_size=fs_page_size)\n\u001b[0;32m--> 567\u001b[0;31m                 \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_fid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muserblock_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfcpl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswmr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswmr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    568\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    569\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlibver\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/h5py/_hl/files.py\u001b[0m in \u001b[0;36mmake_fid\u001b[0;34m(name, mode, userblock_size, fapl, fcpl, swmr)\u001b[0m\n\u001b[1;32m    229\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mswmr\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mswmr_support\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    230\u001b[0m             \u001b[0mflags\u001b[0m \u001b[0;34m|=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mACC_SWMR_READ\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 231\u001b[0;31m         \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfapl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    232\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'r+'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m         \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mACC_RDWR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfapl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mh5py/h5f.pyx\u001b[0m in \u001b[0;36mh5py.h5f.open\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: Unable to open file (bad object header version number)"
     ]
    }
   ],
   "source": [
    "test = h5py.File('./seisbench_data/chunk1_wnoise_data3/waveforms_c1wn.hdf5', 'r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ti_R3vOlmOdi"
   },
   "outputs": [],
   "source": [
    "test_data = test.get('data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 410,
     "status": "ok",
     "timestamp": 1716688648228,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "17877937658835215580"
     },
     "user_tz": 420
    },
    "id": "oUrWaJB-mT4z",
    "outputId": "b79923ee-ac3c-401a-e651-3fad64245171"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hdf5 noise file has 233500 entries.\n"
     ]
    }
   ],
   "source": [
    "keys = test_data.keys()\n",
    "print('Hdf5 noise file has', len(keys), 'entries.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L-5y09TNussB"
   },
   "source": [
    "## Read noise data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "moSWsHlK-Lhe"
   },
   "outputs": [],
   "source": [
    "noise = h5py.File('./noise_data/chunk1.hdf5', 'r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7cAg8DRGwF4P"
   },
   "source": [
    "To observe the structure of the file could use: https://stackoverflow.com/questions/61133916/is-there-in-python-a-single-function-that-shows-the-full-structure-of-a-hdf5-fi\n",
    "\n",
    "However, takes too long for large files like these. The best way to check the structure is running the keys() function to see what references/data is available to access."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "T0s_h-Fl4FVl"
   },
   "outputs": [],
   "source": [
    "# Move down the hierarchy, into the folder 'data'. This is where the data will be.\n",
    "noise_data = noise.get('data')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fhupulH67NCL"
   },
   "source": [
    "Now load the csv file that contains information about the arrays in the hdf5 file. Both the csv file and the data folder in the hdf5 files has 235426 entries.\n",
    "\n",
    "For each row in the csv dataframe, trace_name holds the reference of the corresponding wavefrom stored in the hdf5 file. We should be able to take a partition of the csv file, and the retrieve the corresponding waveforms based on the trace_name column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 342
    },
    "executionInfo": {
     "elapsed": 1336,
     "status": "ok",
     "timestamp": 1714682441252,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "ApYN3i1BZ8Rk",
    "outputId": "c4e354f1-97ed-4737-9e72-d5e47f9af021"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "dataframe",
       "variable_name": "noise_csv"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-1bea2845-bf0f-41c9-a7d9-62fb23c09091\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>network_code</th>\n",
       "      <th>receiver_code</th>\n",
       "      <th>receiver_type</th>\n",
       "      <th>receiver_latitude</th>\n",
       "      <th>receiver_longitude</th>\n",
       "      <th>receiver_elevation_m</th>\n",
       "      <th>p_arrival_sample</th>\n",
       "      <th>p_status</th>\n",
       "      <th>p_weight</th>\n",
       "      <th>p_travel_sec</th>\n",
       "      <th>...</th>\n",
       "      <th>source_magnitude_author</th>\n",
       "      <th>source_mechanism_strike_dip_rake</th>\n",
       "      <th>source_distance_deg</th>\n",
       "      <th>source_distance_km</th>\n",
       "      <th>back_azimuth_deg</th>\n",
       "      <th>snr_db</th>\n",
       "      <th>coda_end_sample</th>\n",
       "      <th>trace_start_time</th>\n",
       "      <th>trace_category</th>\n",
       "      <th>trace_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AE</td>\n",
       "      <td>113A</td>\n",
       "      <td>HH</td>\n",
       "      <td>32.768299</td>\n",
       "      <td>-113.766701</td>\n",
       "      <td>118.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-01-15 00:17:30</td>\n",
       "      <td>noise</td>\n",
       "      <td>113A.AE_20180115001730_NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AE</td>\n",
       "      <td>113A</td>\n",
       "      <td>HH</td>\n",
       "      <td>32.768299</td>\n",
       "      <td>-113.766701</td>\n",
       "      <td>118.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-01-15 00:33:36</td>\n",
       "      <td>noise</td>\n",
       "      <td>113A.AE_20180115003336_NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AE</td>\n",
       "      <td>113A</td>\n",
       "      <td>HH</td>\n",
       "      <td>32.768299</td>\n",
       "      <td>-113.766701</td>\n",
       "      <td>118.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-01-15 02:01:06</td>\n",
       "      <td>noise</td>\n",
       "      <td>113A.AE_20180115020106_NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AE</td>\n",
       "      <td>113A</td>\n",
       "      <td>HH</td>\n",
       "      <td>32.768299</td>\n",
       "      <td>-113.766701</td>\n",
       "      <td>118.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-01-15 02:29:06</td>\n",
       "      <td>noise</td>\n",
       "      <td>113A.AE_20180115022906_NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AE</td>\n",
       "      <td>113A</td>\n",
       "      <td>HH</td>\n",
       "      <td>32.768299</td>\n",
       "      <td>-113.766701</td>\n",
       "      <td>118.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-01-15 03:51:00</td>\n",
       "      <td>noise</td>\n",
       "      <td>113A.AE_20180115035100_NO</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1bea2845-bf0f-41c9-a7d9-62fb23c09091')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-1bea2845-bf0f-41c9-a7d9-62fb23c09091 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-1bea2845-bf0f-41c9-a7d9-62fb23c09091');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-383e7fea-598a-4ecc-b974-a4d2751cfa58\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-383e7fea-598a-4ecc-b974-a4d2751cfa58')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-383e7fea-598a-4ecc-b974-a4d2751cfa58 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "  network_code receiver_code receiver_type  receiver_latitude  \\\n",
       "0           AE          113A            HH          32.768299   \n",
       "1           AE          113A            HH          32.768299   \n",
       "2           AE          113A            HH          32.768299   \n",
       "3           AE          113A            HH          32.768299   \n",
       "4           AE          113A            HH          32.768299   \n",
       "\n",
       "   receiver_longitude  receiver_elevation_m  p_arrival_sample  p_status  \\\n",
       "0         -113.766701                 118.0               NaN       NaN   \n",
       "1         -113.766701                 118.0               NaN       NaN   \n",
       "2         -113.766701                 118.0               NaN       NaN   \n",
       "3         -113.766701                 118.0               NaN       NaN   \n",
       "4         -113.766701                 118.0               NaN       NaN   \n",
       "\n",
       "   p_weight  p_travel_sec  ...  source_magnitude_author  \\\n",
       "0       NaN           NaN  ...                      NaN   \n",
       "1       NaN           NaN  ...                      NaN   \n",
       "2       NaN           NaN  ...                      NaN   \n",
       "3       NaN           NaN  ...                      NaN   \n",
       "4       NaN           NaN  ...                      NaN   \n",
       "\n",
       "   source_mechanism_strike_dip_rake  source_distance_deg  source_distance_km  \\\n",
       "0                               NaN                  NaN                 NaN   \n",
       "1                               NaN                  NaN                 NaN   \n",
       "2                               NaN                  NaN                 NaN   \n",
       "3                               NaN                  NaN                 NaN   \n",
       "4                               NaN                  NaN                 NaN   \n",
       "\n",
       "   back_azimuth_deg  snr_db  coda_end_sample     trace_start_time  \\\n",
       "0               NaN     NaN              NaN  2018-01-15 00:17:30   \n",
       "1               NaN     NaN              NaN  2018-01-15 00:33:36   \n",
       "2               NaN     NaN              NaN  2018-01-15 02:01:06   \n",
       "3               NaN     NaN              NaN  2018-01-15 02:29:06   \n",
       "4               NaN     NaN              NaN  2018-01-15 03:51:00   \n",
       "\n",
       "   trace_category                 trace_name  \n",
       "0           noise  113A.AE_20180115001730_NO  \n",
       "1           noise  113A.AE_20180115003336_NO  \n",
       "2           noise  113A.AE_20180115020106_NO  \n",
       "3           noise  113A.AE_20180115022906_NO  \n",
       "4           noise  113A.AE_20180115035100_NO  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "noise_csv = pd.read_csv('./noise_data/chunk1.csv')\n",
    "noise_csv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 128,
     "status": "ok",
     "timestamp": 1714682437798,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "U8lYUGTibnjZ",
    "outputId": "c35c83ad-6225-4c8b-c75b-73ffc0d13d3a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Noise file has 235426 entries.\n",
      "Hdf5 noise file has 235426 entries.\n"
     ]
    }
   ],
   "source": [
    "# Sanity check for noise file.\n",
    "print('Noise file has', len(noise_csv), 'entries.')\n",
    "\n",
    "keys = noise_data.keys()\n",
    "print('Hdf5 noise file has', len(keys), 'entries.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zntunIqh3Yp2"
   },
   "source": [
    "The noise file contained data from 1160 receivers, based on latitude, longitude and elevation combination. From checking some locations manually, it seems that the noise are from across the globe (Chie, New Zeland, Green Land, Antartica)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 204,
     "status": "ok",
     "timestamp": 1714680316917,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "1RzHPSSG3XK1",
    "outputId": "4c93263a-ece3-4555-950d-ef4062723b7e"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "summary": "{\n  \"name\": \"csv_file\",\n  \"rows\": 1160,\n  \"fields\": [\n    {\n      \"column\": \"receiver_latitude\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 28.18962431217451,\n        \"min\": -77.8492,\n        \"max\": 77.444702,\n        \"num_unique_values\": 1153,\n        \"samples\": [\n          40.25047,\n          35.326,\n          40.7911\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"receiver_longitude\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 88.11339984041986,\n        \"min\": -178.8566,\n        \"max\": 179.6277,\n        \"num_unique_values\": 1158,\n        \"samples\": [\n          13.5782,\n          -99.3939,\n          -70.5987\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"receiver_elevation_m\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 764.4405545152441,\n        \"min\": -413.0,\n        \"max\": 4580.0,\n        \"num_unique_values\": 819,\n        \"samples\": [\n          1227.0,\n          421.0,\n          2009.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"count\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 298,\n        \"min\": 1,\n        \"max\": 4141,\n        \"num_unique_values\": 409,\n        \"samples\": [\n          422,\n          74,\n          236\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
       "type": "dataframe"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-1fccb1f2-4bca-47d5-92d6-0474641ea397\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>receiver_latitude</th>\n",
       "      <th>receiver_longitude</th>\n",
       "      <th>receiver_elevation_m</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-77.849200</td>\n",
       "      <td>166.757200</td>\n",
       "      <td>50.0</td>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-77.551697</td>\n",
       "      <td>167.281998</td>\n",
       "      <td>2370.1</td>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-77.521896</td>\n",
       "      <td>167.150406</td>\n",
       "      <td>3590.1</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-75.106500</td>\n",
       "      <td>123.305000</td>\n",
       "      <td>3240.0</td>\n",
       "      <td>181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-66.664909</td>\n",
       "      <td>140.002075</td>\n",
       "      <td>40.0</td>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1155</th>\n",
       "      <td>71.538383</td>\n",
       "      <td>-53.199581</td>\n",
       "      <td>36.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1156</th>\n",
       "      <td>71.634100</td>\n",
       "      <td>128.866700</td>\n",
       "      <td>40.0</td>\n",
       "      <td>94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1157</th>\n",
       "      <td>74.307098</td>\n",
       "      <td>-20.219299</td>\n",
       "      <td>1.0</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1158</th>\n",
       "      <td>76.537399</td>\n",
       "      <td>-68.823700</td>\n",
       "      <td>38.0</td>\n",
       "      <td>214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1159</th>\n",
       "      <td>77.444702</td>\n",
       "      <td>-51.073799</td>\n",
       "      <td>2513.0</td>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1160 rows × 4 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1fccb1f2-4bca-47d5-92d6-0474641ea397')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-1fccb1f2-4bca-47d5-92d6-0474641ea397 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-1fccb1f2-4bca-47d5-92d6-0474641ea397');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-c5ee3582-723f-4b05-9789-9f6c9650333c\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c5ee3582-723f-4b05-9789-9f6c9650333c')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-c5ee3582-723f-4b05-9789-9f6c9650333c button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "      receiver_latitude  receiver_longitude  receiver_elevation_m  count\n",
       "0            -77.849200          166.757200                  50.0    171\n",
       "1            -77.551697          167.281998                2370.1    120\n",
       "2            -77.521896          167.150406                3590.1     15\n",
       "3            -75.106500          123.305000                3240.0    181\n",
       "4            -66.664909          140.002075                  40.0    120\n",
       "...                 ...                 ...                   ...    ...\n",
       "1155          71.538383          -53.199581                  36.0      7\n",
       "1156          71.634100          128.866700                  40.0     94\n",
       "1157          74.307098          -20.219299                   1.0     52\n",
       "1158          76.537399          -68.823700                  38.0    214\n",
       "1159          77.444702          -51.073799                2513.0    120\n",
       "\n",
       "[1160 rows x 4 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "noise_csv.groupby(['receiver_latitude', 'receiver_longitude', 'receiver_elevation_m']).size().reset_index().rename(columns={0:'count'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BKB8BkM__l7d"
   },
   "source": [
    "## Read Earthquake data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KEqLzmz2_pOE"
   },
   "outputs": [],
   "source": [
    "earthquake = h5py.File('./waveform_chunk2/chunk2.hdf5', 'r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "n3tLUY8XDtYs"
   },
   "outputs": [],
   "source": [
    "eq_data = earthquake.get('./data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4437,
     "status": "ok",
     "timestamp": 1714767524574,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "bX_XNa_QD_TQ",
    "outputId": "d6e2eaad-7fb8-4141-b5ab-e9e57c163977"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-9-e89c729d6222>:1: DtypeWarning: Columns (26) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  eq_csv = pd.read_csv('./waveform_chunk2/chunk2.csv')\n"
     ]
    }
   ],
   "source": [
    "eq_csv = pd.read_csv('./waveform_chunk2/chunk2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 411
    },
    "executionInfo": {
     "elapsed": 243,
     "status": "ok",
     "timestamp": 1714688717275,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "IBlMZ81kFLlM",
    "outputId": "053e35db-4428-46d9-ca83-a1ec00fb040f"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "dataframe",
       "variable_name": "eq_csv"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-d4449f0d-45e4-45d4-8801-cc4e39e9fb21\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>network_code</th>\n",
       "      <th>receiver_code</th>\n",
       "      <th>receiver_type</th>\n",
       "      <th>receiver_latitude</th>\n",
       "      <th>receiver_longitude</th>\n",
       "      <th>receiver_elevation_m</th>\n",
       "      <th>p_arrival_sample</th>\n",
       "      <th>p_status</th>\n",
       "      <th>p_weight</th>\n",
       "      <th>p_travel_sec</th>\n",
       "      <th>...</th>\n",
       "      <th>source_magnitude_author</th>\n",
       "      <th>source_mechanism_strike_dip_rake</th>\n",
       "      <th>source_distance_deg</th>\n",
       "      <th>source_distance_km</th>\n",
       "      <th>back_azimuth_deg</th>\n",
       "      <th>snr_db</th>\n",
       "      <th>coda_end_sample</th>\n",
       "      <th>trace_start_time</th>\n",
       "      <th>trace_category</th>\n",
       "      <th>trace_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TA</td>\n",
       "      <td>109C</td>\n",
       "      <td>BH</td>\n",
       "      <td>32.8889</td>\n",
       "      <td>-117.1051</td>\n",
       "      <td>150.0</td>\n",
       "      <td>700.0</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.5</td>\n",
       "      <td>17.080000</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.92</td>\n",
       "      <td>102.09</td>\n",
       "      <td>159.3</td>\n",
       "      <td>[56.79999924 55.40000153 47.40000153]</td>\n",
       "      <td>[[2896.]]</td>\n",
       "      <td>2006-07-23 15:59:00.960000</td>\n",
       "      <td>earthquake_local</td>\n",
       "      <td>109C.TA_20060723155859_EV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TA</td>\n",
       "      <td>109C</td>\n",
       "      <td>BH</td>\n",
       "      <td>32.8889</td>\n",
       "      <td>-117.1051</td>\n",
       "      <td>150.0</td>\n",
       "      <td>600.0</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.5</td>\n",
       "      <td>16.879999</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.91</td>\n",
       "      <td>101.34</td>\n",
       "      <td>281.7</td>\n",
       "      <td>[65.         65.5        61.40000153]</td>\n",
       "      <td>[[5508.]]</td>\n",
       "      <td>2006-11-03 15:56:53.610000</td>\n",
       "      <td>earthquake_local</td>\n",
       "      <td>109C.TA_20061103155652_EV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TA</td>\n",
       "      <td>109C</td>\n",
       "      <td>BH</td>\n",
       "      <td>32.8889</td>\n",
       "      <td>-117.1051</td>\n",
       "      <td>150.0</td>\n",
       "      <td>500.0</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.5</td>\n",
       "      <td>17.260000</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.92</td>\n",
       "      <td>101.87</td>\n",
       "      <td>280.5</td>\n",
       "      <td>[37.20000076 42.         38.59999847]</td>\n",
       "      <td>[[3114.]]</td>\n",
       "      <td>2006-11-03 16:12:24.700000</td>\n",
       "      <td>earthquake_local</td>\n",
       "      <td>109C.TA_20061103161223_EV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TA</td>\n",
       "      <td>109C</td>\n",
       "      <td>BH</td>\n",
       "      <td>32.8889</td>\n",
       "      <td>-117.1051</td>\n",
       "      <td>150.0</td>\n",
       "      <td>900.0</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.5</td>\n",
       "      <td>17.280001</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.93</td>\n",
       "      <td>103.26</td>\n",
       "      <td>281.6</td>\n",
       "      <td>[54.09999847 54.90000153 45.5       ]</td>\n",
       "      <td>[[3152.]]</td>\n",
       "      <td>2006-11-14 13:32:22.540000</td>\n",
       "      <td>earthquake_local</td>\n",
       "      <td>109C.TA_20061114133221_EV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TA</td>\n",
       "      <td>109C</td>\n",
       "      <td>BH</td>\n",
       "      <td>32.8889</td>\n",
       "      <td>-117.1051</td>\n",
       "      <td>150.0</td>\n",
       "      <td>700.0</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.5</td>\n",
       "      <td>18.139999</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.92</td>\n",
       "      <td>102.48</td>\n",
       "      <td>4.7</td>\n",
       "      <td>[58.20000076 56.20000076 53.79999924]</td>\n",
       "      <td>[[3134.]]</td>\n",
       "      <td>2006-11-27 10:46:41.060000</td>\n",
       "      <td>earthquake_local</td>\n",
       "      <td>109C.TA_20061127104640_EV</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d4449f0d-45e4-45d4-8801-cc4e39e9fb21')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-d4449f0d-45e4-45d4-8801-cc4e39e9fb21 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-d4449f0d-45e4-45d4-8801-cc4e39e9fb21');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-9a403414-f03d-401e-b485-1cf7fb0a4f5e\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9a403414-f03d-401e-b485-1cf7fb0a4f5e')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-9a403414-f03d-401e-b485-1cf7fb0a4f5e button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "  network_code receiver_code receiver_type  receiver_latitude  \\\n",
       "0           TA          109C            BH            32.8889   \n",
       "1           TA          109C            BH            32.8889   \n",
       "2           TA          109C            BH            32.8889   \n",
       "3           TA          109C            BH            32.8889   \n",
       "4           TA          109C            BH            32.8889   \n",
       "\n",
       "   receiver_longitude  receiver_elevation_m  p_arrival_sample p_status  \\\n",
       "0           -117.1051                 150.0             700.0   manual   \n",
       "1           -117.1051                 150.0             600.0   manual   \n",
       "2           -117.1051                 150.0             500.0   manual   \n",
       "3           -117.1051                 150.0             900.0   manual   \n",
       "4           -117.1051                 150.0             700.0   manual   \n",
       "\n",
       "   p_weight  p_travel_sec  ...  source_magnitude_author  \\\n",
       "0       0.5     17.080000  ...                      NaN   \n",
       "1       0.5     16.879999  ...                      NaN   \n",
       "2       0.5     17.260000  ...                      NaN   \n",
       "3       0.5     17.280001  ...                      NaN   \n",
       "4       0.5     18.139999  ...                      NaN   \n",
       "\n",
       "  source_mechanism_strike_dip_rake  source_distance_deg source_distance_km  \\\n",
       "0                              NaN                 0.92             102.09   \n",
       "1                              NaN                 0.91             101.34   \n",
       "2                              NaN                 0.92             101.87   \n",
       "3                              NaN                 0.93             103.26   \n",
       "4                              NaN                 0.92             102.48   \n",
       "\n",
       "  back_azimuth_deg                                 snr_db  coda_end_sample  \\\n",
       "0            159.3  [56.79999924 55.40000153 47.40000153]        [[2896.]]   \n",
       "1            281.7  [65.         65.5        61.40000153]        [[5508.]]   \n",
       "2            280.5  [37.20000076 42.         38.59999847]        [[3114.]]   \n",
       "3            281.6  [54.09999847 54.90000153 45.5       ]        [[3152.]]   \n",
       "4              4.7  [58.20000076 56.20000076 53.79999924]        [[3134.]]   \n",
       "\n",
       "             trace_start_time    trace_category                 trace_name  \n",
       "0  2006-07-23 15:59:00.960000  earthquake_local  109C.TA_20060723155859_EV  \n",
       "1  2006-11-03 15:56:53.610000  earthquake_local  109C.TA_20061103155652_EV  \n",
       "2  2006-11-03 16:12:24.700000  earthquake_local  109C.TA_20061103161223_EV  \n",
       "3  2006-11-14 13:32:22.540000  earthquake_local  109C.TA_20061114133221_EV  \n",
       "4  2006-11-27 10:46:41.060000  earthquake_local  109C.TA_20061127104640_EV  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eq_csv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "IndB_uniFVsd"
   },
   "outputs": [],
   "source": [
    "# Sanity check for earthquake file.\n",
    "print('Noise file has', len(eq_csv), 'entries.')\n",
    "\n",
    "eq_keys = eq_data.keys()\n",
    "print('Hdf5 earthquake file has', len(eq_keys), 'entries.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gb9ghCF4Ghr0"
   },
   "source": [
    "Attempt to parititon the hdf5 file and extract the corresponding entries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 138
    },
    "executionInfo": {
     "elapsed": 5,
     "status": "ok",
     "timestamp": 1714684393958,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "CRpYMXc6IX2C",
    "outputId": "74368d0f-42af-4f71-da44-2677b687b7fc"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-width:800px; border: 1px solid var(--colab-border-color);\"><style>\n",
       "      pre.function-repr-contents {\n",
       "        overflow-x: auto;\n",
       "        padding: 8px 12px;\n",
       "        max-height: 500px;\n",
       "      }\n",
       "\n",
       "      pre.function-repr-contents.function-repr-contents-collapsed {\n",
       "        cursor: pointer;\n",
       "        max-height: 100px;\n",
       "      }\n",
       "    </style>\n",
       "    <pre style=\"white-space: initial; background:\n",
       "         var(--colab-secondary-surface-color); padding: 8px 12px;\n",
       "         border-bottom: 1px solid var(--colab-border-color);\"><b>h5py._hl.group.Group</b><br/>def __init__(bind)</pre><pre class=\"function-repr-contents function-repr-contents-collapsed\" style=\"\"><a class=\"filepath\" style=\"display:none\" href=\"#\">/usr/local/lib/python3.10/dist-packages/h5py/_hl/group.py</a>Represents an HDF5 group.\n",
       "    </pre>\n",
       "      <script>\n",
       "      if (google.colab.kernel.accessAllowed && google.colab.files && google.colab.files.view) {\n",
       "        for (const element of document.querySelectorAll('.filepath')) {\n",
       "          element.style.display = 'block'\n",
       "          element.onclick = (event) => {\n",
       "            event.preventDefault();\n",
       "            event.stopPropagation();\n",
       "            google.colab.files.view(element.textContent, 29);\n",
       "          };\n",
       "        }\n",
       "      }\n",
       "      for (const element of document.querySelectorAll('.function-repr-contents')) {\n",
       "        element.onclick = (event) => {\n",
       "          event.preventDefault();\n",
       "          event.stopPropagation();\n",
       "          element.classList.toggle('function-repr-contents-collapsed');\n",
       "        };\n",
       "      }\n",
       "      </script>\n",
       "      </div>"
      ],
      "text/plain": [
       "h5py._hl.group.Group"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(eq_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1714684432886,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "h09T83h2KAST",
    "outputId": "8538143c-4598-41e6-a146-e96fbdc68433"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ItemsViewHDF5(<HDF5 group \"/data\" (200000 members)>)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eq_data.items()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 155,
     "status": "ok",
     "timestamp": 1714684057947,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "9-1KRNMgHIiE",
    "outputId": "e0fe2367-6b4c-45bb-d3e4-cd4f651d31c0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    109C.TA_20060723155859_EV\n",
       "1    109C.TA_20061103155652_EV\n",
       "Name: trace_name, dtype: object"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eq_csv.iloc[[0,1]]['trace_name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qLpNhbnuHSld"
   },
   "outputs": [],
   "source": [
    "test = eq_data.get(eq_csv.iloc[0]['trace_name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 6,
     "status": "ok",
     "timestamp": 1714686284079,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "LC27mrhwHcKM",
    "outputId": "ad43add6-84f2-416e-af8d-ed1d5e524a4d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.        , -0.        ,  0.        ],\n",
       "       [-0.0085106 , -0.03259867,  0.00775105],\n",
       "       [-0.02285321, -0.07897092,  0.02457174],\n",
       "       ...,\n",
       "       [ 0.10581018, -0.12264204, -0.062743  ],\n",
       "       [ 0.00339233, -0.05204263, -0.01183952],\n",
       "       [-0.        , -0.        , -0.        ]], dtype=float32)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test[0:len(test)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XqqGpD7sKnII"
   },
   "source": [
    "So far it seems that the only way to partition the data is to iterate over the rows and get each array manually with the get() fn.\n",
    "\n",
    "Steps for making a smaller hdf5 file.\n",
    "1. First create a new hdf5 file.\n",
    "2. Then create the group 'data' using the create_group fn.\n",
    "3. Partition the csv file so we have a managable number of entries.\n",
    "4. For each trace_name in the subset, create a dataset using the create_dataset fn. Examples under dataset documentation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aQiwRLh3yTqj"
   },
   "outputs": [],
   "source": [
    "np.random.seed(123)\n",
    "rand_samp = np.random.choice(len(eq_csv), 10, replace=False)\n",
    "rand_samp = np.sort(rand_samp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_m3COJGkM1rZ"
   },
   "outputs": [],
   "source": [
    "subset_names = eq_csv.iloc[rand_samp]['trace_name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "s9lgcQ8yNG6p"
   },
   "outputs": [],
   "source": [
    "with h5py.File('small_subset.hdf5', \"w\") as f:\n",
    "  grp = f.create_group('data')\n",
    "\n",
    "  for name in subset_names:\n",
    "    data = eq_data.get(name)\n",
    "    value = data[0:]\n",
    "    grp.create_dataset(name = name, data = data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BZiUikoczYbN"
   },
   "source": [
    "Write code to crete the subset csv file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 235
    },
    "executionInfo": {
     "elapsed": 16,
     "status": "ok",
     "timestamp": 1714767572487,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "yQj6O8i10fZ9",
    "outputId": "87cd3611-0cc3-4d58-877e-6d7969d2a841"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "dataframe",
       "variable_name": "subset_csv"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-82bafbcf-6b9a-4d27-8ffa-a77972c6f66c\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>network_code</th>\n",
       "      <th>receiver_code</th>\n",
       "      <th>receiver_type</th>\n",
       "      <th>receiver_latitude</th>\n",
       "      <th>receiver_longitude</th>\n",
       "      <th>receiver_elevation_m</th>\n",
       "      <th>p_arrival_sample</th>\n",
       "      <th>p_status</th>\n",
       "      <th>p_weight</th>\n",
       "      <th>p_travel_sec</th>\n",
       "      <th>...</th>\n",
       "      <th>source_magnitude_author</th>\n",
       "      <th>source_mechanism_strike_dip_rake</th>\n",
       "      <th>source_distance_deg</th>\n",
       "      <th>source_distance_km</th>\n",
       "      <th>back_azimuth_deg</th>\n",
       "      <th>snr_db</th>\n",
       "      <th>coda_end_sample</th>\n",
       "      <th>trace_start_time</th>\n",
       "      <th>trace_category</th>\n",
       "      <th>trace_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21386</th>\n",
       "      <td>HV</td>\n",
       "      <td>ALEP</td>\n",
       "      <td>EH</td>\n",
       "      <td>19.541178</td>\n",
       "      <td>-155.644036</td>\n",
       "      <td>2894.0</td>\n",
       "      <td>800.0</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.62</td>\n",
       "      <td>5.350000</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.2193</td>\n",
       "      <td>24.40</td>\n",
       "      <td>5.7</td>\n",
       "      <td>[25.89999962 27.20000076 25.79999924]</td>\n",
       "      <td>[[3199.]]</td>\n",
       "      <td>2017-01-06 01:28:47.960000</td>\n",
       "      <td>earthquake_local</td>\n",
       "      <td>ALEP.HV_20170106012846_EV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33832</th>\n",
       "      <td>AK</td>\n",
       "      <td>ATKA</td>\n",
       "      <td>BH</td>\n",
       "      <td>52.201600</td>\n",
       "      <td>-174.197500</td>\n",
       "      <td>55.0</td>\n",
       "      <td>500.0</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.50</td>\n",
       "      <td>16.629999</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.8700</td>\n",
       "      <td>96.22</td>\n",
       "      <td>0.3</td>\n",
       "      <td>[72.         67.30000305 54.29999924]</td>\n",
       "      <td>[[5864.]]</td>\n",
       "      <td>2013-09-07 13:12:31.220000</td>\n",
       "      <td>earthquake_local</td>\n",
       "      <td>ATKA.AK_20130907131230_EV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78859</th>\n",
       "      <td>PB</td>\n",
       "      <td>B081</td>\n",
       "      <td>EH</td>\n",
       "      <td>33.711167</td>\n",
       "      <td>-116.714167</td>\n",
       "      <td>1467.0</td>\n",
       "      <td>900.0</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.58</td>\n",
       "      <td>5.120000</td>\n",
       "      <td>...</td>\n",
       "      <td>CI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.2666</td>\n",
       "      <td>29.65</td>\n",
       "      <td>139.0</td>\n",
       "      <td>[23.70000076 30.39999962 25.29999924]</td>\n",
       "      <td>[[1833.]]</td>\n",
       "      <td>2012-07-19 13:15:16.520000</td>\n",
       "      <td>earthquake_local</td>\n",
       "      <td>B081.PB_20120719131515_EV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99135</th>\n",
       "      <td>PB</td>\n",
       "      <td>B082</td>\n",
       "      <td>EH</td>\n",
       "      <td>33.598182</td>\n",
       "      <td>-116.596005</td>\n",
       "      <td>1374.8</td>\n",
       "      <td>798.0</td>\n",
       "      <td>autopicker</td>\n",
       "      <td>0.92</td>\n",
       "      <td>5.670000</td>\n",
       "      <td>...</td>\n",
       "      <td>CI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.1845</td>\n",
       "      <td>31.85</td>\n",
       "      <td>265.4</td>\n",
       "      <td>[17.5        16.39999962 13.60000038]</td>\n",
       "      <td>[[1883.]]</td>\n",
       "      <td>2008-08-16 02:04:08.300000</td>\n",
       "      <td>earthquake_local</td>\n",
       "      <td>B082.PB_20080816020407_EV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112674</th>\n",
       "      <td>PB</td>\n",
       "      <td>B082</td>\n",
       "      <td>HH</td>\n",
       "      <td>33.598182</td>\n",
       "      <td>-116.596005</td>\n",
       "      <td>1374.8</td>\n",
       "      <td>700.0</td>\n",
       "      <td>manual</td>\n",
       "      <td>0.63</td>\n",
       "      <td>3.250000</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.1426</td>\n",
       "      <td>15.86</td>\n",
       "      <td>291.3</td>\n",
       "      <td>[15.10000038 14.30000019 11.30000019]</td>\n",
       "      <td>[[1410.]]</td>\n",
       "      <td>2013-06-08 10:45:41.440000</td>\n",
       "      <td>earthquake_local</td>\n",
       "      <td>B082.PB_20130608104540_EV</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-82bafbcf-6b9a-4d27-8ffa-a77972c6f66c')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-82bafbcf-6b9a-4d27-8ffa-a77972c6f66c button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-82bafbcf-6b9a-4d27-8ffa-a77972c6f66c');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-267fdf26-be9e-4d20-9dba-c7f4b4c18dfe\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-267fdf26-be9e-4d20-9dba-c7f4b4c18dfe')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-267fdf26-be9e-4d20-9dba-c7f4b4c18dfe button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "       network_code receiver_code receiver_type  receiver_latitude  \\\n",
       "21386            HV          ALEP            EH          19.541178   \n",
       "33832            AK          ATKA            BH          52.201600   \n",
       "78859            PB          B081            EH          33.711167   \n",
       "99135            PB          B082            EH          33.598182   \n",
       "112674           PB          B082            HH          33.598182   \n",
       "\n",
       "        receiver_longitude  receiver_elevation_m  p_arrival_sample  \\\n",
       "21386          -155.644036                2894.0             800.0   \n",
       "33832          -174.197500                  55.0             500.0   \n",
       "78859          -116.714167                1467.0             900.0   \n",
       "99135          -116.596005                1374.8             798.0   \n",
       "112674         -116.596005                1374.8             700.0   \n",
       "\n",
       "          p_status  p_weight  p_travel_sec  ...  source_magnitude_author  \\\n",
       "21386       manual      0.62      5.350000  ...                      NaN   \n",
       "33832       manual      0.50     16.629999  ...                      NaN   \n",
       "78859       manual      0.58      5.120000  ...                       CI   \n",
       "99135   autopicker      0.92      5.670000  ...                       CI   \n",
       "112674      manual      0.63      3.250000  ...                      NaN   \n",
       "\n",
       "       source_mechanism_strike_dip_rake  source_distance_deg  \\\n",
       "21386                               NaN               0.2193   \n",
       "33832                               NaN               0.8700   \n",
       "78859                               NaN               0.2666   \n",
       "99135                               NaN               0.1845   \n",
       "112674                              NaN               0.1426   \n",
       "\n",
       "       source_distance_km back_azimuth_deg  \\\n",
       "21386               24.40              5.7   \n",
       "33832               96.22              0.3   \n",
       "78859               29.65            139.0   \n",
       "99135               31.85            265.4   \n",
       "112674              15.86            291.3   \n",
       "\n",
       "                                       snr_db  coda_end_sample  \\\n",
       "21386   [25.89999962 27.20000076 25.79999924]        [[3199.]]   \n",
       "33832   [72.         67.30000305 54.29999924]        [[5864.]]   \n",
       "78859   [23.70000076 30.39999962 25.29999924]        [[1833.]]   \n",
       "99135   [17.5        16.39999962 13.60000038]        [[1883.]]   \n",
       "112674  [15.10000038 14.30000019 11.30000019]        [[1410.]]   \n",
       "\n",
       "                  trace_start_time    trace_category  \\\n",
       "21386   2017-01-06 01:28:47.960000  earthquake_local   \n",
       "33832   2013-09-07 13:12:31.220000  earthquake_local   \n",
       "78859   2012-07-19 13:15:16.520000  earthquake_local   \n",
       "99135   2008-08-16 02:04:08.300000  earthquake_local   \n",
       "112674  2013-06-08 10:45:41.440000  earthquake_local   \n",
       "\n",
       "                       trace_name  \n",
       "21386   ALEP.HV_20170106012846_EV  \n",
       "33832   ATKA.AK_20130907131230_EV  \n",
       "78859   B081.PB_20120719131515_EV  \n",
       "99135   B082.PB_20080816020407_EV  \n",
       "112674  B082.PB_20130608104540_EV  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subset_csv = eq_csv.iloc[rand_samp]\n",
    "subset_csv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mXGDqKCW0oFt"
   },
   "outputs": [],
   "source": [
    "subset_csv.to_csv('small_subset.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2AM0_CCobEAT"
   },
   "source": [
    "Checking the code above worked correctly. Seems like it is!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ovGVRcHzRlKB"
   },
   "outputs": [],
   "source": [
    "test_file = h5py.File('./subset.hdf5', 'r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "a1v1RSK1a0F1"
   },
   "outputs": [],
   "source": [
    "test_grp = test_file.get('data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 131,
     "status": "ok",
     "timestamp": 1714688882397,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "FEi4Sax-a4h3",
    "outputId": "2351e8c6-74f3-4b1d-fd10-31901f42fef7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<KeysViewHDF5 ['109C.TA_20060723155859_EV', '109C.TA_20061103155652_EV', '109C.TA_20061103161223_EV', '109C.TA_20061114133221_EV', '109C.TA_20061127104640_EV', '109C.TA_20061129121745_EV', '109C.TA_20061129211102_EV', '109C.TA_20061129211301_EV', '109C.TA_20061129221547_EV', '109C.TA_20070109140205_EV']>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_grp.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 201,
     "status": "ok",
     "timestamp": 1714688894376,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "FuhTkrqBa-A_",
    "outputId": "22c243eb-b46e-4b73-c95e-be5348bb1e74"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.        , -0.        ,  0.        ],\n",
       "       [-0.0085106 , -0.03259867,  0.00775105],\n",
       "       [-0.02285321, -0.07897092,  0.02457174],\n",
       "       ...,\n",
       "       [ 0.10581018, -0.12264204, -0.062743  ],\n",
       "       [ 0.00339233, -0.05204263, -0.01183952],\n",
       "       [-0.        , -0.        , -0.        ]], dtype=float32)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_grp.get('109C.TA_20060723155859_EV')[0:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "luFRCIOybauH"
   },
   "source": [
    "What's the best way to partition the data? Probabaly shouldn't take a straight up subset because that would not be random."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8J86lYSSDOZ2"
   },
   "source": [
    "## Convert data in hdf5 file to npz file\n",
    "\n",
    "Since the training function for PhaseNet only takes in npz files, we need to convert the data in the hdf5 files to the npz formats. Unsure if we would need it for other models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qXYAywP14LIt"
   },
   "outputs": [],
   "source": [
    "bg_acr_data = hf_data.get('BG.ACR..DP.0215954.npz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "yUES1JqF5AJK"
   },
   "outputs": [],
   "source": [
    "# We can get the data as a numpy array this way.\n",
    "# Then can use np.savez to save the arrays into npz format\n",
    "temp = bg_acr_data[0:len(bg_acr_data)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "n27uWgMp6UGC"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tempfile import TemporaryFile\n",
    "outfile = TemporaryFile()\n",
    "np.savez(outfile, temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-xh_rM0S6sCz"
   },
   "outputs": [],
   "source": [
    "_ = outfile.seek(0)\n",
    "npzfile = np.load(outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1714613411795,
     "user": {
      "displayName": "Akari Tajima",
      "userId": "14232977932295828397"
     },
     "user_tz": 420
    },
    "id": "v_Wq6UTJ7AfR",
    "outputId": "c4baebba-8759-4963-ae9e-e40862c2d04a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0.],\n",
       "       [0., 0., 0.],\n",
       "       [0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0.],\n",
       "       [0., 0., 0.],\n",
       "       [0., 0., 0.]], dtype=float32)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "npzfile['arr_0']"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "machine_shape": "hm",
   "provenance": [
    {
     "file_id": "1Djsol5l7fSeqouj9F6Mig6Y0Wi8Hq1AM",
     "timestamp": 1714507426142
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}