Step1_Preliminary_Data_Processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d39daf26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wed Jan  4 17:33:32 2023\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import os \n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.mlab as mlab\n",
    "import glob\n",
    "import time\n",
    "import pandas as pd\n",
    "import fnmatch\n",
    "\n",
    "print (time.ctime())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "164d671a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['fakedata_exp.csv']\n",
      "1\n",
      "['fakedata_exp.csv']\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "#file info\n",
    "f_path=\"Data_fake_exp\" # folder\n",
    "identifier=\"fake\" # identifier that all data we are interested in looking at has\n",
    "\n",
    "l=os.listdir(f_path) # list of all files in a folder \n",
    "l=[ x for x in l if \".csv\" in x ]\n",
    "print (l) \n",
    "print (len(l))\n",
    "\n",
    "\n",
    "i=1\n",
    "for file in os.listdir(f_path):\n",
    "    keyword1='300sec'\n",
    "    keyword2='600sec'\n",
    "    keyword3='1200sec'\n",
    "    keyword4='1800sec'\n",
    "    keyword5='2400sec'\n",
    "    if keyword1 in file:\n",
    "        print ('okay')\n",
    "        b = f'{identifier}' + f'_{keyword1}_' + f'{i}' \n",
    "        oldname = f'{f_path}/'f'{file}'\n",
    "        newName = f'{f_path}/'f'{b}.csv'\n",
    "        os.rename(oldname, newName)\n",
    "    if keyword2 in file:\n",
    "        print ('okay')\n",
    "        b = f'{identifier}' + f'_{keyword2}_' + f'{i}' \n",
    "        oldname = f'{f_path}/'f'{file}'\n",
    "        newName = f'{f_path}/'f'{b}.csv'\n",
    "        os.rename(oldname, newName)\n",
    "    if keyword3 in file:\n",
    "        print ('okay')\n",
    "        b = f'{identifier}' + f'_{keyword3}_' + f'{i}' \n",
    "        oldname = f'{f_path}/'f'{file}'\n",
    "        newName = f'{f_path}/'f'{b}.csv'\n",
    "        os.rename(oldname, newName)\n",
    "    if keyword4 in file:\n",
    "        print ('okay')\n",
    "        b = f'{identifier}' + f'_{keyword4}_' + f'{i}' \n",
    "        oldname = f'{f_path}/'f'{file}'\n",
    "        newName = f'{f_path}/'f'{b}.csv'\n",
    "        os.rename(oldname, newName)\n",
    "    if keyword5 in file:\n",
    "        print ('okay')\n",
    "        b = f'{identifier}' + f'_{keyword5}_' + f'{i}' \n",
    "        oldname = f'{f_path}/'f'{file}'\n",
    "        newName = f'{f_path}/'f'{b}.csv'\n",
    "        os.rename(oldname, newName)\n",
    "    i=i+1\n",
    "        \n",
    "l=os.listdir(f_path) # list of all files in a folder \n",
    "l=[ x for x in l if \".csv\" in x ]\n",
    "print (l) \n",
    "print (len(l))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f6c3cd24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fakedata_exp.csv\n",
      "fakedata_exp\n",
      "         y   x            time  center flux  cluster size\n",
      "633225  92  61               0          206             1\n",
      "739505  93  66        30000000          448             4\n",
      "739506  92  68        80000000         1394             3\n",
      "208326  95  44       290000000         2446            13\n",
      "208327  93  35       310000000         1085            15\n",
      "...     ..  ..             ...          ...           ...\n",
      "314497  93  34  99799300000000         1856            11\n",
      "101052  93  24  99799370000000         1731             9\n",
      "314498  93  33  99799540000000         1719            15\n",
      "739504  95  59  99799650000000         1314             2\n",
      "421339  95  42  99799650000000         1264            15\n",
      "\n",
      "[840797 rows x 5 columns]\n",
      "         y   x      time  center flux  cluster size\n",
      "633225  92  61         0          206             1\n",
      "739505  93  66  30000000          448             4\n",
      "739506  92  68  80000000         1394             3\n"
     ]
    }
   ],
   "source": [
    "#for i in range(len(l)):\n",
    "\n",
    "for count, nam in enumerate(l):\n",
    "    if nam.startswith(identifier)==True:\n",
    "        print (nam) # printing count\n",
    "        filename = str(nam[:-4])\n",
    "        print (filename)#printing file name  \n",
    "        it = pd.read_csv(f'{f_path}/{filename}.csv')\n",
    "        it = it.drop(columns = 'Unnamed: 5')\n",
    "        it = it.rename(\n",
    "            columns = {'#Col': 'y', '#Row': 'x', '#ToA': 'time', '#ToT[arb]': 'center flux', '#Centroid': 'cluster size'}\n",
    "        )  # renames columns\n",
    "        it = it.sort_values(['time'])\n",
    "        #ToA = it['time']\n",
    "        #print (ToA[0])\n",
    "        #print (it['time'][0])\n",
    "        it['time'] = it['time'] - min(it['time'])\n",
    "        it = it.sort_values(['time'])\n",
    "        print (it)\n",
    "        print (it.head(3))\n",
    "        it.to_csv(f'{f_path}/'f'{filename}') ##### changes actual file\n",
    "        #it.head(3) \n",
    "        # all of this is just renaming the file titles into something comprehensible\n",
    "        \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cffd354d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b54835d8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}