edit_h5_seglabel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import h5py\n",
    "from random import *\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "y가 클수록 바닥쪽\n",
    "\n",
    "label 1:  \n",
    "{ 239.014 -239.014  \n",
    "             108.732 -108.358  \n",
    "             239.014 -239.014}  \n",
    "label 2:  \n",
    "{ 213.981 -213.981  \n",
    "             53.668 -53.680  \n",
    "             213.981 -213.981}    \n",
    "label 3:  \n",
    "{ 208.976 -208.976   \n",
    "             120.316 -120.309   \n",
    "             208.976 -208.976}   \n",
    "label 4:  \n",
    "{ 219.977 -219.977   \n",
    "             64.908 -64.943   \n",
    "             219.977 -219.977}    \n",
    "label 5:  \n",
    "{960.863 -960.863  \n",
    "            194.373 -194.371   \n",
    "             960.863 -960.863}    \n",
    "\n",
    "**centralized data & continuous segmentation label ( max, mean, min )**\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "flist=os.listdir(\"./data/npy_10000/list\")\n",
    "max_shards=20\n",
    "ymax, ymin=108.732,-108.358\n",
    "for file in flist:\n",
    "    if 'filelist_1' in file: #file = filelist_1_shards_randomseed_F\n",
    "        fstr=file.split('_') #{filelist, 1, shards, randomseed, F}\n",
    "        filenames=[line.rstrip() for line in open(\"./data/npy_10000/list/\"+file, 'r')] # filenames = generated_shards_0_randomseed_part_partnum\n",
    "        a_data=np.zeros((len(filenames),2048,3))\n",
    "        a_label=np.zeros((len(filenames),1),dtype=np.uint8)\n",
    "        a_seg=np.zeros((len(filenames),3), dtype=np.float64)\n",
    "        temp_list=[]\n",
    "        temp_label=[]\n",
    "        temp_seg=[]        \n",
    "        for i in range(0,len(filenames)):\n",
    "            if filenames[i]==\"\":\n",
    "                continue\n",
    "            npydata=np.load(\"./data/npy_10000/one/\"+filenames[i]+\".npy\")\n",
    "            np.random.shuffle(npydata)\n",
    "            if npydata.shape[0]<2048:\n",
    "                continue\n",
    "            else:\n",
    "                #x,y,z 2048 samping\n",
    "                temp_j_list=[]\n",
    "                for j in range(0,2048):\n",
    "                    temp_j_list.append([npydata[j][0],npydata[j][1], npydata[j][2]])\n",
    "                # seg_label ∈ [0,1]\n",
    "                temp_j_seg=[0,0,0]\n",
    "                temp_j_seg[0]=(np.amax(npydata[:,1])-ymin)/(ymax-ymin)\n",
    "                temp_j_seg[1]=(np.mean(npydata[:,1])-ymin)/(ymax-ymin)\n",
    "                temp_j_seg[2]=(np.amin(npydata[:,1])-ymin)/(ymax-ymin)\n",
    "                    \n",
    "                temp_list.append(temp_j_list)\n",
    "                temp_label.append(int(fstr[1])-1)\n",
    "                temp_seg.append(temp_j_seg)\n",
    "                \n",
    "        if len(temp_list) < max_shards:\n",
    "            for i in range(max_shards-len(temp_list)):\n",
    "                temp_shuffle_list=temp_list\n",
    "                temp_shuffle_seg=temp_seg\n",
    "                rvalue=randrange(len(temp_shuffle_list))\n",
    "                temp_list.append(temp_shuffle_list[rvalue])\n",
    "                temp_label.append(int(fstr[1])-1)\n",
    "                temp_seg.append(temp_shuffle_seg[rvalue])\n",
    "\n",
    "            \n",
    "        a_data=np.array(temp_list)\n",
    "        a_label=np.transpose(np.array(temp_label))\n",
    "        a_seg=np.array(temp_seg)\n",
    "        \n",
    "        target=np.zeros(a_data.shape)\n",
    "        for i in range(a_data.shape[0]):\n",
    "            for j in range(a_data.shape[1]):\n",
    "                for k in range(a_data.shape[2]):\n",
    "                    target[i][j][k]=a_data[i][j][k]-a_data[i].mean(axis=0)[k]\n",
    "\n",
    "        f=h5py.File(\"./data/floatlabel/\"+str(fstr[1])+'_'+str(fstr[2])+'_'+str(fstr[3])+\".h5\",'w')\n",
    "        data=f.create_dataset(\"data\", data=target)\n",
    "        label=f.create_dataset(\"label\", data=a_label)\n",
    "        seg=f.create_dataset(\"seg\", data=a_seg)\n",
    "        f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "flist=os.listdir(\"./data/npy_10000/list\")\n",
    "max_shards=20\n",
    "ymax, ymin=53.668, -53.680\n",
    "for file in flist:\n",
    "    if 'filelist_2' in file: #file = filelist_2_shards_randomseed_F\n",
    "        fstr=file.split('_') #{filelist, 2, shards, randomseed, F}\n",
    "        filenames=[line.rstrip() for line in open(\"./data/npy_10000/list/\"+file, 'r')] # filenames = generated_shards_0_randomseed_part_partnum\n",
    "        a_data=np.zeros((len(filenames),2048,3))\n",
    "        a_label=np.zeros((len(filenames),1),dtype=np.uint8)\n",
    "        a_seg=np.zeros((len(filenames),1), dtype=np.float64)\n",
    "        temp_list=[]\n",
    "        temp_label=[]\n",
    "        temp_seg=[]        \n",
    "        for i in range(0,len(filenames)):\n",
    "            if filenames[i]==\"\":\n",
    "                continue\n",
    "            npydata=np.load(\"./data/npy_10000/two/\"+filenames[i]+\".npy\")\n",
    "            np.random.shuffle(npydata)\n",
    "            if npydata.shape[0]<2048:\n",
    "                continue\n",
    "            else:\n",
    "                #x,y,z 2048 samping\n",
    "                temp_j_list=[]\n",
    "                for j in range(0,2048):\n",
    "                    temp_j_list.append([npydata[j][0],npydata[j][1], npydata[j][2]])\n",
    "                # seg_label ∈ [0,1]\n",
    "                temp_j_seg=[0,0,0]\n",
    "                temp_j_seg[0]=(np.amax(npydata[:,1])-ymin)/(ymax-ymin)\n",
    "                temp_j_seg[1]=(np.mean(npydata[:,1])-ymin)/(ymax-ymin)\n",
    "                temp_j_seg[2]=(np.amin(npydata[:,1])-ymin)/(ymax-ymin)       \n",
    "                \n",
    "                temp_list.append(temp_j_list)\n",
    "                temp_label.append(int(fstr[1])-1)\n",
    "                temp_seg.append(temp_j_seg)\n",
    "                \n",
    "        if len(temp_list) < max_shards:\n",
    "            for i in range(max_shards-len(temp_list)):\n",
    "                temp_shuffle_list=temp_list\n",
    "                temp_shuffle_seg=temp_seg\n",
    "                rvalue=randrange(len(temp_shuffle_list))\n",
    "                temp_list.append(temp_shuffle_list[rvalue])\n",
    "                temp_label.append(int(fstr[1])-1)\n",
    "                temp_seg.append(temp_shuffle_seg[rvalue])\n",
    "\n",
    "            \n",
    "        a_data=np.array(temp_list)\n",
    "        a_label=np.transpose(np.array(temp_label))\n",
    "        a_seg=np.array(temp_seg)\n",
    "        \n",
    "        target=np.zeros(a_data.shape)\n",
    "        for i in range(a_data.shape[0]):\n",
    "            for j in range(a_data.shape[1]):\n",
    "                for k in range(a_data.shape[2]):\n",
    "                    target[i][j][k]=a_data[i][j][k]-a_data[i].mean(axis=0)[k]\n",
    "\n",
    "        f=h5py.File(\"./data/floatlabel/\"+str(fstr[1])+'_'+str(fstr[2])+'_'+str(fstr[3])+\".h5\",'w')\n",
    "        data=f.create_dataset(\"data\", data=target)\n",
    "        label=f.create_dataset(\"label\", data=a_label)\n",
    "        seg=f.create_dataset(\"seg\", data=a_seg)\n",
    "        f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "flist=os.listdir(\"./data/npy_10000/list\")\n",
    "max_shards=20\n",
    "ymax, ymin=120.316, -120.309\n",
    "for file in flist:\n",
    "    if 'filelist_3' in file: #file = filelist_3_shards_randomseed_F\n",
    "        fstr=file.split('_') #{filelist, 3, shards, randomseed, F}\n",
    "        filenames=[line.rstrip() for line in open(\"./data/npy_10000/list/\"+file, 'r')] # filenames = generated_shards_0_randomseed_part_partnum\n",
    "        a_data=np.zeros((len(filenames),2048,3))\n",
    "        a_label=np.zeros((len(filenames),1),dtype=np.uint8)\n",
    "        a_seg=np.zeros((len(filenames),1), dtype=np.float64)\n",
    "        temp_list=[]\n",
    "        temp_label=[]\n",
    "        temp_seg=[]        \n",
    "        for i in range(0,len(filenames)):\n",
    "            if filenames[i]==\"\":\n",
    "                continue\n",
    "            npydata=np.load(\"./data/npy_10000/three/\"+filenames[i]+\".npy\")\n",
    "            np.random.shuffle(npydata)\n",
    "            if npydata.shape[0]<2048:\n",
    "                continue\n",
    "            else:\n",
    "                #x,y,z 2048 samping\n",
    "                temp_j_list=[]\n",
    "                for j in range(0,2048):\n",
    "                    temp_j_list.append([npydata[j][0],npydata[j][1], npydata[j][2]])\n",
    "                # seg_label ∈ [0,1]\n",
    "                temp_j_seg=[0,0,0]\n",
    "                temp_j_seg[0]=(np.amax(npydata[:,1])-ymin)/(ymax-ymin)\n",
    "                temp_j_seg[1]=(np.mean(npydata[:,1])-ymin)/(ymax-ymin)\n",
    "                temp_j_seg[2]=(np.amin(npydata[:,1])-ymin)/(ymax-ymin) \n",
    "                    \n",
    "                temp_list.append(temp_j_list)\n",
    "                temp_label.append(int(fstr[1])-1)\n",
    "                temp_seg.append(temp_j_seg)\n",
    "                \n",
    "        if len(temp_list) < max_shards:\n",
    "            for i in range(max_shards-len(temp_list)):\n",
    "                temp_shuffle_list=temp_list\n",
    "                temp_shuffle_seg=temp_seg\n",
    "                rvalue=randrange(len(temp_shuffle_list))\n",
    "                temp_list.append(temp_shuffle_list[rvalue])\n",
    "                temp_label.append(int(fstr[1])-1)\n",
    "                temp_seg.append(temp_shuffle_seg[rvalue])\n",
    "\n",
    "            \n",
    "        a_data=np.array(temp_list)\n",
    "        a_label=np.transpose(np.array(temp_label))\n",
    "        a_seg=np.array(temp_seg)\n",
    "        \n",
    "        target=np.zeros(a_data.shape)\n",
    "        for i in range(a_data.shape[0]):\n",
    "            for j in range(a_data.shape[1]):\n",
    "                for k in range(a_data.shape[2]):\n",
    "                    target[i][j][k]=a_data[i][j][k]-a_data[i].mean(axis=0)[k]\n",
    "\n",
    "        f=h5py.File(\"./data/floatlabel/\"+str(fstr[1])+'_'+str(fstr[2])+'_'+str(fstr[3])+\".h5\",'w')\n",
    "        data=f.create_dataset(\"data\", data=target)\n",
    "        label=f.create_dataset(\"label\", data=a_label)\n",
    "        seg=f.create_dataset(\"seg\", data=a_seg)\n",
    "        f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "flist=os.listdir(\"./data/npy_10000/list\")\n",
    "max_shards=20\n",
    "ymax, ymin=64.908, -64.943\n",
    "for file in flist:\n",
    "    if 'filelist_4' in file: #file = filelist_4_shards_randomseed_F\n",
    "        fstr=file.split('_') #{filelist, 4, shards, randomseed, F}\n",
    "        filenames=[line.rstrip() for line in open(\"./data/npy_10000/list/\"+file, 'r')] # filenames = generated_shards_0_randomseed_part_partnum\n",
    "        a_data=np.zeros((len(filenames),2048,3))\n",
    "        a_label=np.zeros((len(filenames),1),dtype=np.uint8)\n",
    "        a_seg=np.zeros((len(filenames),1), dtype=np.float64)\n",
    "        temp_list=[]\n",
    "        temp_label=[]\n",
    "        temp_seg=[]        \n",
    "        for i in range(0,len(filenames)):\n",
    "            if filenames[i]==\"\":\n",
    "                continue\n",
    "            npydata=np.load(\"./data/npy_10000/four/\"+filenames[i]+\".npy\")\n",
    "            np.random.shuffle(npydata)\n",
    "            if npydata.shape[0]<2048:\n",
    "                continue\n",
    "            else:\n",
    "                #x,y,z 2048 samping\n",
    "                temp_j_list=[]\n",
    "                for j in range(0,2048):\n",
    "                    temp_j_list.append([npydata[j][0],npydata[j][1], npydata[j][2]])\n",
    "                # seg_label ∈ [0,1]\n",
    "                temp_j_seg=[0,0,0]\n",
    "                temp_j_seg[0]=(np.amax(npydata[:,1])-ymin)/(ymax-ymin)\n",
    "                temp_j_seg[1]=(np.mean(npydata[:,1])-ymin)/(ymax-ymin)\n",
    "                temp_j_seg[2]=(np.amin(npydata[:,1])-ymin)/(ymax-ymin) \n",
    "                    \n",
    "                temp_list.append(temp_j_list)\n",
    "                temp_label.append(int(fstr[1])-1)\n",
    "                temp_seg.append(temp_j_seg)\n",
    "                \n",
    "        if len(temp_list) < max_shards:\n",
    "            for i in range(max_shards-len(temp_list)):\n",
    "                temp_shuffle_list=temp_list\n",
    "                temp_shuffle_seg=temp_seg\n",
    "                rvalue=randrange(len(temp_shuffle_list))\n",
    "                temp_list.append(temp_shuffle_list[rvalue])\n",
    "                temp_label.append(int(fstr[1])-1)\n",
    "                temp_seg.append(temp_shuffle_seg[rvalue])\n",
    "\n",
    "            \n",
    "        a_data=np.array(temp_list)\n",
    "        a_label=np.transpose(np.array(temp_label))\n",
    "        a_seg=np.array(temp_seg)\n",
    "        \n",
    "        target=np.zeros(a_data.shape)\n",
    "        for i in range(a_data.shape[0]):\n",
    "            for j in range(a_data.shape[1]):\n",
    "                for k in range(a_data.shape[2]):\n",
    "                    target[i][j][k]=a_data[i][j][k]-a_data[i].mean(axis=0)[k]\n",
    "\n",
    "        f=h5py.File(\"./data/floatlabel/\"+str(fstr[1])+'_'+str(fstr[2])+'_'+str(fstr[3])+\".h5\",'w')\n",
    "        data=f.create_dataset(\"data\", data=target)\n",
    "        label=f.create_dataset(\"label\", data=a_label)\n",
    "        seg=f.create_dataset(\"seg\", data=a_seg)\n",
    "        f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "flist=os.listdir(\"./data/npy_10000/list\")\n",
    "max_shards=20\n",
    "ymax, ymin=194.373, -194.371\n",
    "for file in flist:\n",
    "    if 'filelist_5' in file: #file = filelist_5_shards_randomseed_F\n",
    "        fstr=file.split('_') #{filelist, 5, shards, randomseed, F}\n",
    "        filenames=[line.rstrip() for line in open(\"./data/npy_10000/list/\"+file, 'r')] # filenames = generated_shards_0_randomseed_part_partnum\n",
    "        a_data=np.zeros((len(filenames),2048,3))\n",
    "        a_label=np.zeros((len(filenames),1),dtype=np.uint8)\n",
    "        a_seg=np.zeros((len(filenames),1), dtype=np.float64)\n",
    "        temp_list=[]\n",
    "        temp_label=[]\n",
    "        temp_seg=[]        \n",
    "        for i in range(0,len(filenames)):\n",
    "            if filenames[i]==\"\":\n",
    "                continue\n",
    "            npydata=np.load(\"./data/npy_10000/five/\"+filenames[i]+\".npy\")\n",
    "            np.random.shuffle(npydata)\n",
    "            if npydata.shape[0]<2048:\n",
    "                continue\n",
    "            else:\n",
    "                #x,y,z 2048 samping\n",
    "                temp_j_list=[]\n",
    "                for j in range(0,2048):\n",
    "                    temp_j_list.append([npydata[j][0],npydata[j][1], npydata[j][2]])\n",
    "                # seg_label ∈ [0,1]\n",
    "                temp_j_seg=[0,0,0]\n",
    "                temp_j_seg[0]=(np.amax(npydata[:,1])-ymin)/(ymax-ymin)\n",
    "                temp_j_seg[1]=(np.mean(npydata[:,1])-ymin)/(ymax-ymin)\n",
    "                temp_j_seg[2]=(np.amin(npydata[:,1])-ymin)/(ymax-ymin) \n",
    "                    \n",
    "                temp_list.append(temp_j_list)\n",
    "                temp_label.append(int(fstr[1])-1)\n",
    "                temp_seg.append(temp_j_seg)\n",
    "                \n",
    "        if len(temp_list) < max_shards:\n",
    "            for i in range(max_shards-len(temp_list)):\n",
    "                temp_shuffle_list=temp_list\n",
    "                temp_shuffle_seg=temp_seg\n",
    "                rvalue=randrange(len(temp_shuffle_list))\n",
    "                temp_list.append(temp_shuffle_list[rvalue])\n",
    "                temp_label.append(int(fstr[1])-1)\n",
    "                temp_seg.append(temp_shuffle_seg[rvalue])\n",
    "\n",
    "            \n",
    "        a_data=np.array(temp_list)\n",
    "        a_label=np.transpose(np.array(temp_label))\n",
    "        a_seg=np.array(temp_seg)\n",
    "        \n",
    "        target=np.zeros(a_data.shape)\n",
    "        for i in range(a_data.shape[0]):\n",
    "            for j in range(a_data.shape[1]):\n",
    "                for k in range(a_data.shape[2]):\n",
    "                    target[i][j][k]=a_data[i][j][k]-a_data[i].mean(axis=0)[k]\n",
    "\n",
    "        f=h5py.File(\"./data/floatlabel/\"+str(fstr[1])+'_'+str(fstr[2])+'_'+str(fstr[3])+\".h5\",'w')\n",
    "        data=f.create_dataset(\"data\", data=target)\n",
    "        label=f.create_dataset(\"label\", data=a_label)\n",
    "        seg=f.create_dataset(\"seg\", data=a_seg)\n",
    "        f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder=os.listdir(\"./data/floatlabel\")\n",
    "f=open(\"./data/total_files.txt\", \"w\")\n",
    "for files in folder:\n",
    "    f.write(\"%s\\n\"%files)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./data/total_files.txt\",\"r\") as f:\n",
    "    data = f.read().split('\\n')\n",
    "random.shuffle(data)\n",
    "\n",
    "train_data=data[:7000]\n",
    "test_data=data[7001:]\n",
    "\n",
    "train=open(\"./data/train_files.txt\",\"w+\")\n",
    "test=open(\"./data/test_files.txt\",\"w+\")\n",
    "\n",
    "for i in range(len(train_data)):\n",
    "    train_name=train_data[i]\n",
    "    train.write(\"%s\\n\"%train_name)\n",
    "    \n",
    "for i in range(len(test_data)):\n",
    "    test_name=test_data[i]\n",
    "    test.write(\"%s\\n\"%test_name)\n",
    "\n",
    "train.close()\n",
    "test.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}