yolox-training.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        s=''\n        #print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-09T19:19:37.679651Z","iopub.execute_input":"2022-01-09T19:19:37.680089Z","iopub.status.idle":"2022-01-09T19:19:43.803304Z","shell.execute_reply.started":"2022-01-09T19:19:37.679973Z","shell.execute_reply":"2022-01-09T19:19:43.802337Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport ast\nimport os\nimport json\nimport pandas as pd\nimport torch\nimport importlib\nimport cv2 \n\nfrom shutil import copyfile\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nfrom sklearn.model_selection import GroupKFold\nfrom PIL import Image\nfrom string import Template\nfrom IPython.display import display\n\nTRAIN_PATH = '/kaggle/input/tensorflow-great-barrier-reef'","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:20:34.349438Z","iopub.execute_input":"2022-01-09T19:20:34.351396Z","iopub.status.idle":"2022-01-09T19:20:36.723561Z","shell.execute_reply.started":"2022-01-09T19:20:34.351313Z","shell.execute_reply":"2022-01-09T19:20:36.722873Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"# check Torch and CUDA version\nprint(f\"Torch: {torch.__version__}\")\n!nvcc --version","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:20:36.725243Z","iopub.execute_input":"2022-01-09T19:20:36.725479Z","iopub.status.idle":"2022-01-09T19:20:37.442457Z","shell.execute_reply.started":"2022-01-09T19:20:36.725445Z","shell.execute_reply":"2022-01-09T19:20:37.441627Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/Megvii-BaseDetection/YOLOX -q\n\n%cd YOLOX\n!pip install -U pip && pip install -r requirements.txt\n!pip install -v -e . ","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:20:40.666637Z","iopub.execute_input":"2022-01-09T19:20:40.666929Z","iopub.status.idle":"2022-01-09T19:21:33.220641Z","shell.execute_reply.started":"2022-01-09T19:20:40.666894Z","shell.execute_reply":"2022-01-09T19:21:33.219834Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"!pip install 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:21:33.223352Z","iopub.execute_input":"2022-01-09T19:21:33.223649Z","iopub.status.idle":"2022-01-09T19:21:50.041802Z","shell.execute_reply.started":"2022-01-09T19:21:33.223609Z","shell.execute_reply":"2022-01-09T19:21:50.040998Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"images_preprocessed_save_directory= '../../working/images_preprocessed'\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_bbox(annots):\n    bboxes = [list(annot.values()) for annot in annots]\n    return bboxes\n\ndef get_path(row):\n    row['image_path'] = f'{TRAIN_PATH}/train_images/video_{row.video_id}/{row.video_frame}.jpg'\n    return row\ndef get_path_preprocessed(row):\n    row['preprocessed_image_path'] = f'{images_preprocessed_save_directory}/video_{row.video_id}_{row.video_frame}.jpg'\n    return row","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:20:58.422556Z","iopub.execute_input":"2022-01-09T20:20:58.424638Z","iopub.status.idle":"2022-01-09T20:20:58.433164Z","shell.execute_reply.started":"2022-01-09T20:20:58.424597Z","shell.execute_reply":"2022-01-09T20:20:58.432269Z"},"trusted":true},"execution_count":56,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\ndf.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:29:55.760397Z","iopub.execute_input":"2022-01-09T19:29:55.760940Z","iopub.status.idle":"2022-01-09T19:29:55.829423Z","shell.execute_reply.started":"2022-01-09T19:29:55.760904Z","shell.execute_reply":"2022-01-09T19:29:55.828704Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"markdown","source":"**TF addons equalize**","metadata":{}},{"cell_type":"code","source":"import glob\nimport cv2\nimport tensorflow_addons as tfa\nimport cv2\nimport os\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:30:50.632646Z","iopub.execute_input":"2022-01-09T19:30:50.632920Z","iopub.status.idle":"2022-01-09T19:30:55.912025Z","shell.execute_reply.started":"2022-01-09T19:30:50.632887Z","shell.execute_reply":"2022-01-09T19:30:55.911123Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"def img_trf(img_path):\n    #print(img_path)\n    img= cv2.imread(img_path)\n    #print(img)\n    return tfa.image.equalize(img)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:30:56.654692Z","iopub.execute_input":"2022-01-09T19:30:56.655262Z","iopub.status.idle":"2022-01-09T19:30:56.663544Z","shell.execute_reply.started":"2022-01-09T19:30:56.655222Z","shell.execute_reply":"2022-01-09T19:30:56.662079Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"def preprocess_images(images_path_list,images_preprocessed_save_directory):\n    #print(images_path_list)\n    for index,path in tqdm(enumerate(images_path_list)):\n        trf_img= img_trf(path).numpy()\n        splits=path.split('video_')\n        #print('video_'+splits[1][0]+'_'+splits[1][2:])\n        #print(os.path.join(images_preprocessed_save_directory,('video_'+splits[1][0]+'_'+splits[1][2:])))\n        #break\n        #print(type(trf_img.numpy()))\n        #print(type(images_preprocessed_save_directory))\n        #print(path.split('images')[1][1:])\n        cv2.imwrite(os.path.join(images_preprocessed_save_directory,('video_'+splits[1][0]+'_'+splits[1][2:])),trf_img)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:43:53.709923Z","iopub.execute_input":"2022-01-09T19:43:53.710772Z","iopub.status.idle":"2022-01-09T19:43:53.718560Z","shell.execute_reply.started":"2022-01-09T19:43:53.710726Z","shell.execute_reply":"2022-01-09T19:43:53.717812Z"},"trusted":true},"execution_count":51,"outputs":[]},{"cell_type":"code","source":"!ls ../../input/tensorflow-great-barrier-reef/train_images/","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:43:54.917724Z","iopub.execute_input":"2022-01-09T19:43:54.918427Z","iopub.status.idle":"2022-01-09T19:43:55.619206Z","shell.execute_reply.started":"2022-01-09T19:43:54.918391Z","shell.execute_reply":"2022-01-09T19:43:55.618295Z"},"trusted":true},"execution_count":52,"outputs":[]},{"cell_type":"code","source":"images_path_list= glob.glob(\"../../input/tensorflow-great-barrier-reef/train_images/*/*.jpg\")\n    ","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:43:55.620967Z","iopub.execute_input":"2022-01-09T19:43:55.621207Z","iopub.status.idle":"2022-01-09T19:43:55.706089Z","shell.execute_reply.started":"2022-01-09T19:43:55.621180Z","shell.execute_reply":"2022-01-09T19:43:55.705285Z"},"trusted":true},"execution_count":53,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists(images_preprocessed_save_directory):\n    os.mkdir(images_preprocessed_save_directory)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:43:55.970254Z","iopub.execute_input":"2022-01-09T19:43:55.971017Z","iopub.status.idle":"2022-01-09T19:43:55.975191Z","shell.execute_reply.started":"2022-01-09T19:43:55.970980Z","shell.execute_reply":"2022-01-09T19:43:55.974136Z"},"trusted":true},"execution_count":54,"outputs":[]},{"cell_type":"code","source":"preprocess_images(images_path_list,images_preprocessed_save_directory)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:43:56.874118Z","iopub.execute_input":"2022-01-09T19:43:56.874378Z","iopub.status.idle":"2022-01-09T20:20:58.417932Z","shell.execute_reply.started":"2022-01-09T19:43:56.874352Z","shell.execute_reply":"2022-01-09T20:20:58.417001Z"},"trusted":true},"execution_count":55,"outputs":[]},{"cell_type":"code","source":"# Taken only annotated photos\ndf[\"num_bbox\"] = df['annotations'].apply(lambda x: str.count(x, 'x'))\ndf_train = df#df[df[\"num_bbox\"]>0]\n\n#Annotations \ndf_train['annotations'] = df_train['annotations'].progress_apply(lambda x: ast.literal_eval(x))\ndf_train['bboxes'] = df_train.annotations.progress_apply(get_bbox)\n\n#Images resolution\ndf_train[\"width\"] = 1280\ndf_train[\"height\"] = 720\n\n#Path of images\ndf_train = df_train.progress_apply(get_path, axis=1)\ndf_train = df_train.progress_apply(get_path_preprocessed, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:24:27.121058Z","iopub.execute_input":"2022-01-09T20:24:27.121329Z","iopub.status.idle":"2022-01-09T20:24:58.555419Z","shell.execute_reply.started":"2022-01-09T20:24:27.121300Z","shell.execute_reply":"2022-01-09T20:24:58.554734Z"},"trusted":true},"execution_count":68,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:24:59.593610Z","iopub.execute_input":"2022-01-09T20:24:59.594177Z","iopub.status.idle":"2022-01-09T20:24:59.615573Z","shell.execute_reply.started":"2022-01-09T20:24:59.594130Z","shell.execute_reply":"2022-01-09T20:24:59.614608Z"},"trusted":true},"execution_count":69,"outputs":[]},{"cell_type":"code","source":"train_5_folds_csv= pd.read_csv('../../input/cots-cv-strategy-subsequence/cross-validation/train-5folds.csv')","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:25:06.027309Z","iopub.execute_input":"2022-01-09T20:25:06.027591Z","iopub.status.idle":"2022-01-09T20:25:06.086421Z","shell.execute_reply.started":"2022-01-09T20:25:06.027563Z","shell.execute_reply":"2022-01-09T20:25:06.085690Z"},"trusted":true},"execution_count":70,"outputs":[]},{"cell_type":"code","source":"dfinal = train_5_folds_csv.merge(df_train, on=\"image_id\", how = 'inner')","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:25:07.033422Z","iopub.execute_input":"2022-01-09T20:25:07.033667Z","iopub.status.idle":"2022-01-09T20:25:07.071971Z","shell.execute_reply.started":"2022-01-09T20:25:07.033639Z","shell.execute_reply":"2022-01-09T20:25:07.071265Z"},"trusted":true},"execution_count":71,"outputs":[]},{"cell_type":"code","source":"dfinal.fold.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:25:39.000557Z","iopub.execute_input":"2022-01-09T20:25:39.000823Z","iopub.status.idle":"2022-01-09T20:25:39.012096Z","shell.execute_reply.started":"2022-01-09T20:25:39.000793Z","shell.execute_reply":"2022-01-09T20:25:39.011060Z"},"trusted":true},"execution_count":75,"outputs":[]},{"cell_type":"code","source":"selected=2","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:25:46.546388Z","iopub.execute_input":"2022-01-09T20:25:46.546635Z","iopub.status.idle":"2022-01-09T20:25:46.550363Z","shell.execute_reply.started":"2022-01-09T20:25:46.546609Z","shell.execute_reply":"2022-01-09T20:25:46.549685Z"},"trusted":true},"execution_count":76,"outputs":[]},{"cell_type":"code","source":"dfinal.columns","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:27:12.322939Z","iopub.execute_input":"2022-01-09T20:27:12.323265Z","iopub.status.idle":"2022-01-09T20:27:12.337177Z","shell.execute_reply.started":"2022-01-09T20:27:12.323225Z","shell.execute_reply":"2022-01-09T20:27:12.336533Z"},"trusted":true},"execution_count":79,"outputs":[]},{"cell_type":"code","source":"df_training= dfinal[dfinal['num_bbox']>0]","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:27:51.417096Z","iopub.execute_input":"2022-01-09T20:27:51.417794Z","iopub.status.idle":"2022-01-09T20:27:51.439735Z","shell.execute_reply.started":"2022-01-09T20:27:51.417742Z","shell.execute_reply":"2022-01-09T20:27:51.439010Z"},"trusted":true},"execution_count":80,"outputs":[]},{"cell_type":"code","source":"df_training.columns","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:28:26.497125Z","iopub.execute_input":"2022-01-09T20:28:26.497394Z","iopub.status.idle":"2022-01-09T20:28:26.505002Z","shell.execute_reply.started":"2022-01-09T20:28:26.497364Z","shell.execute_reply":"2022-01-09T20:28:26.502935Z"},"trusted":true},"execution_count":83,"outputs":[]},{"cell_type":"code","source":"HOME_DIR = '/kaggle/working/' \nDATASET_PATH = 'dataset/images'\n\n!mkdir {HOME_DIR}dataset\n!mkdir {HOME_DIR}{DATASET_PATH}\n!mkdir {HOME_DIR}{DATASET_PATH}/train2017\n!mkdir {HOME_DIR}{DATASET_PATH}/val2017\n!mkdir {HOME_DIR}{DATASET_PATH}/annotations","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:26:27.171305Z","iopub.execute_input":"2022-01-09T20:26:27.171557Z","iopub.status.idle":"2022-01-09T20:26:30.809769Z","shell.execute_reply.started":"2022-01-09T20:26:27.171527Z","shell.execute_reply":"2022-01-09T20:26:30.808764Z"},"trusted":true},"execution_count":78,"outputs":[]},{"cell_type":"code","source":"df_training.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:30:06.693559Z","iopub.execute_input":"2022-01-09T20:30:06.693823Z","iopub.status.idle":"2022-01-09T20:30:06.723561Z","shell.execute_reply.started":"2022-01-09T20:30:06.693794Z","shell.execute_reply":"2022-01-09T20:30:06.722750Z"},"trusted":true},"execution_count":86,"outputs":[]},{"cell_type":"code","source":"SELECTED_FOLD = 2\n\nfor i in tqdm(range(len(df_training))):\n    row = df_training.iloc[i]\n    #print(row)\n    if row.fold != SELECTED_FOLD:\n        copyfile(f'{row.preprocessed_image_path}', f'{HOME_DIR}{DATASET_PATH}/train2017/{row.image_id}.jpg')\n    else:\n        copyfile(f'{row.preprocessed_image_path}', f'{HOME_DIR}{DATASET_PATH}/val2017/{row.image_id}.jpg') ","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:30:37.339848Z","iopub.execute_input":"2022-01-09T20:30:37.340145Z","iopub.status.idle":"2022-01-09T20:31:05.218880Z","shell.execute_reply.started":"2022-01-09T20:30:37.340112Z","shell.execute_reply":"2022-01-09T20:31:05.218178Z"},"trusted":true},"execution_count":88,"outputs":[]},{"cell_type":"code","source":"!rm -rf ../images_preprocessed/","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:35:02.493555Z","iopub.execute_input":"2022-01-09T20:35:02.493833Z","iopub.status.idle":"2022-01-09T20:35:04.023641Z","shell.execute_reply.started":"2022-01-09T20:35:02.493803Z","shell.execute_reply":"2022-01-09T20:35:04.022630Z"},"trusted":true},"execution_count":105,"outputs":[]},{"cell_type":"code","source":"print(f'Number of training files: {len(os.listdir(f\"{HOME_DIR}{DATASET_PATH}/train2017/\"))}')\nprint(f'Number of validation files: {len(os.listdir(f\"{HOME_DIR}{DATASET_PATH}/val2017/\"))}')","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:35:59.037169Z","iopub.execute_input":"2022-01-09T20:35:59.037460Z","iopub.status.idle":"2022-01-09T20:35:59.047189Z","shell.execute_reply.started":"2022-01-09T20:35:59.037428Z","shell.execute_reply":"2022-01-09T20:35:59.046464Z"},"trusted":true},"execution_count":110,"outputs":[]},{"cell_type":"code","source":"def save_annot_json(json_annotation, filename):\n    with open(filename, 'w') as f:\n        output_json = json.dumps(json_annotation)\n        f.write(output_json)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:36:08.982277Z","iopub.execute_input":"2022-01-09T20:36:08.982541Z","iopub.status.idle":"2022-01-09T20:36:08.988348Z","shell.execute_reply.started":"2022-01-09T20:36:08.982513Z","shell.execute_reply":"2022-01-09T20:36:08.987458Z"},"trusted":true},"execution_count":111,"outputs":[]},{"cell_type":"code","source":"annotion_id = 0","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:36:18.218852Z","iopub.execute_input":"2022-01-09T20:36:18.219534Z","iopub.status.idle":"2022-01-09T20:36:18.223534Z","shell.execute_reply.started":"2022-01-09T20:36:18.219494Z","shell.execute_reply":"2022-01-09T20:36:18.222709Z"},"trusted":true},"execution_count":112,"outputs":[]},{"cell_type":"code","source":"def dataset2coco(df, dest_path):\n    \n    global annotion_id\n    \n    annotations_json = {\n        \"info\": [],\n        \"licenses\": [],\n        \"categories\": [],\n        \"images\": [],\n        \"annotations\": []\n    }\n    \n    info = {\n        \"year\": \"2021\",\n        \"version\": \"1\",\n        \"description\": \"COTS dataset - COCO format\",\n        \"contributor\": \"\",\n        \"url\": \"https://kaggle.com\",\n        \"date_created\": \"2021-11-30T15:01:26+00:00\"\n    }\n    annotations_json[\"info\"].append(info)\n    \n    lic = {\n            \"id\": 1,\n            \"url\": \"\",\n            \"name\": \"Unknown\"\n        }\n    annotations_json[\"licenses\"].append(lic)\n\n    classes = {\"id\": 0, \"name\": \"starfish\", \"supercategory\": \"none\"}\n\n    annotations_json[\"categories\"].append(classes)\n\n    \n    for ann_row in df.itertuples():\n            \n        images = {\n            \"id\": ann_row[0],\n            \"license\": 1,\n            \"file_name\": ann_row.image_id + '.jpg',\n            \"height\": ann_row.height,\n            \"width\": ann_row.width,\n            \"date_captured\": \"2021-11-30T15:01:26+00:00\"\n        }\n        \n        annotations_json[\"images\"].append(images)\n        \n        bbox_list = ann_row.bboxes\n        \n        for bbox in bbox_list:\n            b_width = bbox[2]\n            b_height = bbox[3]\n            \n            # some boxes in COTS are outside the image height and width\n            if (bbox[0] + bbox[2] > 1280):\n                b_width = bbox[0] - 1280 \n            if (bbox[1] + bbox[3] > 720):\n                b_height = bbox[1] - 720 \n                \n            image_annotations = {\n                \"id\": annotion_id,\n                \"image_id\": ann_row[0],\n                \"category_id\": 0,\n                \"bbox\": [bbox[0], bbox[1], b_width, b_height],\n                \"area\": bbox[2] * bbox[3],\n                \"segmentation\": [],\n                \"iscrowd\": 0\n            }\n            \n            annotion_id += 1\n            annotations_json[\"annotations\"].append(image_annotations)\n        \n        \n    print(f\"Dataset COTS annotation to COCO json format completed! Files: {len(df)}\")\n    return annotations_json","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:36:38.206888Z","iopub.execute_input":"2022-01-09T20:36:38.207563Z","iopub.status.idle":"2022-01-09T20:36:38.219930Z","shell.execute_reply.started":"2022-01-09T20:36:38.207528Z","shell.execute_reply":"2022-01-09T20:36:38.219245Z"},"trusted":true},"execution_count":113,"outputs":[]},{"cell_type":"code","source":"# Convert COTS dataset to JSON COCO\ntrain_annot_json = dataset2coco(df_training[df_training.fold != SELECTED_FOLD], f\"{HOME_DIR}{DATASET_PATH}/train2017/\")\nval_annot_json = dataset2coco(df_training[df_training.fold == SELECTED_FOLD], f\"{HOME_DIR}{DATASET_PATH}/val2017/\")\n\n# Save converted annotations\nsave_annot_json(train_annot_json, f\"{HOME_DIR}{DATASET_PATH}/annotations/train.json\")\nsave_annot_json(val_annot_json, f\"{HOME_DIR}{DATASET_PATH}/annotations/valid.json\")","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:37:14.067844Z","iopub.execute_input":"2022-01-09T20:37:14.068699Z","iopub.status.idle":"2022-01-09T20:37:14.423652Z","shell.execute_reply.started":"2022-01-09T20:37:14.068652Z","shell.execute_reply":"2022-01-09T20:37:14.422875Z"},"trusted":true},"execution_count":114,"outputs":[]},{"cell_type":"code","source":"# Choose model for your experiments NANO or YOLOX-S (you can adapt for other model type)\n\nNANO = False","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:37:30.341141Z","iopub.execute_input":"2022-01-09T20:37:30.341843Z","iopub.status.idle":"2022-01-09T20:37:30.345094Z","shell.execute_reply.started":"2022-01-09T20:37:30.341802Z","shell.execute_reply":"2022-01-09T20:37:30.344426Z"},"trusted":true},"execution_count":115,"outputs":[]},{"cell_type":"code","source":"config_file_template = '''\n\n#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nfrom yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n    def __init__(self):\n        super(Exp, self).__init__()\n        self.depth = 0.33\n        self.width = 0.50\n        self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(\".\")[0]\n        \n        # Define yourself dataset path\n        self.data_dir = \"/kaggle/working/dataset/images\"\n        self.train_ann = \"train.json\"\n        self.val_ann = \"valid.json\"\n\n        self.num_classes = 1\n\n        self.max_epoch = $max_epoch\n        self.data_num_workers = 2\n        self.eval_interval = 1\n        \n        self.mosaic_prob = 1.0\n        self.mixup_prob = 1.0\n        self.hsv_prob = 1.0\n        self.flip_prob = 0.5\n        self.no_aug_epochs = 2\n        \n        self.input_size = (1280, 720)\n        self.mosaic_scale = (0.5, 1.5)\n        self.random_size = (10, 20)\n        self.test_size = (1280, 720)\n'''","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:38:51.103781Z","iopub.execute_input":"2022-01-09T20:38:51.104055Z","iopub.status.idle":"2022-01-09T20:38:51.110311Z","shell.execute_reply.started":"2022-01-09T20:38:51.104026Z","shell.execute_reply":"2022-01-09T20:38:51.109449Z"},"trusted":true},"execution_count":117,"outputs":[]},{"cell_type":"code","source":"if NANO:\n    config_file_template = '''\n\n#!/usr/bin/env python3\n# -*- coding:utf-8 -*-\n# Copyright (c) Megvii, Inc. and its affiliates.\n\nimport os\n\nimport torch.nn as nn\n\nfrom yolox.exp import Exp as MyExp\n\n\nclass Exp(MyExp):\n    def __init__(self):\n        super(Exp, self).__init__()\n        self.depth = 0.33\n        self.width = 0.25\n        self.input_size = (416, 416)\n        self.mosaic_scale = (0.5, 1.5)\n        self.random_size = (10, 20)\n        self.test_size = (416, 416)\n        self.exp_name = os.path.split(\n            os.path.realpath(__file__))[1].split(\".\")[0]\n        self.enable_mixup = False\n\n        # Define yourself dataset path\n        self.data_dir = \"/kaggle/working/dataset/images\"\n        self.train_ann = \"train.json\"\n        self.val_ann = \"valid.json\"\n\n        self.num_classes = 1\n\n        self.max_epoch = $max_epoch\n        self.data_num_workers = 2\n        self.eval_interval = 1\n\n    def get_model(self, sublinear=False):\n        def init_yolo(M):\n            for m in M.modules():\n                if isinstance(m, nn.BatchNorm2d):\n                    m.eps = 1e-3\n                    m.momentum = 0.03\n\n        if \"model\" not in self.__dict__:\n            from yolox.models import YOLOX, YOLOPAFPN, YOLOXHead\n            in_channels = [256, 512, 1024]\n            # NANO model use depthwise = True, which is main difference.\n            backbone = YOLOPAFPN(self.depth,\n                                 self.width,\n                                 in_channels=in_channels,\n                                 depthwise=True)\n            head = YOLOXHead(self.num_classes,\n                             self.width,\n                             in_channels=in_channels,\n                             depthwise=True)\n            self.model = YOLOX(backbone, head)\n\n        self.model.apply(init_yolo)\n        self.model.head.initialize_biases(1e-2)\n        return self.model\n\n'''\n","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:39:22.134507Z","iopub.execute_input":"2022-01-09T20:39:22.134759Z","iopub.status.idle":"2022-01-09T20:39:22.139807Z","shell.execute_reply.started":"2022-01-09T20:39:22.134732Z","shell.execute_reply":"2022-01-09T20:39:22.139144Z"},"trusted":true},"execution_count":118,"outputs":[]},{"cell_type":"code","source":"PIPELINE_CONFIG_PATH='cots_config.py'\n\npipeline = Template(config_file_template).substitute(max_epoch = 20)\n\nwith open(PIPELINE_CONFIG_PATH, 'w') as f:\n    f.write(pipeline)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:39:41.313922Z","iopub.execute_input":"2022-01-09T20:39:41.314190Z","iopub.status.idle":"2022-01-09T20:39:41.319380Z","shell.execute_reply.started":"2022-01-09T20:39:41.314163Z","shell.execute_reply":"2022-01-09T20:39:41.318095Z"},"trusted":true},"execution_count":119,"outputs":[]},{"cell_type":"code","source":"# ./yolox/data/datasets/voc_classes.py\n\nvoc_cls = '''\nVOC_CLASSES = (\n  \"starfish\",\n)\n'''\nwith open('./yolox/data/datasets/voc_classes.py', 'w') as f:\n    f.write(voc_cls)\n\n# ./yolox/data/datasets/coco_classes.py\n\ncoco_cls = '''\nCOCO_CLASSES = (\n  \"starfish\",\n)\n'''\nwith open('./yolox/data/datasets/coco_classes.py', 'w') as f:\n    f.write(coco_cls)\n\n# check if everything is ok    \n!more ./yolox/data/datasets/coco_classes.py","metadata":{"execution":{"iopub.status.busy":"2022-01-09T20:40:00.091737Z","iopub.execute_input":"2022-01-09T20:40:00.092571Z","iopub.status.idle":"2022-01-09T20:40:00.824689Z","shell.execute_reply.started":"2022-01-09T20:40:00.092532Z","shell.execute_reply":"2022-01-09T20:40:00.823884Z"},"trusted":true},"execution_count":120,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}