CenterPoint copy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# # setup\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd /data/kimgh/CenterPoint-static/det3d/ops/iou3d_nms\n",
    "\n",
    "!python setup.py build_ext --inplace"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# # Prepare data\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd /data/kimgh/CenterPoint\n",
    "\n",
    "!python create_data.py nuscenes_data_prep --root_path '/data/kimgh/CenterPoint/data/nuScenes/' --version 'v1.0-mini' --nsweeps 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/data/kimgh/CenterPoint-custom/CenterPoint-dynamic\n",
      "no apex\n",
      "2023-04-11 13:37:56.986535: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-04-11 13:37:58.113162: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "No Tensorflow\n",
      "['A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_064.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_057.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_012.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_078.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_049.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_001.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_042.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_100.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_017.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_031.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_097.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_079.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_095.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_068.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_004.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_059.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_050.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_053.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_011.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_002.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_056.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_009.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_055.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_022.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_025.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_085.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_045.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_023.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_060.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_067.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_035.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_003.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_074.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_073.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_086.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_061.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_075.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_088.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_096.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_082.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_039.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_052.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_090.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_072.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_044.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_020.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_016.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_005.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_076.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_048.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_046.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_007.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_087.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_058.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_021.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_083.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_018.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_066.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_029.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_065.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_047.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_070.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_089.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_054.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_030.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_006.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_019.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_024.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_094.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_013.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_028.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_034.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_084.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_033.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_071.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_026.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_041.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_008.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_014.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_092.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_063.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_093.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_032.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_027.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_069.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_062.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_043.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_036.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_037.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_015.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_091.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_098.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_040.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_038.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_080.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_081.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_077.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_099.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_010.pcd', 'A/source/A_Clip_07689_03/Lidar/2-048_07689_LR_051.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_036.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_099.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_058.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_006.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_097.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_035.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_061.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_049.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_075.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_011.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_040.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_079.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_054.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_080.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_088.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_048.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_096.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_019.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_074.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_053.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_031.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_092.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_008.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_017.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_032.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_078.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_030.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_095.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_022.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_091.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_052.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_083.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_043.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_081.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_057.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_063.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_089.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_085.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_045.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_041.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_009.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_065.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_020.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_046.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_039.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_084.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_014.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_068.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_047.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_004.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_037.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_100.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_090.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_073.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_064.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_021.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_066.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_038.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_016.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_034.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_059.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_067.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_071.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_028.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_003.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_077.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_023.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_070.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_029.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_024.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_042.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_007.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_033.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_001.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_056.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_013.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_005.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_010.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_018.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_072.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_027.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_069.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_026.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_086.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_062.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_094.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_087.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_051.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_044.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_025.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_012.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_050.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_093.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_055.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_076.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_015.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_002.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_060.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_098.pcd', 'S/source/S_Clip_02245_02/Lidar/2-048_02245_LR_082.pcd']\n",
      "exist train frames: 1200 exist val frames: 200\n",
      " 59%|███████████████████████▌                | 708/1200 [00:08<00:07, 68.83it/s]/data/kimgh/CenterPoint-custom/CenterPoint-dynamic/det3d/datasets/nia/nia_common.py:718: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  if 'ETC' in names:\n",
      "100%|███████████████████████████████████████| 1200/1200 [00:14<00:00, 83.86it/s]\n",
      "100%|█████████████████████████████████████████| 200/200 [00:02<00:00, 94.40it/s]\n",
      "1\n",
      "100%|███████████████████████████████████████| 1200/1200 [08:24<00:00,  2.38it/s]\n",
      "dataset length:  1200\n",
      "load 217 bus database infos\n",
      "load 347 truck database infos\n",
      "load 382 pedestrian database infos\n",
      "load 1816 car database infos\n",
      "load 320 construction_vehicle database infos\n",
      "load 217 bicycle database infos\n",
      "load 100 motorcycle database infos\n"
     ]
    }
   ],
   "source": [
    "%cd /data/kimgh/CenterPoint-custom/CenterPoint-dynamic\n",
    "\n",
    "root_path = '/data/kimgh/CenterPoint-custom/CenterPoint-dynamic/data'\n",
    "sensor ='lidar'\n",
    "\n",
    "!python create_data.py nia_data_prep --root_path {root_path} --sensor {sensor}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd /data/kimgh/CenterPoint-custom/CenterPoint-static\n",
    "\n",
    "root_path = '/data/kimgh/CenterPoint-custom/CenterPoint-static/data'\n",
    "sensor = 'lidar'\n",
    "\n",
    "!python create_data.py nia_data_prep --root_path {root_path} --sensor {sensor}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# # Train\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd /data/kimgh/CenterPoint\n",
    "\n",
    "CONFIG_PATH = '/data/kimgh/CenterPoint/configs/nusc/voxelnet/nusc_centerpoint_voxelnet_01voxel.py'\n",
    "work_dir = './result/nusc_centerpoint_voxelnet_01voxel'\n",
    "\n",
    "f'!torchrun --nproc_per_node 4 train.py {CONFIG_PATH} --work_dir {work_dir} --validate'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd /data/kimgh/CenterPoint-custom/CenterPoint-static\n",
    "\n",
    "CONFIG_PATH = '/data/kimgh/CenterPoint-custom/CenterPoint-static/configs/nia/lidar/nia_centerpoint_voxelnet_01voxel_lidar_normal_copy.py'\n",
    "work_dir = './result/lidar/train'\n",
    "\n",
    "f'!torchrun --nproc_per_node 4 train.py --config {CONFIG_PATH} --work_dir {work_dir} --validate'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd /data/kimgh/CenterPoint-custom/CenterPoint-static\n",
    "\n",
    "CONFIG_PATH = '/data/kimgh/CenterPoint-custom/CenterPoint-static/configs/nia/lidar/nia_centerpoint_voxelnet_01voxel_lidar_normal_copy.py'\n",
    "work_dir = './result/lidar/train'\n",
    "\n",
    "!python train.py --config {CONFIG_PATH} --work_dir {work_dir} --validate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/data/kimgh/CenterPoint-custom/CenterPoint-dynamic\n",
      "no apex\n",
      "2023-04-11 15:18:28.006255: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-04-11 15:18:28.969326: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "No Tensorflow\n",
      "Deformable Convolution not built!\n",
      "No APEX!\n",
      "2023-04-11 15:18:30,356 - INFO - Sensor: lidar\n",
      "2023-04-11 15:18:30,356 - INFO - Config: /data/kimgh/CenterPoint-custom/CenterPoint-dynamic/configs/nia/lidar/nia_centerpoint_voxelnet_01voxel.py\n",
      "2023-04-11 15:18:30,356 - INFO - Distributed training: False\n",
      "2023-04-11 15:18:30,356 - INFO - torch.backends.cudnn.benchmark: False\n",
      "2023-04-11 15:18:30,487 - INFO - Finish RPN Initialization\n",
      "2023-04-11 15:18:30,488 - INFO - num_classes: [1, 2, 1, 2, 1]\n",
      "Use HM Bias:  -2.19\n",
      "2023-04-11 15:18:30,532 - INFO - Finish CenterHead Initialization\n",
      "2023-04-11 15:18:30,560 - INFO - {'car': 5, 'truck': 5, 'bus': 5, 'construction_vehicle': 5, 'motorcycle': 5, 'bicycle': 5, 'pedestrian': 5}\n",
      "2023-04-11 15:18:30,560 - INFO - [-1]\n",
      "2023-04-11 15:18:30,579 - INFO - load 217 bus database infos\n",
      "2023-04-11 15:18:30,579 - INFO - load 347 truck database infos\n",
      "2023-04-11 15:18:30,579 - INFO - load 382 pedestrian database infos\n",
      "2023-04-11 15:18:30,579 - INFO - load 1816 car database infos\n",
      "2023-04-11 15:18:30,579 - INFO - load 320 construction_vehicle database infos\n",
      "2023-04-11 15:18:30,579 - INFO - load 217 bicycle database infos\n",
      "2023-04-11 15:18:30,579 - INFO - load 100 motorcycle database infos\n",
      "2023-04-11 15:18:30,584 - INFO - After filter database:\n",
      "2023-04-11 15:18:30,584 - INFO - load 217 bus database infos\n",
      "2023-04-11 15:18:30,584 - INFO - load 347 truck database infos\n",
      "2023-04-11 15:18:30,584 - INFO - load 381 pedestrian database infos\n",
      "2023-04-11 15:18:30,584 - INFO - load 1809 car database infos\n",
      "2023-04-11 15:18:30,584 - INFO - load 320 construction_vehicle database infos\n",
      "2023-04-11 15:18:30,584 - INFO - load 217 bicycle database infos\n",
      "2023-04-11 15:18:30,584 - INFO - load 100 motorcycle database infos\n",
      "1\n",
      "2023-04-11 15:18:34,516 - INFO - model structure: VoxelNet(\n",
      "  (reader): VoxelFeatureExtractorV3()\n",
      "  (backbone): SpMiddleResNetFHD(\n",
      "    (conv_input): SparseSequential(\n",
      "      (0): SubMConv3d(4, 16, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[0, 0, 0], dilation=[1, 1, 1], output_padding=[0, 0, 0], bias=False, algo=ConvAlgo.MaskImplicitGemm)\n",
      "      (1): BatchNorm1d(16, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "      (2): ReLU(inplace=True)\n",
      "    )\n",
      "    (conv1): SparseSequential(\n",
      "      (0): SparseBasicBlock(\n",
      "        (conv1): SubMConv3d(16, 16, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn1): BatchNorm1d(16, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (relu): ReLU()\n",
      "        (conv2): SubMConv3d(16, 16, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn2): BatchNorm1d(16, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (1): SparseBasicBlock(\n",
      "        (conv1): SubMConv3d(16, 16, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn1): BatchNorm1d(16, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (relu): ReLU()\n",
      "        (conv2): SubMConv3d(16, 16, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn2): BatchNorm1d(16, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (conv2): SparseSequential(\n",
      "      (0): SparseConv3d(16, 32, kernel_size=[3, 3, 3], stride=[2, 2, 2], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], bias=False, algo=ConvAlgo.MaskImplicitGemm)\n",
      "      (1): BatchNorm1d(32, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "      (2): ReLU(inplace=True)\n",
      "      (3): SparseBasicBlock(\n",
      "        (conv1): SubMConv3d(32, 32, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn1): BatchNorm1d(32, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (relu): ReLU()\n",
      "        (conv2): SubMConv3d(32, 32, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn2): BatchNorm1d(32, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (4): SparseBasicBlock(\n",
      "        (conv1): SubMConv3d(32, 32, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn1): BatchNorm1d(32, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (relu): ReLU()\n",
      "        (conv2): SubMConv3d(32, 32, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn2): BatchNorm1d(32, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (conv3): SparseSequential(\n",
      "      (0): SparseConv3d(32, 64, kernel_size=[3, 3, 3], stride=[2, 2, 2], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], bias=False, algo=ConvAlgo.MaskImplicitGemm)\n",
      "      (1): BatchNorm1d(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "      (2): ReLU(inplace=True)\n",
      "      (3): SparseBasicBlock(\n",
      "        (conv1): SubMConv3d(64, 64, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn1): BatchNorm1d(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (relu): ReLU()\n",
      "        (conv2): SubMConv3d(64, 64, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn2): BatchNorm1d(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (4): SparseBasicBlock(\n",
      "        (conv1): SubMConv3d(64, 64, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn1): BatchNorm1d(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (relu): ReLU()\n",
      "        (conv2): SubMConv3d(64, 64, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn2): BatchNorm1d(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (conv4): SparseSequential(\n",
      "      (0): SparseConv3d(64, 128, kernel_size=[3, 3, 3], stride=[2, 2, 2], padding=[0, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], bias=False, algo=ConvAlgo.MaskImplicitGemm)\n",
      "      (1): BatchNorm1d(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "      (2): ReLU(inplace=True)\n",
      "      (3): SparseBasicBlock(\n",
      "        (conv1): SubMConv3d(128, 128, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn1): BatchNorm1d(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (relu): ReLU()\n",
      "        (conv2): SubMConv3d(128, 128, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn2): BatchNorm1d(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "      )\n",
      "      (4): SparseBasicBlock(\n",
      "        (conv1): SubMConv3d(128, 128, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn1): BatchNorm1d(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (relu): ReLU()\n",
      "        (conv2): SubMConv3d(128, 128, kernel_size=[3, 3, 3], stride=[1, 1, 1], padding=[1, 1, 1], dilation=[1, 1, 1], output_padding=[0, 0, 0], algo=ConvAlgo.MaskImplicitGemm)\n",
      "        (bn2): BatchNorm1d(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (extra_conv): SparseSequential(\n",
      "      (0): SparseConv3d(128, 128, kernel_size=[3, 1, 1], stride=[2, 1, 1], padding=[0, 0, 0], dilation=[1, 1, 1], output_padding=[0, 0, 0], bias=False, algo=ConvAlgo.MaskImplicitGemm)\n",
      "      (1): BatchNorm1d(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "      (2): ReLU()\n",
      "    )\n",
      "  )\n",
      "  (neck): RPN(\n",
      "    (blocks): ModuleList(\n",
      "      (0): Sequential(\n",
      "        (0): ZeroPad2d((1, 1, 1, 1))\n",
      "        (1): Conv2d(256, 128, kernel_size=(3, 3), stride=(1, 1), bias=False)\n",
      "        (2): BatchNorm2d(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (3): ReLU()\n",
      "        (4): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "        (5): BatchNorm2d(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (6): ReLU()\n",
      "        (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "        (8): BatchNorm2d(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (9): ReLU()\n",
      "        (10): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "        (11): BatchNorm2d(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (12): ReLU()\n",
      "        (13): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "        (14): BatchNorm2d(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (15): ReLU()\n",
      "        (16): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "        (17): BatchNorm2d(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (18): ReLU()\n",
      "      )\n",
      "      (1): Sequential(\n",
      "        (0): ZeroPad2d((1, 1, 1, 1))\n",
      "        (1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "        (2): BatchNorm2d(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (3): ReLU()\n",
      "        (4): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "        (5): BatchNorm2d(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (6): ReLU()\n",
      "        (7): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "        (8): BatchNorm2d(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (9): ReLU()\n",
      "        (10): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "        (11): BatchNorm2d(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (12): ReLU()\n",
      "        (13): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "        (14): BatchNorm2d(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (15): ReLU()\n",
      "        (16): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "        (17): BatchNorm2d(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (18): ReLU()\n",
      "      )\n",
      "    )\n",
      "    (deblocks): ModuleList(\n",
      "      (0): Sequential(\n",
      "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (1): BatchNorm2d(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (2): ReLU()\n",
      "      )\n",
      "      (1): Sequential(\n",
      "        (0): ConvTranspose2d(256, 256, kernel_size=(2, 2), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)\n",
      "        (2): ReLU()\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (bbox_head): CenterHead(\n",
      "    (crit): FastFocalLoss()\n",
      "    (crit_reg): RegLoss()\n",
      "    (shared_conv): Sequential(\n",
      "      (0): Conv2d(512, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): ReLU(inplace=True)\n",
      "    )\n",
      "    (tasks): ModuleList(\n",
      "      (0): SepHead(\n",
      "        (reg): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (height): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 1, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (dim): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (rot): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (vel): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (hm): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 1, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "      )\n",
      "      (1): SepHead(\n",
      "        (reg): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (height): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 1, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (dim): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (rot): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (vel): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (hm): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "      )\n",
      "      (2): SepHead(\n",
      "        (reg): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (height): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 1, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (dim): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (rot): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (vel): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (hm): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 1, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "      )\n",
      "      (3): SepHead(\n",
      "        (reg): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (height): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 1, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (dim): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (rot): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (vel): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (hm): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "      )\n",
      "      (4): SepHead(\n",
      "        (reg): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (height): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 1, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (dim): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (rot): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (vel): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 2, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (hm): Sequential(\n",
      "          (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "          (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (2): ReLU()\n",
      "          (3): Conv2d(64, 1, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      ")\n",
      "2023-04-11 15:18:34,547 - INFO - Start running, host: ubuntu@t3qai-gpu01, work_dir: /data/kimgh/CenterPoint-custom/CenterPoint-dynamic/result/lidar/train\n",
      "2023-04-11 15:18:34,547 - INFO - workflow: [('train', 1)], max: 25 epochs\n",
      "2023-04-11 15:18:42,021 - INFO - finding looplift candidates\n",
      "2023-04-11 15:18:42,049 - INFO - finding looplift candidates\n",
      "2023-04-11 15:18:42,078 - INFO - finding looplift candidates\n",
      "2023-04-11 15:18:42,082 - INFO - finding looplift candidates\n",
      "2023-04-11 15:18:42,082 - INFO - finding looplift candidates\n",
      "2023-04-11 15:18:42,102 - INFO - finding looplift candidates\n",
      "2023-04-11 15:18:42,152 - INFO - finding looplift candidates\n",
      "2023-04-11 15:18:42,211 - INFO - finding looplift candidates\n",
      "Traceback (most recent call last):\n",
      "  File \"/data/kimgh/CenterPoint-custom/CenterPoint-dynamic/train.py\", line 182, in <module>\n",
      "    main()\n",
      "  File \"/data/kimgh/CenterPoint-custom/CenterPoint-dynamic/train.py\", line 171, in main\n",
      "    train_detector(\n",
      "  File \"/data/kimgh/CenterPoint-custom/CenterPoint-dynamic/det3d/torchie/apis/train.py\", line 328, in train_detector\n",
      "    trainer.run(data_loaders, cfg.workflow, cfg.total_epochs, local_rank=cfg.local_rank)\n",
      "  File \"/data/kimgh/CenterPoint-custom/CenterPoint-dynamic/det3d/torchie/trainer/trainer.py\", line 542, in run\n",
      "    epoch_runner(data_loaders[i], self.epoch, **kwargs)\n",
      "  File \"/data/kimgh/CenterPoint-custom/CenterPoint-dynamic/det3d/torchie/trainer/trainer.py\", line 408, in train\n",
      "    outputs = self.batch_processor_inline(\n",
      "  File \"/data/kimgh/CenterPoint-custom/CenterPoint-dynamic/det3d/torchie/trainer/trainer.py\", line 367, in batch_processor_inline\n",
      "    losses = model(example, return_loss=True)\n",
      "  File \"/home/ubuntu/anaconda3/envs/nia48_310/lib/python3.10/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
      "    return forward_call(*args, **kwargs)\n",
      "  File \"/data/kimgh/CenterPoint-custom/CenterPoint-dynamic/det3d/models/detectors/voxelnet.py\", line 57, in forward\n",
      "    preds, _ = self.bbox_head(x)\n",
      "  File \"/home/ubuntu/anaconda3/envs/nia48_310/lib/python3.10/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
      "    return forward_call(*args, **kwargs)\n",
      "  File \"/data/kimgh/CenterPoint-custom/CenterPoint-dynamic/det3d/models/bbox_heads/center_head.py\", line 242, in forward\n",
      "    ret_dicts.append(task(x))\n",
      "  File \"/home/ubuntu/anaconda3/envs/nia48_310/lib/python3.10/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
      "    return forward_call(*args, **kwargs)\n",
      "  File \"/data/kimgh/CenterPoint-custom/CenterPoint-dynamic/det3d/models/bbox_heads/center_head.py\", line 108, in forward\n",
      "    ret_dict[head] = self.__getattr__(head)(x)\n",
      "  File \"/home/ubuntu/anaconda3/envs/nia48_310/lib/python3.10/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
      "    return forward_call(*args, **kwargs)\n",
      "  File \"/data/kimgh/CenterPoint-custom/CenterPoint-dynamic/det3d/models/utils/misc.py\", line 93, in forward\n",
      "    input = module(input)\n",
      "  File \"/home/ubuntu/anaconda3/envs/nia48_310/lib/python3.10/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
      "    return forward_call(*args, **kwargs)\n",
      "  File \"/home/ubuntu/anaconda3/envs/nia48_310/lib/python3.10/site-packages/torch/nn/modules/batchnorm.py\", line 171, in forward\n",
      "    return F.batch_norm(\n",
      "  File \"/home/ubuntu/anaconda3/envs/nia48_310/lib/python3.10/site-packages/torch/nn/functional.py\", line 2450, in batch_norm\n",
      "    return torch.batch_norm(\n",
      "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 64.00 MiB (GPU 0; 14.76 GiB total capacity; 12.23 GiB already allocated; 49.75 MiB free; 12.75 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF\n"
     ]
    }
   ],
   "source": [
    "%cd /data/kimgh/CenterPoint-custom/CenterPoint-dynamic\n",
    "\n",
    "CONFIG_PATH = '/data/kimgh/CenterPoint-custom/CenterPoint-dynamic/configs/nia/lidar/nia_centerpoint_voxelnet_01voxel.py'\n",
    "work_dir = './result/lidar/train'\n",
    "\n",
    "!python train.py --config {CONFIG_PATH} --work_dir {work_dir} --validate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# # Validation\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lidar-normal\n",
    "\n",
    "%cd /data/kimgh/CenterPoint-custom/CenterPoint-static\n",
    "\n",
    "config = '/data/kimgh/CenterPoint-custom/CenterPoint-static/configs/nia/lidar/nia_centerpoint_voxelnet_01voxel_lidar_normal.py'\n",
    "work_dir = '/data/kimgh/CenterPoint-custom/CenterPoint-static/result/lidar/normal_allsubid'\n",
    "checkpoint = '/data/kimgh/CenterPoint-custom/CenterPoint-static/weights/nia_centerpoint_static_lidar.pth'\n",
    "sensor = 'lidar'\n",
    "\n",
    "f'!CUDA_VISIBLE_DEVICES=1 python dist_test.py --config {config} --work_dir {work_dir} --checkpoint {checkpoint} --sensor {sensor} --speed_test' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lidar-abnormal\n",
    "\n",
    "%cd /data/kimgh/CenterPoint-custom/CenterPoint-static\n",
    "\n",
    "config = '/data/kimgh/CenterPoint-custom/CenterPoint-static/configs/nia/lidar/nia_centerpoint_voxelnet_01voxel_lidar_abnormal.py'\n",
    "work_dir = '/data/kimgh/CenterPoint-custom/CenterPoint-static/result/lidar/abnormal_allsubid'\n",
    "checkpoint = '/data/kimgh/CenterPoint-custom/CenterPoint-static/weights/nia_centerpoint_static_lidar.pth'\n",
    "sensor = 'lidar'\n",
    "\n",
    "f'!CUDA_VISIBLE_DEVICES=2 python dist_test.py --config {config} --work_dir {work_dir} --checkpoint {checkpoint} --sensor {sensor} --speed_test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# radar-normal\n",
    "\n",
    "%cd /data/kimgh/CenterPoint-custom/CenterPoint-static\n",
    "\n",
    "config = '/data/kimgh/CenterPoint-custom/CenterPoint-static/configs/nia/radar/nia_centerpoint_voxelnet_01voxel_radar.py'\n",
    "work_dir = '/data/kimgh/CenterPoint-custom/CenterPoint-static/result/radar/normal'\n",
    "checkpoint = '/data/kimgh/CenterPoint-custom/CenterPoint-static/weights/nia_centerpoint_static_radar.pth'\n",
    "sensor = 'radar'\n",
    "\n",
    "!CUDA_VISIBLE_DEVICES=0 python dist_test.py --config {config} --work_dir {work_dir} --checkpoint {checkpoint} --sensor {sensor} --speed_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# radar-abnormal\n",
    "\n",
    "%cd /data/kimgh/CenterPoint-custom/CenterPoint-static\n",
    "\n",
    "config = '/data/kimgh/CenterPoint-custom/CenterPoint-static/configs/nia/radar/nia_centerpoint_voxelnet_01voxel_radar_abnormal.py'\n",
    "work_dir = '/data/kimgh/CenterPoint-custom/CenterPoint-static/result/radar/abnormal'\n",
    "checkpoint = '/data/kimgh/CenterPoint-custom/CenterPoint-static/weights/nia_centerpoint_static_radar.pth'\n",
    "\n",
    "!CUDA_VISIBLE_DEVICES=3 python dist_test.py --config {config} --work_dir {work_dir} --checkpoint {checkpoint} --speed_test"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nia48_310",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}