diff --git a/README.md b/README.md index bb53720..8efc470 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ [![tests](https://github.com/prime-slam/lidar-labelling/actions/workflows/ci.yml/badge.svg)](https://github.com/prime-slam/lidar-labelling/actions/workflows/ci.yml) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -lidar-labelling is a tool for automatic segmentation of raw lidar clouds based on image segmentation. +lidar-labelling is a tool for automatic segmentation of dense lidar clouds based on image segmentation. Our labelling algorithm consists of two stages. -The first stage is a preliminary marking - a naive transfer of segmentation from images (performed by the [SAM](https://github.com/facebookresearch/segment-anything) algorithm) to the cloud. Then multi-stage processing of the cloud is performed, which allows us to make the cloud more compact before the final labelling without losing points that are significant for labelling. The preprocessing stages include removing points whose corresponding pixels were not marked on a sequence of images, selection of points close to the sensors, removal of noise, cloud voxelization. +The first stage is a initial segmentation - a naive transfer of segmentation from images (performed by the [SAM](https://github.com/facebookresearch/segment-anything) algorithm) to the cloud. Then multi-stage processing of the cloud is performed, which allows us to make the cloud more compact before the final labelling without losing points that are significant for labelling. The preprocessing stages include removing points whose corresponding pixels were not marked on a sequence of images, selection of points close to the sensors, removal of noise, cloud voxelization. The next stage is segmentation itself. The segmentation criterion in this work is the distance between points, which is calculated through the physical distance and the degree of similarity of the labelling of points on several images. Based on the distance matrix, the prepared cloud is segmented using the [GraphCut](https://ieeexplore.ieee.org/abstract/document/937505) algorithm. @@ -16,6 +16,11 @@ This tool currently supports processing of [KITTI](https://www.cvlibs.net/datase ## Usage Please check `example.ipynb` with a example of cloud segmentation from the [KITTI](https://www.cvlibs.net/datasets/kitti/eval_odometry.php) dataset. +## Experiments +Files for reproducing experiments are in folder `experiment`. You will need the [KITTI](https://www.cvlibs.net/datasets/kitti/eval_odometry.php) dataset with its folder structure and pre-performed image segmentation using the [SAM](https://github.com/facebookresearch/segment-anything) algorithm in npz format. + +First run `main_kitti_processing.py` to generate segmentation with our algorithm. Then run `main_kitti_processing_metrics.py` to calculate the segmentation metrics of each run and write them to a csv file. Then run `main_calc_metrics_by_csv.py` to calculate the average values โ€‹โ€‹of the metrics in the csv file. + ## License This project is licensed under the Apache License โ€” see the [LICENSE](https://github.com/prime-slam/lidar-labelling/blob/main/LICENSE) file for details. diff --git a/example.ipynb b/example.ipynb index eebe9b0..9016091 100644 --- a/example.ipynb +++ b/example.ipynb @@ -8,9 +8,9 @@ "source": [ "from src.datasets.kitti_dataset import KittiDataset\n", "\n", - "dataset_path = \"dataset/\"\n", + "dataset_path = \"dataset/\" # kitti\n", "sequence = \"00\"\n", - "image_instances_path = \"pipeline/vfm-labels/sam/00/\"\n", + "image_instances_path = \"pipeline/vfm-labelss/sam/00/\" # images processed by the SAM algorithm in npz format\n", "kitti = KittiDataset(dataset_path, sequence, image_instances_path)" ] }, @@ -20,6 +20,8 @@ "metadata": {}, "outputs": [], "source": [ + "# setting parameter values\n", + "\n", "from src.services.preprocessing.common.config import ConfigDTO\n", "\n", "config = ConfigDTO(\n", @@ -27,9 +29,9 @@ " \"dataset\": kitti,\n", " \"start_index\": 20,\n", " \"end_index\": 24,\n", - " \"start_image_index_offset\": 3,\n", + " \"start_image_index_offset\": 0,\n", " \"alpha_physical_distance\": 5,\n", - " \"beta_instance_distance\": 3,\n", + " \"beta_instance_distance\": 5,\n", " \"T_normalized_cut\": 0.02,\n", " \"reduce_detail_int_to_union_threshold\": 0.5,\n", " \"reduce_detail_int_to_mask_threshold\": 0.6,\n", @@ -48,6 +50,8 @@ "metadata": {}, "outputs": [], "source": [ + "# pcd initialisation and initial segmentation based on images\n", + "\n", "from src.services.preprocessing.init.map import InitMapProcessor\n", "from src.services.preprocessing.init.instances_matrix import InitInstancesMatrixProcessor\n", "\n", @@ -70,6 +74,8 @@ "metadata": {}, "outputs": [], "source": [ + "# visualisation of the initial pcd segmentation masks for a particular image\n", + "\n", "import copy\n", "\n", "from src.utils.pcd_utils import color_pcd_by_labels\n", @@ -85,6 +91,8 @@ "metadata": {}, "outputs": [], "source": [ + "# pcd handler initialisation\n", + "\n", "from src.services.preprocessing.not_zero import SelectionNotZeroProcessor\n", "from src.services.preprocessing.in_cube import SelectionInCubeProcessor\n", "from src.services.preprocessing.statistical_outlier import StatisticalOutlierProcessor\n", @@ -102,6 +110,8 @@ "metadata": {}, "outputs": [], "source": [ + "# pcd processing and saving the state\n", + "\n", "import copy\n", "\n", "pcd = copy.deepcopy(init_pcd)\n", @@ -118,13 +128,15 @@ "metadata": {}, "outputs": [], "source": [ + "# visualisation of the processed pcd before voxelization\n", + "\n", "import copy\n", "\n", "from src.utils.pcd_utils import color_pcd_by_labels\n", "from src.utils.pcd_utils import visualize_pcd\n", "\n", - "colored_pcd_pcd_for_clustering = color_pcd_by_labels(copy.deepcopy(pcd_for_clustering), points2instances_pcd_for_clustering[:, 0])\n", - "visualize_pcd(colored_pcd_pcd_for_clustering)" + "colored_pcd_for_clustering = color_pcd_by_labels(copy.deepcopy(pcd_for_clustering), points2instances_pcd_for_clustering[:, 0])\n", + "visualize_pcd(colored_pcd_for_clustering)" ] }, { @@ -133,6 +145,8 @@ "metadata": {}, "outputs": [], "source": [ + "# final processing step - voxelisation of the pcd\n", + "\n", "from src.services.preprocessing.voxel_down import VoxelDownProcessor\n", "\n", "pcd, points2instances, trace = VoxelDownProcessor().process(config, pcd, points2instances)" @@ -144,6 +158,25 @@ "metadata": {}, "outputs": [], "source": [ + "# visualisation of the voxelised pcd\n", + "\n", + "import copy\n", + "\n", + "from src.utils.pcd_utils import color_pcd_by_labels\n", + "from src.utils.pcd_utils import visualize_pcd\n", + "\n", + "colored_voxel_pcd_for_clustering = color_pcd_by_labels(copy.deepcopy(pcd), points2instances[:, 0])\n", + "visualize_pcd(colored_voxel_pcd_for_clustering)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# calculation of distance matrix for voxelised pcd\n", + "\n", "import numpy as np\n", "\n", "from scipy.spatial.distance import cdist\n", @@ -176,6 +209,8 @@ "metadata": {}, "outputs": [], "source": [ + "# distance matrix processing\n", + "\n", "from src.services.distance.isolated import RemovingIsolatedPointsProcessor\n", "from src.services.distance.connected_component import ExtractionLargestConnectedComponentProcessor\n", "\n", @@ -205,10 +240,11 @@ "metadata": {}, "outputs": [], "source": [ + "# pcd clustering using GraphCut algorithm\n", + "\n", "from src.services.normalized_cut_service import normalized_cut\n", "\n", "eigenval = 2\n", - "\n", "clusters = normalized_cut(\n", " dist,\n", " np.array([i for i in range(len(points))], dtype=int),\n", @@ -232,11 +268,7 @@ "metadata": {}, "outputs": [], "source": [ - "from src.utils.pcd_utils import color_pcd_by_clusters_and_voxels\n", - "from src.utils.pcd_utils import visualize_pcd\n", - "\n", - "pcd_colored = color_pcd_by_clusters_and_voxels(pcd_for_clustering, trace, clusters)\n", - "visualize_pcd(pcd_colored)" + "clusters[0]" ] }, { @@ -245,10 +277,13 @@ "metadata": {}, "outputs": [], "source": [ - "from src.utils.pcd_utils import color_pcd_by_labels\n", + "# visualisation of segmentation results. masks will be drawn on the processed pcd before voxelisation\n", "\n", - "pcd_src_colored = color_pcd_by_labels(pcd_for_clustering, points2instances_pcd_for_clustering[:, 2])\n", - "visualize_pcd(pcd_src_colored)" + "from src.utils.pcd_utils import color_pcd_by_clusters_and_voxels\n", + "from src.utils.pcd_utils import visualize_pcd\n", + "\n", + "pcd_colored = color_pcd_by_clusters_and_voxels(pcd_for_clustering, trace, clusters)\n", + "visualize_pcd(pcd_colored)" ] } ], diff --git a/example_gt.ipynb b/example_gt.ipynb index 64b17bc..35f6b44 100644 --- a/example_gt.ipynb +++ b/example_gt.ipynb @@ -8,10 +8,10 @@ "source": [ "from src.datasets.kitti_dataset import KittiDataset\n", "\n", - "dataset_path = \"dataset/\"\n", + "dataset_path = \"dataset/\" # kitti\n", "sequence = \"00\"\n", - "image_instances_path = \"pipeline/vfm-labelss/sam/00/\"\n", - "gt_labels_path = \"dataset/sequences/00/labels/\"\n", + "image_instances_path = \"pipeline/vfm-labelss/sam/00/\" # images processed by the SAM algorithm in npz format\n", + "gt_labels_path = \"dataset/sequences/00/labels/\" # ground true kitti cloud segmentation\n", "kitti = KittiDataset(dataset_path, sequence, image_instances_path)" ] }, @@ -21,16 +21,18 @@ "metadata": {}, "outputs": [], "source": [ + "# setting parameter values\n", + "\n", "from src.services.preprocessing.common.config import ConfigDTO\n", "\n", "config = ConfigDTO(\n", " **{\n", " \"dataset\": kitti,\n", - " \"start_index\": 20,\n", - " \"end_index\": 24,\n", - " \"start_image_index_offset\": 3,\n", + " \"start_index\": 2024,\n", + " \"end_index\": 2028,\n", + " \"start_image_index_offset\": 0,\n", " \"alpha_physical_distance\": 5,\n", - " \"beta_instance_distance\": 3,\n", + " \"beta_instance_distance\": 5,\n", " \"T_normalized_cut\": 0.02,\n", " \"reduce_detail_int_to_union_threshold\": 0.5,\n", " \"reduce_detail_int_to_mask_threshold\": 0.6,\n", @@ -49,6 +51,8 @@ "metadata": {}, "outputs": [], "source": [ + "# pcd initialisation and initial segmentation based on images\n", + "\n", "from src.services.preprocessing.init.map import InitMapProcessor\n", "from src.services.preprocessing.init.instances_matrix import InitInstancesMatrixProcessor\n", "\n", @@ -62,13 +66,13 @@ "metadata": {}, "outputs": [], "source": [ + "# building an array of gt instance segmentation for each sequence pcd\n", + "\n", "from src.utils.gt_utils import build_sem_inst_label_arrays\n", "\n", - "sem_label_array_src, inst_label_array_src = build_sem_inst_label_arrays(gt_labels_path,\n", - " config.start_index,\n", - " config.end_index)\n", - "print(len(sem_label_array_src) == len(init_pcd.points))\n", - "print(len(inst_label_array_src) == len(init_pcd.points))" + "_, inst_label_array_src = build_sem_inst_label_arrays(\n", + " gt_labels_path, config.start_index, config.end_index\n", + ")" ] }, { @@ -77,7 +81,15 @@ "metadata": {}, "outputs": [], "source": [ - "points2instances.shape" + "# visualisation of gt instance masks on the pcd\n", + "\n", + "import copy\n", + "\n", + "from src.utils.pcd_utils import color_pcd_by_labels\n", + "from src.utils.pcd_utils import visualize_pcd\n", + "\n", + "colored_pcd = color_pcd_by_labels(copy.deepcopy(init_pcd), inst_label_array_src)\n", + "visualize_pcd(colored_pcd)" ] }, { @@ -86,6 +98,8 @@ "metadata": {}, "outputs": [], "source": [ + "# visualisation of the initial pcd segmentation masks for a particular image\n", + "\n", "import copy\n", "\n", "from src.utils.pcd_utils import color_pcd_by_labels\n", @@ -101,6 +115,8 @@ "metadata": {}, "outputs": [], "source": [ + "# pcd handler initialisation\n", + "\n", "from src.services.preprocessing.not_zero import SelectionNotZeroProcessor\n", "from src.services.preprocessing.in_cube import SelectionInCubeProcessor\n", "from src.services.preprocessing.statistical_outlier import StatisticalOutlierProcessor\n", @@ -118,18 +134,18 @@ "metadata": {}, "outputs": [], "source": [ + "# pcd processing and saving the state\n", + "\n", "import copy\n", "\n", "pcd = copy.deepcopy(init_pcd)\n", "for processor in processors:\n", " pcd, points2instances, indices = processor.process(config, pcd, points2instances)\n", - " sem_label_array_src = sem_label_array_src[indices]\n", " inst_label_array_src = inst_label_array_src[indices]\n", "\n", "pcd_for_clustering = copy.deepcopy(pcd)\n", "points2instances_pcd_for_clustering = copy.deepcopy(points2instances)\n", - "inst_label_array_for_clustering = copy.deepcopy(inst_label_array_src)\n", - "sem_label_array_for_clustering = copy.deepcopy(sem_label_array_src)" + "inst_label_array_for_clustering = copy.deepcopy(inst_label_array_src)" ] }, { @@ -138,44 +154,16 @@ "metadata": {}, "outputs": [], "source": [ - "len(inst_label_array_for_clustering) == len(pcd_for_clustering.points)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "len(sem_label_array_for_clustering) == len(pcd_for_clustering.points)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# from src.utils.gt_utils import combine_sem_inst_labels\n", + "# visualisation of gt instance masks on the processed pcd before voxelization\n", "\n", - "# combined_label_array = combine_sem_inst_labels(sem_label_array_for_clustering,\n", - "# inst_label_array_for_clustering)\n", - "# print(len(combined_label_array) == len(pcd_for_clustering.points))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ "import copy\n", "\n", "from src.utils.pcd_utils import color_pcd_by_labels\n", "from src.utils.pcd_utils import visualize_pcd\n", "\n", - "colored_pcd_for_clustering = color_pcd_by_labels(copy.deepcopy(pcd_for_clustering),\n", - " inst_label_array_for_clustering)\n", + "colored_pcd_for_clustering = color_pcd_by_labels(\n", + " copy.deepcopy(pcd_for_clustering), inst_label_array_for_clustering\n", + ")\n", "visualize_pcd(colored_pcd_for_clustering)" ] }, @@ -185,13 +173,16 @@ "metadata": {}, "outputs": [], "source": [ + "# visualisation of the initial pcd segmentation masks before voxelization\n", + "\n", "import copy\n", "\n", "from src.utils.pcd_utils import color_pcd_by_labels\n", "from src.utils.pcd_utils import visualize_pcd\n", "\n", - "colored_pcd_for_clustering = color_pcd_by_labels(copy.deepcopy(pcd_for_clustering),\n", - " points2instances_pcd_for_clustering[:, 0])\n", + "colored_pcd_for_clustering = color_pcd_by_labels(\n", + " copy.deepcopy(pcd_for_clustering), points2instances_pcd_for_clustering[:, 0]\n", + ")\n", "visualize_pcd(colored_pcd_for_clustering)" ] }, @@ -201,6 +192,8 @@ "metadata": {}, "outputs": [], "source": [ + "# final processing step - voxelisation of the pcd\n", + "\n", "from src.services.preprocessing.voxel_down import VoxelDownProcessor\n", "\n", "pcd, points2instances, trace = VoxelDownProcessor().process(config, pcd, points2instances)" @@ -212,6 +205,8 @@ "metadata": {}, "outputs": [], "source": [ + "# calculation of distance matrix for voxelised pcd\n", + "\n", "import numpy as np\n", "\n", "from scipy.spatial.distance import cdist\n", @@ -244,6 +239,8 @@ "metadata": {}, "outputs": [], "source": [ + "# distance matrix processing\n", + "\n", "from src.services.distance.isolated import RemovingIsolatedPointsProcessor\n", "from src.services.distance.connected_component import ExtractionLargestConnectedComponentProcessor\n", "\n", @@ -273,10 +270,11 @@ "metadata": {}, "outputs": [], "source": [ + "# pcd clustering using GraphCut algorithm\n", + "\n", "from src.services.normalized_cut_service import normalized_cut\n", "\n", "eigenval = 2\n", - "\n", "clusters = normalized_cut(\n", " dist,\n", " np.array([i for i in range(len(points))], dtype=int),\n", @@ -300,14 +298,25 @@ "metadata": {}, "outputs": [], "source": [ + "clusters[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# visualisation of segmentation results. masks will be drawn on the processed pcd before voxelisation\n", + "\n", "import copy\n", "\n", "from src.utils.pcd_utils import color_pcd_by_clusters_and_voxels\n", "from src.utils.pcd_utils import visualize_pcd\n", "\n", - "colored_clusters_for_clustering = color_pcd_by_clusters_and_voxels(copy.deepcopy(pcd_for_clustering),\n", - " copy.deepcopy(trace),\n", - " clusters)\n", + "colored_clusters_for_clustering = color_pcd_by_clusters_and_voxels(\n", + " copy.deepcopy(pcd_for_clustering), copy.deepcopy(trace), clusters\n", + ")\n", "visualize_pcd(colored_clusters_for_clustering)" ] }, @@ -317,13 +326,16 @@ "metadata": {}, "outputs": [], "source": [ + "# visualisation of gt instance masks on the processed pcd before voxelization\n", + "\n", "import copy\n", "\n", "from src.utils.pcd_utils import color_pcd_by_labels\n", "from src.utils.pcd_utils import visualize_pcd\n", "\n", - "colored_pcd_for_clustering = color_pcd_by_labels(copy.deepcopy(pcd_for_clustering),\n", - " inst_label_array_for_clustering)\n", + "colored_pcd_for_clustering = color_pcd_by_labels(\n", + " copy.deepcopy(pcd_for_clustering), inst_label_array_for_clustering\n", + ")\n", "visualize_pcd(colored_pcd_for_clustering)" ] }, @@ -333,6 +345,9 @@ "metadata": {}, "outputs": [], "source": [ + "# auxiliary function for calculating metrics\n", + "# if a cluster-prediction point is in the gt label, return the mask id number in the gt label array\n", + "\n", "def find_num_in_inst_label_array(src_points, inst_label_array_for_clustering):\n", " for point in src_points:\n", " if inst_label_array_for_clustering[point] > 0:\n", @@ -346,18 +361,27 @@ "metadata": {}, "outputs": [], "source": [ - "def build_pred_inst_array(inst_label_array_for_clustering, clusters, trace, instance_threshold):\n", + "# building an array of predictions for comparison with gt instance label\n", + "# cell j stores the mask number for the j-th pcd point\n", + "\n", + "def build_pred_inst_array(\n", + " inst_label_array_for_clustering, clusters, trace, instance_threshold\n", + "):\n", " pred_inst_array = np.zeros(len(inst_label_array_for_clustering), dtype=int)\n", " free_id = 1\n", " for cluster in clusters:\n", " voxel_not_in_gt_cluster_count = 0\n", " for voxel in cluster:\n", " src_points = trace[voxel]\n", - " id = find_num_in_inst_label_array(src_points, inst_label_array_for_clustering)\n", + " id = find_num_in_inst_label_array(\n", + " src_points, inst_label_array_for_clustering\n", + " )\n", " if id == -1:\n", " voxel_not_in_gt_cluster_count += 1\n", "\n", - " cluster_in_gt_instance = ((len(cluster) - voxel_not_in_gt_cluster_count) / len(cluster)) * 100\n", + " cluster_in_gt_instance = (\n", + " (len(cluster) - voxel_not_in_gt_cluster_count) / len(cluster)\n", + " ) * 100\n", " if cluster_in_gt_instance >= instance_threshold:\n", " for voxel in cluster:\n", " src_points = trace[voxel]\n", @@ -373,7 +397,16 @@ "metadata": {}, "outputs": [], "source": [ - "pred_inst_array = build_pred_inst_array(copy.deepcopy(inst_label_array_for_clustering), clusters, copy.deepcopy(trace), 20)" + "# an array of predictions\n", + "# if instance_threshold percent or more of the cluster is in the gt instance,\n", + "# consider the cluster to be selected for comparison\n", + "\n", + "pred_inst_array = build_pred_inst_array(\n", + " copy.deepcopy(inst_label_array_for_clustering),\n", + " clusters,\n", + " copy.deepcopy(trace),\n", + " instance_threshold=30\n", + ")" ] }, { @@ -382,6 +415,8 @@ "metadata": {}, "outputs": [], "source": [ + "# metrics calculation\n", + "\n", "from evops.metrics import precision\n", "from evops.metrics import recall\n", "from evops.metrics import fScore\n", @@ -393,6 +428,44 @@ "print(\"recall={}\".format(recall(pred_labels, gt_labels, tp_condition)))\n", "print(\"fScore={}\".format(fScore(pred_labels, gt_labels, tp_condition)))" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# visualisation of prediction masks\n", + "\n", + "import copy\n", + "\n", + "from src.utils.pcd_utils import color_pcd_by_labels\n", + "from src.utils.pcd_utils import visualize_pcd\n", + "\n", + "colored_pcd_for_clustering = color_pcd_by_labels(\n", + " copy.deepcopy(pcd_for_clustering), pred_labels\n", + ")\n", + "visualize_pcd(colored_pcd_for_clustering)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# visualisation of gt instance label masks\n", + "\n", + "import copy\n", + "\n", + "from src.utils.pcd_utils import color_pcd_by_labels\n", + "from src.utils.pcd_utils import visualize_pcd\n", + "\n", + "colored_pcd_for_clustering = color_pcd_by_labels(\n", + " copy.deepcopy(pcd_for_clustering), gt_labels\n", + ")\n", + "visualize_pcd(colored_pcd_for_clustering)" + ] } ], "metadata": { diff --git a/experiment/hdbscan/main_kitti_processing_hdbscan_voxel.py b/experiment/hdbscan/main_kitti_processing_hdbscan_voxel.py new file mode 100644 index 0000000..045db87 --- /dev/null +++ b/experiment/hdbscan/main_kitti_processing_hdbscan_voxel.py @@ -0,0 +1,79 @@ +# Copyright (c) 2023, Sofia Vivdich and Anastasiia Kornilova +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import open3d as o3d +import pickle + +from sklearn.cluster import HDBSCAN + + +def segment_pcds_by_hdbscan(start_index, end_index): + + file_name = "experiment_bin/start{}_end{}.pickle".format(start_index, end_index) + + with open(file_name, "rb") as file: + data = pickle.load(file) # prepared dense cloud + + pcd_hdbscan_voxel_down_points = np.asarray(data[2]["voxel_pcd_original_points"]) + pcd_hdbscan_voxel_down = o3d.geometry.PointCloud() + pcd_hdbscan_voxel_down.points = o3d.utility.Vector3dVector( + pcd_hdbscan_voxel_down_points + ) + + clusterer = HDBSCAN() + clusters = clusterer.fit_predict(np.asarray(pcd_hdbscan_voxel_down.points)) + + return ( + { + "hdbscan_clustered_voxel_pcd_original_points": np.asarray( + pcd_hdbscan_voxel_down.points + ) + }, + {"hdbscan_voxel_trace_original": data[3]["voxel_trace_original"]}, + {"hdbscan_clusters": clusters}, + {"inst_label_array_for_clustering": data[6]["inst_label_array_for_clustering"]}, + ) + + +def process_kitti_hdbscan(from_num, to_num): + + current_from_num = from_num + step = 4 + + while current_from_num < to_num: + start_index = current_from_num + end_index = start_index + step + + result_tuple = segment_pcds_by_hdbscan(start_index, end_index) + + file_name = "experiment_bin_hdbscan/start{}_end{}.pickle".format( + start_index, end_index + ) # hdbscan results + new_file = open(file_name, "w") + new_file.close() + + with open(file_name, "wb") as file: + pickle.dump(result_tuple, file) + + print("start_index={}, end_index={} done".format(start_index, end_index)) + current_from_num = end_index + + +def main(): + process_kitti_hdbscan(from_num=0, to_num=4540) + + +if __name__ == "__main__": + main() diff --git a/experiment/hdbscan/main_kitti_processing_metrics_hdbscan.py b/experiment/hdbscan/main_kitti_processing_metrics_hdbscan.py new file mode 100644 index 0000000..f3bdf2d --- /dev/null +++ b/experiment/hdbscan/main_kitti_processing_metrics_hdbscan.py @@ -0,0 +1,170 @@ +# Copyright (c) 2023, Sofia Vivdich and Anastasiia Kornilova +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import csv +import numpy as np +import pickle + +from evops.metrics import precision +from evops.metrics import recall +from evops.metrics import fScore + + +def find_num_in_inst_label_array(src_points, inst_label_array_for_clustering): + for point in src_points: + if inst_label_array_for_clustering[point] > 0: + return inst_label_array_for_clustering[point] + return -1 + + +def build_pred_inst_array( + inst_label_array_for_clustering, clusters, trace, instance_threshold +): + pred_inst_array = np.zeros(len(inst_label_array_for_clustering), dtype=int) + free_id = 1 + for cluster in clusters: + voxel_not_in_gt_cluster_count = 0 + for voxel in cluster: + src_points = trace[voxel] + id = find_num_in_inst_label_array( + src_points, inst_label_array_for_clustering + ) + if id == -1: + voxel_not_in_gt_cluster_count += 1 + + cluster_in_gt_instance = ( + (len(cluster) - voxel_not_in_gt_cluster_count) / len(cluster) + ) * 100 + if cluster_in_gt_instance >= instance_threshold: + for voxel in cluster: + src_points = trace[voxel] + for src_point in src_points: + pred_inst_array[src_point] = free_id + free_id += 1 + return pred_inst_array + + +def convert_clusters_to_list_of_point_arrays(clusters_arr): + # labels 0 and -1 mean noise, they are equal for us + for ind, label in enumerate(clusters_arr): + if label == -1: + clusters_arr[ind] = 0 + + # initializing a list by the number of unique clusters + clusters_list = [] + for label in set(clusters_arr): + clusters_list.append([]) + + for ind, label in enumerate(clusters_arr): + clusters_list[label].append(ind) + + # converting internal lists to arrays + for i in range(len(clusters_list)): + clusters_list[i] = np.asarray(clusters_list[i]) + return clusters_list + + +def main(): + + from_num = 0 + to_num = 4540 + + instance_thresholds = [30, 50] + + for instance_threshold in instance_thresholds: + print("Start to process instance_threshold={}".format(instance_threshold)) + + current_from_num = from_num + + skipped = 0 + while current_from_num < to_num: + start_index = current_from_num + end_index = start_index + 4 + + file_name = "experiment_bin_hdbscan/start{}_end{}.pickle".format( + start_index, end_index + ) + + with open(file_name, "rb") as file: + data = pickle.load(file) + + trace = data[1]["hdbscan_voxel_trace_original"] + clusters_array = data[2]["hdbscan_clusters"] + inst_label_array_for_clustering = data[3]["inst_label_array_for_clustering"] + + if ( + inst_label_array_for_clustering.sum() == 0 + ): # there are no instances in the cloud => skip + skipped += 1 + print( + "start_index={}, end_index={} skip".format(start_index, end_index) + ) + current_from_num = end_index + continue + + clusters_list_without_noise = convert_clusters_to_list_of_point_arrays( + clusters_array + )[1:] + + pred_inst_array = build_pred_inst_array( + copy.deepcopy(inst_label_array_for_clustering), + copy.deepcopy(clusters_list_without_noise), + copy.deepcopy(trace), + instance_threshold, + ) + + pred_labels = pred_inst_array + gt_labels = inst_label_array_for_clustering + tp_condition = "iou" + precision_res = precision(pred_labels, gt_labels, tp_condition) + recall_res = recall(pred_labels, gt_labels, tp_condition) + fScore_res = fScore(pred_labels, gt_labels, tp_condition) + + gt_labels_unique = set(gt_labels) + gt_labels_unique.discard(0) + + pred_labels_unique = set(pred_labels) + pred_labels_unique.discard(0) + + with open( + "experiment_hdbscan_{}.csv".format(instance_threshold), + "a", + newline="", + ) as file: + writer = csv.writer(file) + + writer.writerow( + [ + str(start_index), + str(end_index), + str(precision_res), + str(recall_res), + str(fScore_res), + len(gt_labels_unique), + len(pred_labels_unique), + len(clusters_list_without_noise), + ] + ) + + print("start_index={}, end_index={} done".format(start_index, end_index)) + + current_from_num = end_index + + print(skipped) + print("Finish to process instance_threshold={}".format(instance_threshold)) + + +if __name__ == "__main__": + main() diff --git a/experiment/main_calc_metrics_by_csv.py b/experiment/main_calc_metrics_by_csv.py new file mode 100644 index 0000000..0eb48cf --- /dev/null +++ b/experiment/main_calc_metrics_by_csv.py @@ -0,0 +1,72 @@ +# Copyright (c) 2023, Sofia Vivdich and Anastasiia Kornilova +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv + + +def calculate_sum_ones_zeros(values): + sum, ones, zeros = 0, 0, 0 + for value in values: + sum += float(value) + if float(value) == 1.0: + ones += 1 + elif float(value) == 0.0: + zeros += 1 + return sum, ones, zeros + + +def calculate_metrics(file_name): + values_pres = [] + values_recall = [] + values_fScore = [] + with open(file_name, "r") as file: + reader = csv.DictReader(file) + for row in reader: + values_pres.append(row["precision"]) + values_recall.append(row["recall"]) + values_fScore.append(row["fScore"]) + + sum_pres, pres1, pres0 = calculate_sum_ones_zeros(values_pres) + sum_recall, recall1, recall0 = calculate_sum_ones_zeros(values_recall) + sum_fScore, fscore1, fscore0 = calculate_sum_ones_zeros(values_fScore) + + print( + "precision={}, 1={}, 0={}".format( + sum_pres / float(len(values_pres)), + pres1 / float(len(values_pres)), + pres0 / float(len(values_pres)), + ) + ) + print( + "recall={}, 1={}, 0={}".format( + sum_recall / float(len(values_recall)), + recall1 / float(len(values_recall)), + recall0 / float(len(values_recall)), + ) + ) + print( + "fscore={}, 1={}, 0={}".format( + sum_fScore / float(len(values_fScore)), + fscore1 / float(len(values_fScore)), + fscore0 / float(len(values_fScore)), + ) + ) + + +def main(): + calculate_metrics("experiment_50.csv") + + +if __name__ == "__main__": + main() diff --git a/experiment/main_kitti_processing.py b/experiment/main_kitti_processing.py new file mode 100644 index 0000000..a42809c --- /dev/null +++ b/experiment/main_kitti_processing.py @@ -0,0 +1,233 @@ +# Copyright (c) 2023, Sofia Vivdich and Anastasiia Kornilova +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import numpy as np +import pickle +import sys + +sys.path.append("/Users/sofiavivdich/proj/lidar-labelling") + +from scipy.spatial.distance import cdist + +from src.datasets.kitti_dataset import KittiDataset + +from src.services.distance.isolated import RemovingIsolatedPointsProcessor +from src.services.distance.connected_component import ( + ExtractionLargestConnectedComponentProcessor, +) +from src.services.normalized_cut_service import normalized_cut +from src.services.preprocessing.common.config import ConfigDTO +from src.services.preprocessing.init.map import InitMapProcessor +from src.services.preprocessing.init.instances_matrix import ( + InitInstancesMatrixProcessor, +) +from src.services.preprocessing.not_zero import SelectionNotZeroProcessor +from src.services.preprocessing.in_cube import SelectionInCubeProcessor +from src.services.preprocessing.statistical_outlier import StatisticalOutlierProcessor +from src.services.preprocessing.voxel_down import VoxelDownProcessor + +from src.utils.distances_utils import sam_label_distance +from src.utils.gt_utils import build_sem_inst_label_arrays + + +dataset_path = "dataset/" +sequence = "00" +image_instances_path = "pipeline/vfm-labelss/sam/00/" +gt_labels_path = "dataset/sequences/00/labels/" +kitti = KittiDataset(dataset_path, sequence, image_instances_path) + + +def build_tuple_bin_saving( + config, + pcd_for_clustering, + voxel_pcd, + voxel_src_trace, + trace, + clusters, + inst_label_array_for_clustering, + sem_label_array_for_clustering, +): + params = {} + params["alpha_physical_distance"] = config.alpha_physical_distance + params["beta_instance_distance"] = config.beta_instance_distance + params["T_normalized_cut"] = config.T_normalized_cut + params["reduce_detail_int_to_union_threshold"] = ( + config.reduce_detail_int_to_union_threshold + ) + params["reduce_detail_int_to_mask_threshold"] = ( + config.reduce_detail_int_to_mask_threshold + ) + + src_trace_arrays = [] + for int_vector in voxel_src_trace: + src_trace_arrays.append(np.asarray(int_vector)) + + trace_arrays = [] + for int_vector in trace: + trace_arrays.append(np.asarray(int_vector)) + + return ( + {"config": params}, + { + "pcd_for_clustering_before_voxelization_points": np.asarray( + pcd_for_clustering.points + ) + }, + {"voxel_pcd_original_points": np.asarray(voxel_pcd.points)}, + {"voxel_trace_original": src_trace_arrays}, + {"trace_graphcut": trace_arrays}, + {"clusters_graphcut": clusters}, + {"inst_label_array_for_clustering": inst_label_array_for_clustering}, + {"sem_label_array_for_clustering": sem_label_array_for_clustering}, + ) + + +def segment_pcds(config): + init_pcd = InitMapProcessor().process(config) + points2instances = InitInstancesMatrixProcessor().process(config, init_pcd) + + sem_label_array_src, inst_label_array_src = build_sem_inst_label_arrays( + gt_labels_path, config.start_index, config.end_index + ) + + processors = [ + SelectionNotZeroProcessor(), + SelectionInCubeProcessor(), + StatisticalOutlierProcessor(), + ] + + pcd = copy.deepcopy(init_pcd) + for processor in processors: + pcd, points2instances, indices = processor.process( + config, pcd, points2instances + ) + inst_label_array_src = inst_label_array_src[indices] + sem_label_array_src = sem_label_array_src[indices] + + pcd_for_clustering = copy.deepcopy(pcd) + points2instances_pcd_for_clustering = copy.deepcopy(points2instances) + inst_label_array_for_clustering = copy.deepcopy(inst_label_array_src) + sem_label_array_for_clustering = copy.deepcopy(sem_label_array_src) + + pcd, points2instances, trace = VoxelDownProcessor().process( + config, pcd, points2instances + ) + voxel_pcd = copy.deepcopy(pcd) + voxel_src_trace = copy.deepcopy(trace) + + points = np.asarray(pcd.points) + spatial_distance = cdist(points, points) + + dist, masks = sam_label_distance( + points2instances, + spatial_distance, + 3, + config.beta_instance_distance, + config.alpha_physical_distance, + ) + + distance_processors = [ + RemovingIsolatedPointsProcessor(), + ExtractionLargestConnectedComponentProcessor(), + ] + + for processor in distance_processors: + dist, points, trace = processor.process(dist, points, trace) + + eigenval = 2 + clusters = normalized_cut( + dist, + np.array([i for i in range(len(points))], dtype=int), + config.T_normalized_cut, + eigenval, + ) + + return build_tuple_bin_saving( + config, + copy.deepcopy(pcd_for_clustering), + copy.deepcopy(voxel_pcd), + copy.deepcopy(voxel_src_trace), + copy.deepcopy(trace), + copy.deepcopy(clusters), + copy.deepcopy(inst_label_array_for_clustering), + copy.deepcopy(sem_label_array_for_clustering), + ) + + +def process_kitti( + from_num, + to_num, + alpha_physical_distance, + beta_instance_distance, + T_normalized_cut, +): + current_from_num = from_num + + while current_from_num < to_num: + start_index = current_from_num + end_index = start_index + 4 + config = ConfigDTO( + **{ + "dataset": kitti, + "start_index": start_index, + "end_index": end_index, + "start_image_index_offset": 0, + "alpha_physical_distance": alpha_physical_distance, + "beta_instance_distance": beta_instance_distance, + "T_normalized_cut": T_normalized_cut, + "reduce_detail_int_to_union_threshold": 0.5, + "reduce_detail_int_to_mask_threshold": 0.6, + "cam_name": "cam2", + "R": 18, + "nb_neighbors": 25, + "std_ratio": 5.0, + "voxel_size": 0.25, + } + ) + + result_tuple = segment_pcds(config) + + file_name = "experiment_bin/start{}_end{}.pickle".format( + config.start_index, config.end_index + ) + new_file = open(file_name, "w") + new_file.close() + + with open(file_name, "wb") as file: + pickle.dump(result_tuple, file) + + print("start_index={}, end_index={} done".format(start_index, end_index)) + current_from_num = end_index + + +def main(): + start_pcd_num = 0 + end_pcd_num = 4540 + + alpha_physical_distance = 5 + beta_instance_distance = 5 + T_normalized_cut = 0.02 + + process_kitti( + start_pcd_num, + end_pcd_num, + alpha_physical_distance, + beta_instance_distance, + T_normalized_cut, + ) + + +if __name__ == "__main__": + main() diff --git a/experiment/main_kitti_processing_metrics.py b/experiment/main_kitti_processing_metrics.py new file mode 100644 index 0000000..a44c11f --- /dev/null +++ b/experiment/main_kitti_processing_metrics.py @@ -0,0 +1,144 @@ +# Copyright (c) 2023, Sofia Vivdich and Anastasiia Kornilova +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import csv +import numpy as np +import pickle + +from evops.metrics import precision +from evops.metrics import recall +from evops.metrics import fScore + + +def find_num_in_inst_label_array(src_points, inst_label_array_for_clustering): + for point in src_points: + if inst_label_array_for_clustering[point] > 0: + return inst_label_array_for_clustering[point] + return -1 + + +def build_pred_inst_array( + inst_label_array_for_clustering, clusters, trace, instance_threshold +): + pred_inst_array = np.zeros(len(inst_label_array_for_clustering), dtype=int) + free_id = 1 + for cluster in clusters: + voxel_not_in_gt_cluster_count = 0 + for voxel in cluster: + src_points = trace[voxel] + id = find_num_in_inst_label_array( + src_points, inst_label_array_for_clustering + ) + if id == -1: + voxel_not_in_gt_cluster_count += 1 + + cluster_in_gt_instance = ( + (len(cluster) - voxel_not_in_gt_cluster_count) / len(cluster) + ) * 100 + if cluster_in_gt_instance >= instance_threshold: + for voxel in cluster: + src_points = trace[voxel] + for src_point in src_points: + pred_inst_array[src_point] = free_id + free_id += 1 + return pred_inst_array + + +def main(): + + from_num = 0 + to_num = 4540 + + instance_thresholds = [30, 50] + + for instance_threshold in instance_thresholds: + print("Start to process instance_threshold={}".format(instance_threshold)) + + current_from_num = from_num + + skipped = 0 + while current_from_num < to_num: + start_index = current_from_num + end_index = start_index + 4 + + file_name = "experiment_bin/start{}_end{}.pickle".format( + start_index, end_index + ) + + with open(file_name, "rb") as file: + data = pickle.load(file) + + trace = data[4]["trace_graphcut"] + clusters = data[5]["clusters_graphcut"] + inst_label_array_for_clustering = data[6]["inst_label_array_for_clustering"] + + if ( + inst_label_array_for_clustering.sum() == 0 + ): # there are no instances in the cloud => skip + skipped += 1 + print( + "start_index={}, end_index={} skip".format(start_index, end_index) + ) + current_from_num = end_index + continue + + pred_inst_array = build_pred_inst_array( + copy.deepcopy(inst_label_array_for_clustering), + clusters, + copy.deepcopy(trace), + instance_threshold, + ) + + pred_labels = pred_inst_array + gt_labels = inst_label_array_for_clustering + tp_condition = "iou" + precision_res = precision(pred_labels, gt_labels, tp_condition) + recall_res = recall(pred_labels, gt_labels, tp_condition) + fScore_res = fScore(pred_labels, gt_labels, tp_condition) + + gt_labels_unique = set(gt_labels) + gt_labels_unique.discard(0) + + pred_labels_unique = set(pred_labels) + pred_labels_unique.discard(0) + + with open( + "experiment_{}.csv".format(instance_threshold), + "a", + newline="", + ) as file: + writer = csv.writer(file) + + writer.writerow( + [ + str(start_index), + str(end_index), + str(precision_res), + str(recall_res), + str(fScore_res), + len(gt_labels_unique), + len(pred_labels_unique), + len(clusters), + ] + ) + + current_from_num = end_index + + print(skipped) + print("Finish to process instance_threshold={}".format(instance_threshold)) + + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py deleted file mode 100644 index 6d632f1..0000000 --- a/main.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (c) 2023, Sofia Vivdich and Anastasiia Kornilova -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy -import numpy as np -import open3d as o3d - -from scipy.spatial.distance import cdist - -from src.datasets.kitti_dataset import KittiDataset - -from src.services.normalized_cut_service import normalized_cut - -from src.services.preprocessing.common.config import ConfigDTO -from src.services.preprocessing.init.map import InitMapProcessor -from src.services.preprocessing.init.instances_matrix import ( - InitInstancesMatrixProcessor, -) -from src.services.preprocessing.not_zero import SelectionNotZeroProcessor -from src.services.preprocessing.in_cube import SelectionInCubeProcessor -from src.services.preprocessing.statistical_outlier import StatisticalOutlierProcessor -from src.services.preprocessing.voxel_down import VoxelDownProcessor - -from src.utils.distances_utils import remove_isolated_points -from src.utils.distances_utils import sam_label_distance -from src.utils.pcd_utils import color_pcd_by_clusters_and_voxels - - -def main(): - dataset_path = "dataset/" - sequence = "00" - image_instances_path = "pipeline/vfm-labels/sam/00/" - kitti = KittiDataset(dataset_path, sequence, image_instances_path) - - config = ConfigDTO( - **{ - "dataset": kitti, - "start_index": 19, - "end_index": 23, - "start_image_index_offset": 3, - "cam_name": "cam2", - "R": 12, - "nb_neighbors": 30, - "std_ratio": 5.0, - "voxel_size": 0.25, - } - ) - - init_pcd = InitMapProcessor().process(config) - points2instances = InitInstancesMatrixProcessor().process(config, init_pcd) - - processors = [ - SelectionNotZeroProcessor(), - SelectionInCubeProcessor(), - StatisticalOutlierProcessor(), - ] - - pcd = copy.deepcopy(init_pcd) - for processor in processors: - pcd, points2instances = processor.process(config, pcd, points2instances) - - pcd_for_clustering = copy.deepcopy(pcd) - - pcd, points2instances, trace = VoxelDownProcessor().process( - config, pcd, points2instances - ) - - points = np.asarray(pcd.points) - spatial_distance = cdist(points, points) - - dist, masks = sam_label_distance( - points2instances, spatial_distance, proximity_threshold=2, beta=10, alpha=2 - ) - - dist, points, trace = remove_isolated_points(dist, points, trace) - - T = 0.2 - eigenval = 2 - clusters = normalized_cut( - dist, np.array([i for i in range(len(points))], dtype=int), T, eigenval - ) - - pcd_clustered = color_pcd_by_clusters_and_voxels( - pcd_for_clustering, trace, clusters - ) - - o3d.visualization.draw_geometries([pcd_clustered]) - - -if __name__ == "__main__": - main() diff --git a/requirements.txt b/requirements.txt index 3720fa0..21df6b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,20 @@ +attrs==23.1.0 +evops==1.0.0 +hdbscan==0.8.33 +jupyter==1.0.0 +matplotlib==3.7.2 numpy==1.25.2 +numpy-indexed==0.3.7 open3d==0.17.0 -opencv_python==4.8.1.78 -Pillow==9.4.0 -pytest==7.2.2 -scipy==1.11.3 -torch==2.0.0 -torchvision==0.15.1 +opencv-python==4.8.1.78 +overrides==7.4.0 +Pillow==10.0.0 pykitti==0.3.1 -numpy_indexed==0.3.7 -matplotlib==3.7.2 -supervision==0.14.0 -segment_anything==1.0 -zope.interface==6.1.0 \ No newline at end of file +pytest==7.4.4 +scikit-learn==1.3.0 +scipy==1.11.2 +segment-anything==1.0 +supervision==0.16.0 +torch==2.1.0 +torchvision==0.16.0 +zope.interface==6.1