add sahi to Megvii yolox

[wgq18]

I am working on Megvii yolox to detect small objects, but I find it hard to add sahi to Megvii yolox. Can you update code to wrap sahi to Megvii yolox? I don't like Mmdetection library and their yolox version. Thank you for your help.

[msaqib17]

I have added the following class to the model for YOLOX but i am getting a lot of False positive for sliced prediction. YOLOX is doing preprocessing and post processing that might have a issue. If the Megvii YOLOX support is added that will be great. Thanks

class YoloXDetectionModel(DetectionModel):
def init(
self,
model_path: Optional[str] = None,
model: Optional[Any] = None,
config_path: Optional[str] = None,
device: Optional[str] = None,
mask_threshold: float = 0.5,
confidence_threshold: float = 0.7,
category_mapping: Optional[Dict] = None,
category_remapping: Optional[Dict] = None,
load_at_init: bool = True,
image_size: int =None,
num_classes: str=None,
nmsthre:float=None,
classes:str=COCO_CLASSES,
):

    super().__init__(
        model_path=model_path,
        model=model,
        config_path=config_path,
        device=device,
        confidence_threshold=confidence_threshold,
        category_mapping=category_mapping,
        category_remapping=category_remapping,
        load_at_init=load_at_init,
        image_size=image_size,
    )

    self.nmsthre=0.45

def load_model(self):
    import torch
    # importing experiment file
    exp = get_exp(self.config_path, None)
    # extract the number of classes
    self.num_classes = exp.num_classes
    # the input size of test image from exp
    self.test_size = exp.input_size
    print(f"The test size is ???{self.test_size}")
    # Non maxima threshold from exp
    self.nms_thresh = exp.nmsthre
    # Loading custom model
    model = exp.get_model()  
    ckpt = torch.load(self.model_path, map_location="cpu")
    self.model = model.load_state_dict(ckpt["model"])
    print("Model loaded.......",self.model)
    self.set_model(model)
    
def set_model(self, model: Any):
    model.eval()
    self.model = model.to(self.device)

    if self.category_mapping is None:
        category_names = {str(i): COCO_CLASSES[i] for i in range(len(COCO_CLASSES))}
        self.category_mapping = category_names
        print("The category mapping",self.category_mapping)


def perform_inference(self, image:np.ndarray, image_size: int=None ): 
    bboxes = []
    bbclasses = []
    scores = []
    print(f"The shape of the input image is :::{image.shape}")
    print(f"The size of the input image is :::{image.size}")
    height, width = image.shape[:2]
    ratio = min(self.test_size[0] / image.shape[0], self.test_size[1] / image.shape[1])
    print(f"The ration is {ratio}")
    
    print(f"The height and width are: {height} and width:{width}")     
    preproc = ValTransform(legacy = False)
    
    if self.test_size is not None:
        print(f"Preprocessing by self.test_size:{self.test_size}::originally trained on")
        tensor_img, _ = preproc(image, None, self.test_size)


    tensor_img = torch.from_numpy(tensor_img).unsqueeze(0)
    tensor_img = tensor_img.float()
    tensor_img = tensor_img.cuda()


    with torch.no_grad():
        outputs = self.model(tensor_img)
        outputs = postprocess(outputs, self.num_classes, self.confidence_threshold, self.nmsthre, class_agnostic=True)
        
    if outputs[0] is None:
        return [], [], []

    outputs = outputs[0].cpu()
    bboxes = outputs[:, 0:4]

    
    bboxes/=ratio
    outputs[:,0:4] = bboxes
    bbclasses = outputs[:, 6]
    scores = outputs[:, 4] * outputs[:, 5]
    self._original_predictions =[outputs]
    return self._original_predictions

@property
def num_categories(self):
    """
    Returns number of categories
    """
    return len(self.category_mapping)

@property
def has_mask(self):
    return False

@property
def category_names(self):
    return list(self.category_mapping.values())


def _create_object_prediction_list_from_original_predictions(
    self,
    shift_amount_list: Optional[List[List[int]]] = [[0, 0]],
    full_shape_list: Optional[List[List[int]]] = None,):

    original_predictions = self._original_predictions
    shift_amount_list = fix_shift_amount_list(shift_amount_list)
    full_shape_list = fix_full_shape_list(full_shape_list)

    # handle all predictions
    object_prediction_list_per_image = []
    for image_ind, image_predictions_in_xyxy_format in enumerate(original_predictions):
        shift_amount = shift_amount_list[image_ind]
        full_shape = None if full_shape_list is None else full_shape_list[image_ind]
        object_prediction_list = []

        # process predictions
        for prediction in image_predictions_in_xyxy_format.cpu().detach().numpy():
            x1 = int(prediction[0])
            y1 = int(prediction[1])
            x2 = int(prediction[2])
            y2 = int(prediction[3])
            bbox = [x1, y1, x2, y2]
            score = prediction[4]
            category_id = int(prediction[5])
            category_name = self.category_mapping[str(category_id)]

            # ignore invalid predictions
            if bbox[0] > bbox[2] or bbox[1] > bbox[3] or bbox[0] < 0 or bbox[1] < 0 or bbox[2] < 0 or bbox[3] < 0:
                logger.warning(f"ignoring invalid prediction with bbox: {bbox}")
                continue
            if full_shape is not None and (
                bbox[1] > full_shape[0]
                or bbox[3] > full_shape[0]
                or bbox[0] > full_shape[1]
                or bbox[2] > full_shape[1]
            ):
                logger.warning(f"ignoring invalid prediction with bbox: {bbox}")
                continue

            object_prediction = ObjectPrediction(
                bbox=bbox,
                category_id=category_id,
                score=score,
                bool_mask=None,
                category_name=category_name,
                shift_amount=shift_amount,
                full_shape=full_shape,
            )
            object_prediction_list.append(object_prediction)
        object_prediction_list_per_image.append(object_prediction_list)

    self._object_prediction_list_per_image = object_prediction_list_per_image

[kadirnar]

Can you share the results? @msaqib17

[kadirnar]

Did you test the original model file? It may be related to the performance of your model.

[msaqib17]

If you see the top left side of the above image you will see original predictions. The original predictions are good.

[kadirnar]

Can you test this model?

Orijinal Model File:
https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/yolox_m.pth

Image File:
https://github.com/obss/sahi/blob/main/demo/demo_data/small-vehicles1.jpeg

I'm getting good results here.

Mmdet-YOLOX:
https://huggingface.co/spaces/fcakyon/sahi-yolox

[msaqib17]

ok thanks let me try.

[msaqib17]

This is the result of sliced prediction i am getting

[msaqib17]

unsliced prediction result

s

[msaqib17]

Can you test this model?

Orijinal Model File: https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/yolox_m.pth

Image File: https://github.com/obss/sahi/blob/main/demo/demo_data/small-vehicles1.jpeg

I'm getting good results here.

Mmdet-YOLOX: https://huggingface.co/spaces/fcakyon/sahi-yolox

For me your model is working. I am not sure why my model is not working i will train and try it again. Thanks for the support.

[kadirnar]

Your model is memorizing. You should pay attention to this while training. The better your model, the more successful SAHI will be. You can change SAHI parameters for better results.

https://github.com/obss/sahi/blob/main/sahi/predict.py#L121-L124

My Works:

https://github.com/kadirnar/yolov5-pytorch-sahi
https://github.com/kadirnar/wideryolo

I wish you success in your work.

[msaqib17]

Thank you Kadir. keep up the amazing work. Yes you are right my model is overfitted.

[wgq18]

Hello, I tested my weights on your 'model.py' script, but my yolox detection result is wrong. Why category result is wrong? This is yolox output result, sahi_yolox output result and terminal output. @msaqib17

[msaqib17]

Hi @wgq18

Your category mappings are wrong. Just look closely at the output of the model. You have trained your model on ten classes with custom labels(as seen in the output), and you are assigning labels from different COCO classes.

[wgq18]

You are right. I trained model on ten VOC classes:('pedestrian', 'people', 'bicycle', 'car', 'van', 'truck', 'tricycle', 'awning-tricycle', 'bus', 'motor'). But as you can see, when I use sahi prepocess and postprocess, I could only get 'pedestrian' class in detection result. Why this error happens?

[kadirnar]

@fcakyon Do you accept PR for Yolox-SAHI?

[wgq18]

@msaqib17 Can you test model precision and speed with sahi-yolox? I am using 'python -m yolox.tools.eval -f exps/example/yolox_darknet53/yolox_voc_darknet53.py -n yolov3 -c my_weights/best_ckpt.pth -b 4 -d 0 --conf 0.001 --fp16 --fuse --tsize 640' to test yolox now. How to test sahi-yolox?

Can you provide your code or command? Thank you.

[fcakyon]

@fcakyon Do you accept PR for Yolox-SAHI?

Yes we are open. The class name can be YoloxDetectionModel.

[msaqib17]

@wgq18 Sorry i don't have testing code for the evaluation. But you can use the following command to get Precision and Recall. For this you have to convert your dataset to coco json format and the result should also be in coco json format.

sahi coco evaluate --dataset_json_path dataset.json --result_json_path result.json

[kadirnar]

Hi @msaqib17 ,

Do you want to add the Yolox model to the SAHI library? I can help.I can do that too. What do you think?

[msaqib17]

@kadirnar Lets do it in a separate repository because my local one is quite messed up.

[kadirnar]

Let's talk about this here.
#534

[wgq18]

I want to detect many pictures in a single script. But my code can only detect one picture at a time. So how to change my code?

# sahi YoloX_0.3.0 test script

import os
os.getcwd()
import torch

# import required functions, classes

from sahi.model import YoloXDetectionModel
from sahi.utils.cv import read_image
# from sahi.utils.file import download_from_url
from sahi.predict import get_prediction, get_sliced_prediction, predict

from yolox.data.datasets.voc import VOC
from sahi.model import YoloXDetectionModel
from sahi.utils.cv import read_image
# from sahi.utils.file import download_from_url
from sahi.predict import get_prediction, get_sliced_prediction, predict

from yolox.data.datasets.voc import Vodel
detection_model = YoloXDetectionModel(
    model_path=YoloX_model_path,
    confidence_threshold=0.25,
    device='cuda:0', # 'cpu' or 'cuda:0'
    config_path='/home/wangpo/Documents/YOLOX-0.3.0/exps/example/yolox_darknet53/yolox_voc_darknet53.py',
)

# result = get_prediction("/home/wangpo/Documents/YOLOX-0.3.0/sahi/demo/demo_data/small-vehicles1.jpeg", detection_model) 
result = get_prediction("/home/wangpo/Documents/YOLOX-0.3.0/sahi/demo/demo_data/0000006_00159_d_0000001.jpg", detection_model) 

print('Test Standard Inference with a YoloX Model!')
print(result.durations_in_seconds)
print()
result.export_visuals(export_dir="/home/wangpo/Documents/YOLOX-0.3.0/sahi/runs/Standard Inference with a YoloX Model/")

# 2. Sliced Inference with a YoloX Model
result = get_sliced_prediction(
    "/home/wangpo/Documents/YOLOX-0.3.0/sahi/demo/demo_data/0000006_00159_d_0000001.jpg",
    detection_model,
    slice_height = 960,
    slice_width = 960,
    overlap_height_ratio = 0.2,
    overlap_width_ratio = 0.2
)

result.export_visuals(export_dir="/home/wangpo/Documents/YOLOX-0.3.0/sahi/runs/Sliced Inference with a YoloX Model/")

print('Test Sliced Inference with a YoloX Model!')
print(result.durations_in_seconds)

[kadirnar]

Hello @msaqib17 ,

You should use predict function.
Example:

import numpy as np
from sahi.model import YoloXDetectionModel
from sahi.utils.cv import read_image
from sahi.predict import get_prediction, get_sliced_prediction, predict

YoloXDetectionModel(
   model_path="model.pt",
   confidence_threshold=0.5,
   device="cuda:0",)

result = predict(
    source = "images/",
    model_type  = "yolox" ,
    model_path  = "models/best.pt",  
    export_visual = True,
    project = "demo_data/")

but first you have to add the yolox library here.

https://github.com/obss/sahi/blob/main/sahi/auto_model.py#L6

[wgq18]

Thank you. When I use command 'sahi coco evaluate --dataset_json_path dataset.json --result_json_path result.json', I only get classwise classes mAP, mAP50 and mAP75 metrics. But the metrics is 0.00.

I don't know why this happened and how to fix this error. Please give me some guide.

[msaqib17]

@wgq18 lower the confidence and then try again. How much is confidence you are using for your experiment? May be your results and ground-truths are not in proper coco json format. First try it with only one image result and ground-truth and when you are confident then apply it to your whole test dataset.

[fcakyon]

May be your results and ground-truths are not in proper coco json format.

This is the most probable reason 👍