Comicdepth

[semjon00]

Spent some time on this: https://github.com/IVRL/ComicsDepth
The idea seems interesting, but the code is garbage and I could not replicate the results. Tried hooking up the model to our software (and use it with leres instead of midas), but the results are quite underwhelming (but maybe it is my fault)...
Patch here:

Subject: [PATCH] Comicdepth
---
Index: src/depthmap_generation.py
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/depthmap_generation.py b/src/depthmap_generation.py
--- a/src/depthmap_generation.py	(revision 16b160eff92abbcdb4e92959a5dd787a5d737d1d)
+++ b/src/depthmap_generation.py	(date 1689868890735)
@@ -72,7 +72,7 @@
 
         # model path and name
         model_dir = "./models/midas"
-        if model_type == 0:
+        if model_type in [0, 10]:
             model_dir = "./models/leres"
         # create paths to model if not present
         os.makedirs(model_dir, exist_ok=True)
@@ -84,7 +84,7 @@
         normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
 
         model = None
-        if model_type == 0:  # "res101"
+        if model_type == 0 or model_type == 10:  # "res101"
             model_path = f"{model_dir}/res101.pth"
             print(model_path)
             ensure_file_downloaded(
@@ -194,13 +194,20 @@
             conf = get_config("zoedepth_nk", "infer")
             model = build_model(conf)
 
-        model.eval()  # prepare for evaluation
+        elif model_type == 10:  # comicdepth + res101
+            import etc.comicdepth
+            comics2natural = etc.comicdepth.load_model(device)
+            model.eval()
+            model.to(device)
+            model = [model, comics2natural]
+
+        #model.eval()  # prepare for evaluation
         # optimize
-        if device == torch.device("cuda") and model_type in [0, 1, 2, 3, 4, 5, 6]:
-            model = model.to(memory_format=torch.channels_last)  # TODO: weird
-            if not self.no_half and model_type != 0 and not boost:  # TODO: zoedepth, too?
-                model = model.half()
-        model.to(device)  # to correct device
+        #if device == torch.device("cuda") and model_type in [0, 1, 2, 3, 4, 5, 6]:
+        #    model = model.to(memory_format=torch.channels_last)  # TODO: weird
+        #    if not self.no_half and model_type != 0 and not boost:  # TODO: zoedepth, too?
+        #        model = model.half()
+        #model.to(device)  # to correct device
 
         self.depth_model = model
         self.depth_model_type = model_type
@@ -238,7 +245,8 @@
             6: [256, 256],
             7: [384, 512],
             8: [384, 768],
-            9: [384, 512]
+            9: [384, 512],
+            10: [448, 448]
         }
         if model_type in sizes:
             return sizes[model_type]
@@ -257,14 +265,20 @@
             self.offloaded = False
 
     def move_models_to(self, device):
+        pass
         if self.depth_model is not None:
-            self.depth_model.to(device)
+            if not isinstance(self.depth_model, list):
+                self.depth_model = [self.depth_model]
+            for model in self.depth_model:
+                model.to(device)
         if self.pix2pix_model is not None:
             pass
             # TODO: pix2pix offloading not implemented
 
     def unload_models(self):
         if self.depth_model is not None or self.pix2pix_model is not None:
+            for model in self.depth_model if isinstance(self.depth_model, list) else [self.depth_model]:
+                del model
             del self.depth_model
             self.depth_model = None
             del self.pix2pix_model
@@ -288,6 +302,11 @@
                 raw_prediction = estimateleres(img, self.depth_model, net_width, net_height)
             elif self.depth_model_type in [7, 8, 9]:
                 raw_prediction = estimatezoedepth(input, self.depth_model, net_width, net_height)
+            elif self.depth_model_type == 10:
+                import etc
+                fake_natural = etc.comicdepth.get_fake_natural(img, self.depth_model[1], net_width, net_height, self.device)
+                fake_natural = cv2.cvtColor(np.asarray(fake_natural), cv2.COLOR_BGR2RGB) / 255.0
+                raw_prediction = estimateleres(fake_natural, self.depth_model[0], net_width, net_height)
             else:
                 raw_prediction = estimatemidas(img, self.depth_model, net_width, net_height,
                                                self.resize_mode, self.normalization, self.no_half,
Index: src/common_ui.py
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/common_ui.py b/src/common_ui.py
--- a/src/common_ui.py	(revision 16b160eff92abbcdb4e92959a5dd787a5d737d1d)
+++ b/src/common_ui.py	(date 1689862000779)
@@ -36,7 +36,8 @@
                                                       'dpt_beit_large_384 (midas 3.1)', 'dpt_large_384 (midas 3.0)',
                                                       'dpt_hybrid_384 (midas 3.0)',
                                                       'midas_v21', 'midas_v21_small',
-                                                      'zoedepth_n (indoor)', 'zoedepth_k (outdoor)', 'zoedepth_nk'],
+                                                      'zoedepth_n (indoor)', 'zoedepth_k (outdoor)', 'zoedepth_nk',
+                                                      'comicdepth + res101'],
                                              value='res101',
                                              type="index")
         with gr.Box():
Index: etc/comicdepth.py
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/etc/comicdepth.py b/etc/comicdepth.py
new file mode 100644
--- /dev/null	(date 1689868988403)
+++ b/etc/comicdepth.py	(date 1689868988403)
@@ -0,0 +1,105 @@
+import torch
+import torch.nn as nn
+import cv2
+import numpy as np
+
+
+class GeneratorResNet(nn.Module):
+    def __init__(self, input_shape=(3, 256, 256), out_channels=3, num_residual_blocks=8):
+        super(GeneratorResNet, self).__init__()
+
+        channels = input_shape[0]
+
+        # Initial convolution block
+        out_features = 64
+        model = [
+            nn.ReflectionPad2d(3),
+            # PrintLayer(),
+            nn.Conv2d(channels, out_features, 7),
+            nn.InstanceNorm2d(out_features),
+            nn.ReLU(inplace=True),
+            # PrintLayer()
+        ]
+        in_features = out_features
+
+        # Downsampling
+        for _ in range(2):
+            out_features *= 2
+            model += [
+                nn.Conv2d(in_features, out_features, 3, stride=2, padding=1),
+                nn.InstanceNorm2d(out_features),
+                nn.ReLU(inplace=True),
+                # PrintLayer()
+            ]
+            in_features = out_features
+
+        # Residual blocks
+        for _ in range(num_residual_blocks):
+            model += [ResidualBlock(out_features)]
+
+        # Upsampling
+        for _ in range(2):
+            out_features //= 2
+            model += [
+                nn.Upsample(scale_factor=2),
+                nn.Conv2d(in_features, out_features, 3, stride=1, padding=1),
+                nn.InstanceNorm2d(out_features),
+                nn.ReLU(inplace=True),
+                # PrintLayer()
+            ]
+            in_features = out_features
+
+        # Output
+        model += [nn.ReflectionPad2d(3),
+                  nn.Conv2d(out_features, out_channels, 7),
+                  nn.Sigmoid(),
+                  # PrintLayer()
+                  ]
+
+        self.model = nn.Sequential(*model)
+
+    def forward(self, x):
+        return self.model(x)
+
+
+class ResidualBlock(nn.Module):
+    def __init__(self, in_features):
+        super(ResidualBlock, self).__init__()
+
+        self.block = nn.Sequential(
+            nn.ReflectionPad2d(1),
+            nn.Conv2d(in_features, in_features, 3),
+            nn.InstanceNorm2d(in_features),
+            nn.ReLU(inplace=True),
+            nn.ReflectionPad2d(1),
+            nn.Conv2d(in_features, in_features, 3),
+            nn.InstanceNorm2d(in_features),
+        )
+
+    def forward(self, x):
+        return x + self.block(x)
+
+def load_model(device):
+    comics2natural = GeneratorResNet()
+    comics2natural = comics2natural.to(device)
+    model_path = 'C:\\StableDiffusion\\comicdepth\\translation.pth'  # TODO: path
+    checkpoint = torch.load(model_path, map_location=device)
+    comics2natural.load_state_dict(checkpoint['comics2natural_state_dict'])
+    comics2natural.eval()
+    comics2natural.to(device)
+    return comics2natural
+
+
+def get_fake_natural(img, comics2natural, net_width, net_height, device):
+    """comicdepth model generates a realistic-looking image from a comic image.
+    Well, this is what it is supposed to do, at least"""
+    # TODO net_width, net_height
+    with torch.no_grad():
+        img = np.flip(np.transpose(img, axes=(2, 0, 1)), axis=(0)).astype(np.float32)
+        tensor = torch.from_numpy(img).unsqueeze(0)
+        # Sanity test - should look regular
+        # Image.fromarray(np.transpose((tensor.squeeze(0).cpu().numpy() * 255.9).astype(np.uint8), axes=(1,2,0))).show()
+        tensor.to(device)
+        fake = comics2natural(tensor)
+
+        return fake.squeeze(0).mul(255).add_(0.5).clamp_(0, 255).permute(1, 2, 0).to("cpu", torch.uint8).numpy()

Buggy: GPU does not work, BOOST does not work, needs to be inverted, maybe incorrect.

If someone finds this interesting, the correctness could be verified - maybe it is good actually, and the fault is in the code, not the approach/model. You need translation.pth from https://drive.google.com/drive/folders/1df-jlouewbr2wLAQtlWzL4Hf5pxT2hHz

[thygate]

Could be cool to inpaint the text balloons and make each picture stereo in a comic book ..