feat(nodes): migrate cnet nodes away from controlnet_aux

invokeai/app/invocations/baseinvocation.py

@@ -60,11 +60,13 @@ class Classification(str, Enum, metaclass=MetaEnum):
     - `Stable`: The invocation, including its inputs/outputs and internal logic, is stable. You may build workflows with it, having confidence that they will not break because of a change in this invocation.
     - `Beta`: The invocation is not yet stable, but is planned to be stable in the future. Workflows built around this invocation may break, but we are committed to supporting this invocation long-term.
     - `Prototype`: The invocation is not yet stable and may be removed from the application at any time. Workflows built around this invocation may break, and we are *not* committed to supporting this invocation.
+    - `Deprecated`: The invocation is deprecated and may be removed in a future version.
     """
 
     Stable = "stable"
     Beta = "beta"
     Prototype = "prototype"
+    Deprecated = "deprecated"
 
 
 class UIConfigBase(BaseModel):

invokeai/app/invocations/canny.py

@@ -0,0 +1,34 @@
+import cv2
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
+from invokeai.app.invocations.fields import ImageField, InputField, WithBoard, WithMetadata
+from invokeai.app.invocations.primitives import ImageOutput
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.image_util.util import cv2_to_pil, pil_to_cv2
+
+
+@invocation(
+    "canny_edge_detection",
+    title="Canny Edge Detection",
+    tags=["controlnet", "canny"],
+    category="controlnet",
+    version="1.0.0",
+)
+class CannyEdgeDetectionInvocation(BaseInvocation, WithMetadata, WithBoard):
+    """Geneartes an edge map using a cv2's Canny algorithm."""
+
+    image: ImageField = InputField(description="The image to process")
+    low_threshold: int = InputField(
+        default=100, ge=0, le=255, description="The low threshold of the Canny pixel gradient (0-255)"
+    )
+    high_threshold: int = InputField(
+        default=200, ge=0, le=255, description="The high threshold of the Canny pixel gradient (0-255)"
+    )
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        image = context.images.get_pil(self.image.image_name, "RGB")
+        np_img = pil_to_cv2(image)
+        edge_map = cv2.Canny(np_img, self.low_threshold, self.high_threshold)
+        edge_map_pil = cv2_to_pil(edge_map)
+        image_dto = context.images.save(image=edge_map_pil)
+        return ImageOutput.build(image_dto)

invokeai/app/invocations/color_map.py

@@ -0,0 +1,41 @@
+import cv2
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
+from invokeai.app.invocations.fields import FieldDescriptions, ImageField, InputField, WithBoard, WithMetadata
+from invokeai.app.invocations.primitives import ImageOutput
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.image_util.util import np_to_pil, pil_to_np
+
+
+@invocation(
+    "color_map",
+    title="Color Map",
+    tags=["controlnet"],
+    category="controlnet",
+    version="1.0.0",
+)
+class ColorMapInvocation(BaseInvocation, WithMetadata, WithBoard):
+    """Generates a color map from the provided image."""
+
+    image: ImageField = InputField(description="The image to process")
+    tile_size: int = InputField(default=64, ge=1, description=FieldDescriptions.tile_size)
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        image = context.images.get_pil(self.image.image_name, "RGB")
+
+        np_image = pil_to_np(image)
+        height, width = np_image.shape[:2]
+
+        width_tile_size = min(self.tile_size, width)
+        height_tile_size = min(self.tile_size, height)
+
+        color_map = cv2.resize(
+            np_image,
+            (width // width_tile_size, height // height_tile_size),
+            interpolation=cv2.INTER_CUBIC,
+        )
+        color_map = cv2.resize(color_map, (width, height), interpolation=cv2.INTER_NEAREST)
+        color_map_pil = np_to_pil(color_map)
+
+        image_dto = context.images.save(image=color_map_pil)
+        return ImageOutput.build(image_dto)

invokeai/app/invocations/content_shuffle.py

@@ -0,0 +1,25 @@
+from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
+from invokeai.app.invocations.fields import ImageField, InputField, WithBoard, WithMetadata
+from invokeai.app.invocations.primitives import ImageOutput
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.image_util.content_shuffle import content_shuffle
+
+
+@invocation(
+    "content_shuffle",
+    title="Content Shuffle",
+    tags=["controlnet", "normal"],
+    category="controlnet",
+    version="1.0.0",
+)
+class ContentShuffleInvocation(BaseInvocation, WithMetadata, WithBoard):
+    """Shuffles the image, similar to a 'liquify' filter."""
+
+    image: ImageField = InputField(description="The image to process")
+    scale_factor: int = InputField(default=256, ge=0, description="The scale factor used for the shuffle")
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        image = context.images.get_pil(self.image.image_name, "RGB")
+        output_image = content_shuffle(input_image=image, scale_factor=self.scale_factor)
+        image_dto = context.images.save(image=output_image)
+        return ImageOutput.build(image_dto)

invokeai/app/invocations/controlnet_image_processors.py

@@ -174,6 +174,7 @@ def invoke(self, context: InvocationContext) -> ImageOutput:
     tags=["controlnet", "canny"],
     category="controlnet",
     version="1.3.3",
+    classification=Classification.Deprecated,
 )
 class CannyImageProcessorInvocation(ImageProcessorInvocation):
     """Canny edge detection for ControlNet"""
@@ -208,6 +209,7 @@ def run_processor(self, image: Image.Image) -> Image.Image:
     tags=["controlnet", "hed", "softedge"],
     category="controlnet",
     version="1.2.3",
+    classification=Classification.Deprecated,
 )
 class HedImageProcessorInvocation(ImageProcessorInvocation):
     """Applies HED edge detection to image"""
@@ -237,6 +239,7 @@ def run_processor(self, image: Image.Image) -> Image.Image:
     tags=["controlnet", "lineart"],
     category="controlnet",
     version="1.2.3",
+    classification=Classification.Deprecated,
 )
 class LineartImageProcessorInvocation(ImageProcessorInvocation):
     """Applies line art processing to image"""
@@ -259,6 +262,7 @@ def run_processor(self, image: Image.Image) -> Image.Image:
     tags=["controlnet", "lineart", "anime"],
     category="controlnet",
     version="1.2.3",
+    classification=Classification.Deprecated,
 )
 class LineartAnimeImageProcessorInvocation(ImageProcessorInvocation):
     """Applies line art anime processing to image"""
@@ -282,6 +286,7 @@ def run_processor(self, image: Image.Image) -> Image.Image:
     tags=["controlnet", "midas"],
     category="controlnet",
     version="1.2.4",
+    classification=Classification.Deprecated,
 )
 class MidasDepthImageProcessorInvocation(ImageProcessorInvocation):
     """Applies Midas depth processing to image"""
@@ -314,6 +319,7 @@ def run_processor(self, image: Image.Image) -> Image.Image:
     tags=["controlnet"],
     category="controlnet",
     version="1.2.3",
+    classification=Classification.Deprecated,
 )
 class NormalbaeImageProcessorInvocation(ImageProcessorInvocation):
     """Applies NormalBae processing to image"""
@@ -330,7 +336,12 @@ def run_processor(self, image: Image.Image) -> Image.Image:
 
 
 @invocation(
-    "mlsd_image_processor", title="MLSD Processor", tags=["controlnet", "mlsd"], category="controlnet", version="1.2.3"
+    "mlsd_image_processor",
+    title="MLSD Processor",
+    tags=["controlnet", "mlsd"],
+    category="controlnet",
+    version="1.2.3",
+    classification=Classification.Deprecated,
 )
 class MlsdImageProcessorInvocation(ImageProcessorInvocation):
     """Applies MLSD processing to image"""
@@ -353,7 +364,12 @@ def run_processor(self, image: Image.Image) -> Image.Image:
 
 
 @invocation(
-    "pidi_image_processor", title="PIDI Processor", tags=["controlnet", "pidi"], category="controlnet", version="1.2.3"
+    "pidi_image_processor",
+    title="PIDI Processor",
+    tags=["controlnet", "pidi"],
+    category="controlnet",
+    version="1.2.3",
+    classification=Classification.Deprecated,
 )
 class PidiImageProcessorInvocation(ImageProcessorInvocation):
     """Applies PIDI processing to image"""
@@ -381,6 +397,7 @@ def run_processor(self, image: Image.Image) -> Image.Image:
     tags=["controlnet", "contentshuffle"],
     category="controlnet",
     version="1.2.3",
+    classification=Classification.Deprecated,
 )
 class ContentShuffleImageProcessorInvocation(ImageProcessorInvocation):
     """Applies content shuffle processing to image"""
@@ -411,6 +428,7 @@ def run_processor(self, image: Image.Image) -> Image.Image:
     tags=["controlnet", "zoe", "depth"],
     category="controlnet",
     version="1.2.3",
+    classification=Classification.Deprecated,
 )
 class ZoeDepthImageProcessorInvocation(ImageProcessorInvocation):
     """Applies Zoe depth processing to image"""
@@ -427,6 +445,7 @@ def run_processor(self, image: Image.Image) -> Image.Image:
     tags=["controlnet", "mediapipe", "face"],
     category="controlnet",
     version="1.2.4",
+    classification=Classification.Deprecated,
 )
 class MediapipeFaceProcessorInvocation(ImageProcessorInvocation):
     """Applies mediapipe face processing to image"""
@@ -454,6 +473,7 @@ def run_processor(self, image: Image.Image) -> Image.Image:
     tags=["controlnet", "leres", "depth"],
     category="controlnet",
     version="1.2.3",
+    classification=Classification.Deprecated,
 )
 class LeresImageProcessorInvocation(ImageProcessorInvocation):
     """Applies leres processing to image"""
@@ -483,6 +503,7 @@ def run_processor(self, image: Image.Image) -> Image.Image:
     tags=["controlnet", "tile"],
     category="controlnet",
     version="1.2.3",
+    classification=Classification.Deprecated,
 )
 class TileResamplerProcessorInvocation(ImageProcessorInvocation):
     """Tile resampler processor"""
@@ -523,6 +544,7 @@ def run_processor(self, image: Image.Image) -> Image.Image:
     tags=["controlnet", "segmentanything"],
     category="controlnet",
     version="1.2.4",
+    classification=Classification.Deprecated,
 )
 class SegmentAnythingProcessorInvocation(ImageProcessorInvocation):
     """Applies segment anything processing to image"""
@@ -570,6 +592,7 @@ def show_anns(self, anns: List[Dict]):
     tags=["controlnet"],
     category="controlnet",
     version="1.2.3",
+    classification=Classification.Deprecated,
 )
 class ColorMapImageProcessorInvocation(ImageProcessorInvocation):
     """Generates a color map from the provided image"""
@@ -609,6 +632,7 @@ def run_processor(self, image: Image.Image) -> Image.Image:
     tags=["controlnet", "depth", "depth anything"],
     category="controlnet",
     version="1.1.3",
+    classification=Classification.Deprecated,
 )
 class DepthAnythingImageProcessorInvocation(ImageProcessorInvocation):
     """Generates a depth map based on the Depth Anything algorithm"""
@@ -643,6 +667,7 @@ def load_depth_anything(model_path: Path):
     tags=["controlnet", "dwpose", "openpose"],
     category="controlnet",
     version="1.1.1",
+    classification=Classification.Deprecated,
 )
 class DWOpenposeImageProcessorInvocation(ImageProcessorInvocation):
     """Generates an openpose pose from an image using DWPose"""

invokeai/app/invocations/depth_anything.py

@@ -0,0 +1,45 @@
+from typing import Literal
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
+from invokeai.app.invocations.fields import ImageField, InputField, WithBoard, WithMetadata
+from invokeai.app.invocations.primitives import ImageOutput
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.image_util.depth_anything.depth_anything_pipeline import DepthAnythingPipeline
+
+DEPTH_ANYTHING_MODEL_SIZES = Literal["large", "base", "small", "small_v2"]
+# DepthAnything V2 Small model is licensed under Apache 2.0 but not the base and large models.
+DEPTH_ANYTHING_MODELS = {
+    "large": "LiheYoung/depth-anything-large-hf",
+    "base": "LiheYoung/depth-anything-base-hf",
+    "small": "LiheYoung/depth-anything-small-hf",
+    "small_v2": "depth-anything/Depth-Anything-V2-Small-hf",
+}
+
+
+@invocation(
+    "depth_anything_depth_estimation",
+    title="Depth Anything Depth Estimation",
+    tags=["controlnet", "depth", "depth anything"],
+    category="controlnet",
+    version="1.0.0",
+)
+class DepthAnythingDepthEstimationInvocation(BaseInvocation, WithMetadata, WithBoard):
+    """Generates a depth map using a Depth Anything model."""
+
+    image: ImageField = InputField(description="The image to process")
+    model_size: DEPTH_ANYTHING_MODEL_SIZES = InputField(
+        default="small_v2", description="The size of the depth model to use"
+    )
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        model_url = DEPTH_ANYTHING_MODELS[self.model_size]
+        image = context.images.get_pil(self.image.image_name, "RGB")
+
+        loaded_model = context.models.load_remote_model(model_url, DepthAnythingPipeline.load_model)
+
+        with loaded_model as depth_anything_detector:
+            assert isinstance(depth_anything_detector, DepthAnythingPipeline)
+            depth_map = depth_anything_detector.generate_depth(image)
+
+        image_dto = context.images.save(image=depth_map)
+        return ImageOutput.build(image_dto)

invokeai/app/invocations/dw_openpose.py

@@ -0,0 +1,50 @@
+import onnxruntime as ort
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
+from invokeai.app.invocations.fields import ImageField, InputField, WithBoard, WithMetadata
+from invokeai.app.invocations.primitives import ImageOutput
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.image_util.dw_openpose import DWOpenposeDetector2
+
+
+@invocation(
+    "dw_openpose_detection",
+    title="DW Openpose Detection",
+    tags=["controlnet", "dwpose", "openpose"],
+    category="controlnet",
+    version="1.1.1",
+)
+class DWOpenposeDetectionInvocation(BaseInvocation, WithMetadata, WithBoard):
+    """Generates an openpose pose from an image using DWPose"""
+
+    image: ImageField = InputField(description="The image to process")
+    draw_body: bool = InputField(default=True)
+    draw_face: bool = InputField(default=False)
+    draw_hands: bool = InputField(default=False)
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        image = context.images.get_pil(self.image.image_name, "RGB")
+
+        onnx_det_path = context.models.download_and_cache_model(DWOpenposeDetector2.get_model_url_det())
+        onnx_pose_path = context.models.download_and_cache_model(DWOpenposeDetector2.get_model_url_pose())
+
+        loaded_session_det = context.models.load_local_model(
+            onnx_det_path, DWOpenposeDetector2.create_onnx_inference_session
+        )
+        loaded_session_pose = context.models.load_local_model(
+            onnx_pose_path, DWOpenposeDetector2.create_onnx_inference_session
+        )
+
+        with loaded_session_det as session_det, loaded_session_pose as session_pose:
+            assert isinstance(session_det, ort.InferenceSession)
+            assert isinstance(session_pose, ort.InferenceSession)
+            detector = DWOpenposeDetector2(session_det=session_det, session_pose=session_pose)
+            detected_image = detector.run(
+                image,
+                draw_face=self.draw_face,
+                draw_hands=self.draw_hands,
+                draw_body=self.draw_body,
+            )
+        image_dto = context.images.save(image=detected_image)
+
+        return ImageOutput.build(image_dto)

invokeai/app/invocations/hed.py

@@ -0,0 +1,33 @@
+from builtins import bool
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, invocation
+from invokeai.app.invocations.fields import FieldDescriptions, ImageField, InputField, WithBoard, WithMetadata
+from invokeai.app.invocations.primitives import ImageOutput
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.image_util.hed import ControlNetHED_Apache2, HEDEdgeDetector
+
+
+@invocation(
+    "hed_edge_detection",
+    title="HED Edge Detection",
+    tags=["controlnet", "hed", "softedge"],
+    category="controlnet",
+    version="1.0.0",
+)
+class HEDEdgeDetectionInvocation(BaseInvocation, WithMetadata, WithBoard):
+    """Geneartes an edge map using the HED (softedge) model."""
+
+    image: ImageField = InputField(description="The image to process")
+    scribble: bool = InputField(default=False, description=FieldDescriptions.scribble_mode)
+
+    def invoke(self, context: InvocationContext) -> ImageOutput:
+        image = context.images.get_pil(self.image.image_name, "RGB")
+        loaded_model = context.models.load_remote_model(HEDEdgeDetector.get_model_url(), HEDEdgeDetector.load_model)
+
+        with loaded_model as model:
+            assert isinstance(model, ControlNetHED_Apache2)
+            hed_processor = HEDEdgeDetector(model)
+            edge_map = hed_processor.run(image=image, scribble=self.scribble)
+
+        image_dto = context.images.save(image=edge_map)
+        return ImageOutput.build(image_dto)