FLUX XLabs IP-Adapter Support (#7157)

## Summary

This PR adds support for the XLabs IP-Adapter
(https://huggingface.co/XLabs-AI/flux-ip-adapter) in workflows. Linear
UI integration is coming in a follow-up PR. The XLabs IP-Adapter can be
installed in the Starter Models tab.

Usage tips:

- Use a `cfg_scale` value of 2.0 to 4.0
- Start with an IP-Adatper weight of ~0.6 and adjust from there.
- Set `cfg_scale_start_step = 1`
- Set `cfg_scale_end_step` to roughly the halfway point (it's
unnecessary to apply CFG to all steps, and this will improve processing
time).

Sample workflow:
<img width="976" alt="image"
src="https://github.com/user-attachments/assets/4627b459-7e5a-4703-80e7-f7575c5fce19">

Result:

![image](https://github.com/user-attachments/assets/220b6a4c-69c6-447f-8df6-8aa6a56f3b3f)

## Related Issues / Discussions

Prerequisite: #7152

## Remaining TODO:

- [ ] Update default workflows.

## QA Instructions

- [x] Test basic happy path
- [x] Test with multiple IP-Adapters (it runs, but results aren't great)
- [ ] ~Test with multiple images to a single IP-Adapter~ (this is not
supported for now)
- [ ] Test automatic runtime installation of CLIP-L, CLIP-H, and CLIP-G
image encoder models if they are not already installed.
- [ ] Test starter model installation of the XLabs FLUX IP-Adapter
- [ ] Test SD and SDXL IP-Adapters for regression.
- [ ] Check peak memory utilization.

## Merge Plan

- [ ] Merge #7152 
- [ ] Change target branch to main

## Checklist

- [x] _The PR has a short but descriptive title, suitable for a
changelog_
- [x] _Tests added / updated (if applicable)_
- [ ] _Documentation added / updated (if applicable)_

invokeai/app/invocations/denoise_latents.py

@@ -547,7 +547,9 @@ def prep_ip_adapter_image_prompts(
                 if not isinstance(single_ipa_image_fields, list):
                     single_ipa_image_fields = [single_ipa_image_fields]
 
-                single_ipa_images = [context.images.get_pil(image.image_name) for image in single_ipa_image_fields]
+                single_ipa_images = [
+                    context.images.get_pil(image.image_name, mode="RGB") for image in single_ipa_image_fields
+                ]
                 with image_encoder_model_info as image_encoder_model:
                     assert isinstance(image_encoder_model, CLIPVisionModelWithProjection)
                     # Get image embeddings from CLIP and ImageProjModel.

invokeai/app/invocations/flux_denoise.py

@@ -1,22 +1,27 @@
 from contextlib import ExitStack
 from typing import Callable, Iterator, Optional, Tuple
 
+import numpy as np
+import numpy.typing as npt
 import torch
 import torchvision.transforms as tv_transforms
 from torchvision.transforms.functional import resize as tv_resize
+from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection
 
 from invokeai.app.invocations.baseinvocation import BaseInvocation, Classification, invocation
 from invokeai.app.invocations.fields import (
     DenoiseMaskField,
     FieldDescriptions,
     FluxConditioningField,
+    ImageField,
     Input,
     InputField,
     LatentsField,
     WithBoard,
     WithMetadata,
 )
 from invokeai.app.invocations.flux_controlnet import FluxControlNetField
+from invokeai.app.invocations.ip_adapter import IPAdapterField
 from invokeai.app.invocations.model import TransformerField, VAEField
 from invokeai.app.invocations.primitives import LatentsOutput
 from invokeai.app.services.shared.invocation_context import InvocationContext
@@ -26,6 +31,8 @@
 from invokeai.backend.flux.extensions.inpaint_extension import InpaintExtension
 from invokeai.backend.flux.extensions.instantx_controlnet_extension import InstantXControlNetExtension
 from invokeai.backend.flux.extensions.xlabs_controlnet_extension import XLabsControlNetExtension
+from invokeai.backend.flux.extensions.xlabs_ip_adapter_extension import XLabsIPAdapterExtension
+from invokeai.backend.flux.ip_adapter.xlabs_ip_adapter_flux import XlabsIpAdapterFlux
 from invokeai.backend.flux.model import Flux
 from invokeai.backend.flux.sampling_utils import (
     clip_timestep_schedule_fractional,
@@ -119,6 +126,10 @@ class FluxDenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
         input=Input.Connection,
     )
 
+    ip_adapter: IPAdapterField | list[IPAdapterField] | None = InputField(
+        description=FieldDescriptions.ip_adapter, title="IP-Adapter", default=None, input=Input.Connection
+    )
+
     @torch.no_grad()
     def invoke(self, context: InvocationContext) -> LatentsOutput:
         latents = self._run_diffusion(context)
@@ -246,6 +257,14 @@ def _run_diffusion(
                 noise=noise,
             )
 
+        # Compute the IP-Adapter image prompt clip embeddings.
+        # We do this before loading other models to minimize peak memory.
+        # TODO(ryand): We should really do this in a separate invocation to benefit from caching.
+        ip_adapter_fields = self._normalize_ip_adapter_fields()
+        pos_image_prompt_clip_embeds, neg_image_prompt_clip_embeds = self._prep_ip_adapter_image_prompt_clip_embeds(
+            ip_adapter_fields, context
+        )
+
         cfg_scale = self.prep_cfg_scale(
             cfg_scale=self.cfg_scale,
             timesteps=timesteps,
@@ -301,6 +320,16 @@ def _run_diffusion(
             else:
                 raise ValueError(f"Unsupported model format: {config.format}")
 
+            # Prepare IP-Adapter extensions.
+            pos_ip_adapter_extensions, neg_ip_adapter_extensions = self._prep_ip_adapter_extensions(
+                pos_image_prompt_clip_embeds=pos_image_prompt_clip_embeds,
+                neg_image_prompt_clip_embeds=neg_image_prompt_clip_embeds,
+                ip_adapter_fields=ip_adapter_fields,
+                context=context,
+                exit_stack=exit_stack,
+                dtype=inference_dtype,
+            )
+
             x = denoise(
                 model=transformer,
                 img=x,
@@ -317,6 +346,8 @@ def _run_diffusion(
                 cfg_scale=cfg_scale,
                 inpaint_extension=inpaint_extension,
                 controlnet_extensions=controlnet_extensions,
+                pos_ip_adapter_extensions=pos_ip_adapter_extensions,
+                neg_ip_adapter_extensions=neg_ip_adapter_extensions,
             )
 
         x = unpack(x.float(), self.height, self.width)
@@ -510,6 +541,112 @@ def _prep_controlnet_extensions(
 
         return controlnet_extensions
 
+    def _normalize_ip_adapter_fields(self) -> list[IPAdapterField]:
+        if self.ip_adapter is None:
+            return []
+        elif isinstance(self.ip_adapter, IPAdapterField):
+            return [self.ip_adapter]
+        elif isinstance(self.ip_adapter, list):
+            return self.ip_adapter
+        else:
+            raise ValueError(f"Unsupported IP-Adapter type: {type(self.ip_adapter)}")
+
+    def _prep_ip_adapter_image_prompt_clip_embeds(
+        self,
+        ip_adapter_fields: list[IPAdapterField],
+        context: InvocationContext,
+    ) -> tuple[list[torch.Tensor], list[torch.Tensor]]:
+        """Run the IPAdapter CLIPVisionModel, returning image prompt embeddings."""
+        clip_image_processor = CLIPImageProcessor()
+
+        pos_image_prompt_clip_embeds: list[torch.Tensor] = []
+        neg_image_prompt_clip_embeds: list[torch.Tensor] = []
+        for ip_adapter_field in ip_adapter_fields:
+            # `ip_adapter_field.image` could be a list or a single ImageField. Normalize to a list here.
+            ipa_image_fields: list[ImageField]
+            if isinstance(ip_adapter_field.image, ImageField):
+                ipa_image_fields = [ip_adapter_field.image]
+            elif isinstance(ip_adapter_field.image, list):
+                ipa_image_fields = ip_adapter_field.image
+            else:
+                raise ValueError(f"Unsupported IP-Adapter image type: {type(ip_adapter_field.image)}")
+
+            if len(ipa_image_fields) != 1:
+                raise ValueError(
+                    f"FLUX IP-Adapter only supports a single image prompt (received {len(ipa_image_fields)})."
+                )
+
+            ipa_images = [context.images.get_pil(image.image_name, mode="RGB") for image in ipa_image_fields]
+
+            pos_images: list[npt.NDArray[np.uint8]] = []
+            neg_images: list[npt.NDArray[np.uint8]] = []
+            for ipa_image in ipa_images:
+                assert ipa_image.mode == "RGB"
+                pos_image = np.array(ipa_image)
+                # We use a black image as the negative image prompt for parity with
+                # https://github.com/XLabs-AI/x-flux-comfyui/blob/45c834727dd2141aebc505ae4b01f193a8414e38/nodes.py#L592-L593
+                # An alternative scheme would be to apply zeros_like() after calling the clip_image_processor.
+                neg_image = np.zeros_like(pos_image)
+                pos_images.append(pos_image)
+                neg_images.append(neg_image)
+
+            with context.models.load(ip_adapter_field.image_encoder_model) as image_encoder_model:
+                assert isinstance(image_encoder_model, CLIPVisionModelWithProjection)
+
+                clip_image: torch.Tensor = clip_image_processor(images=pos_images, return_tensors="pt").pixel_values
+                clip_image = clip_image.to(device=image_encoder_model.device, dtype=image_encoder_model.dtype)
+                pos_clip_image_embeds = image_encoder_model(clip_image).image_embeds
+
+                clip_image = clip_image_processor(images=neg_images, return_tensors="pt").pixel_values
+                clip_image = clip_image.to(device=image_encoder_model.device, dtype=image_encoder_model.dtype)
+                neg_clip_image_embeds = image_encoder_model(clip_image).image_embeds
+
+            pos_image_prompt_clip_embeds.append(pos_clip_image_embeds)
+            neg_image_prompt_clip_embeds.append(neg_clip_image_embeds)
+
+        return pos_image_prompt_clip_embeds, neg_image_prompt_clip_embeds
+
+    def _prep_ip_adapter_extensions(
+        self,
+        ip_adapter_fields: list[IPAdapterField],
+        pos_image_prompt_clip_embeds: list[torch.Tensor],
+        neg_image_prompt_clip_embeds: list[torch.Tensor],
+        context: InvocationContext,
+        exit_stack: ExitStack,
+        dtype: torch.dtype,
+    ) -> tuple[list[XLabsIPAdapterExtension], list[XLabsIPAdapterExtension]]:
+        pos_ip_adapter_extensions: list[XLabsIPAdapterExtension] = []
+        neg_ip_adapter_extensions: list[XLabsIPAdapterExtension] = []
+        for ip_adapter_field, pos_image_prompt_clip_embed, neg_image_prompt_clip_embed in zip(
+            ip_adapter_fields, pos_image_prompt_clip_embeds, neg_image_prompt_clip_embeds, strict=True
+        ):
+            ip_adapter_model = exit_stack.enter_context(context.models.load(ip_adapter_field.ip_adapter_model))
+            assert isinstance(ip_adapter_model, XlabsIpAdapterFlux)
+            ip_adapter_model = ip_adapter_model.to(dtype=dtype)
+            if ip_adapter_field.mask is not None:
+                raise ValueError("IP-Adapter masks are not yet supported in Flux.")
+            ip_adapter_extension = XLabsIPAdapterExtension(
+                model=ip_adapter_model,
+                image_prompt_clip_embed=pos_image_prompt_clip_embed,
+                weight=ip_adapter_field.weight,
+                begin_step_percent=ip_adapter_field.begin_step_percent,
+                end_step_percent=ip_adapter_field.end_step_percent,
+            )
+            ip_adapter_extension.run_image_proj(dtype=dtype)
+            pos_ip_adapter_extensions.append(ip_adapter_extension)
+
+            ip_adapter_extension = XLabsIPAdapterExtension(
+                model=ip_adapter_model,
+                image_prompt_clip_embed=neg_image_prompt_clip_embed,
+                weight=ip_adapter_field.weight,
+                begin_step_percent=ip_adapter_field.begin_step_percent,
+                end_step_percent=ip_adapter_field.end_step_percent,
+            )
+            ip_adapter_extension.run_image_proj(dtype=dtype)
+            neg_ip_adapter_extensions.append(ip_adapter_extension)
+
+        return pos_ip_adapter_extensions, neg_ip_adapter_extensions
+
     def _lora_iterator(self, context: InvocationContext) -> Iterator[Tuple[LoRAModelRaw, float]]:
         for lora in self.transformer.loras:
             lora_info = context.models.load(lora.lora)

invokeai/app/invocations/flux_ip_adapter.py

@@ -0,0 +1,89 @@
+from builtins import float
+from typing import List, Literal, Union
+
+from pydantic import field_validator, model_validator
+from typing_extensions import Self
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, Classification, invocation
+from invokeai.app.invocations.fields import InputField, UIType
+from invokeai.app.invocations.ip_adapter import (
+    CLIP_VISION_MODEL_MAP,
+    IPAdapterField,
+    IPAdapterInvocation,
+    IPAdapterOutput,
+)
+from invokeai.app.invocations.model import ModelIdentifierField
+from invokeai.app.invocations.primitives import ImageField
+from invokeai.app.invocations.util import validate_begin_end_step, validate_weights
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.model_manager.config import (
+    IPAdapterCheckpointConfig,
+    IPAdapterInvokeAIConfig,
+)
+
+
+@invocation(
+    "flux_ip_adapter",
+    title="FLUX IP-Adapter",
+    tags=["ip_adapter", "control"],
+    category="ip_adapter",
+    version="1.0.0",
+    classification=Classification.Prototype,
+)
+class FluxIPAdapterInvocation(BaseInvocation):
+    """Collects FLUX IP-Adapter info to pass to other nodes."""
+
+    # FLUXIPAdapterInvocation is based closely on IPAdapterInvocation, but with some unsupported features removed.
+
+    image: ImageField = InputField(description="The IP-Adapter image prompt(s).")
+    ip_adapter_model: ModelIdentifierField = InputField(
+        description="The IP-Adapter model.", title="IP-Adapter Model", ui_type=UIType.IPAdapterModel
+    )
+    # Currently, the only known ViT model used by FLUX IP-Adapters is ViT-L.
+    clip_vision_model: Literal["ViT-L"] = InputField(description="CLIP Vision model to use.", default="ViT-L")
+    weight: Union[float, List[float]] = InputField(
+        default=1, description="The weight given to the IP-Adapter", title="Weight"
+    )
+    begin_step_percent: float = InputField(
+        default=0, ge=0, le=1, description="When the IP-Adapter is first applied (% of total steps)"
+    )
+    end_step_percent: float = InputField(
+        default=1, ge=0, le=1, description="When the IP-Adapter is last applied (% of total steps)"
+    )
+
+    @field_validator("weight")
+    @classmethod
+    def validate_ip_adapter_weight(cls, v: float) -> float:
+        validate_weights(v)
+        return v
+
+    @model_validator(mode="after")
+    def validate_begin_end_step_percent(self) -> Self:
+        validate_begin_end_step(self.begin_step_percent, self.end_step_percent)
+        return self
+
+    def invoke(self, context: InvocationContext) -> IPAdapterOutput:
+        # Lookup the CLIP Vision encoder that is intended to be used with the IP-Adapter model.
+        ip_adapter_info = context.models.get_config(self.ip_adapter_model.key)
+        assert isinstance(ip_adapter_info, (IPAdapterInvokeAIConfig, IPAdapterCheckpointConfig))
+
+        # Note: There is a IPAdapterInvokeAIConfig.image_encoder_model_id field, but it isn't trustworthy.
+        image_encoder_starter_model = CLIP_VISION_MODEL_MAP[self.clip_vision_model]
+        image_encoder_model_id = image_encoder_starter_model.source
+        image_encoder_model_name = image_encoder_starter_model.name
+        image_encoder_model = IPAdapterInvocation.get_clip_image_encoder(
+            context, image_encoder_model_id, image_encoder_model_name
+        )
+
+        return IPAdapterOutput(
+            ip_adapter=IPAdapterField(
+                image=self.image,
+                ip_adapter_model=self.ip_adapter_model,
+                image_encoder_model=ModelIdentifierField.from_config(image_encoder_model),
+                weight=self.weight,
+                target_blocks=[],  # target_blocks is currently unused for FLUX IP-Adapters.
+                begin_step_percent=self.begin_step_percent,
+                end_step_percent=self.end_step_percent,
+                mask=None,  # mask is currently unused for FLUX IP-Adapters.
+            ),
+        )

invokeai/app/invocations/ip_adapter.py

@@ -9,6 +9,7 @@
 from invokeai.app.invocations.model import ModelIdentifierField
 from invokeai.app.invocations.primitives import ImageField
 from invokeai.app.invocations.util import validate_begin_end_step, validate_weights
+from invokeai.app.services.model_records.model_records_base import ModelRecordChanges
 from invokeai.app.services.shared.invocation_context import InvocationContext
 from invokeai.backend.model_manager.config import (
     AnyModelConfig,
@@ -17,6 +18,12 @@
     IPAdapterInvokeAIConfig,
     ModelType,
 )
+from invokeai.backend.model_manager.starter_models import (
+    StarterModel,
+    clip_vit_l_image_encoder,
+    ip_adapter_sd_image_encoder,
+    ip_adapter_sdxl_image_encoder,
+)
 
 
 class IPAdapterField(BaseModel):
@@ -55,10 +62,14 @@ class IPAdapterOutput(BaseInvocationOutput):
     ip_adapter: IPAdapterField = OutputField(description=FieldDescriptions.ip_adapter, title="IP-Adapter")
 
 
-CLIP_VISION_MODEL_MAP = {"ViT-H": "ip_adapter_sd_image_encoder", "ViT-G": "ip_adapter_sdxl_image_encoder"}
+CLIP_VISION_MODEL_MAP: dict[Literal["ViT-L", "ViT-H", "ViT-G"], StarterModel] = {
+    "ViT-L": clip_vit_l_image_encoder,
+    "ViT-H": ip_adapter_sd_image_encoder,
+    "ViT-G": ip_adapter_sdxl_image_encoder,
+}
 
 
-@invocation("ip_adapter", title="IP-Adapter", tags=["ip_adapter", "control"], category="ip_adapter", version="1.4.1")
+@invocation("ip_adapter", title="IP-Adapter", tags=["ip_adapter", "control"], category="ip_adapter", version="1.5.0")
 class IPAdapterInvocation(BaseInvocation):
     """Collects IP-Adapter info to pass to other nodes."""
 
@@ -70,7 +81,7 @@ class IPAdapterInvocation(BaseInvocation):
         ui_order=-1,
         ui_type=UIType.IPAdapterModel,
     )
-    clip_vision_model: Literal["ViT-H", "ViT-G"] = InputField(
+    clip_vision_model: Literal["ViT-H", "ViT-G", "ViT-L"] = InputField(
         description="CLIP Vision model to use. Overrides model settings. Mandatory for checkpoint models.",
         default="ViT-H",
         ui_order=2,
@@ -111,9 +122,11 @@ def invoke(self, context: InvocationContext) -> IPAdapterOutput:
             image_encoder_model_id = ip_adapter_info.image_encoder_model_id
             image_encoder_model_name = image_encoder_model_id.split("/")[-1].strip()
         else:
-            image_encoder_model_name = CLIP_VISION_MODEL_MAP[self.clip_vision_model]
+            image_encoder_starter_model = CLIP_VISION_MODEL_MAP[self.clip_vision_model]
+            image_encoder_model_id = image_encoder_starter_model.source
+            image_encoder_model_name = image_encoder_starter_model.name
 
-        image_encoder_model = self._get_image_encoder(context, image_encoder_model_name)
+        image_encoder_model = self.get_clip_image_encoder(context, image_encoder_model_id, image_encoder_model_name)
 
         if self.method == "style":
             if ip_adapter_info.base == "sd-1":
@@ -147,7 +160,10 @@ def invoke(self, context: InvocationContext) -> IPAdapterOutput:
             ),
         )
 
-    def _get_image_encoder(self, context: InvocationContext, image_encoder_model_name: str) -> AnyModelConfig:
+    @classmethod
+    def get_clip_image_encoder(
+        cls, context: InvocationContext, image_encoder_model_id: str, image_encoder_model_name: str
+    ) -> AnyModelConfig:
         image_encoder_models = context.models.search_by_attrs(
             name=image_encoder_model_name, base=BaseModelType.Any, type=ModelType.CLIPVision
         )
@@ -159,7 +175,11 @@ def _get_image_encoder(self, context: InvocationContext, image_encoder_model_nam
             )
 
             installer = context._services.model_manager.install
-            job = installer.heuristic_import(f"InvokeAI/{image_encoder_model_name}")
+            # Note: We hard-code the type to CLIPVision here because if the model contains both a CLIPVision and a
+            # CLIPText model, the probe may treat it as a CLIPText model.
+            job = installer.heuristic_import(
+                image_encoder_model_id, ModelRecordChanges(name=image_encoder_model_name, type=ModelType.CLIPVision)
+            )
             installer.wait_for_job(job, timeout=600)  # Wait for up to 10 minutes
             image_encoder_models = context.models.search_by_attrs(
                 name=image_encoder_model_name, base=BaseModelType.Any, type=ModelType.CLIPVision

invokeai/app/invocations/metadata.py

@@ -40,7 +40,7 @@ class IPAdapterMetadataField(BaseModel):
 
     image: ImageField = Field(description="The IP-Adapter image prompt.")
     ip_adapter_model: ModelIdentifierField = Field(description="The IP-Adapter model.")
-    clip_vision_model: Literal["ViT-H", "ViT-G"] = Field(description="The CLIP Vision model")
+    clip_vision_model: Literal["ViT-L", "ViT-H", "ViT-G"] = Field(description="The CLIP Vision model")
     method: Literal["full", "style", "composition"] = Field(description="Method to apply IP Weights with")
     weight: Union[float, list[float]] = Field(description="The weight given to the IP-Adapter")
     begin_step_percent: float = Field(description="When the IP-Adapter is first applied (% of total steps)")