Ryan/feat/01 lora module

demo/realtime-img2img/frontend/src/routes/+page.svelte

@@ -8,6 +8,7 @@
   import PipelineOptions from '$lib/components/PipelineOptions.svelte';
   import ControlNetConfig from '$lib/components/ControlNetConfig.svelte';
   import IPAdapterConfig from '$lib/components/IPAdapterConfig.svelte';
+  import LoRAConfig from '$lib/components/LoRAConfig.svelte';
   import BlendingControl from '$lib/components/BlendingControl.svelte';
   import PipelineHooksConfig from '$lib/components/PipelineHooksConfig.svelte';
   import ResolutionPicker from '$lib/components/ResolutionPicker.svelte';
@@ -25,6 +26,7 @@
   let pipelineInfo: PipelineInfo;
   let controlnetInfo: any = null;
   let ipadapterInfo: any = null;
+  let loraInfo: any = null;
   let imagePreprocessingInfo: any = null;
   let imagePostprocessingInfo: any = null;
   let latentPreprocessingInfo: any = null;
@@ -131,6 +133,7 @@
 
       controlnetInfo = settings.controlnet || null;
       ipadapterInfo = settings.ipadapter || null;
+      loraInfo = settings.lora || null;
 
       // Load pipeline hooks info
       try {
@@ -1025,6 +1028,11 @@
             currentWeightType={ipadapterWeightType}
           ></IPAdapterConfig>
 
+          <LoRAConfig 
+            {loraInfo}
+            on:loraConfigChanged={getSettings}
+          ></LoRAConfig>
+
           <PipelineHooksConfig 
             hookType="image_preprocessing"
             hookInfo={imagePreprocessingInfo}

demo/realtime-img2img/img2img.py

@@ -104,6 +104,7 @@ def __init__(self, args: Args, device: torch.device, torch_dtype: torch.dtype, w
         self.pipeline_mode = "img2img"  # default mode
         self.has_controlnet = False
         self.has_ipadapter = False
+        self.has_lora = False
 
         if args.controlnet_config:
             try:
@@ -115,6 +116,7 @@ def __init__(self, args: Args, device: torch.device, torch_dtype: torch.dtype, w
                 # Check what features are enabled
                 self.has_controlnet = 'controlnets' in self.config and len(self.config['controlnets']) > 0
                 self.has_ipadapter = 'ipadapters' in self.config and len(self.config['ipadapters']) > 0
+                self.has_lora = 'loras' in self.config and len(self.config['loras']) > 0
 
 
 

src/streamdiffusion/config.py

@@ -141,6 +141,14 @@ def _extract_wrapper_params(config: Dict[str, Any]) -> Dict[str, Any]:
         param_map['use_ipadapter'] = config.get('use_ipadapter', False)
         param_map['ipadapter_config'] = config.get('ipadapter_config')
 
+    # Set LoRA usage if LoRAs are configured
+    if 'loras' in config and config['loras']:
+        param_map['use_lora'] = True
+        param_map['lora_config'] = _prepare_lora_configs(config)
+    else:
+        param_map['use_lora'] = config.get('use_lora', False)
+        param_map['lora_config'] = config.get('lora_config')
+
     # Pipeline hook configurations (Phase 4: Configuration Integration)
     hook_configs = _prepare_pipeline_hook_configs(config)
     param_map.update(hook_configs)
@@ -219,6 +227,24 @@ def _prepare_ipadapter_configs(config: Dict[str, Any]) -> List[Dict[str, Any]]:
     return ipadapter_configs
 
 
+def _prepare_lora_configs(config: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Prepare LoRA configurations for wrapper"""
+    lora_configs = []
+
+    for lora_config in config['loras']:
+        lora_config_prepared = {
+            'lora_path': lora_config['lora_path'],
+            'scale': lora_config.get('scale', 1.0),
+            'enabled': lora_config.get('enabled', True),
+            'lora_type': lora_config.get('lora_type'),
+            'display_name': lora_config.get('display_name'),
+            'description': lora_config.get('description'),
+        }
+        lora_configs.append(lora_config_prepared)
+
+    return lora_configs
+
+
 def _prepare_pipeline_hook_configs(config: Dict[str, Any]) -> Dict[str, Any]:
     """Prepare pipeline hook configurations for wrapper following ControlNet/IPAdapter pattern"""
     hook_configs = {}
@@ -418,6 +444,24 @@ def _validate_config(config: Dict[str, Any]) -> None:
             if 'image_encoder_path' not in ipadapter:
                 raise ValueError(f"_validate_config: IPAdapter {i} missing required 'image_encoder_path'")
 
+    # Validate loras if present
+    if 'loras' in config:
+        if not isinstance(config['loras'], list):
+            raise ValueError("_validate_config: 'loras' must be a list")
+
+        for i, lora in enumerate(config['loras']):
+            if not isinstance(lora, dict):
+                raise ValueError(f"_validate_config: LoRA {i} must be a dictionary")
+
+            if 'lora_path' not in lora:
+                raise ValueError(f"_validate_config: LoRA {i} missing required 'lora_path'")
+
+            # Validate scale if provided
+            if 'scale' in lora:
+                scale = lora['scale']
+                if not isinstance(scale, (int, float)) or scale < 0:
+                    raise ValueError(f"_validate_config: LoRA {i} 'scale' must be a non-negative number, got {scale}")
+
     # Validate prompt blending configuration if present
     if 'prompt_blending' in config:
         blend_config = config['prompt_blending']