feat: Get rid of EvaluationPipeline

README.md

@@ -2,7 +2,7 @@
 
 [![](https://img.shields.io/badge/Hemm-docs-blue)](https://wandb.github.io/Hemm/)
 
-Hemm is a library for performing comprehensive benchmark of text-to-image diffusion models on image quality and prompt comprehension integrated with [Weights & Biases](https://wandb.ai/site) and [Weave](https://wandb.github.io/weave/).
+Hemm is a library for performing comprehensive benchmark of text-to-image diffusion models on image quality and prompt comprehension integrated with [Weave](https://wandb.github.io/weave/), a lightweight toolkit for tracking and evaluating LLM applications, built by [Weights & Biases](https://wandb.ai/site).
 
 Hemm is highly inspired by the following projects:
 - [Holistic Evaluation of Text-To-Image Models](https://crfm.stanford.edu/helm/heim/v1.0.0/)
@@ -37,49 +37,32 @@ First, you need to publish your evaluation dataset to Weave. Check out [this tut
 Once you have a dataset on your Weave project, you can evaluate a text-to-image generation model on the metrics.
 
 ```python
-import wandb
+import asyncio
 import weave
+from hemm.metrics.vqa import MultiModalLLMEvaluationMetric
+from hemm.metrics.vqa.judges.mmllm_judges import OpenAIJudge
+from hemm.models import DiffusersModel
 
-
-from hemm.eval_pipelines import EvaluationPipeline
-from hemm.metrics.prompt_alignment import CLIPImageQualityScoreMetric, CLIPScoreMetric
-from hemm.models import BaseDiffusionModel
-
-
-# Initialize Weave and WandB
-wandb.init(project="image-quality-leaderboard", job_type="evaluation")
+# Initialize Weave
 weave.init(project_name="image-quality-leaderboard")
 
+# The `DiffusersModel` is a `weave.Model` that uses a
+# `diffusers.DiffusionPipeline` under the hood.
+# You can write your own model `weave.Model` if your
+# model is not diffusers compatible.
+model = DiffusersModel(
+    diffusion_model_name_or_path="stabilityai/stable-diffusion-2-1",
+    image_height=1024,
+    image_width=1024,
+)
 
-# Initialize the diffusion model to be evaluated as a `weave.Model` using `BaseWeaveModel`
-# The `BaseDiffusionModel` class uses a `diffusers.DiffusionPipeline` under the hood.
-# You can write your own model `weave.Model` if your model is not diffusers compatible.
-model = BaseDiffusionModel(diffusion_model_name_or_path="CompVis/stable-diffusion-v1-4")
-
-
-# Add the model to the evaluation pipeline
-evaluation_pipeline = EvaluationPipeline(model=model)
-
-
-# Add PSNR Metric to the evaluation pipeline
-psnr_metric = PSNRMetric(image_size=evaluation_pipeline.image_size)
-evaluation_pipeline.add_metric(psnr_metric)
-
-
-# Add SSIM Metric to the evaluation pipeline
-ssim_metric = SSIMMetric(image_size=evaluation_pipeline.image_size)
-evaluation_pipeline.add_metric(ssim_metric)
-
-
-# Add LPIPS Metric to the evaluation pipeline
-lpips_metric = LPIPSMetric(image_size=evaluation_pipeline.image_size)
-evaluation_pipeline.add_metric(lpips_metric)
-
+# Define the metric
+metric = MultiModalLLMEvaluationMetric(judge=OpenAIJudge())
 
 # Get the Weave dataset reference
-dataset = weave.ref("COCO:v0").get()
-
+dataset=weave.ref("Dataset:v2").get()
 
 # Evaluate!
-evaluation_pipeline(dataset=dataset)
+evaluation = weave.Evaluation(dataset=dataset, scorers=[metric])
+summary = asyncio.run(evaluation.evaluate(model))
 ```

docs/index.md

@@ -1,6 +1,6 @@
 # Hemm: Holistic Evaluation of Multi-modal Generative Models
 
-Hemm is a library for performing comprehensive benchmark of text-to-image diffusion models on image quality and prompt comprehension integrated with [Weights & Biases](https://wandb.ai/site) and [Weave](https://wandb.github.io/weave/). 
+Hemm is a library for performing comprehensive benchmark of text-to-image diffusion models on image quality and prompt comprehension integrated with [Weave](https://wandb.github.io/weave/), a lightweight toolkit for tracking and evaluating LLM applications, built by [Weights & Biases](https://wandb.ai/site).
 
 Hemm is highly inspired by the following projects:
 
@@ -39,48 +39,32 @@ First, you need to publish your evaluation dataset to Weave. Check out [this tut
 Once you have a dataset on your Weave project, you can evaluate a text-to-image generation model on the metrics.
 
 ```python
-import wandb
+import asyncio
 import weave
+from hemm.metrics.vqa import MultiModalLLMEvaluationMetric
+from hemm.metrics.vqa.judges.mmllm_judges import OpenAIJudge
+from hemm.models import DiffusersModel
 
-
-from hemm.eval_pipelines import BaseDiffusionModel, EvaluationPipeline
-from hemm.metrics.prompt_alignment import CLIPImageQualityScoreMetric, CLIPScoreMetric
-
-
-# Initialize Weave and WandB
-wandb.init(project="image-quality-leaderboard", job_type="evaluation")
+# Initialize Weave
 weave.init(project_name="image-quality-leaderboard")
 
+# The `DiffusersModel` is a `weave.Model` that uses a
+# `diffusers.DiffusionPipeline` under the hood.
+# You can write your own model `weave.Model` if your
+# model is not diffusers compatible.
+model = DiffusersModel(
+    diffusion_model_name_or_path="stabilityai/stable-diffusion-2-1",
+    image_height=1024,
+    image_width=1024,
+)
 
-# Initialize the diffusion model to be evaluated as a `weave.Model` using `BaseWeaveModel`
-# The `BaseDiffusionModel` class uses a `diffusers.DiffusionPipeline` under the hood.
-# You can write your own model `weave.Model` if your model is not diffusers compatible.
-model = BaseDiffusionModel(diffusion_model_name_or_path="CompVis/stable-diffusion-v1-4")
-
-
-# Add the model to the evaluation pipeline
-evaluation_pipeline = EvaluationPipeline(model=model)
-
-
-# Add PSNR Metric to the evaluation pipeline
-psnr_metric = PSNRMetric(image_size=evaluation_pipeline.image_size)
-evaluation_pipeline.add_metric(psnr_metric)
-
-
-# Add SSIM Metric to the evaluation pipeline
-ssim_metric = SSIMMetric(image_size=evaluation_pipeline.image_size)
-evaluation_pipeline.add_metric(ssim_metric)
-
-
-# Add LPIPS Metric to the evaluation pipeline
-lpips_metric = LPIPSMetric(image_size=evaluation_pipeline.image_size)
-evaluation_pipeline.add_metric(lpips_metric)
-
+# Define the metric
+metric = MultiModalLLMEvaluationMetric(judge=OpenAIJudge())
 
 # Get the Weave dataset reference
-dataset = weave.ref("COCO:v0").get()
-
+dataset=weave.ref("Dataset:v2").get()
 
 # Evaluate!
-evaluation_pipeline(dataset=dataset)
+evaluation = weave.Evaluation(dataset=dataset, scorers=[metric])
+summary = asyncio.run(evaluation.evaluate(model))
 ```

docs/metrics/spatial_relationship.md

@@ -20,21 +20,18 @@ This module aims to implement the Spatial relationship metric described in secti
     ## Step 2: Evaluate
 
     ```python
-    import wandb
+    import asyncio
     import weave
 
-    from hemm.eval_pipelines import BaseDiffusionModel, EvaluationPipeline
+    from hemm.models import DiffusersModel
+    from hemm.metrics.spatial_relationship import SpatialRelationshipMetric2D
     from hemm.metrics.image_quality import LPIPSMetric, PSNRMetric, SSIMMetric
 
-    # Initialize Weave and WandB
-    wandb.init(project="image-quality-leaderboard", job_type="evaluation")
+    # Initialize Weave
     weave.init(project_name="image-quality-leaderboard")
 
-    # Initialize the diffusion model to be evaluated as a `weave.Model` using `BaseWeaveModel`
-    model = BaseDiffusionModel(diffusion_model_name_or_path="CompVis/stable-diffusion-v1-4")
-
-    # Add the model to the evaluation pipeline
-    evaluation_pipeline = EvaluationPipeline(model=model)
+    # Initialize the diffusion model to be evaluated as a `weave.Model`
+    model = DiffusersModel(diffusion_model_name_or_path="CompVis/stable-diffusion-v1-4")
 
     # Define the judge model for 2d spatial relationship metric
     judge = DETRSpatialRelationShipJudge(
@@ -43,10 +40,11 @@ This module aims to implement the Spatial relationship metric described in secti
 
     # Add 2d spatial relationship Metric to the evaluation pipeline
     metric = SpatialRelationshipMetric2D(judge=judge, name="2d_spatial_relationship_score")
-    evaluation_pipeline.add_metric(metric)
 
     # Evaluate!
-    evaluation_pipeline(dataset="t2i_compbench_spatial_prompts:v0")
+    dataset = weave.ref("2d-spatial-prompts-mscoco:v0").get()
+    evaluation = weave.Evaluation(dataset=dataset, scorers=[metric])
+    summary = asyncio.run(evaluation.evaluate(model))
     ```
 
 ## Metrics

docs/metrics/vqa/disentangled_vqa.md

@@ -33,25 +33,29 @@ This module aims to implement the Disentangled VQA metric inspired by Section 4.
     ## Step 2: Evaluate
 
     ```python
-    import wandb
+    import asyncio
+
     import weave
 
-    wandb.init(project=project, entity=entity, job_type="evaluation")
+    from hemm.metrics.vqa import DisentangledVQAMetric
+    from hemm.metrics.vqa.judges import BlipVQAJudge
+    from hemm.models import DiffusersModel
+
     weave.init(project_name=project)
 
-    diffusion_model = BaseDiffusionModel(
+    diffusion_model = DiffusersModel(
         diffusion_model_name_or_path=diffusion_model_address,
         enable_cpu_offfload=diffusion_model_enable_cpu_offfload,
         image_height=image_size[0],
         image_width=image_size[1],
     )
-    evaluation_pipeline = EvaluationPipeline(model=diffusion_model)
 
     judge = BlipVQAJudge()
     metric = DisentangledVQAMetric(judge=judge, name="disentangled_blip_metric")
     evaluation_pipeline.add_metric(metric)
 
-    evaluation_pipeline(dataset=dataset)
+    evaluation = weave.Evaluation(dataset=dataset, scorers=[metric])
+    asyncio.run(evaluation.evaluate(model))
     ```
 
 ## Metrics

docs/metrics/vqa/multi_modal_llm.md

@@ -22,31 +22,25 @@ This module aims to implement the Multi-modal LLM based metric inspired by
     ```
     Finallly, you can run the following snippet to evaluate your model:
     ```python  
-    import wandb
+    import asyncio
+
     import weave
 
-    from hemm.eval_pipelines import BaseDiffusionModel, EvaluationPipeline
     from hemm.metrics.vqa import MultiModalLLMEvaluationMetric
-    from hemm.metrics.vqa.judges.mmllm_judges import OpenAIJudge, PromptCategory
-
-    wandb.init(project="mllm-eval", job_type="evaluation")
-    weave.init(project_name="mllm-eval")
+    from hemm.metrics.vqa.judges.mmllm_judges import OpenAIJudge
+    from hemm.models import DiffusersModel
 
-    dataset = weave.ref(dataset_ref).get()
+    weave.init(project_name="hemm-eval/mllm-eval")
 
-    diffusion_model = BaseDiffusionModel(
+    model = DiffusersModel(
         diffusion_model_name_or_path="stabilityai/stable-diffusion-2-1",
-        enable_cpu_offfload=False,
-        image_height=512,
-        image_width=512,
+        image_height=1024,
+        image_width=1024,
     )
-    evaluation_pipeline = EvaluationPipeline(model=diffusion_model)
-
-    judge = OpenAIJudge(prompt_property=PromptCategory.complex)
-    metric = MultiModalLLMEvaluationMetric(judge=judge)
-    evaluation_pipeline.add_metric(metric)
+    metric = MultiModalLLMEvaluationMetric(judge=OpenAIJudge())
 
-    evaluation_pipeline(dataset=dataset)
+    evaluation = weave.Evaluation(dataset=weave.ref("Dataset:v2").get(), scorers=[metric])
+    asyncio.run(evaluation.evaluate(model))
     ```
 
 ## Metrics

examples/2d_spatial_eval/evaluate_spatial_relationship_detr.py

@@ -1,13 +1,12 @@
-from typing import Optional, Tuple
+import asyncio
+from typing import Optional
 
 import fire
 import weave
 
-import wandb
-from hemm.eval_pipelines import EvaluationPipeline
 from hemm.metrics.spatial_relationship import SpatialRelationshipMetric2D
 from hemm.metrics.spatial_relationship.judges import DETRSpatialRelationShipJudge
-from hemm.models import BaseDiffusionModel
+from hemm.models import DiffusersModel
 
 
 def main(
@@ -17,33 +16,31 @@ def main(
     dataset_limit: Optional[int] = None,
     diffusion_model_address: str = "stabilityai/stable-diffusion-2-1",
     diffusion_model_enable_cpu_offfload: bool = False,
-    image_size: Tuple[int, int] = (1024, 1024),
+    image_height: int = 1024,
+    image_width: int = 1024,
     detr_model_address: str = "facebook/detr-resnet-50",
     detr_revision: str = "no_timm",
+    iou_threshold: Optional[float] = 0.1,
 ):
-    wandb.init(project=project, entity=entity, job_type="evaluation")
     weave.init(project_name=f"{entity}/{project}")
 
     dataset = weave.ref(dataset_ref).get()
     dataset = dataset.rows[:dataset_limit] if dataset_limit else dataset
 
-    diffusion_model = BaseDiffusionModel(
+    model = DiffusersModel(
         diffusion_model_name_or_path=diffusion_model_address,
         enable_cpu_offfload=diffusion_model_enable_cpu_offfload,
-        image_height=image_size[0],
-        image_width=image_size[1],
+        image_height=image_height,
+        image_width=image_width,
     )
-    evaluation_pipeline = EvaluationPipeline(model=diffusion_model)
 
     judge = DETRSpatialRelationShipJudge(
         model_address=detr_model_address, revision=detr_revision
     )
-    metric = SpatialRelationshipMetric2D(
-        judge=judge, name="2d_spatial_relationship_score"
-    )
-    evaluation_pipeline.add_metric(metric)
+    metric = SpatialRelationshipMetric2D(judge=judge, iou_threshold=iou_threshold)
 
-    evaluation_pipeline(dataset=dataset)
+    evaluation = weave.Evaluation(dataset=dataset, scorers=[metric])
+    asyncio.run(evaluation.evaluate(model))
 
 
 if __name__ == "__main__":

examples/disentangled_vqa/evaluate_disentangled_vqa.py

@@ -1,13 +1,12 @@
-from typing import Optional, Tuple
+import asyncio
+from typing import Optional
 
 import fire
 import weave
 
-import wandb
-from hemm.eval_pipelines import EvaluationPipeline
 from hemm.metrics.vqa import DisentangledVQAMetric
 from hemm.metrics.vqa.judges import BlipVQAJudge
-from hemm.models import BaseDiffusionModel
+from hemm.models import DiffusersModel
 
 
 def main(
@@ -17,29 +16,27 @@ def main(
     dataset_limit: Optional[int] = None,
     diffusion_model_address: str = "stabilityai/stable-diffusion-2-1",
     diffusion_model_enable_cpu_offfload: bool = False,
-    image_size: Tuple[int, int] = (1024, 1024),
+    image_height: int = 1024,
+    image_width: int = 1024,
 ):
-    wandb.init(project=project, entity=entity, job_type="evaluation")
     weave.init(project_name=f"{entity}/{project}")
 
     dataset = weave.ref(dataset_ref).get()
     dataset = dataset.rows[:dataset_limit] if dataset_limit else dataset
 
-    diffusion_model = BaseDiffusionModel(
+    model = DiffusersModel(
         diffusion_model_name_or_path=diffusion_model_address,
         enable_cpu_offfload=diffusion_model_enable_cpu_offfload,
-        image_height=image_size[0],
-        image_width=image_size[1],
-        pipeline_configs={"variant": "fp16", "use_safetensors": True},
+        image_height=image_height,
+        image_width=image_width,
     )
-    diffusion_model._pipeline.set_progress_bar_config(disable=True)
-    evaluation_pipeline = EvaluationPipeline(model=diffusion_model)
+    model._pipeline.set_progress_bar_config(disable=True)
 
     judge = BlipVQAJudge()
     metric = DisentangledVQAMetric(judge=judge, name="disentangled_blip_metric")
-    evaluation_pipeline.add_metric(metric)
 
-    evaluation_pipeline(dataset=dataset)
+    evaluation = weave.Evaluation(dataset=dataset, scorers=[metric])
+    asyncio.run(evaluation.evaluate(model))
 
 
 if __name__ == "__main__":