Add per_step diffusion measurments (#303)

optimum_benchmark/backends/base.py

@@ -55,8 +55,8 @@ def __init__(self, config: BackendConfigT):
         if self.config.library == "diffusers":
             self.logger.info("\t+ Benchmarking a Diffusers pipeline")
             self.pretrained_config = get_diffusers_pretrained_config(self.config.model, **self.config.model_kwargs)
-            self.model_shapes = extract_diffusers_shapes_from_model(self.config.model, **self.config.model_kwargs)
             self.automodel_loader = get_diffusers_auto_pipeline_class_for_task(self.config.task)
+            self.model_shapes = extract_diffusers_shapes_from_model()
             self.pretrained_processor = None
             self.generation_config = None
 

optimum_benchmark/backends/diffusers_utils.py

@@ -1,8 +1,5 @@
-import warnings
 from typing import Dict
 
-from hydra.utils import get_class
-
 from ..import_utils import is_diffusers_available
 from ..task_utils import TASKS_TO_AUTO_PIPELINE_CLASS_NAMES, map_from_synonym_task
 
@@ -34,41 +31,10 @@ def get_diffusers_pretrained_config(model: str, **kwargs) -> Dict[str, int]:
     return pipeline_config
 
 
-def extract_diffusers_shapes_from_model(model: str, **kwargs) -> Dict[str, int]:
+def extract_diffusers_shapes_from_model(**kwargs) -> Dict[str, int]:
     if not is_diffusers_available():
         raise ImportError("diffusers is not available. Please, pip install diffusers.")
 
-    model_config = get_diffusers_pretrained_config(model, **kwargs)
-
     shapes = {}
-    if "vae" in model_config:
-        vae_import_path = model_config["vae"]
-        vae_class = get_class(f"{vae_import_path[0]}.{vae_import_path[1]}")
-        vae_config = vae_class.load_config(model, subfolder="vae", **kwargs)
-        shapes["num_channels"] = vae_config["out_channels"]
-        shapes["height"] = vae_config["sample_size"]
-        shapes["width"] = vae_config["sample_size"]
-
-    elif "vae_decoder" in model_config:
-        vae_import_path = model_config["vae_decoder"]
-        vae_class = get_class(f"{vae_import_path[0]}.{vae_import_path[1]}")
-        vae_config = vae_class.load_config(model, subfolder="vae_decoder", **kwargs)
-        shapes["num_channels"] = vae_config["out_channels"]
-        shapes["height"] = vae_config["sample_size"]
-        shapes["width"] = vae_config["sample_size"]
-
-    elif "vae_encoder" in model_config:
-        vae_import_path = model_config["vae_encoder"]
-        vae_class = get_class(f"{vae_import_path[0]}.{vae_import_path[1]}")
-        vae_config = vae_class.load_config(model, subfolder="vae_encoder", **kwargs)
-        shapes["num_channels"] = vae_config["out_channels"]
-        shapes["height"] = vae_config["sample_size"]
-        shapes["width"] = vae_config["sample_size"]
-
-    else:
-        warnings.warn("Could not extract shapes [num_channels, height, width] from diffusion pipeline.")
-        shapes["num_channels"] = -1
-        shapes["height"] = -1
-        shapes["width"] = -1
 
     return shapes

optimum_benchmark/backends/ipex/backend.py

@@ -63,7 +63,6 @@ def _load_ipexmodel_from_pretrained(self) -> None:
         self.pretrained_model = self.ipexmodel_class.from_pretrained(
             self.config.model,
             export=self.config.export,
-            device=self.config.device,
             **self.config.model_kwargs,
             **self.automodel_kwargs,
         )

optimum_benchmark/scenarios/energy_star/scenario.py

@@ -137,7 +137,6 @@ def track(self, task_name: str):
             if self.config.memory:
                 context_stack.enter_context(self.memory_tracker.track())
             if self.config.latency:
-                self.latency_tracker.reset()
                 context_stack.enter_context(self.latency_tracker.track())
             yield
 
@@ -173,17 +172,17 @@ def run_dataset_preprocessing_tracking(self):
 
         if self.config.energy:
             preprocess_energy = self.energy_tracker.get_energy()
-            preprocess_volume = self.dataset_preprocess_volume
+
             self.report.preprocess_dataset.energy = preprocess_energy
             self.report.preprocess_dataset.efficiency = Efficiency.from_energy(
-                preprocess_energy, preprocess_volume, unit=PREPROCESS_EFFICIENCY_UNIT
+                preprocess_energy, self.dataset_preprocess_volume, unit=PREPROCESS_EFFICIENCY_UNIT
             )
         if self.config.latency:
             preprocess_latency = self.latency_tracker.get_latency()
-            preprocess_volume = self.dataset_preprocess_volume
+
             self.report.preprocess_dataset.latency = preprocess_latency
             self.report.preprocess_dataset.throughput = Throughput.from_latency(
-                preprocess_latency, preprocess_volume, unit=PREPROCESS_THROUGHPUT_UNIT
+                preprocess_latency, self.dataset_preprocess_volume, unit=PREPROCESS_THROUGHPUT_UNIT
             )
         if self.config.memory:
             self.report.preprocess_dataset.memory = self.memory_tracker.get_max_memory()
@@ -237,17 +236,17 @@ def run_text_generation_tracking(self):
 
         if self.config.energy:
             prefill_energy = self.energy_tracker.get_energy()
-            decode_energy = self.dataset_prefill_volume
+
             self.report.prefill.energy = prefill_energy
             self.report.prefill.efficiency = Efficiency.from_energy(
-                prefill_energy, decode_energy, unit=PREFILL_EFFICIENCY_UNIT
+                prefill_energy, self.dataset_prefill_volume, unit=PREFILL_EFFICIENCY_UNIT
             )
         if self.config.latency:
             prefill_latency = self.latency_tracker.get_latency()
-            prefill_volume = self.dataset_prefill_volume
+
             self.report.prefill.latency = prefill_latency
             self.report.prefill.throughput = Throughput.from_latency(
-                prefill_latency, prefill_volume, unit=PREFILL_THROUGHPUT_UNIT
+                prefill_latency, self.dataset_prefill_volume, unit=PREFILL_THROUGHPUT_UNIT
             )
         if self.config.memory:
             self.report.prefill.memory = self.memory_tracker.get_max_memory()
@@ -260,18 +259,18 @@ def run_text_generation_tracking(self):
         if self.config.energy:
             generate_energy = self.energy_tracker.get_energy()
             decode_energy = generate_energy - prefill_energy
-            decode_volume = self.dataset_decode_volume
+
             self.report.decode.energy = decode_energy
             self.report.decode.efficiency = Efficiency.from_energy(
-                decode_energy, decode_volume, unit=DECODE_EFFICIENCY_UNIT
+                decode_energy, self.dataset_decode_volume, unit=DECODE_EFFICIENCY_UNIT
             )
         if self.config.latency:
             generate_latency = self.latency_tracker.get_latency()
             decode_latency = generate_latency - prefill_latency
-            decode_volume = self.dataset_decode_volume
+
             self.report.decode.latency = decode_latency
             self.report.decode.throughput = Throughput.from_latency(
-                decode_latency, decode_volume, unit=DECODE_THROUGHPUT_UNIT
+                decode_latency, self.dataset_decode_volume, unit=DECODE_THROUGHPUT_UNIT
             )
         if self.config.memory:
             self.report.decode.memory = self.memory_tracker.get_max_memory()