diff --git a/examples/energy_star/_base_.yaml b/examples/energy_star/_base_.yaml new file mode 100644 index 00000000..57110fb1 --- /dev/null +++ b/examples/energy_star/_base_.yaml @@ -0,0 +1,20 @@ +log_report: true +print_report: true + +# hydra/cli specific settings +hydra: + run: + # define run directory + dir: runs/${name} + sweep: + # define sweep directory + dir: sweeps/${name} + subdir: ${hydra.job.override_dirname} + job: + # change working directory to the job directory + # so that artifacts are stored there + chdir: true + env_set: + # set environment variable OVERRIDE_BENCHMARKS to 1 + # to not skip benchmarks that have been run before + OVERRIDE_BENCHMARKS: 1 diff --git a/examples/energy_star/automatic_speech_recognition.yaml b/examples/energy_star/automatic_speech_recognition.yaml index dc02497c..03827067 100644 --- a/examples/energy_star/automatic_speech_recognition.yaml +++ b/examples/energy_star/automatic_speech_recognition.yaml @@ -17,12 +17,12 @@ backend: device_ids: 0 no_weights: true model: openai/whisper-large-v3 - processor: openai/whisper-large-v3 task: automatic-speech-recognition scenario: dataset_name: EnergyStarAI/ASR - num_samples: 1000 audio_column_name: audio + num_samples: 1000 + input_shapes: batch_size: 1 diff --git a/examples/energy_star/image_classification.yaml b/examples/energy_star/image_classification.yaml index cba83b24..57b81000 100644 --- a/examples/energy_star/image_classification.yaml +++ b/examples/energy_star/image_classification.yaml @@ -16,12 +16,13 @@ backend: device: cuda device_ids: 0 no_weights: true - model: google/vit-base-patch16-224 task: image-classification + model: google/vit-base-patch16-224 scenario: dataset_name: EnergyStarAI/image_classification - num_samples: 1000 image_column_name: image + num_samples: 1000 + input_shapes: batch_size: 1 diff --git a/examples/energy_star/image_generation_xl.yaml b/examples/energy_star/image_generation_xl.yaml deleted file mode 100644 index f591a650..00000000 --- a/examples/energy_star/image_generation_xl.yaml +++ /dev/null @@ -1,27 +0,0 @@ -defaults: - - benchmark - - backend: pytorch - - launcher: process - - scenario: energy_star - - _base_ - - _self_ - -experiment_name: image_generation - -launcher: - device_isolation: true - device_isolation_action: warn - -backend: - device: cuda - device_ids: 0 - no_weights: false - model: ByteDance/SDXL-Lightning - task: stable-diffusion - -scenario: - dataset_name: EnergyStarAI/image_generation - num_samples: 1000 - text_column_name: prompt - input_shapes: - batch_size: 1 diff --git a/examples/energy_star/image_to_text.yaml b/examples/energy_star/image_to_text.yaml index c3ec4d94..6e9ee839 100644 --- a/examples/energy_star/image_to_text.yaml +++ b/examples/energy_star/image_to_text.yaml @@ -16,13 +16,13 @@ backend: device: cuda device_ids: 0 no_weights: true - model: sashakunitsyn/vlrm-blip2-opt-2.7b - processor: sashakunitsyn/vlrm-blip2-opt-2.7b task: image-to-text + model: sashakunitsyn/vlrm-blip2-opt-2.7b scenario: dataset_name: EnergyStarAI/image_captioning - num_samples: 1000 image_column_name: image + num_samples: 1000 + input_shapes: batch_size: 1 diff --git a/examples/energy_star/object_detection.yaml b/examples/energy_star/object_detection.yaml index 266b3119..3598b3c5 100644 --- a/examples/energy_star/object_detection.yaml +++ b/examples/energy_star/object_detection.yaml @@ -16,13 +16,13 @@ backend: device: cuda device_ids: 0 no_weights: true - model: facebook/detr-resnet-50 - processor: facebook/detr-resnet-50 task: object-detection + model: facebook/detr-resnet-50 scenario: dataset_name: EnergyStarAI/object_detection - num_samples: 1000 image_column_name: image + num_samples: 1000 + input_shapes: batch_size: 1 diff --git a/examples/energy_star/question_answering.yaml b/examples/energy_star/question_answering.yaml index e587ec76..480737cc 100644 --- a/examples/energy_star/question_answering.yaml +++ b/examples/energy_star/question_answering.yaml @@ -16,13 +16,14 @@ backend: device: cuda device_ids: 0 no_weights: true - model: deepset/electra-base-squad2 task: question-answering + model: deepset/electra-base-squad2 scenario: dataset_name: EnergyStarAI/extractive_qa - num_samples: 1000 question_column_name: question context_column_name: context + num_samples: 1000 + input_shapes: batch_size: 1 diff --git a/examples/energy_star/sentence_similarity.yaml b/examples/energy_star/sentence_similarity.yaml index 471e0b88..341632c0 100644 --- a/examples/energy_star/sentence_similarity.yaml +++ b/examples/energy_star/sentence_similarity.yaml @@ -16,14 +16,15 @@ backend: device: cuda device_ids: 0 no_weights: true - model: izhx/udever-bloom-7b1 - task: sentence-similarity library: transformers + task: sentence-similarity + model: izhx/udever-bloom-7b1 scenario: dataset_name: EnergyStarAI/sentence_similarity - num_samples: 1000 sentence1_column_name: sentence1 sentence2_column_name: sentence2 + num_samples: 1000 + input_shapes: batch_size: 1 diff --git a/examples/energy_star/summarization.yaml b/examples/energy_star/summarization.yaml index 0adffedb..1811420d 100644 --- a/examples/energy_star/summarization.yaml +++ b/examples/energy_star/summarization.yaml @@ -16,14 +16,15 @@ backend: device: cuda device_ids: 0 no_weights: true - model: sshleifer/distilbart-cnn-12-6 task: summarization + model: sshleifer/distilbart-cnn-12-6 scenario: dataset_name: EnergyStarAI/summarization - num_samples: 1000 text_column_name: text + num_samples: 1000 truncation: True + input_shapes: batch_size: 1 diff --git a/examples/energy_star/question_answering_t5.yaml b/examples/energy_star/t5_question_answering.yaml similarity index 80% rename from examples/energy_star/question_answering_t5.yaml rename to examples/energy_star/t5_question_answering.yaml index d3470c9e..8d30c5ce 100644 --- a/examples/energy_star/question_answering_t5.yaml +++ b/examples/energy_star/t5_question_answering.yaml @@ -21,12 +21,12 @@ backend: scenario: dataset_name: EnergyStarAI/extractive_qa - num_samples: 1000 - text_column_name: question question_column_name: question context_column_name: context + dataset_prefix1: "question: " + dataset_prefix2: " context: " + t5_task: question-answering + num_samples: 1000 + input_shapes: batch_size: 1 - dataset_prefix1: 'question: ' - dataset_prefix2: ' context: ' - t5_task: question_answering diff --git a/examples/energy_star/summarization_t5.yaml b/examples/energy_star/t5_summarization.yaml similarity index 94% rename from examples/energy_star/summarization_t5.yaml rename to examples/energy_star/t5_summarization.yaml index 98023b11..16066645 100644 --- a/examples/energy_star/summarization_t5.yaml +++ b/examples/energy_star/t5_summarization.yaml @@ -21,13 +21,14 @@ backend: scenario: dataset_name: EnergyStarAI/summarization - num_samples: 1000 + dataset_prefix1: "summarize: " text_column_name: text + t5_task: summarization + num_samples: 1000 truncation: True + input_shapes: batch_size: 1 - dataset_prefix1: 'summarize: ' - t5_task: summarization generate_kwargs: max_new_tokens: 10 diff --git a/examples/energy_star/text_classification_t5.yaml b/examples/energy_star/t5_text_classification.yaml similarity index 88% rename from examples/energy_star/text_classification_t5.yaml rename to examples/energy_star/t5_text_classification.yaml index a0f08c02..86da8c66 100644 --- a/examples/energy_star/text_classification_t5.yaml +++ b/examples/energy_star/t5_text_classification.yaml @@ -21,13 +21,15 @@ backend: scenario: dataset_name: EnergyStarAI/text_classification - num_samples: 1000 + dataset_prefix1: "sst2 sentence: " + t5_task: text-classification text_column_name: text + + num_samples: 1000 truncation: True + input_shapes: batch_size: 1 - dataset_prefix1: 'sst2 sentence: ' - t5_task: text_classification generate_kwargs: max_new_tokens: 10 diff --git a/examples/energy_star/text2text_generation.yaml b/examples/energy_star/text2text_generation.yaml index 8878f012..49fbb8bd 100644 --- a/examples/energy_star/text2text_generation.yaml +++ b/examples/energy_star/text2text_generation.yaml @@ -16,18 +16,19 @@ backend: device: cuda device_ids: 0 no_weights: true - model: CohereForAI/aya-101 task: text2text-generation + model: CohereForAI/aya-101 scenario: dataset_name: EnergyStarAI/text_generation - num_samples: 1000 + t5_task: text_generation text_column_name: text + dataset_prefix1: "" + num_samples: 1000 truncation: True + input_shapes: batch_size: 1 - dataset_prefix1: "" - t5_task: text_generation generate_kwargs: max_new_tokens: 10 diff --git a/examples/energy_star/text_classification.yaml b/examples/energy_star/text_classification.yaml index 3fccb01c..c501cde7 100644 --- a/examples/energy_star/text_classification.yaml +++ b/examples/energy_star/text_classification.yaml @@ -16,14 +16,15 @@ backend: device: cuda device_ids: 0 no_weights: true + task: text-classification model: lvwerra/distilbert-imdb processor: lvwerra/distilbert-imdb - task: text-classification scenario: dataset_name: EnergyStarAI/text_classification - num_samples: 1000 text_column_name: text + num_samples: 1000 truncation: True + input_shapes: batch_size: 1 diff --git a/examples/energy_star/text_generation.yaml b/examples/energy_star/text_generation.yaml index 10ce5306..65b3e2cf 100644 --- a/examples/energy_star/text_generation.yaml +++ b/examples/energy_star/text_generation.yaml @@ -16,15 +16,16 @@ backend: device: cuda device_ids: 0 no_weights: true + task: text-generation model: facebook/opt-125m processor: facebook/opt-125m - task: text-generation scenario: dataset_name: EnergyStarAI/text_generation - num_samples: 1000 text_column_name: text + num_samples: 1000 truncation: True + input_shapes: batch_size: 1 diff --git a/examples/energy_star/text_generation_multi_gpu.yaml b/examples/energy_star/text_generation_multi_gpu.yaml deleted file mode 100644 index b36bf044..00000000 --- a/examples/energy_star/text_generation_multi_gpu.yaml +++ /dev/null @@ -1,33 +0,0 @@ -defaults: - - benchmark - - backend: pytorch - - launcher: process - - scenario: energy_star - - _base_ - - _self_ - -name: text_generation_mixtral-8x22B - -launcher: - device_isolation: true - device_isolation_action: warn - -backend: - device: cuda - device_ids: 0,1,2,3,4,5,6,7 - no_weights: true - model: mistral-community/Mixtral-8x22B-v0.1 - task: text-generation - device_map: auto - -scenario: - dataset_name: EnergyStarAI/text_generation - num_samples: 1000 - text_column_name: text - truncation: True - input_shapes: - batch_size: 1 - - generate_kwargs: - max_new_tokens: 10 - min_new_tokens: 10 diff --git a/examples/energy_star/image_generation.yaml b/examples/energy_star/text_to_image.yaml similarity index 84% rename from examples/energy_star/image_generation.yaml rename to examples/energy_star/text_to_image.yaml index 2b86bc29..1be89f8a 100644 --- a/examples/energy_star/image_generation.yaml +++ b/examples/energy_star/text_to_image.yaml @@ -6,7 +6,7 @@ defaults: - _base_ - _self_ -name: image_generation_tiny_sd +name: image_generation_tiny launcher: device_isolation: true @@ -17,11 +17,14 @@ backend: device_ids: 0 no_weights: false model: segmind/tiny-sd - task: stable-diffusion scenario: dataset_name: EnergyStarAI/image_generation - num_samples: 1000 text_column_name: prompt + num_samples: 1000 + input_shapes: batch_size: 1 + + call_kwargs: + num_images_per_prompt: 1