Re-enable PyTorch Benchmarks

nod-ai · Oct 12, 2023 · 4e06f74 · 4e06f74
1 parent 8380941
commit 4e06f74
Show file tree

Hide file tree

Showing 4 changed files with 359 additions and 110 deletions.
diff --git a/build_tools/python/e2e_test_framework/models/model_groups.py b/build_tools/python/e2e_test_framework/models/model_groups.py
@@ -9,7 +9,7 @@
 from e2e_test_framework.models import (
     matmul,
     tflite_models,
-    # torch_models,
+    torch_models,
     tf_models,
     jax_models,
 )
@@ -65,9 +65,6 @@
     common_definitions.CpuBenchmarkConfig(
         model=tf_models.MINILM_L12_H384_UNCASED_INT32_SEQLEN128, threads=[1, 8]
     ),
-    # common_definitions.CpuBenchmarkConfig(
-    #     model=torch_models.EFFICIENTNET_V2_S_FP32_TORCH, threads=[1, 8]
-    # ),
     # Large models.
     # TODO: These models should be running at 8, 13, 28 threads but we use 8 for now until new hardware becomes available.
     common_definitions.CpuBenchmarkConfig(
@@ -76,9 +73,6 @@
     common_definitions.CpuBenchmarkConfig(
         model=tf_models.BERT_LARGE_TF_FP32_SEQLEN384, threads=[8]
     ),
-    # common_definitions.CpuBenchmarkConfig(
-    #     model=torch_models.EFFICIENTNET_B7_FP32_TORCH, threads=[8]
-    # ),
 ]
 
 # A subset of `x86_64_MODELS_AND_THREADS`.
@@ -122,16 +116,10 @@
     common_definitions.CpuBenchmarkConfig(
         model=tf_models.MINILM_L12_H384_UNCASED_INT32_SEQLEN128, threads=[8]
     ),
-    # common_definitions.CpuBenchmarkConfig(
-    #     model=torch_models.EFFICIENTNET_V2_S_FP32_TORCH, threads=[8]
-    # ),
     # Large models.
     common_definitions.CpuBenchmarkConfig(
         model=tf_models.BERT_LARGE_TF_FP32_SEQLEN384, threads=[8]
     ),
-    # common_definitions.CpuBenchmarkConfig(
-    #     model=torch_models.EFFICIENTNET_B7_FP32_TORCH, threads=[8]
-    # ),
 ]
 
 X86_64_BENCHMARK_CONFIG_LONG = [
@@ -184,34 +172,10 @@
 
 # Batched Torch models.
 
-# BERT_LARGE_TORCH_BATCHES = [
-#     model
-#     for batch_size, model in torch_models.BERT_LARGE_384_FP32_TORCH_BATCHES.items()
-#     # Higher batch sizes disabled due to OOM https://github.com/openxla/iree/issues/14668.
-#     if batch_size < 64
-# ]
-#
-# BERT_LARGE_FP16_TORCH_BATCHES = [
-#     model
-#     for batch_size, model in torch_models.BERT_LARGE_384_FP16_TORCH_BATCHES.items()
-#     # Batchsize 1 is included separately in CUDA_MODELS
-#     # Higher batch sizes disabled due to OOM https://github.com/openxla/iree/issues/14668.
-#     if batch_size != 1 and batch_size < 64
-# ]
-#
-# RESNET50_TORCH_BATCHES = [
-#     model
-#     for batch_size, model in torch_models.RESNET50_3X224X224_FP32_TORCH_BATCHES.items()
-#     # Higher batch sizes disabled due to OOM https://github.com/openxla/iree/issues/14668.
-#     if batch_size < 2048
-# ]
-#
-# RESNET50_FP16_TORCH_BATCHES = list(
-#     model
-#     for batch_size, model in torch_models.RESNET50_3X224X224_FP16_TORCH_BATCHES.items()
-#     # Higher batch sizes disabled due to OOM https://github.com/openxla/iree/issues/14668.
-#     if batch_size < 2048
-# )
+BERT_LARGE_TORCH_BATCHES = [
+    model
+    for batch_size, model in torch_models.BERT_LARGE_384_FP32_TORCH_BATCHES.items()
+]
 
 # Batched Tensorflow models.
 BERT_LARGE_TF_BATCHES = [
@@ -267,25 +231,20 @@
     # PyTorch model are disabled due to https://github.com/openxla/iree/issues/14993.
     # torch_models.MODEL_CLIP_TEXT_SEQLEN64_FP32_TORCH,
     # torch_models.MODEL_UNET_2D_FP32_TORCH,
-    # torch_models.EFFICIENTNET_B7_FP32_TORCH,
-    # torch_models.BERT_LARGE_384_FP16_TORCH_BATCHES[1],
-    # torch_models.EFFICIENTNET_V2_S_FP16_TORCH,
 ]
 
 CUDA_MODELS_LONG = (
     RESNET50_TF_BATCHES
     + BERT_LARGE_TF_BATCHES
     + T5_LARGE_TF_BATCHES
-    # + BERT_LARGE_TORCH_BATCHES
-    # + RESNET50_TORCH_BATCHES
-    # + RESNET50_FP16_TORCH_BATCHES
-    # + BERT_LARGE_FP16_TORCH_BATCHES
+    + BERT_LARGE_TORCH_BATCHES
     + BERT_LARGE_JAX_BATCHES
     + RESNET50_JAX_BATCHES
     + T5_LARGE_JAX_BATCHES
 )
 
 VULKAN_MODELS = [
+    # PyTorch model are disabled due to https://github.com/openxla/iree/issues/14993.
     # torch_models.MODEL_CLIP_TEXT_SEQLEN64_FP32_TORCH,
     # torch_models.MODEL_UNET_2D_FP32_TORCH,
 ]
diff --git a/build_tools/python/e2e_test_framework/models/torch_models.py b/build_tools/python/e2e_test_framework/models/torch_models.py
@@ -92,17 +92,11 @@
 ID_FORMAT = string.Template("${model_id}-batch-${batch_size}")
 NAME_FORMAT = string.Template("${name}Batch${batch_size}")
 BERT_LARGE_FP32_URL = string.Template(
-    "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20230813.929_1692010793/BERT_LARGE_FP32_PT_384XI32_BATCH${batch_size}/linalg.mlirbc"
+    "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20231010.987_1696982151/BERT_LARGE_FP32_PT_384XI32_BATCH${batch_size}/linalg.mlirbc"
 )
 BERT_LARGE_FP16_URL = string.Template(
     "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20230816.932_1692245822/BERT_LARGE_FP16_PT_384XI32_BATCH${batch_size}/linalg.mlirbc"
 )
-RESNET50_FP32_URL = string.Template(
-    "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20230813.929_1692010793/RESNET50_FP32_PT_3X224X224XF32_BATCH${batch_size}/linalg.mlirbc"
-)
-RESNET50_FP16_URL = string.Template(
-    "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20230816.932_1692245822/RESNET50_FP16_PT_3X224X224XF16_BATCH${batch_size}/linalg.mlirbc"
-)
 
 # Converted from https://huggingface.co/docs/transformers/v4.27.2/en/model_doc/bert#transformers.BertModel
 BERT_LARGE_384_FP32_TORCH_BATCHES = model_utils.generate_batch_models(
@@ -122,58 +116,3 @@
     ],
     batch_sizes=[1, 16, 24, 32, 48, 64, 512, 1024, 1280],
 )
-
-# FP16 Versions
-BERT_LARGE_384_FP16_TORCH_BATCHES = model_utils.generate_batch_models(
-    id_template=model_utils.partial_template_substitute(
-        ID_FORMAT, model_id=unique_ids.MODEL_BERT_LARGE_384_FP16_TORCH
-    ),
-    name_template=model_utils.partial_template_substitute(
-        NAME_FORMAT, name="BertLargefp16PT"
-    ),
-    tags=["fp16", "transformer", "seqlen384"],
-    source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
-    source_url_template=BERT_LARGE_FP16_URL,
-    entry_function="forward",
-    input_type_templates=[
-        string.Template("${batch_size}x384xi64"),
-        string.Template("${batch_size}x384xi64"),
-    ],
-    # Batches >=512 are disabled due to OOM: https://github.com/openxla/openxla-benchmark/issues/122.
-    # batch_sizes=[1, 16, 24, 32, 48, 64, 512, 1024, 1280],
-    batch_sizes=[1, 16, 24, 32, 48, 64],
-)
-
-# Converted from https://pytorch.org/vision/main/models/generated/torchvision.models.resnet50.html
-RESNET50_3X224X224_FP32_TORCH_BATCHES = model_utils.generate_batch_models(
-    id_template=model_utils.partial_template_substitute(
-        ID_FORMAT, model_id=unique_ids.MODEL_RESNET50_3X224X224_FP32_TORCH
-    ),
-    name_template=model_utils.partial_template_substitute(
-        NAME_FORMAT, name="Resnet50PT"
-    ),
-    tags=["fp32", "cnn"],
-    source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
-    source_url_template=RESNET50_FP32_URL,
-    entry_function="forward",
-    input_type_templates=[string.Template("${batch_size}x3x224x224xf32")],
-    batch_sizes=[1, 8, 64, 128, 256, 2048],
-)
-
-# FP16 Versions
-RESNET50_3X224X224_FP16_TORCH_BATCHES = model_utils.generate_batch_models(
-    id_template=model_utils.partial_template_substitute(
-        ID_FORMAT, model_id=unique_ids.MODEL_RESNET50_3X224X224_FP16_TORCH
-    ),
-    name_template=model_utils.partial_template_substitute(
-        NAME_FORMAT, name="Resnet50fp16PT"
-    ),
-    tags=["fp32", "cnn"],
-    source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR,
-    source_url_template=RESNET50_FP16_URL,
-    entry_function="forward",
-    input_type_templates=[string.Template("${batch_size}x3x224x224xf16")],
-    # Batch 2048 is disabled due to OOM: https://github.com/openxla/openxla-benchmark/issues/122
-    # batch_sizes=[1, 8, 64, 128, 256, 2048],
-    batch_sizes=[1, 8, 64, 128, 256],
-)
diff --git a/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake b/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake
@@ -285,6 +285,69 @@ iree_fetch_artifact(
   UNPACK
 )
 
+iree_fetch_artifact(
+  NAME "model-BertLargePTBatch1"
+  SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20231010.987_1696982151/BERT_LARGE_FP32_PT_384XI32_BATCH1/linalg.mlirbc"
+  OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BertLargePTBatch1.mlirbc"
+  UNPACK
+)
+
+iree_fetch_artifact(
+  NAME "model-BertLargePTBatch16"
+  SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20231010.987_1696982151/BERT_LARGE_FP32_PT_384XI32_BATCH16/linalg.mlirbc"
+  OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BertLargePTBatch16.mlirbc"
+  UNPACK
+)
+
+iree_fetch_artifact(
+  NAME "model-BertLargePTBatch24"
+  SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20231010.987_1696982151/BERT_LARGE_FP32_PT_384XI32_BATCH24/linalg.mlirbc"
+  OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BertLargePTBatch24.mlirbc"
+  UNPACK
+)
+
+iree_fetch_artifact(
+  NAME "model-BertLargePTBatch32"
+  SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20231010.987_1696982151/BERT_LARGE_FP32_PT_384XI32_BATCH32/linalg.mlirbc"
+  OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BertLargePTBatch32.mlirbc"
+  UNPACK
+)
+
+iree_fetch_artifact(
+  NAME "model-BertLargePTBatch48"
+  SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20231010.987_1696982151/BERT_LARGE_FP32_PT_384XI32_BATCH48/linalg.mlirbc"
+  OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BertLargePTBatch48.mlirbc"
+  UNPACK
+)
+
+iree_fetch_artifact(
+  NAME "model-BertLargePTBatch64"
+  SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20231010.987_1696982151/BERT_LARGE_FP32_PT_384XI32_BATCH64/linalg.mlirbc"
+  OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BertLargePTBatch64.mlirbc"
+  UNPACK
+)
+
+iree_fetch_artifact(
+  NAME "model-BertLargePTBatch512"
+  SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20231010.987_1696982151/BERT_LARGE_FP32_PT_384XI32_BATCH512/linalg.mlirbc"
+  OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BertLargePTBatch512.mlirbc"
+  UNPACK
+)
+
+iree_fetch_artifact(
+  NAME "model-BertLargePTBatch1024"
+  SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20231010.987_1696982151/BERT_LARGE_FP32_PT_384XI32_BATCH1024/linalg.mlirbc"
+  OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BertLargePTBatch1024.mlirbc"
+  UNPACK
+)
+
+iree_fetch_artifact(
+  NAME "model-BertLargePTBatch1280"
+  SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20231010.987_1696982151/BERT_LARGE_FP32_PT_384XI32_BATCH1280/linalg.mlirbc"
+  OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BertLargePTBatch1280.mlirbc"
+  UNPACK
+)
+
 iree_fetch_artifact(
   NAME "model-BERT_LARGE_JAX_384XI32_BATCH1"
   SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.14_1691969180/BERT_LARGE_FP32_JAX_384XI32_BATCH1/stablehlo.mlirbc"