moved InferenceGraphPlotter

InferenceGraphPlotter/README.md

@@ -0,0 +1,8 @@
+# InferenceGraphPlotter
+
+## How to run?
+1. Clone the repo and cd into the repo
+2. Spin up a simple webserver to serve the files. One way is by using python.
+    - for python 2: python -m SimpleHTTPServer
+    - for python 3: python -m http.server
+3. Open a webbrowser and go to http://localhost:8000

InferenceGraphPlotter/data/Best_Hardware_and_Framework/config.json

@@ -0,0 +1,33 @@
+{
+    "Filters":[
+        {
+            "name": "Hardware",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Num of Hardware",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Framework",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Model",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Input Output Length",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Batch Size",
+            "multi_select": "yes"
+        }
+    ],
+    "Selectors":[
+        {"id":"x", "name": "X-axis", "multi_select": "no", "values":["", "Input Output Length", "Batch Size", "Num of Hardware"]},
+        {"id":"y", "name": "Y-axis", "multi_select": "no", "values":["", "Latency", "Throughput"]},
+        {"id":"c", "name": "Color By", "multi_select": "yes", "values":["Num of Hardware", "Hardware", "Input Output Length", "Batch Size", "Framework", "Model"]}
+    ]
+}

InferenceGraphPlotter/data/KV_Cache/config.json

@@ -0,0 +1,37 @@
+{
+    "Filters":[
+        {
+            "name": "Hardware",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Num of Hardware",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Framework",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Model",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Input Output Length",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Batch Size",
+            "multi_select": "yes"
+        },
+        {
+            "name": "KV Cache",
+            "multi_select": "yes"
+        }
+    ],
+    "Selectors":[
+        {"id":"x", "name": "X-axis", "multi_select": "no", "values":["", "Input Output Length", "Batch Size", "Num of Hardware"]},
+        {"id":"y", "name": "Y-axis", "multi_select": "no", "values":["", "Latency", "Throughput"]},
+        {"id":"c", "name": "Color By", "multi_select": "yes", "values":["Num of Hardware", "Hardware", "Input Output Length", "Batch Size", "Framework", "Model", "KV Cache"]}
+    ]
+}

InferenceGraphPlotter/data/KV_Cache_Block_Size/All_results.csv

@@ -0,0 +1,67 @@
+Hardware,Num of Hardware,Framework,Model,Input Output Length,Batch Size,KV Cache Block_size,Latency,Throughput
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,1,8,14.005365249700844,146.22967437737802
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,1,16,13.99969663657248,146.28888419266548
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,1,32,13.991018221713603,146.37962495264077
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,1,64,14.020640316419303,146.07036153702813
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,1,128,14.012758919969201,146.15251797998545
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,16,8,24.89088204409927,1316.4660031711535
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,16,16,24.966458911076188,1312.4808815183123
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,16,32,24.879735534079373,1317.0558004973793
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,16,64,24.77487556822598,1322.6302553876508
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,16,128,24.704433063976467,1326.4016184925804
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,32,8,46.57918341457844,1406.9804405263237
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,32,16,45.90180201642215,1427.743511606655
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,32,32,45.857902373187244,1429.1102865254122
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,32,64,45.6841256422922,1434.546444275818
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,32,128,45.61047025676817,1436.8630630436237
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,64,8,89.63302907068282,1462.3180914330055
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,64,16,88.2489915471524,1485.2520997927415
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,64,32,87.91284341551363,1490.931187158829
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,64,64,88.24184815306216,1485.3723345939638
+Nvidia A100 GPU,1,vLLM,meta-llama/Llama-2-7b-hf,1024,64,128,88.05219635833055,1488.5716134393663
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,1,8,15.115392192266881,135.49102623005496
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,1,16,14.871934588067234,137.7090510903168
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,1,32,14.874929436482489,137.68132539688173
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,1,64,14.870830445550382,137.71927583323352
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,1,128,14.858755989000201,137.83118866183113
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,16,8,21.613671293482184,1516.0774657418572
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,16,16,19.85494950134307,1650.369344821725
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,16,32,19.713430639356375,1662.2170234835312
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,16,64,19.591813604347408,1672.5353079476424
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,16,128,19.57982941623777,1673.559013380634
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,32,8,30.602605057880282,2141.5170334698105
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,32,16,26.22699354030192,2498.799563102556
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,32,32,26.077006820589304,2513.1718701801137
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,32,64,25.92175380885601,2528.2239960789234
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,32,128,25.908147785812616,2529.551727965969
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,64,8,44.88313675392419,2920.2950034132855
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,64,16,35.96128025930375,3644.809057266242
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,64,32,35.615573917515576,3680.1877825570964
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,64,64,35.39454714395106,3703.1692895214865
+Nvidia A100 GPU,1,vLLM,meta-llama/Meta-Llama-3-8B,1024,64,128,35.31509536970407,3711.500666438618
+Nvidia A100 GPU,1,vLLM,mistralai/Mistral-7B-v0.1,1024,1,8,14.42210394423455,142.00424625414786
+Nvidia A100 GPU,1,vLLM,mistralai/Mistral-7B-v0.1,1024,1,16,14.224905307404697,143.97283888659172
+Nvidia A100 GPU,1,vLLM,mistralai/Mistral-7B-v0.1,1024,1,32,14.732134027406573,139.0158408951515
+Nvidia A100 GPU,1,vLLM,mistralai/Mistral-7B-v0.1,1024,16,8,20.867154153063893,1570.3147520568214
+Nvidia A100 GPU,1,vLLM,mistralai/Mistral-7B-v0.1,1024,16,16,19.872619120404124,1648.9019289035537
+Nvidia A100 GPU,1,vLLM,mistralai/Mistral-7B-v0.1,1024,16,32,19.976278366521,1640.3455838359325
+Nvidia A100 GPU,1,vLLM,mistralai/Mistral-7B-v0.1,1024,32,8,29.716878558509052,2205.346024851409
+Nvidia A100 GPU,1,vLLM,mistralai/Mistral-7B-v0.1,1024,32,16,27.352903434075415,2395.943090939196
+Nvidia A100 GPU,1,vLLM,mistralai/Mistral-7B-v0.1,1024,32,32,31.50867538433522,2079.9351036058383
+Nvidia A100 GPU,1,vLLM,mistralai/Mistral-7B-v0.1,1024,64,8,43.781486575491726,2993.776827883508
+Nvidia A100 GPU,1,vLLM,mistralai/Mistral-7B-v0.1,1024,64,16,39.10564037691802,3351.741557910015
+Nvidia A100 GPU,1,vLLM,mistralai/Mistral-7B-v0.1,1024,64,32,45.57801699638367,2875.772327049677
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,1,8,10.722455237992108,191.00103050497876
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,1,16,10.356212973594666,197.7556859077546
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,1,32,10.367651543579996,197.53750320323908
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,1,64,10.378672692924738,197.32773742794157
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,1,128,10.361898914910853,197.6471703514606
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,16,8,23.723289482295513,1381.2587004198754
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,16,16,23.969889169558883,1367.0484568453692
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,16,32,23.87044494319707,1372.7435780093692
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,16,64,23.93308106996119,1369.1509214468697
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,16,128,23.845554472878575,1374.1764754210108
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,32,8,29.383708734996617,2230.3515390467114
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,32,16,28.68344722595066,2284.8020840642853
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,32,32,28.448902206495404,2303.6389778526122
+Nvidia A100 GPU,4,vLLM,mistralai/Mixtral-8x7B-v0.1,1024,32,64,28.33799762930721,2312.6545798078037

InferenceGraphPlotter/data/KV_Cache_Block_Size/config.json

@@ -0,0 +1,37 @@
+{
+    "Filters":[
+        {
+            "name": "Hardware",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Num of Hardware",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Framework",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Model",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Input Output Length",
+            "multi_select": "yes"
+        },
+        {
+            "name": "Batch Size",
+            "multi_select": "yes"
+        },
+        {
+            "name": "KV Cache Block_size",
+            "multi_select": "yes"
+        }
+    ],
+    "Selectors":[
+        {"id":"x", "name": "X-axis", "multi_select": "no", "values":["", "Input Output Length", "Batch Size", "Num of Hardware", "KV Cache Block_size"]},
+        {"id":"y", "name": "Y-axis", "multi_select": "no", "values":["", "Latency", "Throughput"]},
+        {"id":"c", "name": "Color By", "multi_select": "yes", "values":["Num of Hardware", "Hardware", "Input Output Length", "Batch Size", "Framework", "Model", "KV Cache Block_size"]}
+    ]
+}