Bring back TorchBench AO dashboard

torchci/clickhouse_queries/torchao_query/params.json

@@ -1,15 +1,13 @@
 {
-  "branches": "String",
-  "commits": "String",
-  "compilers": "String",
+  "branches": "Array(String)",
+  "commits": "Array(String)",
   "device": "String",
-  "dtypes": "String",
-  "getJobId": "Bool",
+  "dtypes": "Array(String)",
   "granularity": "String",
   "mode": "String",
+  "repo": "String",
   "startTime": "DateTime64(3)",
   "stopTime": "DateTime64(3)",
-  "suites": "String",
-  "timezone": "String",
+  "suites": "Array(String)",
   "workflowId": "Int64"
-}
+}

torchci/clickhouse_queries/torchao_query/query.sql

@@ -1,160 +1,76 @@
--- !!! Query is not converted to CH syntax yet.  Delete this line when it gets converted
-WITH performance_results AS (
-  SELECT
-  name,
-  IF(speedup = 'infra_error', NULL, speedup) AS speedup, -- Handle the recent burst of infra error
-  REPLACE(
-    filename,
-    CONCAT(
-      '_', : dtypes, '_', : mode, '_', : device,
-      '_performance'
-    )
-  ) AS filename,
-  compilation_latency,
-  compression_ratio,
-  abs_latency,
-  mfu,
-  memory_bandwidth,
-  dynamo_peak_mem,
-  eager_peak_mem,
-  workflow_id,
-  CAST(job_id AS INT) AS job_id,
-  FORMAT_ISO8601(
-    DATE_TRUNC(: granularity, _event_time)
-  ) AS granularity_bucket,
-  head_sha,
-  head_branch,
-FROM
-  inductor.torchao_perf_stats
-WHERE
-  filename LIKE '%_performance'
-  AND filename LIKE CONCAT(
-    '%_', : dtypes, '_', : mode, '_', : device,
-    '_%'
-  )
-  AND _event_time >= PARSE_DATETIME_ISO8601(:startTime)
-  AND _event_time < PARSE_DATETIME_ISO8601(:stopTime)
-  AND (workflow_id = :workflowId OR :workflowId = 0)
-),
-accuracy_results AS (
-  SELECT
-    name,
-    accuracy,
-    REPLACE(
-      filename,
-      CONCAT(
-        '_', : dtypes, '_', : mode, '_', : device,
-        '_accuracy'
-      )
-    ) AS filename,
-    workflow_id,
-    CAST(job_id AS INT) AS job_id,
-  FROM
-    inductor.torchao_perf_stats
-  WHERE
-    filename LIKE '%_accuracy'
-    AND filename LIKE CONCAT(
-      '%_', : dtypes, '_', : mode, '_', : device,
-      '_%'
-    )
-    AND _event_time >= PARSE_DATETIME_ISO8601(:startTime)
-    AND _event_time < PARSE_DATETIME_ISO8601(:stopTime)
-    AND (workflow_id = :workflowId OR :workflowId = 0)
-    AND accuracy != 'model_fail_to_load'
-    AND accuracy != 'eager_fail_to_run'
-),
-results AS (
-  SELECT
-    performance_results.granularity_bucket AS granularity_bucket,
-    performance_results.workflow_id AS workflow_id,
-    performance_results.job_id AS job_id,
-    performance_results.head_branch AS head_branch,
-    performance_results.head_sha AS head_sha,
-    CASE
-      WHEN performance_results.filename LIKE '%_torchbench' THEN 'torchbench'
-      WHEN performance_results.filename LIKE '%_timm_models' THEN 'timm_models'
-      WHEN performance_results.filename LIKE '%_huggingface' THEN 'huggingface'
-      ELSE NULL
-    END AS suite,
-    CASE
-      WHEN performance_results.filename LIKE '%_torchbench' THEN REPLACE(
-        performance_results.filename, '_torchbench'
-      )
-      WHEN performance_results.filename LIKE '%_timm_models' THEN REPLACE(
-        performance_results.filename, '_timm_models'
-      )
-      WHEN performance_results.filename LIKE '%_huggingface' THEN REPLACE(
-        performance_results.filename, '_huggingface'
-      )
-      ELSE NULL
-    END AS compiler,
-    performance_results.name,
-    IF(TRY_CAST(speedup AS FLOAT) IS NOT NULL,
-      CAST(speedup AS FLOAT),
-      0.0
-    ) AS speedup,
-    accuracy_results.accuracy AS accuracy,
-    IF(TRY_CAST(compilation_latency AS FLOAT) IS NOT NULL,
-      CAST(compilation_latency AS FLOAT),
-      0.0
-    ) AS compilation_latency,
-    IF(TRY_CAST(compression_ratio AS FLOAT) IS NOT NULL,
-      CAST(compression_ratio AS FLOAT),
-      0.0
-    ) AS compression_ratio,
-    IF(TRY_CAST(abs_latency AS FLOAT) IS NOT NULL,
-      CAST(abs_latency AS FLOAT),
-      0.0
-    ) AS abs_latency,
-    IF(TRY_CAST(mfu AS FLOAT) IS NOT NULL,
-      CAST(mfu AS FLOAT),
-      0.0
-    ) AS mfu,
-    IF(TRY_CAST(memory_bandwidth AS FLOAT) IS NOT NULL,
-      CAST(memory_bandwidth AS FLOAT),
-      0.0
-    ) AS memory_bandwidth,
-    IF(TRY_CAST(dynamo_peak_mem AS FLOAT) IS NOT NULL,
-      CAST(dynamo_peak_mem AS FLOAT),
-      0.0
-    ) AS dynamo_peak_mem,
-    IF(TRY_CAST(eager_peak_mem AS FLOAT) IS NOT NULL,
-      CAST(eager_peak_mem AS FLOAT),
-      0.0
-    ) AS eager_peak_mem,
-  FROM
-    performance_results
-  LEFT JOIN accuracy_results ON performance_results.name = accuracy_results.name
-    AND performance_results.filename = accuracy_results.filename
-    AND performance_results.workflow_id = accuracy_results.workflow_id
+-- This powers HUD TorchAO benchmarks dashboards
+WITH benchmarks AS (
+    SELECT
+        o.model.origins [ 1 ] AS suite,
+        o.model.name AS model,
+        tupleElement(o.benchmark, 'extra_info') [ 'quantization' ] AS dtype,
+        o.metric.name AS metric,
+        floor(arrayAvg(o.metric.benchmark_values), 2) AS value,
+        tupleElement(o.metric, 'extra_info') AS extra_info,
+        replaceOne(o.head_branch, 'refs/heads/', '') AS head_branch,
+        o.head_sha AS head_sha,
+        o.workflow_id AS workflow_id,
+        o.job_id AS job_id,
+        DATE_TRUNC(
+            {granularity: String },
+            fromUnixTimestamp(o.timestamp)
+        ) AS granularity_bucket
+    FROM
+        benchmark.oss_ci_benchmark_v3 o
+    WHERE
+        o.timestamp >= toUnixTimestamp({startTime: DateTime64(3) })
+        AND o.timestamp < toUnixTimestamp({stopTime: DateTime64(3) })
+        AND o.repo = {repo: String }
+        AND (
+            has({commits: Array(String) }, o.head_sha)
+            OR empty({commits: Array(String) })
+        )
+        AND (
+            has({suites: Array(String) }, suite)
+            OR empty({suites: Array(String) })
+        )
+        AND (
+            has({dtypes: Array(String) }, dtype)
+            OR empty({dtypes: Array(String) })
+        )
+        AND tupleElement(o.benchmark, 'mode') = {mode: String }
+        AND tupleElement(o.benchmark, 'extra_info') [ 'device' ] = {device: String }
+        AND (
+            workflow_id = {workflowId: Int64}
+            OR {workflowId: Int64} = 0
+        )
+        AND (
+            o.metric.name in [ 'accuracy',
+            'speedup',
+            'compilation_latency',
+            'compression_ratio',
+            'abs_latency',
+            'mfu',
+            'memory_bandwidth',
+            'dynamo_peak_mem',
+            'eager_peak_mem' ]
+        )
 )
-SELECT DISTINCT
-  results.workflow_id,
-  -- As the JSON response is pretty big, only return the field if it's needed
-  IF(:getJobId, results.job_id, NULL) AS job_id,
-  results.suite,
-  results.compiler,
-  results.name,
-  results.speedup,
-  results.accuracy,
-  results.compilation_latency,
-  results.compression_ratio,
-  results.abs_latency,
-  results.mfu,
-  results.memory_bandwidth,
-  results.dynamo_peak_mem,
-  results.eager_peak_mem,
-  results.granularity_bucket,
+SELECT
+    suite,
+    model,
+    dtype,
+    metric,
+    value,
+    extra_info,
+    workflow_id,
+    job_id,
+    granularity_bucket
 FROM
-  results
+    benchmarks
 WHERE
-  ARRAY_CONTAINS(SPLIT(:suites, ','), LOWER(results.suite))
-  AND (ARRAY_CONTAINS(SPLIT(:compilers, ','), LOWER(results.compiler)) OR :compilers = '')
-  AND (ARRAY_CONTAINS(SPLIT(:branches, ','), results.head_branch) OR :branches = '')
-  AND (ARRAY_CONTAINS(SPLIT(:commits, ','), results.head_sha) OR :commits = '')
+    (
+        has({branches: Array(String) }, head_branch)
+        OR empty({branches: Array(String) })
+    )
 ORDER BY
-  granularity_bucket DESC,
-  workflow_id DESC,
-  suite ASC,
-  compiler ASC,
-  name ASC
+    granularity_bucket DESC,
+    workflow_id DESC,
+    suite ASC,
+    dtype ASC,
+    model ASC

torchci/clickhouse_queries/torchao_query_branches/params.json

@@ -1,8 +1,8 @@
 {
+  "dtypes": "Array(String)",
   "device": "String",
-  "dtypes": "String",
-  "granularity": "String",
   "mode": "String",
+  "repo": "String",
   "startTime": "DateTime64(3)",
   "stopTime": "DateTime64(3)"
-}
+}

torchci/clickhouse_queries/torchao_query_branches/query.sql

@@ -1,22 +1,35 @@
--- !!! Query is not converted to CH syntax yet.  Delete this line when it gets converted
+-- This query is used to get the list of branches and commits used by different
+-- OSS CI benchmark experiments. This powers HUD TorchAO benchmarks dashboards
+WITH benchmarks AS (
+    SELECT
+        o.head_branch AS head_branch,
+        o.head_sha AS head_sha,
+        o.workflow_id AS id,
+        toStartOfDay(fromUnixTimestamp(o.timestamp)) AS event_time
+    FROM
+        benchmark.oss_ci_benchmark_v3 o
+    WHERE
+        o.timestamp >= toUnixTimestamp({startTime: DateTime64(3) })
+        AND o.timestamp < toUnixTimestamp({stopTime: DateTime64(3) })
+        AND o.repo = {repo: String }
+        AND tupleElement(o.benchmark, 'extra_info') [ 'performance' ] = 'true'
+        AND (
+            has(
+                {dtypes: Array(String) },
+                tupleElement(o.benchmark, 'extra_info') [ 'quantization' ]
+            )
+            OR empty({dtypes: Array(String) })
+        )
+        AND tupleElement(o.benchmark, 'mode') = {mode: String }
+        AND tupleElement(o.benchmark, 'extra_info') [ 'device' ] = {device: String }
+)
 SELECT
-  DISTINCT head_branch,
-  head_sha,
-  FORMAT_ISO8601(
-    DATE_TRUNC(
-      : granularity, _event_time
-    )
-  ) AS event_time,
+    DISTINCT replaceOne(head_branch, 'refs/heads/', '') AS head_branch,
+    head_sha,
+    id,
+    event_time
 FROM
-  inductor.torchao_perf_stats
-WHERE
-  torchao_perf_stats._event_time >= PARSE_DATETIME_ISO8601(: startTime)
-  AND torchao_perf_stats._event_time < PARSE_DATETIME_ISO8601(: stopTime)
-  AND torchao_perf_stats.filename LIKE '%_performance'
-  AND torchao_perf_stats.filename LIKE CONCAT(
-    '%_', : dtypes, '_', : mode, '_', : device,
-    '_%'
-  )
+    benchmarks
 ORDER BY
-  head_branch,
-  event_time DESC
+    head_branch,
+    event_time DESC

torchci/components/benchmark/compilers/CompilerGraphGroup.tsx

@@ -12,25 +12,34 @@ const GraphCardGroup = styled(Card)({
 /** Mui Styles */
 
 export default function CompilerGraphGroup({
+  dashboard,
   suiteConfig,
   queryParams,
   granularity,
   lBranchAndCommit,
   rBranchAndCommit,
 }: {
-  lBranchAndCommit: BranchAndCommit;
-  rBranchAndCommit: BranchAndCommit;
+  dashboard: string;
+  suiteConfig: SuiteConfig;
   queryParams: { [key: string]: any };
   granularity: Granularity;
-  suiteConfig: SuiteConfig;
+  lBranchAndCommit: BranchAndCommit;
+  rBranchAndCommit: BranchAndCommit;
 }) {
+  // TODO (huydhn): Remove this once TorchInductor dashboard is migrated to the
+  // new database schema
+  const queryName =
+    dashboard === "torchao"
+      ? "torchao_query"
+      : "compilers_benchmark_performance";
+
   return (
     <>
       <GraphCardGroup>
         <CardHeader title={`Suite: ${suiteConfig.name}`} />
         <CardContent>
           <GraphPanel
-            queryName={"compilers_benchmark_performance"}
+            queryName={queryName}
             queryParams={queryParams}
             granularity={granularity}
             suite={suiteConfig.id}

torchci/components/benchmark/compilers/ModelGraphPanel.tsx

@@ -10,6 +10,7 @@ import {
   TimeSeriesPanelWithData,
 } from "components/metrics/panels/TimeSeriesPanel";
 import dayjs from "dayjs";
+import { convertToCompilerPerformanceData } from "lib/benchmark/aoUtils";
 import { augmentData } from "lib/benchmark/compilerUtils";
 import { fetcher } from "lib/GeneralUtils";
 import { CompilerPerformanceData } from "lib/types";
@@ -49,6 +50,12 @@ export function GraphPanel({
   let { data, error } = useSWR(url, fetcher, {
     refreshInterval: 60 * 60 * 1000, // refresh every hour
   });
+  // TODO (huydhn): Remove this once TorchInductor dashboard is migrated to the
+  // new database schema
+  data =
+    queryName === "torchao_query"
+      ? convertToCompilerPerformanceData(data)
+      : data;
   data = augmentData(data);
 
   if (data === undefined || data.length === 0) {