From 1feb9da2d3d33b5bb3ed5e28e114443f8e8c9601 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 19 Dec 2024 13:07:33 -0800 Subject: [PATCH 1/5] Resurrect TorchAO dynamobench dashboard --- .../torchao_query/params.json | 2 +- .../torchao_query/query.sql | 127 ++++++++------- .../torchao_query_branches/params.json | 6 +- .../torchao_query_branches/query.sql | 51 +++--- .../components/benchmark/torchao/common.tsx | 4 +- torchci/pages/benchmark/torchao.tsx | 152 ++++++------------ 6 files changed, 155 insertions(+), 187 deletions(-) diff --git a/torchci/clickhouse_queries/torchao_query/params.json b/torchci/clickhouse_queries/torchao_query/params.json index f91e82cbb1..c0a529cf62 100644 --- a/torchci/clickhouse_queries/torchao_query/params.json +++ b/torchci/clickhouse_queries/torchao_query/params.json @@ -12,4 +12,4 @@ "suites": "String", "timezone": "String", "workflowId": "Int64" -} \ No newline at end of file +} diff --git a/torchci/clickhouse_queries/torchao_query/query.sql b/torchci/clickhouse_queries/torchao_query/query.sql index b5ae2b7bd0..e580354f51 100644 --- a/torchci/clickhouse_queries/torchao_query/query.sql +++ b/torchci/clickhouse_queries/torchao_query/query.sql @@ -1,67 +1,75 @@ --- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted +-- This powers HUD TorchAO benchmarks dashboards WITH performance_results AS ( - SELECT - name, - IF(speedup = 'infra_error', NULL, speedup) AS speedup, -- Handle the recent burst of infra error - REPLACE( - filename, - CONCAT( - '_', : dtypes, '_', : mode, '_', : device, - '_performance' - ) - ) AS filename, - compilation_latency, - compression_ratio, - abs_latency, - mfu, - memory_bandwidth, - dynamo_peak_mem, - eager_peak_mem, - workflow_id, - CAST(job_id AS INT) AS job_id, - FORMAT_ISO8601( - DATE_TRUNC(: granularity, _event_time) - ) AS granularity_bucket, - head_sha, - head_branch, -FROM - inductor.torchao_perf_stats -WHERE - filename LIKE '%_performance' - AND filename LIKE CONCAT( - '%_', : dtypes, '_', : mode, '_', : device, - '_%' - ) - AND _event_time >= PARSE_DATETIME_ISO8601(:startTime) - AND _event_time < PARSE_DATETIME_ISO8601(:stopTime) - AND (workflow_id = :workflowId OR :workflowId = 0) + SELECT + o.model.name AS model, + o.model.backend AS backend, + o.metric.name AS metric, + floor(arrayAvg(o.metric.benchmark_values), 2) AS actual, + o.head_branch AS head_branch, + o.head_sha AS head_sha, + o.workflow_id AS workflow_id, + o.job_id AS job_id, + o.timestamp AS timestamp + FROM + benchmark.oss_ci_benchmark_v3 o + WHERE + o.timestamp >= toUnixTimestamp({startTime: DateTime64(3) }) + AND o.timestamp < toUnixTimestamp({stopTime: DateTime64(3) }) + AND o.repo = {repo: String } + AND tupleElement(o.benchmark, 'extra_info') [ 'performance' ] = 'true' + AND ( + has( + {dtypes: Array(String) }, + tupleElement(o.benchmark, 'extra_info') [ 'quantization' ] + ) + OR empty({dtypes: Array(String) }) + ) + AND tupleElement(o.benchmark, 'mode') = {mode: String } + AND tupleElement(o.benchmark, 'extra_info') [ 'device' ] = {device: String } + AND ( + workflow_id = {workflowId: Int64} + OR {workflowId: Int64} = 0 + ) + AND ( + o.metric.name in [ 'speedup', + 'compilation_latency', + 'compression_ratio', + 'abs_latency', + 'mfu', + 'memory_bandwidth', + 'dynamo_peak_mem', + 'eager_peak_mem' ] + ) ), accuracy_results AS ( SELECT - name, + o.model.name AS model, accuracy, - REPLACE( - filename, - CONCAT( - '_', : dtypes, '_', : mode, '_', : device, - '_accuracy' - ) - ) AS filename, - workflow_id, - CAST(job_id AS INT) AS job_id, + o.workflow_id AS workflow_id, + o.job_id AS job_id, FROM - inductor.torchao_perf_stats + benchmark.oss_ci_benchmark_v3 o WHERE - filename LIKE '%_accuracy' - AND filename LIKE CONCAT( - '%_', : dtypes, '_', : mode, '_', : device, - '_%' - ) - AND _event_time >= PARSE_DATETIME_ISO8601(:startTime) - AND _event_time < PARSE_DATETIME_ISO8601(:stopTime) - AND (workflow_id = :workflowId OR :workflowId = 0) - AND accuracy != 'model_fail_to_load' - AND accuracy != 'eager_fail_to_run' + o.timestamp >= toUnixTimestamp({startTime: DateTime64(3) }) + AND o.timestamp < toUnixTimestamp({stopTime: DateTime64(3) }) + AND o.repo = {repo: String } + AND tupleElement(o.benchmark, 'extra_info') [ 'accuracy' ] = 'true' + AND ( + has( + {dtypes: Array(String) }, + tupleElement(o.benchmark, 'extra_info') [ 'quantization' ] + ) + OR empty({dtypes: Array(String) }) + ) + AND tupleElement(o.benchmark, 'mode') = {mode: String } + AND tupleElement(o.benchmark, 'extra_info') [ 'device' ] = {device: String } + AND ( + workflow_id = {workflowId: Int64} + OR {workflowId: Int64} = 0 + ) + + --AND accuracy != 'model_fail_to_load' + --AND accuracy != 'eager_fail_to_run' ), results AS ( SELECT @@ -145,6 +153,9 @@ SELECT DISTINCT results.dynamo_peak_mem, results.eager_peak_mem, results.granularity_bucket, + FORMAT_ISO8601( + DATE_TRUNC(: granularity, _event_time) + ) AS granularity_bucket, FROM results WHERE @@ -157,4 +168,4 @@ ORDER BY workflow_id DESC, suite ASC, compiler ASC, - name ASC \ No newline at end of file + name ASC diff --git a/torchci/clickhouse_queries/torchao_query_branches/params.json b/torchci/clickhouse_queries/torchao_query_branches/params.json index 3f7013e3ee..64518741c7 100644 --- a/torchci/clickhouse_queries/torchao_query_branches/params.json +++ b/torchci/clickhouse_queries/torchao_query_branches/params.json @@ -1,8 +1,8 @@ { + "dtypes": "Array(String)", "device": "String", - "dtypes": "String", - "granularity": "String", "mode": "String", + "repo": "String", "startTime": "DateTime64(3)", "stopTime": "DateTime64(3)" -} \ No newline at end of file +} diff --git a/torchci/clickhouse_queries/torchao_query_branches/query.sql b/torchci/clickhouse_queries/torchao_query_branches/query.sql index 088cf20957..d22b615615 100644 --- a/torchci/clickhouse_queries/torchao_query_branches/query.sql +++ b/torchci/clickhouse_queries/torchao_query_branches/query.sql @@ -1,22 +1,35 @@ --- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted +-- This query is used to get the list of branches and commits used by different +-- OSS CI benchmark experiments. This powers HUD TorchAO benchmarks dashboards +WITH benchmarks AS ( + SELECT + o.head_branch AS head_branch, + o.head_sha AS head_sha, + o.workflow_id AS id, + toStartOfDay(fromUnixTimestamp(o.timestamp)) AS event_time + FROM + benchmark.oss_ci_benchmark_v3 o + WHERE + o.timestamp >= toUnixTimestamp({startTime: DateTime64(3) }) + AND o.timestamp < toUnixTimestamp({stopTime: DateTime64(3) }) + AND o.repo = {repo: String } + AND tupleElement(o.benchmark, 'extra_info') [ 'performance' ] = 'true' + AND ( + has( + {dtypes: Array(String) }, + tupleElement(o.benchmark, 'extra_info') [ 'quantization' ] + ) + OR empty({dtypes: Array(String) }) + ) + AND tupleElement(o.benchmark, 'mode') = {mode: String } + AND tupleElement(o.benchmark, 'extra_info') [ 'device' ] = {device: String } +) SELECT - DISTINCT head_branch, - head_sha, - FORMAT_ISO8601( - DATE_TRUNC( - : granularity, _event_time - ) - ) AS event_time, + DISTINCT replaceOne(head_branch, 'refs/heads/', '') AS head_branch, + head_sha, + id, + event_time FROM - inductor.torchao_perf_stats -WHERE - torchao_perf_stats._event_time >= PARSE_DATETIME_ISO8601(: startTime) - AND torchao_perf_stats._event_time < PARSE_DATETIME_ISO8601(: stopTime) - AND torchao_perf_stats.filename LIKE '%_performance' - AND torchao_perf_stats.filename LIKE CONCAT( - '%_', : dtypes, '_', : mode, '_', : device, - '_%' - ) + benchmarks ORDER BY - head_branch, - event_time DESC + head_branch, + event_time DESC diff --git a/torchci/components/benchmark/torchao/common.tsx b/torchci/components/benchmark/torchao/common.tsx index 2d4ec110df..c342b851cc 100644 --- a/torchci/components/benchmark/torchao/common.tsx +++ b/torchci/components/benchmark/torchao/common.tsx @@ -1,5 +1,5 @@ -export const DEFAULT_REPO_NAME = "pytorch/ao"; -export const DTYPES = ["amp", "bfloat16"]; +export const DEFAULT_REPO_NAME = "pytorch/benchmark"; +export const DTYPES = ["autoquant", "int8dynamic", "int8weightonly", "noquant"]; export const DEFAULT_MODE = "inference"; export const DEFAULT_DEVICE_NAME = "cuda (a100)"; diff --git a/torchci/pages/benchmark/torchao.tsx b/torchci/pages/benchmark/torchao.tsx index 844f316762..c4c19567ac 100644 --- a/torchci/pages/benchmark/torchao.tsx +++ b/torchci/pages/benchmark/torchao.tsx @@ -3,7 +3,7 @@ import { BranchAndCommitPicker } from "components/benchmark/BranchAndCommitPicke import { CommitPanel } from "components/benchmark/CommitPanel"; import { LAST_N_DAYS, MAIN_BRANCH } from "components/benchmark/common"; import { BenchmarkLogs } from "components/benchmark/compilers/BenchmarkLogs"; -import { GraphPanel } from "components/benchmark/compilers/SummaryGraphPanel"; +import CompilerGraphGroup from "components/benchmark/compilers/CompilerGraphGroup"; import { SummaryPanel } from "components/benchmark/compilers/SummaryPanel"; import { DTypePicker, @@ -17,18 +17,18 @@ import { DISPLAY_NAMES_TO_DEVICE_NAMES, DTYPES, } from "components/benchmark/torchao/common"; -import { SuitePicker, SUITES } from "components/benchmark/torchao/SuitePicker"; +import { SUITES } from "components/benchmark/torchao/SuitePicker"; import CopyLink from "components/CopyLink"; import GranularityPicker from "components/GranularityPicker"; import { Granularity } from "components/metrics/panels/TimeSeriesPanel"; import dayjs from "dayjs"; import { augmentData } from "lib/benchmark/compilerUtils"; import { fetcher } from "lib/GeneralUtils"; -import { RocksetParam } from "lib/rockset"; import { BranchAndCommit } from "lib/types"; import { useRouter } from "next/router"; import { useEffect, useState } from "react"; import useSWR from "swr"; +import { COMPILER_SUITES_MAP } from "../../lib/benchmark/compliers/CompilerSuites"; import { TimeRangePicker } from "../metrics"; function Report({ @@ -36,46 +36,29 @@ function Report({ startTime, stopTime, granularity, - suite, mode, dtype, deviceName, lBranchAndCommit, rBranchAndCommit, }: { - queryParams: RocksetParam[]; + queryParams: { [key: string]: any }; startTime: dayjs.Dayjs; stopTime: dayjs.Dayjs; granularity: Granularity; - suite: string; mode: string; dtype: string; deviceName: string; lBranchAndCommit: BranchAndCommit; rBranchAndCommit: BranchAndCommit; }) { - const queryCollection = "inductor"; const queryName = "torchao_query"; - - const queryParamsWithL: RocksetParam[] = [ - { - name: "suites", - type: "string", - value: Object.keys(SUITES).join(","), - }, - { - name: "branches", - type: "string", - value: lBranchAndCommit.branch, - }, - { - name: "commits", - type: "string", - value: lBranchAndCommit.commit, - }, + const queryParamsWithL: { [key: string]: any } = { ...queryParams, - ]; - const lUrl = `/api/query/${queryCollection}/${queryName}?parameters=${encodeURIComponent( + branches: [lBranchAndCommit.branch], + commits: lBranchAndCommit.commit ? [lBranchAndCommit.commit] : [], + }; + const lUrl = `/api/clickhouse/${queryName}?parameters=${encodeURIComponent( JSON.stringify(queryParamsWithL) )}`; @@ -84,20 +67,12 @@ function Report({ }); lData = augmentData(lData); - const queryParamsWithR: RocksetParam[] = [ - { - name: "branches", - type: "string", - value: rBranchAndCommit.branch, - }, - { - name: "commits", - type: "string", - value: rBranchAndCommit.commit, - }, + const queryParamsWithR: { [key: string]: any } = { ...queryParams, - ]; - const rUrl = `/api/query/${queryCollection}/${queryName}?parameters=${encodeURIComponent( + branches: [rBranchAndCommit.branch], + commits: rBranchAndCommit.commit ? [rBranchAndCommit.commit] : [], + }; + const rUrl = `/api/clickhouse/${queryName}?parameters=${encodeURIComponent( JSON.stringify(queryParamsWithR) )}`; @@ -152,15 +127,21 @@ function Report({ }} all_suites={SUITES} /> - + {Array.from(Object.values(COMPILER_SUITES_MAP)).map((suiteConfig) => { + return ( + suiteConfig.showGraph && ( +
+ +
+ ) + ); + })} ); } @@ -177,7 +158,7 @@ export default function Page() { const [granularity, setGranularity] = useState("hour"); const [suite, setSuite] = useState(Object.keys(SUITES)[0]); const [mode, setMode] = useState(DEFAULT_MODE); - const [dtype, setDType] = useState(MODES[DEFAULT_MODE]); + const [dtype, setDType] = useState(DTYPES[0]); const [lBranch, setLBranch] = useState(MAIN_BRANCH); const [lCommit, setLCommit] = useState(""); const [rBranch, setRBranch] = useState(MAIN_BRANCH); @@ -258,43 +239,20 @@ export default function Page() { ); }, [router.query]); - const queryParams: RocksetParam[] = [ - { - name: "timezone", - type: "string", - value: Intl.DateTimeFormat().resolvedOptions().timeZone, - }, - { - name: "startTime", - type: "string", - value: startTime, - }, - { - name: "stopTime", - type: "string", - value: stopTime, - }, - { - name: "granularity", - type: "string", - value: granularity, - }, - { - name: "mode", - type: "string", - value: mode, - }, - { - name: "dtypes", - type: "string", - value: dtype, - }, - { - name: "device", - type: "string", - value: DISPLAY_NAMES_TO_DEVICE_NAMES[deviceName], - }, - ]; + const queryParams: { [key: string]: any } = { + branches: [], + commits: [], + compilers: [], + device: DISPLAY_NAMES_TO_DEVICE_NAMES[deviceName], + dtypes: [dtype], + granularity: granularity, + mode: mode, + repo: DEFAULT_REPO_NAME, + startTime: dayjs(startTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + stopTime: dayjs(stopTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + suites: Object.keys(SUITES), + workflowId: 0, + }; return (
@@ -307,7 +265,7 @@ export default function Page() { startTime.toString() )}&stopTime=${encodeURIComponent( stopTime.toString() - )}&granularity=${granularity}&suite=${suite}&mode=${mode}&dtype=${dtype}&deviceName=${encodeURIComponent( + )}&granularity=${granularity}&mode=${mode}&dtype=${dtype}&deviceName=${encodeURIComponent( deviceName )}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`} /> @@ -326,13 +284,12 @@ export default function Page() { granularity={granularity} setGranularity={setGranularity} /> - —Diff→ @@ -367,22 +324,9 @@ export default function Page() { titlePrefix={"New"} fallbackIndex={0} // Default to the latest commit timeRange={timeRange} - useClickHouse={false} + useClickHouse={true} /> - -
); } From 6f411fde0876bc3c987d3e69bb062e1d16ad296a Mon Sep 17 00:00:00 2001 From: Huy Do Date: Sun, 22 Dec 2024 02:49:09 -0800 Subject: [PATCH 2/5] Bring back TorchBench AO dashboard --- .../torchao_query/params.json | 12 +- .../torchao_query/query.sql | 177 ++++-------------- .../compilers/CompilerGraphGroup.tsx | 17 +- .../benchmark/compilers/ModelGraphPanel.tsx | 7 + .../benchmark/compilers/SummaryGraphPanel.tsx | 10 +- .../components/benchmark/torchao/common.tsx | 7 +- torchci/lib/types.ts | 12 ++ .../[suite]/[compiler]/[[...page]].tsx | 72 +++++-- torchci/pages/benchmark/compilers.tsx | 1 + torchci/pages/benchmark/torchao.tsx | 38 ++-- 10 files changed, 174 insertions(+), 179 deletions(-) diff --git a/torchci/clickhouse_queries/torchao_query/params.json b/torchci/clickhouse_queries/torchao_query/params.json index c0a529cf62..67bc42b2b5 100644 --- a/torchci/clickhouse_queries/torchao_query/params.json +++ b/torchci/clickhouse_queries/torchao_query/params.json @@ -1,15 +1,13 @@ { - "branches": "String", - "commits": "String", - "compilers": "String", + "branches": "Array(String)", + "commits": "Array(String)", "device": "String", - "dtypes": "String", - "getJobId": "Bool", + "dtypes": "Array(String)", "granularity": "String", "mode": "String", + "repo": "String", "startTime": "DateTime64(3)", "stopTime": "DateTime64(3)", - "suites": "String", - "timezone": "String", + "suites": "Array(String)", "workflowId": "Int64" } diff --git a/torchci/clickhouse_queries/torchao_query/query.sql b/torchci/clickhouse_queries/torchao_query/query.sql index e580354f51..71538b126d 100644 --- a/torchci/clickhouse_queries/torchao_query/query.sql +++ b/torchci/clickhouse_queries/torchao_query/query.sql @@ -1,27 +1,36 @@ -- This powers HUD TorchAO benchmarks dashboards -WITH performance_results AS ( +WITH benchmarks AS ( SELECT + o.model.origins [ 1 ] AS suite, o.model.name AS model, - o.model.backend AS backend, + tupleElement(o.benchmark, 'extra_info') [ 'quantization' ] AS dtype, o.metric.name AS metric, - floor(arrayAvg(o.metric.benchmark_values), 2) AS actual, - o.head_branch AS head_branch, + floor(arrayAvg(o.metric.benchmark_values), 2) AS value, + tupleElement(o.metric, 'extra_info') AS extra_info, + replaceOne(o.head_branch, 'refs/heads/', '') AS head_branch, o.head_sha AS head_sha, o.workflow_id AS workflow_id, o.job_id AS job_id, - o.timestamp AS timestamp + DATE_TRUNC( + {granularity: String }, + fromUnixTimestamp(o.timestamp) + ) AS granularity_bucket FROM benchmark.oss_ci_benchmark_v3 o WHERE o.timestamp >= toUnixTimestamp({startTime: DateTime64(3) }) AND o.timestamp < toUnixTimestamp({stopTime: DateTime64(3) }) AND o.repo = {repo: String } - AND tupleElement(o.benchmark, 'extra_info') [ 'performance' ] = 'true' AND ( - has( - {dtypes: Array(String) }, - tupleElement(o.benchmark, 'extra_info') [ 'quantization' ] - ) + has({commits: Array(String) }, o.head_sha) + OR empty({commits: Array(String) }) + ) + AND ( + has({suites: Array(String) }, suite) + OR empty({suites: Array(String) }) + ) + AND ( + has({dtypes: Array(String) }, dtype) OR empty({dtypes: Array(String) }) ) AND tupleElement(o.benchmark, 'mode') = {mode: String } @@ -31,7 +40,8 @@ WITH performance_results AS ( OR {workflowId: Int64} = 0 ) AND ( - o.metric.name in [ 'speedup', + o.metric.name in [ 'accuracy', + 'speedup', 'compilation_latency', 'compression_ratio', 'abs_latency', @@ -40,132 +50,27 @@ WITH performance_results AS ( 'dynamo_peak_mem', 'eager_peak_mem' ] ) -), -accuracy_results AS ( - SELECT - o.model.name AS model, - accuracy, - o.workflow_id AS workflow_id, - o.job_id AS job_id, - FROM - benchmark.oss_ci_benchmark_v3 o - WHERE - o.timestamp >= toUnixTimestamp({startTime: DateTime64(3) }) - AND o.timestamp < toUnixTimestamp({stopTime: DateTime64(3) }) - AND o.repo = {repo: String } - AND tupleElement(o.benchmark, 'extra_info') [ 'accuracy' ] = 'true' - AND ( - has( - {dtypes: Array(String) }, - tupleElement(o.benchmark, 'extra_info') [ 'quantization' ] - ) - OR empty({dtypes: Array(String) }) - ) - AND tupleElement(o.benchmark, 'mode') = {mode: String } - AND tupleElement(o.benchmark, 'extra_info') [ 'device' ] = {device: String } - AND ( - workflow_id = {workflowId: Int64} - OR {workflowId: Int64} = 0 - ) - - --AND accuracy != 'model_fail_to_load' - --AND accuracy != 'eager_fail_to_run' -), -results AS ( - SELECT - performance_results.granularity_bucket AS granularity_bucket, - performance_results.workflow_id AS workflow_id, - performance_results.job_id AS job_id, - performance_results.head_branch AS head_branch, - performance_results.head_sha AS head_sha, - CASE - WHEN performance_results.filename LIKE '%_torchbench' THEN 'torchbench' - WHEN performance_results.filename LIKE '%_timm_models' THEN 'timm_models' - WHEN performance_results.filename LIKE '%_huggingface' THEN 'huggingface' - ELSE NULL - END AS suite, - CASE - WHEN performance_results.filename LIKE '%_torchbench' THEN REPLACE( - performance_results.filename, '_torchbench' - ) - WHEN performance_results.filename LIKE '%_timm_models' THEN REPLACE( - performance_results.filename, '_timm_models' - ) - WHEN performance_results.filename LIKE '%_huggingface' THEN REPLACE( - performance_results.filename, '_huggingface' - ) - ELSE NULL - END AS compiler, - performance_results.name, - IF(TRY_CAST(speedup AS FLOAT) IS NOT NULL, - CAST(speedup AS FLOAT), - 0.0 - ) AS speedup, - accuracy_results.accuracy AS accuracy, - IF(TRY_CAST(compilation_latency AS FLOAT) IS NOT NULL, - CAST(compilation_latency AS FLOAT), - 0.0 - ) AS compilation_latency, - IF(TRY_CAST(compression_ratio AS FLOAT) IS NOT NULL, - CAST(compression_ratio AS FLOAT), - 0.0 - ) AS compression_ratio, - IF(TRY_CAST(abs_latency AS FLOAT) IS NOT NULL, - CAST(abs_latency AS FLOAT), - 0.0 - ) AS abs_latency, - IF(TRY_CAST(mfu AS FLOAT) IS NOT NULL, - CAST(mfu AS FLOAT), - 0.0 - ) AS mfu, - IF(TRY_CAST(memory_bandwidth AS FLOAT) IS NOT NULL, - CAST(memory_bandwidth AS FLOAT), - 0.0 - ) AS memory_bandwidth, - IF(TRY_CAST(dynamo_peak_mem AS FLOAT) IS NOT NULL, - CAST(dynamo_peak_mem AS FLOAT), - 0.0 - ) AS dynamo_peak_mem, - IF(TRY_CAST(eager_peak_mem AS FLOAT) IS NOT NULL, - CAST(eager_peak_mem AS FLOAT), - 0.0 - ) AS eager_peak_mem, - FROM - performance_results - LEFT JOIN accuracy_results ON performance_results.name = accuracy_results.name - AND performance_results.filename = accuracy_results.filename - AND performance_results.workflow_id = accuracy_results.workflow_id ) -SELECT DISTINCT - results.workflow_id, - -- As the JSON response is pretty big, only return the field if it's needed - IF(:getJobId, results.job_id, NULL) AS job_id, - results.suite, - results.compiler, - results.name, - results.speedup, - results.accuracy, - results.compilation_latency, - results.compression_ratio, - results.abs_latency, - results.mfu, - results.memory_bandwidth, - results.dynamo_peak_mem, - results.eager_peak_mem, - results.granularity_bucket, - FORMAT_ISO8601( - DATE_TRUNC(: granularity, _event_time) - ) AS granularity_bucket, +SELECT + suite, + model, + dtype, + metric, + value, + extra_info, + workflow_id, + job_id, + granularity_bucket FROM - results + benchmarks WHERE - ARRAY_CONTAINS(SPLIT(:suites, ','), LOWER(results.suite)) - AND (ARRAY_CONTAINS(SPLIT(:compilers, ','), LOWER(results.compiler)) OR :compilers = '') - AND (ARRAY_CONTAINS(SPLIT(:branches, ','), results.head_branch) OR :branches = '') - AND (ARRAY_CONTAINS(SPLIT(:commits, ','), results.head_sha) OR :commits = '') + ( + has({branches: Array(String) }, head_branch) + OR empty({branches: Array(String) }) + ) ORDER BY - granularity_bucket DESC, - workflow_id DESC, - suite ASC, - compiler ASC, - name ASC + granularity_bucket DESC, + workflow_id DESC, + suite ASC, + dtype ASC, + model ASC diff --git a/torchci/components/benchmark/compilers/CompilerGraphGroup.tsx b/torchci/components/benchmark/compilers/CompilerGraphGroup.tsx index 6d9b70704b..061f3f7453 100644 --- a/torchci/components/benchmark/compilers/CompilerGraphGroup.tsx +++ b/torchci/components/benchmark/compilers/CompilerGraphGroup.tsx @@ -12,25 +12,34 @@ const GraphCardGroup = styled(Card)({ /** Mui Styles */ export default function CompilerGraphGroup({ + dashboard, suiteConfig, queryParams, granularity, lBranchAndCommit, rBranchAndCommit, }: { - lBranchAndCommit: BranchAndCommit; - rBranchAndCommit: BranchAndCommit; + dashboard: string; + suiteConfig: SuiteConfig; queryParams: { [key: string]: any }; granularity: Granularity; - suiteConfig: SuiteConfig; + lBranchAndCommit: BranchAndCommit; + rBranchAndCommit: BranchAndCommit; }) { + // TODO (huydhn): Remove this once TorchInductor dashboard is migrated to the + // new database schema + const queryName = + dashboard === "torchao" + ? "torchao_query" + : "compilers_benchmark_performance"; + return ( <> ; } + // TODO (huydhn): Remove this once TorchInductor dashboard is migrated to the + // new database schema + data = + queryName === "torchao_query" + ? convertToCompilerPerformanceData(data) + : data; + // Clamp to the nearest granularity (e.g. nearest hour) so that the times will // align with the data we get from the database const startTime = dayjs(queryParams["startTime"]).startOf(granularity); diff --git a/torchci/components/benchmark/torchao/common.tsx b/torchci/components/benchmark/torchao/common.tsx index c342b851cc..708112690d 100644 --- a/torchci/components/benchmark/torchao/common.tsx +++ b/torchci/components/benchmark/torchao/common.tsx @@ -1,5 +1,10 @@ export const DEFAULT_REPO_NAME = "pytorch/benchmark"; -export const DTYPES = ["autoquant", "int8dynamic", "int8weightonly", "noquant"]; +export const QUANTIZATIONS = [ + "autoquant", + "int8dynamic", + "int8weightonly", + "noquant", +]; export const DEFAULT_MODE = "inference"; export const DEFAULT_DEVICE_NAME = "cuda (a100)"; diff --git a/torchci/lib/types.ts b/torchci/lib/types.ts index b2ccd56962..82167618f8 100644 --- a/torchci/lib/types.ts +++ b/torchci/lib/types.ts @@ -204,6 +204,18 @@ export interface CompilerPerformanceData { job_id?: number; } +export interface BenchmarkData { + extra_info: { [key: string]: string }; + granularity_bucket: string; + job_id: number; + metric: string; + model: string; + quantization?: string; + suite: string; + value: number; + workflow_id: number; +} + export interface BranchAndCommit { branch: string; commit: string; diff --git a/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx b/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx index a132aa49b7..087a13a866 100644 --- a/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx +++ b/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx @@ -24,10 +24,12 @@ import { ModePicker, MODES, } from "components/benchmark/ModeAndDTypePicker"; +import { QUANTIZATIONS } from "components/benchmark/torchao/common"; import CopyLink from "components/CopyLink"; import GranularityPicker from "components/GranularityPicker"; import { Granularity } from "components/metrics/panels/TimeSeriesPanel"; import dayjs from "dayjs"; +import { convertToCompilerPerformanceData } from "lib/benchmark/aoUtils"; import { augmentData } from "lib/benchmark/compilerUtils"; import { fetcher } from "lib/GeneralUtils"; import { BranchAndCommit, CompilerPerformanceData } from "lib/types"; @@ -80,6 +82,10 @@ function Report({ let { data: lData, error: _lError } = useSWR(lUrl, fetcher, { refreshInterval: 60 * 60 * 1000, // refresh every hour }); + // TODO (huydhn): Remove this once TorchInductor dashboard is migrated to the + // new database schema + lData = + dashboard === "torchao" ? convertToCompilerPerformanceData(lData) : lData; lData = augmentData(lData); lData = lData ? lData.filter((e: CompilerPerformanceData) => e.suite === suite) @@ -98,6 +104,10 @@ function Report({ let { data: rData, error: _rError } = useSWR(rUrl, fetcher, { refreshInterval: 60 * 60 * 1000, // refresh every hour }); + // TODO (huydhn): Remove this once TorchInductor dashboard is migrated to the + // new database schema + rData = + dashboard === "torchao" ? convertToCompilerPerformanceData(rData) : rData; rData = augmentData(rData); rData = rData ? rData.filter((e: CompilerPerformanceData) => e.suite === suite) @@ -171,7 +181,7 @@ export default function Page() { const compiler: string = (router.query.compiler as string) ?? undefined; const model: string = (router.query.model as string) ?? undefined; const dashboard: string = - (router.query.dashboard as string) ?? "TorchInductor"; + (router.query.dashboard as string) ?? "torchinductor"; const queryName: string = DASHBOARD_QUERY_MAP[dashboard] ?? "compilers_benchmark_performance"; const branchQueryName = queryName + "_branches"; @@ -264,19 +274,37 @@ export default function Page() { return ; } - const queryParams: { [key: string]: any } = { - commits: [], - compilers: [compiler], - device: DISPLAY_NAMES_TO_DEVICE_NAMES[deviceName], - dtypes: dtype, - getJobId: false, - granularity: granularity, - mode: mode, - startTime: dayjs(startTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), - stopTime: dayjs(stopTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), - suites: [], - workflowId: 0, - }; + // TODO (huydhn): Remove this once TorchInductor dashboard is migrated to the + // new database schema + const queryParams: { [key: string]: any } = + dashboard === "torchao" + ? { + branches: [], + commits: [], + compilers: [compiler], + device: DISPLAY_NAMES_TO_DEVICE_NAMES[deviceName], + dtypes: [dtype], + granularity: granularity, + mode: mode, + repo: "pytorch/benchmark", + startTime: dayjs(startTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + stopTime: dayjs(stopTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + suites: [], + workflowId: 0, + } + : { + commits: [], + compilers: [compiler], + device: DISPLAY_NAMES_TO_DEVICE_NAMES[deviceName], + dtypes: dtype, + getJobId: false, + granularity: granularity, + mode: mode, + startTime: dayjs(startTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + stopTime: dayjs(stopTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + suites: [], + workflowId: 0, + }; return (
@@ -312,12 +340,22 @@ export default function Page() { granularity={granularity} setGranularity={setGranularity} /> - + {dashboard === "torchao" && ( + + )} + {dashboard === "torchinductor" && ( + + )} ; } + lData = convertToCompilerPerformanceData(lData); + rData = convertToCompilerPerformanceData(rData); + return (
("hour"); const [suite, setSuite] = useState(Object.keys(SUITES)[0]); const [mode, setMode] = useState(DEFAULT_MODE); - const [dtype, setDType] = useState(DTYPES[0]); + const [dtype, setDType] = useState(QUANTIZATIONS[0]); const [lBranch, setLBranch] = useState(MAIN_BRANCH); const [lCommit, setLCommit] = useState(""); const [rBranch, setRBranch] = useState(MAIN_BRANCH); @@ -284,11 +282,16 @@ export default function Page() { granularity={granularity} setGranularity={setGranularity} /> - + +
); } From 2ba9e1d7d19688877635e922388a50eeb5d9ffdf Mon Sep 17 00:00:00 2001 From: Huy Do Date: Sun, 22 Dec 2024 03:03:25 -0800 Subject: [PATCH 3/5] Add aoUtils --- torchci/lib/benchmark/aoUtils.ts | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 torchci/lib/benchmark/aoUtils.ts diff --git a/torchci/lib/benchmark/aoUtils.ts b/torchci/lib/benchmark/aoUtils.ts new file mode 100644 index 0000000000..923724ca7f --- /dev/null +++ b/torchci/lib/benchmark/aoUtils.ts @@ -0,0 +1,45 @@ +import { BenchmarkData, CompilerPerformanceData } from "lib/types"; + +// TODO (huydhn): Use this function to convert the generic benchmark data to the old +// CompilerPerformanceData format. This is needed until the TorchInductor dashboard +// is migrated to the new format +export function convertToCompilerPerformanceData(data: BenchmarkData[]) { + const convertData: { [model: string]: CompilerPerformanceData } = {}; + if (data === undefined || data === null) { + return []; + } + + data.forEach((r: BenchmarkData) => { + const k = `${r.granularity_bucket} ${r.model}`; + + if (!(k in convertData)) { + convertData[k] = { + abs_latency: 0, + accuracy: "", + compilation_latency: 0, + compiler: "default", + compression_ratio: 0, + dynamo_peak_mem: 0, + eager_peak_mem: 0, + granularity_bucket: r.granularity_bucket, + name: r.model, + speedup: 0, + suite: r.suite, + workflow_id: r.workflow_id, + job_id: r.job_id, + }; + } + + // Accuracy metric has a string value instead of a number https://github.com/pytorch/pytorch/pull/143611 + if (r.metric === "accuracy") { + convertData[k][r.metric] = JSON.parse( + r.extra_info["benchmark_values"] + )[0]; + } else { + // @ts-expect-error + convertData[k][r.metric] = r.value; + } + }); + + return Object.values(convertData); +} From 7c82303f8897d0e066e2d301369c3c904148bd52 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Sun, 22 Dec 2024 03:16:36 -0800 Subject: [PATCH 4/5] Another tweak --- .../pages/benchmark/[suite]/[compiler]/[[...page]].tsx | 8 ++++++-- torchci/pages/benchmark/torchao.tsx | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx b/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx index 087a13a866..13fb9e8506 100644 --- a/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx +++ b/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx @@ -120,7 +120,9 @@ function Report({ return (
diff --git a/torchci/pages/benchmark/torchao.tsx b/torchci/pages/benchmark/torchao.tsx index f04b6783f0..3879a5915d 100644 --- a/torchci/pages/benchmark/torchao.tsx +++ b/torchci/pages/benchmark/torchao.tsx @@ -102,7 +102,7 @@ function Report({ ? rData[0].granularity_bucket : undefined, }} - workflowName={"torchao-nightly"} + workflowName={"Torchao nightly workflow (A100)".toLowerCase()} > From 5078a08c4dc5c440ccb2d54395feef699498c3f3 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Sun, 22 Dec 2024 03:23:52 -0800 Subject: [PATCH 5/5] Another tweak --- torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx b/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx index 13fb9e8506..763fee376b 100644 --- a/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx +++ b/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx @@ -135,10 +135,9 @@ function Report({ : undefined, }} workflowName={ - DISPLAY_NAMES_TO_WORKFLOW_NAMES[deviceName] ?? - (dashboard === "torchao" + dashboard === "torchao" ? "Torchao nightly workflow (A100)".toLowerCase() - : "inductor-A100-perf-nightly") + : DISPLAY_NAMES_TO_WORKFLOW_NAMES[deviceName] } >