diff --git a/torchci/clickhouse_queries/torchao_query/params.json b/torchci/clickhouse_queries/torchao_query/params.json index f91e82cbb1..67bc42b2b5 100644 --- a/torchci/clickhouse_queries/torchao_query/params.json +++ b/torchci/clickhouse_queries/torchao_query/params.json @@ -1,15 +1,13 @@ { - "branches": "String", - "commits": "String", - "compilers": "String", + "branches": "Array(String)", + "commits": "Array(String)", "device": "String", - "dtypes": "String", - "getJobId": "Bool", + "dtypes": "Array(String)", "granularity": "String", "mode": "String", + "repo": "String", "startTime": "DateTime64(3)", "stopTime": "DateTime64(3)", - "suites": "String", - "timezone": "String", + "suites": "Array(String)", "workflowId": "Int64" -} \ No newline at end of file +} diff --git a/torchci/clickhouse_queries/torchao_query/query.sql b/torchci/clickhouse_queries/torchao_query/query.sql index b5ae2b7bd0..71538b126d 100644 --- a/torchci/clickhouse_queries/torchao_query/query.sql +++ b/torchci/clickhouse_queries/torchao_query/query.sql @@ -1,160 +1,76 @@ --- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted -WITH performance_results AS ( - SELECT - name, - IF(speedup = 'infra_error', NULL, speedup) AS speedup, -- Handle the recent burst of infra error - REPLACE( - filename, - CONCAT( - '_', : dtypes, '_', : mode, '_', : device, - '_performance' - ) - ) AS filename, - compilation_latency, - compression_ratio, - abs_latency, - mfu, - memory_bandwidth, - dynamo_peak_mem, - eager_peak_mem, - workflow_id, - CAST(job_id AS INT) AS job_id, - FORMAT_ISO8601( - DATE_TRUNC(: granularity, _event_time) - ) AS granularity_bucket, - head_sha, - head_branch, -FROM - inductor.torchao_perf_stats -WHERE - filename LIKE '%_performance' - AND filename LIKE CONCAT( - '%_', : dtypes, '_', : mode, '_', : device, - '_%' - ) - AND _event_time >= PARSE_DATETIME_ISO8601(:startTime) - AND _event_time < PARSE_DATETIME_ISO8601(:stopTime) - AND (workflow_id = :workflowId OR :workflowId = 0) -), -accuracy_results AS ( - SELECT - name, - accuracy, - REPLACE( - filename, - CONCAT( - '_', : dtypes, '_', : mode, '_', : device, - '_accuracy' - ) - ) AS filename, - workflow_id, - CAST(job_id AS INT) AS job_id, - FROM - inductor.torchao_perf_stats - WHERE - filename LIKE '%_accuracy' - AND filename LIKE CONCAT( - '%_', : dtypes, '_', : mode, '_', : device, - '_%' - ) - AND _event_time >= PARSE_DATETIME_ISO8601(:startTime) - AND _event_time < PARSE_DATETIME_ISO8601(:stopTime) - AND (workflow_id = :workflowId OR :workflowId = 0) - AND accuracy != 'model_fail_to_load' - AND accuracy != 'eager_fail_to_run' -), -results AS ( - SELECT - performance_results.granularity_bucket AS granularity_bucket, - performance_results.workflow_id AS workflow_id, - performance_results.job_id AS job_id, - performance_results.head_branch AS head_branch, - performance_results.head_sha AS head_sha, - CASE - WHEN performance_results.filename LIKE '%_torchbench' THEN 'torchbench' - WHEN performance_results.filename LIKE '%_timm_models' THEN 'timm_models' - WHEN performance_results.filename LIKE '%_huggingface' THEN 'huggingface' - ELSE NULL - END AS suite, - CASE - WHEN performance_results.filename LIKE '%_torchbench' THEN REPLACE( - performance_results.filename, '_torchbench' - ) - WHEN performance_results.filename LIKE '%_timm_models' THEN REPLACE( - performance_results.filename, '_timm_models' - ) - WHEN performance_results.filename LIKE '%_huggingface' THEN REPLACE( - performance_results.filename, '_huggingface' - ) - ELSE NULL - END AS compiler, - performance_results.name, - IF(TRY_CAST(speedup AS FLOAT) IS NOT NULL, - CAST(speedup AS FLOAT), - 0.0 - ) AS speedup, - accuracy_results.accuracy AS accuracy, - IF(TRY_CAST(compilation_latency AS FLOAT) IS NOT NULL, - CAST(compilation_latency AS FLOAT), - 0.0 - ) AS compilation_latency, - IF(TRY_CAST(compression_ratio AS FLOAT) IS NOT NULL, - CAST(compression_ratio AS FLOAT), - 0.0 - ) AS compression_ratio, - IF(TRY_CAST(abs_latency AS FLOAT) IS NOT NULL, - CAST(abs_latency AS FLOAT), - 0.0 - ) AS abs_latency, - IF(TRY_CAST(mfu AS FLOAT) IS NOT NULL, - CAST(mfu AS FLOAT), - 0.0 - ) AS mfu, - IF(TRY_CAST(memory_bandwidth AS FLOAT) IS NOT NULL, - CAST(memory_bandwidth AS FLOAT), - 0.0 - ) AS memory_bandwidth, - IF(TRY_CAST(dynamo_peak_mem AS FLOAT) IS NOT NULL, - CAST(dynamo_peak_mem AS FLOAT), - 0.0 - ) AS dynamo_peak_mem, - IF(TRY_CAST(eager_peak_mem AS FLOAT) IS NOT NULL, - CAST(eager_peak_mem AS FLOAT), - 0.0 - ) AS eager_peak_mem, - FROM - performance_results - LEFT JOIN accuracy_results ON performance_results.name = accuracy_results.name - AND performance_results.filename = accuracy_results.filename - AND performance_results.workflow_id = accuracy_results.workflow_id +-- This powers HUD TorchAO benchmarks dashboards +WITH benchmarks AS ( + SELECT + o.model.origins [ 1 ] AS suite, + o.model.name AS model, + tupleElement(o.benchmark, 'extra_info') [ 'quantization' ] AS dtype, + o.metric.name AS metric, + floor(arrayAvg(o.metric.benchmark_values), 2) AS value, + tupleElement(o.metric, 'extra_info') AS extra_info, + replaceOne(o.head_branch, 'refs/heads/', '') AS head_branch, + o.head_sha AS head_sha, + o.workflow_id AS workflow_id, + o.job_id AS job_id, + DATE_TRUNC( + {granularity: String }, + fromUnixTimestamp(o.timestamp) + ) AS granularity_bucket + FROM + benchmark.oss_ci_benchmark_v3 o + WHERE + o.timestamp >= toUnixTimestamp({startTime: DateTime64(3) }) + AND o.timestamp < toUnixTimestamp({stopTime: DateTime64(3) }) + AND o.repo = {repo: String } + AND ( + has({commits: Array(String) }, o.head_sha) + OR empty({commits: Array(String) }) + ) + AND ( + has({suites: Array(String) }, suite) + OR empty({suites: Array(String) }) + ) + AND ( + has({dtypes: Array(String) }, dtype) + OR empty({dtypes: Array(String) }) + ) + AND tupleElement(o.benchmark, 'mode') = {mode: String } + AND tupleElement(o.benchmark, 'extra_info') [ 'device' ] = {device: String } + AND ( + workflow_id = {workflowId: Int64} + OR {workflowId: Int64} = 0 + ) + AND ( + o.metric.name in [ 'accuracy', + 'speedup', + 'compilation_latency', + 'compression_ratio', + 'abs_latency', + 'mfu', + 'memory_bandwidth', + 'dynamo_peak_mem', + 'eager_peak_mem' ] + ) ) -SELECT DISTINCT - results.workflow_id, - -- As the JSON response is pretty big, only return the field if it's needed - IF(:getJobId, results.job_id, NULL) AS job_id, - results.suite, - results.compiler, - results.name, - results.speedup, - results.accuracy, - results.compilation_latency, - results.compression_ratio, - results.abs_latency, - results.mfu, - results.memory_bandwidth, - results.dynamo_peak_mem, - results.eager_peak_mem, - results.granularity_bucket, +SELECT + suite, + model, + dtype, + metric, + value, + extra_info, + workflow_id, + job_id, + granularity_bucket FROM - results + benchmarks WHERE - ARRAY_CONTAINS(SPLIT(:suites, ','), LOWER(results.suite)) - AND (ARRAY_CONTAINS(SPLIT(:compilers, ','), LOWER(results.compiler)) OR :compilers = '') - AND (ARRAY_CONTAINS(SPLIT(:branches, ','), results.head_branch) OR :branches = '') - AND (ARRAY_CONTAINS(SPLIT(:commits, ','), results.head_sha) OR :commits = '') + ( + has({branches: Array(String) }, head_branch) + OR empty({branches: Array(String) }) + ) ORDER BY - granularity_bucket DESC, - workflow_id DESC, - suite ASC, - compiler ASC, - name ASC \ No newline at end of file + granularity_bucket DESC, + workflow_id DESC, + suite ASC, + dtype ASC, + model ASC diff --git a/torchci/clickhouse_queries/torchao_query_branches/params.json b/torchci/clickhouse_queries/torchao_query_branches/params.json index 3f7013e3ee..64518741c7 100644 --- a/torchci/clickhouse_queries/torchao_query_branches/params.json +++ b/torchci/clickhouse_queries/torchao_query_branches/params.json @@ -1,8 +1,8 @@ { + "dtypes": "Array(String)", "device": "String", - "dtypes": "String", - "granularity": "String", "mode": "String", + "repo": "String", "startTime": "DateTime64(3)", "stopTime": "DateTime64(3)" -} \ No newline at end of file +} diff --git a/torchci/clickhouse_queries/torchao_query_branches/query.sql b/torchci/clickhouse_queries/torchao_query_branches/query.sql index 088cf20957..d22b615615 100644 --- a/torchci/clickhouse_queries/torchao_query_branches/query.sql +++ b/torchci/clickhouse_queries/torchao_query_branches/query.sql @@ -1,22 +1,35 @@ --- !!! Query is not converted to CH syntax yet. Delete this line when it gets converted +-- This query is used to get the list of branches and commits used by different +-- OSS CI benchmark experiments. This powers HUD TorchAO benchmarks dashboards +WITH benchmarks AS ( + SELECT + o.head_branch AS head_branch, + o.head_sha AS head_sha, + o.workflow_id AS id, + toStartOfDay(fromUnixTimestamp(o.timestamp)) AS event_time + FROM + benchmark.oss_ci_benchmark_v3 o + WHERE + o.timestamp >= toUnixTimestamp({startTime: DateTime64(3) }) + AND o.timestamp < toUnixTimestamp({stopTime: DateTime64(3) }) + AND o.repo = {repo: String } + AND tupleElement(o.benchmark, 'extra_info') [ 'performance' ] = 'true' + AND ( + has( + {dtypes: Array(String) }, + tupleElement(o.benchmark, 'extra_info') [ 'quantization' ] + ) + OR empty({dtypes: Array(String) }) + ) + AND tupleElement(o.benchmark, 'mode') = {mode: String } + AND tupleElement(o.benchmark, 'extra_info') [ 'device' ] = {device: String } +) SELECT - DISTINCT head_branch, - head_sha, - FORMAT_ISO8601( - DATE_TRUNC( - : granularity, _event_time - ) - ) AS event_time, + DISTINCT replaceOne(head_branch, 'refs/heads/', '') AS head_branch, + head_sha, + id, + event_time FROM - inductor.torchao_perf_stats -WHERE - torchao_perf_stats._event_time >= PARSE_DATETIME_ISO8601(: startTime) - AND torchao_perf_stats._event_time < PARSE_DATETIME_ISO8601(: stopTime) - AND torchao_perf_stats.filename LIKE '%_performance' - AND torchao_perf_stats.filename LIKE CONCAT( - '%_', : dtypes, '_', : mode, '_', : device, - '_%' - ) + benchmarks ORDER BY - head_branch, - event_time DESC + head_branch, + event_time DESC diff --git a/torchci/components/benchmark/compilers/CompilerGraphGroup.tsx b/torchci/components/benchmark/compilers/CompilerGraphGroup.tsx index 6d9b70704b..061f3f7453 100644 --- a/torchci/components/benchmark/compilers/CompilerGraphGroup.tsx +++ b/torchci/components/benchmark/compilers/CompilerGraphGroup.tsx @@ -12,25 +12,34 @@ const GraphCardGroup = styled(Card)({ /** Mui Styles */ export default function CompilerGraphGroup({ + dashboard, suiteConfig, queryParams, granularity, lBranchAndCommit, rBranchAndCommit, }: { - lBranchAndCommit: BranchAndCommit; - rBranchAndCommit: BranchAndCommit; + dashboard: string; + suiteConfig: SuiteConfig; queryParams: { [key: string]: any }; granularity: Granularity; - suiteConfig: SuiteConfig; + lBranchAndCommit: BranchAndCommit; + rBranchAndCommit: BranchAndCommit; }) { + // TODO (huydhn): Remove this once TorchInductor dashboard is migrated to the + // new database schema + const queryName = + dashboard === "torchao" + ? "torchao_query" + : "compilers_benchmark_performance"; + return ( <> ; } + // TODO (huydhn): Remove this once TorchInductor dashboard is migrated to the + // new database schema + data = + queryName === "torchao_query" + ? convertToCompilerPerformanceData(data) + : data; + // Clamp to the nearest granularity (e.g. nearest hour) so that the times will // align with the data we get from the database const startTime = dayjs(queryParams["startTime"]).startOf(granularity); diff --git a/torchci/components/benchmark/torchao/common.tsx b/torchci/components/benchmark/torchao/common.tsx index 2d4ec110df..708112690d 100644 --- a/torchci/components/benchmark/torchao/common.tsx +++ b/torchci/components/benchmark/torchao/common.tsx @@ -1,5 +1,10 @@ -export const DEFAULT_REPO_NAME = "pytorch/ao"; -export const DTYPES = ["amp", "bfloat16"]; +export const DEFAULT_REPO_NAME = "pytorch/benchmark"; +export const QUANTIZATIONS = [ + "autoquant", + "int8dynamic", + "int8weightonly", + "noquant", +]; export const DEFAULT_MODE = "inference"; export const DEFAULT_DEVICE_NAME = "cuda (a100)"; diff --git a/torchci/lib/benchmark/aoUtils.ts b/torchci/lib/benchmark/aoUtils.ts new file mode 100644 index 0000000000..923724ca7f --- /dev/null +++ b/torchci/lib/benchmark/aoUtils.ts @@ -0,0 +1,45 @@ +import { BenchmarkData, CompilerPerformanceData } from "lib/types"; + +// TODO (huydhn): Use this function to convert the generic benchmark data to the old +// CompilerPerformanceData format. This is needed until the TorchInductor dashboard +// is migrated to the new format +export function convertToCompilerPerformanceData(data: BenchmarkData[]) { + const convertData: { [model: string]: CompilerPerformanceData } = {}; + if (data === undefined || data === null) { + return []; + } + + data.forEach((r: BenchmarkData) => { + const k = `${r.granularity_bucket} ${r.model}`; + + if (!(k in convertData)) { + convertData[k] = { + abs_latency: 0, + accuracy: "", + compilation_latency: 0, + compiler: "default", + compression_ratio: 0, + dynamo_peak_mem: 0, + eager_peak_mem: 0, + granularity_bucket: r.granularity_bucket, + name: r.model, + speedup: 0, + suite: r.suite, + workflow_id: r.workflow_id, + job_id: r.job_id, + }; + } + + // Accuracy metric has a string value instead of a number https://github.com/pytorch/pytorch/pull/143611 + if (r.metric === "accuracy") { + convertData[k][r.metric] = JSON.parse( + r.extra_info["benchmark_values"] + )[0]; + } else { + // @ts-expect-error + convertData[k][r.metric] = r.value; + } + }); + + return Object.values(convertData); +} diff --git a/torchci/lib/types.ts b/torchci/lib/types.ts index b2ccd56962..82167618f8 100644 --- a/torchci/lib/types.ts +++ b/torchci/lib/types.ts @@ -204,6 +204,18 @@ export interface CompilerPerformanceData { job_id?: number; } +export interface BenchmarkData { + extra_info: { [key: string]: string }; + granularity_bucket: string; + job_id: number; + metric: string; + model: string; + quantization?: string; + suite: string; + value: number; + workflow_id: number; +} + export interface BranchAndCommit { branch: string; commit: string; diff --git a/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx b/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx index a132aa49b7..763fee376b 100644 --- a/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx +++ b/torchci/pages/benchmark/[suite]/[compiler]/[[...page]].tsx @@ -24,10 +24,12 @@ import { ModePicker, MODES, } from "components/benchmark/ModeAndDTypePicker"; +import { QUANTIZATIONS } from "components/benchmark/torchao/common"; import CopyLink from "components/CopyLink"; import GranularityPicker from "components/GranularityPicker"; import { Granularity } from "components/metrics/panels/TimeSeriesPanel"; import dayjs from "dayjs"; +import { convertToCompilerPerformanceData } from "lib/benchmark/aoUtils"; import { augmentData } from "lib/benchmark/compilerUtils"; import { fetcher } from "lib/GeneralUtils"; import { BranchAndCommit, CompilerPerformanceData } from "lib/types"; @@ -80,6 +82,10 @@ function Report({ let { data: lData, error: _lError } = useSWR(lUrl, fetcher, { refreshInterval: 60 * 60 * 1000, // refresh every hour }); + // TODO (huydhn): Remove this once TorchInductor dashboard is migrated to the + // new database schema + lData = + dashboard === "torchao" ? convertToCompilerPerformanceData(lData) : lData; lData = augmentData(lData); lData = lData ? lData.filter((e: CompilerPerformanceData) => e.suite === suite) @@ -98,6 +104,10 @@ function Report({ let { data: rData, error: _rError } = useSWR(rUrl, fetcher, { refreshInterval: 60 * 60 * 1000, // refresh every hour }); + // TODO (huydhn): Remove this once TorchInductor dashboard is migrated to the + // new database schema + rData = + dashboard === "torchao" ? convertToCompilerPerformanceData(rData) : rData; rData = augmentData(rData); rData = rData ? rData.filter((e: CompilerPerformanceData) => e.suite === suite) @@ -110,7 +120,9 @@ function Report({ return (
@@ -171,7 +184,7 @@ export default function Page() { const compiler: string = (router.query.compiler as string) ?? undefined; const model: string = (router.query.model as string) ?? undefined; const dashboard: string = - (router.query.dashboard as string) ?? "TorchInductor"; + (router.query.dashboard as string) ?? "torchinductor"; const queryName: string = DASHBOARD_QUERY_MAP[dashboard] ?? "compilers_benchmark_performance"; const branchQueryName = queryName + "_branches"; @@ -264,19 +277,37 @@ export default function Page() { return ; } - const queryParams: { [key: string]: any } = { - commits: [], - compilers: [compiler], - device: DISPLAY_NAMES_TO_DEVICE_NAMES[deviceName], - dtypes: dtype, - getJobId: false, - granularity: granularity, - mode: mode, - startTime: dayjs(startTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), - stopTime: dayjs(stopTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), - suites: [], - workflowId: 0, - }; + // TODO (huydhn): Remove this once TorchInductor dashboard is migrated to the + // new database schema + const queryParams: { [key: string]: any } = + dashboard === "torchao" + ? { + branches: [], + commits: [], + compilers: [compiler], + device: DISPLAY_NAMES_TO_DEVICE_NAMES[deviceName], + dtypes: [dtype], + granularity: granularity, + mode: mode, + repo: "pytorch/benchmark", + startTime: dayjs(startTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + stopTime: dayjs(stopTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + suites: [], + workflowId: 0, + } + : { + commits: [], + compilers: [compiler], + device: DISPLAY_NAMES_TO_DEVICE_NAMES[deviceName], + dtypes: dtype, + getJobId: false, + granularity: granularity, + mode: mode, + startTime: dayjs(startTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + stopTime: dayjs(stopTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + suites: [], + workflowId: 0, + }; return (
@@ -312,12 +343,22 @@ export default function Page() { granularity={granularity} setGranularity={setGranularity} /> - + {dashboard === "torchao" && ( + + )} + {dashboard === "torchinductor" && ( + + )} ; } + lData = convertToCompilerPerformanceData(lData); + rData = convertToCompilerPerformanceData(rData); + return (
@@ -152,15 +124,22 @@ function Report({ }} all_suites={SUITES} /> - + {Array.from(Object.values(COMPILER_SUITES_MAP)).map((suiteConfig) => { + return ( + suiteConfig.showGraph && ( +
+ +
+ ) + ); + })}
); } @@ -177,7 +156,7 @@ export default function Page() { const [granularity, setGranularity] = useState("hour"); const [suite, setSuite] = useState(Object.keys(SUITES)[0]); const [mode, setMode] = useState(DEFAULT_MODE); - const [dtype, setDType] = useState(MODES[DEFAULT_MODE]); + const [dtype, setDType] = useState(QUANTIZATIONS[0]); const [lBranch, setLBranch] = useState(MAIN_BRANCH); const [lCommit, setLCommit] = useState(""); const [rBranch, setRBranch] = useState(MAIN_BRANCH); @@ -258,43 +237,20 @@ export default function Page() { ); }, [router.query]); - const queryParams: RocksetParam[] = [ - { - name: "timezone", - type: "string", - value: Intl.DateTimeFormat().resolvedOptions().timeZone, - }, - { - name: "startTime", - type: "string", - value: startTime, - }, - { - name: "stopTime", - type: "string", - value: stopTime, - }, - { - name: "granularity", - type: "string", - value: granularity, - }, - { - name: "mode", - type: "string", - value: mode, - }, - { - name: "dtypes", - type: "string", - value: dtype, - }, - { - name: "device", - type: "string", - value: DISPLAY_NAMES_TO_DEVICE_NAMES[deviceName], - }, - ]; + const queryParams: { [key: string]: any } = { + branches: [], + commits: [], + compilers: [], + device: DISPLAY_NAMES_TO_DEVICE_NAMES[deviceName], + dtypes: [dtype], + granularity: granularity, + mode: mode, + repo: DEFAULT_REPO_NAME, + startTime: dayjs(startTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + stopTime: dayjs(stopTime).utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), + suites: Object.keys(SUITES), + workflowId: 0, + }; return (
@@ -307,7 +263,7 @@ export default function Page() { startTime.toString() )}&stopTime=${encodeURIComponent( stopTime.toString() - )}&granularity=${granularity}&suite=${suite}&mode=${mode}&dtype=${dtype}&deviceName=${encodeURIComponent( + )}&granularity=${granularity}&mode=${mode}&dtype=${dtype}&deviceName=${encodeURIComponent( deviceName )}&lBranch=${lBranch}&lCommit=${lCommit}&rBranch=${rBranch}&rCommit=${rCommit}`} /> @@ -326,13 +282,17 @@ export default function Page() { granularity={granularity} setGranularity={setGranularity} /> - - + —Diff→ @@ -367,16 +327,14 @@ export default function Page() { titlePrefix={"New"} fallbackIndex={0} // Default to the latest commit timeRange={timeRange} - useClickHouse={false} + useClickHouse={true} /> -