diff --git a/benchmarks/graphics_pipeline/GraphicsBenchmarkApp.cpp b/benchmarks/graphics_pipeline/GraphicsBenchmarkApp.cpp index f92a9a860..45b2965d3 100644 --- a/benchmarks/graphics_pipeline/GraphicsBenchmarkApp.cpp +++ b/benchmarks/graphics_pipeline/GraphicsBenchmarkApp.cpp @@ -234,6 +234,56 @@ void GraphicsBenchmarkApp::Setup() } } +void GraphicsBenchmarkApp::SetupMetrics() +{ + Application::SetupMetrics(); + ppx::metrics::MetricMetadata metadata = {ppx::metrics::MetricType::GAUGE, "CPU Submission Time", "ms", ppx::metrics::MetricInterpretation::LOWER_IS_BETTER, {0.f, 10000.f}}; + mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime] = AddMetric(metadata); + PPX_ASSERT_MSG(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime] != ppx::metrics::kInvalidMetricID, "Failed to add CPU Submission Time metric"); + + metadata = {ppx::metrics::MetricType::GAUGE, "Bandwidth", "GB/s", ppx::metrics::MetricInterpretation::HIGHER_IS_BETTER, {0.f, 10000.f}}; + mMetricsData.metrics[MetricsData::kTypeBandwidth] = AddMetric(metadata); + PPX_ASSERT_MSG(mMetricsData.metrics[MetricsData::kTypeBandwidth] != ppx::metrics::kInvalidMetricID, "Failed to add Bandwidth metric"); +} + +void GraphicsBenchmarkApp::UpdateMetrics() +{ + uint64_t frequency = 0; + GetGraphicsQueue()->GetTimestampFrequency(&frequency); + + ppx::metrics::MetricData data = {ppx::metrics::MetricType::GAUGE}; + data.gauge.seconds = GetElapsedSeconds(); + data.gauge.value = mCPUSubmissionTime; + RecordMetricData(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime], data); + + const float gpuWorkDurationInSec = static_cast(mGpuWorkDuration / static_cast(frequency)); + const uint32_t width = GetSwapchain()->GetWidth(); + const uint32_t height = GetSwapchain()->GetHeight(); + const uint32_t quadCount = pFullscreenQuadsCount->GetValue(); + + if (quadCount) { + // Skip the first kSkipFrameCount frames after the knob of quad count being changed to avoid noise + constexpr uint32_t kSkipFrameCount = 2; + static uint32_t skipFrameCounter = 0; + if (pFullscreenQuadsCount->DigestUpdate()) { + skipFrameCounter = kSkipFrameCount; + } + + if (skipFrameCounter == 0) { + const float dataWriteInGb = (static_cast(width) * static_cast(height) * 4.f * quadCount) / (1024.f * 1024.f * 1024.f); + const float bandwidth = dataWriteInGb / gpuWorkDurationInSec; + + ppx::metrics::MetricData data = {ppx::metrics::MetricType::GAUGE}; + data.gauge.seconds = GetElapsedSeconds(); + data.gauge.value = bandwidth; + RecordMetricData(mMetricsData.metrics[MetricsData::kTypeBandwidth], data); + } + else { + --skipFrameCounter; + } + } +} + void GraphicsBenchmarkApp::SetupSkyBoxResources() { // Textures @@ -889,18 +939,7 @@ void GraphicsBenchmarkApp::Render() Timer timerSubmit; PPX_ASSERT_MSG(timerSubmit.Start() == TIMER_RESULT_SUCCESS, "Error starting the Timer"); PPX_CHECKED_CALL(GetGraphicsQueue()->Submit(&submitInfo)); - double t = timerSubmit.MillisSinceStart(); - - uint64_t frameCount = GetFrameCount(); - mSubmissionTime.average = (mSubmissionTime.average * frameCount + t) / (frameCount + 1); - if (frameCount == 0) { - mSubmissionTime.min = t; - mSubmissionTime.max = t; - } - else { - mSubmissionTime.min = std::min(mSubmissionTime.min, t); - mSubmissionTime.max = std::max(mSubmissionTime.max, t); - } + mCPUSubmissionTime = timerSubmit.MillisSinceStart(); #if defined(PPX_BUILD_XR) // No need to present when XR is enabled. @@ -949,25 +988,27 @@ void GraphicsBenchmarkApp::UpdateGUI() void GraphicsBenchmarkApp::DrawExtraInfo() { - uint64_t frequency = 0; - GetGraphicsQueue()->GetTimestampFrequency(&frequency); + const auto cpuSubmissionTime = GetGaugeBasicStatistics(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime]); + const auto bandwidth = GetGaugeBasicStatistics(mMetricsData.metrics[MetricsData::kTypeBandwidth]); ImGui::Columns(2); ImGui::Text("CPU Average Submission Time"); ImGui::NextColumn(); - ImGui::Text("%.2f ms ", mSubmissionTime.average); + ImGui::Text("%.2f ms", cpuSubmissionTime.average); ImGui::NextColumn(); ImGui::Text("CPU Min Submission Time"); ImGui::NextColumn(); - ImGui::Text("%.2f ms ", mSubmissionTime.min); + ImGui::Text("%.2f ms", cpuSubmissionTime.min); ImGui::NextColumn(); ImGui::Text("CPU Max Submission Time"); ImGui::NextColumn(); - ImGui::Text("%.2f ms ", mSubmissionTime.max); + ImGui::Text("%.2f ms", cpuSubmissionTime.max); ImGui::NextColumn(); + uint64_t frequency = 0; + GetGraphicsQueue()->GetTimestampFrequency(&frequency); ImGui::Columns(2); const float gpuWorkDurationInSec = static_cast(mGpuWorkDuration / static_cast(frequency)); const float gpuWorkDurationInMs = gpuWorkDurationInSec * 1000.0f; @@ -997,10 +1038,19 @@ void GraphicsBenchmarkApp::DrawExtraInfo() ImGui::Text("%.2f GB", dataWriteInGb); ImGui::NextColumn(); - const float bandwidth = dataWriteInGb / gpuWorkDurationInSec; - ImGui::Text("Write Bandwidth"); + ImGui::Text("Average Write Bandwidth"); + ImGui::NextColumn(); + ImGui::Text("%.2f GB/s", bandwidth.average); + ImGui::NextColumn(); + + ImGui::Text("Min Write Bandwidth"); + ImGui::NextColumn(); + ImGui::Text("%.2f GB/s", bandwidth.min); + ImGui::NextColumn(); + + ImGui::Text("Max Write Bandwidth"); ImGui::NextColumn(); - ImGui::Text("%.2f GB/s", bandwidth); + ImGui::Text("%.2f GB/s", bandwidth.max); ImGui::NextColumn(); } ImGui::Columns(1); diff --git a/benchmarks/graphics_pipeline/GraphicsBenchmarkApp.h b/benchmarks/graphics_pipeline/GraphicsBenchmarkApp.h index bd6dbef29..32737ac2a 100644 --- a/benchmarks/graphics_pipeline/GraphicsBenchmarkApp.h +++ b/benchmarks/graphics_pipeline/GraphicsBenchmarkApp.h @@ -225,7 +225,7 @@ class GraphicsBenchmarkApp uint64_t mGpuWorkDuration; grfx::SamplerPtr mLinearSampler; grfx::DescriptorPoolPtr mDescriptorPool; - SubmissionTime mSubmissionTime; + double mCPUSubmissionTime = 0.0; // SkyBox resources Entity mSkyBox; @@ -254,6 +254,20 @@ class GraphicsBenchmarkApp std::array mQuadsPipelineInterfaces; std::array mQuadsPs; + // Metrics Data + struct MetricsData + { + enum EMetricsTypes : size_t + { + kTypeCPUSubmissionTime = 0, + kTypeBandwidth, + kCount + }; + + ppx::metrics::MetricID metrics[kCount] = {}; + }; + MetricsData mMetricsData; + private: std::shared_ptr> pKnobVs; std::shared_ptr> pKnobPs; @@ -298,6 +312,10 @@ class GraphicsBenchmarkApp void SetupSpheresPipelines(); void SetupFullscreenQuadsPipelines(); + // Metrics related functions + virtual void SetupMetrics() override; + virtual void UpdateMetrics() override; + Result CompileSpherePipeline(const SpherePipelineKey& key); // Update descriptors diff --git a/include/ppx/application.h b/include/ppx/application.h index c6e4262fe..8004a703f 100644 --- a/include/ppx/application.h +++ b/include/ppx/application.h @@ -384,6 +384,8 @@ class Application // Thus it should always be called once per frame. Virtual for unit testing purposes. virtual void UpdateMetrics() {} + virtual metrics::GaugeBasicStatistics GetGaugeBasicStatistics(metrics::MetricID id) const; + void TakeScreenshot(); void DrawImGui(grfx::CommandBuffer* pCommandBuffer); diff --git a/include/ppx/metrics.h b/include/ppx/metrics.h index 21c510b67..c403300a9 100644 --- a/include/ppx/metrics.h +++ b/include/ppx/metrics.h @@ -29,9 +29,9 @@ namespace ppx { namespace metrics { -#define METRICS_NO_COPY(TYPE__) \ - TYPE__(TYPE__&&) = delete; \ - TYPE__(const TYPE__&) = delete; \ +#define METRICS_NO_COPY(TYPE__) \ + TYPE__(TYPE__&&) = delete; \ + TYPE__(const TYPE__&) = delete; \ TYPE__& operator=(const TYPE__&) = delete; //////////////////////////////////////////////////////////////////////////////// @@ -109,6 +109,31 @@ class Metric // derived from the sampling process. // The most typical case is the frame time, but memory consumption and image // quality are also good examples. + +// Basic statistics are computed on the fly as metrics entries are recorded. +// They can be retrieved without any significant run-time cost. +struct GaugeBasicStatistics +{ + double min = std::numeric_limits::max(); + double max = std::numeric_limits::min(); + double average = 0.0; + double timeRatio = 0.0; +}; + +// Complex statistics cannot be computed on the fly. +// They require significant computation (e.g. sorting). +struct GaugeComplexStatistics +{ + double median = 0.0; + double standardDeviation = 0.0; + double percentile01 = 0.0; + double percentile05 = 0.0; + double percentile10 = 0.0; + double percentile90 = 0.0; + double percentile95 = 0.0; + double percentile99 = 0.0; +}; + class MetricGauge final : public Metric { friend class Run; @@ -133,6 +158,8 @@ class MetricGauge final : public Metric return MetricType::GAUGE; } + const GaugeBasicStatistics GetBasicStatistics() const { return mStats.basic; }; + private: struct TimeSeriesEntry { @@ -142,21 +169,8 @@ class MetricGauge final : public Metric struct Stats { - // Basic - updated every entry. - double min = std::numeric_limits::max(); - double max = std::numeric_limits::min(); - double average = 0.0; - double timeRatio = 0.0; - - // Complex - computed on request. - double median = 0.0; - double standardDeviation = 0.0; - double percentile01 = 0.0; - double percentile05 = 0.0; - double percentile10 = 0.0; - double percentile90 = 0.0; - double percentile95 = 0.0; - double percentile99 = 0.0; + GaugeBasicStatistics basic; + GaugeComplexStatistics complex; }; private: @@ -172,7 +186,7 @@ class MetricGauge final : public Metric private: MetricMetadata mMetadata; std::vector mTimeSeries; - Stats mBasicStats; + Stats mStats; double mAccumulatedValue = 0.0; }; @@ -292,6 +306,9 @@ class Manager final // current run. Report CreateReport(const std::string& reportPath) const; + // Get Gauge Basic Statistics, only works for type GAUGE + GaugeBasicStatistics GetGaugeBasicStatistics(MetricID id) const; + private: METRICS_NO_COPY(Manager) diff --git a/src/ppx/application.cpp b/src/ppx/application.cpp index 8b0066ab6..0d1437e8d 100644 --- a/src/ppx/application.cpp +++ b/src/ppx/application.cpp @@ -786,6 +786,15 @@ void Application::ShutdownMetrics() StopMetricsRun(); } +metrics::GaugeBasicStatistics Application::GetGaugeBasicStatistics(metrics::MetricID id) const +{ + if (!mStandardOpts.pEnableMetrics->GetValue()) { + PPX_LOG_WARN("Metrics is not enabled."); + return metrics::GaugeBasicStatistics(); + } + return mMetrics.manager.GetGaugeBasicStatistics(id); +} + void Application::SaveMetricsReportToDisk() { // Ensure the base metrics knob was initialized by the KnobManager. @@ -855,8 +864,9 @@ void Application::InitStandardKnobs() "Prints a list of the available GPUs on the current system with their " "index and exits (see --gpu)."); + const std::string defaultPath = std::filesystem::current_path().u8string(); mStandardOpts.pMetricsFilename = - mKnobManager.CreateKnob>("metrics-filename", ""); + mKnobManager.CreateKnob>("metrics-filename", defaultPath); mStandardOpts.pMetricsFilename->SetFlagDescription( "If metrics are enabled, save the metrics report to the " "provided filename (including path). If used, any `@` " @@ -1708,7 +1718,7 @@ void Application::StartMetricsRun(const std::string& name) metadata.unit = ""; metadata.interpretation = metrics::MetricInterpretation::HIGHER_IS_BETTER; mMetrics.framerateId = mMetrics.manager.AddMetric(metadata); - PPX_ASSERT_MSG(mMetrics.cpuFrameTimeId != metrics::kInvalidMetricID, "Failed to create framerate metric"); + PPX_ASSERT_MSG(mMetrics.framerateId != metrics::kInvalidMetricID, "Failed to create framerate metric"); } { metrics::MetricMetadata metadata = {}; @@ -1717,7 +1727,7 @@ void Application::StartMetricsRun(const std::string& name) metadata.unit = ""; metadata.interpretation = metrics::MetricInterpretation::NONE; mMetrics.frameCountId = mMetrics.manager.AddMetric(metadata); - PPX_ASSERT_MSG(mMetrics.cpuFrameTimeId != metrics::kInvalidMetricID, "Failed to create frame count metric"); + PPX_ASSERT_MSG(mMetrics.frameCountId != metrics::kInvalidMetricID, "Failed to create frame count metric"); } mMetrics.resetFramerateTracking = true; diff --git a/src/ppx/metrics.cpp b/src/ppx/metrics.cpp index d42917277..2b59bc3f2 100644 --- a/src/ppx/metrics.cpp +++ b/src/ppx/metrics.cpp @@ -62,13 +62,13 @@ bool MetricGauge::RecordEntry(const MetricData& data) // Update the basic stats. mAccumulatedValue += entry.value; - mBasicStats.min = std::min(mBasicStats.min, entry.value); - mBasicStats.max = std::max(mBasicStats.max, entry.value); - mBasicStats.average = mAccumulatedValue / entryCount; + mStats.basic.min = std::min(mStats.basic.min, entry.value); + mStats.basic.max = std::max(mStats.basic.max, entry.value); + mStats.basic.average = mAccumulatedValue / entryCount; // Above checks guarantee the 'seconds' field monotonically increases with each entry. - mBasicStats.timeRatio = (entryCount > 1) - ? mAccumulatedValue / (entry.seconds - mTimeSeries.front().seconds) - : entry.value; + mStats.basic.timeRatio = (entryCount > 1) + ? mAccumulatedValue / (entry.seconds - mTimeSeries.front().seconds) + : entry.value; mTimeSeries.emplace_back(std::move(entry)); return true; @@ -76,7 +76,7 @@ bool MetricGauge::RecordEntry(const MetricData& data) MetricGauge::Stats MetricGauge::ComputeStats() const { - Stats stats = mBasicStats; + Stats stats = mStats; size_t entryCount = mTimeSeries.size(); if (entryCount == 0) { return stats; @@ -90,24 +90,24 @@ MetricGauge::Stats MetricGauge::ComputeStats() const auto medianIndex = entryCount / 2; // medianIndex is guaranteed to be > 0 when even from above check. - stats.median = (entryCount % 2 == 0) - ? (sorted[medianIndex - 1].value + sorted[medianIndex].value) * 0.5 - : sorted[medianIndex].value; + stats.complex.median = (entryCount % 2 == 0) + ? (sorted[medianIndex - 1].value + sorted[medianIndex].value) * 0.5 + : sorted[medianIndex].value; double squareDiffSum = 0.0; for (const auto& entry : mTimeSeries) { - double diff = entry.value - stats.average; + double diff = entry.value - stats.basic.average; squareDiffSum += (diff * diff); } - double variance = squareDiffSum / entryCount; - stats.standardDeviation = sqrt(variance); + double variance = squareDiffSum / entryCount; + stats.complex.standardDeviation = sqrt(variance); - stats.percentile01 = sorted[entryCount * 1 / 100].value; - stats.percentile05 = sorted[entryCount * 5 / 100].value; - stats.percentile10 = sorted[entryCount * 10 / 100].value; - stats.percentile90 = sorted[entryCount * 90 / 100].value; - stats.percentile95 = sorted[entryCount * 95 / 100].value; - stats.percentile99 = sorted[entryCount * 99 / 100].value; + stats.complex.percentile01 = sorted[entryCount * 1 / 100].value; + stats.complex.percentile05 = sorted[entryCount * 5 / 100].value; + stats.complex.percentile10 = sorted[entryCount * 10 / 100].value; + stats.complex.percentile90 = sorted[entryCount * 90 / 100].value; + stats.complex.percentile95 = sorted[entryCount * 95 / 100].value; + stats.complex.percentile99 = sorted[entryCount * 99 / 100].value; return stats; } @@ -120,18 +120,18 @@ nlohmann::json MetricGauge::Export() const metricObject["metadata"] = mMetadata.Export(); Stats stats = ComputeStats(); - statsObject["min"] = stats.min; - statsObject["max"] = stats.max; - statsObject["average"] = stats.average; - statsObject["time_ratio"] = stats.timeRatio; - statsObject["median"] = stats.median; - statsObject["standard_deviation"] = stats.standardDeviation; - statsObject["percentile_01"] = stats.percentile01; - statsObject["percentile_05"] = stats.percentile05; - statsObject["percentile_10"] = stats.percentile10; - statsObject["percentile_90"] = stats.percentile90; - statsObject["percentile_95"] = stats.percentile95; - statsObject["percentile_99"] = stats.percentile99; + statsObject["min"] = stats.basic.min; + statsObject["max"] = stats.basic.max; + statsObject["average"] = stats.basic.average; + statsObject["time_ratio"] = stats.basic.timeRatio; + statsObject["median"] = stats.complex.median; + statsObject["standard_deviation"] = stats.complex.standardDeviation; + statsObject["percentile_01"] = stats.complex.percentile01; + statsObject["percentile_05"] = stats.complex.percentile05; + statsObject["percentile_10"] = stats.complex.percentile10; + statsObject["percentile_90"] = stats.complex.percentile90; + statsObject["percentile_95"] = stats.complex.percentile95; + statsObject["percentile_99"] = stats.complex.percentile99; metricObject["statistics"] = statsObject; @@ -160,8 +160,8 @@ bool MetricCounter::RecordEntry(const MetricData& data) nlohmann::json MetricCounter::Export() const { nlohmann::json metricObject; - metricObject["metadata"] = mMetadata.Export(); - metricObject["value"] = mCounter; + metricObject["metadata"] = mMetadata.Export(); + metricObject["value"] = mCounter; metricObject["entry_count"] = mEntryCount; return metricObject; } @@ -205,7 +205,7 @@ bool Run::HasMetric(const std::string& name) const nlohmann::json Run::Export() const { nlohmann::json object; - object["name"] = mName; + object["name"] = mName; object["gauges"] = nlohmann::json::array(); object["counters"] = nlohmann::json::array(); for (const auto& metric : mMetrics) { @@ -294,6 +294,26 @@ Report Manager::CreateReport(const std::string& reportPath) const return Report(std::move(content), reportPath); } +GaugeBasicStatistics Manager::GetGaugeBasicStatistics(MetricID id) const +{ + if (mActiveRun == nullptr) { + PPX_LOG_WARN("Attempted to record a metric entry with no active run."); + return GaugeBasicStatistics(); + } + auto findResult = mActiveMetrics.find(id); + if (findResult == mActiveMetrics.end()) { + PPX_LOG_ERROR("Attempted to record a metric entry against an invalid ID."); + return GaugeBasicStatistics(); + } + + if (findResult->second->GetType() != MetricType::GAUGE) { + PPX_LOG_ERROR("Attempted to get gauge basic statistics from non MetricType::GAUGE type."); + return GaugeBasicStatistics(); + } + + return static_cast(findResult->second)->GetBasicStatistics(); +} + //////////////////////////////////////////////////////////////////////////////// Report::Report(const nlohmann::json& content, const std::string& reportPath)