Use metrics system for benchmark cpu submission time and bandwidth

benchmarks/graphics_pipeline/GraphicsBenchmarkApp.cpp

@@ -234,6 +234,56 @@ void GraphicsBenchmarkApp::Setup()
     }
 }
 
+void GraphicsBenchmarkApp::SetupMetrics()
+{
+    Application::SetupMetrics();
+    ppx::metrics::MetricMetadata metadata                     = {ppx::metrics::MetricType::GAUGE, "CPU Submission Time", "ms", ppx::metrics::MetricInterpretation::LOWER_IS_BETTER, {0.f, 10000.f}};
+    mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime] = AddMetric(metadata);
+    PPX_ASSERT_MSG(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime] != ppx::metrics::kInvalidMetricID, "Failed to add CPU Submission Time metric");
+
+    metadata                                          = {ppx::metrics::MetricType::GAUGE, "Bandwidth", "GB/s", ppx::metrics::MetricInterpretation::HIGHER_IS_BETTER, {0.f, 10000.f}};
+    mMetricsData.metrics[MetricsData::kTypeBandwidth] = AddMetric(metadata);
+    PPX_ASSERT_MSG(mMetricsData.metrics[MetricsData::kTypeBandwidth] != ppx::metrics::kInvalidMetricID, "Failed to add Bandwidth metric");
+}
+
+void GraphicsBenchmarkApp::UpdateMetrics()
+{
+    uint64_t frequency = 0;
+    GetGraphicsQueue()->GetTimestampFrequency(&frequency);
+
+    ppx::metrics::MetricData data = {ppx::metrics::MetricType::GAUGE};
+    data.gauge.seconds            = GetElapsedSeconds();
+    data.gauge.value              = mCPUSubmissionTime;
+    RecordMetricData(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime], data);
+
+    const float    gpuWorkDurationInSec = static_cast<float>(mGpuWorkDuration / static_cast<double>(frequency));
+    const uint32_t width                = GetSwapchain()->GetWidth();
+    const uint32_t height               = GetSwapchain()->GetHeight();
+    const uint32_t quadCount            = pFullscreenQuadsCount->GetValue();
+
+    if (quadCount) {
+        // Skip the first kSkipFrameCount frames after the knob of quad count being changed to avoid noise
+        constexpr uint32_t kSkipFrameCount  = 2;
+        static uint32_t    skipFrameCounter = 0;
+        if (pFullscreenQuadsCount->DigestUpdate()) {
+            skipFrameCounter = kSkipFrameCount;
+        }
+
+        if (skipFrameCounter == 0) {
+            const float dataWriteInGb = (static_cast<float>(width) * static_cast<float>(height) * 4.f * quadCount) / (1024.f * 1024.f * 1024.f);
+            const float bandwidth     = dataWriteInGb / gpuWorkDurationInSec;
+
+            ppx::metrics::MetricData data = {ppx::metrics::MetricType::GAUGE};
+            data.gauge.seconds            = GetElapsedSeconds();
+            data.gauge.value              = bandwidth;
+            RecordMetricData(mMetricsData.metrics[MetricsData::kTypeBandwidth], data);
+        }
+        else {
+            --skipFrameCounter;
+        }
+    }
+}
+
 void GraphicsBenchmarkApp::SetupSkyBoxResources()
 {
     // Textures
@@ -889,18 +939,7 @@ void GraphicsBenchmarkApp::Render()
     Timer timerSubmit;
     PPX_ASSERT_MSG(timerSubmit.Start() == TIMER_RESULT_SUCCESS, "Error starting the Timer");
     PPX_CHECKED_CALL(GetGraphicsQueue()->Submit(&submitInfo));
-    double t = timerSubmit.MillisSinceStart();
-
-    uint64_t frameCount     = GetFrameCount();
-    mSubmissionTime.average = (mSubmissionTime.average * frameCount + t) / (frameCount + 1);
-    if (frameCount == 0) {
-        mSubmissionTime.min = t;
-        mSubmissionTime.max = t;
-    }
-    else {
-        mSubmissionTime.min = std::min(mSubmissionTime.min, t);
-        mSubmissionTime.max = std::max(mSubmissionTime.max, t);
-    }
+    mCPUSubmissionTime = timerSubmit.MillisSinceStart();
 
 #if defined(PPX_BUILD_XR)
     // No need to present when XR is enabled.
@@ -949,25 +988,27 @@ void GraphicsBenchmarkApp::UpdateGUI()
 
 void GraphicsBenchmarkApp::DrawExtraInfo()
 {
-    uint64_t frequency = 0;
-    GetGraphicsQueue()->GetTimestampFrequency(&frequency);
+    const auto cpuSubmissionTime = GetGaugeBasicStatistics(mMetricsData.metrics[MetricsData::kTypeCPUSubmissionTime]);
+    const auto bandwidth         = GetGaugeBasicStatistics(mMetricsData.metrics[MetricsData::kTypeBandwidth]);
 
     ImGui::Columns(2);
     ImGui::Text("CPU Average Submission Time");
     ImGui::NextColumn();
-    ImGui::Text("%.2f ms ", mSubmissionTime.average);
+    ImGui::Text("%.2f ms", cpuSubmissionTime.average);
     ImGui::NextColumn();
 
     ImGui::Text("CPU Min Submission Time");
     ImGui::NextColumn();
-    ImGui::Text("%.2f ms ", mSubmissionTime.min);
+    ImGui::Text("%.2f ms", cpuSubmissionTime.min);
     ImGui::NextColumn();
 
     ImGui::Text("CPU Max Submission Time");
     ImGui::NextColumn();
-    ImGui::Text("%.2f ms ", mSubmissionTime.max);
+    ImGui::Text("%.2f ms", cpuSubmissionTime.max);
     ImGui::NextColumn();
 
+    uint64_t frequency = 0;
+    GetGraphicsQueue()->GetTimestampFrequency(&frequency);
     ImGui::Columns(2);
     const float gpuWorkDurationInSec = static_cast<float>(mGpuWorkDuration / static_cast<double>(frequency));
     const float gpuWorkDurationInMs  = gpuWorkDurationInSec * 1000.0f;
@@ -997,10 +1038,19 @@ void GraphicsBenchmarkApp::DrawExtraInfo()
         ImGui::Text("%.2f GB", dataWriteInGb);
         ImGui::NextColumn();
 
-        const float bandwidth = dataWriteInGb / gpuWorkDurationInSec;
-        ImGui::Text("Write Bandwidth");
+        ImGui::Text("Average Write Bandwidth");
+        ImGui::NextColumn();
+        ImGui::Text("%.2f GB/s", bandwidth.average);
+        ImGui::NextColumn();
+
+        ImGui::Text("Min Write Bandwidth");
+        ImGui::NextColumn();
+        ImGui::Text("%.2f GB/s", bandwidth.min);
+        ImGui::NextColumn();
+
+        ImGui::Text("Max Write Bandwidth");
         ImGui::NextColumn();
-        ImGui::Text("%.2f GB/s", bandwidth);
+        ImGui::Text("%.2f GB/s", bandwidth.max);
         ImGui::NextColumn();
     }
     ImGui::Columns(1);

benchmarks/graphics_pipeline/GraphicsBenchmarkApp.h

@@ -225,7 +225,7 @@ class GraphicsBenchmarkApp
     uint64_t                          mGpuWorkDuration;
     grfx::SamplerPtr                  mLinearSampler;
     grfx::DescriptorPoolPtr           mDescriptorPool;
-    SubmissionTime                    mSubmissionTime;
+    double                            mCPUSubmissionTime = 0.0;
 
     // SkyBox resources
     Entity                mSkyBox;
@@ -254,6 +254,20 @@ class GraphicsBenchmarkApp
     std::array<grfx::PipelineInterfacePtr, kFullscreenQuadsTypes.size()> mQuadsPipelineInterfaces;
     std::array<grfx::ShaderModulePtr, kFullscreenQuadsTypes.size()>      mQuadsPs;
 
+    // Metrics Data
+    struct MetricsData
+    {
+        enum EMetricsTypes : size_t
+        {
+            kTypeCPUSubmissionTime = 0,
+            kTypeBandwidth,
+            kCount
+        };
+
+        ppx::metrics::MetricID metrics[kCount] = {};
+    };
+    MetricsData mMetricsData;
+
 private:
     std::shared_ptr<KnobDropdown<std::string>> pKnobVs;
     std::shared_ptr<KnobDropdown<std::string>> pKnobPs;
@@ -298,6 +312,10 @@ class GraphicsBenchmarkApp
     void SetupSpheresPipelines();
     void SetupFullscreenQuadsPipelines();
 
+    // Metrics related functions
+    virtual void SetupMetrics() override;
+    virtual void UpdateMetrics() override;
+
     Result CompileSpherePipeline(const SpherePipelineKey& key);
 
     // Update descriptors

include/ppx/application.h

@@ -384,6 +384,8 @@ class Application
     // Thus it should always be called once per frame. Virtual for unit testing purposes.
     virtual void UpdateMetrics() {}
 
+    virtual metrics::GaugeBasicStatistics GetGaugeBasicStatistics(metrics::MetricID id) const;
+
     void TakeScreenshot();
 
     void DrawImGui(grfx::CommandBuffer* pCommandBuffer);

include/ppx/metrics.h

@@ -29,9 +29,9 @@
 namespace ppx {
 namespace metrics {
 
-#define METRICS_NO_COPY(TYPE__)     \
-    TYPE__(TYPE__&&)      = delete; \
-    TYPE__(const TYPE__&) = delete; \
+#define METRICS_NO_COPY(TYPE__)                \
+    TYPE__(TYPE__&&)                 = delete; \
+    TYPE__(const TYPE__&)            = delete; \
     TYPE__& operator=(const TYPE__&) = delete;
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -109,6 +109,31 @@ class Metric
 // derived from the sampling process.
 // The most typical case is the frame time, but memory consumption and image
 // quality are also good examples.
+
+// Basic statistics are computed on the fly as metrics entries are recorded.
+// They can be retrieved without any significant run-time cost.
+struct GaugeBasicStatistics
+{
+    double min       = std::numeric_limits<double>::max();
+    double max       = std::numeric_limits<double>::min();
+    double average   = 0.0;
+    double timeRatio = 0.0;
+};
+
+// Complex statistics cannot be computed on the fly.
+// They require significant computation (e.g. sorting).
+struct GaugeComplexStatistics
+{
+    double median            = 0.0;
+    double standardDeviation = 0.0;
+    double percentile01      = 0.0;
+    double percentile05      = 0.0;
+    double percentile10      = 0.0;
+    double percentile90      = 0.0;
+    double percentile95      = 0.0;
+    double percentile99      = 0.0;
+};
+
 class MetricGauge final : public Metric
 {
     friend class Run;
@@ -133,6 +158,8 @@ class MetricGauge final : public Metric
         return MetricType::GAUGE;
     }
 
+    const GaugeBasicStatistics GetBasicStatistics() const { return mStats.basic; };
+
 private:
     struct TimeSeriesEntry
     {
@@ -142,21 +169,8 @@ class MetricGauge final : public Metric
 
     struct Stats
     {
-        // Basic - updated every entry.
-        double min       = std::numeric_limits<double>::max();
-        double max       = std::numeric_limits<double>::min();
-        double average   = 0.0;
-        double timeRatio = 0.0;
-
-        // Complex - computed on request.
-        double median            = 0.0;
-        double standardDeviation = 0.0;
-        double percentile01      = 0.0;
-        double percentile05      = 0.0;
-        double percentile10      = 0.0;
-        double percentile90      = 0.0;
-        double percentile95      = 0.0;
-        double percentile99      = 0.0;
+        GaugeBasicStatistics   basic;
+        GaugeComplexStatistics complex;
     };
 
 private:
@@ -172,7 +186,7 @@ class MetricGauge final : public Metric
 private:
     MetricMetadata               mMetadata;
     std::vector<TimeSeriesEntry> mTimeSeries;
-    Stats                        mBasicStats;
+    Stats                        mStats;
     double                       mAccumulatedValue = 0.0;
 };
 
@@ -292,6 +306,9 @@ class Manager final
     // current run.
     Report CreateReport(const std::string& reportPath) const;
 
+    // Get Gauge Basic Statistics, only works for type GAUGE
+    GaugeBasicStatistics GetGaugeBasicStatistics(MetricID id) const;
+
 private:
     METRICS_NO_COPY(Manager)
 

src/ppx/application.cpp

@@ -786,6 +786,15 @@ void Application::ShutdownMetrics()
     StopMetricsRun();
 }
 
+metrics::GaugeBasicStatistics Application::GetGaugeBasicStatistics(metrics::MetricID id) const
+{
+    if (!mStandardOpts.pEnableMetrics->GetValue()) {
+        PPX_LOG_WARN("Metrics is not enabled.");
+        return metrics::GaugeBasicStatistics();
+    }
+    return mMetrics.manager.GetGaugeBasicStatistics(id);
+}
+
 void Application::SaveMetricsReportToDisk()
 {
     // Ensure the base metrics knob was initialized by the KnobManager.
@@ -855,8 +864,9 @@ void Application::InitStandardKnobs()
         "Prints a list of the available GPUs on the current system with their "
         "index and exits (see --gpu).");
 
+    const std::string defaultPath = std::filesystem::current_path().u8string();
     mStandardOpts.pMetricsFilename =
-        mKnobManager.CreateKnob<KnobFlag<std::string>>("metrics-filename", "");
+        mKnobManager.CreateKnob<KnobFlag<std::string>>("metrics-filename", defaultPath);
     mStandardOpts.pMetricsFilename->SetFlagDescription(
         "If metrics are enabled, save the metrics report to the "
         "provided filename (including path). If used, any `@` "
@@ -1708,7 +1718,7 @@ void Application::StartMetricsRun(const std::string& name)
         metadata.unit                    = "";
         metadata.interpretation          = metrics::MetricInterpretation::HIGHER_IS_BETTER;
         mMetrics.framerateId             = mMetrics.manager.AddMetric(metadata);
-        PPX_ASSERT_MSG(mMetrics.cpuFrameTimeId != metrics::kInvalidMetricID, "Failed to create framerate metric");
+        PPX_ASSERT_MSG(mMetrics.framerateId != metrics::kInvalidMetricID, "Failed to create framerate metric");
     }
     {
         metrics::MetricMetadata metadata = {};
@@ -1717,7 +1727,7 @@ void Application::StartMetricsRun(const std::string& name)
         metadata.unit                    = "";
         metadata.interpretation          = metrics::MetricInterpretation::NONE;
         mMetrics.frameCountId            = mMetrics.manager.AddMetric(metadata);
-        PPX_ASSERT_MSG(mMetrics.cpuFrameTimeId != metrics::kInvalidMetricID, "Failed to create frame count metric");
+        PPX_ASSERT_MSG(mMetrics.frameCountId != metrics::kInvalidMetricID, "Failed to create frame count metric");
     }
 
     mMetrics.resetFramerateTracking = true;