Fix WIFF SIM/SRM RTWindows bug (#2488)

- changed WIFF SIM/SRM chromatogram extraction to operate on the entire time range instead of within the scheduled limits; works around a bug(?) with Sciex WIFF where it records the wrong scheduled limits but the data is actually there if you tell it to ignore the limits (reported by Phillip and Celeste)
* updated expected values for TestSrmTutorialLegacy, TestPeakPickingTutorial, and TestAsymCEOpt (thanks Nick)

Co-authored-by: Nicholas Shulman <[email protected]>

pwiz/data/vendor_readers/ABI/ChromatogramList_ABI.cpp

@@ -251,7 +251,8 @@ PWIZ_API_DECL ChromatogramPtr ChromatogramList_ABI::chromatogram(size_t index, D
             result->setTimeIntensityArrays(std::vector<double>(), std::vector<double>(), UO_minute, MS_number_of_detector_counts);
 
             pwiz::util::BinaryData<double> times, intensities;
-            experiment->getSIC(ie.transition, times, intensities);
+            bool ignoreScheduledLimitsForChromatograms = true;
+            experiment->getSIC(ie.transition, times, intensities, ignoreScheduledLimitsForChromatograms);
             result->defaultArrayLength = times.size();
 
             if (getBinaryData)
@@ -290,7 +291,8 @@ PWIZ_API_DECL ChromatogramPtr ChromatogramList_ABI::chromatogram(size_t index, D
             result->setTimeIntensityArrays(std::vector<double>(), std::vector<double>(), UO_minute, MS_number_of_detector_counts);
 
             pwiz::util::BinaryData<double> times, intensities;
-            experiment->getSIC(ie.transition, times, intensities);
+            bool ignoreScheduledLimitsForChromatograms = true;
+            experiment->getSIC(ie.transition, times, intensities, ignoreScheduledLimitsForChromatograms);
             result->defaultArrayLength = times.size();
 
             if (getBinaryData)

pwiz/utility/bindings/CLI/msdata/Reader.hpp

@@ -67,52 +67,82 @@ public ref class ReaderConfig
 {
     public:
 
+    /// <summary>
     /// return Selected Ion Monitoring as spectra
+    /// </summary>
     bool simAsSpectra;
 
+    /// <summary>
     /// return Selected Reaction Monitoring as spectra
+    /// </summary>
     bool srmAsSpectra;
 
+    /// <summary>
 	/// when true, allows for skipping 0 length checks (and thus skip re-reading data for ABI)
+    /// </summary>
     bool acceptZeroLengthSpectra;
 
+    /// <summary>
     /// when true, allows certain vendor readers to produce profile data without zero intensity samples flanking each peak profile
+    /// </summary>
     bool ignoreZeroIntensityPoints;
 
+    /// <summary>
     /// when true, all drift bins/scans in a frame/block are written in combined form instead of as individual spectra
+    /// </summary>
     bool combineIonMobilitySpectra;
 
+    /// <summary>
     // when true, scans with only calibration data will be ignored (currently Waters lockmass only)
+    /// </summary>
     bool ignoreCalibrationScans;
 
+    /// <summary>
     /// when true, Waters SONAR data populates "drift" information with bin number instead of pseudo ion mobility values
+    /// </summary>
     bool reportSonarBins;
 
+    /// <summary>
     /// when true, if a reader cannot identify an instrument, an exception will be thrown asking users to report it
+    /// </summary>
     bool unknownInstrumentIsError;
 
+    /// <summary>
     /// when true, if a reader does not know what time zone was used to record a time, it will assume the time refers to the host's local time;
     /// when false, the reader will treat times with unknown time zone as UTC
+    /// </summary>
     bool adjustUnknownTimeZonesToHostTimeZone;
 
+    /// <summary>
     /// when nonzero, if reader can enumerate only spectra of ms level, it will (currently only supported by Bruker TDF)
+    /// </summary>
     int preferOnlyMsLevel;
 
+    /// <summary>
     /// when true, MS2 spectra without precursor/isolation information will be included in the output (currently only affects Bruker PASEF data)
+    /// </summary>
     bool allowMsMsWithoutPrecursor;
 
 
+    /// <summary>
     /// temporary(?) variable to avoid needing to regenerate Bruker test data
+    /// </summary>
     bool sortAndJitter;
 
+    /// <summary>
     /// when non-empty, only scans from precursors matching one of the included m/z and/or mobility windows will be enumerated; MS1 scans are affected only by the mobility filter
+    /// </summary>
     System::Collections::Generic::IList<MzMobilityWindow^>^ isolationMzAndMobilityFilter;
 
+    /// <summary>
     /// when true, global TIC and BPC chromatograms consist of only MS1 spectra (thus the number of time points cannot be assumed to be equal to the number of spectra)
+    /// </summary>
     bool globalChromatogramsAreMs1Only;
 
+    /// <summary>
     // When true, vendor-specific DDA processing is enabled. For Waters data, this involves combining MS2 scans which refer
     // to the same precursor from the same survey scan, and lockmass-correcting the precursor mass (if the lockmass refiner is active).
+    /// </summary>
     bool ddaProcessing;
 
     ReaderConfig()

pwiz_aux/msrc/utility/vendor_api/ABI/WiffFile.cpp

@@ -40,9 +40,11 @@
 #pragma managed
 #include "pwiz/utility/misc/cpp_cli_utilities.hpp"
 #include <msclr/auto_gcroot.h>
+#using <System.dll>
 #using <System.Xml.dll>
 using namespace pwiz::util;
 using namespace System;
+using namespace System::Text::RegularExpressions;
 using namespace Clearcore2::Data;
 using namespace Clearcore2::Data::AnalystDataProvider;
 using namespace Clearcore2::Data::Client;
@@ -128,9 +130,9 @@ struct ExperimentImpl : public Experiment
     virtual size_t getSRMSize() const;
     virtual void getSRM(size_t index, Target& target) const;
 
-    virtual void getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities) const;
+    virtual double getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities, bool ignoreScheduledLimits) const;
     virtual void getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities,
-                        double& basePeakX, double& basePeakY) const;
+                        double& basePeakX, double& basePeakY, bool ignoreScheduledLimits) const;
 
     virtual void getAcquisitionMassRange(double& startMz, double& stopMz) const;
     virtual ScanType getScanType() const;
@@ -147,6 +149,7 @@ struct ExperimentImpl : public Experiment
     const WiffFileImpl* wifffile_;
     gcroot<MSExperiment^> msExperiment;
     int sample, period, experiment;
+    bool hasHalfSizeRTWindow;
 
     ExperimentType experimentType;
     size_t simCount;
@@ -439,6 +442,27 @@ ExperimentImpl::ExperimentImpl(const WiffFileImpl* wifffile, int sample, int per
             transitionCount = msExperiment->Details->MassRangeInfo->Length;
         else if (experimentType == SIM)
             simCount = msExperiment->Details->MassRangeInfo->Length;
+
+        hasHalfSizeRTWindow = false;
+        try
+        {
+            auto softwareVersion = wifffile_->batch->GetSample(sample)->Details->SoftwareVersion;
+            auto sciexOsVersionRegex = gcnew Regex(R"(SCIEX OS (\d+)\.(\d+))");
+
+            auto match = sciexOsVersionRegex->Match(softwareVersion);
+            if (match->Success)
+            {
+                int major = Convert::ToInt32(match->Groups[1]->Value);
+                int minor = Convert::ToInt32(match->Groups[2]->Value);
+                hasHalfSizeRTWindow = !(major >= 3 && minor >= 1); // currently assumed present in SCIEX OS lower than v3.1
+                //if (hasHalfSizeRTWindow)
+                //    Console::Error->WriteLine("NOTE: data from " + softwareVersion + " has bugged half-width RTWindows");
+            }
+        }
+        catch (Exception^)
+        {
+            // ignore read past end of stream: no version details? probably acquired with Analyst?
+        }
     }
     CATCH_AND_FORWARD
 }
@@ -523,11 +547,12 @@ void ExperimentImpl::getSIM(size_t index, Target& target) const
 
         SIMMassRange^ transition = (SIMMassRange^) msExperiment->Details->MassRangeInfo[index];
 
+        double rtWindowMultiplier = hasHalfSizeRTWindow ? 1 : 0.5;
         target.type = TargetType_SIM;
         target.Q1 = transition->Mass;
         target.dwellTime = transition->DwellTime;
-        target.startTime = transition->ExpectedRT - transition->RTWindow / 2;
-        target.endTime = transition->ExpectedRT + transition->RTWindow / 2;
+        target.startTime = transition->ExpectedRT - transition->RTWindow * rtWindowMultiplier;
+        target.endTime = transition->ExpectedRT + transition->RTWindow * rtWindowMultiplier;
         target.compoundID = ToStdString(transition->Name);
 
         auto parameters = transition->CompoundDepParameters;
@@ -561,12 +586,13 @@ void ExperimentImpl::getSRM(size_t index, Target& target) const
 
         MRMMassRange^ transition = (MRMMassRange^) msExperiment->Details->MassRangeInfo[index];
 
+        double rtWindowMultiplier = hasHalfSizeRTWindow ? 1 : 0.5;
         target.type = TargetType_SRM;
         target.Q1 = transition->Q1Mass;
         target.Q3 = transition->Q3Mass;
         target.dwellTime = transition->DwellTime;
-        target.startTime = transition->ExpectedRT - transition->RTWindow;
-        target.endTime = transition->ExpectedRT + transition->RTWindow;
+        target.startTime = transition->ExpectedRT - transition->RTWindow * rtWindowMultiplier;
+        target.endTime = transition->ExpectedRT + transition->RTWindow * rtWindowMultiplier;
         target.compoundID = ToStdString(transition->Name);
 
         auto parameters = transition->CompoundDepParameters;
@@ -583,14 +609,30 @@ void ExperimentImpl::getSRM(size_t index, Target& target) const
     CATCH_AND_FORWARD
 }
 
-void ExperimentImpl::getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities) const
+double ExperimentImpl::getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities, bool ignoreScheduledLimits) const
 {
     try
     {
         if (index >= transitionCount+simCount)
-            throw std::out_of_range("[Experiment::getSIC()] index out of range");
+            throw std::out_of_range("[Experiment::getSIC()] index " + lexical_cast<string>(index) + " out of range");
+
+        Target target;
+        getSRM(index, target);
 
         ExtractedIonChromatogramSettings^ option = gcnew ExtractedIonChromatogramSettings(index);
+        if (ignoreScheduledLimits)
+        {
+            option->StartCycle = 0;
+            option->EndCycle = convertRetentionTimeToCycle(cycleTimes().back());
+            option->UseStartEndCycle = true;
+        }
+        else if (target.startTime != target.endTime)
+        {
+            option->StartCycle = convertRetentionTimeToCycle(target.startTime);
+            option->EndCycle = convertRetentionTimeToCycle(target.endTime);
+            option->UseStartEndCycle = true;
+        }
+
         ExtractedIonChromatogram^ xic = msExperiment->GetExtractedIonChromatogram(option);
 
         ToBinaryData(xic->GetActualXValues(), times);
@@ -600,20 +642,12 @@ void ExperimentImpl::getSIC(size_t index, pwiz::util::BinaryData<double>& times,
 }
 
 void ExperimentImpl::getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities,
-                            double& basePeakX, double& basePeakY) const
+                            double& basePeakX, double& basePeakY, bool ignoreScheduledLimits) const
 {
+    basePeakY = getSIC(index, times, intensities, ignoreScheduledLimits);
+
     try
     {
-        if (index >= transitionCount)
-            throw std::out_of_range("[Experiment::getSIC()] index " + lexical_cast<string>(index) + " out of range");
-
-        ExtractedIonChromatogramSettings^ option = gcnew ExtractedIonChromatogramSettings(index);
-        ExtractedIonChromatogram^ xic = msExperiment->GetExtractedIonChromatogram(option);
-
-        ToBinaryData(xic->GetActualXValues(), times);
-        ToBinaryData(xic->GetActualYValues(), intensities);
-
-        basePeakY = xic->MaximumYValue;
         basePeakX = 0;
         for (size_t i=0; i < intensities.size(); ++i)
             if (intensities[i] == basePeakY)

pwiz_aux/msrc/utility/vendor_api/ABI/WiffFile.hpp

@@ -190,9 +190,9 @@ struct PWIZ_API_DECL Experiment
 
     virtual size_t getSRMSize() const = 0;
     virtual void getSRM(size_t index, Target& target) const = 0;
-    virtual void getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities) const = 0;
+    virtual double getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities, bool ignoreScheduledLimits) const = 0;
     virtual void getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities,
-                        double& basePeakX, double& basePeakY) const = 0;
+                        double& basePeakX, double& basePeakY, bool ignoreScheduledLimits) const = 0;
 
     virtual void getAcquisitionMassRange(double& startMz, double& stopMz) const = 0;
     virtual ScanType getScanType() const = 0;

pwiz_aux/msrc/utility/vendor_api/ABI/WiffFile2.ipp

@@ -130,9 +130,9 @@ struct Experiment2Impl : public Experiment
     virtual size_t getSRMSize() const;
     virtual void getSRM(size_t index, Target& target) const;
 
-    virtual void getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities) const;
+    virtual double getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities, bool ignoreScheduledLimits) const;
     virtual void getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities,
-                        double& basePeakX, double& basePeakY) const;
+                        double& basePeakX, double& basePeakY, bool ignoreScheduledLimits) const;
 
     virtual void getAcquisitionMassRange(double& startMz, double& stopMz) const;
     virtual ScanType getScanType() const;
@@ -568,14 +568,15 @@ void Experiment2Impl::getSRM(size_t index, Target& target) const
     CATCH_AND_FORWARD
 }
 
-void Experiment2Impl::getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities) const
+double Experiment2Impl::getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities, bool ignoreScheduledLimits) const
 {
     double x, y;
-    getSIC(index, times, intensities, x, y);
+    getSIC(index, times, intensities, x, y, ignoreScheduledLimits);
+    return y;
 }
 
 void Experiment2Impl::getSIC(size_t index, pwiz::util::BinaryData<double>& times, pwiz::util::BinaryData<double>& intensities,
-                            double& basePeakX, double& basePeakY) const
+                            double& basePeakX, double& basePeakY, bool ignoreScheduledLimits) const
 {
     try
     {

pwiz_tools/Skyline/TestData/Results/AsymCEOptTest.cs

@@ -64,7 +64,7 @@ private void ExportImport(ResultsTestDocumentContainer docContainer, string resu
             var chromSet = new ChromatogramSet("Optimize", new[] {resultsUri}, Annotations.EMPTY, optRegression);
             var measuredResults = new MeasuredResults(new[] { chromSet });
 
-            docContainer.ChangeMeasuredResults(measuredResults, 2, optSteps1 + optSteps2, 5*optSteps1 + 5*optSteps2);
+            docContainer.ChangeMeasuredResults(measuredResults, 2, optSteps1 + optSteps2, 86);
 
             // Check expected optimization data with missing values for steps below 10 volts CE
             int expectedMissingSteps = optSteps - optSteps1;

pwiz_tools/Skyline/TestTutorial/PeakPickingTutorialTest.cs

@@ -96,7 +96,7 @@ private string GetTestPath(string relativePath)
 
         private readonly string[] EXPECTED_COEFFICIENTS =
         {
-            "-0.0783|-0.7492|2.2424|1.2062|0.0331|0.1515|0.1767| null |0.4983|6.3997|-0.0482|0.5476|0.6281| null | null | null | null | null ",
+            "-0.1095|-0.7689|1.9147|0.9647|0.0265|0.1822|0.2229| null |0.5529|6.5433|-0.0357|0.5285|0.6585| null | null | null | null | null ",
             "0.2900| null | null |5.9842|-0.0624|0.6681|0.7968| null | null | null | null | null | null | null | null | null | null | null ",
         };
 
@@ -193,7 +193,7 @@ protected override void DoTest()
             var editDlg = ShowDialog<EditPeakScoringModelDlg>(reintegrateDlg.AddPeakScoringModel);
             RunUI(() => editDlg.TrainModel());
             PauseForScreenShot<EditPeakScoringModelDlg.ModelTab>("Edit Peak Scoring Model form trained model", 6);
-            RunUI(() => Assert.AreEqual(0.5992, editDlg.PeakCalculatorsGrid.Items[3].PercentContribution ?? 0, 0.005));
+            RunUI(() => Assert.AreEqual(0.5893, editDlg.PeakCalculatorsGrid.Items[3].PercentContribution ?? 0, 0.005));
 
             RunUI(() => editDlg.SelectedGraphTab = 2);
             PauseForScreenShot<EditPeakScoringModelDlg.PvalueTab>("Edit Peak Scoring Model form p value graph metafile", 7);