Explore challenging intermittent nightly test failures

Jamroot.jam

@@ -362,6 +362,8 @@ project pwiz
         <toolset>msvc:<cxxflags>/wd4456                 # warning: declaration of 'type' hides previous local declaration
         <toolset>msvc:<cxxflags>/wd4459                 # warning: declaration of 'type' hides global declaration
         <toolset>msvc:<cxxflags>/wd4458                 # warning: declaration of 'type' hides class member
+        #<toolset>msvc,<debug-symbols>on:<cxxflags>-fsanitize=address      # address sanitizer
+        #<toolset>msvc,<debug-symbols>on:<cxxflags>/bigobj      # address sanitizer makes for big object files: "fatal error C1128: number of sections exceeded object file format limit"
         <toolset>msvc,<debug-symbols>off:<linkflags>"/OPT:REF,ICF"
         <toolset>msvc,<toolset-msvc:version>12.0:<define>_HAS_TR1
         <toolset>msvc,<toolset-msvc:version>14.0:<define>_HAS_TR1

pwiz_tools/Skyline/Model/DiaUmpireDdaConverter.cs

@@ -111,6 +111,7 @@ public override bool Run(IProgressMonitor progressMonitor, IProgressStatus statu
 
                     // CONSIDER: read the file description to see what settings were used to generate the file;
                     // if the same settings were used, we can re-use the file, else regenerate
+/* TODO(bspratt) temp removal - is this the trouble spot causing memory errors? TODO restore this?
                     if (MsDataFileImpl.IsValidFile(outputFilepath))
                     {
                         var outputFileConfig = DiaUmpire.Config.GetConfigFromDiaUmpireOutput(outputFilepath);
@@ -139,7 +140,8 @@ public override bool Run(IProgressMonitor progressMonitor, IProgressStatus statu
                             }
                         }
                     }
-
+end temp removal */
+
                     if (File.Exists(outputFilepath))
                         FileEx.SafeDelete(outputFilepath);
 

pwiz_tools/Skyline/Program.cs

@@ -95,6 +95,7 @@ public static bool SkylineOffscreen
         } // Set true to move Skyline windows offscreen.
 
         public static bool DemoMode { get; set; }                   // Set to true in demo mode (main window is full screen and pauses at screenshots)
+        public static bool ReportTutorialTestProgress { get; set; } // When True, write PauseForScreenshot messages to console even if offscreen and even if not pausing for screenshots
         public static bool NoVendorReaders { get; set; }            // Set true to avoid calling vendor readers.
         public static bool UseOriginalURLs { get; set; }            // Set true to use original URLs for downloading tools instead of our S3 copies
         public static bool IsPassZero { get { return NoVendorReaders; } }   // Currently the only time NoVendorReaders gets set is pass0

pwiz_tools/Skyline/SkylineTester/SkylineTesterWindow.resx

@@ -154,6 +154,9 @@
   <metadata name="TestName.UserAddedColumn" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
     <value>True</value>
   </metadata>
+  <metadata name="FirstAppearance.UserAddedColumn" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>True</value>
+  </metadata>
   <metadata name="Iterations.UserAddedColumn" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
     <value>True</value>
   </metadata>
@@ -172,6 +175,9 @@
   <metadata name="menuStrip1.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
     <value>17, 17</value>
   </metadata>
+  <metadata name="DetectedLeak.UserAddedColumn" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>True</value>
+  </metadata>
   <metadata name="$this.TrayHeight" type="System.Int32, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
     <value>25</value>
   </metadata>

pwiz_tools/Skyline/SkylineTester/TabBuild.cs

@@ -196,15 +196,17 @@ public static bool CreateBuildCommands(
 
                 commandShell.Add("#@ Checking out {0} source files...\n", branchName);
                 commandShell.Add("# Checking out {0} source files...", branchName);
+
                 // Add the --progress flag for richer logging - git leaves out most progress info when it isn't writing to an actual terminal
+                // Add the "-c http.postBuffer=524288000" argument to help with clone on unstable networks
                 if (branchName.Contains("master"))
                 {
-                    commandShell.AddWithRetry("{0} clone {1} --recurse-submodules --progress {2}", git.Quote(), branchUrl.Quote(), buildRoot.Quote());
+                    commandShell.AddWithRetry("{0} -c http.postBuffer=524288000 clone {1} --recurse-submodules --progress {2}", git.Quote(), branchUrl.Quote(), buildRoot.Quote());
                 }
                 else
                 {
                     var branch = branchUrl.Split(new[] {"tree/"}, StringSplitOptions.None)[1];
-                    commandShell.AddWithRetry("{0} clone {1} --recurse-submodules --progress -b {2} {3}", git.Quote(), GetMasterUrl().Quote(), branch.Quote(), buildRoot.Quote());
+                    commandShell.AddWithRetry("{0} -c http.postBuffer=524288000 clone {1} --recurse-submodules --progress -b {2} {3}", git.Quote(), GetMasterUrl().Quote(), branch.Quote(), buildRoot.Quote());
                 }
             }
 
@@ -214,7 +216,7 @@ public static bool CreateBuildCommands(
             foreach (int architecture in architectures)
             {
                 commandShell.Add("#@ Building Skyline {0} bit...\n", architecture);
-                commandShell.Add("{0} {1} {2} --i-agree-to-the-vendor-licenses {3} nolog",
+                commandShell.Add("{0} {1} {2} --i-agree-to-the-vendor-licenses {3} debug-symbols=on nolog",
                     Path.Combine(buildRoot, @"pwiz_tools\build-apps.bat").Quote(),
                     architecture,
                     runBuildTests ? "pwiz_tools/Skyline" : "pwiz_tools/Skyline//Skyline.exe",