During automation, set up external script for signing installer. (#538)

This updates the Windows automation scripts to optionally download a repo containing external scripts for signing the LLVM installer.   The external scripts are Microsoft-specific and we keep them in a private git repo in Visual Studio Online.   We clone the repo to automation/Windows/sign.

The tricky part about this was authenticating git commands involving VSO access.   VSO has an OAuth token that you can provide to build phases that can be passed through git using an extra http header.   In our automated builds, we add a separate VSO build task for signing an installer once one has been built.

automation/Windows/build-and-test.bat

@@ -23,6 +23,7 @@
 @rem  BUILD_PACKAGE: Build an installation package.  May be one of Yes or No.
 @rem                 Defaults to No.  If this is Yes and the build is a Release
 @rem                 build, assertions are enabled during the build.
+@rem  SIGN_INSTALLER: Sign the installer package.
 @rem  TEST_TARGET_ARCH: the target architecuture on which testing will be
 @rem                    run.  May be one of X86 or AMD64.  Defaults to X86.
 @rem
@@ -33,13 +34,14 @@
 @rem  CHECKEDC_BRANCH: defaults to master
 @rem  CHECKEDC_COMMIT: defaults to HEAD
 @rem  LLVM_BRANCH: defaults to master
-@rem  LLVM_BRANCH: defaults to HEAD
+@rem  LLVM_COMMIT: defaults to HEAD
 @rem  CLANG_BRANCH: If not set, uses BUILD_SOURCEBRANCHNAME if defined.
 @rem                If BUILD_SOURCEBRANCHNAME is not defined, defaults
 @rem                 to master
 @rem  CLANG_COMMIT: If not set, uses BUILD_SOURCEVERSION if defined.
 @rem                If BUILD_SOURCEVERSION is not default, defaults to
 @rem                HEAD.
+@rem  SIGN_BRANCH: signing automation branch to checkout.
 
 @setlocal
 @set DIRNAME=%CD%

automation/Windows/config-vars.bat

@@ -83,6 +83,24 @@ if NOT DEFINED BUILD_PACKAGE (
   )
 )
 
+if NOT DEFINED SIGN_INSTALLER (
+    set SIGN_INSTALLER=No
+) else if "%SIGN_INSTALLER%"=="Test" (
+    if "%BUILD_PACKAGE"=="No" (
+      echo "BUILD_PACKAGE must be Yes when SIGN_INSTALLER is Test"
+      exit /b /1
+    )
+) else if "%SIGN_INSTALLER%"=="Release" (
+    if "%BUILD_PACKAGE"=="No" (
+      echo "BUILD_PACKAGE must be Yes when SIGN_INSTALLER is Release"
+      exit /b /1
+    )
+) else (
+    echo Unknown SIGN_INSTALLER value %SIGN_INSTALLER%: must be one of Test or Release
+    exit /b /1
+  )
+)
+
 if not defined BUILD_BINARIESDIRECTORY (
   echo BUILD_BINARIESDIRECTORY not set.  Set it the directory that will contain the object directory.
   exit /b 1
@@ -149,6 +167,12 @@ if not defined CLANG_BRANCH (
     set CLANG_BRANCH=master
 )
 
+if not defined SIGN_BRANCH (
+  set SIGN_BRANCH=master
+) else if "%SIGN_BRANCH%"=="" (
+  set SIGN_BRANCH=master
+)
+
 rem set up source versions (Git commit number)
 if not defined LLVM_COMMIT (
   set LLVM_COMMIT=HEAD
@@ -188,7 +212,8 @@ echo.  TEST_TARGET_ARCH: %TEST_TARGET_ARCH%
 echo.  TEST_SUITE: %TEST_SUITE%
 echo.  SKIP_CHECKEDC_TESTS: %SKIP_CHECKEDC_TESTS%
 echo.  BUILD_CHECKEDC_CLEAN: %BUILD_CHECKEDC_CLEAN%
-echo   BUILD_PACKAGE: %BUILD_PACKAGE%
+echo.  BUILD_PACKAGE: %BUILD_PACKAGE%
+echo.  SIGN_INSTALLER: %SIGN_INSTALLER%
 echo.
 echo.  Directories:
 echo.    BUILD_SOURCESDIRECTORY: %BUILD_SOURCESDIRECTORY%
@@ -202,6 +227,7 @@ echo.    LLVM_BRANCH: %LLVM_BRANCH%
 echo.    LLVM_COMMIT: %LLVM_COMMIT%
 echo.    CHECKEDC BRANCH: %CHECKEDC_BRANCH%
 echo.    CHECKEDC_COMMIT: %CHECKEDC_COMMIT%
+echo.    SIGN_BRANCH: %SIGN_BRANCH%
 echo.
 echo.  MSBUILD_BIN: %MSBUILD_BIN%
 echo.  MSBUILD_CPU_COUNT: %MSBUILD_CPU_COUNT%

automation/Windows/setup-files.bat

@@ -28,6 +28,16 @@ if not exist %BUILD_SOURCESDIRECTORY%\llvm\projects\checkedc-wrapper\checkedc\.g
   if ERRORLEVEL 1 (goto cmdfailed)
 )
 
+
+if "%SIGN_INSTALLER%" NEQ "No" (
+  if not exist %BUILD_SOURCESDIRECTORY%\automation\Windows\sign\.git (
+    rem VSO automation runs scripts from a top-level clang repo that its cloned.
+    rem Place the signing scripts there, not within the cloned compiler repos.
+    git -c http.extraheader="Authorization: bearer %SYSTEM_ACCESSTOKEN%" clone https://msresearch.visualstudio.com/DefaultCollection/CheckedC/_git/checkedc-sign %BUILD_SOURCESDIRECTORY%\automation\Windows\sign
+    if ERRORLEVEL 1 (goto cmdfailed)
+  )
+)
+
 rem set up LLVM sources
 cd %BUILD_SOURCESDIRECTORY%\llvm
 if ERRORLEVEL 1 (goto cmdfailed)
@@ -72,6 +82,18 @@ if not exist %LLVM_OBJ_DIR% (
   if ERRORLEVEL 1 (goto cmdfailed)
 )
 
+rem Set up sources for scripts for signing installer
+if "%SIGN_INSTALLER%" NEQ "No" (
+    cd %BUILD_SOURCESDIRECTORY%\automation\Windows\sign
+    if ERRORLEVEL 1 (goto cmdfailed)
+    git -c http.extraheader="Authorization: bearer %SYSTEM_ACCESSTOKEN%" fetch origin
+    if ERRORLEVEL 1 (goto cmdfailed)
+    git -c http.extraheader="Authorization: bearer %SYSTEM_ACCESSTOKEN%" checkout -f %SIGN_BRANCH%
+    if ERRORLEVEL 1 (goto cmdfailed)
+    git -c http.extraheader="Authorization: bearer %SYSTEM_ACCESSTOKEN%" pull -f origin %SIGN_BRANCH%
+    if ERRORLEVEL 1 (goto cmdfailed)
+)
+
 rem Set up directory for package
 if exist %LLVM_OBJ_DIR%\package (
   rmdir /s /q %LLVM_OBJ_DIR%\package
@@ -83,6 +105,17 @@ if "%BUILD_PACKAGE%"=="Yes" (
   if ERRORLEVEL 1 (goto cmdfailed)
 )
 
+rem Set up directory for signing
+if exist %LLVM_OBJ_DIR%\signed-package (
+  rmdir /s /q %LLVM_OBJ_DIR%\signed-package
+  if ERRORLEVEL 1 (goto cmdfailed)
+)
+
+if "%SIGN_INSTALLER%" NEQ "No" (
+  mkdir %LLVM_OBJ_DIR%\signed-package
+  if ERRORLEVEL 1 (goto cmdfailed)
+)
+
 :succeeded
   cd %OLD_DIR%
   exit /b 0