ARM64 image support (#274)

* WIP ARM64 image support close #269 * added `build.fsx` support * added ARM64 integration testing * added Linux integration testing * fixing integration testing FAKE * fixing up `docker build` instructions * added more detailed logging * have `docker build` working again * fixed Docker images and ARM build * fixed Windows image name * allow cluster to actually form * fixed Azure Pipelines template * added fail gate back * added Windows runtime tests back * fixed powershell invocation * update ARM64 file to use `dotnet tool install -g -a arm64` * target Ubuntu 20.04 * removed `-a arm64` param * fixed formatting * ensure .NET Core 3.1 SDK is installed * skip installing .NET from CLI * disable PBM install on ARM64 * add a DockerInstall stage to AzDo * added `dotnet` commands back to ARM image * disable caching on `docker build` * try 3.1 base image * changed base image URIs * testing .NET 6 images * Revert "testing .NET 6 images" This reverts commit 4abd075. * enable Docker BuiltKit * removed Docker install, BuildKit, but added QEMU * clean up QEMU
petabridge · Aug 9, 2022 · d5be872 · d5be872
1 parent a32f665
commit d5be872
Show file tree

Hide file tree

Showing 11 changed files with 159 additions and 230 deletions.
diff --git a/build-system/azure-pipeline.template.yaml b/build-system/azure-pipeline.template.yaml
@@ -16,7 +16,19 @@ jobs:
         clean: false  # whether to fetch clean each time
         submodules: recursive  # set to 'true' for a single level of submodules or 'recursive' to get submodules of submodules
         persistCredentials: true
+      - task: UseDotNet@2
+        displayName: 'Use .NET 6 SDK 6.0.100'
+        inputs:
+          version: 6.0.100
+      - task: UseDotNet@2
+        displayName: "Use .NET Core SDK 3.1.415"
+        inputs:
+          version: 3.1.415
       # Linux or macOS
+      - script: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+        displayName: Docker - Register QEMU
+        continueOnError: true
+        condition: in( variables['Agent.OS'], 'Linux', 'Darwin' )
       - task: Bash@3 
         displayName: Linux / OSX Build
         inputs:
@@ -38,22 +50,6 @@ jobs:
           testResultsFiles: '**/*.trx' #TestResults folder usually
           testRunTitle: ${{ parameters.name }}
           mergeTestResults: true
-      #Docker build 
-      # Build on Linux
-      - task: Bash@3 
-        displayName: Docker Build (Linux)
-        inputs:
-          filePath: ${{ parameters.scriptFileName }}
-          arguments: ${{ parameters.dockerBuildArgs }}
-        continueOnError: false
-        condition: in( variables['Agent.OS'], 'Linux', 'Darwin' )
-      - task: BatchScript@1
-        displayName: Docker Build (Windows)
-        inputs:
-          filename: ${{ parameters.scriptFileName }}
-          arguments: ${{ parameters.dockerBuildArgs }}
-        continueOnError: true
-        condition: eq( variables['Agent.OS'], 'Windows_NT' )
       - script: 'echo 1>&2'
         failOnStderr: true
         displayName: 'If above is partially succeeded, then fail'

diff --git a/build-system/linux-pr-validation.yaml b/build-system/linux-pr-validation.yaml
@@ -20,3 +20,9 @@ jobs:
     vmImage: 'ubuntu-20.04'
     scriptFileName: ./build.sh
     scriptArgs: all
+- template: azure-pipeline.template.yaml
+  parameters:
+    name: LinuxRuntimesTests
+    vmImage: 'ubuntu-20.04'
+    scriptFileName: ./build.sh
+    scriptArgs: RunTestsOnRuntimes
diff --git a/build-system/windows-release.yaml b/build-system/windows-release.yaml
@@ -87,7 +87,7 @@ jobs:
 
   - job:  windowsImageDeploy
     pool:
-      vmImage: windows-2019
+      vmImage: windows-latest
     dependsOn: publishNetCore
     steps:
       - task: DownloadBuildArtifacts@0

diff --git a/build.fsx b/build.fsx
@@ -37,10 +37,17 @@ let toolsDir = __SOURCE_DIRECTORY__ @@ "tools"
 let output = __SOURCE_DIRECTORY__  @@ "bin"
 let outputTests = __SOURCE_DIRECTORY__ @@ "TestResults"
 let outputPerfTests = __SOURCE_DIRECTORY__ @@ "PerfResults"
+let dockerScriptDir = __SOURCE_DIRECTORY__ @@ "src" @@ "Lighthouse"
 let outputNuGet = output @@ "nuget"
 
 exception ConnectionFailure of string
 
+let composedGetDirName (p:string) =
+    System.IO.Path.GetDirectoryName p
+
+let composedGetFileNameWithoutExtension (p:string) =
+    System.IO.Path.GetFileNameWithoutExtension p
+
 Target "Clean" (fun _ ->
     ActivateFinalTarget "KillCreatedProcesses"
 
@@ -131,24 +138,24 @@ Target "NBench" <| fun _ ->
 
     projects |> Seq.iter runSingleProject
 
+let dockerFilesWithTags =
+    match (isWindows) with
+        | true -> [("src/Lighthouse/Dockerfile-windows", ["windows-latest"])] 
+        | _ -> [("src/Lighthouse/Dockerfile-linux", ["latest";"linux-latest"]); ("src/Lighthouse/Dockerfile-arm64", ["arm64-latest"])] 
+
 Target "RunTestsOnRuntimes" (fun _ ->
 
     let LighthouseConnectTimeout = 20.0 // in seconds
-
-    let dockerFileForTest = 
-        match (isWindows) with
-        | true -> "src/Lighthouse/Dockerfile-windows" 
-        | _ -> "src/Lighthouse/Dockerfile-linux" 
 
     let installPbm () =
         // Install pbm client to test connections
         ExecProcess(fun info ->
             info.FileName <- "dotnet"
             info.Arguments <- "tool install --global pbm") (TimeSpan.FromMinutes 5.0) |> ignore // this is fine if tool is already installed
 
-    let startLighthouseDocker dockerFile =
-        printfn "Starting Lighthouse..."
-        let runArgs = "run -d --name lighthouse --hostname lighthouse1 -p 4053:4053 -p 9110:9110 --env CLUSTER_IP=127.0.0.1 --env CLUSTER_SEEDS=akka.tcp://some@lighthouse1:4053 --env CLUSTER_PORT=4053 lighthouse:latest"
+    let startLighthouseDocker dockerFile tag =
+        printfn "Starting Lighthouse w/ Dockerfile %s" dockerFile
+        let runArgs = sprintf "run -d --name lighthouse --hostname lighthouse1 -p 4053:4053 -p 9110:9110 --env ACTORSYSTEM=some --env CLUSTER_IP=127.0.0.1 --env CLUSTER_SEEDS=akka.tcp://some@127.0.0.1:4053 --env CLUSTER_PORT=4053 lighthouse:%s" tag
         let runResult = ExecProcess(fun info -> 
             info.FileName <- "docker"
             info.WorkingDirectory <- (Directory.GetParent dockerFile).FullName
@@ -162,7 +169,7 @@ Target "RunTestsOnRuntimes" (fun _ ->
                 info.WorkingDirectory <- (Directory.GetParent dockerFile).FullName
                 info.Arguments <- "rm -f lighthouse") (System.TimeSpan.FromMinutes 5.0) |> ignore // cleanup failure should not fail the test
 
-    let startLighhouseLocally exePath =
+    let startLighthouseLocally exePath =
         printfn "Starting Lighthouse locally..."
         try
             let runResult = ExecProcess(fun info -> 
@@ -184,21 +191,15 @@ Target "RunTestsOnRuntimes" (fun _ ->
             | _ -> printfn "Lighthouse was connected successfully"
 
     installPbm()
-    startLighthouseDocker dockerFileForTest
-    try       
-        connectLighthouse()
-    finally
-        stopLighthouseDocker dockerFileForTest
-
-    // Test Full .NET Framework version under windows only
-    // TODO: To make this work, need to start lighthouse and pbm as two parallel processes
-    (*
-    match (isWindows) with
-            | true -> 
-                startLighhouseLocally "src/Lighthouse/bin/Release/net461/Lighthouse.exe"
-                connectLighthouse()
-            | _ -> ()
-    *)
+
+    let runSpec (dockerFile, tags) =    
+        startLighthouseDocker dockerFile (tags |> List.head)
+        try       
+            connectLighthouse()
+        finally
+            stopLighthouseDocker dockerFile
+
+    dockerFilesWithTags |> Seq.iter runSpec
 )
 
 
@@ -243,7 +244,7 @@ Target "SignPackages" (fun _ ->
                 info.FileName <- signPath
                 info.WorkingDirectory <- __SOURCE_DIRECTORY__
                 info.Arguments <- args) (System.TimeSpan.FromMinutes 5.0) (* Reasonably long-running task. *)
-            if result <> 0 then failwithf "SignClient failed.%s" args
+            if result <> 0 then failwithf "SignClient failed. %s" args
 
         assemblies |> Seq.iter (signAssembly)
     else
@@ -326,74 +327,19 @@ let mapDockerImageName (projectName:string) =
     | "Lighthouse" -> Some("lighthouse")
     | _ -> None
 
-let composedGetDirName (p:string) =
-    System.IO.Path.GetDirectoryName p
-
-let composedGetFileNameWithoutExtension (p:string) =
-    System.IO.Path.GetFileNameWithoutExtension p
-
 Target "BuildDockerImages" (fun _ ->
-    let projects = !! "src/**/*.csproj" 
-                   -- "src/**/*Tests.csproj" // Don't publish unit tests
-                   -- "src/**/*Tests*.csproj"
-
-    let dockerFile = 
-        match (isWindows) with 
-        | true -> "Dockerfile-windows"
-        | _ -> "Dockerfile-linux"
-
-    let dockerTags (imageName:string, assemblyVersion:string) =
-        match(isWindows) with
-        | true -> [| imageName + ":" + releaseNotes.AssemblyVersion; imageName + ":" + releaseNotes.AssemblyVersion + "-nanoserver1803"; imageName + ":latest" |]
-        | _ -> [| imageName + ":" + releaseNotes.AssemblyVersion; imageName + ":" + releaseNotes.AssemblyVersion + "-linux"; imageName + ":latest" |]
-
-
-    let remoteRegistryUrl = getBuildParamOrDefault "remoteRegistry" ""
-
-    let buildDockerImage imageName projectPath =
-
-        let args = 
-            if(hasBuildParam "remoteRegistry") then
-                StringBuilder()
-                    |> append "build"
-                    |> append "-f"
-                    |> append dockerFile
-                    |> append "-t"
-                    |> append (imageName + ":" + releaseNotes.AssemblyVersion) 
-                    |> append "-t"
-                    |> append (imageName + ":latest") 
-                    |> append "-t"
-                    |> append (remoteRegistryUrl + "/" + imageName + ":" + releaseNotes.AssemblyVersion) 
-                    |> append "-t"
-                    |> append (remoteRegistryUrl + "/" + imageName + ":latest") 
-                    |> append "."
-                    |> toText
-            else
-                StringBuilder()
-                    |> append "build"
-                    |> append "-f"
-                    |> append dockerFile
-                    |> append "-t"
-                    |> append (imageName + ":" + releaseNotes.AssemblyVersion) 
-                    |> append "-t"
-                    |> append (imageName + ":latest") 
-                    |> append "."
-                    |> toText
-
-        ExecProcess(fun info -> 
-                info.FileName <- "docker"
-                info.WorkingDirectory <- composedGetDirName projectPath
-                info.Arguments <- args) (System.TimeSpan.FromMinutes 5.0) (* Reasonably long-running task. *)
-
-    let runSingleProject project =
-        let projectName = composedGetFileNameWithoutExtension project
-        let imageName = mapDockerImageName projectName
-        let result = match imageName with
-                        | None -> 0
-                        | Some(name) -> buildDockerImage name project
-        if result <> 0 then failwithf "docker build failed. %s" project
-
-    projects |> Seq.iter (runSingleProject)
+   if(isWindows) then
+        let result = ExecProcess(fun info -> 
+                info.FileName <- "powershell"
+                info.WorkingDirectory <- dockerScriptDir
+                info.Arguments <- sprintf " ./buildWindowsDockerImages.ps1 -tagVersion %s" releaseNotes.AssemblyVersion) (System.TimeSpan.FromMinutes 5.0) (* Reasonably long-running task. *)
+        if result <> 0 then failwithf "Unable to build Lighthouse Dockerfiles"
+   else
+       let result = ExecProcess(fun info -> 
+                info.FileName <- "sh"
+                info.WorkingDirectory <- dockerScriptDir
+                info.Arguments <- sprintf " ./buildLinuxDockerImages.sh %s" releaseNotes.AssemblyVersion) (System.TimeSpan.FromMinutes 5.0) (* Reasonably long-running task. *)
+       if result <> 0 then failwithf "Unable to build Lighthouse Dockerfiles"
 )
 
 //--------------------------------------------------------------------------------
@@ -460,7 +406,7 @@ Target "Nuget" DoNothing
 
 // tests dependencies
 "Build" ==> "RunTests"
-"PublishCode" ==> "BuildDockerImages" ==> "RunTestsOnRuntimes"
+"BuildDockerImages" ==> "RunTestsOnRuntimes"
 
 // nuget dependencies
 "Clean" ==> "Build" ==> "CreateNuget"
@@ -470,7 +416,7 @@ Target "Nuget" DoNothing
 "Clean" ==> "BuildRelease" ==> "Docfx"
 
 // Docker
-"BuildRelease" ==> "PublishCode" ==> "BuildDockerImages" ==> "Docker"
+"BuildDockerImages" ==> "Docker"
 
 // all
 "BuildRelease" ==> "All"

diff --git a/build.ps1 b/build.ps1
@@ -46,50 +46,6 @@ if (!(Test-Path $ToolPath)) {
     New-Item -Path $ToolPath -Type directory | out-null
 }
 
-###########################################################################
-# INSTALL .NET CORE CLI
-###########################################################################
-
-Function Remove-PathVariable([string]$VariableToRemove)
-{
-    $path = [Environment]::GetEnvironmentVariable("PATH", "User")
-    if ($path -ne $null)
-    {
-        $newItems = $path.Split(';', [StringSplitOptions]::RemoveEmptyEntries) | Where-Object { "$($_)" -inotlike $VariableToRemove }
-        [Environment]::SetEnvironmentVariable("PATH", [System.String]::Join(';', $newItems), "User")
-    }
-
-    $path = [Environment]::GetEnvironmentVariable("PATH", "Process")
-    if ($path -ne $null)
-    {
-        $newItems = $path.Split(';', [StringSplitOptions]::RemoveEmptyEntries) | Where-Object { "$($_)" -inotlike $VariableToRemove }
-        [Environment]::SetEnvironmentVariable("PATH", [System.String]::Join(';', $newItems), "Process")
-    }
-}
-
-# Get .NET Core CLI path if installed.
-$FoundDotNetCliVersion = $null;
-if (Get-Command dotnet -ErrorAction SilentlyContinue) {
-    $FoundDotNetCliVersion = dotnet --version;
-    $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
-    $env:DOTNET_CLI_TELEMETRY_OPTOUT=1
-}
-
-if($FoundDotNetCliVersion -ne $DotNetVersion) {
-    $InstallPath = Join-Path $PSScriptRoot ".dotnet"
-    if (!(Test-Path $InstallPath)) {
-        mkdir -Force $InstallPath | Out-Null;
-    }
-    (New-Object System.Net.WebClient).DownloadFile($DotNetInstallerUri, "$InstallPath\dotnet-install.ps1");
-    & $InstallPath\dotnet-install.ps1 -Channel $DotNetChannel -Version $DotNetVersion -InstallDir $InstallPath -Architecture x64;
-
-    Remove-PathVariable "$InstallPath"
-    $env:PATH = "$InstallPath;$env:PATH"
-    $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
-    $env:DOTNET_CLI_TELEMETRY_OPTOUT=1
-    $env:DOTNET_ROOT=$InstallPath
-}
-
 ###########################################################################
 # INSTALL NUGET
 ###########################################################################

diff --git a/build.sh b/build.sh
@@ -42,22 +42,6 @@ if [ ! -d "$TOOLS_DIR" ]; then
   mkdir "$TOOLS_DIR"
 fi
 
-###########################################################################
-# INSTALL .NET CORE CLI
-###########################################################################
-
-echo "Installing .NET CLI..."
-if [ ! -d "$SCRIPT_DIR/.dotnet" ]; then
-  mkdir "$SCRIPT_DIR/.dotnet"
-fi
-curl -Lsfo "$SCRIPT_DIR/.dotnet/dotnet-install.sh" $DOTNET_INSTALLER_URL
-bash "$SCRIPT_DIR/.dotnet/dotnet-install.sh" --version $DOTNET_VERSION --channel $DOTNET_CHANNEL --install-dir .dotnet --no-path
-export PATH="$SCRIPT_DIR/.dotnet":$PATH
-export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
-export DOTNET_CLI_TELEMETRY_OPTOUT=1
-chmod -R 0755 ".dotnet"
-"$SCRIPT_DIR/.dotnet/dotnet" --info
-
 
 ###########################################################################
 # INSTALL NUGET
@@ -112,7 +96,7 @@ fi
 # INSTALL SignTool
 ###########################################################################
 if [ ! -f "$SIGNTOOL_EXE" ]; then
-    "$SCRIPT_DIR/.dotnet/dotnet" tool install SignClient --version 1.0.82 --tool-path "$SIGNCLIENT_DIR"
+    dotnet tool install SignClient --version 1.0.82 --tool-path "$SIGNCLIENT_DIR"
     if [ $? -ne 0 ]; then
         echo "SignClient already installed."
     fi

diff --git a/src/Lighthouse/Dockerfile-arm64 b/src/Lighthouse/Dockerfile-arm64
@@ -0,0 +1,34 @@
+FROM mcr.microsoft.com/dotnet/sdk:3.1 AS base
+WORKDIR /app
+
+# Install Petabridge.Cmd client so it can be invoked remotely via
+# Docker or K8s 'exec` commands
+RUN dotnet tool install --global pbm 
+
+# RUN pbm help
+
+COPY ./bin/Release/netcoreapp3.1/publish/ /app
+
+FROM mcr.microsoft.com/dotnet/runtime:3.1 AS app
+WORKDIR /app
+
+COPY --from=base /app /app
+
+# copy .NET Core global tool
+COPY --from=base /root/.dotnet /root/.dotnet/
+
+# Needed because https://stackoverflow.com/questions/51977474/install-dotnet-core-tool-dockerfile
+ENV PATH="${PATH}:/root/.dotnet/tools"
+
+# should be a comma-delimited list
+ENV CLUSTER_SEEDS "[]"
+ENV CLUSTER_IP ""
+ENV CLUSTER_PORT "4053"
+ENV AKKA__CLUSTER__SPLIT_BRAIN_RESOLVER__ACTIVE_STRATEGY "keep-majority"
+ENV AKKA__REMOTE__DOT-NETTY__TCP__BATCHING__ENABLED "false"
+
+# 9110 - Petabridge.Cmd
+# 4053 - Akka.Cluster
+EXPOSE 9110 4053
+
+CMD ["dotnet", "Lighthouse.dll"]
diff --git a/src/Lighthouse/Lighthouse.csproj b/src/Lighthouse/Lighthouse.csproj
@@ -6,7 +6,7 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Akka.Cluster.Hosting" Version="0.4.1" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Xml" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Xml" Version="3.1.0" />
     <PackageReference Include="Petabridge.Cmd.Cluster" Version="$(PbmVersion)" />
     <PackageReference Include="Petabridge.Cmd.Remote" Version="$(PbmVersion)" />
     <PackageReference Include="Akka.Bootstrap.Docker">