Merge pull request #26 from myk002/myk_modernize

Update msvc build image

msvc/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 LABEL org.opencontainers.image.source https://github.com/DFHack/build-env
 
@@ -8,17 +8,22 @@ ARG BUILDER_UID=1001
 RUN apt-get update \
  && apt-get upgrade -y \
  && apt-get install -y wine64-development python3 msitools python3-simplejson python3-six ca-certificates \
- && apt-get install -y gcc g++ ninja-build git zlib1g-dev libsdl1.2-dev libxml-libxml-perl libxml-libxslt-perl make libncurses-dev libssl-dev \
+ && apt-get install -y git gcc g++ ninja-build libxml-libxslt-perl make libssl-dev \
  && apt-get install -y dh-autoreconf libcurl4-gnutls-dev libexpat1-dev gettext libz-dev \
- && apt-get install -y wget curl unzip vim bash-completion xvfb python3-pip \
+ && apt-get install -y wget curl unzip python3-pip \
  && apt-get clean -y \
  && rm -rf /var/lib/apt/lists/* \
  && pip3 install --compile sphinx \
  && useradd --uid $BUILDER_UID --create-home --shell /bin/bash buildmaster
 
+# Initialize the wine environment for root. Wait until the wineserver process has
+# exited before closing the session, to avoid corrupting the wine prefix.
+RUN wine64 wineboot --init && \
+    while pgrep wineserver > /dev/null; do sleep 1; done
+
 # set up msvc
 WORKDIR /opt/msvc
-COPY lowercase fixinclude install.sh vsdownload.py msvcenv-native.sh ./
+COPY lowercase fixinclude install.sh vsdownload.py msvctricks.cpp ./
 COPY wrappers/* ./wrappers/
 
 # This removes the arm and x86 bits to save space (around 5.5G)
@@ -29,47 +34,36 @@ RUN PYTHONUNBUFFERED=1 ./vsdownload.py --accept-license --dest /opt/msvc \
  && find /opt/msvc -depth -type d -iregex '.*/.*arm[0-9]*$' -exec rm -fr {} \; \
  && find /opt/msvc -depth -type d -iregex '.*/.*x86$' -exec rm -fr {} \;
 
-ENV BIN=/opt/msvc/bin/x64 \
-    PATH=/opt/msvc:/opt/msvc/bin/x64:/opt/git/bin:/opt/ccache/bin:/opt/cmake/bin:$PATH \
-    WINEARCH=win64 \
-    WINEDLLOVERRIDES='ngen.exe,mscorsvw.exe=b' \
-    WINEDEBUG=-all
-
-# compile more recent git for ssh keysigning
-RUN mkdir -p /opt/git/src && cd /opt/git/src \
- && wget https://github.com/git/git/archive/refs/tags/v2.38.1.tar.gz \
- && tar xzf "v2.38.1.tar.gz" \
- && cd git-2.38.1 \
- && make configure && ./configure --prefix=/opt/git \
- && make -j$(nproc) all && make install \
- && cd && rm -rf /opt/git/src
+COPY msvcenv-native.sh /opt/msvc
 
 # compile custom cmake for msvc and ninja compatibility
 RUN mkdir -p /opt/cmake/src && cd /opt/cmake/src \
  && git clone -b msvc-3.22.1 --depth=1 https://gitlab.kitware.com/mstorsjo/cmake.git \
- && mkdir cmake/build && cd cmake/build && ../configure --prefix=/opt/cmake --parallel=$(nproc) -- -DBUILD_CursesDialog=ON \
+ && mkdir cmake/build && cd cmake/build && ../configure --prefix=/opt/cmake --parallel=$(nproc) \
  && make -j$(nproc) && make install \
  && cd && rm -rf /opt/cmake/src
 
-# compile custom ccache for msvc caching
-COPY ccache-4.7.4.patch /tmp
-RUN mkdir -p /opt/ccache/src && cd /opt/ccache/src \
- && git clone -b "v4.7.4" --depth=1 https://github.com/ccache/ccache.git \
- && mkdir ccache/build && cd ccache && git apply /tmp/ccache-4.7.4.patch \
- && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/ccache .. \
- && make -j$(nproc) && make install && cd && rm -rf /opt/ccache/src
+# install newer ccache for msvc caching
+RUN mkdir -p /opt/ccache && cd /opt/ccache \
+ && wget 'https://github.com/ccache/ccache/releases/download/v4.8.3/ccache-4.8.3-linux-x86_64.tar.xz' -O- | tar -xJ \
+ && mv * bin
+
+ENV BIN=/opt/msvc/bin/x64 \
+    PATH=/opt/msvc:/opt/msvc/bin/x64:/opt/ccache/bin:/opt/cmake/bin:$PATH \
+    WINEARCH=win64 \
+    WINEDLLOVERRIDES='ngen.exe,mscorsvw.exe=b' \
+    WINEDEBUG=-all
 
 # set up dfhack build environment
 USER buildmaster
 WORKDIR /home/buildmaster
 
 # Add native build directory for DFHack cross compiling
-# maintainer: Update cloned git tag when we update protobuf to keep them in sync
 RUN mkdir /home/buildmaster/dfhack-native \
  && cd /home/buildmaster/dfhack-native \
- && git clone -b "0.47.05-r8" --depth=1 https://github.com/DFHack/dfhack.git \
+ && git clone --depth=1 https://github.com/DFHack/dfhack.git \
  && cd dfhack && git submodule update --init && cd .. \
- && cmake dfhack -GNinja -DCMAKE_BUILD_TYPE=Release -DDOWNLOAD_RUBY=OFF \
+ && cmake dfhack -GNinja -DCMAKE_BUILD_TYPE=Release \
  && ninja protoc-bin \
  && cd .. \
  && mkdir -p dfhack-native-bin/depends/protobuf \
@@ -78,7 +72,7 @@ RUN mkdir /home/buildmaster/dfhack-native \
  && rm -rf dfhack-native \
  && mv dfhack-native-bin dfhack-native
 
-# set up wine (get rid of the "install mono?" and "install gecko?" prompts) and ccache
+# set up wine and ccache for the buildmaster user
 RUN WINEDLLOVERRIDES="mscoree,mshtml=" wine64 wineboot --init \
  && wineserver -w \
  && ccache -C

msvc/README.md

@@ -1,35 +1,44 @@
 Cross compilation with MSVC on Linux
 ====================================
 
-This is a reproducible Dockerfile for cross compiling with MSVC on Linux,
-usable as base image for CI style setups.
+This is a set of tools for using MSVC on Linux (and other Unix OSes).
+
+It consists of two main parts:
+
+- A Python script that downloads and unpacks MSVC and the Windows SDK
+
+- Wrapping and tweaking of the unpacked MSVC/WinSDK to make MSVC work
+  transparently from Unix based build tools with Wine
+
+The first part also works without the second, as a source of MSVC
+and WinSDK for use with e.g. clang-cl.
 
 This downloads and unpacks the necessary Visual Studio components using
 the same installer manifests as Visual Studio 2017/2019's installer
 uses. Downloading and installing it requires accepting the license,
 available at https://go.microsoft.com/fwlink/?LinkId=2086102 for the
 currently latest version.
 
-As Visual Studio isn't redistributable, the resulting docker image isn't
+As Visual Studio isn't redistributable, the installed toolchain isn't
 either.
 
-Build the docker image like this:
-
-    docker build .
-
-After building the docker image, there are 4 directories with tools,
-in `/opt/msvc/bin/<arch>`, for all architectures out of `x86`,
-`x64`, `arm` and `arm64`, that should be added to the PATH before building
-with it.
-
-The installer scripts also work fine without docker; just run the following two commands:
+To install, just run the following two commands:
 
     ./vsdownload.py --dest <dir>
     ./install.sh <dir>
 
 The unpacking requires recent versions of msitools (0.98) and libgcab
 (1.2); sufficiently new versions are available in e.g. Ubuntu 19.04.
 
+After installing the toolchain this way, there are 4 directories with tools,
+in `<dest>/bin/<arch>`, for all architectures out of `x86`,
+`x64`, `arm` and `arm64`, that should be added to the PATH before building
+with it.
+
+There's also a reproducible dockerfile, that creates a docker image with
+the MSVC tools available in `/opt/msvc`. (This also serves as a testable
+example of an environment where the install is known to work.)
+
 
 # Build instructions for local installation
 
@@ -52,31 +61,20 @@ We're going to install it into `~/my_msvc` to avoid needing root privileges on a
 # Add wrapper scripts, do minor cleanup of the unpacked MSVC installation
 ./install.sh ~/my_msvc/opt/msvc
 
-# Custom CMake
-git clone https://gitlab.kitware.com/mstorsjo/cmake.git
-cd cmake
-git checkout 844ccd2280d11ada286d0e2547c0fa5ff22bd4db
-mkdir build 
-cd build
-../configure --prefix=~/my_msvc/opt/cmake --parallel=$(nproc) -- -DCMAKE_USE_OPENSSL=OFF
-make -j$(nproc)
-make install
-
-# Run wine at least once
-wineserver -k # kills server (optional)
-wineserver -p
-wine64 wineboot
+# Optional: Start a persistent wineserver
+wineserver -k # Kill a potential old server
+wineserver -p # Start a new server
+wine64 wineboot # Run a process to start up all background wine processes
 ```
 
 ### Setting up your project with CMake
 
-You need to set the path to prioritize our custom CMake, and also to see our MSVC installation.
+You need to add the our MSVC installation to the path.
 After that we just run CMake command with a few extra settings:
 
 ```bash
-export PATH=~/my_msvc/opt/cmake/bin:$PATH
 export PATH=~/my_msvc/opt/msvc/bin/x64:$PATH
-CC=cl CXX=cl cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_PROGRAMS=ON -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_CROSSCOMPILING=ON
+CC=cl CXX=cl cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_SYSTEM_NAME=Windows
 make
 ```
 
@@ -135,9 +133,29 @@ Yes, but the install scripts won't work because `msitools` is too old. You'll ne
 
 ## Does it work with CMake?
 
-Yes, but a custom version is needed or else CMake will complain that `/opt/msvc/bin/x64/cl` can't build a simple program.
-
-It also fixes the `Ninja` Generator which otherwise has trouble finding RC.
+Yes, but you need CMake 3.23, and either need `winbind` installed, or
+need to configure the build to always use embedded debug info (or a
+custom build of CMake that doesn't try to use separate PDB file debug
+info by default).
+
+Even if configuring CMake with `-DCMAKE_BUILD_TYPE=Release`, CMake does
+call the compiler in `Debug` mode when it does the first few test
+invocations of the compiler. By default, CMake uses separate PDB
+file debug info, when compiling in `Debug` mode, i.e. using the
+`/Zi` compiler option (as opposed to the `/Z7` option, storing the
+debug info in the individual object files).
+
+When MSVC is invoked with the `/Zi` and `/FS` options, it spawns a
+background `mspdbsrv.exe` process and communicates with it. This
+requires the `winbind` package to be installed for this communication
+to work.
+
+With CMake 3.25, it's possible to override the type of debug info
+even for the first few probing steps. This requires the CMake project
+to either set `cmake_minimum_required(VERSION 3.25.0)`, or set
+`cmake_policy(SET CMP0141 NEW)`, and requires the user to configure it
+with `-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded`; in such a
+configuration, `winbind` isn't needed.
 
 ## Can it build Debug versions?
 

msvc/dfhack-configure

@@ -54,7 +54,7 @@ esac
 wineserver -p
 wine64 wineboot
 
-CC=cl CXX=cl cmake .. -GNinja -DDFHACK_BUILD_ARCH=$bits -DCMAKE_BUILD_TYPE=$mode -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_CROSSCOMPILING=ON -DDFHACK_NATIVE_BUILD_DIR=/home/buildmaster/dfhack-native "$@" <&0
+CC=cl CXX=cl cmake .. -GNinja -DDFHACK_BUILD_ARCH=$bits -DCMAKE_BUILD_TYPE=$mode -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_CROSSCOMPILING=ON -DDFHACK_NATIVE_BUILD_DIR=/home/buildmaster/dfhack-native -DGITCOMMAND=$(which git) -DGIT_EXECUTABLE=$(which git) "$@" <&0
 status=$?
 
 wineserver -k

msvc/fixinclude

@@ -18,6 +18,8 @@ use strict;
 
 sub dodir($);
 
+my %mapping;
+
 sub dodir($) {
   my $dir = shift;
 
@@ -40,14 +42,21 @@ sub dodir($) {
       open FHOUT,">$dir/$i.out" || die("Can't write to $dir/$i.out");
 
       while(my $line = <FHIN>) {
-        if($line =~ m/^#include/) {
-	  my @values = split('//', $line);
-	  $values[0] =~ tr [A-Z\\] [a-z/];
-
-	  $line = join('//', @values);
-	}
-	$line =~ s/[\r\n]//g;
-	print FHOUT "$line\n";
+        # Make sure to match '#include <foo>' or '#include "bar"', but not
+        # '#include IDENTIFIER'.
+        if($line =~ m/^#\s*include\s+["<][\w\.\/\\]+[">]/) {
+          my @values = split('//', $line);
+          $values[0] =~ tr [A-Z\\] [a-z/];
+
+          foreach my $from (keys %mapping) {
+            my $to = $mapping{$from};
+            $values[0] =~ s,$from,$to,;
+          }
+
+          $line = join('//', @values);
+        }
+        $line =~ s/[\r\n]//g;
+        print FHOUT "$line\n";
       }
 
       close(FHIN);
@@ -58,5 +67,15 @@ sub dodir($) {
   }
 }
 
-die("Usage: fixinclude dir\n") if(@ARGV != 1);
-dodir($ARGV[0]);
+my @paths;
+for (my $i = 0; $i < @ARGV; $i++) {
+  my $arg = $ARGV[$i];
+  if ($arg eq "-map_winsdk") {
+    # Map references to e.g. GL/gl.h to keep that canonical spelling.
+    $mapping{lc("GL/")} = "GL/";
+  } else {
+    push @paths, $arg;
+  }
+}
+die("Usage: fixinclude dir\n") if(@paths != 1);
+dodir($paths[0]);