From a58cc13ca6a65ba8c7819b9e01a0c88b43a5184a Mon Sep 17 00:00:00 2001 From: Myk Taylor Date: Fri, 1 Sep 2023 16:18:28 -0700 Subject: [PATCH 1/2] update to latest msvc-wine --- msvc/README.md | 84 ++++++++++++--------- msvc/fixinclude | 39 +++++++--- msvc/install.sh | 79 ++++++++++++++++---- msvc/lowercase | 5 ++ msvc/msvcenv-native.sh | 10 ++- msvc/msvctricks.cpp | 147 +++++++++++++++++++++++++++++++++++++ msvc/vsdownload.py | 16 +++- msvc/wrappers/cl | 32 ++++++++ msvc/wrappers/dumpbin | 6 ++ msvc/wrappers/msvcenv.sh | 2 +- msvc/wrappers/wine-msvc.sh | 58 ++++++++++++++- 11 files changed, 410 insertions(+), 68 deletions(-) create mode 100644 msvc/msvctricks.cpp diff --git a/msvc/README.md b/msvc/README.md index 7603d2e..874cc43 100644 --- a/msvc/README.md +++ b/msvc/README.md @@ -1,8 +1,17 @@ Cross compilation with MSVC on Linux ==================================== -This is a reproducible Dockerfile for cross compiling with MSVC on Linux, -usable as base image for CI style setups. +This is a set of tools for using MSVC on Linux (and other Unix OSes). + +It consists of two main parts: + +- A Python script that downloads and unpacks MSVC and the Windows SDK + +- Wrapping and tweaking of the unpacked MSVC/WinSDK to make MSVC work + transparently from Unix based build tools with Wine + +The first part also works without the second, as a source of MSVC +and WinSDK for use with e.g. clang-cl. This downloads and unpacks the necessary Visual Studio components using the same installer manifests as Visual Studio 2017/2019's installer @@ -10,19 +19,10 @@ uses. Downloading and installing it requires accepting the license, available at https://go.microsoft.com/fwlink/?LinkId=2086102 for the currently latest version. -As Visual Studio isn't redistributable, the resulting docker image isn't +As Visual Studio isn't redistributable, the installed toolchain isn't either. -Build the docker image like this: - - docker build . - -After building the docker image, there are 4 directories with tools, -in `/opt/msvc/bin/`, for all architectures out of `x86`, -`x64`, `arm` and `arm64`, that should be added to the PATH before building -with it. - -The installer scripts also work fine without docker; just run the following two commands: +To install, just run the following two commands: ./vsdownload.py --dest ./install.sh @@ -30,6 +30,15 @@ The installer scripts also work fine without docker; just run the following two The unpacking requires recent versions of msitools (0.98) and libgcab (1.2); sufficiently new versions are available in e.g. Ubuntu 19.04. +After installing the toolchain this way, there are 4 directories with tools, +in `/bin/`, for all architectures out of `x86`, +`x64`, `arm` and `arm64`, that should be added to the PATH before building +with it. + +There's also a reproducible dockerfile, that creates a docker image with +the MSVC tools available in `/opt/msvc`. (This also serves as a testable +example of an environment where the install is known to work.) + # Build instructions for local installation @@ -52,31 +61,20 @@ We're going to install it into `~/my_msvc` to avoid needing root privileges on a # Add wrapper scripts, do minor cleanup of the unpacked MSVC installation ./install.sh ~/my_msvc/opt/msvc -# Custom CMake -git clone https://gitlab.kitware.com/mstorsjo/cmake.git -cd cmake -git checkout 844ccd2280d11ada286d0e2547c0fa5ff22bd4db -mkdir build -cd build -../configure --prefix=~/my_msvc/opt/cmake --parallel=$(nproc) -- -DCMAKE_USE_OPENSSL=OFF -make -j$(nproc) -make install - -# Run wine at least once -wineserver -k # kills server (optional) -wineserver -p -wine64 wineboot +# Optional: Start a persistent wineserver +wineserver -k # Kill a potential old server +wineserver -p # Start a new server +wine64 wineboot # Run a process to start up all background wine processes ``` ### Setting up your project with CMake -You need to set the path to prioritize our custom CMake, and also to see our MSVC installation. +You need to add the our MSVC installation to the path. After that we just run CMake command with a few extra settings: ```bash -export PATH=~/my_msvc/opt/cmake/bin:$PATH export PATH=~/my_msvc/opt/msvc/bin/x64:$PATH -CC=cl CXX=cl cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_PROGRAMS=ON -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_CROSSCOMPILING=ON +CC=cl CXX=cl cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_SYSTEM_NAME=Windows make ``` @@ -135,9 +133,29 @@ Yes, but the install scripts won't work because `msitools` is too old. You'll ne ## Does it work with CMake? -Yes, but a custom version is needed or else CMake will complain that `/opt/msvc/bin/x64/cl` can't build a simple program. - -It also fixes the `Ninja` Generator which otherwise has trouble finding RC. +Yes, but you need CMake 3.23, and either need `winbind` installed, or +need to configure the build to always use embedded debug info (or a +custom build of CMake that doesn't try to use separate PDB file debug +info by default). + +Even if configuring CMake with `-DCMAKE_BUILD_TYPE=Release`, CMake does +call the compiler in `Debug` mode when it does the first few test +invocations of the compiler. By default, CMake uses separate PDB +file debug info, when compiling in `Debug` mode, i.e. using the +`/Zi` compiler option (as opposed to the `/Z7` option, storing the +debug info in the individual object files). + +When MSVC is invoked with the `/Zi` and `/FS` options, it spawns a +background `mspdbsrv.exe` process and communicates with it. This +requires the `winbind` package to be installed for this communication +to work. + +With CMake 3.25, it's possible to override the type of debug info +even for the first few probing steps. This requires the CMake project +to either set `cmake_minimum_required(VERSION 3.25.0)`, or set +`cmake_policy(SET CMP0141 NEW)`, and requires the user to configure it +with `-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded`; in such a +configuration, `winbind` isn't needed. ## Can it build Debug versions? diff --git a/msvc/fixinclude b/msvc/fixinclude index 57c32e1..9f2be68 100755 --- a/msvc/fixinclude +++ b/msvc/fixinclude @@ -18,6 +18,8 @@ use strict; sub dodir($); +my %mapping; + sub dodir($) { my $dir = shift; @@ -40,14 +42,21 @@ sub dodir($) { open FHOUT,">$dir/$i.out" || die("Can't write to $dir/$i.out"); while(my $line = ) { - if($line =~ m/^#include/) { - my @values = split('//', $line); - $values[0] =~ tr [A-Z\\] [a-z/]; - - $line = join('//', @values); - } - $line =~ s/[\r\n]//g; - print FHOUT "$line\n"; + # Make sure to match '#include ' or '#include "bar"', but not + # '#include IDENTIFIER'. + if($line =~ m/^#\s*include\s+["<][\w\.\/\\]+[">]/) { + my @values = split('//', $line); + $values[0] =~ tr [A-Z\\] [a-z/]; + + foreach my $from (keys %mapping) { + my $to = $mapping{$from}; + $values[0] =~ s,$from,$to,; + } + + $line = join('//', @values); + } + $line =~ s/[\r\n]//g; + print FHOUT "$line\n"; } close(FHIN); @@ -58,5 +67,15 @@ sub dodir($) { } } -die("Usage: fixinclude dir\n") if(@ARGV != 1); -dodir($ARGV[0]); +my @paths; +for (my $i = 0; $i < @ARGV; $i++) { + my $arg = $ARGV[$i]; + if ($arg eq "-map_winsdk") { + # Map references to e.g. GL/gl.h to keep that canonical spelling. + $mapping{lc("GL/")} = "GL/"; + } else { + push @paths, $arg; + } +} +die("Usage: fixinclude dir\n") if(@paths != 1); +dodir($paths[0]); diff --git a/msvc/install.sh b/msvc/install.sh index 9443498..c32e232 100755 --- a/msvc/install.sh +++ b/msvc/install.sh @@ -34,13 +34,19 @@ mkdir -p $DEST cd $DEST DEST=$(pwd) +ln_s() { + if [ ! -e "$2" ]; then + ln -s "$1" "$2" + fi +} + if [ -n "$VC_ZIP" ]; then unzip $VC_ZIP fi -ln -s kits "Windows Kits" -ln -s VC vc -ln -s Tools vc/tools -ln -s MSVC vc/tools/msvc +ln_s kits "Windows Kits" +ln_s VC vc +ln_s Tools vc/tools +ln_s MSVC vc/tools/msvc # Add symlinks like LIBCMT.lib -> libcmt.lib. These are properly lowercased # out of the box, but MSVC produces directives like /DEFAULTLIB:"LIBCMT" @@ -54,17 +60,38 @@ for arch in x86 x64 arm arm64; do fi cd $arch for i in libcmt libcmtd msvcrt msvcrtd oldnames; do - ln -s $i.lib $(echo $i | tr [a-z] [A-Z]).lib + ln_s $i.lib $(echo $i | tr [a-z] [A-Z]).lib done cd .. done -cd ../bin +cd .. +# Fix casing issues in the MSVC headers. These headers mostly have consistent +# lowercase naming among themselves, but they do reference some WinSDK headers +# with mixed case names (in a spelling that isn't present in the WinSDK). +# Thus process them to reference the other headers with lowercase names. +# Also lowercase these files, as a few of them do have non-lowercase names, +# and the call to fixinclude lowercases those references. +$ORIG/lowercase -symlink include +$ORIG/fixinclude include +cd bin # vctip.exe is known to cause problems at some times; just remove it. # See https://bugs.chromium.org/p/chromium/issues/detail?id=735226 and # https://github.com/mstorsjo/msvc-wine/issues/23 for references. for i in $(find . -iname vctip.exe); do rm $i done +if [ -d HostARM64 ]; then + # 17.2 - 17.3 + mv HostARM64 Hostarm64 +fi +if [ -d HostArm64 ]; then + # 17.4 + mv HostArm64 Hostarm64 +fi +if [ -d Hostarm64/ARM64 ]; then + # 17.2 - 17.3 + mv Hostarm64/ARM64 Hostarm64/arm64 +fi cd "$DEST" if [ -d kits/10 ]; then @@ -75,8 +102,8 @@ else unzip $SDK_ZIP cd 10 fi -ln -s Lib lib -ln -s Include include +ln_s Lib lib +ln_s Include include cd ../.. SDKVER=$(basename $(echo kits/10/include/* | awk '{print $NF}')) @@ -92,26 +119,48 @@ SDKVER=$(basename $(echo kits/10/include/* | awk '{print $NF}')) # an option, because the headers aren't self consistent (headers are # included with a different mix of upper/lower case than what they have # on disk). -$ORIG/lowercase kits/10/include/$SDKVER/um -$ORIG/lowercase kits/10/include/$SDKVER/shared -$ORIG/fixinclude kits/10/include/$SDKVER/um -$ORIG/fixinclude kits/10/include/$SDKVER/shared +# +# The original casing of file names is preserved though, by adding lowercase +# symlinks instead of doing a plain rename, so files can be referred to with +# either the out of the box filename or with the lowercase name. +$ORIG/lowercase -map_winsdk -symlink kits/10/include/$SDKVER/um +$ORIG/lowercase -map_winsdk -symlink kits/10/include/$SDKVER/shared +$ORIG/fixinclude -map_winsdk kits/10/include/$SDKVER/um +$ORIG/fixinclude -map_winsdk kits/10/include/$SDKVER/shared for arch in x86 x64 arm arm64; do if [ ! -d "kits/10/lib/$SDKVER/um/$arch" ]; then continue fi - $ORIG/lowercase kits/10/lib/$SDKVER/um/$arch + $ORIG/lowercase -symlink kits/10/lib/$SDKVER/um/$arch done +host=x64 +if [ "$(uname -m)" = "aarch64" ]; then + host=arm64 +fi + SDKVER=$(basename $(echo kits/10/include/* | awk '{print $NF}')) MSVCVER=$(basename $(echo vc/tools/msvc/* | awk '{print $1}')) -cat $ORIG/wrappers/msvcenv.sh | sed 's/MSVCVER=.*/MSVCVER='$MSVCVER/ | sed 's/SDKVER=.*/SDKVER='$SDKVER/ > msvcenv.sh +cat $ORIG/wrappers/msvcenv.sh | sed 's/MSVCVER=.*/MSVCVER='$MSVCVER/ | sed 's/SDKVER=.*/SDKVER='$SDKVER/ | sed s/x64/$host/ > msvcenv.sh for arch in x86 x64 arm arm64; do if [ ! -d "vc/tools/msvc/$MSVCVER/bin/Hostx64/$arch" ]; then continue fi mkdir -p bin/$arch - cp $ORIG/wrappers/* bin/$arch + cp -a $ORIG/wrappers/* bin/$arch cat msvcenv.sh | sed 's/ARCH=.*/ARCH='$arch/ > bin/$arch/msvcenv.sh done rm msvcenv.sh + +if [ -d "$DEST/bin/$host" ] && [ -x "$(which wine64 2>/dev/null)" ]; then + WINEDEBUG=-all wine64 wineboot &>/dev/null + echo "Build msvctricks ..." + "$DEST/bin/$host/cl" /EHsc /O2 "$ORIG/msvctricks.cpp" + if [ $? -eq 0 ]; then + mv msvctricks.exe bin/ + rm msvctricks.obj + echo "Build msvctricks done." + else + echo "Build msvctricks failed." + fi +fi diff --git a/msvc/lowercase b/msvc/lowercase index f655ce8..ee10cc7 100755 --- a/msvc/lowercase +++ b/msvc/lowercase @@ -147,6 +147,11 @@ for (my $i = 0; $i < @ARGV; $i++) { if ($i + 1 < @ARGV) { readMapping($ARGV[$i+1]); } $map_files = 1; $i += 1; + } elsif ($arg eq "-map_winsdk") { + $map_paths = 1; + # Keep the GL header directory in upper case, as that's the canonical + # cross platform spelling of that directory. + $mapping{lc("GL")} = "GL"; } else { push @paths, $arg; } diff --git a/msvc/msvcenv-native.sh b/msvc/msvcenv-native.sh index 43c2c75..e83c687 100755 --- a/msvc/msvcenv-native.sh +++ b/msvc/msvcenv-native.sh @@ -34,6 +34,14 @@ else echo $ENV doesn\'t exist else export INCLUDE="$(bash -c ". $ENV && /bin/echo \"\$INCLUDE\"" | sed s/z://g | sed 's/\\/\//g')" - export LIB="$(bash -c ". $ENV && /bin/echo \"\$LIB\"" | sed s/z://g | sed 's/\\/\//g' | sed 's/;/;/g')" + export LIB="$(bash -c ". $ENV && /bin/echo \"\$LIB\"" | sed s/z://g | sed 's/\\/\//g')" + MSVCARCH="$(bash -c ". $ENV && /bin/echo \"\$ARCH\"")" + case $MSVCARCH in + x86) TARGET_ARCH=i686 ;; + x64) TARGET_ARCH=x86_64 ;; + arm) TARGET_ARCH=armv7 ;; + arm64) TARGET_ARCH=aarch64 ;; + esac + TARGET_TRIPLE=$TARGET_ARCH-windows-msvc fi fi diff --git a/msvc/msvctricks.cpp b/msvc/msvctricks.cpp new file mode 100644 index 0000000..0d4ea4b --- /dev/null +++ b/msvc/msvctricks.cpp @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2023 Huang Qinjin + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include + + +#pragma comment(lib, "shell32.lib") +#pragma comment(lib, "shlwapi.lib") +#pragma comment(linker, "/ENTRY:wWinMainCRTStartup") +#pragma comment(linker, "/SUBSYSTEM:CONSOLE") + + +static HANDLE hStdIn = INVALID_HANDLE_VALUE; +static HANDLE hStdOut = INVALID_HANDLE_VALUE; +static HANDLE hStdErr = INVALID_HANDLE_VALUE; +static HANDLE hChildJob; + +static DWORD run(LPWSTR lpCmdLine) +{ + STARTUPINFOW si = {}; + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + si.hStdInput = hStdIn; + si.hStdOutput = hStdOut; + si.hStdError = hStdErr; + + PROCESS_INFORMATION pi = {}; + + DWORD dwExitCode; + if (CreateProcessW(nullptr, lpCmdLine, nullptr, nullptr, TRUE, 0, nullptr, nullptr, &si, &pi)) + { + if (hChildJob) + { + AssignProcessToJobObject(hChildJob, pi.hProcess); + } + + WaitForSingleObject(pi.hProcess, INFINITE); + + if (!GetExitCodeProcess(pi.hProcess, &dwExitCode)) + { + dwExitCode = GetLastError(); + } + + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + } + else + { + dwExitCode = GetLastError(); + } + + return dwExitCode; +} + +static DWORD mt(LPWSTR lpCmdLine) +{ + DWORD dwExitCode = run(lpCmdLine); + // https://gitlab.kitware.com/cmake/cmake/-/blob/v3.26.0/Source/cmcmd.cxx#L2405 + if (dwExitCode == 0x41020001) + dwExitCode = 0xbb; + return dwExitCode; +} + +int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPWSTR lpCmdLine, int nCmdShow) +{ + (void) hInstance; + (void) hPrevInstance; + (void) nCmdShow; + + int argc = 0; + wchar_t** argv = CommandLineToArgvW(lpCmdLine, &argc); + if (argc <= 0) return 0; + + wchar_t buf[32768]; + if (GetEnvironmentVariableW(L"WINE_MSVC_STDIN", buf, ARRAYSIZE(buf))) + { + SECURITY_ATTRIBUTES attr = {}; + attr.nLength = sizeof(attr); + attr.bInheritHandle = TRUE; + + hStdIn = CreateFileW(buf, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &attr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + } + if (GetEnvironmentVariableW(L"WINE_MSVC_STDOUT", buf, ARRAYSIZE(buf))) + { + SECURITY_ATTRIBUTES attr = {}; + attr.nLength = sizeof(attr); + attr.bInheritHandle = TRUE; + + hStdOut = CreateFileW(buf, GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &attr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + } + if (GetEnvironmentVariableW(L"WINE_MSVC_STDERR", buf, ARRAYSIZE(buf))) + { + SECURITY_ATTRIBUTES attr = {}; + attr.nLength = sizeof(attr); + attr.bInheritHandle = TRUE; + + hStdErr = CreateFileW(buf, GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &attr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + } + + if (hStdIn == INVALID_HANDLE_VALUE) + { + hStdIn = GetStdHandle(STD_INPUT_HANDLE); + } + if (hStdOut == INVALID_HANDLE_VALUE) + { + hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); + } + if (hStdErr == INVALID_HANDLE_VALUE) + { + hStdErr = GetStdHandle(STD_ERROR_HANDLE); + } + + if (hChildJob = CreateJobObjectW(nullptr, nullptr)) + { + JOBOBJECT_EXTENDED_LIMIT_INFORMATION info = {}; + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + | JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK; + SetInformationJobObject(hChildJob, JobObjectExtendedLimitInformation, &info, sizeof(info)); + } + + LPCWSTR const exe = PathFindFileNameW(argv[0]); + + if (PathMatchSpecW(exe, L"mt.exe")) + return mt(lpCmdLine); + + return run(lpCmdLine); +} diff --git a/msvc/vsdownload.py b/msvc/vsdownload.py index 5b20b89..147e29c 100755 --- a/msvc/vsdownload.py +++ b/msvc/vsdownload.py @@ -511,9 +511,12 @@ def unpackWin10SDK(src, payloads, dest): if name.endswith(".msi"): print("Extracting " + name) srcfile = os.path.join(src, name) - log = open(os.path.join(dest, "WinSDK-" + getPayloadName(payload) + "-listing.txt"), "w") - subprocess.check_call(["msiextract", "-C", dest, srcfile], stdout=log) - log.close() + if sys.platform == "win32": + cmd = ["msiexec", "/a", srcfile, "/qn", "TARGETDIR=" + os.path.abspath(dest)] + else: + cmd = ["msiextract", "-C", dest, srcfile] + with open(os.path.join(dest, "WinSDK-" + getPayloadName(payload) + "-listing.txt"), "w") as log: + subprocess.check_call(cmd, stdout=log) def extractPackages(selected, cache, dest): makedirs(dest) @@ -538,7 +541,12 @@ def moveVCSDK(unpack, dest): # files to be removed. makedirs(os.path.join(dest, "kits")) mergeTrees(os.path.join(unpack, "VC"), os.path.join(dest, "VC")) - mergeTrees(os.path.join(unpack, "Program Files", "Windows Kits", "10"), os.path.join(dest, "kits", "10")) + kitsPath = unpack + # msiexec extracts to Windows Kits rather than Program Files\Windows Kits + if sys.platform != "win32": + kitsPath = os.path.join(kitsPath, "Program Files") + kitsPath = os.path.join(kitsPath, "Windows Kits", "10") + mergeTrees(kitsPath, os.path.join(dest, "kits", "10")) # The DIA SDK isn't necessary for normal use, but can be used when e.g. # compiling LLVM. mergeTrees(os.path.join(unpack, "DIA SDK"), os.path.join(dest, "DIA SDK")) diff --git a/msvc/wrappers/cl b/msvc/wrappers/cl index 30b5336..8e89090 100755 --- a/msvc/wrappers/cl +++ b/msvc/wrappers/cl @@ -15,4 +15,36 @@ # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. . $(dirname $0)/msvcenv.sh + +# /showIncludes +unixify_path='/^Note: including file: /{ s/z:([\\/])/\1/i; s,\\,/,g; }' +# /E +unixify_line='/^[[:blank:]]*#[[:blank:]]*line[[:blank:]]/{ s/z:([\\/])/\1/i; s,\\\\,/,g; }' + +export WINE_MSVC_STDOUT_SED="$unixify_path;$unixify_line" +export WINE_MSVC_STDERR_SED="$unixify_path" + $(dirname $0)/wine-msvc.sh $BINDIR/cl.exe "$@" + +ec=$? +[ $ec -ne 0 ] && exit $ec + +# Postprocess +for a in "$@"; do + case $a in + [-/]P) arg_P=$a ;; + [-/]Fi*) arg_Fi=${a:3} ;; + esac +done + +# Unixify paths for /P +if [ -n "$arg_P" ] && [ -f "$arg_Fi" ]; then + if sed --help 2>&1 | grep '\-i extension' >/dev/null; then + inplace=(-i '') # BSD sed + else + inplace=(-i) # GNU sed + fi + sed "${inplace[@]}" -E 's/\r//;'"$unixify_line" "$arg_Fi" +fi + +exit $ec diff --git a/msvc/wrappers/dumpbin b/msvc/wrappers/dumpbin index 93fe37b..531ac80 100755 --- a/msvc/wrappers/dumpbin +++ b/msvc/wrappers/dumpbin @@ -15,4 +15,10 @@ # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. . $(dirname $0)/msvcenv.sh + +# /PDBPATH +unixify_path='/^(Dump of file | PDB file found at )/{ s/z:([\\/])/\1/i; s,\\,/,g; }' + +export WINE_MSVC_STDOUT_SED="$unixify_path" + $(dirname $0)/wine-msvc.sh $BINDIR/dumpbin.exe "$@" diff --git a/msvc/wrappers/msvcenv.sh b/msvc/wrappers/msvcenv.sh index a55b060..f647f3d 100755 --- a/msvc/wrappers/msvcenv.sh +++ b/msvc/wrappers/msvcenv.sh @@ -36,4 +36,4 @@ export LIB="$MSVCDIR\\lib\\$ARCH;$SDKLIB\\ucrt\\$ARCH;$SDKLIB\\um\\$ARCH" export LIBPATH="$LIB" # "$MSVCDIR\\bin\\Hostx64\\x64" is included in PATH for DLLs. export WINEPATH="${BINDIR//\//\\};${SDKBINDIR//\//\\};$MSVCDIR\\bin\\Hostx64\\x64" -export WINEDLLOVERRIDES="vcruntime140_1=n" +export WINEDLLOVERRIDES="vcruntime140=n;vcruntime140_1=n" diff --git a/msvc/wrappers/wine-msvc.sh b/msvc/wrappers/wine-msvc.sh index 06138c6..878f371 100755 --- a/msvc/wrappers/wine-msvc.sh +++ b/msvc/wrappers/wine-msvc.sh @@ -14,20 +14,46 @@ # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -EXE=$1 -shift +EXE=$(dirname $0)/../msvctricks.exe +if [ -f "$EXE" ]; then + HAS_MSVCTRICKS=true +else + HAS_MSVCTRICKS=false + EXE=$1 + shift +fi + ARGS=() while [ $# -gt 0 ]; do a=$1 case $a in + [-/][A-Za-z]/*) + opt=${a:0:2} + path=${a:2} + # Rewrite options like -I/absolute/path into -Iz:/absolute/path. + # This is needed to avoid what seems like a cl.exe/Wine bug combination + # in some very rare cases, see https://bugs.winehq.org/show_bug.cgi?id=55200 + # for details. In those rare cases, cl.exe fails to find includes in + # some directories specified with -I/absolute/path but does find them if + # they have been specified as -Iz:/absolute/path. + if [ -d "$(dirname "$path")" ] && [ "$(dirname "$path")" != "/" ]; then + a=${opt}z:$path + fi + ;; [-/][A-Za-z][A-Za-z]/*) opt=${a:0:3} path=${a:3} + # Rewrite options like -Fo/absolute/path into -Foz:/absolute/path. + # This doesn't seem to be strictly needed for any known case at the moment, but + # might have been needed with some version of MSVC or Wine earlier. if [ -d "$(dirname "$path")" ] && [ "$(dirname "$path")" != "/" ]; then a=${opt}z:$path fi ;; /*) + # Rewrite options like /absolute/path into z:/absolute/path. + # This is essential for disambiguating e.g. /home/user/file from the + # tool option /h with the value ome/user/file. if [ -d "$(dirname "$a")" ] && [ "$(dirname "$a")" != "/" ]; then a=z:$a fi @@ -38,5 +64,29 @@ while [ $# -gt 0 ]; do ARGS+=("$a") shift done -wine64 "$EXE" "${ARGS[@]}" 2> >(grep -v '^[[:alnum:]]*:\?fixme' | grep -v ^err:bcrypt:hash_init | sed 's/\r//' >&2) | sed 's/\r//' | sed 's/z:\([\\/]\)/\1/i' | sed '/^Note:/s,\\,/,g' -exit $PIPESTATUS + +WINE_MSVC_STDOUT_SED='s/\r//;'"$WINE_MSVC_STDOUT_SED" +WINE_MSVC_STDERR_SED='s/\r//;'"$WINE_MSVC_STDERR_SED" + +if ! $HAS_MSVCTRICKS; then + WINEDEBUG=-all wine64 "$EXE" "${ARGS[@]}" 2> >(sed -E "$WINE_MSVC_STDERR_SED" >&2) | sed -E "$WINE_MSVC_STDOUT_SED" + exit $PIPESTATUS +else + export WINE_MSVC_STDOUT=${TMPDIR:-/tmp}/wine-msvc.stdout.$$ + export WINE_MSVC_STDERR=${TMPDIR:-/tmp}/wine-msvc.stderr.$$ + + cleanup() { + wait + rm -f $WINE_MSVC_STDOUT $WINE_MSVC_STDERR + } + + trap cleanup EXIT + + cleanup && mkfifo $WINE_MSVC_STDOUT $WINE_MSVC_STDERR || exit 1 + + WINEDEBUG=-all wine64 "$EXE" "${ARGS[@]}" &>/dev/null & + pid=$! + sed -E "$WINE_MSVC_STDOUT_SED" <$WINE_MSVC_STDOUT || kill $pid &>/dev/null & + sed -E "$WINE_MSVC_STDERR_SED" <$WINE_MSVC_STDERR >&2 || kill $pid &>/dev/null & + wait $pid &>/dev/null +fi From 7e6bfa0fa42eb6441d2195d457418790f7806da9 Mon Sep 17 00:00:00 2001 From: Myk Taylor Date: Fri, 1 Sep 2023 17:22:11 -0700 Subject: [PATCH 2/2] update and simplify image config --- msvc/Dockerfile | 54 ++++++++++++++++++----------------------- msvc/ccache-4.7.4.patch | 13 ---------- msvc/dfhack-configure | 2 +- 3 files changed, 25 insertions(+), 44 deletions(-) delete mode 100644 msvc/ccache-4.7.4.patch diff --git a/msvc/Dockerfile b/msvc/Dockerfile index 01ee9c4..72a8bdf 100644 --- a/msvc/Dockerfile +++ b/msvc/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:22.04 LABEL org.opencontainers.image.source https://github.com/DFHack/build-env @@ -8,17 +8,22 @@ ARG BUILDER_UID=1001 RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y wine64-development python3 msitools python3-simplejson python3-six ca-certificates \ - && apt-get install -y gcc g++ ninja-build git zlib1g-dev libsdl1.2-dev libxml-libxml-perl libxml-libxslt-perl make libncurses-dev libssl-dev \ + && apt-get install -y git gcc g++ ninja-build libxml-libxslt-perl make libssl-dev \ && apt-get install -y dh-autoreconf libcurl4-gnutls-dev libexpat1-dev gettext libz-dev \ - && apt-get install -y wget curl unzip vim bash-completion xvfb python3-pip \ + && apt-get install -y wget curl unzip python3-pip \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* \ && pip3 install --compile sphinx \ && useradd --uid $BUILDER_UID --create-home --shell /bin/bash buildmaster +# Initialize the wine environment for root. Wait until the wineserver process has +# exited before closing the session, to avoid corrupting the wine prefix. +RUN wine64 wineboot --init && \ + while pgrep wineserver > /dev/null; do sleep 1; done + # set up msvc WORKDIR /opt/msvc -COPY lowercase fixinclude install.sh vsdownload.py msvcenv-native.sh ./ +COPY lowercase fixinclude install.sh vsdownload.py msvctricks.cpp ./ COPY wrappers/* ./wrappers/ # This removes the arm and x86 bits to save space (around 5.5G) @@ -29,47 +34,36 @@ RUN PYTHONUNBUFFERED=1 ./vsdownload.py --accept-license --dest /opt/msvc \ && find /opt/msvc -depth -type d -iregex '.*/.*arm[0-9]*$' -exec rm -fr {} \; \ && find /opt/msvc -depth -type d -iregex '.*/.*x86$' -exec rm -fr {} \; -ENV BIN=/opt/msvc/bin/x64 \ - PATH=/opt/msvc:/opt/msvc/bin/x64:/opt/git/bin:/opt/ccache/bin:/opt/cmake/bin:$PATH \ - WINEARCH=win64 \ - WINEDLLOVERRIDES='ngen.exe,mscorsvw.exe=b' \ - WINEDEBUG=-all - -# compile more recent git for ssh keysigning -RUN mkdir -p /opt/git/src && cd /opt/git/src \ - && wget https://github.com/git/git/archive/refs/tags/v2.38.1.tar.gz \ - && tar xzf "v2.38.1.tar.gz" \ - && cd git-2.38.1 \ - && make configure && ./configure --prefix=/opt/git \ - && make -j$(nproc) all && make install \ - && cd && rm -rf /opt/git/src +COPY msvcenv-native.sh /opt/msvc # compile custom cmake for msvc and ninja compatibility RUN mkdir -p /opt/cmake/src && cd /opt/cmake/src \ && git clone -b msvc-3.22.1 --depth=1 https://gitlab.kitware.com/mstorsjo/cmake.git \ - && mkdir cmake/build && cd cmake/build && ../configure --prefix=/opt/cmake --parallel=$(nproc) -- -DBUILD_CursesDialog=ON \ + && mkdir cmake/build && cd cmake/build && ../configure --prefix=/opt/cmake --parallel=$(nproc) \ && make -j$(nproc) && make install \ && cd && rm -rf /opt/cmake/src -# compile custom ccache for msvc caching -COPY ccache-4.7.4.patch /tmp -RUN mkdir -p /opt/ccache/src && cd /opt/ccache/src \ - && git clone -b "v4.7.4" --depth=1 https://github.com/ccache/ccache.git \ - && mkdir ccache/build && cd ccache && git apply /tmp/ccache-4.7.4.patch \ - && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/ccache .. \ - && make -j$(nproc) && make install && cd && rm -rf /opt/ccache/src +# install newer ccache for msvc caching +RUN mkdir -p /opt/ccache && cd /opt/ccache \ + && wget 'https://github.com/ccache/ccache/releases/download/v4.8.3/ccache-4.8.3-linux-x86_64.tar.xz' -O- | tar -xJ \ + && mv * bin + +ENV BIN=/opt/msvc/bin/x64 \ + PATH=/opt/msvc:/opt/msvc/bin/x64:/opt/ccache/bin:/opt/cmake/bin:$PATH \ + WINEARCH=win64 \ + WINEDLLOVERRIDES='ngen.exe,mscorsvw.exe=b' \ + WINEDEBUG=-all # set up dfhack build environment USER buildmaster WORKDIR /home/buildmaster # Add native build directory for DFHack cross compiling -# maintainer: Update cloned git tag when we update protobuf to keep them in sync RUN mkdir /home/buildmaster/dfhack-native \ && cd /home/buildmaster/dfhack-native \ - && git clone -b "0.47.05-r8" --depth=1 https://github.com/DFHack/dfhack.git \ + && git clone --depth=1 https://github.com/DFHack/dfhack.git \ && cd dfhack && git submodule update --init && cd .. \ - && cmake dfhack -GNinja -DCMAKE_BUILD_TYPE=Release -DDOWNLOAD_RUBY=OFF \ + && cmake dfhack -GNinja -DCMAKE_BUILD_TYPE=Release \ && ninja protoc-bin \ && cd .. \ && mkdir -p dfhack-native-bin/depends/protobuf \ @@ -78,7 +72,7 @@ RUN mkdir /home/buildmaster/dfhack-native \ && rm -rf dfhack-native \ && mv dfhack-native-bin dfhack-native -# set up wine (get rid of the "install mono?" and "install gecko?" prompts) and ccache +# set up wine and ccache for the buildmaster user RUN WINEDLLOVERRIDES="mscoree,mshtml=" wine64 wineboot --init \ && wineserver -w \ && ccache -C diff --git a/msvc/ccache-4.7.4.patch b/msvc/ccache-4.7.4.patch deleted file mode 100644 index 5f558aa..0000000 --- a/msvc/ccache-4.7.4.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/argprocessing.cpp b/src/argprocessing.cpp -index 0bc37e57..75b7cfa2 100644 ---- a/src/argprocessing.cpp -+++ b/src/argprocessing.cpp -@@ -287,7 +287,7 @@ process_option_arg(const Context& ctx, - } - - bool changed_from_slash = false; -- if (ctx.config.is_compiler_group_msvc() && util::starts_with(args[i], "/")) { -+ if (ctx.config.is_compiler_group_msvc() && !state.found_c_opt && util::starts_with(args[i], "/")) { - // MSVC understands both /option and -option, so convert all /option to - // -option to simplify our handling. - args[i][0] = '-'; diff --git a/msvc/dfhack-configure b/msvc/dfhack-configure index 58370b8..37694cb 100755 --- a/msvc/dfhack-configure +++ b/msvc/dfhack-configure @@ -54,7 +54,7 @@ esac wineserver -p wine64 wineboot -CC=cl CXX=cl cmake .. -GNinja -DDFHACK_BUILD_ARCH=$bits -DCMAKE_BUILD_TYPE=$mode -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_CROSSCOMPILING=ON -DDFHACK_NATIVE_BUILD_DIR=/home/buildmaster/dfhack-native "$@" <&0 +CC=cl CXX=cl cmake .. -GNinja -DDFHACK_BUILD_ARCH=$bits -DCMAKE_BUILD_TYPE=$mode -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_CROSSCOMPILING=ON -DDFHACK_NATIVE_BUILD_DIR=/home/buildmaster/dfhack-native -DGITCOMMAND=$(which git) -DGIT_EXECUTABLE=$(which git) "$@" <&0 status=$? wineserver -k