chore: Streamline usage

Write a helper script for managing docker setups. Use JSON instead of
tab separated files for environment configuration. Simplify the
docker-compose files and remove legacy scripts.

Remove obsolete stuff related to spawner patching and integration
testing from Makefile.

Make spawner patchable with standard tools by including patches in the
repository.

Remove redundant UID args from docker containers.

Makefile

@@ -6,20 +6,16 @@ VERSION = SOFT_VERSION-$(shell git rev-parse --short HEAD)
 
 REPO_FILES = $(shell git ls-tree -r --name-only HEAD) 
 TESTS := $(patsubst %.cpp,%.exe,$(subst tests/,bin/,$(shell find tests/ -iname "test_*.cpp")))
-PYTHON := python3
 
 export CMAKE_TOOLCHAIN_FILE ?= toolchains/mingw-w64-i686.cmake
 BASE_DIR := $(BUILDDIR)/$(notdir $(basename $(CMAKE_TOOLCHAIN_FILE)))-$(CMAKE_BUILD_TYPE)
 BUILD_DIR = $(BASE_DIR)/build
 export DEST_DIR = $(BASE_DIR)/pkg
 BUILDER ?= builder
 
-export CPPCHECK ?= cppcheck
 export CM_FILES = $(filter %CMakeLists.txt, $(REPO_FILES))
 export CPP_SOURCES = $(filter %.cpp %.hpp %.c %.h, $(REPO_FILES))
 export W32_FILES := process.cpp state_parser.cpp dll_inject.cpp network.cpp addscn/addscn.cpp
-GAMEMD_PATCHED := $(DEST_DIR)/bin/gamemd-spawn-patched.exe
-EXTRA_PATCHES ?=
 
 export UID := $(shell id -u)
 export GID := $(shell id -g)
@@ -28,16 +24,9 @@ export CMAKE_TARGET ?= all
 export CMAKE_EXTRA_ARGS ?=
 export CXXFLAGS ?= -Wall -Wextra
 
-DLL_LOADER_UNIX = $(BUILD_DIR)/load_dll.bin
-DLL_LOADER = $(subst \,\\,$(shell winepath -w $(DLL_LOADER_UNIX)))
-
-INTEGRATION_TEST ?= docker-compose.integration.yml
-INTEGRATION_TEST_TARGET ?= ./pyra2yr/test_sell_mcv.py
-COMPOSE_ARGS ?= --abort-on-container-exit pyra2yr tunnel wm vnc novnc game-0 game-1
-compose_cmd := docker-compose -f docker-compose.yml -f $(INTEGRATION_TEST)
 # need -T flag for this to work properly in shell scripts, but this causes ctrl+c not to work.
 # TODO: find a workaround
-compose_build = docker-compose run -e BUILDDIR -e CMAKE_TOOLCHAIN_FILE -e CMAKE_TARGET -e NPROC -e CMAKE_BUILD_TYPE -e EXTRA_PATCHES -e CMAKE_EXTRA_ARGS -e CXXFLAGS -e TAG_NAME -T --rm $(BUILDER)
+compose_build = docker-compose run -e BUILDDIR -e CMAKE_TOOLCHAIN_FILE -e CMAKE_TARGET -e NPROC -e CMAKE_BUILD_TYPE -e CMAKE_EXTRA_ARGS -e CXXFLAGS -e TAG_NAME -T --rm $(BUILDER)
 
 
 doc:
@@ -77,33 +66,7 @@ build_cpp: $(BUILD_DIR)
 	cmake --build $(BUILD_DIR) --config $(CMAKE_BUILD_TYPE) --target $(CMAKE_TARGET) -j $(NPROC)
 	cmake --build $(BUILD_DIR) --config $(CMAKE_BUILD_TYPE) --target install/fast
 
-build: build_cpp $(GAMEMD_PATCHED)
-
-$(BUILD_DIR)/.gamemd-spawn.exe: gamemd-spawn.exe
-	cp $< $@
-
-# FIXME: if building just core lib, ra2yrcppcli.exe is unavailable
-$(BUILD_DIR)/p_text2.txt: $(BUILD_DIR)/.gamemd-spawn.exe
-	$(DEST_DIR)/bin/ra2yrcppcli.exe \
-		--address-GetProcAddr=0x7e1250 \
-		--address-LoadLibraryA=0x7e1220 \
-		--generate-dll-loader=$(DLL_LOADER)
-	$(DEST_DIR)/bin/addscn.exe $< .p_text2 0x1000 0x60000020 > $@
-	wineserver -w
-
-# FIXME: autodetect detour address
-$(GAMEMD_PATCHED): $(BUILD_DIR)/p_text2.txt
-	$(eval s_ptext2 := $(shell cat $(<)))
-	$(eval s_ptext2_addr := $(shell cat $(<) | cut -f 3 -d":"))
-	$(PYTHON) ./scripts/patch_gamemd.py \
-		-p "d0x7cd80f:$(BUILD_DIR)/load_dll.bin" \
-		$(EXTRA_PATCHES) \
-		-d $(s_ptext2_addr) \
-		-s ".p_text:0x00004d66:0x00b7a000:0x0047e000" \
-		-s ".text:0x003df38d:0x00401000:0x00001000" \
-		-s "$(s_ptext2)" \
-		-i $(BUILD_DIR)/.gamemd-spawn.exe > $(BUILD_DIR)/.gamemd-spawn-patched.exe
-	install -D $(BUILD_DIR)/.gamemd-spawn-patched.exe $@
+build: build_cpp
 
 build_protobuf:
 	mkdir -p $(BUILD_DIR)
@@ -129,20 +92,16 @@ test:
 		wineboot -s; \
 		wine $(DEST_DIR)/$$f; done
 
-test_integration: $(GAMEMD_PATCHED)
-	BUILDDIR=$(BUILDDIR) COMMAND='./scripts/run_gamemd.sh' COMMAND_PYRA2YR='python3 $(INTEGRATION_TEST_TARGET)' $(compose_cmd) up $(COMPOSE_ARGS)
-
-docker_base:
-	docker-compose build --build-arg USER_ID=$(UID)
+docker-base:
+	docker-compose build builder pyra2yr tunnel vnc
 
 docker_test:
 	set -e; for f in $(TESTS); do \
 		docker-compose down --remove-orphans; \
 		COMMAND="sh -c 'UID=$(UID) BUILDDIR=$(BUILDDIR) make BUILDDIR=$(BUILDDIR) DEST_DIR=$(DEST_DIR) TESTS=$$f test'" docker-compose up --abort-on-container-exit builder; done
 
 # NB. using "run" the env. vars need to be specified with -e flag
-# actually we dont wanna pass TC in env var, because it overrides the --toolchain flag, which we use to transform relative path
-docker_build:
+docker-build:
 	$(compose_build) make build
 
 cppcheck:
@@ -151,15 +110,12 @@ cppcheck:
 check: cmake_format lint format
 	./scripts/check.sh
 
-clean-gamemd:
-	rm -f $(GAMEMD_PATCHED) $(BUILD_DIR)/p_text2.txt $(BUILD_DIR)/.gamemd-spawn.exe
-
 clean:
 	rm -rf $(BUILDDIR); mkdir -p $(BUILDDIR)
-	rm -f test_data/*.status $(GAMEMD_PATCHED)
+	rm -f test_data/*.status
 
 # TODO: check out the special $(MAKE) variable that presumably passes flags
 docker-release:
 	$(compose_build) ./scripts/create-release.sh
 
-.PHONY: build build_cpp doc lint format test docker docker_build check cppcheck clean clean-gamemd
+.PHONY: build build_cpp doc lint format test docker docker_build check cppcheck clean

data/patches.txt

@@ -0,0 +1,7 @@
+d0x7cd80f:609c5589e568646c6c00686370702e6861327972686c6962728d45f050a120127e00ffd089ec506a656872766963685f69736568696e69748d45ec508b45fc50a150127e00ffd06a0068b93800006a10ffd089ec5d9d61:s10
+d0x55de4f:s7
+d0x72dfb0:s6
+d0x7b3d6f:s6
+d0x7b3f15:s6
+d0x643c62:s6
+d0x4068e0:s6

docker-compose.integration.yml

@@ -1,42 +1,31 @@
 version: "3.9"
+
+x-common: &common
+  cap_add:
+    - SYS_PTRACE
+  command: "${COMMAND}"
+  depends_on:
+    - wm
+  env_file:
+    - integration.env
+  image: shmocz/ra2yrcpp:latest
+  network_mode: service:vnc
+  stop_signal: SIGKILL
+  user: "${UID}:${GID}"
+  volumes:
+    - .:/home/user/project
+    - gamedata:/home/user/RA2
+    - ./.wine-dir:/home/user/.wine
+  working_dir: /home/user/project
+
 services:
   game-0:
-    cap_add:
-      - SYS_PTRACE
-    stop_signal: SIGKILL
-    image: shmocz/ra2yrcpp:latest
-    depends_on:
-      - wm
-    user: "${UID}:${GID}"
-    working_dir: /home/user/project
-    volumes:
-      - gamedata:/home/user/RA2
-      - ./wine-dir:/home/user/.wine
-      - .:/home/user/project
-    command: "${COMMAND}"
-    env_file:
-      - integration.env
+    <<: *common
     environment:
       PLAYER_ID: player_0
       RA2YRCPP_PORT: 14521
-      RA2YRCPP_WS_PORT: 14525
-    network_mode: service:vnc
   game-1:
-    stop_signal: SIGKILL
-    image: shmocz/ra2yrcpp:latest
-    depends_on:
-      - wm
-    user: "${UID}:${GID}"
-    working_dir: /home/user/project
-    volumes:
-      - gamedata:/home/user/RA2
-      - ./wine-dir:/home/user/.wine
-      - .:/home/user/project
-    command: "${COMMAND}"
-    env_file:
-      - integration.env
+    <<: *common
     environment:
       PLAYER_ID: player_1
       RA2YRCPP_PORT: 14522
-      RA2YRCPP_WS_PORT: 14526
-    network_mode: service:vnc

docker-compose.yml

@@ -1,4 +1,5 @@
 version: "3.9"
+
 services:
   vnc:
     build:
@@ -13,7 +14,6 @@ services:
       - "12001:5901" # vnc
       - "50000:50000" # tunnel
       - "14521:14521" # ra2yrcpp
-      - "14525:14525" # ws proxy
       - "12340:12340" # debugger
   novnc:
     image: shmocz/vnc:latest

docker/tunnel.Dockerfile

@@ -1,11 +1,10 @@
 FROM alpine:3.16.2
-ARG USER_ID
 
 RUN apk add --update --no-cache python3 git
 RUN python3 -m ensurepip
 RUN pip3 install --no-cache --upgrade pip setuptools
 RUN python3 -m pip install 'pycncnettunnel @ git+https://github.com/shmocz/pycncnettunnel.git'
-RUN adduser -D -u $USER_ID user
+RUN adduser -D user
 USER user
 
 CMD pycncnettunnel

docker/vnc.Dockerfile

@@ -7,10 +7,9 @@ COPY --from=src /noVNC /
 RUN apk add --no-cache bash git nodejs python3
 
 FROM novnc
-ARG USER_ID
 
 RUN apk add --no-cache tigervnc openbox xterm terminus-font bash
-RUN adduser -D -u $USER_ID user
+RUN adduser -D user
 
 USER user
 WORKDIR /home/user

integration.env

@@ -1,19 +1,6 @@
 DISPLAY=:1
 HOME=/home/user
-PLAYERS_CONFIG=test_data/envs.tsv
 RA2YRCPP_GAME_DIR=/home/user/RA2
-RA2YRCPP_TEST_INSTANCES_DIR=/home/user/project/${BUILDDIR}/test_instances
 WINEARCH=win32
-# TODO: find a workaround for this
-DEST_DIR=${DEST_DIR}
 USER=${UID}
 GID=${GID}
-MAP_PATH=test_data/dry_heat.map
-INI_OVERRIDE=test_data/cheap_items.ini
-# set to 1 for 60FPS in single player
-GAME_SPEED=0
-FRAME_SEND_RATE=1
-RA2_MODE=False
-PROTOCOL_VERSION=0
-TUNNEL_ADDRESS="0.0.0.0"
-TUNNEL_PORT=50000

scripts/debug.sh

@@ -7,11 +7,12 @@ HOMEDIR="/home/user/project"
 : ${BUILDDIR:="cbuild_docker"}
 TOOLCHAIN="$(echo $CMAKE_TOOLCHAIN_FILE | sed -E 's/.+\/(.+)\.cmake/\1/g')-${CMAKE_BUILD_TYPE}"
 : ${TARGET:="localhost:12340"}
+: ${INTEGRATION_TEST_TARGET:="./pyra2yr/test_sell_mcv.py"}
 
 # Executable to be passed to wine and it's args, example:
 # EXE="$HOMEDIR/$BUILDDIR/$TOOLCHAIN/pkg/bin/test_dll_inject.exe --gtest_repeat=-1 --gtest_filter=*IServiceDLL*"
 : ${EXE:="$HOMEDIR/$BUILDDIR/$TOOLCHAIN/pkg/bin/test_dll_inject.exe --gtest_repeat=-1 --gtest_filter=*IServiceDLL*"}
-GDB_COMMAND='x86_64-w64-mingw32-gdb -iex "set pagination off" -ex "target extended-remote '"$TARGET"'" -ex "set pagination off" -ex "set logging overwrite on" -ex "set logging on" -ex "set disassembly-flavor intel"'
+GDB_COMMAND='gdb'
 : ${GDB_SCRIPT:="$HOMEDIR/scripts/debug.gdb"}
 
 function dcmd_generic() {
@@ -23,18 +24,21 @@ function dcmd_generic() {
 
 function dcmd_integration() {
 	: ${user:="root"}
-	cmd="$(printf 'WINEPREFIX=${HOME}/project/%s/test_instances/${PLAYER_ID}/.wine %s' "$BUILDDIR" "$1")"
-	docker-compose -f docker-compose.yml -f docker-compose.integration.yml exec --user "$user" \
-		-w "$HOMEDIR"/$BUILDDIR/test_instances/player_${PLAYER_ID} \
-		game-$PLAYER_ID bash -c "$cmd"
+	: ${it=""}
+	docker exec $it --user "$user" -w "$HOMEDIR"/$BUILDDIR/test_instances/player_${PLAYER_ID} \
+		game-0-$PLAYER_ID bash -c "$1"
 }
 
 function gdb_connect() {
-	dcmd_integration "$(printf '%s %s' "$GDB_COMMAND" "$1")"
+	it="-it" dcmd_integration "$(printf '%s %s' "$GDB_COMMAND" "$1")"
 }
 
 function debug_integration_test() {
-	a="$(printf '%s "source "%s""' "-ex" "$GDB_SCRIPT")"
+	# get target PID
+	game_pid="$(dcmd_integration "pgrep -f gamemd-spawn-ra2yrcpp.exe")"
+	a="$(printf -- '-p %s %s "source "%s""' "$game_pid" "-ex" "$GDB_SCRIPT")"
+
+	# attach (and load symbols)
 	gdb_connect "$a"
 }
 
@@ -48,8 +52,13 @@ function tmux_send_keys() {
 }
 
 function debug_integration() {
-	make INTEGRATION_TEST_TARGET="$INTEGRATION_TEST_TARGET" test_integration &
-	sleep 3
+	python ./scripts/run-gamemd.py \
+		-b $BUILDDIR/test_instances \
+		-B $BUILDDIR/mingw-w64-i686-Release/pkg/bin \
+		-S maingame/gamemd-spawn-n.exe \
+		run-docker-instance \
+		-e pyra2yr/test_sell_mcv.py &
+	sleep 5
 	tmux_send_keys C-c
 	tmux_send_keys "CMAKE_TOOLCHAIN_FILE=$CMAKE_TOOLCHAIN_FILE CMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE DEBUG_FN=debug_integration_test ./scripts/debug.sh" C-m
 	wait