Protobuf 22.5 cpp

COMPILING.md

@@ -44,11 +44,11 @@ This section is only necessary if you are making changes to the code
 generation. Most users only need to use `skipCodegen=true` as discussed above.
 
 ### Build Protobuf
-The codegen plugin is C++ code and requires protobuf 21.7 or later.
+The codegen plugin is C++ code and requires protobuf 22.5 or later.
 
 For Linux, Mac and MinGW:
 ```
-$ PROTOBUF_VERSION=21.7
+$ PROTOBUF_VERSION=22.5
 $ curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-all-$PROTOBUF_VERSION.tar.gz
 $ tar xzf protobuf-all-$PROTOBUF_VERSION.tar.gz
 $ cd protobuf-$PROTOBUF_VERSION

buildscripts/grpc-java-artifacts/Dockerfile

@@ -27,7 +27,10 @@ RUN mkdir -p "$ANDROID_HOME/cmdline-tools" && \
     mv "$ANDROID_HOME/cmdline-tools/cmdline-tools" "$ANDROID_HOME/cmdline-tools/latest" && \
     yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --licenses
 
+RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
+    tar xz -C /var/local    
+
 # Install Maven
 RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \
     tar xz -C /var/local
-ENV PATH /var/local/apache-maven-3.8.8/bin:$PATH
+ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH

buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base

@@ -10,5 +10,15 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         g++-aarch64-linux-gnu \
         g++-powerpc64le-linux-gnu \
         openjdk-8-jdk \
+        pkg-config \
         && \
     rm -rf /var/lib/apt/lists/*
+
+RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
+    tar xz -C /var/local
+
+# Install Maven
+RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \
+    tar xz -C /var/local
+ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH
+

buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base

@@ -9,5 +9,14 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         curl \
         g++-s390x-linux-gnu \
         openjdk-8-jdk \
+        pkg-config \
         && \
     rm -rf /var/lib/apt/lists/*
+
+RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
+    tar xz -C /var/local    
+
+# Install Maven
+RUN curl -Ls https://dlcdn.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz | \
+    tar xz -C /var/local
+ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.8.8/bin:$PATH    

buildscripts/kokoro/android-interop.sh

@@ -7,8 +7,8 @@ set -exu -o pipefail
 
 cd github/grpc-java
 
-export LDFLAGS=-L/tmp/protobuf/lib
-export CXXFLAGS=-I/tmp/protobuf/include
+export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)"
+export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)"
 export LD_LIBRARY_PATH=/tmp/protobuf/lib
 export OS_NAME=$(uname)
 

buildscripts/kokoro/android.sh

@@ -9,9 +9,6 @@ BASE_DIR="$(pwd)"
 
 cd "$BASE_DIR/github/grpc-java"
 
-export LDFLAGS=-L/tmp/protobuf/lib
-export CXXFLAGS=-I/tmp/protobuf/include
-export LD_LIBRARY_PATH=/tmp/protobuf/lib
 export OS_NAME=$(uname)
 
 cat <<EOF >> gradle.properties
@@ -30,10 +27,18 @@ unzip -qd "${ANDROID_HOME}/cmdline-tools" cmdline.zip
 rm cmdline.zip
 mv "${ANDROID_HOME}/cmdline-tools/cmdline-tools" "${ANDROID_HOME}/cmdline-tools/latest"
 (yes || true) | "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" --licenses
-
+curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
+    tar xz -C /tmp
+export PATH=/tmp/cmake-3.26.3-linux-x86_64/bin:$PATH
+
 # Proto deps
 buildscripts/make_dependencies.sh
 
+sudo apt-get update && sudo apt-get install pkg-config
+export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)"
+export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)"
+export LD_LIBRARY_PATH=/tmp/protobuf/lib
+
 # Build Android with Java 11, this adds it to the PATH
 sudo update-java-alternatives --set java-1.11.0-openjdk-amd64
 # Unset any existing JAVA_HOME env var to stop Gradle from using it
@@ -98,6 +103,7 @@ cd $BASE_DIR/github/grpc-java
 ./gradlew clean
 git checkout HEAD^
 ./gradlew --stop  # use a new daemon to build the previous commit
+GRADLE_FLAGS="${GRADLE_FLAGS} -PskipCodegen=true"
 ./gradlew publishToMavenLocal $GRADLE_FLAGS
 cd examples/android/helloworld/
 ../../gradlew build $GRADLE_FLAGS

buildscripts/kokoro/unix.sh

@@ -51,9 +51,9 @@ fi
 export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'"
 
 # Make protobuf discoverable by :grpc-compiler
-export LD_LIBRARY_PATH=/tmp/protobuf/lib
-export LDFLAGS=-L/tmp/protobuf/lib
-export CXXFLAGS="-I/tmp/protobuf/include"
+export LDFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --libs protobuf)"
+export CXXFLAGS="$(PKG_CONFIG_PATH=/tmp/protobuf/lib/pkgconfig pkg-config --cflags protobuf)"
+export LIBRARY_PATH=/tmp/protobuf/lib
 
 ./gradlew grpc-compiler:clean $GRADLE_FLAGS
 

buildscripts/kokoro/windows32.bat

@@ -15,20 +15,22 @@ set ESCWORKSPACE=%WORKSPACE:\=\\%
 
 @rem Clear JAVA_HOME to prevent a different Java version from being used
 set JAVA_HOME=
-set PATH=C:\Program Files\OpenJDK\openjdk-11.0.12_7\bin;%PATH%
 
 mkdir grpc-java-helper32
 cd grpc-java-helper32
-call "%VS140COMNTOOLS%\vsvars32.bat" || exit /b 1
+call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars32.bat" || exit /b 1
 call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1
 
 cd "%WORKSPACE%"
 
 SET TARGET_ARCH=x86_32
 SET FAIL_ON_WARNINGS=true
-SET VC_PROTOBUF_LIBS=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\Release
-SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\include
-SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true
+SET PROTOBUF_VER=22.5
+SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib\\pkgconfig
+SET VC_PROTOBUF_LIB_PATHS=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib
+SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper32\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include
+call :Get_Libs
+SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibPaths=%VC_PROTOBUF_LIB_PATHS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true
 SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'"
 
 cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% build"
@@ -50,3 +52,32 @@ IF NOT %GRADLEEXIT% == 0 (
 cmd.exe /C "%WORKSPACE%\gradlew.bat --stop"
 
 cmd.exe /C "%WORKSPACE%\gradlew.bat  %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts clean grpc-compiler:build grpc-compiler:publish" || exit /b 1
+
+goto :eof
+:Get_Libs
+SetLocal EnableDelayedExpansion
+set "libs_list="
+for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do (
+  for %%b in (%%a) do (
+    set lib=%%b
+    set libfirst2char=!lib:~0,2!
+    if !libfirst2char!==-l (
+      @rem remove the leading -l
+      set lib=!lib:~2!
+      @rem remove spaces
+      set lib=!lib: =!
+      @rem Because protobuf is specified as libprotobuf and elsewhere
+      if !lib! NEQ protobuf (
+        set lib=!lib!.lib
+        if "!libs_list!"=="" (
+          set libs_list=!lib!
+        ) else (
+          set libs_list=!libs_list!,!lib!
+        )
+      )
+    )
+  )
+)
+EndLocal & set "VC_PROTOBUF_LIBS=%libs_list%" 
+exit /b 0
+

buildscripts/kokoro/windows64.bat

@@ -1,36 +1,67 @@
-@rem ##########################################################################
-@rem
-@rem Builds artifacts for x86_64 into %WORKSPACE%\artifacts\
-@rem
-@rem ##########################################################################
-
-type c:\VERSION
-
-@rem Enter repo root
-cd /d %~dp0\..\..
-
-set WORKSPACE=T:\src\github\grpc-java
-set ESCWORKSPACE=%WORKSPACE:\=\\%
-
-@rem Clear JAVA_HOME to prevent a different Java version from being used
-set JAVA_HOME=
-set PATH=C:\Program Files\OpenJDK\openjdk-11.0.12_7\bin;%PATH%
-
-mkdir grpc-java-helper64
-cd grpc-java-helper64
-call "%VS140COMNTOOLS%\..\..\VC\bin\amd64\vcvars64.bat" || exit /b 1
-call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1
-
-cd "%WORKSPACE%"
-
-SET TARGET_ARCH=x86_64
-SET FAIL_ON_WARNINGS=true
-SET VC_PROTOBUF_LIBS=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\Release
-SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\include
-SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true
-SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'"
-
-@rem make sure no daemons have any files open
-cmd.exe /C "%WORKSPACE%\gradlew.bat --stop"
-
-cmd.exe /C "%WORKSPACE%\gradlew.bat  %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts grpc-compiler:clean grpc-compiler:build grpc-compiler:publish" || exit /b 1
+@rem ##########################################################################
+@rem
+@rem Builds artifacts for x86_64 into %WORKSPACE%\artifacts\
+@rem
+@rem ##########################################################################
+
+type c:\VERSION
+
+@rem Enter repo root
+cd /d %~dp0\..\..
+
+set WORKSPACE=T:\src\github\grpc-java
+set ESCWORKSPACE=%WORKSPACE:\=\\%
+
+@rem Clear JAVA_HOME to prevent a different Java version from being used
+set JAVA_HOME=
+
+mkdir grpc-java-helper64
+cd grpc-java-helper64
+call "%VS170COMNTOOLS%\..\..\VC\Auxiliary\Build\vcvars64.bat" || exit /b 1
+call "%WORKSPACE%\buildscripts\make_dependencies.bat" || exit /b 1
+
+cd "%WORKSPACE%"
+
+SET TARGET_ARCH=x86_64
+SET FAIL_ON_WARNINGS=true
+SET PROTOBUF_VER=22.5
+SET PKG_CONFIG_PATH=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib\\pkgconfig
+SET VC_PROTOBUF_LIB_PATHS=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\lib
+SET VC_PROTOBUF_INCLUDE=%ESCWORKSPACE%\\grpc-java-helper64\\protobuf-%PROTOBUF_VER%\\build\\protobuf-%PROTOBUF_VER%\\include
+call :Get_Libs
+SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% -PvcProtobufLibPaths=%VC_PROTOBUF_LIB_PATHS% -PvcProtobufLibs=%VC_PROTOBUF_LIBS% -PvcProtobufInclude=%VC_PROTOBUF_INCLUDE% -PskipAndroid=true
+SET GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'"
+
+@rem make sure no daemons have any files open
+cmd.exe /C "%WORKSPACE%\gradlew.bat --stop"
+
+cmd.exe /C "%WORKSPACE%\gradlew.bat  %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts grpc-compiler:clean grpc-compiler:build grpc-compiler:publish" || exit /b 1
+
+goto :eof
+:Get_Libs
+SetLocal EnableDelayedExpansion
+set "libs_list="
+for /f "tokens=*" %%a in ('pkg-config --libs protobuf') do (
+  for %%b in (%%a) do (
+    set lib=%%b
+    set libfirst2char=!lib:~0,2!
+    if !libfirst2char!==-l (
+      @rem remove the leading -l
+      set lib=!lib:~2!
+      @rem remove spaces
+      set lib=!lib: =!
+      @rem Because protobuf is specified as libprotobuf and elsewhere
+      if !lib! NEQ protobuf (
+        set lib=!lib!.lib
+        if "!libs_list!"=="" (
+          set libs_list=!lib!
+        ) else (
+          set libs_list=!libs_list!,!lib!
+        )
+      )
+    )
+  )
+)
+EndLocal & set "VC_PROTOBUF_LIBS=%libs_list%" 
+exit /b 0
+

buildscripts/make_dependencies.bat

@@ -1,44 +1,61 @@
-set PROTOBUF_VER=21.7
-set CMAKE_NAME=cmake-3.3.2-win32-x86
+echo on
+@rem set PROTOBUF_VER=21.7
+choco install -y gradle git curl pkgconfiglite
+choco install -y openjdk --version=17.0
+set JAVA_HOME="c:\Program Files\OpenJDK\jdk-17"
+set PATH=%PATH%;"c:\Program Files\OpenJDK\jdk-17\bin"
+set PROTOBUF_VER=22.5
+set ABSL_VERSION=20230125.4
 
 if not exist "protobuf-%PROTOBUF_VER%\build\Release\" (
   call :installProto || exit /b 1
 )
 
 echo Compile gRPC-Java with something like:
-echo -PtargetArch=x86_32 -PvcProtobufLibs=%cd%\protobuf-%PROTOBUF_VER%\build\Release -PvcProtobufInclude=%cd%\protobuf-%PROTOBUF_VER%\build\include
+echo -PtargetArch=x86_32 -PvcProtobufLibPath=%cd%\protobuf-%PROTOBUF_VER%\build\protobuf-%PROTOBUF_VER%\lib -PvcProtobufInclude=%cd%\protobuf-%PROTOBUF_VER%\build\protobuf-%PROTOBUF_VER%\include -PvcProtobufLibs=insert-list-of-libs-from-pkg-config-output-here
 goto :eof
 
 
 :installProto
 
 where /q cmake
 if not ERRORLEVEL 1 goto :hasCmake
+set CMAKE_NAME=cmake-3.26.3-windows-x86_64
 if not exist "%CMAKE_NAME%" (
   call :installCmake || exit /b 1
 )
 set PATH=%PATH%;%cd%\%CMAKE_NAME%\bin
 :hasCmake
 @rem GitHub requires TLSv1.2, and for whatever reason our powershell doesn't have it enabled
-powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/archive/v%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1
+powershell -command "$ProgressPreference = 'SilentlyContinue'; $ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1
 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('protobuf.zip', '.') }" || exit /b 1
 del protobuf.zip
+powershell -command "$ProgressPreference = 'SilentlyContinue'; $ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/abseil/abseil-cpp/archive/refs/tags/%ABSL_VERSION%.zip -OutFile absl.zip }" || exit /b 1
+powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('absl.zip', '.') }" || exit /b 1
+del absl.zip
+rmdir protobuf-%PROTOBUF_VER%\third_party\abseil-cpp
+move abseil-cpp-%ABSL_VERSION% protobuf-%PROTOBUF_VER%\third_party\abseil-cpp
 mkdir protobuf-%PROTOBUF_VER%\build
 pushd protobuf-%PROTOBUF_VER%\build
 
-@rem Workaround https://github.com/protocolbuffers/protobuf/issues/10174
-powershell -command "(Get-Content ..\cmake\extract_includes.bat.in) -replace '\.\.\\', '' | Out-File -encoding ascii ..\cmake\extract_includes.bat.in"
 @rem cmake does not detect x86_64 from the vcvars64.bat variables.
-@rem If vcvars64.bat has set PLATFORM to X64, then inform cmake to use the Win64 version of VS
-if "%PLATFORM%" == "X64" (
-  @rem Note the space
-  SET CMAKE_VSARCH= Win64
+@rem If vcvars64.bat has set PLATFORM to X64, then inform cmake to use the Win64 version of VS, likewise for x32
+if "%PLATFORM%" == "x64" (
+  SET CMAKE_VSARCH=-A x64
+) else if "%PLATFORM%" == "x86" (
+  @rem -A x86 doesn't work: https://github.com/microsoft/vcpkg/issues/15465
+  SET CMAKE_VSARCH=-DCMAKE_GENERATOR_PLATFORM=WIN32
 ) else (
   SET CMAKE_VSARCH=
 )
-cmake -Dprotobuf_BUILD_TESTS=OFF -G "Visual Studio %VisualStudioVersion:~0,2%%CMAKE_VSARCH%" .. || exit /b 1
-msbuild /maxcpucount /p:Configuration=Release /verbosity:minimal libprotoc.vcxproj || exit /b 1
-call extract_includes.bat || exit /b 1
+for /f "tokens=4 delims=\" %%a in ("%VCINSTALLDIR%") do (
+  SET VC_YEAR=%%a
+)
+for /f "tokens=1 delims=." %%a in ("%VisualStudioVersion%") do (
+  SET visual_studio_major_version=%%a
+)
+cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=%cd%\protobuf-%PROTOBUF_VER% -DCMAKE_PREFIX_PATH=%cd%\protobuf-%PROTOBUF_VER% -G "Visual Studio %visual_studio_major_version% %VC_YEAR%" %CMAKE_VSARCH% ..
+cmake --build . --config Release --target install
 popd
 goto :eof
 
@@ -49,3 +66,4 @@ powershell -command "$ErrorActionPreference = 'stop'; & { iwr https://cmake.org/
 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('cmake.zip', '.') }" || exit /b 1
 del cmake.zip
 goto :eof
+