[improve][misc] DO NOT MERGE

.github/workflows/ci-maven-cache-update.yaml

@@ -63,7 +63,7 @@ jobs:
             mvn_arguments: ''
 
           - name: all modules - macos
-            runs-on: macos-11
+            runs-on: macos-latest
             cache_name: 'm2-dependencies-all'
 
           - name: core-modules

.github/workflows/pulsar-ci.yaml

@@ -1209,7 +1209,7 @@ jobs:
 
   macos-build:
     name: Build Pulsar on MacOS
-    runs-on: macos-11
+    runs-on: macos-latest
     timeout-minutes: 120
     needs: ['preconditions', 'integration-tests']
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}

.gitignore

@@ -97,3 +97,5 @@ test-reports/
 
 # Gradle Enterprise
 .mvn/.gradle-enterprise/
+# Gradle Develocity
+.mvn/.develocity/

.mvn/gradle-enterprise.xml → .mvn/develocity.xml

@@ -19,22 +19,21 @@
     under the License.
 
 -->
-<gradleEnterprise
-        xmlns="https://www.gradle.com/gradle-enterprise-maven" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="https://www.gradle.com/gradle-enterprise-maven https://www.gradle.com/schema/gradle-enterprise-maven.xsd">
+<develocity xmlns="https://www.gradle.com/develocity-maven" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+            xsi:schemaLocation="https://www.gradle.com/develocity-maven https://www.gradle.com/schema/develocity-maven.xsd">
+<!-- Enable Gradle Develocity extension when GRADLE_ENTERPRISE_ACCESS_KEY/DEVELOCITY_ACCESS_KEY is set and the build isn't
+       a pull request from a branch or forked repository with a name that indicates it's a work in progress. -->
+  <enabled>#{(env['GRADLE_ENTERPRISE_ACCESS_KEY']?.trim() > '' or env['DEVELOCITY_ACCESS_KEY']?.trim() > '') and !(env['GITHUB_HEAD_REF']?.matches('(?i).*(experiment|wip|private).*') or env['GITHUB_REPOSITORY']?.matches('(?i).*(experiment|wip|private).*'))}</enabled>
   <server>
     <url>https://ge.apache.org</url>
     <allowUntrusted>false</allowUntrusted>
   </server>
   <buildScan>
     <capture>
-      <goalInputFiles>true</goalInputFiles>
       <buildLogging>true</buildLogging>
       <testLogging>true</testLogging>
     </capture>
     <backgroundBuildScanUpload>#{isFalse(env['GITHUB_ACTIONS'])}</backgroundBuildScanUpload>
-    <publish>ALWAYS</publish>
-    <publishIfAuthenticated>true</publishIfAuthenticated>
     <obfuscation>
       <ipAddresses>#{{'0.0.0.0'}}</ipAddresses>
     </obfuscation>
@@ -47,4 +46,4 @@
       <enabled>false</enabled>
     </remote>
   </buildCache>
-</gradleEnterprise>
+</develocity>

.mvn/extensions.xml

@@ -23,12 +23,12 @@
             xsi:schemaLocation="http://maven.apache.org/EXTENSIONS/1.0.0 http://maven.apache.org/xsd/core-extensions-1.0.0.xsd">
   <extension>
     <groupId>com.gradle</groupId>
-    <artifactId>gradle-enterprise-maven-extension</artifactId>
-    <version>1.17.1</version>
+    <artifactId>develocity-maven-extension</artifactId>
+    <version>1.21.4</version>
   </extension>
   <extension>
     <groupId>com.gradle</groupId>
     <artifactId>common-custom-user-data-maven-extension</artifactId>
-    <version>1.11.1</version>
+    <version>2.0</version>
   </extension>
 </extensions>

README.md

@@ -141,8 +141,6 @@ components in the Pulsar ecosystem, including connectors, adapters, and other la
 >
 > This project includes a [Maven Wrapper](https://maven.apache.org/wrapper/) that can be used instead of a system-installed Maven.
 > Use it by replacing `mvn` by `./mvnw` on Linux and `mvnw.cmd` on Windows in the commands below.    
->
-> It's better to use CMD rather than Powershell on Windows. Because maven will activate the `windows` profile which runs `rename-netty-native-libs.cmd`.
 
 ### Build
 

bin/pulsar-daemon

@@ -157,7 +157,7 @@ start ()
     echo starting $command, logging to $logfile
     echo Note: Set immediateFlush to true in conf/log4j2.yaml will guarantee the logging event is flushing to disk immediately. The default behavior is switched off due to performance considerations.
     pulsar=$PULSAR_HOME/bin/pulsar
-    nohup $pulsar $command "$1" > "$out" 2>&1 < /dev/null &
+    nohup $pulsar $command "$@" > "$out" 2>&1 < /dev/null &
     echo $! > $pid
     sleep 1; head $out
     sleep 2;
@@ -216,29 +216,28 @@ stop ()
 
 case $startStop in
   (start)
-    start "$*"
+    start "$@"
     ;;
 
   (stop)
     stop $1
     ;;
 
   (restart)
-    forceStopFlag=$(echo "$*"|grep "\-force")
-    if [[ "$forceStopFlag" != "" ]]
+    if [[ "$1" == "-force" ]]
     then
-      stop "-force"
+      stop -force
+      # remove "-force" from the arguments
+      shift
     else
       stop
     fi
     if [ "$?" == 0 ]
     then
-       sleep 3
-       paramaters="$*"
-       startParamaters=${paramaters//-force/}
-       start "$startParamaters"
+      sleep 3
+      start "$@"
     else
-       echo "WARNNING :  $command failed restart, for $command is not stopped completely."
+      echo "WARNNING :  $command failed restart, for $command is not stopped completely."
     fi
     ;;
 

distribution/server/src/assemble/LICENSE.bin.txt

@@ -306,6 +306,7 @@ The Apache Software License, Version 2.0
     - io.netty-netty-resolver-dns-native-macos-4.1.108.Final-osx-x86_64.jar
     - io.netty-netty-transport-4.1.108.Final.jar
     - io.netty-netty-transport-classes-epoll-4.1.108.Final.jar
+    - io.netty-netty-transport-native-epoll-4.1.108.Final-linux-aarch_64.jar
     - io.netty-netty-transport-native-epoll-4.1.108.Final-linux-x86_64.jar
     - io.netty-netty-transport-native-unix-common-4.1.108.Final.jar
     - io.netty-netty-transport-native-unix-common-4.1.108.Final-linux-x86_64.jar
@@ -345,34 +346,34 @@ The Apache Software License, Version 2.0
     - net.java.dev.jna-jna-jpms-5.12.1.jar
     - net.java.dev.jna-jna-platform-jpms-5.12.1.jar
  * BookKeeper
-    - org.apache.bookkeeper-bookkeeper-common-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-proto-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-server-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.5.jar
-    - org.apache.bookkeeper-circe-checksum-4.16.5.jar
-    - org.apache.bookkeeper-cpu-affinity-4.16.5.jar
-    - org.apache.bookkeeper-statelib-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-api-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-common-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-java-client-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-java-client-base-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-proto-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-server-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-service-api-4.16.5.jar
-    - org.apache.bookkeeper-stream-storage-service-impl-4.16.5.jar
-    - org.apache.bookkeeper.http-http-server-4.16.5.jar
-    - org.apache.bookkeeper.http-vertx-http-server-4.16.5.jar
-    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.5.jar
-    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.5.jar
-    - org.apache.distributedlog-distributedlog-common-4.16.5.jar
-    - org.apache.distributedlog-distributedlog-core-4.16.5-tests.jar
-    - org.apache.distributedlog-distributedlog-core-4.16.5.jar
-    - org.apache.distributedlog-distributedlog-protocol-4.16.5.jar
-    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.5.jar
-    - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.5.jar
-    - org.apache.bookkeeper-native-io-4.16.5.jar
+    - org.apache.bookkeeper-bookkeeper-common-4.16.6.jar
+    - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.6.jar
+    - org.apache.bookkeeper-bookkeeper-proto-4.16.6.jar
+    - org.apache.bookkeeper-bookkeeper-server-4.16.6.jar
+    - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.6.jar
+    - org.apache.bookkeeper-circe-checksum-4.16.6.jar
+    - org.apache.bookkeeper-cpu-affinity-4.16.6.jar
+    - org.apache.bookkeeper-statelib-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-api-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-common-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-java-client-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-java-client-base-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-proto-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-server-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-service-api-4.16.6.jar
+    - org.apache.bookkeeper-stream-storage-service-impl-4.16.6.jar
+    - org.apache.bookkeeper.http-http-server-4.16.6.jar
+    - org.apache.bookkeeper.http-vertx-http-server-4.16.6.jar
+    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.6.jar
+    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.6.jar
+    - org.apache.distributedlog-distributedlog-common-4.16.6.jar
+    - org.apache.distributedlog-distributedlog-core-4.16.6-tests.jar
+    - org.apache.distributedlog-distributedlog-core-4.16.6.jar
+    - org.apache.distributedlog-distributedlog-protocol-4.16.6.jar
+    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.6.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.6.jar
+    - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.6.jar
+    - org.apache.bookkeeper-native-io-4.16.6.jar
   * Apache HTTP Client
     - org.apache.httpcomponents-httpclient-4.5.13.jar
     - org.apache.httpcomponents-httpcore-4.4.15.jar
@@ -500,11 +501,12 @@ The Apache Software License, Version 2.0
     - io.etcd-jetcd-core-0.7.7.jar
     - io.etcd-jetcd-grpc-0.7.7.jar
   * IPAddress
-    - com.github.seancfoley-ipaddress-5.3.3.jar
+    - com.github.seancfoley-ipaddress-5.5.0.jar
   * RxJava
     - io.reactivex.rxjava3-rxjava-3.0.1.jar
   * RoaringBitmap
     - org.roaringbitmap-RoaringBitmap-0.9.44.jar
+    - org.roaringbitmap-shims-0.9.44.jar
 
 BSD 3-clause "New" or "Revised" License
  * Google auth library
@@ -537,7 +539,6 @@ Protocol Buffers License
 
 CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt
  * Java Annotations API
-    - javax.annotation-javax.annotation-api-1.3.2.jar
     - com.sun.activation-javax.activation-1.2.0.jar
     - javax.xml.bind-jaxb-api-2.3.1.jar
  * Java Servlet API -- javax.servlet-javax.servlet-api-3.1.0.jar

distribution/server/src/assemble/bin.xml

@@ -126,21 +126,23 @@
     <dependencySet>
       <outputDirectory>lib</outputDirectory>
       <unpack>false</unpack>
-      <scope>compile</scope>
+      <scope>runtime</scope>
       <useProjectArtifact>false</useProjectArtifact>
       <!-- Include 'groupId' in the dependencies Jar names to better identify
            the provenance of the jar -->
       <outputFileNameMapping>${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension}</outputFileNameMapping>
 
       <excludes>
         <exclude>com.datastax.oss:pulsar-functions-runtime-all</exclude>
-
-        <exclude>org.projectlombok:lombok</exclude>
-
         <!-- prevent adding pulsar-functions-api-examples in lib -->
         <exclude>com.datastax.oss:pulsar-functions-api-examples</exclude>
         <!-- prevent adding any distribution .tar.gz files in lib -->
         <exclude>*:tar.gz</exclude>
+        <!-- prevent adding annotation libraries -->
+        <exclude>org.codehaus.mojo:animal-sniffer-annotations</exclude>
+        <exclude>com.google.android:annotations</exclude>
+        <!-- Needed only in the pulsar-shell distro only -->
+        <exclude>net.java.dev.jna:jna</exclude>
       </excludes>
     </dependencySet>
   </dependencySets>

distribution/shell/src/assemble/LICENSE.bin.txt

@@ -357,6 +357,7 @@ The Apache Software License, Version 2.0
     - netty-resolver-dns-4.1.108.Final.jar
     - netty-transport-4.1.108.Final.jar
     - netty-transport-classes-epoll-4.1.108.Final.jar
+    - netty-transport-native-epoll-4.1.108.Final-linux-aarch_64.jar
     - netty-transport-native-epoll-4.1.108.Final-linux-x86_64.jar
     - netty-transport-native-unix-common-4.1.108.Final.jar
     - netty-transport-native-unix-common-4.1.108.Final-linux-x86_64.jar
@@ -386,9 +387,9 @@ The Apache Software License, Version 2.0
     - log4j-web-2.18.0.jar
 
  * BookKeeper
-    - bookkeeper-common-allocator-4.16.5.jar
-    - cpu-affinity-4.16.5.jar
-    - circe-checksum-4.16.5.jar
+    - bookkeeper-common-allocator-4.16.6.jar
+    - cpu-affinity-4.16.6.jar
+    - circe-checksum-4.16.6.jar
   * AirCompressor
      - aircompressor-0.27.jar
  * AsyncHttpClient

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java

@@ -1112,6 +1112,13 @@ public long getEstimatedSizeSinceMarkDeletePosition() {
         return ledger.estimateBacklogFromPosition(markDeletePosition);
     }
 
+    private long getNumberOfEntriesInBacklog() {
+        if (markDeletePosition.compareTo(ledger.getLastPosition()) >= 0) {
+            return 0;
+        }
+        return getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition()));
+    }
+
     @Override
     public long getNumberOfEntriesInBacklog(boolean isPrecise) {
         if (log.isDebugEnabled()) {
@@ -1120,16 +1127,13 @@ public long getNumberOfEntriesInBacklog(boolean isPrecise) {
                     messagesConsumedCounter, markDeletePosition, readPosition);
         }
         if (isPrecise) {
-            if (markDeletePosition.compareTo(ledger.getLastPosition()) >= 0) {
-                return 0;
-            }
-            return getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition()));
+            return getNumberOfEntriesInBacklog();
         }
 
         long backlog = ManagedLedgerImpl.ENTRIES_ADDED_COUNTER_UPDATER.get(ledger) - messagesConsumedCounter;
         if (backlog < 0) {
             // In some case the counters get incorrect values, fall back to the precise backlog count
-            backlog = getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition()));
+            backlog = getNumberOfEntriesInBacklog();
         }
 
         return backlog;