merge 4.5 into wip-4.6

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [cwensel]

.github/workflows/publish-release.yaml

@@ -0,0 +1,54 @@
+name: Publish Release
+
+on:
+  push:
+    branches:
+      - 'wip-[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+'
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set Build Number Environment Variable
+        env:
+          BUILD_NUMBER_OFFSET: 26
+        run: |
+          echo "BUILD_NUMBER=$((${{ github.run_number }} + $BUILD_NUMBER_OFFSET))" >> "$GITHUB_ENV"
+
+      - name: Install gpg secret key
+        run: |
+          cat <(echo -e "${{ secrets.GPG_SECRET_KEY }}") | gpg2 --batch --import
+          gpg2 --list-secret-keys --keyid-format LONG
+
+      - uses: actions/checkout@v2
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: 8
+          distribution: 'zulu'
+
+      - name: Grant execute permission for gradlew
+        run: chmod +x gradlew
+
+      - name: Build and Test
+        run: ./gradlew clean check
+
+      - name: Publish WIP to Github Packages
+        if: ${{ contains( github.ref, 'wip-' ) }}
+        env:
+          GRADLE_OPTS: -Dbuild.number=${{ env.BUILD_NUMBER }} -Dbuild.vcs.number=${{ github.sha }} -Dbuild.vcs.branch=${{ github.ref }}
+          GPR_USERNAME: ${{ github.actor }}
+          GPR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: ./gradlew publishAllPublicationsToGitHubPackagesRepository -Psigning.gnupg.keyName=${{ secrets.GPG_SECRET_KEY_ID }} -Psigning.gnupg.passphrase=${{ secrets.GPG_SECRET_KEY_PASSWORD }}
+
+      - name: Publish Release to Maven Central
+        if: ${{ !contains( github.ref, 'wip-' ) }}
+        env:
+          GRADLE_OPTS: -Dcascading.release.final=true -Dbuild.number=${{ env.BUILD_NUMBER }} -Dbuild.vcs.number=${{ github.sha }} -Dbuild.vcs.branch=${{ github.ref }}
+          MCR_USERNAME: ${{ secrets.MCR_USERNAME }}
+          MCR_PASSWORD: ${{ secrets.MCR_PASSWORD }}
+        run: ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -Psigning.gnupg.keyName=${{ secrets.GPG_SECRET_KEY_ID }} -Psigning.gnupg.passphrase=${{ secrets.GPG_SECRET_KEY_PASSWORD }}

CHANGES.txt

@@ -19,11 +19,23 @@ Cascading Change Log
 
   Updated jackson to 2.15.2
 
-4.5.1 [unreleased]
+4.5.2
+
+  Fixed issue in c.t.Fields where the pos map was the source of a memory leak under certain conditions.
+  The IdentityHashMap was replaced with a WeakHashMap to prevent the leak.
+
+4.5.1
+
+  Updated Apache Tez to version 0.10.2.
+
+  Updated Apache Hadoop to version 3.3.6.
+
+  Updated c.p.AppProps#getApplicationJarClass to test if property is a j.l.Class instance in order to retain
+  backwards compatibility.
 
   Created c.CascadingTesting to provide common utilities to all tests without Junit dependencies.
 
-  Fix for '.tsv' extension on TextDelimited implementations being unused.
+  Fixed '.tsv' extension on TextDelimited implementations being unused.
 
 4.5.0
 

build.gradle

@@ -1,7 +1,5 @@
-import java.text.SimpleDateFormat
-
 /*
- * Copyright (c) 2007-2022 The Cascading Authors. All Rights Reserved.
+ * Copyright (c) 2007-2024 The Cascading Authors. All Rights Reserved.
  *
  * Project and contact information: https://cascading.wensel.net/
  *
@@ -20,6 +18,8 @@ import java.text.SimpleDateFormat
  * limitations under the License.
  */
 
+import java.text.SimpleDateFormat
+
 /*
  Using:
 
@@ -34,8 +34,8 @@ import java.text.SimpleDateFormat
 */
 plugins {
   id 'io.github.gradle-nexus.publish-plugin' version '1.0.0'
-  id 'me.champeau.jmh' version '0.6.5' apply false
-  id 'io.morethan.jmhreport' version '0.9.0' apply false
+  id 'me.champeau.jmh' version '0.7.2' apply false
+  id 'io.morethan.jmhreport' version '0.9.6' apply false
 }
 
 apply from: 'etc/version.gradle'
@@ -240,6 +240,7 @@ subprojects {
   }
 
   signing {
+    useGpgCmd()
     sign publishing.publications.maven
   }
 

cascading-core/build.gradle

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2007-2022 The Cascading Authors. All Rights Reserved.
+ * Copyright (c) 2007-2024 The Cascading Authors. All Rights Reserved.
  *
  * Project and contact information: https://cascading.wensel.net/
  *

cascading-core/src/jmh/java/cascading/fields/FieldsPosCacheBench.java

@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2007-2024 The Cascading Authors. All Rights Reserved.
+ *
+ * Project and contact information: https://cascading.wensel.net/
+ *
+ * This file is part of the Cascading project.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cascading.fields;
+
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+import cascading.tuple.Fields;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@State(Scope.Thread)
+@Warmup(iterations = 1, time = 500, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 5, time = 250, timeUnit = TimeUnit.MILLISECONDS)
+@Fork(1)
+@BenchmarkMode({Mode.AverageTime})
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+public class FieldsPosCacheBench
+  {
+  private static Comparable[] comparables = {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o"};
+  private static Fields fieldsUntyped = new Fields( comparables );
+
+  private static Fields[] fields;
+  private static Comparable<?>[][] elements;
+
+  static
+    {
+    fields = new Fields[ comparables.length ];
+    elements = new Comparable[ comparables.length ][];
+    for( int i = 0; i < comparables.length; i++ )
+      {
+      elements[ i ] = Arrays.copyOf( comparables, i + 1 );
+      fields[ i ] = new Fields( elements[ i ] );
+      }
+    }
+
+  @Param({"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14"})
+  private int index;
+
+  /**
+   * using IdentityHashMap
+   * # Benchmark: cascading.fields.FieldsPosCacheBench.containsIdentity
+   * # Parameters: (index = 14)
+   * Result "cascading.fields.FieldsPosCacheBench.containsIdentity":
+   *   3.744 ±(99.9%) 0.200 ns/op [Average]
+   *   (min, avg, max) = (3.705, 3.744, 3.824), stdev = 0.052
+   *   CI (99.9%): [3.545, 3.944] (assumes normal distribution)
+   *
+   * using HashMap
+   * Result "cascading.fields.FieldsPosCacheBench.containsIdentity":
+   *   3.758 ±(99.9%) 0.178 ns/op [Average]
+   *   (min, avg, max) = (3.719, 3.758, 3.826), stdev = 0.046
+   *   CI (99.9%): [3.579, 3.936] (assumes normal distribution)
+   *
+   * using WeakHashMap
+   * Result "cascading.fields.FieldsPosCacheBench.containsIdentity":
+   *   3.938 ±(99.9%) 0.202 ns/op [Average]
+   *   (min, avg, max) = (3.908, 3.938, 4.032), stdev = 0.053
+   *   CI (99.9%): [3.736, 4.141] (assumes normal distribution)
+   *
+   * @param bh
+   */
+  @Benchmark
+  public void containsIdentity( Blackhole bh )
+    {
+    bh.consume( fieldsUntyped.contains( fields[ index ] ) );
+    }
+
+  /**
+   * using IdentityHashMap
+   * # Benchmark: cascading.fields.FieldsPosCacheBench.containsEquals
+   * # Parameters: (index = 14)
+   * Result "cascading.fields.FieldsPosCacheBench.containsEquals":
+   *   549.168 ±(99.9%) 288.971 ns/op [Average]
+   *   (min, avg, max) = (493.761, 549.168, 673.781), stdev = 75.045
+   *   CI (99.9%): [260.197, 838.139] (assumes normal distribution)
+   *
+   * using HashMap
+   * Result "cascading.fields.FieldsPosCacheBench.containsEquals":
+   *   143.050 ±(99.9%) 13.049 ns/op [Average]
+   *   (min, avg, max) = (141.058, 143.050, 149.062), stdev = 3.389
+   *   CI (99.9%): [130.001, 156.099] (assumes normal distribution)
+   *
+   * using WeakHashMap
+   * Result "cascading.fields.FieldsPosCacheBench.containsEquals":
+   *   146.248 ±(99.9%) 1.322 ns/op [Average]
+   *   (min, avg, max) = (145.972, 146.248, 146.766), stdev = 0.343
+   *   CI (99.9%): [144.926, 147.571] (assumes normal distribution)
+   *
+   * @param bh
+   */
+  @Benchmark
+  public void containsEquals( Blackhole bh )
+    {
+    Fields value = new Fields( elements[ index ] );
+    bh.consume( fieldsUntyped.contains( value ) );
+    }
+  }

cascading-core/src/main/java/cascading/property/AppProps.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2007-2022 The Cascading Authors. All Rights Reserved.
+ * Copyright (c) 2007-2023 The Cascading Authors. All Rights Reserved.
  *
  * Project and contact information: https://cascading.wensel.net/
  *
@@ -100,19 +100,35 @@ public static void setApplicationJarClass( Map<Object, Object> properties, Class
    * @param properties of type Map
    * @return Class
    */
-  public static Class getApplicationJarClass( Map<Object, Object> properties )
+  public static Class<?> getApplicationJarClass( Map<Object, Object> properties )
     {
-    String className = PropertyUtil.getProperty( properties, DEP_APP_JAR_CLASS, (String) null );
+    // updated to retain compatibility with older versions
+    Object type = PropertyUtil.getProperty( properties, DEP_APP_JAR_CLASS, null );
+
+    if( type instanceof Class )
+      {
+      LOG.warn( "using deprecated property: {}, use instead: {}", DEP_APP_JAR_CLASS, APP_JAR_CLASS );
+      return (Class<?>) type;
+      }
+
+    String className = (String) type;
 
     if( className != null )
       {
       LOG.warn( "using deprecated property: {}, use instead: {}", DEP_APP_JAR_CLASS, APP_JAR_CLASS );
       return Util.loadClassSafe( className );
       }
 
-    className = PropertyUtil.getProperty( properties, APP_JAR_CLASS, (String) null );
+    type = PropertyUtil.getProperty( properties, APP_JAR_CLASS, null );
+
+    if( type instanceof Class )
+      return (Class<?>) type;
+
+    className = (String) type;
+
     if( className == null )
       return null;
+
     return Util.loadClassSafe( className );
     }
 
@@ -180,7 +196,9 @@ private static String getAppID()
     return appID;
     }
 
-  /** Sets the static appID value to null. For debugging purposes. */
+  /**
+   * Sets the static appID value to null. For debugging purposes.
+   */
   public static void resetAppID()
     {
     appID = null;

cascading-core/src/main/java/cascading/tuple/Fields.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2007-2022 The Cascading Authors. All Rights Reserved.
+ * Copyright (c) 2007-2024 The Cascading Authors. All Rights Reserved.
  *
  * Project and contact information: https://cascading.wensel.net/
  *
@@ -29,13 +29,13 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.WeakHashMap;
 import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.stream.IntStream;
@@ -956,7 +956,7 @@ private int[] makeThisPos()
   private Map<Fields, int[]> getPosCache()
     {
     if( posCache == null )
-      posCache = new IdentityHashMap<>();
+      posCache = new WeakHashMap<>();
 
     return posCache;
     }

cascading-hadoop3-common/build.gradle

@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-ext.hadoop3Version = '3.3.4'
+ext.hadoop3Version = '3.3.6'
 
 if( System.properties[ 'hadoop3.release.final' ] )
   hadoop3Version = System.properties[ 'hadoop3.release.final' ]

cascading-hadoop3-io/build.gradle

@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-ext.hadoop3Version = '3.3.4'
+ext.hadoop3Version = '3.3.6'
 
 if ( System.properties [ 'hadoop3.release.final'] )
   hadoop3Version = System.properties[ 'hadoop3.release.final' ]

cascading-hadoop3-mr1/build.gradle

@@ -20,7 +20,7 @@
 
 apply from: '../etc/testingPlatform.gradle'
 
-ext.hadoop3Version = '3.3.4'
+ext.hadoop3Version = '3.3.6'
 
 if ( System.properties [ 'hadoop3.release.final'] )
   hadoop3Version = System.properties[ 'hadoop3.release.final' ]

cascading-hadoop3-parquet-thrift/build.gradle

@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-ext.hadoop3Version = '3.3.4'
+ext.hadoop3Version = '3.3.6'
 
 if ( System.properties [ 'hadoop3.release.final'] )
   hadoop3Version = System.properties[ 'hadoop3.release.final' ]

cascading-hadoop3-parquet/build.gradle

@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-ext.hadoop3Version = '3.3.4'
+ext.hadoop3Version = '3.3.6'
 
 if ( System.properties [ 'hadoop3.release.final'] )
   hadoop3Version = System.properties[ 'hadoop3.release.final' ]

cascading-hadoop3-tez-stats/build.gradle

@@ -18,7 +18,7 @@
  * limitations under the License.
  */
 
-ext.hadoop3Version = '3.3.4'
+ext.hadoop3Version = '3.3.6'
 ext.tezVersion = '0.10.2'
 
 if ( System.properties [ 'hadoop3.release.final'] )