From 5049de6b60b3e3aeb12fe55e8cbc35605497acc3 Mon Sep 17 00:00:00 2001 From: liurenjie1024 Date: Fri, 25 Oct 2024 11:57:19 +0800 Subject: [PATCH 1/4] Add some utility methods Signed-off-by: liurenjie1024 --- java/src/main/java/ai/rapids/cudf/Arms.java | 78 +++++++++++++++++++ .../java/ai/rapids/cudf/HostMemoryBuffer.java | 2 +- java/src/main/java/ai/rapids/cudf/Pair.java | 44 +++++++++++ .../java/ai/rapids/cudf/Preconditions.java | 43 ++++++++++ .../main/java/ai/rapids/cudf/SlicedTable.java | 65 ++++++++++++++++ 5 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 java/src/main/java/ai/rapids/cudf/Arms.java create mode 100644 java/src/main/java/ai/rapids/cudf/Pair.java create mode 100644 java/src/main/java/ai/rapids/cudf/Preconditions.java create mode 100644 java/src/main/java/ai/rapids/cudf/SlicedTable.java diff --git a/java/src/main/java/ai/rapids/cudf/Arms.java b/java/src/main/java/ai/rapids/cudf/Arms.java new file mode 100644 index 00000000000..f15a6755088 --- /dev/null +++ b/java/src/main/java/ai/rapids/cudf/Arms.java @@ -0,0 +1,78 @@ +/* + * + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ai.rapids.cudf; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.function.Function; + +/** + * This class contains utility methods for automatic resource management. + */ +public class Arms { + /** + * This method close the resource if an exception is thrown while executing the function. + */ + public static T closeIfException(R resource, Function function) { + try { + return function.apply(resource); + } catch (Exception e) { + if (resource != null) { + try { + resource.close(); + } catch (Exception inner) { + e.addSuppressed(inner); + } + } + throw e; + } + } + + /** + * This method safes closes the resources. + */ + public static void close(Iterator resources) { + Throwable t = null; + while (resources.hasNext()) { + try { + resources.next().close(); + } catch (Exception e) { + if (t == null) { + t = e; + } else { + t.addSuppressed(e); + } + } + } + } + + /** + * This method safes closes the resources. + */ + public static void close(R... resources) { + close(Arrays.asList(resources)); + } + + /** + * This method safes closes the resources. + */ + public static void close(Iterable resources) { + close(resources.iterator()); + } +} diff --git a/java/src/main/java/ai/rapids/cudf/HostMemoryBuffer.java b/java/src/main/java/ai/rapids/cudf/HostMemoryBuffer.java index e4106574a19..3c52388504b 100644 --- a/java/src/main/java/ai/rapids/cudf/HostMemoryBuffer.java +++ b/java/src/main/java/ai/rapids/cudf/HostMemoryBuffer.java @@ -246,7 +246,7 @@ public final void copyFromHostBuffer(long destOffset, HostMemoryBuffer srcData, * @param in input stream to copy bytes from * @param byteLength number of bytes to copy */ - final void copyFromStream(long destOffset, InputStream in, long byteLength) throws IOException { + public final void copyFromStream(long destOffset, InputStream in, long byteLength) throws IOException { addressOutOfBoundsCheck(address + destOffset, byteLength, "copy from stream"); byte[] arrayBuffer = new byte[(int) Math.min(1024 * 128, byteLength)]; long left = byteLength; diff --git a/java/src/main/java/ai/rapids/cudf/Pair.java b/java/src/main/java/ai/rapids/cudf/Pair.java new file mode 100644 index 00000000000..76bfbbefdcb --- /dev/null +++ b/java/src/main/java/ai/rapids/cudf/Pair.java @@ -0,0 +1,44 @@ +/* + * + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ai.rapids.cudf; + +/** + * A utility class for holding a pair of values. + */ +public class Pair { + private final K left; + private final V right; + + public Pair(K left, V right) { + this.left = left; + this.right = right; + } + + public K getLeft() { + return left; + } + + public V getRight() { + return right; + } + + public static Pair of(K left, V right) { + return new Pair<>(left, right); + } +} diff --git a/java/src/main/java/ai/rapids/cudf/Preconditions.java b/java/src/main/java/ai/rapids/cudf/Preconditions.java new file mode 100644 index 00000000000..f276d058be6 --- /dev/null +++ b/java/src/main/java/ai/rapids/cudf/Preconditions.java @@ -0,0 +1,43 @@ +/* + * + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package ai.rapids.cudf; + +import java.util.function.Supplier; + +/** + * This class contains utility methods for checking preconditions. + */ +public class Preconditions { + /** + * Check if the condition is true, otherwise throw an IllegalStateException with the given message. + */ + public static void ensure(boolean condition, String message) { + if (!condition) { + throw new IllegalStateException(message); + } + } + + /** + * Check if the condition is true, otherwise throw an IllegalStateException with the given message supplier. + */ + public static void ensure(boolean condition, Supplier messageSupplier) { + if (!condition) { + throw new IllegalStateException(messageSupplier.get()); + } + } +} diff --git a/java/src/main/java/ai/rapids/cudf/SlicedTable.java b/java/src/main/java/ai/rapids/cudf/SlicedTable.java new file mode 100644 index 00000000000..a2fc8477c3e --- /dev/null +++ b/java/src/main/java/ai/rapids/cudf/SlicedTable.java @@ -0,0 +1,65 @@ +/* + * + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ai.rapids.cudf; + +import java.util.Objects; + +import static ai.rapids.cudf.Preconditions.ensure; + +/** + * A sliced view to underlying table. + * + * This is a simple wrapper around a table that represents a slice of the table, and it doesn't change ownership of the + * underlying table, so it's always the caller's responsibility to manage the lifecycle of the underlying table. + */ +public class SlicedTable { + private final int startRow; + private final int numRows; + private final Table table; + + public SlicedTable(int startRow, int numRows, Table table) { + Objects.requireNonNull(table, "table must not be null"); + ensure(startRow >= 0, "startRow must be >= 0"); + ensure(startRow < table.getRowCount(), + () -> "startRow " + startRow + " is larger than table row count " + table.getRowCount()); + ensure(numRows >= 0, () -> "numRows " + numRows + " is negative"); + ensure(startRow + numRows <= table.getRowCount(), () -> "startRow + numRows is " + (startRow + numRows) + + ", must be less than table row count " + table.getRowCount()); + + this.startRow = startRow; + this.numRows = numRows; + this.table = table; + } + + public int getStartRow() { + return startRow; + } + + public int getNumRows() { + return numRows; + } + + public Table getBaseTable() { + return table; + } + + public static SlicedTable from(Table table, int startRow, int numRows) { + return new SlicedTable(startRow, numRows, table); + } +} From fecbdc17390efcc4e207d6e0d7a2e80d57ee17d8 Mon Sep 17 00:00:00 2001 From: liurenjie1024 Date: Mon, 28 Oct 2024 10:57:58 +0800 Subject: [PATCH 2/4] Fix comments --- java/src/main/java/ai/rapids/cudf/Arms.java | 24 ++++++++++++------- .../java/ai/rapids/cudf/HostMemoryBuffer.java | 6 +++-- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/Arms.java b/java/src/main/java/ai/rapids/cudf/Arms.java index f15a6755088..f9284439199 100644 --- a/java/src/main/java/ai/rapids/cudf/Arms.java +++ b/java/src/main/java/ai/rapids/cudf/Arms.java @@ -19,6 +19,7 @@ package ai.rapids.cudf; import java.util.Arrays; +import java.util.Collection; import java.util.Iterator; import java.util.function.Function; @@ -45,9 +46,13 @@ public static T closeIfException(R resource, Functi } /** - * This method safes closes the resources. + * This method safely closes all the resources. + *

+ * This method will iterate through all the resources and closes them. If any exception happened during the + * traversal, exception will be captured and rethrown after all resources closed. + *

*/ - public static void close(Iterator resources) { + public static void closeAll(Iterator resources) { Throwable t = null; while (resources.hasNext()) { try { @@ -60,19 +65,22 @@ public static void close(Iterator resources) { } } } + + if (t != null) throw new RuntimeException(t); } + /** - * This method safes closes the resources. + * This method safely closes all the resources. See {@link #closeAll(Iterator)} for more details. */ - public static void close(R... resources) { - close(Arrays.asList(resources)); + public static void closeAll(R... resources) { + closeAll(Arrays.asList(resources)); } /** - * This method safes closes the resources. + * This method safely closes the resources. See {@link #closeAll(Iterator)} for more details. */ - public static void close(Iterable resources) { - close(resources.iterator()); + public static void closeAll(Collection resources) { + closeAll(resources.iterator()); } } diff --git a/java/src/main/java/ai/rapids/cudf/HostMemoryBuffer.java b/java/src/main/java/ai/rapids/cudf/HostMemoryBuffer.java index 3c52388504b..bec86b0c618 100644 --- a/java/src/main/java/ai/rapids/cudf/HostMemoryBuffer.java +++ b/java/src/main/java/ai/rapids/cudf/HostMemoryBuffer.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -245,6 +245,8 @@ public final void copyFromHostBuffer(long destOffset, HostMemoryBuffer srcData, * @param destOffset offset in bytes in this buffer to start copying to * @param in input stream to copy bytes from * @param byteLength number of bytes to copy + * @throws EOFException If there are not enough bytes in the stream to copy. + * @throws IOException If there is an error reading from the stream. */ public final void copyFromStream(long destOffset, InputStream in, long byteLength) throws IOException { addressOutOfBoundsCheck(address + destOffset, byteLength, "copy from stream"); @@ -254,7 +256,7 @@ public final void copyFromStream(long destOffset, InputStream in, long byteLengt int amountToCopy = (int) Math.min(arrayBuffer.length, left); int amountRead = in.read(arrayBuffer, 0, amountToCopy); if (amountRead < 0) { - throw new EOFException(); + throw new EOFException("Unexpected end of stream, expected " + left + " more bytes"); } setBytes(destOffset, arrayBuffer, 0, amountRead); destOffset += amountRead; From 403e302240e79b24ac0027c3709a258f38c5d484 Mon Sep 17 00:00:00 2001 From: liurenjie1024 Date: Mon, 28 Oct 2024 11:01:06 +0800 Subject: [PATCH 3/4] Remove SlicedTable --- .../main/java/ai/rapids/cudf/SlicedTable.java | 65 ------------------- 1 file changed, 65 deletions(-) delete mode 100644 java/src/main/java/ai/rapids/cudf/SlicedTable.java diff --git a/java/src/main/java/ai/rapids/cudf/SlicedTable.java b/java/src/main/java/ai/rapids/cudf/SlicedTable.java deleted file mode 100644 index a2fc8477c3e..00000000000 --- a/java/src/main/java/ai/rapids/cudf/SlicedTable.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package ai.rapids.cudf; - -import java.util.Objects; - -import static ai.rapids.cudf.Preconditions.ensure; - -/** - * A sliced view to underlying table. - * - * This is a simple wrapper around a table that represents a slice of the table, and it doesn't change ownership of the - * underlying table, so it's always the caller's responsibility to manage the lifecycle of the underlying table. - */ -public class SlicedTable { - private final int startRow; - private final int numRows; - private final Table table; - - public SlicedTable(int startRow, int numRows, Table table) { - Objects.requireNonNull(table, "table must not be null"); - ensure(startRow >= 0, "startRow must be >= 0"); - ensure(startRow < table.getRowCount(), - () -> "startRow " + startRow + " is larger than table row count " + table.getRowCount()); - ensure(numRows >= 0, () -> "numRows " + numRows + " is negative"); - ensure(startRow + numRows <= table.getRowCount(), () -> "startRow + numRows is " + (startRow + numRows) - + ", must be less than table row count " + table.getRowCount()); - - this.startRow = startRow; - this.numRows = numRows; - this.table = table; - } - - public int getStartRow() { - return startRow; - } - - public int getNumRows() { - return numRows; - } - - public Table getBaseTable() { - return table; - } - - public static SlicedTable from(Table table, int startRow, int numRows) { - return new SlicedTable(startRow, numRows, table); - } -} From eadea60b0f344a1a33dddb69283a7e4fab75836a Mon Sep 17 00:00:00 2001 From: liurenjie1024 Date: Tue, 29 Oct 2024 10:05:45 +0800 Subject: [PATCH 4/4] Remove utility class and move them into spark-rapids-jni --- java/src/main/java/ai/rapids/cudf/Arms.java | 86 ------------------- java/src/main/java/ai/rapids/cudf/Pair.java | 44 ---------- .../java/ai/rapids/cudf/Preconditions.java | 43 ---------- 3 files changed, 173 deletions(-) delete mode 100644 java/src/main/java/ai/rapids/cudf/Arms.java delete mode 100644 java/src/main/java/ai/rapids/cudf/Pair.java delete mode 100644 java/src/main/java/ai/rapids/cudf/Preconditions.java diff --git a/java/src/main/java/ai/rapids/cudf/Arms.java b/java/src/main/java/ai/rapids/cudf/Arms.java deleted file mode 100644 index f9284439199..00000000000 --- a/java/src/main/java/ai/rapids/cudf/Arms.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package ai.rapids.cudf; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Iterator; -import java.util.function.Function; - -/** - * This class contains utility methods for automatic resource management. - */ -public class Arms { - /** - * This method close the resource if an exception is thrown while executing the function. - */ - public static T closeIfException(R resource, Function function) { - try { - return function.apply(resource); - } catch (Exception e) { - if (resource != null) { - try { - resource.close(); - } catch (Exception inner) { - e.addSuppressed(inner); - } - } - throw e; - } - } - - /** - * This method safely closes all the resources. - *

- * This method will iterate through all the resources and closes them. If any exception happened during the - * traversal, exception will be captured and rethrown after all resources closed. - *

- */ - public static void closeAll(Iterator resources) { - Throwable t = null; - while (resources.hasNext()) { - try { - resources.next().close(); - } catch (Exception e) { - if (t == null) { - t = e; - } else { - t.addSuppressed(e); - } - } - } - - if (t != null) throw new RuntimeException(t); - } - - - /** - * This method safely closes all the resources. See {@link #closeAll(Iterator)} for more details. - */ - public static void closeAll(R... resources) { - closeAll(Arrays.asList(resources)); - } - - /** - * This method safely closes the resources. See {@link #closeAll(Iterator)} for more details. - */ - public static void closeAll(Collection resources) { - closeAll(resources.iterator()); - } -} diff --git a/java/src/main/java/ai/rapids/cudf/Pair.java b/java/src/main/java/ai/rapids/cudf/Pair.java deleted file mode 100644 index 76bfbbefdcb..00000000000 --- a/java/src/main/java/ai/rapids/cudf/Pair.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package ai.rapids.cudf; - -/** - * A utility class for holding a pair of values. - */ -public class Pair { - private final K left; - private final V right; - - public Pair(K left, V right) { - this.left = left; - this.right = right; - } - - public K getLeft() { - return left; - } - - public V getRight() { - return right; - } - - public static Pair of(K left, V right) { - return new Pair<>(left, right); - } -} diff --git a/java/src/main/java/ai/rapids/cudf/Preconditions.java b/java/src/main/java/ai/rapids/cudf/Preconditions.java deleted file mode 100644 index f276d058be6..00000000000 --- a/java/src/main/java/ai/rapids/cudf/Preconditions.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package ai.rapids.cudf; - -import java.util.function.Supplier; - -/** - * This class contains utility methods for checking preconditions. - */ -public class Preconditions { - /** - * Check if the condition is true, otherwise throw an IllegalStateException with the given message. - */ - public static void ensure(boolean condition, String message) { - if (!condition) { - throw new IllegalStateException(message); - } - } - - /** - * Check if the condition is true, otherwise throw an IllegalStateException with the given message supplier. - */ - public static void ensure(boolean condition, Supplier messageSupplier) { - if (!condition) { - throw new IllegalStateException(messageSupplier.get()); - } - } -}