Skip to content

Commit

Permalink
Expose data length of host merge result
Browse files Browse the repository at this point in the history
  • Loading branch information
liurenjie1024 committed Nov 15, 2024
1 parent e405a88 commit 0328fee
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,14 @@ public void close() throws Exception {
hostBuf = null;
}

/**
* Get the length of the data in the host buffer.
* @return the length of the data in the host buffer
*/
public long getDataLength() {
return hostBuf.getLength();
}

/**
* Convert the host buffer into a cudf table.
* @return the cudf table
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,12 @@
package com.nvidia.spark.rapids.jni.kudo;

import ai.rapids.cudf.*;
import com.nvidia.spark.rapids.jni.Arms;
import com.nvidia.spark.rapids.jni.Pair;
import com.nvidia.spark.rapids.jni.schema.Visitors;

import java.io.*;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.LongConsumer;
import java.util.function.Supplier;
import java.util.stream.IntStream;
Expand Down Expand Up @@ -168,9 +165,8 @@ public KudoSerializer(Schema schema) {
}

/**
* Write partition of a table to a stream.
* Write partition of a table to a stream. This method is used for test only.
* <br/>
* <p>
* The caller should ensure that table's schema matches the schema used to create this serializer, otherwise behavior
* is undefined.
*
Expand All @@ -180,7 +176,7 @@ public KudoSerializer(Schema schema) {
* @param numRows number of rows to write
* @return number of bytes written
*/
public long writeToStream(Table table, OutputStream out, int rowOffset, int numRows) {
long writeToStream(Table table, OutputStream out, int rowOffset, int numRows) {
HostColumnVector[] columns = null;
try {
columns = IntStream.range(0, table.getNumberOfColumns())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,10 @@
* limitations under the License.
*/

package com.nvidia.spark.rapids.jni;
package com.nvidia.spark.rapids.jni.kudo;

import ai.rapids.cudf.*;
import com.nvidia.spark.rapids.jni.kudo.KudoSerializer;
import com.nvidia.spark.rapids.jni.kudo.KudoTable;
import com.nvidia.spark.rapids.jni.kudo.KudoTableHeader;
import com.nvidia.spark.rapids.jni.Arms;
import org.junit.jupiter.api.Test;

import java.io.ByteArrayInputStream;
Expand Down

0 comments on commit 0328fee

Please sign in to comment.