Skip to content

Commit

Permalink
Addressing the feedback from 10/25/22
Browse files Browse the repository at this point in the history
  • Loading branch information
yash-puligundla committed Jan 12, 2023
1 parent f797944 commit 8464eb7
Show file tree
Hide file tree
Showing 5 changed files with 105 additions and 219 deletions.
5 changes: 4 additions & 1 deletion src/main/java/htsjdk/samtools/cram/compression/rans/rans4x8/RANS4x8Params.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package htsjdk.samtools.cram.compression.rans.rans4x8;

import htsjdk.samtools.cram.compression.rans.RANSParams;
import htsjdk.samtools.cram.compression.rans.ransnx16.RANSNx16Params;

public class RANS4x8Params implements RANSParams {

Expand All @@ -21,7 +22,9 @@ public ORDER getOrder() {
}

public int getFormatFlags(){
return order.ordinal();
return order == ORDER.ONE ?
RANSNx16Params.ORDER_FLAG_MASK :
0;
}

}
77 changes: 54 additions & 23 deletions src/test/java/htsjdk/samtools/cram/RANSInteropTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,12 @@ public class RANSInteropTest extends HtsjdkTest {
public static final String COMPRESSED_RANS4X8_DIR = "r4x8";
public static final String COMPRESSED_RANSNX16_DIR = "r4x16";

/////////////////////////////////////////////////////////////////////////////////////////////////
// RANS tests
/////////////////////////////////////////////////////////////////////////////////////////////////

// RANS4x8 codecs and testdata
public Object[][] getRANS4x8TestData() throws IOException {
// cache/reuse this for each test case to eliminate excessive garbage collection

// params:
// uncompressed testfile path, RANS encoder, RANS decoder,
// RANS params, compressed testfile directory name
final List<RANSParams.ORDER> rans4x8ParamsOrderList = Arrays.asList(
RANSParams.ORDER.ZERO,
RANSParams.ORDER.ONE);
Expand All @@ -65,7 +64,11 @@ public Object[][] getRANS4x8TestData() throws IOException {
}

// RANSNx16 codecs and testdata
public Object[][] getRANS4x16TestData() throws IOException {
public Object[][] getRANSNx16TestData() throws IOException {

// params:
// uncompressed testfile path, RANS encoder, RANS decoder,
// RANS params, compressed testfile directory name
final List<Integer> ransNx16ParamsFormatFlagList = Arrays.asList(
0x00,
RANSNx16Params.ORDER_FLAG_MASK,
Expand All @@ -76,10 +79,30 @@ public Object[][] getRANS4x16TestData() throws IOException {
RANSNx16Params.PACK_FLAG_MASK,
RANSNx16Params.PACK_FLAG_MASK | RANSNx16Params.ORDER_FLAG_MASK,
RANSNx16Params.RLE_FLAG_MASK | RANSNx16Params.PACK_FLAG_MASK,
RANSNx16Params.RLE_FLAG_MASK | RANSNx16Params.PACK_FLAG_MASK | RANSNx16Params.ORDER_FLAG_MASK,
RANSNx16Params.RLE_FLAG_MASK | RANSNx16Params.PACK_FLAG_MASK | RANSNx16Params.ORDER_FLAG_MASK);
final List<Object[]> testCases = new ArrayList<>();
getInteropRANSTestFiles()
.forEach(path ->
ransNx16ParamsFormatFlagList.stream().map(ransNx16ParamsFormatFlag -> new Object[]{
path,
new RANSNx16Encode(),
new RANSNx16Decode(),
new RANSNx16Params(ransNx16ParamsFormatFlag),
COMPRESSED_RANSNX16_DIR
}).forEach(testCases::add));
return testCases.toArray(new Object[][]{});
}

public Object[][] getRansNx16DecodeOnlyTestData() throws IOException {

// params:
// uncompressed testfile path, RANS encoder, RANS decoder,
// RANS params, compressed testfile directory name

// Stripe is implemented in the Decoder. It is not implemented in the Encoder.
final List<Integer> ransNx16ParamsFormatFlagList = Arrays.asList(
RANSNx16Params.STRIPE_FLAG_MASK,
RANSNx16Params.STRIPE_FLAG_MASK | RANSNx16Params.ORDER_FLAG_MASK
);
RANSNx16Params.STRIPE_FLAG_MASK | RANSNx16Params.ORDER_FLAG_MASK);
final List<Object[]> testCases = new ArrayList<>();
getInteropRANSTestFiles()
.forEach(path ->
Expand All @@ -93,14 +116,29 @@ public Object[][] getRANS4x16TestData() throws IOException {
return testCases.toArray(new Object[][]{});
}

@DataProvider(name = "allRansCodecsAndDataForRoundtrip")
public Object[][] getAllRansCodecsForRoundTrip() throws IOException {

// params:
// uncompressed testfile path, RANS encoder, RANS decoder,
// RANS params, compressed testfile directory name

// Since, Stripe is not implemented in the Encoder,
// we don't test round tripping for the cases where Stripe Flag = 1
return Stream.concat(Arrays.stream(getRANS4x8TestData()), Arrays.stream(getRANSNx16TestData()))
.toArray(Object[][]::new);
}

@DataProvider(name = "allRansCodecsAndData")
public Object[][] getAllRansCodecs() throws IOException {
// concatenate RANS4x8 and RANSNx16 codecs and testdata
return Stream.concat(Arrays.stream(getRANS4x8TestData()), Arrays.stream(getRANS4x16TestData()))

// params:
// uncompressed testfile path, RANS encoder, RANS decoder,
// RANS params, compressed testfile directory name
return Stream.concat(Arrays.stream(getAllRansCodecsForRoundTrip()), Arrays.stream(getRansNx16DecodeOnlyTestData()))
.toArray(Object[][]::new);
}

// TODO: testHtslibVersion should depend on SamtoolsTestUtilsTest.testSamtoolsVersion
@Test(description = "Test if CRAM Interop Test Data is available")
public void testGetHTSCodecsCorpus() {
if (!CRAMInteropTestUtils.isInteropTestDataAvailable()) {
Expand All @@ -111,7 +149,7 @@ public void testGetHTSCodecsCorpus() {

@Test (
dependsOnMethods = "testGetHTSCodecsCorpus",
dataProvider = "allRansCodecsAndData",
dataProvider = "allRansCodecsAndDataForRoundtrip",
description = "Roundtrip using htsjdk RANS. Compare the output with the original file" )
public void testRANSRoundTrip(
final Path uncompressedInteropPath,
Expand All @@ -125,13 +163,6 @@ public void testRANSRoundTrip(
// by filtering out the embedded newlines, and then round trip through RANS and compare the
// results
final ByteBuffer uncompressedInteropBytes = ByteBuffer.wrap(filterEmbeddedNewlines(IOUtils.toByteArray(uncompressedInteropStream)));

// If Stripe Flag is set, skip the round trip test as encoding is not implemented for this case.
if ((params.getFormatFlags() & RANSNx16Params.STRIPE_FLAG_MASK)==0) {
final ByteBuffer compressedHtsjdkBytes = ransEncode.compress(uncompressedInteropBytes, params);
uncompressedInteropBytes.rewind();
Assert.assertEquals(ransDecode.uncompress(compressedHtsjdkBytes), uncompressedInteropBytes);
}
}
}

Expand Down Expand Up @@ -170,7 +201,7 @@ public void testRANSPreCompressed(
}
}

// return a list of all RANS test data files in the InteropTest/RANS directory
// return a list of all RANS test data files in the htscodecs/tests directory
private List<Path> getInteropRANSTestFiles() throws IOException {
final List<Path> paths = new ArrayList<>();
Files.newDirectoryStream(
Expand All @@ -184,7 +215,7 @@ private List<Path> getInteropRANSTestFiles() throws IOException {
}

// the input files have embedded newlines that the test remove before round-tripping...
final byte[] filterEmbeddedNewlines(final byte[] rawBytes) throws IOException {
private final byte[] filterEmbeddedNewlines(final byte[] rawBytes) throws IOException {
// 1. filters new lines if any.
// 2. "q40+dir" file has an extra column delimited by tab. This column provides READ1 vs READ2 flag.
// This file is also new-line separated. The extra column, '\t' and '\n' are filtered.
Expand All @@ -206,7 +237,7 @@ final byte[] filterEmbeddedNewlines(final byte[] rawBytes) throws IOException {
}

// Given a test file name, map it to the corresponding rans compressed path
final Path getCompressedRANSPath(final String ransType,final Path uncompressedInteropPath, RANSParams params) {
private final Path getCompressedRANSPath(final String ransType,final Path uncompressedInteropPath, RANSParams params) {

// Example compressedFileName: r4x16/q4.193
// the substring after "." in the compressedFileName is the formatFlags (aka. the first byte of the compressed stream)
Expand Down
122 changes: 0 additions & 122 deletions src/test/java/htsjdk/samtools/cram/compression/rans/RansOrder1DemoTest.java
View file

This file was deleted.

Loading

0 comments on commit 8464eb7

Please sign in to comment.