Add RANS Nx16 codec (Update CRAM Codecs to CRAM 3.1)

src/main/java/htsjdk/samtools/cram/compression/CompressionUtils.java

@@ -0,0 +1,179 @@
+package htsjdk.samtools.cram.compression;
+
+import htsjdk.samtools.cram.CRAMException;
+import htsjdk.samtools.cram.compression.rans.Constants;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+public class CompressionUtils {
+    public static void writeUint7(final int i, final ByteBuffer cp) {
+        int s = 0;
+        int X = i;
+        do {
+            s += 7;
+            X >>= 7;
+        } while (X > 0);
+        do {
+            s -= 7;
+            //writeByte
+            final int s_ = (s > 0) ? 1 : 0;
+            cp.put((byte) (((i >> s) & 0x7f) + (s_ << 7)));
+        } while (s > 0);
+    }
+
+    public static int readUint7(final ByteBuffer cp) {
+        int i = 0;
+        int c;
+        do {
+            //read byte
+            c = cp.get();
+            i = (i << 7) | (c & 0x7f);
+        } while ((c & 0x80) != 0);
+        return i;
+    }
+
+    public static ByteBuffer encodePack(
+            final ByteBuffer inBuffer,
+            final ByteBuffer outBuffer,
+            final int[] frequencyTable,
+            final int[] packMappingTable,
+            final int numSymbols){
+        final int inSize = inBuffer.remaining();
+        final ByteBuffer encodedBuffer;
+        if (numSymbols <= 1) {
+            encodedBuffer = CompressionUtils.allocateByteBuffer(0);
+        } else if (numSymbols <= 2) {
+
+            // 1 bit per value
+            final int encodedBufferSize = (int) Math.ceil((double) inSize/8);
+            encodedBuffer = CompressionUtils.allocateByteBuffer(encodedBufferSize);
+            int j = -1;
+            for (int i = 0; i < inSize; i ++) {
+                if (i % 8 == 0) {
+                    encodedBuffer.put(++j, (byte) 0);
+                }
+                encodedBuffer.put(j, (byte) (encodedBuffer.get(j) + (packMappingTable[inBuffer.get(i) & 0xFF] << (i % 8))));
+            }
+        } else if (numSymbols <= 4) {
+
+            // 2 bits per value
+            final int encodedBufferSize = (int) Math.ceil((double) inSize/4);
+            encodedBuffer = CompressionUtils.allocateByteBuffer(encodedBufferSize);
+            int j = -1;
+            for (int i = 0; i < inSize; i ++) {
+                if (i % 4 == 0) {
+                    encodedBuffer.put(++j, (byte) 0);
+                }
+                encodedBuffer.put(j, (byte) (encodedBuffer.get(j) + (packMappingTable[inBuffer.get(i) & 0xFF] << ((i % 4) * 2))));
+            }
+        } else {
+
+            // 4 bits per value
+            final int encodedBufferSize = (int) Math.ceil((double)inSize/2);
+            encodedBuffer = CompressionUtils.allocateByteBuffer(encodedBufferSize);
+            int j = -1;
+            for (int i = 0; i < inSize; i ++) {
+                if (i % 2 == 0) {
+                    encodedBuffer.put(++j, (byte) 0);
+                }
+                encodedBuffer.put(j, (byte) (encodedBuffer.get(j) + (packMappingTable[inBuffer.get(i) & 0xFF] << ((i % 2) * 4))));
+            }
+        }
+
+        // write numSymbols
+        outBuffer.put((byte) numSymbols);
+
+        // write mapping table "packMappingTable" that converts mapped value to original symbol
+        for(int i = 0; i < Constants.NUMBER_OF_SYMBOLS; i ++) {
+            if (frequencyTable[i] > 0) {
+                outBuffer.put((byte) i);
+            }
+        }
+
+        // write the length of data
+        CompressionUtils.writeUint7(encodedBuffer.limit(), outBuffer);
+        return encodedBuffer; // Here position = 0 since we have always accessed the data buffer using index
+    }
+
+    public static ByteBuffer decodePack(
+            final ByteBuffer inBuffer,
+            final byte[] packMappingTable,
+            final int numSymbols,
+            final int uncompressedPackOutputLength) {
+        final ByteBuffer outBufferPack = CompressionUtils.allocateByteBuffer(uncompressedPackOutputLength);
+        int j = 0;
+        if (numSymbols <= 1) {
+            for (int i=0; i < uncompressedPackOutputLength; i++){
+                outBufferPack.put(i, packMappingTable[0]);
+            }
+        }
+
+        // 1 bit per value
+        else if (numSymbols <= 2) {
+            int v = 0;
+            for (int i=0; i < uncompressedPackOutputLength; i++){
+                if (i % 8 == 0){
+                    v = inBuffer.get(j++);
+                }
+                outBufferPack.put(i, packMappingTable[v & 1]);
+                v >>=1;
+            }
+        }
+
+        // 2 bits per value
+        else if (numSymbols <= 4){
+            int v = 0;
+            for(int i=0; i < uncompressedPackOutputLength; i++){
+                if (i % 4 == 0){
+                    v = inBuffer.get(j++);
+                }
+                outBufferPack.put(i, packMappingTable[v & 3]);
+                v >>=2;
+            }
+        }
+
+        // 4 bits per value
+        else if (numSymbols <= 16){
+            int v = 0;
+            for(int i=0; i < uncompressedPackOutputLength; i++){
+                if (i % 2 == 0){
+                    v = inBuffer.get(j++);
+                }
+                outBufferPack.put(i, packMappingTable[v & 15]);
+                v >>=4;
+            }
+        }
+        return outBufferPack;
+    }
+
+
+
+    public static ByteBuffer allocateOutputBuffer(final int inSize) {
+        // This calculation is identical to the one in samtools rANS_static.c
+        // Presumably the frequency table (always big enough for order 1) = 257*257,
+        // then * 3 for each entry (byte->symbol, 2 bytes -> scaled frequency),
+        // + 9 for the header (order byte, and 2 int lengths for compressed/uncompressed lengths).
+        final int compressedSize = (int) (inSize + 257 * 257 * 3 + 9);
+        final ByteBuffer outputBuffer = ByteBuffer.allocate(compressedSize).order(ByteOrder.LITTLE_ENDIAN);
+        if (outputBuffer.remaining() < compressedSize) {
+            throw new CRAMException("Failed to allocate sufficient buffer size for RANS coder.");
+        }
+        return outputBuffer;
+    }
+
+    // returns a new LITTLE_ENDIAN ByteBuffer of size = bufferSize
+    public static ByteBuffer allocateByteBuffer(final int bufferSize){
+        return ByteBuffer.allocate(bufferSize).order(ByteOrder.LITTLE_ENDIAN);
+    }
+
+    // returns a LITTLE_ENDIAN ByteBuffer that is created by wrapping a byte[]
+    public static ByteBuffer wrap(final byte[] inputBytes){
+        return ByteBuffer.wrap(inputBytes).order(ByteOrder.LITTLE_ENDIAN);
+    }
+
+    // returns a LITTLE_ENDIAN ByteBuffer that is created by inputBuffer.slice()
+    public static ByteBuffer slice(final ByteBuffer inputBuffer){
+        return inputBuffer.slice().order(ByteOrder.LITTLE_ENDIAN);
+    }
+}

src/main/java/htsjdk/samtools/cram/compression/ExternalCompressor.java

@@ -1,6 +1,7 @@
 package htsjdk.samtools.cram.compression;
 
-import htsjdk.samtools.cram.compression.rans.RANS;
+import htsjdk.samtools.cram.compression.rans.rans4x8.RANS4x8Decode;
+import htsjdk.samtools.cram.compression.rans.rans4x8.RANS4x8Encode;
 import htsjdk.samtools.cram.structure.block.BlockCompressionMethod;
 import htsjdk.utils.ValidationUtils;
 
@@ -71,8 +72,8 @@ public static ExternalCompressor getCompressorForMethod(
 
             case RANS:
                 return compressorSpecificArg == NO_COMPRESSION_ARG ?
-                        new RANSExternalCompressor(new RANS()) :
-                        new RANSExternalCompressor(compressorSpecificArg, new RANS());
+                        new RANSExternalCompressor(new RANS4x8Encode(), new RANS4x8Decode()) :
+                        new RANSExternalCompressor(compressorSpecificArg, new RANS4x8Encode(), new RANS4x8Decode());
 
             case BZIP2:
                 ValidationUtils.validateArg(
@@ -85,5 +86,4 @@ public static ExternalCompressor getCompressorForMethod(
         }
     }
 
-}
-
+}

src/main/java/htsjdk/samtools/cram/compression/RANSExternalCompressor.java

@@ -24,48 +24,60 @@
  */
 package htsjdk.samtools.cram.compression;
 
-import htsjdk.samtools.cram.compression.rans.RANS;
+import htsjdk.samtools.cram.compression.rans.RANSParams;
+import htsjdk.samtools.cram.compression.rans.rans4x8.RANS4x8Decode;
+import htsjdk.samtools.cram.compression.rans.rans4x8.RANS4x8Encode;
+import htsjdk.samtools.cram.compression.rans.rans4x8.RANS4x8Params;
 import htsjdk.samtools.cram.structure.block.BlockCompressionMethod;
 
 import java.nio.ByteBuffer;
 import java.util.Objects;
 
 public final class RANSExternalCompressor extends ExternalCompressor {
-    private final RANS.ORDER order;
-    private final RANS rans;
+    private final RANSParams.ORDER order;
+    private final RANS4x8Encode ransEncode;
+    private final RANS4x8Decode ransDecode;
 
     /**
      * We use a shared RANS instance for all compressors.
      * @param rans
      */
-    public RANSExternalCompressor(final RANS rans) {
-        this(RANS.ORDER.ZERO, rans);
+    public RANSExternalCompressor(
+            final RANS4x8Encode ransEncode,
+            final RANS4x8Decode ransDecode) {
+        this(RANSParams.ORDER.ZERO, ransEncode, ransDecode);
     }
 
-    public RANSExternalCompressor(final int order, final RANS rans) {
-        this(RANS.ORDER.fromInt(order), rans);
+    public RANSExternalCompressor(
+            final int order,
+            final RANS4x8Encode ransEncode,
+            final RANS4x8Decode ransDecode) {
+        this(RANSParams.ORDER.fromInt(order), ransEncode, ransDecode);
     }
 
-    public RANSExternalCompressor(final RANS.ORDER order, final RANS rans) {
+    public RANSExternalCompressor(
+            final RANSParams.ORDER order,
+            final RANS4x8Encode ransEncode,
+            final RANS4x8Decode ransDecode) {
         super(BlockCompressionMethod.RANS);
-        this.rans = rans;
+        this.ransEncode = ransEncode;
+        this.ransDecode = ransDecode;
         this.order = order;
     }
 
     @Override
     public byte[] compress(final byte[] data) {
-        final ByteBuffer buffer = rans.compress(ByteBuffer.wrap(data), order);
+        final RANS4x8Params params = new RANS4x8Params(order);
+        final ByteBuffer buffer = ransEncode.compress(ByteBuffer.wrap(data), params);
         return toByteArray(buffer);
     }
 
     @Override
     public byte[] uncompress(byte[] data) {
-        final ByteBuffer buf = rans.uncompress(ByteBuffer.wrap(data));
+        final ByteBuffer buf = ransDecode.uncompress(ByteBuffer.wrap(data));
         return toByteArray(buf);
     }
 
-    public RANS.ORDER getOrder() { return order; }
-
     @Override
     public String toString() {
         return String.format("%s(%s)", this.getMethod(), order);
@@ -96,4 +108,4 @@ private byte[] toByteArray(final ByteBuffer buffer) {
         return bytes;
     }
 
-}
+}

src/main/java/htsjdk/samtools/cram/compression/rans/ArithmeticDecoder.java

@@ -24,25 +24,25 @@
  */
 package htsjdk.samtools.cram.compression.rans;
 
-final class ArithmeticDecoder {
-    final FC[] fc = new FC[256];
+final public class ArithmeticDecoder {
+    public final int[] frequencies = new int[Constants.NUMBER_OF_SYMBOLS];
 
-    // reverse lookup table ?
-    byte[] R = new byte[Constants.TOTFREQ];
+    // reverse lookup table
+    public final byte[] reverseLookup = new byte[Constants.TOTAL_FREQ];
 
     public ArithmeticDecoder() {
-        for (int i = 0; i < 256; i++) {
-            fc[i] = new FC();
+        for (int i = 0; i < Constants.NUMBER_OF_SYMBOLS; i++) {
+            frequencies[i] = 0;
         }
     }
 
     public void reset() {
-        for (int i = 0; i < 256; i++) {
-            fc[i].reset();
+        for (int i = 0; i < Constants.NUMBER_OF_SYMBOLS; i++) {
+            frequencies[i] = 0;
         }
-        for (int i = 0; i < Constants.TOTFREQ; i++) {
-            R[i] = 0;
+        for (int i = 0; i < Constants.TOTAL_FREQ; i++) {
+            reverseLookup[i] = 0;
         }
     }
 
-}
+}

src/main/java/htsjdk/samtools/cram/compression/rans/Constants.java

@@ -1,7 +1,13 @@
 package htsjdk.samtools.cram.compression.rans;
 
-final class Constants {
-    static final int TF_SHIFT = 12;
-    static final int TOTFREQ = (1 << TF_SHIFT); // 4096
-    static final int RANS_BYTE_L = 1 << 23;
-}
+final public class Constants {
+    public static final int TOTAL_FREQ_SHIFT = 12;
+    public static final int TOTAL_FREQ = (1 << TOTAL_FREQ_SHIFT); // 4096
+    public static final int NUMBER_OF_SYMBOLS = 256;
+    public static final int RANS_4x8_LOWER_BOUND = 1 << 23;
+    public static final int RANS_4x8_ORDER_BYTE_LENGTH = 1;
+    public static final int RANS_4x8_COMPRESSED_BYTE_LENGTH = 4;
+    public static final int RANS_4x8_RAW_BYTE_LENGTH = 4;
+    public static final int RANS_4x8_PREFIX_BYTE_LENGTH = RANS_4x8_ORDER_BYTE_LENGTH + RANS_4x8_COMPRESSED_BYTE_LENGTH + RANS_4x8_RAW_BYTE_LENGTH;
+    public static final int RANS_Nx16_LOWER_BOUND = 1 << 15;
+}