Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Removed XZ compression feature of serialized IValues #253

Merged
merged 2 commits into from
Apr 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -153,11 +153,6 @@
<artifactId>capsule</artifactId>
<version>0.7.1</version>
</dependency>
<dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@
import java.io.OutputStream;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.tukaani.xz.LZMA2Options;
import org.tukaani.xz.XZInputStream;
import org.tukaani.xz.XZOutputStream;
import com.github.luben.zstd.ZstdInputStream;
import com.github.luben.zstd.ZstdOutputStream;
import com.github.luben.zstd.util.Native;
Expand Down Expand Up @@ -60,8 +57,8 @@ public void setLevel(int level) {
result.setLevel(level);
return result;
}
case Header.Compression.XZ: {
return new XZOutputStream(rawStream, new LZMA2Options(level));
case Header.Compression.UNSUPPORTED_XZ: {
throw new UnsupportedOperationException("XZ compression is not supported anymore, please use an older version of rascal/vallang to open the file and store it with a different level of compression");
}
case Header.Compression.ZSTD: {
return new ZstdOutputStream(rawStream, level);
Expand All @@ -77,8 +74,8 @@ public static InputStream wrapStream(InputStream raw, int algorithm) throws IOEx
return raw;
case Header.Compression.GZIP:
return new GZIPInputStream(raw);
case Header.Compression.XZ:
return new XZInputStream(raw);
case Header.Compression.UNSUPPORTED_XZ:
throw new UnsupportedOperationException("XZ compression is not supported anymore, please use an older version of rascal/vallang to open the file and store it with a different level of compression");
case Header.Compression.ZSTD:
if (Compressor.zstdAvailable()) {
if (raw instanceof ByteBufferInputStream && ((ByteBufferInputStream)raw).getByteBuffer().isDirect()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
public static final class Compression {
public static final byte NONE = 0;
public static final byte GZIP = 1;
public static final byte XZ = 2;
/** Never officially available, but still reserved */
public static final byte UNSUPPORTED_XZ = 2;
public static final byte ZSTD = 3;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.nio.channels.FileChannel;
import java.util.Arrays;
import java.util.function.Supplier;

import org.checkerframework.checker.nullness.qual.Nullable;
Expand Down Expand Up @@ -49,13 +50,25 @@ public IValueInputStream(InputStream in, IValueFactory vf, Supplier<TypeStore> t
while (read < currentHeader.length) {
read += in.read(currentHeader, read, currentHeader.length - read);
}
if (!Arrays.equals(currentHeader, Header.MAIN)) {
throw new IOException("Incorrect file header, expected: [" + toHex(Header.MAIN) + "] but found: [" + toHex(currentHeader) + "]");
}

int compression = in.read();
in = Compressor.wrapStream(in, compression);
reader = new BinaryWireInputStream(in);
}


private static String toHex(byte[] main) {
String result = "";
for (byte b : main) {
result += String.format("%#04x ", b);
}
return result.trim();
}


public IValueInputStream(FileChannel channel, IValueFactory vf, Supplier<TypeStore> typeStoreSupplier) throws IOException {
this(new FileChannelDirectInputStream(channel), vf, typeStoreSupplier);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public enum CompressionRate {
Light(Header.Compression.ZSTD, 1),
Normal(Header.Compression.ZSTD, 5),
Strong(Header.Compression.ZSTD, 13),
Extreme(Header.Compression.XZ, 6),
Extreme(Header.Compression.ZSTD, 19),
DavyLandman marked this conversation as resolved.
Show resolved Hide resolved
XML(Compression.NONE, 0)
;

Expand Down
Loading