Test binary io compression modes

pom.xml

@@ -36,11 +36,6 @@
 				</includes>
 			</resource>
 		</resources>
-		<testResources>
-			<testResource>
-				<directory>test/main/resources</directory>
-			</testResource>
-		</testResources>
 		<plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>

src/main/java/io/usethesource/vallang/io/binary/util/ByteBufferOutputStream.java

@@ -31,10 +31,20 @@ public void flush() throws IOException {
         if (closed) {
             throw new IOException("Stream closed");
         }
+
         target.flip();
         if (target.hasRemaining()) {
             target = flush(target);
+            if (!target.hasRemaining()) {
+                throw new IOException("flush implementation didn't correctly provide a new buffer to write to: " + target);
+            }
+        }
+        else {
+            // the buffer was empty, so flush was called on an already flushed stream
+            // so we'll reset the buffer to make sure we don't continue with empty buffer
+            target.clear();
         }
+        assert target.hasRemaining(): "after a flush, we should have a buffer with some room. (it was: " + target + ")";
     }
 
     @Override

src/main/java/io/usethesource/vallang/io/binary/wire/binary/BinaryWireOutputStream.java

@@ -36,7 +36,6 @@ public BinaryWireOutputStream(OutputStream stream, int stringSharingWindowSize)
         this(stream, stringSharingWindowSize, 8*1024);
     }
     public BinaryWireOutputStream(OutputStream stream, int stringSharingWindowSize, int bufferSize) throws IOException {
-        assert stringSharingWindowSize > 0;
         if (stream instanceof BufferedOutputStream || stream instanceof ByteBufferOutputStream) {
             __stream = stream;
         }

src/test/java/io/usethesource/vallang/basic/BinaryIoSmokeTest.java

@@ -10,23 +10,24 @@
  *******************************************************************************/
 package io.usethesource.vallang.basic;
 
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.fail;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
+import java.io.FilterInputStream;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.nio.file.StandardOpenOption;
 import java.util.HashMap;
-
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ArgumentsSource;
-
 import io.usethesource.vallang.ArgumentsMaxDepth;
 import io.usethesource.vallang.ArgumentsMaxWidth;
 import io.usethesource.vallang.ExpectedType;
@@ -42,6 +43,7 @@
 import io.usethesource.vallang.io.binary.message.IValueWriter;
 import io.usethesource.vallang.io.binary.stream.IValueInputStream;
 import io.usethesource.vallang.io.binary.stream.IValueOutputStream;
+import io.usethesource.vallang.io.binary.stream.IValueOutputStream.CompressionRate;
 import io.usethesource.vallang.io.binary.util.WindowSizes;
 import io.usethesource.vallang.io.binary.wire.IWireInputStream;
 import io.usethesource.vallang.io.binary.wire.IWireOutputStream;
@@ -54,11 +56,44 @@
 
 public final class BinaryIoSmokeTest extends BooleanStoreProvider {
 
+    @ParameterizedTest @ArgumentsSource(ValueProvider.class)
+    public void testSingleValueIO(IValueFactory vf, TypeStore ts) throws IOException {
+        ioRoundTrip(vf, ts, vf.integer(1));
+    }
+
+    @ParameterizedTest @ArgumentsSource(ValueProvider.class)
+    public void testSingleValue2IO(IValueFactory vf, TypeStore ts) throws IOException {
+        ioRoundTrip(vf, ts, vf.string("a"));
+    }
+
     @ParameterizedTest @ArgumentsSource(ValueProvider.class)
     public void testSmallBinaryIO(IValueFactory vf, TypeStore ts, IValue value) throws IOException {
         ioRoundTrip(vf, ts, value);
     }
 
+    @ParameterizedTest @ArgumentsSource(ValueProvider.class)
+    void testReadReferenceSerializedFile(IValueFactory vf, TypeStore ts) throws IOException {
+        try (var files = new ZipInputStream(this.getClass().getResourceAsStream("/io/reference-serialized-binary-values.zip"))) {
+            ZipEntry current;
+            while ((current = files.getNextEntry()) != null) {
+                try (var read = new IValueInputStream(new FilterInputStream(files) {
+                    public void close() throws IOException {
+                        // we have to redirect the close to the entry close, not the global close
+                        files.closeEntry();
+                    };
+                }, vf, () -> ts)) {
+                    assertNotNull(read.read());
+                }
+                catch (Throwable e) {
+                    fail("Failed for " + current.getName(), e);
+                }
+
+            }
+        }
+    }
+
+
+
     @ParameterizedTest @ArgumentsSource(ValueProvider.class)
     public void testRegression40(IValueFactory vf, TypeStore store, 
             @GivenValue("twotups(<\\true(),twotups(<not(\\true()),and(\\false(),\\true())>,<twotups(<couples([]),\\true()>,<or([]),friends([])>),twotups(<or([]),or([])>,<or([]),\\true()>)>)>,<twotups(<not(\\true()),and(\\true(),\\true())>,<twotups(<couples([]),couples([])>,<\\true(),couples([])>),not(\\true())>),and(or([\\true()]),twotups(<or([]),\\true()>,<or([]),\\false()>))>)") 
@@ -152,7 +187,7 @@ public void testDeepRandomValuesIO(IValueFactory vf, TypeStore ts, IValue val) t
             ValueStreams.bottomupbf(val).forEach(v -> {
                 try {
                     ioRoundTrip(vf, ts, v);
-                } catch (IOException error) {
+                } catch (Throwable error) {
                     fail(error);
                 }
             });
@@ -203,9 +238,22 @@ public void iopRoundTrip(IValueFactory vf, TypeStore ts, Type tp) throws IOExcep
         }
     }
 
+    private final CompressionRate[] RATES_TO_TESTS = {CompressionRate.Normal, CompressionRate.Extreme, CompressionRate.None, CompressionRate.NoSharing};
+
     private void ioRoundTrip(IValueFactory vf, TypeStore ts, IValue value) throws IOException {
+        for (var rate: RATES_TO_TESTS) {
+            try {
+                ioRoundTrip(vf, ts, value, rate);
+            }
+            catch (Throwable e) {
+                fail("Error with "+ rate + " compression", e);
+            }
+        }
+    }
+
+    private void ioRoundTrip(IValueFactory vf, TypeStore ts, IValue value, IValueOutputStream.CompressionRate compression) throws IOException {
         ByteArrayOutputStream buffer = new ByteArrayOutputStream();
-        try (IValueOutputStream w = new IValueOutputStream(buffer, vf, IValueOutputStream.CompressionRate.Normal)) {
+        try (IValueOutputStream w = new IValueOutputStream(buffer, vf, compression)) {
             w.write(value);
         }
         try (IValueInputStream read = new IValueInputStream(new ByteArrayInputStream(buffer.toByteArray()), vf, () -> ts)) {
@@ -239,10 +287,20 @@ else if (value instanceof IConstructor) {
     }
 
     private void ioRoundTripFile(IValueFactory vf, TypeStore ts, IValue value) throws IOException {
+        for (var rate: RATES_TO_TESTS) {
+            try {
+                ioRoundTripFile(vf, ts, value, rate);
+            }
+            catch (Throwable e) {
+                fail("Error with "+ rate + " compression", e);
+            }
+        }
+    }
+    private void ioRoundTripFile(IValueFactory vf, TypeStore ts, IValue value, IValueOutputStream.CompressionRate compression) throws IOException {
         long fileSize = 0;
         File target = File.createTempFile("valllang-test-file", "something");
         target.deleteOnExit();
-        try (IValueOutputStream w = new IValueOutputStream(FileChannel.open(target.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE), vf, IValueOutputStream.CompressionRate.Normal)) {
+        try (IValueOutputStream w = new IValueOutputStream(FileChannel.open(target.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE), vf, compression)) {
             w.write(value);
         }
         fileSize = Files.size(target.toPath());
@@ -261,10 +319,20 @@ private void ioRoundTripFile(IValueFactory vf, TypeStore ts, IValue value) throw
     }
 
     private void ioRoundTripFile2(IValueFactory vf, TypeStore ts, IValue value) throws FileNotFoundException, IOException {
+        for (var rate: RATES_TO_TESTS) {
+            try {
+                ioRoundTripFile2(vf, ts, value, rate);
+            }
+            catch (Throwable e) {
+                fail("Error with "+ rate + " compression", e);
+            }
+        }
+    }
+    private void ioRoundTripFile2(IValueFactory vf, TypeStore ts, IValue value, IValueOutputStream.CompressionRate compression) throws FileNotFoundException, IOException {
         long fileSize = 0;
         File target = File.createTempFile("valllang-test-file", "something");
         target.deleteOnExit();
-        try (IValueOutputStream w = new IValueOutputStream(new FileOutputStream(target), vf, IValueOutputStream.CompressionRate.Normal)) {
+        try (IValueOutputStream w = new IValueOutputStream(new FileOutputStream(target), vf, compression)) {
             w.write(value);
         }
         fileSize = Files.size(target.toPath());

src/test/java/io/usethesource/vallang/io/FileChannelOutputStreamTest.java

@@ -0,0 +1,105 @@
+package io.usethesource.vallang.io;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Random;
+import org.junit.jupiter.api.Test;
+import io.usethesource.vallang.io.binary.util.FileChannelDirectInputStream;
+import io.usethesource.vallang.io.binary.util.FileChannelDirectOutputStream;
+
+class FileChannelOutputStreamTest {
+    private static final Path targetFile;
+    static {
+        Path file = null;
+        try {
+            file = Files.createTempFile("file-channel-test", "bin");
+        } catch (IOException e) {
+            System.err.println(e);
+        }
+        targetFile = file;
+    } 
+
+    private FileChannel openWriteChannel() throws IOException {
+        return FileChannel.open(targetFile, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
+    }
+
+    private FileChannel openReadChannel() throws IOException {
+        return FileChannel.open(targetFile, StandardOpenOption.READ);
+    }
+
+    @Test
+    void testSimpleWrite() throws IOException {
+        roundTripChannel(new byte[][]{{42}});
+    }
+
+    @Test
+    void testBigWrite() throws IOException {
+        byte[] buffer = new byte[1024*1024];
+        new Random().nextBytes(buffer);
+        roundTripChannel(new byte[][]{buffer});
+    }
+
+    @Test
+    void testChunkedBigWrite() throws IOException {
+        byte[][] buffers = new byte[1024][];
+        var r = new Random();
+        for (int i = 0; i < buffers.length; i++) {
+            buffers[i] = new byte[i * 128];
+            r.nextBytes(buffers[i]);
+        }
+        roundTripChannel(buffers);
+    }
+
+
+    private void roundTripChannel(byte[][] buffers) throws IOException {
+        writeChannelInBulk(buffers);
+        verifyChannelInBulk(buffers);
+        writeChannelBytePerByte(buffers);
+        verifyChannelBytePerByte(buffers);
+    }
+
+    private void verifyChannelBytePerByte(byte[][] buffers) throws IOException {
+        try (var reader = new FileChannelDirectInputStream(openReadChannel())) {
+            for (byte[] expected: buffers) {
+                for (byte expect: expected) {
+                    assertEquals(expect & 0xFF, reader.read());
+                }
+            }
+        }
+    }
+
+
+    private void verifyChannelInBulk(byte[][] buffers) throws IOException {
+        try (var reader = new FileChannelDirectInputStream(openReadChannel())) {
+            for (byte[] expected: buffers) {
+                byte[] actual = new byte[expected.length];
+                reader.read(actual);
+                assertArrayEquals(expected, actual);
+            }
+        }
+    }
+
+    private void writeChannelBytePerByte(byte[][] buffers) throws IOException {
+        try (var writer = new FileChannelDirectOutputStream(openWriteChannel(), 1)) {
+            for (byte[] buf: buffers) {
+                for (byte b: buf) {
+                    writer.write(b);
+                }
+            }
+        }
+    }
+
+    private void writeChannelInBulk(byte[][] buffers) throws IOException {
+        try (var writer = new FileChannelDirectOutputStream(openWriteChannel(), 1)) {
+            for (byte[] buf: buffers) {
+                writer.write(buf);
+            }
+        }
+    }
+
+}