vng: Use uint64 for Data Lengths

vng/header.go

@@ -9,15 +9,15 @@ import (
 
 const (
 	Version     = 4
-	HeaderSize  = 16
+	HeaderSize  = 24
 	MaxMetaSize = 100 * 1024 * 1024
 	MaxDataSize = 2 * 1024 * 1024 * 1024
 )
 
 type Header struct {
 	Version  uint32
-	MetaSize uint32
-	DataSize uint32
+	MetaSize uint64
+	DataSize uint64
 }
 
 func (h Header) Serialize() []byte {
@@ -26,8 +26,8 @@ func (h Header) Serialize() []byte {
 	bytes[1] = 'N'
 	bytes[2] = 'G'
 	binary.LittleEndian.PutUint32(bytes[4:], h.Version)
-	binary.LittleEndian.PutUint32(bytes[8:], h.MetaSize)
-	binary.LittleEndian.PutUint32(bytes[12:], h.DataSize)
+	binary.LittleEndian.PutUint64(bytes[8:], h.MetaSize)
+	binary.LittleEndian.PutUint64(bytes[16:], h.DataSize)
 	return bytes[:]
 }
 
@@ -36,8 +36,8 @@ func (h *Header) Deserialize(bytes []byte) error {
 		return errors.New("invalid VNG header")
 	}
 	h.Version = binary.LittleEndian.Uint32(bytes[4:])
-	h.MetaSize = binary.LittleEndian.Uint32(bytes[8:])
-	h.DataSize = binary.LittleEndian.Uint32(bytes[12:])
+	h.MetaSize = binary.LittleEndian.Uint64(bytes[8:])
+	h.DataSize = binary.LittleEndian.Uint64(bytes[16:])
 	if h.Version != Version {
 		return fmt.Errorf("unsupport VNG version %d: expected version %d", h.Version, Version)
 	}

vng/primitive.go

@@ -141,9 +141,9 @@ func (p *PrimitiveEncoder) Metadata(off uint64) (uint64, Metadata) {
 		}
 	}
 	loc := Segment{
-		Offset:            int64(off),
-		Length:            int32(len(p.out)),
-		MemLength:         int32(p.bytesLen),
+		Offset:            off,
+		Length:            uint64(len(p.out)),
+		MemLength:         p.bytesLen,
 		CompressionFormat: p.format,
 	}
 	off += uint64(len(p.out))

vng/segment.go

@@ -16,10 +16,10 @@ const (
 )
 
 type Segment struct {
-	Offset            int64 // Offset relative to start of file
-	Length            int32 // Length in file
-	MemLength         int32 // Length in memory
-	CompressionFormat uint8 // Compression format in file
+	Offset            uint64 // Offset relative to start of file
+	Length            uint64 // Length in file
+	MemLength         uint64 // Length in memory
+	CompressionFormat uint8  // Compression format in file
 }
 
 var zbufPool = sync.Pool{
@@ -36,13 +36,13 @@ func (s *Segment) Read(r io.ReaderAt, b []byte) error {
 	b = b[:s.MemLength]
 	switch s.CompressionFormat {
 	case CompressionFormatNone:
-		_, err := r.ReadAt(b, s.Offset)
+		_, err := r.ReadAt(b, int64(s.Offset))
 		return err
 	case CompressionFormatLZ4:
 		zbuf := zbufPool.Get().(*[]byte)
 		defer zbufPool.Put(zbuf)
 		*zbuf = slices.Grow((*zbuf)[:0], int(s.Length))[:s.Length]
-		if _, err := r.ReadAt(*zbuf, s.Offset); err != nil {
+		if _, err := r.ReadAt(*zbuf, int64(s.Offset)); err != nil {
 			return err
 		}
 		n, err := lz4.UncompressBlock(*zbuf, b)

vng/writer.go

@@ -63,7 +63,7 @@ func (w *Writer) finalize() error {
 	zw.EndStream()
 	metaSize := zw.Position()
 	// Header
-	if _, err := w.writer.Write(Header{Version, uint32(metaSize), uint32(dataSize)}.Serialize()); err != nil {
+	if _, err := w.writer.Write(Header{Version, uint64(metaSize), dataSize}.Serialize()); err != nil {
 		return fmt.Errorf("system error: could not write VNG header: %w", err)
 	}
 	// Metadata section

vng/ztests/const.yaml

@@ -12,5 +12,5 @@ inputs:
 outputs:
   - name: stdout
     data: |
-      {Version:4(uint32),MetaSize:35(uint32),DataSize:0(uint32)}
+      {Version:4(uint32),MetaSize:35(uint64),DataSize:0(uint64)}
       {Value:1,Count:3(uint32)}(=Const)