Merge pull request #352 from onflow/fxamacker/array-map-inlining-tweaks

Update for Cadence integration for atree inlining and deduplication

array.go

@@ -132,6 +132,7 @@ func (a *ArrayDataSlab) StoredValue(storage SlabStorage) (Value, error) {
 }
 
 var _ ArraySlab = &ArrayDataSlab{}
+var _ ContainerStorable = &ArrayDataSlab{}
 
 // ArrayMetaDataSlab is internal node, implementing ArraySlab.
 type ArrayMetaDataSlab struct {
@@ -697,14 +698,14 @@ func DecodeInlinedArrayStorable(
 	}, nil
 }
 
-// encodeAsInlined encodes inlined array data slab. Encoding is
+// EncodeAsElement encodes inlined array data slab. Encoding is
 // version 1 with CBOR tag having tag number CBORTagInlinedArray,
 // and tag contant as 3-element array:
 //
 //	+------------------+----------------+----------+
 //	| extra data index | value ID index | elements |
 //	+------------------+----------------+----------+
-func (a *ArrayDataSlab) encodeAsInlined(enc *Encoder, inlinedTypeInfo *inlinedExtraData) error {
+func (a *ArrayDataSlab) EncodeAsElement(enc *Encoder, inlinedTypeInfo *InlinedExtraData) error {
 	if a.extraData == nil {
 		return NewEncodingError(
 			fmt.Errorf("failed to encode non-root array data slab as inlined"))
@@ -754,7 +755,8 @@ func (a *ArrayDataSlab) encodeAsInlined(enc *Encoder, inlinedTypeInfo *inlinedEx
 	// element 2: array elements
 	err = a.encodeElements(enc, inlinedTypeInfo)
 	if err != nil {
-		return NewEncodingError(err)
+		// err is already categorized by ArrayDataSlab.encodeElements().
+		return err
 	}
 
 	err = enc.CBOR.Flush()
@@ -817,7 +819,7 @@ func (a *ArrayDataSlab) Encode(enc *Encoder) error {
 		return NewEncodingError(err)
 	}
 
-	if a.hasPointer() {
+	if a.HasPointer() {
 		h.setHasPointers()
 	}
 
@@ -885,7 +887,7 @@ func (a *ArrayDataSlab) Encode(enc *Encoder) error {
 	return nil
 }
 
-func (a *ArrayDataSlab) encodeElements(enc *Encoder, inlinedTypeInfo *inlinedExtraData) error {
+func (a *ArrayDataSlab) encodeElements(enc *Encoder, inlinedTypeInfo *InlinedExtraData) error {
 	// Encode CBOR array size manually for fix-sized encoding
 
 	enc.Scratch[0] = 0x80 | 25
@@ -906,10 +908,10 @@ func (a *ArrayDataSlab) encodeElements(enc *Encoder, inlinedTypeInfo *inlinedExt
 
 	// Encode data slab content (array of elements)
 	for _, e := range a.elements {
-		err = encodeStorableAsElement(enc, e, inlinedTypeInfo)
+		err = EncodeStorableAsElement(enc, e, inlinedTypeInfo)
 		if err != nil {
-			// Wrap err as external error (if needed) because err is returned by Storable interface.
-			return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode array element")
+			// err is already categorized by encodeStorableAsElement().
+			return err
 		}
 	}
 
@@ -1000,7 +1002,7 @@ func (a *ArrayDataSlab) Uninline(storage SlabStorage) error {
 	return nil
 }
 
-func (a *ArrayDataSlab) hasPointer() bool {
+func (a *ArrayDataSlab) HasPointer() bool {
 	for _, e := range a.elements {
 		if hasPointer(e) {
 			return true

array_debug.go

@@ -345,8 +345,16 @@ func (v *arrayVerifier) verifyDataSlab(
 	}
 
 	// Verify that only root data slab can be inlined
-	if level > 0 && dataSlab.Inlined() {
-		return 0, nil, nil, NewFatalError(fmt.Errorf("non-root slab %s is inlined", id))
+	if dataSlab.Inlined() {
+		if level > 0 {
+			return 0, nil, nil, NewFatalError(fmt.Errorf("non-root slab %s is inlined", id))
+		}
+		if dataSlab.extraData == nil {
+			return 0, nil, nil, NewFatalError(fmt.Errorf("inlined slab %s doesn't have extra data", id))
+		}
+		if dataSlab.next != SlabIDUndefined {
+			return 0, nil, nil, NewFatalError(fmt.Errorf("inlined slab %s has next slab ID", id))
+		}
 	}
 
 	// Verify that aggregated element size + slab prefix is the same as header.size
@@ -524,6 +532,11 @@ func VerifyArraySerialization(
 	decodeTypeInfo TypeInfoDecoder,
 	compare StorableComparator,
 ) error {
+	// Skip verification of inlined array serialization.
+	if a.Inlined() {
+		return nil
+	}
+
 	v := &serializationVerifier{
 		storage:        a.Storage,
 		cborDecMode:    cborDecMode,
@@ -550,7 +563,7 @@ func (v *serializationVerifier) verifyArraySlab(slab ArraySlab) error {
 	id := slab.SlabID()
 
 	// Encode slab
-	data, err := Encode(slab, v.cborEncMode)
+	data, err := EncodeSlab(slab, v.cborEncMode)
 	if err != nil {
 		// Don't need to wrap error as external error because err is already categorized by Encode().
 		return err
@@ -564,7 +577,7 @@ func (v *serializationVerifier) verifyArraySlab(slab ArraySlab) error {
 	}
 
 	// Re-encode decoded slab
-	dataFromDecodedSlab, err := Encode(decodedSlab, v.cborEncMode)
+	dataFromDecodedSlab, err := EncodeSlab(decodedSlab, v.cborEncMode)
 	if err != nil {
 		// Don't need to wrap error as external error because err is already categorized by Encode().
 		return err

array_test.go

@@ -142,6 +142,30 @@ func _testArray(
 	require.Equal(t, 1, len(rootIDs))
 	require.Equal(t, array.SlabID(), rootIDs[0])
 
+	// Encode all non-nil slab
+	encodedSlabs := make(map[SlabID][]byte)
+	for id, slab := range storage.deltas {
+		if slab != nil {
+			b, err := EncodeSlab(slab, storage.cborEncMode)
+			require.NoError(t, err)
+			encodedSlabs[id] = b
+		}
+	}
+
+	// Test decoded array from new storage to force slab decoding
+	decodedArray, err := NewArrayWithRootID(
+		newTestPersistentStorageWithBaseStorageAndDeltas(t, storage.baseStorage, encodedSlabs),
+		array.SlabID())
+	require.NoError(t, err)
+
+	// Verify decoded array elements
+	for i, expected := range expectedValues {
+		actual, err := decodedArray.Get(uint64(i))
+		require.NoError(t, err)
+
+		valueEqual(t, expected, actual)
+	}
+
 	if !hasNestedArrayMapElement {
 		// Need to call Commit before calling storage.Count() for PersistentSlabStorage.
 		err = storage.Commit()

cmd/stress/typeinfo.go

@@ -53,7 +53,7 @@ func (i arrayTypeInfo) IsComposite() bool {
 	return false
 }
 
-func (i arrayTypeInfo) ID() string {
+func (i arrayTypeInfo) Identifier() string {
 	return fmt.Sprintf("array(%d)", i)
 }
 
@@ -88,7 +88,7 @@ func (i mapTypeInfo) IsComposite() bool {
 	return false
 }
 
-func (i mapTypeInfo) ID() string {
+func (i mapTypeInfo) Identifier() string {
 	return fmt.Sprintf("map(%d)", i)
 }
 
@@ -153,7 +153,7 @@ func (i compositeTypeInfo) IsComposite() bool {
 	return true
 }
 
-func (i compositeTypeInfo) ID() string {
+func (i compositeTypeInfo) Identifier() string {
 	return fmt.Sprintf("composite(%d_%d)", i.fieldStartIndex, i.fieldEndIndex)
 }
 

cmd/stress/utils.go

@@ -540,5 +540,5 @@ func (v mapValue) Storable(atree.SlabStorage, atree.Address, uint64) (atree.Stor
 }
 
 var typeInfoComparator = func(a atree.TypeInfo, b atree.TypeInfo) bool {
-	return a.ID() == b.ID()
+	return a.Identifier() == b.Identifier()
 }

encode.go

@@ -42,23 +42,24 @@ func NewEncoder(w io.Writer, encMode cbor.EncMode) *Encoder {
 	}
 }
 
-// encodeStorableAsElement encodes storable as Array or OrderedMap element.
+// EncodeStorableAsElement encodes storable as Array or OrderedMap element.
 // Storable is encode as an inlined ArrayDataSlab or MapDataSlab if it is ArrayDataSlab or MapDataSlab.
-func encodeStorableAsElement(enc *Encoder, storable Storable, inlinedTypeInfo *inlinedExtraData) error {
+func EncodeStorableAsElement(enc *Encoder, storable Storable, inlinedTypeInfo *InlinedExtraData) error {
 
 	switch storable := storable.(type) {
 
-	case *ArrayDataSlab:
-		return storable.encodeAsInlined(enc, inlinedTypeInfo)
-
-	case *MapDataSlab:
-		return storable.encodeAsInlined(enc, inlinedTypeInfo)
+	case ContainerStorable:
+		err := storable.EncodeAsElement(enc, inlinedTypeInfo)
+		if err != nil {
+			// Wrap err as external error (if needed) because err is returned by ContainerStorable interface.
+			return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode container storable as element")
+		}
 
 	default:
 		err := storable.Encode(enc)
 		if err != nil {
 			// Wrap err as external error (if needed) because err is returned by Storable interface.
-			return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode map value")
+			return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode storable as element")
 		}
 	}