Merge branch 'next-major' of github.com:realm/realm-core into nc/merg…

…e_all_together

src/realm/array.cpp

@@ -245,7 +245,6 @@ struct Array::VTableForEncodedArray {
 template <size_t width>
 const typename Array::VTableForWidth<width>::PopulatedVTable Array::VTableForWidth<width>::vtable;
 const typename Array::VTableForEncodedArray::PopulatedVTableEncoded Array::VTableForEncodedArray::vtable;
-
 void Array::init_from_mem(MemRef mem) noexcept
 {
     // Header is the type of header that has been allocated, in case we are decompressing,
@@ -358,73 +357,6 @@ void Array::destroy_children(size_t offset) noexcept
 //     return num_bytes;
 // }
 
-ref_type Array::write(_impl::ArrayWriterBase& out, bool deep, bool only_if_modified, bool compress_in_flight) const
-{
-    REALM_ASSERT_DEBUG(is_attached());
-    // The default allocator cannot be trusted wrt is_read_only():
-    REALM_ASSERT_DEBUG(!only_if_modified || &m_alloc != &Allocator::get_default());
-    if (only_if_modified && m_alloc.is_read_only(m_ref))
-        return m_ref;
-
-    if (!deep || !m_has_refs) {
-        // however - creating an array using ANYTHING BUT the default allocator during commit is also wrong....
-        // it only works by accident, because the whole slab area is reinitialized after commit.
-        // We should have: Array encoded_array{Allocator::get_default()};
-        Array compressed_array{Allocator::get_default()};
-        if (compress_in_flight && size() != 0 && compress_array(compressed_array)) {
-#ifdef REALM_DEBUG
-            const auto encoding = compressed_array.m_integer_compressor.get_encoding();
-            REALM_ASSERT_DEBUG(encoding == Encoding::Flex || encoding == Encoding::Packed);
-            REALM_ASSERT_DEBUG(size() == compressed_array.size());
-            for (size_t i = 0; i < compressed_array.size(); ++i) {
-                REALM_ASSERT_DEBUG(get(i) == compressed_array.get(i));
-            }
-#endif
-            auto ref = compressed_array.do_write_shallow(out);
-            compressed_array.destroy();
-            return ref;
-        }
-        return do_write_shallow(out); // Throws
-    }
-
-    return do_write_deep(out, only_if_modified, compress_in_flight); // Throws
-}
-
-ref_type Array::write(ref_type ref, Allocator& alloc, _impl::ArrayWriterBase& out, bool only_if_modified,
-                      bool compress_in_flight)
-{
-    // The default allocator cannot be trusted wrt is_read_only():
-    REALM_ASSERT_DEBUG(!only_if_modified || &alloc != &Allocator::get_default());
-    if (only_if_modified && alloc.is_read_only(ref))
-        return ref;
-
-    Array array(alloc);
-    array.init_from_ref(ref);
-    REALM_ASSERT_DEBUG(array.is_attached());
-
-    if (!array.m_has_refs) {
-        Array compressed_array{Allocator::get_default()};
-        if (compress_in_flight && array.size() != 0 && array.compress_array(compressed_array)) {
-#ifdef REALM_DEBUG
-            const auto encoding = compressed_array.m_integer_compressor.get_encoding();
-            REALM_ASSERT_DEBUG(encoding == Encoding::Flex || encoding == Encoding::Packed);
-            REALM_ASSERT_DEBUG(array.size() == compressed_array.size());
-            for (size_t i = 0; i < compressed_array.size(); ++i) {
-                REALM_ASSERT_DEBUG(array.get(i) == compressed_array.get(i));
-            }
-#endif
-            auto ref = compressed_array.do_write_shallow(out);
-            compressed_array.destroy();
-            return ref;
-        }
-        else {
-            return array.do_write_shallow(out); // Throws
-        }
-    }
-    return array.do_write_deep(out, only_if_modified, compress_in_flight); // Throws
-}
-
-
 ref_type Array::do_write_shallow(_impl::ArrayWriterBase& out) const
 {
     // here we might want to compress the array and write down.
@@ -1228,11 +1160,14 @@ void Array::typed_print(std::string prefix) const
     }
     else {
         std::cout << " Leaf of unknown type }" << std::endl;
-        /*
-        for (unsigned n = 0; n < size(); ++n) {
-            auto pref = prefix + to_string(n) + ":\t";
-            std::cout << pref << get(n) << std::endl;
-        }
-        */
     }
 }
+
+ref_type ArrayPayload::typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc)
+{
+    Array arr(alloc);
+    arr.init_from_ref(ref);
+    // By default we are not compressing
+    constexpr bool compress = false;
+    return arr.write(out, true, out.only_modified, compress);
+}

src/realm/array.hpp

@@ -497,7 +497,7 @@ class Array : public Node, public ArrayParent {
 
     /// Takes a 64-bit value and returns the minimum number of bits needed
     /// to fit the value. For alignment this is rounded up to nearest
-    /// log2. Posssible results {0, 1, 2, 4, 8, 16, 32, 64}
+    /// log2. Possible results {0, 1, 2, 4, 8, 16, 32, 64}
     static size_t bit_width(int64_t value);
 
     void typed_print(std::string prefix) const;
@@ -614,6 +614,23 @@ class Array : public Node, public ArrayParent {
     friend class FlexCompressor;
 };
 
+class TempArray : public Array {
+public:
+    TempArray(size_t sz, Type type = Type::type_HasRefs)
+        : Array(Allocator::get_default())
+    {
+        create(type, false, sz);
+    }
+    ~TempArray()
+    {
+        destroy();
+    }
+    ref_type write(_impl::ArrayWriterBase& out)
+    {
+        return Array::write(out, false, false, false);
+    }
+};
+
 // Implementation:
 
 inline bool Array::is_compressed() const
@@ -1032,6 +1049,73 @@ inline void Array::ensure_minimum_width(int_fast64_t value)
     do_ensure_minimum_width(value);
 }
 
+inline ref_type Array::write(_impl::ArrayWriterBase& out, bool deep, bool only_if_modified,
+                             bool compress_in_flight) const
+{
+    REALM_ASSERT_DEBUG(is_attached());
+    // The default allocator cannot be trusted wrt is_read_only():
+    REALM_ASSERT_DEBUG(!only_if_modified || &m_alloc != &Allocator::get_default());
+    if (only_if_modified && m_alloc.is_read_only(m_ref))
+        return m_ref;
+
+    if (!deep || !m_has_refs) {
+        // however - creating an array using ANYTHING BUT the default allocator during commit is also wrong....
+        // it only works by accident, because the whole slab area is reinitialized after commit.
+        // We should have: Array encoded_array{Allocator::get_default()};
+        Array compressed_array{Allocator::get_default()};
+        if (compress_in_flight && size() != 0 && compress_array(compressed_array)) {
+#ifdef REALM_DEBUG
+            const auto encoding = compressed_array.m_integer_compressor.get_encoding();
+            REALM_ASSERT_DEBUG(encoding == Encoding::Flex || encoding == Encoding::Packed);
+            REALM_ASSERT_DEBUG(size() == compressed_array.size());
+            for (size_t i = 0; i < compressed_array.size(); ++i) {
+                REALM_ASSERT_DEBUG(get(i) == compressed_array.get(i));
+            }
+#endif
+            auto ref = compressed_array.do_write_shallow(out);
+            compressed_array.destroy();
+            return ref;
+        }
+        return do_write_shallow(out); // Throws
+    }
+
+    return do_write_deep(out, only_if_modified, compress_in_flight); // Throws
+}
+
+inline ref_type Array::write(ref_type ref, Allocator& alloc, _impl::ArrayWriterBase& out, bool only_if_modified,
+                             bool compress_in_flight)
+{
+    // The default allocator cannot be trusted wrt is_read_only():
+    REALM_ASSERT_DEBUG(!only_if_modified || &alloc != &Allocator::get_default());
+    if (only_if_modified && alloc.is_read_only(ref))
+        return ref;
+
+    Array array(alloc);
+    array.init_from_ref(ref);
+    REALM_ASSERT_DEBUG(array.is_attached());
+
+    if (!array.m_has_refs) {
+        Array compressed_array{Allocator::get_default()};
+        if (compress_in_flight && array.size() != 0 && array.compress_array(compressed_array)) {
+#ifdef REALM_DEBUG
+            const auto encoding = compressed_array.m_integer_compressor.get_encoding();
+            REALM_ASSERT_DEBUG(encoding == Encoding::Flex || encoding == Encoding::Packed);
+            REALM_ASSERT_DEBUG(array.size() == compressed_array.size());
+            for (size_t i = 0; i < compressed_array.size(); ++i) {
+                REALM_ASSERT_DEBUG(array.get(i) == compressed_array.get(i));
+            }
+#endif
+            auto ref = compressed_array.do_write_shallow(out);
+            compressed_array.destroy();
+            return ref;
+        }
+        else {
+            return array.do_write_shallow(out); // Throws
+        }
+    }
+    return array.do_write_deep(out, only_if_modified, compress_in_flight); // Throws
+}
+
 
 } // namespace realm
 

src/realm/array_integer.hpp

@@ -64,6 +64,14 @@ class ArrayInteger : public Array, public ArrayPayload {
     }
     template <class cond>
     bool find(value_type value, size_t start, size_t end, QueryStateBase* state) const;
+
+    template <class T>
+    static ref_type typed_write(ref_type ref, T& out, Allocator& alloc)
+    {
+        Array arr(alloc);
+        arr.init_from_ref(ref);
+        return arr.write(out, false, out.only_modified, out.compress);
+    }
 };
 
 class ArrayIntNull : public Array, public ArrayPayload {
@@ -133,6 +141,14 @@ class ArrayIntNull : public Array, public ArrayPayload {
 
     size_t find_first(value_type value, size_t begin = 0, size_t end = npos) const;
 
+    template <class T>
+    static ref_type typed_write(ref_type ref, T& out, Allocator& alloc)
+    {
+        Array arr(alloc);
+        arr.init_from_ref(ref);
+        return arr.write(out, false, out.only_modified, out.compress);
+    }
+
 protected:
     void avoid_null_collision(int64_t value);
 

src/realm/array_mixed.cpp

@@ -18,6 +18,8 @@
 
 #include <realm/array_mixed.hpp>
 #include <realm/array_basic.hpp>
+#include <realm/dictionary.hpp>
+#include <realm/impl/array_writer.hpp>
 
 using namespace realm;
 
@@ -328,6 +330,89 @@ void ArrayMixed::verify() const
     // TODO: Implement
 }
 
+ref_type ArrayMixed::typed_write(ref_type top_ref, _impl::ArrayWriterBase& out, Allocator& alloc)
+{
+    if (out.only_modified && alloc.is_read_only(top_ref))
+        return top_ref;
+
+    ArrayRef top(alloc);
+    top.init_from_ref(top_ref);
+    size_t sz = top.size();
+    TempArray written_leaf(sz);
+
+    /*
+     Mixed stores things using different arrays. We need to take into account this in order to
+     understand what we need to compress and what we can instead leave not compressed.
+
+     The main subarrays are:
+
+     composite array : index 0
+     int array : index 1
+     pair_int array: index 2
+     string array: index 3
+     ref array: index 4
+     key array: index 5
+
+     Description of each array:
+     1. composite array: the data stored here is either a small int (< 32 bits) or an offset to one of
+        the other arrays where the actual data is.
+     2. int and pair int arrays, they are used for storing integers, timestamps, floats, doubles,
+     decimals, links. In general we can compress them, but we need to be careful, controlling the col_type
+     should prevent compressing data that we want to leave in the current format.
+     3. string array is for strings and binary data (no compression for now)
+     4. ref array is actually storing refs to collections. they can only be BPlusTree<int, Mixed> or
+     BPlusTree<string, Mixed>.
+     5. key array stores unique identifiers for collections in mixed (integers that can be compressed)
+     */
+    Array composite(alloc);
+    composite.init_from_ref(top.get_as_ref(0));
+    written_leaf.set_as_ref(0, composite.write(out, true, out.only_modified, false));
+    for (size_t i = 1; i < sz; ++i) {
+        auto ref = top.get(i);
+        ref_type new_ref = ref;
+        if (ref && !(out.only_modified && alloc.is_read_only(ref))) {
+            if (i < 3) { // int, and pair_int
+                // integer arrays
+                new_ref = Array::write(ref, alloc, out, out.only_modified, out.compress);
+            }
+            else if (i == 4) { // collection in mixed
+                ArrayRef arr_ref(alloc);
+                arr_ref.init_from_ref(ref);
+                auto ref_sz = arr_ref.size();
+                TempArray written_ref_leaf(ref_sz);
+
+                for (size_t k = 0; k < ref_sz; k++) {
+                    ref_type new_sub_ref = 0;
+                    if (auto sub_ref = arr_ref.get(k)) {
+                        auto header = alloc.translate(sub_ref);
+                        // Now we have to find out if the nested collection is a
+                        // dictionary or a list. If the top array has a size of 2
+                        // and it is not a BplusTree inner node, then it is a dictionary
+                        if (NodeHeader::get_size_from_header(header) == 2 &&
+                            !NodeHeader::get_is_inner_bptree_node_from_header(header)) {
+                            new_sub_ref = Dictionary::typed_write(sub_ref, out, alloc);
+                        }
+                        else {
+                            new_sub_ref = BPlusTree<Mixed>::typed_write(sub_ref, out, alloc);
+                        }
+                    }
+                    written_ref_leaf.set_as_ref(k, new_sub_ref);
+                }
+                new_ref = written_ref_leaf.write(out);
+            }
+            else if (i == 5) { // unique keys associated to collections in mixed
+                new_ref = Array::write(ref, alloc, out, out.only_modified, out.compress);
+            }
+            else {
+                // all the rest we don't want to compress it, at least for now (strings will be needed)
+                new_ref = Array::write(ref, alloc, out, out.only_modified, false);
+            }
+        }
+        written_leaf.set(i, new_ref);
+    }
+    return written_leaf.write(out);
+}
+
 void ArrayMixed::ensure_array_accessor(Array& arr, size_t ndx_in_parent) const
 {
     if (!arr.is_attached()) {

src/realm/array_mixed.hpp

@@ -100,6 +100,7 @@ class ArrayMixed : public ArrayPayload, private Array {
     int64_t get_key(size_t ndx) const;
 
     void verify() const;
+    static ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& outs, Allocator& alloc);
 
 private:
     enum {

src/realm/array_timestamp.cpp

@@ -18,6 +18,7 @@
 
 #include <realm/array_timestamp.hpp>
 #include <realm/array_integer_tpl.hpp>
+#include <realm/impl/array_writer.hpp>
 
 using namespace realm;
 
@@ -244,4 +245,25 @@ void ArrayTimestamp::verify() const
     REALM_ASSERT(m_seconds.size() == m_nanoseconds.size());
 #endif
 }
+
+ref_type ArrayTimestamp::typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc)
+{
+    // timestamps could be compressed, but the formats we support at the moment are not producing
+    // noticeable gains.
+    Array top(alloc);
+    top.init_from_ref(ref);
+    REALM_ASSERT_DEBUG(top.size() == 2);
+
+    TempArray written_top(2);
+
+    auto rot0 = top.get_as_ref_or_tagged(0);
+    auto rot1 = top.get_as_ref_or_tagged(1);
+    REALM_ASSERT_DEBUG(rot0.is_ref() && rot0.get_as_ref());
+    REALM_ASSERT_DEBUG(rot1.is_ref() && rot1.get_as_ref());
+    written_top.set_as_ref(0, Array::write(rot0.get_as_ref(), alloc, out, out.only_modified, false));
+    written_top.set_as_ref(1, Array::write(rot1.get_as_ref(), alloc, out, out.only_modified, false));
+
+    return written_top.write(out);
+}
+
 } // namespace realm

src/realm/array_timestamp.hpp

@@ -109,6 +109,7 @@ class ArrayTimestamp : public ArrayPayload, private Array {
     size_t find_first(Timestamp value, size_t begin, size_t end) const noexcept;
 
     void verify() const;
+    static ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc);
 
 private:
     ArrayIntNull m_seconds;