Merge branch 'master' of github.com:realm/realm-core into mwb/configu…

…rable-retry-timers
realm · Mar 13, 2024 · cb2a1c3 · cb2a1c3
2 parents abd0b78 + 687bb98
commit cb2a1c3
Show file tree

Hide file tree

Showing 64 changed files with 3,229 additions and 1,034 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,23 +1,27 @@
 # NEXT RELEASE
 
 ### Enhancements
-* <New feature description> (PR [#????](https://github.com/realm/realm-core/pull/????))
+* Add support to synchronize collections embedded in Mixed properties and other collections (except sets) ([PR #7353](https://github.com/realm/realm-core/pull/7353)).
+* Improve performance of change notifications on nested collections somewhat ([PR #7402](https://github.com/realm/realm-core/pull/7402)).
 * Added Resumption delay configuration to SyncClientTimeouts. ([PR #7441](https://github.com/realm/realm-core/pull/7441))
 
 ### Fixed
-* <How do the end-user experience this issue? what was the impact?> ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?)
-* None.
+* Fixed conflict resolution bug which may result in an crash when the AddInteger instruction on Mixed properties is merged against updates to a non-integer type ([PR #7353](https://github.com/realm/realm-core/pull/7353)).
+* Fix a spurious crash related to opening a Realm on background thread while the process was in the middle of exiting ([#7420](https://github.com/realm/realm-core/issues/7420jj))
+* Fix a data race in change notification delivery when running at debug log level ([PR #7402](https://github.com/realm/realm-core/pull/7402), since v14.0.0).
+* Fix a 10-15% performance regression when reading data from the Realm resulting from Obj being made a non-trivial type ([PR #7402](https://github.com/realm/realm-core/pull/7402), since v14.0.0).
 
 ### Breaking changes
-* None.
+* Remove `realm_scheduler_set_default_factory()` and `realm_scheduler_has_default_factory()`, and change the `Scheduler` factory function to a bare function pointer rather than a `UniqueFunction` so that it does not have a non-trivial destructor.
 
 ### Compatibility
 * Fileformat: Generates files with format v24. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier.
 
 -----------
 
 ### Internals
-* None.
+* The CMake option `REALM_MONGODB_ENDPOINT` for running the object-store-tests against baas has been deprecated in favor of an environment variable of the same name ([PR #7423](https://github.com/realm/realm-core/pull/7423)).
+* The object-store-tests test suite can now launch baas containers on its own by specifying a `BAASAAS_API_KEY` in the environment ([PR #7423](https://github.com/realm/realm-core/pull/7423)).
 
 ----------------------------------------------
 

diff --git a/dependencies.yml b/dependencies.yml
@@ -3,5 +3,5 @@ VERSION: 14.2.0
 OPENSSL_VERSION: 3.2.0
 ZLIB_VERSION: 1.2.13
 # https://github.com/10gen/baas/commits
-# dd016 is 2024 Feb 22
-BAAS_VERSION: dd01629d83b86292af9c59ebe2a28673c2e559cf
+# acb71d0 is 2024 Mar 12
+BAAS_VERSION: acb71d0183b33eb304bb496390567efcfb8a6e60
diff --git a/evergreen/config.yml b/evergreen/config.yml
@@ -78,7 +78,6 @@ functions:
           if [ -z "${disable_tests_against_baas|}" ]; then
               scheme="http"
               set_cmake_var baas_vars REALM_ENABLE_AUTH_TESTS BOOL On
-              set_cmake_var baas_vars REALM_MONGODB_ENDPOINT STRING "$scheme://localhost:9090"
               if [ -n "${baas_admin_port|}" ]; then
                 set_cmake_var baas_vars REALM_ADMIN_ENDPOINT STRING "$scheme://localhost:${baas_admin_port}"
               fi
@@ -156,7 +155,7 @@ functions:
           if [ -n "${curl_base|}" ]; then
               set_cmake_var curl_vars CURL_LIBRARY PATH "$(./evergreen/abspath.sh ${curl_base}/lib/libcurl.dll.a)"
               set_cmake_var curl_vars CURL_INCLUDE_DIR PATH "$(./evergreen/abspath.sh ${curl_base}/include)"
-              set_cmake_var baas_vars REALM_CURL_CACERTS PATH "$(./evergreen/abspath.sh "${curl_base}/bin/cacert.pem")"
+              set_cmake_var baas_vars REALM_CURL_CACERTS PATH "$(./evergreen/abspath.sh "${curl_base}/bin/curl-ca-bundle.crt")"
           fi
 
           set_cmake_var realm_vars REALM_NO_TESTS BOOL ${no_tests|Off}
@@ -208,10 +207,17 @@ functions:
         file: './realm-core/benchmark_results/results.latest.json'
 
   "run tests":
+    - command: expansions.update
+      params:
+        file: realm-core/dependencies.yml
     - command: shell.exec
       params:
         working_dir: realm-core
         shell: bash
+        env:
+          BAASAAS_API_KEY: "${baasaas_api_key}"
+          BAASAAS_REF_SPEC: "${BAAS_VERSION}"
+          BAASAAS_START_MODE: "githash"
         script: |-
           set -o errexit
           set -o verbose
@@ -244,6 +250,12 @@ functions:
 
           TEST_FLAGS="--no-tests=error $TEST_FLAGS ${test_flags|}"
 
+          if [[ -n "${disable_tests_against_baas|}" ]]; then
+              unset BAASAAS_API_KEY
+              unset BAASAAS_REF_SPEC
+              unset BAASAAS_START_MODE
+          fi
+
           if [[ -n "${llvm_symbolizer}" ]]; then
               export ASAN_SYMBOLIZER_PATH="$(./evergreen/abspath.sh ${llvm_symbolizer})"
           fi
@@ -977,18 +989,17 @@ tasks:
       test_label: objstore-local
       test_executable_name: "realm-object-store-tests"
       verbose_test_output: true
+      disable_tests_against_baas: true
   - func: "check branch state"
 
 # These are baas object store tests that run against baas running on a remote host
 - name: baas-integration-tests
   tags: [ "test_suite", "for_pull_requests", "requires_baas" ]
   exec_timeout_secs: 3600
   commands:
-  - func: "launch remote baas"
   - func: "compile"
     vars:
       target_to_build: ObjectStoreTests
-  - func: "wait for remote baas to start"
   - func: "run tests"
     vars:
       test_label: objstore-baas

diff --git a/how-to-build.md b/how-to-build.md
@@ -145,39 +145,41 @@ These are the available variables:
    testing process as soon as a check fails or an unexpected exception is thrown
    in a test.
 
-## Running [app] tests against a local MongoDB BAAS
-
-Due to MongoDB security policies, running baas requires company issued AWS account credentials.
-These are for MongoDB employees only, if you do not have these, reach out to #realm-core.
-Once you have them, they need to be set in the shell environment.
-
-First, log in to aws using their command line tool. On mac this requries `brew install awscli`.
-Then login using `aws configure` and input your access key and secret acess key. The other 
-configuration options can be left as none. This creates a correctly formatted file locally at
-`~/.aws/credentials` which we will use later.
-
-If you do not want to install the aws command line tools, you can also create the aws file
-manually in the correct location (`~/.aws/credentials`) with the following contents:
+## Running [app] tests against a local BAAS instance
 
+If you already have a baas instance running, you can specify that directly via the
+`BAAS_BASE_URL` environment variable. You can run baas in a local docker container using
+instructions from [the wiki](https://wiki.corp.mongodb.com/display/10GEN/%28Device+Sync%29+Using+Docker+to+run+a+BAAS+server+instance).
 ```
-AWS_ACCESS_KEY_ID = <your-key-id>
-AWS_SECRET_ACCESS_KEY = <your-secret-key>
+export BAAS_BASE_URL=http://localhost:9090
+mkdir build.sync.ninja
+cmake -B build.sync.ninja -G Ninja -DREALM_ENABLE_AUTH_TESTS=1
+cmake --build build.sync.ninja --target realm-object-store-tests
+./build.sync.ninja/test/object-store/realm-object-store-tests -d=1
 ```
 
-We use a script to fetch the dependencies for and run baas locally. Use the `-b sha` to use a particular version from https://github.com/10gen/baas/
-The script uses the configuration from https://github.com/10gen/baas/blob/master/etc/configs/test_rcore_config.json
+## Running [app] tests against an on-demand BAASAAS container
 
+Due to MongoDB security policies, running baas requires company issued credentials.
+These are for MongoDB employees only, if you do not have these, reach out to
+#appx-device-sync-internal. Once you have a baasaas API key, it needs to be set
+in the shell environment.
 ```
-./evergreen/install_baas.sh -w baas
-```
-
-To run the [app] tests against the local baas, you need to configure a build with some cmake options to tell the tests where to point to.
-```
+export BAASAAS_API_KEY=<your API key here>
 mkdir build.sync.ninja
-cmake -B build.sync.ninja -G Ninja -DREALM_ENABLE_AUTH_TESTS=1 -DREALM_MONGODB_ENDPOINT=http://localhost:9090
+cmake -B build.sync.ninja -G Ninja -DREALM_ENABLE_AUTH_TESTS=1
 cmake --build build.sync.ninja --target realm-object-store-tests
 ./build.sync.ninja/test/object-store/realm-object-store-tests -d=1
 ```
+You can tell the object-store tests to use a specific version of baas with the
+`BAASAAS_START_MODE` environment variable, which can either be `githash`, `patchid`,
+or `branch`. If you specify a start mode, you need to tell it which githash or
+branch name to start with via the `BAASAAS_REF_SPEC` environment variable. Omitting
+these will use the latest available commit from the main branch of baas.
+
+If you've started a baasaas container already via the baasaas CLI, you can tell
+the object-store tests to use that with the `BAASAAS_INSTANCE_ID` environment variable.
+
 
 ### Developing inside a container
 

diff --git a/src/realm.h b/src/realm.h
@@ -953,33 +953,6 @@ RLM_API realm_scheduler_t* realm_scheduler_make_default(void);
  */
 RLM_API const realm_scheduler_t* realm_scheduler_get_frozen(void);
 
-/**
- * Returns true if there is a default scheduler implementation for the current
- * platform, or one has been set with `realm_scheduler_set_default_factory()`.
- *
- * If there is no default factory, and no scheduler is provided in the config,
- * `realm_open()` will fail. Note that `realm_scheduler_get_frozen()` always
- * returns a valid scheduler.
- *
- * This function is thread-safe, and cannot fail.
- */
-RLM_API bool realm_scheduler_has_default_factory(void);
-
-/**
- * For platforms with no default scheduler implementation, register a factory
- * function which can produce custom schedulers. If there is a platform-specific
- * scheduler, this function will fail. If a custom scheduler is desired for
- * platforms that already have a default scheduler implementation, the caller
- * must call `realm_open()` with a config that indicates the desired scheduler.
- *
- * The provided callback may produce a scheduler by calling
- * `realm_scheduler_new()`.
- *
- * This function is thread-safe, but should generally only be called once.
- */
-RLM_API bool realm_scheduler_set_default_factory(realm_userdata_t userdata, realm_free_userdata_func_t userdata_free,
-                                                 realm_scheduler_default_factory_func_t);
-
 /**
  * Open a Realm file.
  *

diff --git a/src/realm/collection.cpp b/src/realm/collection.cpp
@@ -237,4 +237,25 @@ void Collection::get_any(QueryCtrlBlock& ctrl, Mixed val, size_t index)
     }
 }
 
+UpdateStatus CollectionBase::do_init_from_parent(BPlusTreeBase* tree, ref_type ref, bool allow_create)
+{
+    if (ref) {
+        tree->init_from_ref(ref);
+    }
+    else {
+        if (tree->init_from_parent()) {
+            // All is well
+            return UpdateStatus::Updated;
+        }
+        if (!allow_create) {
+            tree->detach();
+            return UpdateStatus::Detached;
+        }
+        // The ref in the column was NULL, create the tree in place.
+        tree->create();
+        REALM_ASSERT(tree->is_attached());
+    }
+    return UpdateStatus::Updated;
+}
+
 } // namespace realm
diff --git a/src/realm/collection.hpp b/src/realm/collection.hpp
@@ -53,18 +53,18 @@ class DummyParent : public CollectionParent {
     {
         return m_obj;
     }
+    uint32_t parent_version() const noexcept final
+    {
+        return 0;
+    }
 
 protected:
     Obj m_obj;
     ref_type m_ref;
-    UpdateStatus update_if_needed_with_status() const final
+    UpdateStatus update_if_needed() const final
     {
         return UpdateStatus::Updated;
     }
-    bool update_if_needed() const final
-    {
-        return true;
-    }
     ref_type get_collection_ref(Index, CollectionType) const final
     {
         return m_ref;
@@ -255,6 +255,7 @@ class CollectionBase : public Collection {
     CollectionBase& operator=(CollectionBase&&) noexcept = default;
 
     void validate_index(const char* msg, size_t index, size_t size) const;
+    static UpdateStatus do_init_from_parent(BPlusTreeBase* tree, ref_type ref, bool allow_create);
 };
 
 inline std::string_view collection_type_name(CollectionType col_type, bool uppercase = false)
@@ -492,7 +493,7 @@ class CollectionBaseImpl : public Interface, protected ArrayParent {
         if (m_parent) {
             try {
                 // Update the parent. Will throw if parent is not existing.
-                switch (m_parent->update_if_needed_with_status()) {
+                switch (m_parent->update_if_needed()) {
                     case UpdateStatus::Updated:
                         // Make sure to update next time around
                         m_content_version = 0;
@@ -524,7 +525,7 @@ class CollectionBaseImpl : public Interface, protected ArrayParent {
     {
         try {
             // `has_changed()` sneakily modifies internal state.
-            update_if_needed_with_status();
+            update_if_needed();
             if (m_last_content_version != m_content_version) {
                 m_last_content_version = m_content_version;
                 return true;
@@ -563,24 +564,27 @@ class CollectionBaseImpl : public Interface, protected ArrayParent {
         m_content_version = 0;
     }
 
+    CollectionParent* get_owner() const noexcept
+    {
+        return m_parent;
+    }
+
     void to_json(std::ostream&, JSONOutputMode, util::FunctionRef<void(const Mixed&)>) const override;
 
     using Interface::get_owner_key;
     using Interface::get_table;
     using Interface::get_target_table;
 
 protected:
-    Obj m_obj_mem;
+    ObjCollectionParent m_obj_mem;
     std::shared_ptr<CollectionParent> m_col_parent;
     CollectionParent::Index m_index;
-    mutable size_t m_my_version = 0;
     ColKey m_col_key;
-    bool m_nullable = false;
-
     mutable uint_fast64_t m_content_version = 0;
-
     // Content version used by `has_changed()`.
     mutable uint_fast64_t m_last_content_version = 0;
+    mutable uint32_t m_parent_version = 0;
+    bool m_nullable = false;
 
     CollectionBaseImpl() = default;
     CollectionBaseImpl(const CollectionBaseImpl& other)
@@ -650,13 +654,14 @@ class CollectionBaseImpl : public Interface, protected ArrayParent {
 
     UpdateStatus get_update_status() const
     {
-        UpdateStatus status = m_parent ? m_parent->update_if_needed_with_status() : UpdateStatus::Detached;
+        UpdateStatus status = m_parent ? m_parent->update_if_needed() : UpdateStatus::Detached;
 
         if (status != UpdateStatus::Detached) {
             auto content_version = m_alloc->get_content_version();
-            if (content_version != m_content_version || m_my_version != m_parent->m_parent_version) {
+            auto parent_version = m_parent->parent_version();
+            if (content_version != m_content_version || m_parent_version != parent_version) {
                 m_content_version = content_version;
-                m_my_version = m_parent->m_parent_version;
+                m_parent_version = parent_version;
                 status = UpdateStatus::Updated;
             }
         }
@@ -667,18 +672,14 @@ class CollectionBaseImpl : public Interface, protected ArrayParent {
     /// Refresh the parent object (if needed) and compare version numbers.
     /// Return true if the collection should initialize from parent
     /// Throws if the owning object no longer exists.
-    bool should_update()
+    bool should_update() const
     {
         check_parent();
-        bool changed = m_parent->update_if_needed(); // Throws if the object does not exist.
-        auto content_version = m_alloc->get_content_version();
-
-        if (changed || content_version != m_content_version || m_my_version != m_parent->m_parent_version) {
-            m_content_version = content_version;
-            m_my_version = m_parent->m_parent_version;
-            return true;
+        auto status = get_update_status();
+        if (status == UpdateStatus::Detached) {
+            throw StaleAccessor("Parent no longer exists");
         }
-        return false;
+        return status == UpdateStatus::Updated;
     }
 
     void bump_content_version()
@@ -728,19 +729,19 @@ class CollectionBaseImpl : public Interface, protected ArrayParent {
     void set_backlink(ColKey col_key, ObjLink new_link) const
     {
         check_parent();
-        m_parent->set_backlink(col_key, new_link);
+        m_parent->get_object().set_backlink(col_key, new_link);
     }
     // Used when replacing a link, return true if CascadeState contains objects to remove
     bool replace_backlink(ColKey col_key, ObjLink old_link, ObjLink new_link, CascadeState& state) const
     {
         check_parent();
-        return m_parent->replace_backlink(col_key, old_link, new_link, state);
+        return m_parent->get_object().replace_backlink(col_key, old_link, new_link, state);
     }
     // Used when removing a backlink, return true if CascadeState contains objects to remove
     bool remove_backlink(ColKey col_key, ObjLink old_link, CascadeState& state) const
     {
         check_parent();
-        return m_parent->remove_backlink(col_key, old_link, state);
+        return m_parent->get_object().remove_backlink(col_key, old_link, state);
     }
 
     /// Reset the accessor's tracking of the content version. Derived classes
@@ -796,7 +797,7 @@ class CollectionBaseImpl : public Interface, protected ArrayParent {
     ///
     /// If no change has happened to the data, this function returns
     /// `UpdateStatus::NoChange`, and the caller is allowed to not do anything.
-    virtual UpdateStatus update_if_needed_with_status() const = 0;
+    virtual UpdateStatus update_if_needed() const = 0;
 };
 
 namespace _impl {
@@ -884,7 +885,7 @@ class ObjCollectionBase : public Interface, public _impl::ObjListProxy {
     /// `BPlusTree<T>`.
     virtual BPlusTree<ObjKey>* get_mutable_tree() const = 0;
 
-    /// Implements update_if_needed() in a way that ensures the consistency of
+    /// Implements `update_if_needed()` in a way that ensures the consistency of
     /// the unresolved list. Derived classes should call this instead of calling
     /// `update_if_needed()` on their inner accessor.
     UpdateStatus update_if_needed() const