fix(datasets): BI-5245 fix source refresh with deleted fields, do not…

… force refresh always (#604)

lib/dl_api_lib/dl_api_lib/dataset/validator.py

@@ -1098,12 +1098,15 @@ def get_schema_info(exists: bool) -> Optional[SchemaInfo]:
 
         self._reload_sources()
 
-        if len(new_raw_schema) != len(self._ds.result_schema.get_direct_fields_by_source(source_id)):
-            force_update_fields = True
-        if not self._are_schemas_identical(new_raw_schema, old_raw_schema) or force_update_fields:
+        avatar_ids = [avatar.id for avatar in self._ds_accessor.get_avatar_list(source_id=source_id)]
+        new_direct_result_fields = self._ds.result_schema.get_direct_fields_for_avatars(avatar_ids)
+        if (
+            not self._are_schemas_identical(new_raw_schema, old_raw_schema)
+            or len(new_raw_schema) != len(new_direct_result_fields)
+            or force_update_fields
+        ):
             # try to match old and new schemas against each other
             # and update result_schema fields accordingly
-            avatar_ids = [avatar.id for avatar in self._ds_accessor.get_avatar_list(source_id=source_id)]
             self._update_direct_fields_for_updated_raw_schema(
                 old_raw_schema=old_raw_schema,  # type: ignore  # TODO: fix
                 new_raw_schema=new_raw_schema,

lib/dl_core/dl_core/fields.py

@@ -475,9 +475,9 @@ def remove_multiple(self, field_ids: Collection[str]) -> None:
             self.fields.remove(field)
         self.clear_caches()
 
-    def get_direct_fields_by_source(self, source_id: str) -> List[BIField]:
+    def get_direct_fields_for_avatars(self, avatar_ids: Collection[str]) -> List[BIField]:
         return [
             field
             for field in self.fields
-            if isinstance(field.calc_spec, DirectCalculationSpec) and field.source == source_id
+            if isinstance(field.calc_spec, DirectCalculationSpec) and field.avatar_id in avatar_ids
         ]