[Do not review] New schema with reference separation

src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs

@@ -93,5 +93,6 @@ public enum SchemaVersion
         V81 = 81,
         V82 = 82,
         V83 = 83,
+        V84 = 84,
     }
 }

src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs

@@ -8,7 +8,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema
     public static class SchemaVersionConstants
     {
         public const int Min = (int)SchemaVersion.V80;
-        public const int Max = (int)SchemaVersion.V83;
+        public const int Max = (int)SchemaVersion.V84;
         public const int MinForUpgrade = (int)SchemaVersion.V80; // this is used for upgrade tests only
         public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6;
         public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7;

src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/Sequences.sql

@@ -13,3 +13,12 @@ CREATE SEQUENCE dbo.ResourceSurrogateIdUniquifierSequence
         CYCLE
         CACHE 1000000
 GO
+CREATE SEQUENCE dbo.ResourceIdIntMapSequence
+        AS int
+        START WITH 0
+        INCREMENT BY 1
+        MINVALUE 0
+        MAXVALUE 79999
+        CYCLE
+        CACHE 1000000
+GO

src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql

@@ -19,6 +19,6 @@ Go
 
 INSERT INTO dbo.SchemaVersion
 VALUES
-    (83, 'started')
+    (84, 'started')
 
 Go

src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/AssignResourceIdInts.sql

@@ -0,0 +1,29 @@
+CREATE PROCEDURE dbo.AssignResourceIdInts @Count int, @FirstIdInt bigint OUT
+AS
+set nocount on
+DECLARE @SP varchar(100) = 'AssignResourceIdInts'
+       ,@Mode varchar(200) = 'Cnt='+convert(varchar,@Count)
+       ,@st datetime = getUTCdate()
+       ,@FirstValueVar sql_variant
+       ,@LastValueVar sql_variant
+       ,@SequenceRangeFirstValue int
+
+BEGIN TRY
+  SET @FirstValueVar = NULL
+  WHILE @FirstValueVar IS NULL
+  BEGIN
+    EXECUTE sys.sp_sequence_get_range @sequence_name = 'dbo.ResourceIdIntMapSequence', @range_size = @Count, @range_first_value = @FirstValueVar OUT, @range_last_value = @LastValueVar OUT
+    SET @SequenceRangeFirstValue = convert(int,@FirstValueVar)
+    IF @SequenceRangeFirstValue > convert(int,@LastValueVar)
+      SET @FirstValueVar = NULL
+  END
+
+  SET @FirstIdInt = datediff_big(millisecond,'0001-01-01',sysUTCdatetime()) * 80000 + @SequenceRangeFirstValue
+END TRY
+BEGIN CATCH
+  IF error_number() = 1750 THROW -- Real error is before 1750, cannot trap in SQL.
+  IF @@trancount > 0 ROLLBACK TRANSACTION
+  EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error';
+  THROW
+END CATCH
+GO

src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/DisableIndexes.sql

@@ -39,7 +39,7 @@ BEGIN TRY
       WHERE NOT EXISTS (SELECT * FROM dbo.IndexProperties WHERE TableName = Tbl AND IndexName = Ind)
   EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Info',@Target='IndexProperties',@Action='Insert',@Rows=@@rowcount
 
-  DELETE FROM @Indexes WHERE Tbl = 'Resource' OR IndId = 1
+  DELETE FROM @Indexes WHERE Tbl IN ('Resource','ResourceCurrent','ResourceHistory') OR IndId = 1
   EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Info',@Target='@Indexes',@Action='Delete',@Rows=@@rowcount
 
   WHILE EXISTS (SELECT * FROM @Indexes)

src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourceVersions.sql

@@ -22,12 +22,17 @@ BEGIN TRY
                    END
         ,MatchedVersion = isnull(D.Version,0)
         ,MatchedRawResource = D.RawResource
+        ,MatchedTransactionId = D.TransactionId
+        ,MatchedOffsetInFile = D.OffsetInFile
         -- ResourceIndex allows to deal with more than one late arrival per resource 
-    FROM (SELECT TOP (@DummyTop) *, ResourceIndex = convert(int,row_number() OVER (PARTITION BY ResourceTypeId, ResourceId ORDER BY ResourceSurrogateId DESC)) FROM @ResourceDateKeys) A
-         OUTER APPLY (SELECT TOP 1 * FROM dbo.Resource B WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId_Version) WHERE B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId AND B.Version > 0 AND B.ResourceSurrogateId < A.ResourceSurrogateId ORDER BY B.ResourceSurrogateId DESC) L -- lower
-         OUTER APPLY (SELECT TOP 1 * FROM dbo.Resource B WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId_Version) WHERE B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId AND B.Version > 0 AND B.ResourceSurrogateId > A.ResourceSurrogateId ORDER BY B.ResourceSurrogateId) U -- upper
-         OUTER APPLY (SELECT TOP 1 * FROM dbo.Resource B WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId_Version) WHERE B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId AND B.Version < 0 ORDER BY B.Version) M -- minus
-         OUTER APPLY (SELECT TOP 1 * FROM dbo.Resource B WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId_Version) WHERE B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId AND B.ResourceSurrogateId BETWEEN A.ResourceSurrogateId AND A.ResourceSurrogateId + 79999) D -- date
+    FROM (SELECT TOP (@DummyTop) A.*, M.ResourceIdInt, ResourceIndex = convert(int,row_number() OVER (PARTITION BY A.ResourceTypeId, A.ResourceId ORDER BY ResourceSurrogateId DESC)) 
+            FROM @ResourceDateKeys A
+                 LEFT OUTER JOIN dbo.ResourceIdIntMap M WITH (INDEX = U_ResourceIdIntMap_ResourceId_ResourceTypeId) ON M.ResourceTypeId = A.ResourceTypeId AND M.ResourceId = A.ResourceId
+         ) A
+         OUTER APPLY (SELECT TOP 1 * FROM dbo.Resource B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.ResourceIdInt = A.ResourceIdInt AND B.Version > 0 AND B.ResourceSurrogateId < A.ResourceSurrogateId ORDER BY B.ResourceSurrogateId DESC) L -- lower
+         OUTER APPLY (SELECT TOP 1 * FROM dbo.Resource B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.ResourceIdInt = A.ResourceIdInt AND B.Version > 0 AND B.ResourceSurrogateId > A.ResourceSurrogateId ORDER BY B.ResourceSurrogateId) U -- upper
+         OUTER APPLY (SELECT TOP 1 * FROM dbo.Resource B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.ResourceIdInt = A.ResourceIdInt AND B.Version < 0 ORDER BY B.Version) M -- minus
+         OUTER APPLY (SELECT TOP 1 * FROM dbo.Resource B WHERE B.ResourceTypeId = A.ResourceTypeId AND B.ResourceIdInt = A.ResourceIdInt AND B.ResourceSurrogateId BETWEEN A.ResourceSurrogateId AND A.ResourceSurrogateId + 79999) D -- date
     OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
 
   EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@@rowcount

src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResources.sql

@@ -6,7 +6,6 @@ set nocount on
 DECLARE @st datetime = getUTCdate()
        ,@SP varchar(100) = 'GetResources'
        ,@InputRows int
-       ,@DummyTop bigint = 9223372036854775807
        ,@NotNullVersionExists bit 
        ,@NullVersionExists bit
        ,@MinRT smallint
@@ -22,57 +21,69 @@ BEGIN TRY
       SELECT B.ResourceTypeId
             ,B.ResourceId
             ,ResourceSurrogateId
-            ,B.Version
+            ,C.Version
             ,IsDeleted
             ,IsHistory
             ,RawResource
             ,IsRawResourceMetaSet
             ,SearchParamHash
-        FROM (SELECT TOP (@DummyTop) * FROM @ResourceKeys) A
-             JOIN dbo.Resource B WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId_Version) ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId AND B.Version = A.Version
-        OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
+            ,FileId
+            ,OffsetInFile
+        FROM (SELECT * FROM @ResourceKeys) A
+             INNER LOOP JOIN dbo.ResourceIdIntMap B WITH (INDEX = U_ResourceIdIntMap_ResourceId_ResourceTypeId) ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId
+             INNER LOOP JOIN dbo.Resource C ON C.ResourceTypeId = A.ResourceTypeId AND C.ResourceIdInt = B.ResourceIdInt AND C.Version = A.Version
+        OPTION (MAXDOP 1)
     ELSE
       SELECT *
         FROM (SELECT B.ResourceTypeId
                     ,B.ResourceId
                     ,ResourceSurrogateId
-                    ,B.Version
+                    ,C.Version
                     ,IsDeleted
                     ,IsHistory
                     ,RawResource
                     ,IsRawResourceMetaSet
                     ,SearchParamHash
-                FROM (SELECT TOP (@DummyTop) * FROM @ResourceKeys WHERE Version IS NOT NULL) A
-                     JOIN dbo.Resource B WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId_Version) ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId AND B.Version = A.Version
+                    ,FileId
+                    ,OffsetInFile
+                FROM (SELECT * FROM @ResourceKeys WHERE Version IS NOT NULL) A
+                     INNER LOOP JOIN dbo.ResourceIdIntMap B WITH (INDEX = U_ResourceIdIntMap_ResourceId_ResourceTypeId) ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId
+                     INNER LOOP JOIN dbo.Resource C ON C.ResourceTypeId = A.ResourceTypeId AND C.ResourceIdInt = B.ResourceIdInt AND C.Version = A.Version
               UNION ALL
               SELECT B.ResourceTypeId
                     ,B.ResourceId
-                    ,ResourceSurrogateId
-                    ,B.Version
+                    ,C.ResourceSurrogateId
+                    ,C.Version
                     ,IsDeleted
                     ,IsHistory
                     ,RawResource
                     ,IsRawResourceMetaSet
                     ,SearchParamHash
-                FROM (SELECT TOP (@DummyTop) * FROM @ResourceKeys WHERE Version IS NULL) A
-                     JOIN dbo.Resource B WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId) ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId
-                WHERE IsHistory = 0
+                    ,FileId
+                    ,OffsetInFile
+                FROM (SELECT * FROM @ResourceKeys WHERE Version IS NULL) A
+                     INNER LOOP JOIN dbo.ResourceIdIntMap B WITH (INDEX = U_ResourceIdIntMap_ResourceId_ResourceTypeId) ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId
+                     INNER LOOP JOIN dbo.CurrentResources C ON C.ResourceTypeId = A.ResourceTypeId AND C.ResourceIdInt = B.ResourceIdInt AND C.IsHistory = 0
+                     LEFT OUTER JOIN dbo.RawResources D ON D.ResourceTypeId = A.ResourceTypeId AND D.ResourceSurrogateId = C.ResourceSurrogateId
              ) A
-        OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
+        OPTION (MAXDOP 1)
   ELSE
     SELECT B.ResourceTypeId
           ,B.ResourceId
-          ,ResourceSurrogateId
-          ,B.Version
+          ,C.ResourceSurrogateId
+          ,C.Version
           ,IsDeleted
           ,IsHistory
           ,RawResource
           ,IsRawResourceMetaSet
           ,SearchParamHash
-      FROM (SELECT TOP (@DummyTop) * FROM @ResourceKeys) A
-           JOIN dbo.Resource B WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId) ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId
-      WHERE IsHistory = 0
-      OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
+          ,FileId
+          ,OffsetInFile
+      FROM (SELECT * FROM @ResourceKeys) A
+           INNER LOOP JOIN dbo.ResourceIdIntMap B WITH (INDEX = U_ResourceIdIntMap_ResourceId_ResourceTypeId) ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId
+           INNER LOOP JOIN dbo.CurrentResources C ON C.ResourceTypeId = A.ResourceTypeId AND C.ResourceIdInt = B.ResourceIdInt
+           LEFT OUTER JOIN dbo.RawResources D ON D.ResourceTypeId = A.ResourceTypeId AND D.ResourceSurrogateId = C.ResourceSurrogateId
+      OPTION (MAXDOP 1)
 
   EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@@rowcount
 END TRY
@@ -83,5 +94,5 @@ BEGIN CATCH
 END CATCH
 GO
 --DECLARE @ResourceKeys dbo.ResourceKeyList
---INSERT INTO @ResourceKeys SELECT TOP 1 ResourceTypeId, ResourceId, NULL FROM Resource
+--INSERT INTO @ResourceKeys SELECT 96, newid(), NULL
 --EXECUTE dbo.GetResources @ResourceKeys

src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTransactionId.sql

@@ -6,23 +6,8 @@ set nocount on
 DECLARE @SP varchar(100) = object_name(@@procid)
        ,@Mode varchar(100) = 'T='+convert(varchar,@TransactionId)+' H='+convert(varchar,@IncludeHistory)
        ,@st datetime = getUTCdate()
-       ,@DummyTop bigint = 9223372036854775807
-       ,@TypeId smallint
 
 BEGIN TRY
-  DECLARE @Types TABLE (TypeId smallint PRIMARY KEY, Name varchar(100))
-  INSERT INTO @Types EXECUTE dbo.GetUsedResourceTypes
-
-  DECLARE @Keys TABLE (TypeId smallint, SurrogateId bigint PRIMARY KEY (TypeId, SurrogateId))
-  WHILE EXISTS (SELECT * FROM @Types)
-  BEGIN
-    SET @TypeId = (SELECT TOP 1 TypeId FROM @Types ORDER BY TypeId)
-
-    INSERT INTO @Keys SELECT @TypeId, ResourceSurrogateId FROM dbo.Resource WHERE ResourceTypeId = @TypeId AND TransactionId = @TransactionId
-
-    DELETE FROM @Types WHERE TypeId = @TypeId
-  END
-
   IF @ReturnResourceKeysOnly = 0
     SELECT ResourceTypeId
           ,ResourceId
@@ -34,20 +19,20 @@ BEGIN TRY
           ,IsRawResourceMetaSet
           ,SearchParamHash
           ,RequestMethod
-      FROM (SELECT TOP (@DummyTop) * FROM @Keys) A
-           JOIN dbo.Resource B ON ResourceTypeId = TypeId AND ResourceSurrogateId = SurrogateId
-      WHERE IsHistory = 0 OR @IncludeHistory = 1
-      OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
+          ,FileId
+          ,OffsetInFile
+      FROM dbo.Resource
+      WHERE TransactionId = @TransactionId AND (IsHistory = 0 OR @IncludeHistory = 1)
+      OPTION (MAXDOP 1)
   ELSE
     SELECT ResourceTypeId
           ,ResourceId
           ,ResourceSurrogateId
           ,Version
           ,IsDeleted
-      FROM (SELECT TOP (@DummyTop) * FROM @Keys) A
-           JOIN dbo.Resource B ON ResourceTypeId = TypeId AND ResourceSurrogateId = SurrogateId
-      WHERE IsHistory = 0 OR @IncludeHistory = 1
-      OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
+      FROM dbo.Resource
+      WHERE TransactionId = @TransactionId AND (IsHistory = 0 OR @IncludeHistory = 1)
+      OPTION (MAXDOP 1)
 
   EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@@rowcount
 END TRY

src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql

@@ -42,14 +42,14 @@ BEGIN TRY
           OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1))
   END
 
-  SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource 
+  SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource, FileId, OffsetInFile
     FROM dbo.Resource
     WHERE ResourceTypeId = @ResourceTypeId 
       AND ResourceSurrogateId BETWEEN @StartId AND @EndId 
       AND (IsHistory = 0 OR @IncludeHistory = 1)
       AND (IsDeleted = 0 OR @IncludeDeleted = 1)
   UNION ALL
-  SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource 
+  SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource, FileId, OffsetInFile
     FROM @SurrogateIds
          JOIN dbo.Resource ON ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId = MaxSurrogateId
     WHERE IsHistory = 1

src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/HardDeleteResource.sql

@@ -41,7 +41,7 @@ BEGIN TRY
   BEGIN
     -- PAGLOCK allows deallocation of empty page without waiting for ghost cleanup 
     DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.ResourceWriteClaim B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1)
-    DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.ReferenceSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1)
+    DELETE FROM dbo.ReferenceSearchParam WHERE ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId IN (SELECT ResourceSurrogateId FROM @SurrogateIds)
     DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.TokenSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1)
     DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.TokenText B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1)
     DELETE FROM B FROM @SurrogateIds A INNER LOOP JOIN dbo.StringSearchParam B WITH (INDEX = 1, FORCESEEK, PAGLOCK) ON B.ResourceTypeId = @ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId OPTION (MAXDOP 1)