diff --git a/codebase2/codebase-sqlite-hashing-v2/package.yaml b/codebase2/codebase-sqlite-hashing-v2/package.yaml index d2301bd918..9e32e8546b 100644 --- a/codebase2/codebase-sqlite-hashing-v2/package.yaml +++ b/codebase2/codebase-sqlite-hashing-v2/package.yaml @@ -20,6 +20,7 @@ dependencies: - unison-hashing-v2 - unison-prelude - unison-sqlite + - unison-syntax - unison-util-base32hex - unison-util-term - vector diff --git a/codebase2/codebase-sqlite-hashing-v2/src/U/Codebase/Decl/Hashing.hs b/codebase2/codebase-sqlite-hashing-v2/src/U/Codebase/Decl/Hashing.hs new file mode 100644 index 0000000000..c16b0d931e --- /dev/null +++ b/codebase2/codebase-sqlite-hashing-v2/src/U/Codebase/Decl/Hashing.hs @@ -0,0 +1,54 @@ +module U.Codebase.Decl.Hashing where + +import Control.Lens +import Data.Foldable qualified as Foldable +import Data.Map qualified as Map +import U.Codebase.Decl qualified as C +import U.Codebase.Decl qualified as C.Decl +import U.Codebase.HashTags +import U.Codebase.Reference qualified as Reference +import U.Codebase.Sqlite.Decl.Format qualified as DeclFormat +import U.Codebase.Sqlite.HashHandle (HashMismatch (..)) +import U.Codebase.Sqlite.HashHandle qualified as HH +import U.Codebase.Sqlite.LocalIds qualified as LocalIds +import U.Codebase.Sqlite.Queries qualified as Q +import U.Codebase.Sqlite.Symbol qualified as S +import U.Codebase.Sqlite.Symbol qualified as Sqlite +import Unison.Hash32 +import Unison.Hash32 qualified as Hash32 +import Unison.Hashing.V2 qualified as H2 +import Unison.Hashing.V2.Convert2 qualified as H2 +import Unison.Prelude +import Unison.Symbol qualified as Unison +import Unison.Syntax.Name qualified as Name +import Unison.Var qualified as Var + +verifyDeclFormatHash :: ComponentHash -> DeclFormat.HashDeclFormat -> Maybe HH.DeclHashingError +verifyDeclFormatHash (ComponentHash hash) (DeclFormat.Decl (DeclFormat.LocallyIndexedComponent elements)) = + Foldable.toList elements + & fmap s2cDecl + & Reference.component hash + & fmap (\(decl, refId) -> (refId, (C.Decl.vmap symbol2to1 decl, ()))) + & Map.fromList + & C.Decl.unhashComponent hash Var.unnamedRef + & Map.toList + & fmap (\(_refId, (v, decl, ())) -> (v, either H2.toDataDecl id $ H2.v2ToH2Decl decl)) + & Map.fromList + & H2.hashDecls Name.unsafeFromVar + & \case + Left _err -> Just HH.DeclHashResolutionFailure + Right m -> + m + & altMap \(_, H2.ReferenceId hash' _, _) -> + if hash == hash' + then Nothing + else Just (HH.DeclHashMismatch $ HashMismatch hash hash') + where + symbol2to1 :: S.Symbol -> Unison.Symbol + symbol2to1 (S.Symbol i t) = Unison.Symbol i (Var.User t) + +s2cDecl :: (LocalIds.LocalIds' Text Hash32, DeclFormat.Decl Sqlite.Symbol) -> C.Decl Sqlite.Symbol +s2cDecl (ids, decl) = + let Identity (substText, substHash) = Q.localIdsToLookups Identity pure (bimap id Hash32.toHash ids) + refmap = (bimap substText (fmap substHash)) + in Q.x2cDecl refmap decl diff --git a/codebase2/codebase-sqlite-hashing-v2/src/U/Codebase/Sqlite/V2/HashHandle.hs b/codebase2/codebase-sqlite-hashing-v2/src/U/Codebase/Sqlite/V2/HashHandle.hs index d08f2906e3..be5030e386 100644 --- a/codebase2/codebase-sqlite-hashing-v2/src/U/Codebase/Sqlite/V2/HashHandle.hs +++ b/codebase2/codebase-sqlite-hashing-v2/src/U/Codebase/Sqlite/V2/HashHandle.hs @@ -7,13 +7,15 @@ import Data.Function ((&)) import Data.Set qualified as Set import U.Codebase.Branch.Hashing qualified as H2 import U.Codebase.Causal.Hashing qualified as H2 -import U.Codebase.HashTags (BranchHash (..)) +import U.Codebase.Decl.Hashing qualified as H2 +import U.Codebase.HashTags (BranchHash (..), PatchHash (..)) import U.Codebase.Sqlite.Branch.Format qualified as BranchFormat import U.Codebase.Sqlite.HashHandle +import U.Codebase.Sqlite.Patch.Format qualified as PatchFormat import U.Codebase.Term.Hashing as H2 import U.Util.Type (removeAllEffectVars) import Unison.Hashing.V2 qualified as H2 -import Unison.Hashing.V2.Convert2 (h2ToV2Reference, hashBranchFormatToH2Branch, v2ToH2Type, v2ToH2TypeD) +import Unison.Hashing.V2.Convert2 (h2ToV2Reference, hashBranchFormatToH2Branch, hashPatchFormatToH2Patch, v2ToH2Type, v2ToH2TypeD) v2HashHandle :: HashHandle v2HashHandle = @@ -25,10 +27,19 @@ v2HashHandle = hashBranch = H2.hashBranch, hashBranchV3 = H2.hashBranchV3, hashCausal = H2.hashCausal, - hashBranchFormatFull = \localIds localBranch -> - BranchFormat.localToHashBranch localIds localBranch - & hashBranchFormatToH2Branch - & H2.contentHash - & BranchHash, - verifyTermFormatHash = H2.verifyTermFormatHash + hashBranchFormatFull, + hashPatchFormatFull, + verifyTermFormatHash = H2.verifyTermFormatHash, + verifyDeclFormatHash = H2.verifyDeclFormatHash } + where + hashBranchFormatFull localIds localBranch = + BranchFormat.localToHashBranch localIds localBranch + & hashBranchFormatToH2Branch + & H2.contentHash + & BranchHash + hashPatchFormatFull localIds localPatch = + PatchFormat.localPatchToHashPatch localIds localPatch + & hashPatchFormatToH2Patch + & H2.contentHash + & PatchHash diff --git a/codebase2/codebase-sqlite-hashing-v2/src/Unison/Hashing/V2/Convert2.hs b/codebase2/codebase-sqlite-hashing-v2/src/Unison/Hashing/V2/Convert2.hs index 2c6c51c66a..1f3b4476fb 100644 --- a/codebase2/codebase-sqlite-hashing-v2/src/Unison/Hashing/V2/Convert2.hs +++ b/codebase2/codebase-sqlite-hashing-v2/src/Unison/Hashing/V2/Convert2.hs @@ -6,22 +6,29 @@ module Unison.Hashing.V2.Convert2 h2ToV2Reference, v2ToH2Branch, v2ToH2Term, + v2ToH2Decl, hashBranchFormatToH2Branch, + hashPatchFormatToH2Patch, ) where import Data.Map qualified as Map import Data.Set qualified as Set +import Data.Text qualified as Text import U.Codebase.Branch qualified as V2 import U.Codebase.Branch qualified as V2Branch import U.Codebase.BranchV3 (BranchV3 (..)) import U.Codebase.Causal qualified as Causal +import U.Codebase.Decl qualified as V2.Decl import U.Codebase.HashTags import U.Codebase.Kind qualified as V2 import U.Codebase.Reference qualified as V2 import U.Codebase.Reference qualified as V2Reference import U.Codebase.Referent qualified as V2Referent import U.Codebase.Sqlite.Branch.Full qualified as Memory.BranchFull +import U.Codebase.Sqlite.Patch.Full qualified as Memory.PatchFull +import U.Codebase.Sqlite.Patch.TermEdit qualified as Memory.TermEdit +import U.Codebase.Sqlite.Patch.TypeEdit qualified as Memory.TypeEdit import U.Codebase.Term qualified as V2 (TypeRef) import U.Codebase.Term qualified as V2.Term import U.Codebase.Type qualified as V2.Type @@ -30,6 +37,7 @@ import Unison.Hash (Hash) import Unison.Hashing.V2 qualified as H2 import Unison.NameSegment (NameSegment (..)) import Unison.Prelude +import Unison.Symbol qualified as Unison import Unison.Util.Map qualified as Map -- | Convert a V3 branch to a hashing branch. @@ -135,6 +143,29 @@ hashBranchFormatToH2Branch Memory.BranchFull.Branch {terms, types, patches, chil V2Referent.Con typeRef conId -> do (H2.ReferentCon (v2ToH2Reference $ second unComponentHash typeRef) conId) +hashPatchFormatToH2Patch :: Memory.PatchFull.HashPatch -> H2.Patch +hashPatchFormatToH2Patch Memory.PatchFull.Patch {termEdits, typeEdits} = + H2.Patch + { termEdits = Map.bimap cvreferent (Set.map cvTermEdit) termEdits, + typeEdits = Map.bimap cvreference (Set.map cvTypeEdit) typeEdits + } + where + cvTermEdit :: Memory.TermEdit.HashTermEdit -> H2.TermEdit + cvTermEdit = \case + Memory.TermEdit.Replace ref _typing -> H2.TermEditReplace (v2ToH2Referent . coerce $ ref) + Memory.TermEdit.Deprecate -> H2.TermEditDeprecate + cvTypeEdit :: Memory.TypeEdit.HashTypeEdit -> H2.TypeEdit + cvTypeEdit = \case + Memory.TypeEdit.Replace ref -> H2.TypeEditReplace (v2ToH2Reference . coerce $ ref) + Memory.TypeEdit.Deprecate -> H2.TypeEditDeprecate + cvreference :: V2Reference.Reference' Text ComponentHash -> H2.Reference + cvreference = v2ToH2Reference . second unComponentHash + cvreferent :: Memory.BranchFull.Referent'' Text ComponentHash -> H2.Referent + cvreferent = \case + V2Referent.Ref ref -> (H2.ReferentRef (v2ToH2Reference $ second unComponentHash ref)) + V2Referent.Con typeRef conId -> do + (H2.ReferentCon (v2ToH2Reference $ second unComponentHash typeRef) conId) + v2ToH2Term :: forall v. Ord v => V2.Term.HashableTerm v -> H2.Term v () v2ToH2Term = ABT.transform convertF where @@ -189,3 +220,27 @@ v2ToH2Term = ABT.transform convertF V2.Term.PCons -> H2.Cons V2.Term.PSnoc -> H2.Snoc V2.Term.PConcat -> H2.Concat + +v2ToH2Decl :: V2.Decl.HashableDecl Unison.Symbol -> H2.Decl Unison.Symbol () +v2ToH2Decl (V2.Decl.DataDeclaration {declType, modifier, bound, constructorTypes}) = + let tag = case declType of + V2.Decl.Effect -> Left . H2.EffectDeclaration + V2.Decl.Data -> Right + in tag $ + H2.DataDeclaration + { modifier = v2ToH2Modifier modifier, + annotation = (), + bound = bound, + constructors' = + constructorTypes + & zip [0 ..] + & fmap mkCtor + } + where + mkCtor :: (Int, V2.Type.TypeR V2.Decl.HashableTypeRef Unison.Symbol) -> ((), Unison.Symbol, H2.Type Unison.Symbol ()) + mkCtor (n, t) = ((), Unison.symbol . Text.pack $ "Constructor" ++ show n, v2ToH2Type t) + + v2ToH2Modifier :: V2.Decl.Modifier -> H2.Modifier + v2ToH2Modifier = \case + V2.Decl.Structural -> H2.Structural + V2.Decl.Unique t -> H2.Unique t diff --git a/codebase2/codebase-sqlite-hashing-v2/unison-codebase-sqlite-hashing-v2.cabal b/codebase2/codebase-sqlite-hashing-v2/unison-codebase-sqlite-hashing-v2.cabal index 0bdc069dc8..67e88874b7 100644 --- a/codebase2/codebase-sqlite-hashing-v2/unison-codebase-sqlite-hashing-v2.cabal +++ b/codebase2/codebase-sqlite-hashing-v2/unison-codebase-sqlite-hashing-v2.cabal @@ -21,6 +21,7 @@ library other-modules: U.Codebase.Branch.Hashing U.Codebase.Causal.Hashing + U.Codebase.Decl.Hashing U.Codebase.Term.Hashing Unison.Hashing.V2.Convert2 hs-source-dirs: @@ -69,6 +70,7 @@ library , unison-hashing-v2 , unison-prelude , unison-sqlite + , unison-syntax , unison-util-base32hex , unison-util-term , vector diff --git a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Decl/Format.hs b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Decl/Format.hs index 76a3d1069d..5a6f401964 100644 --- a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Decl/Format.hs +++ b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Decl/Format.hs @@ -10,12 +10,18 @@ import U.Codebase.Sqlite.LocalIds (LocalDefnId, LocalIds', LocalTextId) import U.Codebase.Sqlite.Symbol (Symbol) import U.Codebase.Type qualified as Type import U.Core.ABT qualified as ABT +import Unison.Hash32 (Hash32) import Unison.Prelude -- | Add new formats here -data DeclFormat = Decl LocallyIndexedComponent +data DeclFormat' text defn = Decl (LocallyIndexedComponent' text defn) deriving (Show) +type DeclFormat = DeclFormat' TextId ObjectId + +-- | A DeclFormat which uses hash references instead of database ids. +type HashDeclFormat = DeclFormat' Text Hash32 + -- | V1: Decls included `Hash`es inline -- V2: Instead of `Hash`, we use a smaller index. type LocallyIndexedComponent = diff --git a/codebase2/codebase-sqlite/U/Codebase/Sqlite/HashHandle.hs b/codebase2/codebase-sqlite/U/Codebase/Sqlite/HashHandle.hs index cd73192a30..028c4d827f 100644 --- a/codebase2/codebase-sqlite/U/Codebase/Sqlite/HashHandle.hs +++ b/codebase2/codebase-sqlite/U/Codebase/Sqlite/HashHandle.hs @@ -1,6 +1,7 @@ module U.Codebase.Sqlite.HashHandle ( HashHandle (..), HashMismatch (..), + DeclHashingError (..), ) where @@ -10,6 +11,9 @@ import U.Codebase.HashTags import U.Codebase.Reference qualified as C import U.Codebase.Sqlite.Branch.Format (HashBranchLocalIds) import U.Codebase.Sqlite.Branch.Full (LocalBranch) +import U.Codebase.Sqlite.Decl.Format qualified as DeclFormat +import U.Codebase.Sqlite.Patch.Format (HashPatchLocalIds) +import U.Codebase.Sqlite.Patch.Full (LocalPatch) import U.Codebase.Sqlite.Symbol (Symbol) import U.Codebase.Sqlite.Term.Format qualified as TermFormat import U.Codebase.Term qualified as C.Term @@ -22,6 +26,10 @@ data HashMismatch = HashMismatch actualHash :: Hash } +data DeclHashingError + = DeclHashMismatch HashMismatch + | DeclHashResolutionFailure + data HashHandle = HashHandle { -- | Hash type toReference :: C.Term.Type Symbol -> C.Reference, @@ -43,5 +51,16 @@ data HashHandle = HashHandle HashBranchLocalIds -> LocalBranch -> BranchHash, - verifyTermFormatHash :: ComponentHash -> TermFormat.HashTermFormat -> Maybe (HashMismatch) + hashPatchFormatFull :: + HashPatchLocalIds -> + LocalPatch -> + PatchHash, + verifyTermFormatHash :: + ComponentHash -> + TermFormat.HashTermFormat -> + Maybe (HashMismatch), + verifyDeclFormatHash :: + ComponentHash -> + DeclFormat.HashDeclFormat -> + Maybe DeclHashingError } diff --git a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/Format.hs b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/Format.hs index 263fec4bb9..a732887c2e 100644 --- a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/Format.hs +++ b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/Format.hs @@ -2,11 +2,13 @@ module U.Codebase.Sqlite.Patch.Format ( PatchFormat (..), PatchLocalIds, PatchLocalIds' (..), + HashPatchLocalIds, SyncPatchFormat, SyncPatchFormat' (..), applyPatchDiffs, localPatchToPatch, localPatchDiffToPatchDiff, + localPatchToHashPatch, ) where @@ -14,11 +16,12 @@ import Data.Map.Strict qualified as Map import Data.Set qualified as Set import Data.Vector (Vector) import Data.Vector qualified as Vector +import U.Codebase.HashTags import U.Codebase.Sqlite.DbId (HashId, ObjectId, PatchObjectId, TextId) import U.Codebase.Sqlite.LocalIds (LocalDefnId (LocalDefnId), LocalHashId (LocalHashId), LocalTextId (LocalTextId)) import U.Codebase.Sqlite.Patch.Diff (LocalPatchDiff, PatchDiff, PatchDiff' (..)) import U.Codebase.Sqlite.Patch.Diff qualified as Patch.Diff -import U.Codebase.Sqlite.Patch.Full (LocalPatch, Patch, Patch' (..)) +import U.Codebase.Sqlite.Patch.Full (HashPatch, LocalPatch, Patch, Patch' (..)) import U.Codebase.Sqlite.Patch.Full qualified as Patch.Full import Unison.Prelude @@ -28,6 +31,9 @@ data PatchFormat type PatchLocalIds = PatchLocalIds' TextId HashId ObjectId +-- | LocalIds type which can be used in hashing the Patch. +type HashPatchLocalIds = PatchLocalIds' Text ComponentHash ComponentHash + data PatchLocalIds' t h d = LocalIds { patchTextLookup :: Vector t, patchHashLookup :: Vector h, @@ -64,10 +70,17 @@ applyPatchDiffs = let diff = Set.difference src del in if Set.null diff then Nothing else Just diff -localPatchToPatch :: PatchLocalIds -> LocalPatch -> Patch -localPatchToPatch li = +localToPatch' :: (Ord t, Ord h, Ord d) => PatchLocalIds' t h d -> (Patch' LocalTextId LocalHashId LocalDefnId) -> Patch' t h d +localToPatch' li = Patch.Full.trimap (lookupPatchLocalText li) (lookupPatchLocalHash li) (lookupPatchLocalDefn li) +-- | Type specialized version of `localToPatch'`. +localPatchToPatch :: PatchLocalIds -> LocalPatch -> Patch +localPatchToPatch = localToPatch' + +localPatchToHashPatch :: HashPatchLocalIds -> LocalPatch -> HashPatch +localPatchToHashPatch = localToPatch' + localPatchDiffToPatchDiff :: PatchLocalIds -> LocalPatchDiff -> PatchDiff localPatchDiffToPatchDiff li = Patch.Diff.trimap @@ -75,11 +88,11 @@ localPatchDiffToPatchDiff li = (lookupPatchLocalHash li) (lookupPatchLocalDefn li) -lookupPatchLocalText :: PatchLocalIds -> LocalTextId -> TextId +lookupPatchLocalText :: PatchLocalIds' t h d -> LocalTextId -> t lookupPatchLocalText li (LocalTextId w) = patchTextLookup li Vector.! fromIntegral w -lookupPatchLocalHash :: PatchLocalIds -> LocalHashId -> HashId +lookupPatchLocalHash :: PatchLocalIds' t h d -> LocalHashId -> h lookupPatchLocalHash li (LocalHashId w) = patchHashLookup li Vector.! fromIntegral w -lookupPatchLocalDefn :: PatchLocalIds -> LocalDefnId -> ObjectId +lookupPatchLocalDefn :: PatchLocalIds' t h d -> LocalDefnId -> d lookupPatchLocalDefn li (LocalDefnId w) = patchDefnLookup li Vector.! fromIntegral w diff --git a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/Full.hs b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/Full.hs index 7f3252848b..39cecdb6fb 100644 --- a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/Full.hs +++ b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/Full.hs @@ -4,6 +4,8 @@ import Control.Lens import Data.Map (Map) import Data.Set (Set) import Data.Set qualified as Set +import Data.Text (Text) +import U.Codebase.HashTags import U.Codebase.Reference (Reference') import U.Codebase.Reference qualified as Reference import U.Codebase.Referent (Referent') @@ -26,6 +28,9 @@ import Unison.Util.Set qualified as Set -- @ type Patch = Patch' Db.TextId Db.HashId Db.ObjectId +-- | A version of Patch' which can be used for hashing. +type HashPatch = Patch' Text ComponentHash ComponentHash + -- | -- @ -- LocalPatch diff --git a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/TermEdit.hs b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/TermEdit.hs index 4b07b975b0..bc93dd166c 100644 --- a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/TermEdit.hs +++ b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/TermEdit.hs @@ -3,6 +3,8 @@ module U.Codebase.Sqlite.Patch.TermEdit where import Control.Lens import Data.Bifoldable (Bifoldable (bifoldMap)) import Data.Bitraversable (Bitraversable (bitraverse)) +import Data.Text (Text) +import U.Codebase.HashTags import U.Codebase.Reference (Reference') import U.Codebase.Reference qualified as Reference import U.Codebase.Referent qualified as Referent @@ -11,6 +13,8 @@ import U.Codebase.Sqlite.LocalIds (LocalDefnId, LocalTextId) type TermEdit = TermEdit' Db.TextId Db.ObjectId +type HashTermEdit = TermEdit' Text ComponentHash + type LocalTermEdit = TermEdit' LocalTextId LocalDefnId type Referent' t h = Referent.Referent' (Reference' t h) (Reference' t h) diff --git a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/TypeEdit.hs b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/TypeEdit.hs index 9223812532..6b8d3ea48c 100644 --- a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/TypeEdit.hs +++ b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Patch/TypeEdit.hs @@ -3,6 +3,8 @@ module U.Codebase.Sqlite.Patch.TypeEdit where import Control.Lens import Data.Bifoldable (Bifoldable (bifoldMap)) import Data.Bitraversable (Bitraversable (bitraverse)) +import Data.Text (Text) +import U.Codebase.HashTags import U.Codebase.Reference (Reference') import U.Codebase.Reference qualified as Reference import U.Codebase.Sqlite.DbId qualified as Db @@ -12,6 +14,8 @@ type LocalTypeEdit = TypeEdit' LocalTextId LocalDefnId type TypeEdit = TypeEdit' Db.TextId Db.ObjectId +type HashTypeEdit = TypeEdit' Text ComponentHash + data TypeEdit' t h = Replace (Reference' t h) | Deprecate deriving (Eq, Ord, Show) diff --git a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Queries.hs b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Queries.hs index 98e645b367..ab44fa29ef 100644 --- a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Queries.hs +++ b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Queries.hs @@ -273,6 +273,7 @@ module U.Codebase.Sqlite.Queries schemaVersion, x2cTType, x2cTerm, + x2cDecl, checkBranchExistsForCausalHash, -- * Types @@ -3045,9 +3046,12 @@ saveTermComponent hh@HashHandle {toReference, toReferenceMentions} maybeEncodedT -- | Unlocalize a decl. s2cDecl :: LocalIds -> S.Decl.Decl Symbol -> Transaction (C.Decl Symbol) -s2cDecl ids (C.Decl.DataDeclaration dt m b ct) = do +s2cDecl ids decl = do substTypeRef <- localIdsToTypeRefLookup ids - pure (C.Decl.DataDeclaration dt m b (C.Type.rmap substTypeRef <$> ct)) + pure $ x2cDecl substTypeRef decl + +x2cDecl :: (r -> r1) -> (C.Decl.DeclR r Symbol -> C.Decl.DeclR r1 Symbol) +x2cDecl substTypeRef (C.Decl.DataDeclaration dt m b ct) = C.Decl.DataDeclaration dt m b (C.Type.rmap substTypeRef <$> ct) saveDeclComponent :: HashHandle -> diff --git a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Serialization.hs b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Serialization.hs index fddaafba02..9aa6380db6 100644 --- a/codebase2/codebase-sqlite/U/Codebase/Sqlite/Serialization.hs +++ b/codebase2/codebase-sqlite/U/Codebase/Sqlite/Serialization.hs @@ -12,6 +12,7 @@ module U.Codebase.Sqlite.Serialization getDeclElementNumConstructors, getDeclFormat, getPatchFormat, + getLocalPatch, getTempCausalFormat, getTempDeclFormat, getTempNamespaceFormat, @@ -583,15 +584,10 @@ putPatchFormat = \case getPatchFormat :: (MonadGet m) => m PatchFormat.PatchFormat getPatchFormat = getWord8 >>= \case - 0 -> PatchFormat.Full <$> getPatchLocalIds <*> getPatchFull + 0 -> PatchFormat.Full <$> getPatchLocalIds <*> getLocalPatch 1 -> PatchFormat.Diff <$> getVarInt <*> getPatchLocalIds <*> getPatchDiff x -> unknownTag "getPatchFormat" x where - getPatchFull :: (MonadGet m) => m PatchFull.LocalPatch - getPatchFull = - PatchFull.Patch - <$> getMap getReferent (getSet getTermEdit) - <*> getMap getReference (getSet getTypeEdit) getPatchDiff :: (MonadGet m) => m PatchDiff.LocalPatchDiff getPatchDiff = PatchDiff.PatchDiff @@ -599,12 +595,20 @@ getPatchFormat = <*> getMap getReference (getSet getTypeEdit) <*> getMap getReferent (getSet getTermEdit) <*> getMap getReference (getSet getTypeEdit) - getTermEdit :: (MonadGet m) => m TermEdit.LocalTermEdit - getTermEdit = - getWord8 >>= \case - 0 -> pure TermEdit.Deprecate - 1 -> TermEdit.Replace <$> getReferent <*> getTyping - x -> unknownTag "getTermEdit" x + +getLocalPatch :: (MonadGet m) => m PatchFull.LocalPatch +getLocalPatch = + PatchFull.Patch + <$> getMap getReferent (getSet getTermEdit) + <*> getMap getReference (getSet getTypeEdit) + +getTermEdit :: (MonadGet m) => m TermEdit.LocalTermEdit +getTermEdit = + getWord8 >>= \case + 0 -> pure TermEdit.Deprecate + 1 -> TermEdit.Replace <$> getReferent <*> getTyping + x -> unknownTag "getTermEdit" x + where getTyping :: (MonadGet m) => m TermEdit.Typing getTyping = getWord8 >>= \case @@ -612,12 +616,13 @@ getPatchFormat = 1 -> pure TermEdit.Subtype 2 -> pure TermEdit.Different x -> unknownTag "getTyping" x - getTypeEdit :: (MonadGet m) => m TypeEdit.LocalTypeEdit - getTypeEdit = - getWord8 >>= \case - 0 -> pure TypeEdit.Deprecate - 1 -> TypeEdit.Replace <$> getReference - x -> unknownTag "getTypeEdit" x + +getTypeEdit :: (MonadGet m) => m TypeEdit.LocalTypeEdit +getTypeEdit = + getWord8 >>= \case + 0 -> pure TypeEdit.Deprecate + 1 -> TypeEdit.Replace <$> getReference + x -> unknownTag "getTypeEdit" x getPatchLocalIds :: (MonadGet m) => m PatchFormat.PatchLocalIds getPatchLocalIds = diff --git a/codebase2/codebase/U/Codebase/Decl.hs b/codebase2/codebase/U/Codebase/Decl.hs index e725150041..26172ed1db 100644 --- a/codebase2/codebase/U/Codebase/Decl.hs +++ b/codebase2/codebase/U/Codebase/Decl.hs @@ -1,8 +1,15 @@ module U.Codebase.Decl where +import Control.Lens hiding (List) +import Control.Monad.State +import Data.Map qualified as Map +import Data.Set qualified as Set import U.Codebase.Reference (Reference') +import U.Codebase.Reference qualified as Reference import U.Codebase.Type (TypeR) import U.Codebase.Type qualified as Type +import U.Core.ABT qualified as ABT +import U.Core.ABT.Var qualified as ABT import Unison.Hash (Hash) import Unison.Prelude @@ -13,10 +20,16 @@ data DeclType = Data | Effect type Decl v = DeclR TypeRef v +type HashableDecl v = DeclR HashableTypeRef v + type TypeRef = Reference' Text (Maybe Hash) +type HashableTypeRef = Reference' Text Hash + type Type v = TypeR TypeRef v +type HashableType v = TypeR HashableTypeRef v + data Modifier = Structural | Unique Text deriving (Eq, Ord, Show) @@ -28,6 +41,28 @@ data DeclR r v = DataDeclaration } deriving (Show) +allVars :: Ord v => DeclR r v -> Set v +allVars (DataDeclaration _ _ bound constructorTypes) = + (Set.fromList $ foldMap ABT.allVars constructorTypes) <> Set.fromList bound + +vmap :: Ord v' => (v -> v') -> DeclR r v -> DeclR r v' +vmap f (DataDeclaration {declType, modifier, bound, constructorTypes}) = + DataDeclaration + { declType, + modifier, + bound = f <$> bound, + constructorTypes = ABT.vmap f <$> constructorTypes + } + +rmap :: (Ord v) => (r -> r') -> DeclR r v -> DeclR r' v +rmap f (DataDeclaration {declType, modifier, bound, constructorTypes}) = + DataDeclaration + { declType, + modifier, + bound, + constructorTypes = Type.rmap f <$> constructorTypes + } + -- * Hashing stuff dependencies :: (Ord r, Ord v) => DeclR r v -> Set r @@ -41,3 +76,71 @@ data F a | Constructors [a] | Modified DeclType Modifier a deriving (Functor, Foldable, Show) + +-- | Given the pieces of a single decl component, +-- replaces all 'Nothing' self-referential hashes with a variable reference +-- to the relevant piece of the component in the component map. +unhashComponent :: + forall v extra. + ABT.Var v => + Hash -> + -- | A function to convert a reference to a variable. The actual var names aren't important. + (Reference.Id -> v) -> + -- A SINGLE decl component. Self references should have a 'Nothing' hash in term + -- references/term links + Map Reference.Id (Decl v, extra) -> + -- | The component with all self-references replaced with variable references. + Map Reference.Id (v, HashableDecl v, extra) +unhashComponent componentHash refToVar m = + withGeneratedVars + & traversed . _2 %~ fillSelfReferences + where + usedVars :: Set v + usedVars = foldMapOf (folded . _1) allVars m + withGeneratedVars :: Map Reference.Id (v, Decl v, extra) + withGeneratedVars = evalState (Map.traverseWithKey assignVar m) usedVars + assignVar :: Reference.Id -> (trm, extra) -> StateT (Set v) Identity (v, trm, extra) + assignVar r (trm, extra) = (,trm,extra) <$> ABT.freshenS (refToVar r) + fillSelfReferences :: Decl v -> HashableDecl v + fillSelfReferences DataDeclaration {declType, modifier, bound, constructorTypes} = + DataDeclaration + { declType, + modifier, + bound, + constructorTypes = ABT.cata alg <$> constructorTypes + } + where + rewriteTypeReference :: Reference.Id' (Maybe Hash) -> Either v Reference.Reference + rewriteTypeReference rid@(Reference.Id mayH pos) = + case mayH of + Just h -> + case Map.lookup (Reference.Id h pos) withGeneratedVars of + -- No entry in the component map, so this is NOT a self-reference, keep it but + -- replace the 'Maybe Hash' with a 'Hash'. + Nothing -> Right (Reference.ReferenceDerived (Reference.Id h pos)) + -- Entry in the component map, so this is a self-reference, replace it with a + -- Var. + Just (v, _, _) -> Left v + Nothing -> + -- This is a self-reference, so we expect to find it in the component map. + case Map.lookup (fromMaybe componentHash <$> rid) withGeneratedVars of + Nothing -> error "unhashComponent: self-reference not found in component map" + Just (v, _, _) -> Left v + alg :: () -> ABT.ABT (Type.F' TypeRef) v (HashableType v) -> HashableType v + alg () = \case + ABT.Var v -> ABT.var () v + ABT.Cycle body -> ABT.cycle () body + ABT.Abs v body -> ABT.abs () v body + ABT.Tm t -> case t of + Type.Ref (Reference.ReferenceDerived rid) -> + rewriteTypeReference rid + & either (ABT.var ()) (ABT.tm () . Type.Ref) + Type.Ref (Reference.ReferenceBuiltin t) -> + ABT.tm () $ Type.Ref (Reference.ReferenceBuiltin t) + Type.Arrow a b -> ABT.tm () $ Type.Arrow a b + Type.Ann a k -> ABT.tm () $ Type.Ann a k + Type.App a b -> ABT.tm () $ Type.App a b + Type.Effect a b -> ABT.tm () $ Type.Effect a b + Type.Effects as -> ABT.tm () $ Type.Effects as + Type.Forall a -> ABT.tm () $ Type.Forall a + Type.IntroOuter a -> ABT.tm () $ Type.IntroOuter a diff --git a/unison-cli/src/Unison/CommandLine/OutputMessages.hs b/unison-cli/src/Unison/CommandLine/OutputMessages.hs index 7f321bddfd..d58d1cf9b2 100644 --- a/unison-cli/src/Unison/CommandLine/OutputMessages.hs +++ b/unison-cli/src/Unison/CommandLine/OutputMessages.hs @@ -2352,6 +2352,12 @@ prettyEntityValidationError = \case "", P.wrap $ "The error was: " <> P.text err ] + Share.HashResolutionFailure hash -> + -- See https://github.com/unisonweb/unison/pull/4381#discussion_r1452652087 for discussion. + P.lines + [ P.wrap $ "Failed to resolve data when hashing " <> prettyHash32 hash <> ".", + "Please create an issue and report this to the Unison team" + ] prettyEntityType :: Share.EntityType -> Pretty prettyEntityType = \case diff --git a/unison-share-api/src/Unison/Sync/EntityValidation.hs b/unison-share-api/src/Unison/Sync/EntityValidation.hs index 0477bcc9bb..fdd986f0d8 100644 --- a/unison-share-api/src/Unison/Sync/EntityValidation.hs +++ b/unison-share-api/src/Unison/Sync/EntityValidation.hs @@ -14,10 +14,12 @@ import Data.Text qualified as Text import U.Codebase.HashTags import U.Codebase.Sqlite.Branch.Format qualified as BranchFormat import U.Codebase.Sqlite.Causal qualified as CausalFormat +import U.Codebase.Sqlite.Decl.Format qualified as DeclFormat import U.Codebase.Sqlite.Decode qualified as Decode import U.Codebase.Sqlite.Entity qualified as Entity import U.Codebase.Sqlite.HashHandle qualified as HH import U.Codebase.Sqlite.Orphans () +import U.Codebase.Sqlite.Patch.Format qualified as PatchFormat import U.Codebase.Sqlite.Serialization qualified as Serialization import U.Codebase.Sqlite.Term.Format qualified as TermFormat import U.Codebase.Sqlite.V2.HashHandle (v2HashHandle) @@ -33,20 +35,43 @@ import Unison.Sync.Types qualified as Share -- We should add more validation as more entities are shared. validateEntity :: Hash32 -> Share.Entity Text Hash32 Hash32 -> Maybe Share.EntityValidationError validateEntity expectedHash32 entity = do - case Share.entityToTempEntity id entity of - Entity.TC (TermFormat.SyncTerm localComp) -> do - validateTerm expectedHash localComp - Entity.N (BranchFormat.SyncDiff {}) -> do - (Just $ Share.UnsupportedEntityType expectedHash32 Share.NamespaceDiffType) - Entity.N (BranchFormat.SyncFull localIds (BranchFormat.LocalBranchBytes bytes)) -> do - validateBranchFull expectedHash localIds bytes - Entity.C CausalFormat.SyncCausalFormat {valueHash, parents} -> do - validateCausal expectedHash32 valueHash (toList parents) - _ -> Nothing + case Share.entityToTempEntity id entity of + Entity.TC (TermFormat.SyncTerm localComp) -> do + validateTerm expectedHash localComp + Entity.DC (DeclFormat.SyncDecl localComp) -> do + validateDecl expectedHash localComp + Entity.N (BranchFormat.SyncDiff {}) -> do + Just $ Share.UnsupportedEntityType expectedHash32 Share.NamespaceDiffType + Entity.N (BranchFormat.SyncFull localIds (BranchFormat.LocalBranchBytes bytes)) -> do + validateBranchFull expectedHash localIds bytes + Entity.C CausalFormat.SyncCausalFormat {valueHash, parents} -> do + validateCausal expectedHash32 valueHash (toList parents) + Entity.P (PatchFormat.SyncDiff {}) -> do + Just $ Share.UnsupportedEntityType expectedHash32 Share.PatchDiffType + Entity.P (PatchFormat.SyncFull localIds bytes) -> do + validatePatchFull expectedHash32 localIds bytes where expectedHash :: Hash expectedHash = Hash32.toHash expectedHash32 +validatePatchFull :: Hash32 -> PatchFormat.PatchLocalIds' Text Hash32 Hash32 -> BS.ByteString -> Maybe Share.EntityValidationError +validatePatchFull expectedHash32 localIds bytes = do + let expectedHash = Hash32.toHash expectedHash32 + case runGetS Serialization.getLocalPatch bytes of + Left e -> Just $ Share.InvalidByteEncoding expectedHash32 Share.PatchType (Text.pack e) + Right localPatch -> do + let localIds' = + localIds + { PatchFormat.patchTextLookup = PatchFormat.patchTextLookup localIds, + PatchFormat.patchHashLookup = ComponentHash . Hash32.toHash <$> PatchFormat.patchHashLookup localIds, + PatchFormat.patchDefnLookup = ComponentHash . Hash32.toHash <$> PatchFormat.patchDefnLookup localIds + } + let actualHash = + HH.hashPatchFormatFull v2HashHandle localIds' localPatch + if actualHash == PatchHash expectedHash + then Nothing + else Just $ Share.EntityHashMismatch Share.NamespaceType (mismatch expectedHash (unPatchHash actualHash)) + validateBranchFull :: Hash -> BranchFormat.BranchLocalIds' Text Hash32 Hash32 (Hash32, Hash32) -> @@ -79,6 +104,16 @@ validateTerm expectedHash syncLocalComp = do Nothing -> Nothing Just (HH.HashMismatch {expectedHash, actualHash}) -> Just . Share.EntityHashMismatch Share.TermComponentType $ mismatch expectedHash actualHash +validateDecl :: Hash -> (DeclFormat.SyncLocallyIndexedComponent' Text Hash32) -> (Maybe Share.EntityValidationError) +validateDecl expectedHash syncLocalComp = do + case Decode.unsyncDeclComponent syncLocalComp of + Left decodeErr -> Just (Share.InvalidByteEncoding (Hash32.fromHash expectedHash) Share.DeclComponentType (tShow decodeErr)) + Right localComp -> do + case HH.verifyDeclFormatHash v2HashHandle (ComponentHash expectedHash) (DeclFormat.Decl localComp) of + Nothing -> Nothing + Just (HH.DeclHashMismatch (HH.HashMismatch {expectedHash, actualHash})) -> Just . Share.EntityHashMismatch Share.TermComponentType $ mismatch expectedHash actualHash + Just HH.DeclHashResolutionFailure -> Just $ Share.HashResolutionFailure (Hash32.fromHash expectedHash) + validateCausal :: Hash32 -> Hash32 -> [Hash32] -> Maybe Share.EntityValidationError validateCausal expectedHash32 valueHash32 parentHashes32 = do let expectedHash = Hash32.toHash expectedHash32 diff --git a/unison-share-api/src/Unison/Sync/Types.hs b/unison-share-api/src/Unison/Sync/Types.hs index a44194d0a0..105f7b49e6 100644 --- a/unison-share-api/src/Unison/Sync/Types.hs +++ b/unison-share-api/src/Unison/Sync/Types.hs @@ -619,6 +619,7 @@ data EntityValidationError = EntityHashMismatch EntityType HashMismatchForEntity | UnsupportedEntityType Hash32 EntityType | InvalidByteEncoding Hash32 EntityType Text {- decoding err msg -} + | HashResolutionFailure Hash32 deriving stock (Show, Eq, Ord) deriving anyclass (Exception) @@ -627,6 +628,7 @@ instance ToJSON EntityValidationError where EntityHashMismatch typ mismatch -> jsonUnion "mismatched_hash" (object ["type" .= typ, "mismatch" .= mismatch]) UnsupportedEntityType hash typ -> jsonUnion "unsupported_entity_type" (object ["hash" .= hash, "type" .= typ]) InvalidByteEncoding hash typ errMsg -> jsonUnion "invalid_byte_encoding" (object ["hash" .= hash, "type" .= typ, "error" .= errMsg]) + HashResolutionFailure hash -> jsonUnion "hash_resolution_failure" hash instance FromJSON EntityValidationError where parseJSON = Aeson.withObject "EntityValidationError" \obj ->