Skip to content

Commit

Permalink
Small mem improvements on BulkImport
Browse files Browse the repository at this point in the history
  • Loading branch information
josephschorr committed Mar 27, 2024
1 parent 98b04e6 commit 2630f70
Showing 1 changed file with 8 additions and 11 deletions.
19 changes: 8 additions & 11 deletions internal/services/v1/experimental.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,11 +146,7 @@ func (a *bulkLoadAdapter) Next(_ context.Context) (*core.RelationTuple, error) {
}

a.current.Caveat = &a.caveat
tuple.CopyRelationshipToRelationTuple[
*v1.ObjectReference,
*v1.SubjectReference,
*v1.ContextualizedCaveat,
](a.currentBatch[a.numSent], &a.current)
tuple.CopyRelationshipToRelationTuple(a.currentBatch[a.numSent], &a.current)

if err := relationships.ValidateOneRelationship(
a.referencedNamespaceMap,
Expand All @@ -170,8 +166,8 @@ func extractBatchNewReferencedNamespacesAndCaveats(
existingNamespaces map[string]*typesystem.TypeSystem,
existingCaveats map[string]*core.CaveatDefinition,
) ([]string, []string) {
newNamespaces := make(map[string]struct{})
newCaveats := make(map[string]struct{})
newNamespaces := make(map[string]struct{}, 2)
newCaveats := make(map[string]struct{}, 0)
for _, rel := range batch {
if _, ok := existingNamespaces[rel.Resource.ObjectType]; !ok {
newNamespaces[rel.Resource.ObjectType] = struct{}{}
Expand All @@ -198,8 +194,8 @@ func (es *experimentalServer) BulkImportRelationships(stream v1.ExperimentalServ

var numWritten uint64
if _, err := ds.ReadWriteTx(stream.Context(), func(ctx context.Context, rwt datastore.ReadWriteTransaction) error {
loadedNamespaces := make(map[string]*typesystem.TypeSystem)
loadedCaveats := make(map[string]*core.CaveatDefinition)
loadedNamespaces := make(map[string]*typesystem.TypeSystem, 2)
loadedCaveats := make(map[string]*core.CaveatDefinition, 0)

adapter := &bulkLoadAdapter{
stream: stream,
Expand All @@ -211,21 +207,22 @@ func (es *experimentalServer) BulkImportRelationships(stream v1.ExperimentalServ
},
caveat: core.ContextualizedCaveat{},
}
resolver := typesystem.ResolverForDatastoreReader(rwt)

var streamWritten uint64
var err error
for ; adapter.err == nil && err == nil; streamWritten, err = rwt.BulkLoad(stream.Context(), adapter) {
numWritten += streamWritten

// The stream has terminated because we're awaiting namespace and caveat information
// The stream has terminated because we're awaiting namespace and/or caveat information
if len(adapter.awaitingNamespaces) > 0 {
nsDefs, err := rwt.LookupNamespacesWithNames(stream.Context(), adapter.awaitingNamespaces)
if err != nil {
return err
}

for _, nsDef := range nsDefs {
nts, err := typesystem.NewNamespaceTypeSystem(nsDef.Definition, typesystem.ResolverForDatastoreReader(rwt))
nts, err := typesystem.NewNamespaceTypeSystem(nsDef.Definition, resolver)
if err != nil {
return err
}
Expand Down

0 comments on commit 2630f70

Please sign in to comment.