From 6b1d88bb24a09772a80507581cf33c4feb530289 Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Tue, 28 Jan 2025 12:38:14 -0500 Subject: [PATCH 01/30] Create foundation for top-level alert --- src/api/storageMappings.ts | 23 ++++++- .../materialization/TrialOnlyPrefixAlert.tsx | 48 +++++++++++++ src/components/materialization/types.ts | 3 + src/components/shared/Entity/Edit/index.tsx | 7 ++ src/hooks/useTrialStorageOnly.ts | 67 +++++++++++++++++++ src/lang/en-US/Workflows.ts | 3 + src/services/types.ts | 1 + src/stores/Tenant/Store.ts | 54 +++++++++++++-- src/stores/Tenant/types.ts | 3 + 9 files changed, 204 insertions(+), 5 deletions(-) create mode 100644 src/components/materialization/TrialOnlyPrefixAlert.tsx create mode 100644 src/components/materialization/types.ts create mode 100644 src/hooks/useTrialStorageOnly.ts diff --git a/src/api/storageMappings.ts b/src/api/storageMappings.ts index f923e4858..c6410f7c9 100644 --- a/src/api/storageMappings.ts +++ b/src/api/storageMappings.ts @@ -2,6 +2,8 @@ import { PostgrestSingleResponse } from '@supabase/postgrest-js'; import { supabaseClient } from 'context/GlobalProviders'; import { defaultTableFilter, + handleFailure, + handleSuccess, Pagination, RPCS, SortingProps, @@ -51,6 +53,20 @@ const getStorageMapping = (catalog_prefix: string) => { .returns(); }; +const getStorageMappingStores = async (prefixes: string[]) => { + return supabaseRetry( + () => + supabaseClient + .from(TABLES.STORAGE_MAPPINGS) + .select('catalog_prefix,spec') + .in('catalog_prefix', prefixes), + 'getStorageMappingStores' + ).then( + handleSuccess[]>, + handleFailure + ); +}; + const republishPrefix = async (prefix: string) => { return supabaseRetry>( () => @@ -61,4 +77,9 @@ const republishPrefix = async (prefix: string) => { ); }; -export { getStorageMapping, getStorageMappings, republishPrefix }; +export { + getStorageMapping, + getStorageMappingStores, + getStorageMappings, + republishPrefix, +}; diff --git a/src/components/materialization/TrialOnlyPrefixAlert.tsx b/src/components/materialization/TrialOnlyPrefixAlert.tsx new file mode 100644 index 000000000..87d2210d0 --- /dev/null +++ b/src/components/materialization/TrialOnlyPrefixAlert.tsx @@ -0,0 +1,48 @@ +import { Typography } from '@mui/material'; +import AlertBox from 'components/shared/AlertBox'; +import useTrialStorageOnly from 'hooks/useTrialStorageOnly'; +import { useEffect } from 'react'; +import { useIntl } from 'react-intl'; +import { useBinding_collections } from 'stores/Binding/hooks'; +import { useBindingStore } from 'stores/Binding/Store'; +import { useTenantStore } from 'stores/Tenant/Store'; +import { hasLength, stripPathing } from 'utils/misc-utils'; +import { TrialOnlyPrefixAlertProps } from './types'; + +export default function TrialOnlyPrefixAlert({ + messageId, +}: TrialOnlyPrefixAlertProps) { + const intl = useIntl(); + + const bindingsHydrated = useBindingStore((state) => state.hydrated); + const collections = useBinding_collections(); + + const trialStorageOnlyTenantsExist = useTenantStore((state) => + hasLength(state.trialStorageOnly) + ); + + const getTrialOnlyPrefixes = useTrialStorageOnly(); + + useEffect(() => { + if (bindingsHydrated) { + const prefixes = collections.map((collection) => + stripPathing(collection, true) + ); + + getTrialOnlyPrefixes(prefixes).then( + () => {}, + () => {} + ); + } + }, [bindingsHydrated, collections, getTrialOnlyPrefixes]); + + if (!bindingsHydrated || !trialStorageOnlyTenantsExist) { + return null; + } + + return ( + + {intl.formatMessage({ id: messageId })} + + ); +} diff --git a/src/components/materialization/types.ts b/src/components/materialization/types.ts new file mode 100644 index 000000000..973874575 --- /dev/null +++ b/src/components/materialization/types.ts @@ -0,0 +1,3 @@ +export interface TrialOnlyPrefixAlertProps { + messageId: string; +} diff --git a/src/components/shared/Entity/Edit/index.tsx b/src/components/shared/Entity/Edit/index.tsx index 8bb01abd0..570960ae0 100644 --- a/src/components/shared/Entity/Edit/index.tsx +++ b/src/components/shared/Entity/Edit/index.tsx @@ -7,6 +7,7 @@ import { useEditorStore_persistedDraftId, useEditorStore_setId, } from 'components/editor/Store/hooks'; +import TrialOnlyPrefixAlert from 'components/materialization/TrialOnlyPrefixAlert'; import CatalogEditor from 'components/shared/Entity/CatalogEditor'; import DetailsForm from 'components/shared/Entity/DetailsForm'; import EndpointConfig from 'components/shared/Entity/EndpointConfig'; @@ -186,6 +187,12 @@ function EntityEdit({ ) : null} + {entityType === 'materialization' ? ( + + + + ) : null} + {draftInitializationError ? ( diff --git a/src/hooks/useTrialStorageOnly.ts b/src/hooks/useTrialStorageOnly.ts new file mode 100644 index 000000000..dc1bfa147 --- /dev/null +++ b/src/hooks/useTrialStorageOnly.ts @@ -0,0 +1,67 @@ +import { getStorageMappingStores } from 'api/storageMappings'; +import { isEqual } from 'lodash'; +import { useCallback } from 'react'; +import { logRocketEvent } from 'services/shared'; +import { CustomEvents } from 'services/types'; +import { useEntitiesStore_capabilities_adminable } from 'stores/Entities/hooks'; +import { useTenantStore } from 'stores/Tenant/Store'; +import { hasLength } from 'utils/misc-utils'; + +const ESTUARY_TRIAL_STORAGE = { + provider: 'GCS', + bucket: 'estuary-trial', + prefix: 'collection-data/', +}; + +const getTrialStorageOnlyPrefixes = async ( + prefixes: string[] +): Promise => { + const { data, error } = await getStorageMappingStores(prefixes); + + if (error || !data) { + logRocketEvent(CustomEvents.TRIAL_STORAGE_UNKNOWN, { prefixes, error }); + + return []; + } + + return data + .filter( + ({ spec }) => + spec.stores.length === 1 && + isEqual(data[0].spec.stores[0], ESTUARY_TRIAL_STORAGE) + ) + .map(({ catalog_prefix }) => catalog_prefix); +}; + +export default function useTrialStorageOnly() { + const objectRoles = useEntitiesStore_capabilities_adminable(); + + const addTrialStorageOnly = useTenantStore( + (state) => state.addTrialStorageOnly + ); + + return useCallback( + async (prefixes: string[]) => { + if (!hasLength(prefixes)) { + return []; + } + + const filteredPrefixes = prefixes.filter((prefix) => + objectRoles.includes(prefix) + ); + + if (!hasLength(filteredPrefixes)) { + return []; + } + + const trialOnlyPrefixes = await getTrialStorageOnlyPrefixes( + filteredPrefixes + ); + + addTrialStorageOnly(trialOnlyPrefixes); + + return trialOnlyPrefixes; + }, + [addTrialStorageOnly, objectRoles] + ); +} diff --git a/src/lang/en-US/Workflows.ts b/src/lang/en-US/Workflows.ts index e93ba73b6..049603aa6 100644 --- a/src/lang/en-US/Workflows.ts +++ b/src/lang/en-US/Workflows.ts @@ -10,6 +10,9 @@ export const Workflows: Record = { 'workflows.error.endpointConfig.empty': `${endpointConfigHeader} empty`, 'workflows.error.initForm': `An issue was encountered initializing the form.`, 'workflows.error.initFormSection': `An issue was encountered initializing this section of the form.`, + 'workflows.error.oldBoundCollection.generic': `Your account uses Estuary's Trial bucket which includes 20 days of storage. There are collections bound to this materialization that are older than that. Please backfill the following sources after publishing this materialization:`, + 'workflows.error.oldBoundCollection.binding': `Your account uses Estuary's Trial bucket which includes 20 days of storage and this collection is older than that. To ensure you have all data, please also backfill this collection from the source after adding it to the materialization.`, + 'workflows.error.oldBoundCollection.backfill': `Your account uses Estuary's Trial bucket which includes 20 days of storage and this collection is older than that. Data will be missing if you backfill from the materialization so we recommend backfilling from the source.`, 'workflows.initTask.alert.title.initFailed': `Form Initialization Error`, 'workflows.initTask.alert.message.initFailed': `An issue was encountered initializing the form. Try refreshing the page and if the issue persists {docLink}.`, diff --git a/src/services/types.ts b/src/services/types.ts index 9598601e0..927e6ff34 100644 --- a/src/services/types.ts +++ b/src/services/types.ts @@ -63,6 +63,7 @@ export enum CustomEvents { SUPABASE_CALL_UNAUTHENTICATED = 'Supabase_Call_Unauthenticated', SWR_LOADING_SLOW = 'SWR_Loading_Slow', TRANSLATION_KEY_MISSING = 'Translation_Key_Missing', + TRIAL_STORAGE_UNKNOWN = 'TrialStorage:Unknown', UPDATE_AVAILABLE = 'Update_Available', URL_FORMAT_ERROR = 'URLFormatError', } diff --git a/src/stores/Tenant/Store.ts b/src/stores/Tenant/Store.ts index 0aa6af5c9..70ea6791c 100644 --- a/src/stores/Tenant/Store.ts +++ b/src/stores/Tenant/Store.ts @@ -1,12 +1,18 @@ import produce from 'immer'; +import { pull, union } from 'lodash'; +import { hasLength } from 'utils/misc-utils'; import { devtoolsOptions } from 'utils/store-utils'; -import { StoreApi, create } from 'zustand'; -import { NamedSet, devtools, persist } from 'zustand/middleware'; -import { TenantState } from './types'; +import { create, StoreApi } from 'zustand'; +import { devtools, NamedSet, persist } from 'zustand/middleware'; import { persistOptions } from './shared'; +import { TenantState } from './types'; -const getInitialStateData = (): Pick => ({ +const getInitialStateData = (): Pick< + TenantState, + 'selectedTenant' | 'trialStorageOnly' +> => ({ selectedTenant: '', + trialStorageOnly: [], }); const getInitialState = ( @@ -15,6 +21,46 @@ const getInitialState = ( ): TenantState => ({ ...getInitialStateData(), + addTrialStorageOnly: (values) => { + set( + produce((state: TenantState) => { + if ( + typeof values === 'string' && + !state.trialStorageOnly.includes(values) + ) { + state.trialStorageOnly.push(values); + } + + if (typeof values !== 'string') { + state.trialStorageOnly = hasLength(values) + ? union(state.trialStorageOnly, values) + : []; + } + }), + false, + 'Tenants with trial storage only added' + ); + }, + + removeTrialStorageOnly: (value) => { + set( + produce((state: TenantState) => { + if (value) { + state.trialStorageOnly = pull( + state.trialStorageOnly, + value + ); + + return; + } + + state.trialStorageOnly = []; + }), + false, + 'Tenants with trial storage only removed' + ); + }, + setSelectedTenant: (value) => { set( produce((state: TenantState) => { diff --git a/src/stores/Tenant/types.ts b/src/stores/Tenant/types.ts index 62b330ca1..d3a981fdd 100644 --- a/src/stores/Tenant/types.ts +++ b/src/stores/Tenant/types.ts @@ -1,4 +1,7 @@ export interface TenantState { + addTrialStorageOnly: (value: string | string[]) => void; + removeTrialStorageOnly: (value?: string) => void; selectedTenant: string; setSelectedTenant: (value: string) => void; + trialStorageOnly: string[]; } From c54e8484d9b0f30f156498d55f65acf539f64646 Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Tue, 28 Jan 2025 16:22:33 -0500 Subject: [PATCH 02/30] Display an alert when backfilling a old collection --- src/components/collection/ResourceConfig.tsx | 1 + .../Bindings/Backfill/BackfillButton.tsx | 14 +++++- .../Bindings/Backfill/SectionWrapper.tsx | 37 +++++++++----- .../editor/Bindings/Backfill/index.tsx | 6 ++- .../editor/Bindings/Backfill/types.ts | 7 +++ .../materialization/Create/index.tsx | 46 +++++++++-------- src/components/materialization/Edit.tsx | 49 ++++++++++--------- .../materialization/TrialOnlyPrefixAlert.tsx | 48 +++++++----------- .../TrialOnlyPrefixHydrator.tsx | 43 ++++++++++++++++ src/components/materialization/types.ts | 1 + src/components/shared/Entity/Backfill.tsx | 2 +- src/components/shared/Entity/Edit/index.tsx | 7 --- src/lang/en-US/Workflows.ts | 4 +- src/stores/Binding/Store.ts | 19 ++++++- src/stores/Binding/types.ts | 4 +- 15 files changed, 189 insertions(+), 99 deletions(-) create mode 100644 src/components/materialization/TrialOnlyPrefixHydrator.tsx diff --git a/src/components/collection/ResourceConfig.tsx b/src/components/collection/ResourceConfig.tsx index 289e745dc..2adae22da 100644 --- a/src/components/collection/ResourceConfig.tsx +++ b/src/components/collection/ResourceConfig.tsx @@ -72,6 +72,7 @@ function ResourceConfig({ diff --git a/src/components/editor/Bindings/Backfill/BackfillButton.tsx b/src/components/editor/Bindings/Backfill/BackfillButton.tsx index 38af8d05f..c94ad249a 100644 --- a/src/components/editor/Bindings/Backfill/BackfillButton.tsx +++ b/src/components/editor/Bindings/Backfill/BackfillButton.tsx @@ -20,7 +20,9 @@ import { useFormStateStore_setFormState, } from 'stores/FormState/hooks'; import { FormStatus } from 'stores/FormState/types'; +import { useTenantStore } from 'stores/Tenant/Store'; import { BindingMetadata } from 'types'; +import { useShallow } from 'zustand/react/shallow'; import { useEditorStore_queryResponse_draftSpecs } from '../../Store/hooks'; import BackfillCount from './BackfillCount'; import BackfillDataFlowOption from './BackfillDataFlowOption'; @@ -61,6 +63,11 @@ function BackfillButton({ const formActive = useFormStateStore_isActive(); const setFormState = useFormStateStore_setFormState(); + // Tenant Store + const trialOnlyPrefixes = useTenantStore( + useShallow((state) => state.trialStorageOnly) + ); + const disabled = formActive || collectionsCount < 1 || @@ -150,7 +157,11 @@ function BackfillButton({ ? currentBindingUUID : undefined; - setBackfilledBindings(increment, targetBindingUUID); + setBackfilledBindings( + increment, + targetBindingUUID, + trialOnlyPrefixes + ); setFormState({ status: FormStatus.UPDATED }); }, (error) => { @@ -173,6 +184,7 @@ function BackfillButton({ evaluateServerDifferences, setBackfilledBindings, setFormState, + trialOnlyPrefixes, updateBackfillCounter, ] ); diff --git a/src/components/editor/Bindings/Backfill/SectionWrapper.tsx b/src/components/editor/Bindings/Backfill/SectionWrapper.tsx index 55a1e295f..8753fedc8 100644 --- a/src/components/editor/Bindings/Backfill/SectionWrapper.tsx +++ b/src/components/editor/Bindings/Backfill/SectionWrapper.tsx @@ -1,23 +1,36 @@ import { Box, Stack, Typography } from '@mui/material'; +import TrialOnlyPrefixAlert from 'components/materialization/TrialOnlyPrefixAlert'; +import { useEntityType } from 'context/EntityContext'; import { useIntl } from 'react-intl'; -import { BaseComponentProps } from 'types'; +import { SectionWrapperProps } from './types'; -export default function SectionWrapper({ children }: BaseComponentProps) { +export default function SectionWrapper({ + alertMessageId, + bindingUUID, + children, +}: SectionWrapperProps) { const intl = useIntl(); + const entityType = useEntityType(); + return ( - - {intl.formatMessage({ - id: 'workflows.collectionSelector.manualBackfill.header', - })} - + + + {intl.formatMessage({ + id: 'workflows.collectionSelector.manualBackfill.header', + })} + + + {entityType === 'materialization' ? ( + + ) : null} - {children} + {children} + ); } diff --git a/src/components/editor/Bindings/Backfill/index.tsx b/src/components/editor/Bindings/Backfill/index.tsx index de0dbbbc1..6bb58b596 100644 --- a/src/components/editor/Bindings/Backfill/index.tsx +++ b/src/components/editor/Bindings/Backfill/index.tsx @@ -7,6 +7,7 @@ import { BackfillProps } from './types'; export default function Backfill({ bindingIndex, + bindingUUID, collectionEnabled, }: BackfillProps) { const entityType = useEntityType(); @@ -15,7 +16,10 @@ export default function Backfill({ const showBackfillButton = isEdit && bindingIndex > -1 && collectionEnabled; return showBackfillButton ? ( - + - - } - primaryButtonProps={{ - disabled: !draftId, - logEvent: CustomEvents.MATERIALIZATION_CREATE, - }} - secondaryButtonProps={{ - disabled: !hasConnectors, - logEvent: CustomEvents.MATERIALIZATION_TEST, - }} - /> - } - /> + + + } + primaryButtonProps={{ + disabled: !draftId, + logEvent: + CustomEvents.MATERIALIZATION_CREATE, + }} + secondaryButtonProps={{ + disabled: !hasConnectors, + logEvent: CustomEvents.MATERIALIZATION_TEST, + }} + /> + } + /> + ); diff --git a/src/components/materialization/Edit.tsx b/src/components/materialization/Edit.tsx index 16c833a97..2a81bc90d 100644 --- a/src/components/materialization/Edit.tsx +++ b/src/components/materialization/Edit.tsx @@ -17,6 +17,7 @@ import usePageTitle from 'hooks/usePageTitle'; import { useCallback } from 'react'; import { CustomEvents } from 'services/types'; import WorkflowHydrator from 'stores/Workflow/Hydrator'; +import TrialOnlyPrefixHydrator from './TrialOnlyPrefixHydrator'; function MaterializationEdit() { usePageTitle({ @@ -48,29 +49,31 @@ function MaterializationEdit() { return ( - - } - primaryButtonProps={{ - disabled: !draftId, - logEvent: CustomEvents.MATERIALIZATION_EDIT, - }} - secondaryButtonProps={{ - disabled: !hasConnectors, - logEvent: CustomEvents.MATERIALIZATION_TEST, - }} - /> - } - /> + + + } + primaryButtonProps={{ + disabled: !draftId, + logEvent: CustomEvents.MATERIALIZATION_EDIT, + }} + secondaryButtonProps={{ + disabled: !hasConnectors, + logEvent: CustomEvents.MATERIALIZATION_TEST, + }} + /> + } + /> + ); diff --git a/src/components/materialization/TrialOnlyPrefixAlert.tsx b/src/components/materialization/TrialOnlyPrefixAlert.tsx index 87d2210d0..7be5c03a1 100644 --- a/src/components/materialization/TrialOnlyPrefixAlert.tsx +++ b/src/components/materialization/TrialOnlyPrefixAlert.tsx @@ -1,42 +1,32 @@ import { Typography } from '@mui/material'; import AlertBox from 'components/shared/AlertBox'; -import useTrialStorageOnly from 'hooks/useTrialStorageOnly'; -import { useEffect } from 'react'; import { useIntl } from 'react-intl'; -import { useBinding_collections } from 'stores/Binding/hooks'; -import { useBindingStore } from 'stores/Binding/Store'; -import { useTenantStore } from 'stores/Tenant/Store'; -import { hasLength, stripPathing } from 'utils/misc-utils'; +import { + useBinding_resourceConfigOfMetaBindingProperty, + useBinding_resourceConfigs, +} from 'stores/Binding/hooks'; import { TrialOnlyPrefixAlertProps } from './types'; export default function TrialOnlyPrefixAlert({ + bindingUUID, messageId, }: TrialOnlyPrefixAlertProps) { const intl = useIntl(); - const bindingsHydrated = useBindingStore((state) => state.hydrated); - const collections = useBinding_collections(); - - const trialStorageOnlyTenantsExist = useTenantStore((state) => - hasLength(state.trialStorageOnly) - ); - - const getTrialOnlyPrefixes = useTrialStorageOnly(); - - useEffect(() => { - if (bindingsHydrated) { - const prefixes = collections.map((collection) => - stripPathing(collection, true) - ); - - getTrialOnlyPrefixes(prefixes).then( - () => {}, - () => {} - ); - } - }, [bindingsHydrated, collections, getTrialOnlyPrefixes]); - - if (!bindingsHydrated || !trialStorageOnlyTenantsExist) { + const resourceConfigs = useBinding_resourceConfigs(); + const bindingSourceBackfillRecommended = + useBinding_resourceConfigOfMetaBindingProperty( + bindingUUID, + 'sourceBackfillRecommended' + ); + + if ( + (bindingUUID && !bindingSourceBackfillRecommended) || + (!bindingUUID && + Object.values(resourceConfigs).some( + (config) => !config.meta.sourceBackfillRecommended + )) + ) { return null; } diff --git a/src/components/materialization/TrialOnlyPrefixHydrator.tsx b/src/components/materialization/TrialOnlyPrefixHydrator.tsx new file mode 100644 index 000000000..d8c920121 --- /dev/null +++ b/src/components/materialization/TrialOnlyPrefixHydrator.tsx @@ -0,0 +1,43 @@ +import useTrialStorageOnly from 'hooks/useTrialStorageOnly'; +import { difference } from 'lodash'; +import { useEffect, useMemo } from 'react'; +import { useBinding_collections } from 'stores/Binding/hooks'; +import { useBindingStore } from 'stores/Binding/Store'; +import { useTenantStore } from 'stores/Tenant/Store'; +import { BaseComponentProps } from 'types'; +import { stripPathing } from 'utils/misc-utils'; +import { useShallow } from 'zustand/react/shallow'; + +export default function TrialOnlyPrefixHydrator({ + children, +}: BaseComponentProps) { + const bindingsHydrated = useBindingStore((state) => state.hydrated); + const collections = useBinding_collections(); + + const trialOnlyPrefixes = useTenantStore( + useShallow((state) => state.trialStorageOnly) + ); + + const getTrialOnlyPrefixes = useTrialStorageOnly(); + + const newPrefixes = useMemo( + () => + difference( + collections.map((collection) => stripPathing(collection, true)), + trialOnlyPrefixes + ), + [collections, trialOnlyPrefixes] + ); + + useEffect(() => { + if (bindingsHydrated) { + getTrialOnlyPrefixes(newPrefixes).then( + () => {}, + () => {} + ); + } + }, [bindingsHydrated, getTrialOnlyPrefixes, newPrefixes]); + + // eslint-disable-next-line react/jsx-no-useless-fragment + return <>{children}; +} diff --git a/src/components/materialization/types.ts b/src/components/materialization/types.ts index 973874575..52cb80fc8 100644 --- a/src/components/materialization/types.ts +++ b/src/components/materialization/types.ts @@ -1,3 +1,4 @@ export interface TrialOnlyPrefixAlertProps { messageId: string; + bindingUUID?: string; } diff --git a/src/components/shared/Entity/Backfill.tsx b/src/components/shared/Entity/Backfill.tsx index 783b1fc0a..8df6b94ed 100644 --- a/src/components/shared/Entity/Backfill.tsx +++ b/src/components/shared/Entity/Backfill.tsx @@ -15,7 +15,7 @@ export default function Backfill() { } return ( - + - {entityType === 'materialization' ? ( - - - - ) : null} - {draftInitializationError ? ( diff --git a/src/lang/en-US/Workflows.ts b/src/lang/en-US/Workflows.ts index 049603aa6..5d63cc9f2 100644 --- a/src/lang/en-US/Workflows.ts +++ b/src/lang/en-US/Workflows.ts @@ -10,8 +10,8 @@ export const Workflows: Record = { 'workflows.error.endpointConfig.empty': `${endpointConfigHeader} empty`, 'workflows.error.initForm': `An issue was encountered initializing the form.`, 'workflows.error.initFormSection': `An issue was encountered initializing this section of the form.`, - 'workflows.error.oldBoundCollection.generic': `Your account uses Estuary's Trial bucket which includes 20 days of storage. There are collections bound to this materialization that are older than that. Please backfill the following sources after publishing this materialization:`, - 'workflows.error.oldBoundCollection.binding': `Your account uses Estuary's Trial bucket which includes 20 days of storage and this collection is older than that. To ensure you have all data, please also backfill this collection from the source after adding it to the materialization.`, + 'workflows.error.oldBoundCollection.added': `Your account uses Estuary's Trial bucket which includes 20 days of storage and this collection is older than that. To ensure you have all data, please also backfill this collection from the source after adding it to the materialization.`, + 'workflows.error.oldBoundCollection.backfillAll': `Your account uses Estuary's Trial bucket which includes 20 days of storage. There are collections bound to this materialization that are older than that.`, 'workflows.error.oldBoundCollection.backfill': `Your account uses Estuary's Trial bucket which includes 20 days of storage and this collection is older than that. Data will be missing if you backfill from the materialization so we recommend backfilling from the source.`, 'workflows.initTask.alert.title.initFailed': `Form Initialization Error`, diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index 6e164984a..aeb7f2895 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -849,7 +849,11 @@ const getInitialState = ( set(newState, false, 'Binding State Reset'); }, - setBackfilledBindings: (increment, targetBindingUUID) => { + setBackfilledBindings: ( + increment, + targetBindingUUID, + trialOnlyPrefixes + ) => { set( produce((state: BindingState) => { const existingBindingUUIDs = Object.keys(state.resourceConfigs); @@ -869,6 +873,19 @@ const getInitialState = ( hasLength(existingBindingUUIDs) && existingBindingUUIDs.length === state.backfilledBindings.length; + + if (trialOnlyPrefixes && trialOnlyPrefixes.length > 0) { + existingBindingUUIDs.forEach((uuid) => { + state.resourceConfigs[ + uuid + ].meta.sourceBackfillRecommended = + trialOnlyPrefixes.some((prefix) => + state.resourceConfigs[ + uuid + ].meta.collectionName.startsWith(prefix) + ) && state.backfilledBindings.includes(uuid); + }); + } }), false, 'Backfilled Collections Set' diff --git a/src/stores/Binding/types.ts b/src/stores/Binding/types.ts index f04db3542..f3412316a 100644 --- a/src/stores/Binding/types.ts +++ b/src/stores/Binding/types.ts @@ -26,6 +26,7 @@ export interface ResourceConfig extends JsonFormsData { disable?: boolean; onIncompatibleSchemaChange?: string; previouslyDisabled?: boolean; // Used to store if the binding was disabled last time we loaded in bindings + sourceBackfillRecommended?: boolean; }; } @@ -86,7 +87,8 @@ export interface BindingState backfilledBindings: string[]; setBackfilledBindings: ( increment: BooleanString, - targetBindingUUID?: string + targetBindingUUID?: string, + trialOnlyPrefixes?: string[] ) => void; backfillAllBindings: boolean; From 6e3d79faa9d80221cc7a94f812b397280d156a3f Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Fri, 31 Jan 2025 10:03:35 -0500 Subject: [PATCH 03/30] Move trial only storage-related logic out of tenant store --- src/api/liveSpecsExt.ts | 40 ++++++++++ .../Bindings/Backfill/BackfillButton.tsx | 14 +--- .../Bindings/Backfill/SectionWrapper.tsx | 12 +++ .../materialization/TrialOnlyPrefixAlert.tsx | 19 ++--- .../TrialOnlyPrefixHydrator.tsx | 32 ++++++-- src/components/materialization/types.ts | 1 + src/hooks/useTrialStorageOnly.ts | 10 +-- src/stores/Binding/Store.ts | 60 ++++++++++----- src/stores/Binding/types.ts | 8 +- src/stores/Tenant/Store.ts | 48 +----------- src/stores/Tenant/types.ts | 3 - src/stores/TrialMetadata/Store.ts | 73 +++++++++++++++++++ 12 files changed, 217 insertions(+), 103 deletions(-) create mode 100644 src/stores/TrialMetadata/Store.ts diff --git a/src/api/liveSpecsExt.ts b/src/api/liveSpecsExt.ts index e5decc08d..99c38e577 100644 --- a/src/api/liveSpecsExt.ts +++ b/src/api/liveSpecsExt.ts @@ -1,6 +1,7 @@ import { PostgrestResponse } from '@supabase/postgrest-js'; import { supabaseClient } from 'context/GlobalProviders'; import { ProtocolLabel } from 'data-plane-gateway/types/gen/consumer/protocol/consumer'; +import { DateTime } from 'luxon'; import pLimit from 'p-limit'; import { CONNECTOR_IMAGE, @@ -429,6 +430,44 @@ const getLiveSpecShards = (tenant: string, entityType: Entity) => { .eq('spec_type', entityType); }; +export interface TrialCollectionQuery { + catalog_name: string; + updated_at: string; +} + +const getTrialCollections = async (prefixes: string[]) => { + const limiter = pLimit(3); + const promises: Promise>[] = []; + let index = 0; + + const promiseGenerator = (idx: number) => { + const trialThreshold = DateTime.utc().minus({ days: 20 }); + const prefixFilter = prefixes + .slice(idx, idx + CHUNK_SIZE) + .map((prefix) => `catalog_name.like.${prefix}%`) + .join(','); + + return supabaseClient + .from(TABLES.LIVE_SPECS_EXT) + .select('catalog_name,updated_at') + .or(prefixFilter) + .eq('spec_type', 'collection') + .lt('updated_at', trialThreshold); + }; + + while (index < prefixes.length) { + const prom = promiseGenerator(index); + promises.push(limiter(() => prom)); + + index = index + CHUNK_SIZE; + } + + const response = await Promise.all(promises); + const errors = response.filter((r) => r.error); + + return errors[0] ?? { data: response.flatMap((r) => r.data) }; +}; + const liveSpecsExtRelatedColumns = ['catalog_name', 'reads_from', 'id']; export const liveSpecsExtRelatedQuery = liveSpecsExtRelatedColumns.join(','); export interface LiveSpecsExt_Related { @@ -482,4 +521,5 @@ export { getLiveSpecs_entitySelector, getLiveSpecs_existingTasks, getLiveSpecs_materializations, + getTrialCollections, }; diff --git a/src/components/editor/Bindings/Backfill/BackfillButton.tsx b/src/components/editor/Bindings/Backfill/BackfillButton.tsx index c94ad249a..38af8d05f 100644 --- a/src/components/editor/Bindings/Backfill/BackfillButton.tsx +++ b/src/components/editor/Bindings/Backfill/BackfillButton.tsx @@ -20,9 +20,7 @@ import { useFormStateStore_setFormState, } from 'stores/FormState/hooks'; import { FormStatus } from 'stores/FormState/types'; -import { useTenantStore } from 'stores/Tenant/Store'; import { BindingMetadata } from 'types'; -import { useShallow } from 'zustand/react/shallow'; import { useEditorStore_queryResponse_draftSpecs } from '../../Store/hooks'; import BackfillCount from './BackfillCount'; import BackfillDataFlowOption from './BackfillDataFlowOption'; @@ -63,11 +61,6 @@ function BackfillButton({ const formActive = useFormStateStore_isActive(); const setFormState = useFormStateStore_setFormState(); - // Tenant Store - const trialOnlyPrefixes = useTenantStore( - useShallow((state) => state.trialStorageOnly) - ); - const disabled = formActive || collectionsCount < 1 || @@ -157,11 +150,7 @@ function BackfillButton({ ? currentBindingUUID : undefined; - setBackfilledBindings( - increment, - targetBindingUUID, - trialOnlyPrefixes - ); + setBackfilledBindings(increment, targetBindingUUID); setFormState({ status: FormStatus.UPDATED }); }, (error) => { @@ -184,7 +173,6 @@ function BackfillButton({ evaluateServerDifferences, setBackfilledBindings, setFormState, - trialOnlyPrefixes, updateBackfillCounter, ] ); diff --git a/src/components/editor/Bindings/Backfill/SectionWrapper.tsx b/src/components/editor/Bindings/Backfill/SectionWrapper.tsx index 8753fedc8..ca6824af5 100644 --- a/src/components/editor/Bindings/Backfill/SectionWrapper.tsx +++ b/src/components/editor/Bindings/Backfill/SectionWrapper.tsx @@ -2,6 +2,8 @@ import { Box, Stack, Typography } from '@mui/material'; import TrialOnlyPrefixAlert from 'components/materialization/TrialOnlyPrefixAlert'; import { useEntityType } from 'context/EntityContext'; import { useIntl } from 'react-intl'; +import { useBinding_backfilledBindings } from 'stores/Binding/hooks'; +import { useBindingStore } from 'stores/Binding/Store'; import { SectionWrapperProps } from './types'; export default function SectionWrapper({ @@ -13,6 +15,11 @@ export default function SectionWrapper({ const entityType = useEntityType(); + const backfillAllBindings = useBindingStore( + (state) => state.backfillAllBindings + ); + const backfilledBindings = useBinding_backfilledBindings(); + return ( @@ -26,6 +33,11 @@ export default function SectionWrapper({ ) : null} diff --git a/src/components/materialization/TrialOnlyPrefixAlert.tsx b/src/components/materialization/TrialOnlyPrefixAlert.tsx index 7be5c03a1..399003d8c 100644 --- a/src/components/materialization/TrialOnlyPrefixAlert.tsx +++ b/src/components/materialization/TrialOnlyPrefixAlert.tsx @@ -10,22 +10,23 @@ import { TrialOnlyPrefixAlertProps } from './types'; export default function TrialOnlyPrefixAlert({ bindingUUID, messageId, + triggered, }: TrialOnlyPrefixAlertProps) { const intl = useIntl(); const resourceConfigs = useBinding_resourceConfigs(); - const bindingSourceBackfillRecommended = - useBinding_resourceConfigOfMetaBindingProperty( - bindingUUID, - 'sourceBackfillRecommended' - ); + const trialOnlyStorage = useBinding_resourceConfigOfMetaBindingProperty( + bindingUUID, + 'trialOnlyStorage' + ); if ( - (bindingUUID && !bindingSourceBackfillRecommended) || + (bindingUUID && !trialOnlyStorage) || (!bindingUUID && - Object.values(resourceConfigs).some( - (config) => !config.meta.sourceBackfillRecommended - )) + Object.values(resourceConfigs).every( + (config) => !config.meta.trialOnlyStorage + )) || + !triggered ) { return null; } diff --git a/src/components/materialization/TrialOnlyPrefixHydrator.tsx b/src/components/materialization/TrialOnlyPrefixHydrator.tsx index d8c920121..5838adfd3 100644 --- a/src/components/materialization/TrialOnlyPrefixHydrator.tsx +++ b/src/components/materialization/TrialOnlyPrefixHydrator.tsx @@ -1,20 +1,24 @@ +import { getTrialCollections } from 'api/liveSpecsExt'; import useTrialStorageOnly from 'hooks/useTrialStorageOnly'; import { difference } from 'lodash'; import { useEffect, useMemo } from 'react'; import { useBinding_collections } from 'stores/Binding/hooks'; import { useBindingStore } from 'stores/Binding/Store'; -import { useTenantStore } from 'stores/Tenant/Store'; +import { useTrialMetadataStore } from 'stores/TrialMetadata/Store'; import { BaseComponentProps } from 'types'; -import { stripPathing } from 'utils/misc-utils'; +import { hasLength, stripPathing } from 'utils/misc-utils'; import { useShallow } from 'zustand/react/shallow'; export default function TrialOnlyPrefixHydrator({ children, }: BaseComponentProps) { const bindingsHydrated = useBindingStore((state) => state.hydrated); + const setSourceBackfillRecommended = useBindingStore( + (state) => state.setSourceBackfillRecommended + ); const collections = useBinding_collections(); - const trialOnlyPrefixes = useTenantStore( + const trialOnlyPrefixes = useTrialMetadataStore( useShallow((state) => state.trialStorageOnly) ); @@ -32,11 +36,29 @@ export default function TrialOnlyPrefixHydrator({ useEffect(() => { if (bindingsHydrated) { getTrialOnlyPrefixes(newPrefixes).then( - () => {}, + (trialPrefixes) => { + if (hasLength(trialPrefixes)) { + getTrialCollections(trialPrefixes).then( + (response) => { + if (response.error) { + return; + } + + setSourceBackfillRecommended(response.data); + }, + () => {} + ); + } + }, () => {} ); } - }, [bindingsHydrated, getTrialOnlyPrefixes, newPrefixes]); + }, [ + bindingsHydrated, + getTrialOnlyPrefixes, + newPrefixes, + setSourceBackfillRecommended, + ]); // eslint-disable-next-line react/jsx-no-useless-fragment return <>{children}; diff --git a/src/components/materialization/types.ts b/src/components/materialization/types.ts index 52cb80fc8..a918f9d45 100644 --- a/src/components/materialization/types.ts +++ b/src/components/materialization/types.ts @@ -1,4 +1,5 @@ export interface TrialOnlyPrefixAlertProps { messageId: string; + triggered: boolean; bindingUUID?: string; } diff --git a/src/hooks/useTrialStorageOnly.ts b/src/hooks/useTrialStorageOnly.ts index dc1bfa147..3febc0d70 100644 --- a/src/hooks/useTrialStorageOnly.ts +++ b/src/hooks/useTrialStorageOnly.ts @@ -4,7 +4,7 @@ import { useCallback } from 'react'; import { logRocketEvent } from 'services/shared'; import { CustomEvents } from 'services/types'; import { useEntitiesStore_capabilities_adminable } from 'stores/Entities/hooks'; -import { useTenantStore } from 'stores/Tenant/Store'; +import { useTrialMetadataStore } from 'stores/TrialMetadata/Store'; import { hasLength } from 'utils/misc-utils'; const ESTUARY_TRIAL_STORAGE = { @@ -36,7 +36,7 @@ const getTrialStorageOnlyPrefixes = async ( export default function useTrialStorageOnly() { const objectRoles = useEntitiesStore_capabilities_adminable(); - const addTrialStorageOnly = useTenantStore( + const addTrialStorageOnly = useTrialMetadataStore( (state) => state.addTrialStorageOnly ); @@ -54,13 +54,13 @@ export default function useTrialStorageOnly() { return []; } - const trialOnlyPrefixes = await getTrialStorageOnlyPrefixes( + const trialPrefixes = await getTrialStorageOnlyPrefixes( filteredPrefixes ); - addTrialStorageOnly(trialOnlyPrefixes); + addTrialStorageOnly(trialPrefixes); - return trialOnlyPrefixes; + return trialPrefixes; }, [addTrialStorageOnly, objectRoles] ); diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index aeb7f2895..2da16136c 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -849,11 +849,7 @@ const getInitialState = ( set(newState, false, 'Binding State Reset'); }, - setBackfilledBindings: ( - increment, - targetBindingUUID, - trialOnlyPrefixes - ) => { + setBackfilledBindings: (increment, targetBindingUUID) => { set( produce((state: BindingState) => { const existingBindingUUIDs = Object.keys(state.resourceConfigs); @@ -873,19 +869,6 @@ const getInitialState = ( hasLength(existingBindingUUIDs) && existingBindingUUIDs.length === state.backfilledBindings.length; - - if (trialOnlyPrefixes && trialOnlyPrefixes.length > 0) { - existingBindingUUIDs.forEach((uuid) => { - state.resourceConfigs[ - uuid - ].meta.sourceBackfillRecommended = - trialOnlyPrefixes.some((prefix) => - state.resourceConfigs[ - uuid - ].meta.collectionName.startsWith(prefix) - ) && state.backfilledBindings.includes(uuid); - }); - } }), false, 'Backfilled Collections Set' @@ -1027,6 +1010,45 @@ const getInitialState = ( ); }, + setSourceBackfillRecommended: (values) => { + if (!hasLength(values)) { + return; + } + + set( + produce((state: BindingState) => { + const targetCollections = values.map( + ({ catalog_name }) => catalog_name + ); + + const resourceMap: { [collection: string]: string[] } = {}; + + Object.entries(state.resourceConfigs) + .filter(([_uuid, config]) => + targetCollections.includes(config.meta.collectionName) + ) + .forEach(([uuid, config]) => { + const { collectionName } = config.meta; + + if (Object.keys(resourceMap).includes(collectionName)) { + resourceMap[collectionName].push(uuid); + } else { + resourceMap[collectionName] = [uuid]; + } + }); + + values.forEach(({ catalog_name }) => { + resourceMap[catalog_name].forEach((uuid) => { + state.resourceConfigs[uuid].meta.trialOnlyStorage = + true; + }); + }); + }), + false, + 'Source Backfill Recommended Set' + ); + }, + setSpecOnIncompatibleSchemaChange: (value) => { set( produce((state: BindingState) => { @@ -1183,6 +1205,8 @@ const getInitialState = ( meta: { collectionName: targetCollection, bindingIndex: targetResourceConfig.meta.bindingIndex, + trialOnlyStorage: + targetResourceConfig.meta.trialOnlyStorage, }, }; diff --git a/src/stores/Binding/types.ts b/src/stores/Binding/types.ts index f3412316a..117cbf9fa 100644 --- a/src/stores/Binding/types.ts +++ b/src/stores/Binding/types.ts @@ -1,4 +1,5 @@ import { EvolvedCollections } from 'api/evolutions'; +import { TrialCollectionQuery } from 'api/liveSpecsExt'; import { BooleanString } from 'components/shared/buttons/types'; import { LiveSpecsExt_MaterializeOrTransform } from 'hooks/useLiveSpecsExt'; import { DurationObjectUnits } from 'luxon'; @@ -26,7 +27,7 @@ export interface ResourceConfig extends JsonFormsData { disable?: boolean; onIncompatibleSchemaChange?: string; previouslyDisabled?: boolean; // Used to store if the binding was disabled last time we loaded in bindings - sourceBackfillRecommended?: boolean; + trialOnlyStorage?: boolean; }; } @@ -87,8 +88,7 @@ export interface BindingState backfilledBindings: string[]; setBackfilledBindings: ( increment: BooleanString, - targetBindingUUID?: string, - trialOnlyPrefixes?: string[] + targetBindingUUID?: string ) => void; backfillAllBindings: boolean; @@ -109,6 +109,8 @@ export interface BindingState backfillSupported: boolean; setBackfillSupported: (val: BindingState['backfillSupported']) => void; + setSourceBackfillRecommended: (values: TrialCollectionQuery[]) => void; + // Control sourceCapture optional settings sourceCaptureTargetSchemaSupported: boolean; sourceCaptureDeltaUpdatesSupported: boolean; diff --git a/src/stores/Tenant/Store.ts b/src/stores/Tenant/Store.ts index 70ea6791c..f1322dee2 100644 --- a/src/stores/Tenant/Store.ts +++ b/src/stores/Tenant/Store.ts @@ -1,18 +1,12 @@ import produce from 'immer'; -import { pull, union } from 'lodash'; -import { hasLength } from 'utils/misc-utils'; import { devtoolsOptions } from 'utils/store-utils'; import { create, StoreApi } from 'zustand'; import { devtools, NamedSet, persist } from 'zustand/middleware'; import { persistOptions } from './shared'; import { TenantState } from './types'; -const getInitialStateData = (): Pick< - TenantState, - 'selectedTenant' | 'trialStorageOnly' -> => ({ +const getInitialStateData = (): Pick => ({ selectedTenant: '', - trialStorageOnly: [], }); const getInitialState = ( @@ -21,46 +15,6 @@ const getInitialState = ( ): TenantState => ({ ...getInitialStateData(), - addTrialStorageOnly: (values) => { - set( - produce((state: TenantState) => { - if ( - typeof values === 'string' && - !state.trialStorageOnly.includes(values) - ) { - state.trialStorageOnly.push(values); - } - - if (typeof values !== 'string') { - state.trialStorageOnly = hasLength(values) - ? union(state.trialStorageOnly, values) - : []; - } - }), - false, - 'Tenants with trial storage only added' - ); - }, - - removeTrialStorageOnly: (value) => { - set( - produce((state: TenantState) => { - if (value) { - state.trialStorageOnly = pull( - state.trialStorageOnly, - value - ); - - return; - } - - state.trialStorageOnly = []; - }), - false, - 'Tenants with trial storage only removed' - ); - }, - setSelectedTenant: (value) => { set( produce((state: TenantState) => { diff --git a/src/stores/Tenant/types.ts b/src/stores/Tenant/types.ts index d3a981fdd..62b330ca1 100644 --- a/src/stores/Tenant/types.ts +++ b/src/stores/Tenant/types.ts @@ -1,7 +1,4 @@ export interface TenantState { - addTrialStorageOnly: (value: string | string[]) => void; - removeTrialStorageOnly: (value?: string) => void; selectedTenant: string; setSelectedTenant: (value: string) => void; - trialStorageOnly: string[]; } diff --git a/src/stores/TrialMetadata/Store.ts b/src/stores/TrialMetadata/Store.ts new file mode 100644 index 000000000..01fa0e862 --- /dev/null +++ b/src/stores/TrialMetadata/Store.ts @@ -0,0 +1,73 @@ +import produce from 'immer'; +import { pull, union } from 'lodash'; +import { hasLength } from 'utils/misc-utils'; +import { devtoolsOptions } from 'utils/store-utils'; +import { create, StoreApi } from 'zustand'; +import { devtools, NamedSet } from 'zustand/middleware'; + +export interface TrialMetadataState { + addTrialStorageOnly: (value: string | string[]) => void; + removeTrialStorageOnly: (value?: string) => void; + trialStorageOnly: string[]; +} + +const getInitialStateData = (): Pick< + TrialMetadataState, + 'trialStorageOnly' +> => ({ + trialStorageOnly: [], +}); + +const getInitialState = ( + set: NamedSet, + _get: StoreApi['getState'] +): TrialMetadataState => ({ + ...getInitialStateData(), + + addTrialStorageOnly: (values) => { + set( + produce((state: TrialMetadataState) => { + if ( + typeof values === 'string' && + !state.trialStorageOnly.includes(values) + ) { + state.trialStorageOnly.push(values); + } + + if (typeof values !== 'string') { + state.trialStorageOnly = hasLength(values) + ? union(state.trialStorageOnly, values) + : []; + } + }), + false, + 'Prefixes with trial storage only added' + ); + }, + + removeTrialStorageOnly: (value) => { + set( + produce((state: TrialMetadataState) => { + if (value) { + state.trialStorageOnly = pull( + state.trialStorageOnly, + value + ); + + return; + } + + state.trialStorageOnly = []; + }), + false, + 'Prefixes with trial storage only removed' + ); + }, +}); + +export const useTrialMetadataStore = create()( + devtools( + (set, get) => getInitialState(set, get), + devtoolsOptions('trial-metadata') + ) +); From 36fc4ac6facc7b1bab7c1bcbe6614444cecb9ddf Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Mon, 3 Feb 2025 10:00:03 -0500 Subject: [PATCH 04/30] Replace trial only storage hydrator with hook --- .../Bindings/Backfill/BackfillButton.tsx | 6 ++ .../materialization/Create/index.tsx | 46 ++++++------- src/components/materialization/Edit.tsx | 49 +++++++------- .../AddSourceCaptureToSpecButton.tsx | 3 + .../TrialOnlyPrefixHydrator.tsx | 65 ------------------ src/hooks/useTrialCollections.ts | 67 +++++++++++++++++++ src/services/types.ts | 1 + src/stores/Binding/Store.ts | 2 +- src/stores/Binding/types.ts | 2 +- 9 files changed, 123 insertions(+), 118 deletions(-) delete mode 100644 src/components/materialization/TrialOnlyPrefixHydrator.tsx create mode 100644 src/hooks/useTrialCollections.ts diff --git a/src/components/editor/Bindings/Backfill/BackfillButton.tsx b/src/components/editor/Bindings/Backfill/BackfillButton.tsx index 38af8d05f..1b5f3a0d4 100644 --- a/src/components/editor/Bindings/Backfill/BackfillButton.tsx +++ b/src/components/editor/Bindings/Backfill/BackfillButton.tsx @@ -2,6 +2,7 @@ import { Box, Stack, Typography } from '@mui/material'; import BooleanToggleButton from 'components/shared/buttons/BooleanToggleButton'; import { BooleanString } from 'components/shared/buttons/types'; import { useEntityWorkflow } from 'context/Workflow'; +import useTrialCollections from 'hooks/useTrialCollections'; import { useCallback, useMemo } from 'react'; import { useIntl } from 'react-intl'; import { @@ -38,6 +39,7 @@ function BackfillButton({ const { updateBackfillCounter } = useUpdateBackfillCounter(); const workflow = useEntityWorkflow(); + const evaluateTrialCollections = useTrialCollections(); const evolvedCollections = useBindingStore( (state) => state.evolvedCollections @@ -151,6 +153,9 @@ function BackfillButton({ : undefined; setBackfilledBindings(increment, targetBindingUUID); + evaluateTrialCollections( + bindingMetadata.map(({ collection }) => collection) + ); setFormState({ status: FormStatus.UPDATED }); }, (error) => { @@ -171,6 +176,7 @@ function BackfillButton({ currentCollection, draftSpec, evaluateServerDifferences, + evaluateTrialCollections, setBackfilledBindings, setFormState, updateBackfillCounter, diff --git a/src/components/materialization/Create/index.tsx b/src/components/materialization/Create/index.tsx index 03bab902b..323c60e7b 100644 --- a/src/components/materialization/Create/index.tsx +++ b/src/components/materialization/Create/index.tsx @@ -16,7 +16,6 @@ import { useCallback, useEffect } from 'react'; import { CustomEvents } from 'services/types'; import { useDetailsFormStore } from 'stores/DetailsForm/Store'; import WorkflowHydrator from 'stores/Workflow/Hydrator'; -import TrialOnlyPrefixHydrator from '../TrialOnlyPrefixHydrator'; function MaterializationCreate() { usePageTitle({ @@ -59,30 +58,27 @@ function MaterializationCreate() { return ( - - - } - primaryButtonProps={{ - disabled: !draftId, - logEvent: - CustomEvents.MATERIALIZATION_CREATE, - }} - secondaryButtonProps={{ - disabled: !hasConnectors, - logEvent: CustomEvents.MATERIALIZATION_TEST, - }} - /> - } - /> - + + } + primaryButtonProps={{ + disabled: !draftId, + logEvent: CustomEvents.MATERIALIZATION_CREATE, + }} + secondaryButtonProps={{ + disabled: !hasConnectors, + logEvent: CustomEvents.MATERIALIZATION_TEST, + }} + /> + } + /> ); diff --git a/src/components/materialization/Edit.tsx b/src/components/materialization/Edit.tsx index 2a81bc90d..16c833a97 100644 --- a/src/components/materialization/Edit.tsx +++ b/src/components/materialization/Edit.tsx @@ -17,7 +17,6 @@ import usePageTitle from 'hooks/usePageTitle'; import { useCallback } from 'react'; import { CustomEvents } from 'services/types'; import WorkflowHydrator from 'stores/Workflow/Hydrator'; -import TrialOnlyPrefixHydrator from './TrialOnlyPrefixHydrator'; function MaterializationEdit() { usePageTitle({ @@ -49,31 +48,29 @@ function MaterializationEdit() { return ( - - - } - primaryButtonProps={{ - disabled: !draftId, - logEvent: CustomEvents.MATERIALIZATION_EDIT, - }} - secondaryButtonProps={{ - disabled: !hasConnectors, - logEvent: CustomEvents.MATERIALIZATION_TEST, - }} - /> - } - /> - + + } + primaryButtonProps={{ + disabled: !draftId, + logEvent: CustomEvents.MATERIALIZATION_EDIT, + }} + secondaryButtonProps={{ + disabled: !hasConnectors, + logEvent: CustomEvents.MATERIALIZATION_TEST, + }} + /> + } + /> ); diff --git a/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx b/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx index e243949f9..77ed6a009 100644 --- a/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx +++ b/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx @@ -1,6 +1,7 @@ import { Button } from '@mui/material'; import { AddCollectionDialogCTAProps } from 'components/shared/Entity/types'; import invariableStores from 'context/Zustand/invariableStores'; +import useTrialCollections from 'hooks/useTrialCollections'; import { FormattedMessage } from 'react-intl'; import { useBinding_prefillResourceConfigs } from 'stores/Binding/hooks'; import { useBindingStore } from 'stores/Binding/Store'; @@ -18,6 +19,7 @@ function AddSourceCaptureToSpecButton({ toggle }: AddCollectionDialogCTAProps) { ); const { existingSourceCapture, updateDraft } = useSourceCapture(); + const evaluateTrialCollections = useTrialCollections(); const [ sourceCaptureDeltaUpdatesSupported, @@ -80,6 +82,7 @@ function AddSourceCaptureToSpecButton({ toggle }: AddCollectionDialogCTAProps) { if (selectedRow?.writes_to) { prefillResourceConfigs(selectedRow.writes_to, true); + evaluateTrialCollections(selectedRow.writes_to as string[]); } } diff --git a/src/components/materialization/TrialOnlyPrefixHydrator.tsx b/src/components/materialization/TrialOnlyPrefixHydrator.tsx deleted file mode 100644 index 5838adfd3..000000000 --- a/src/components/materialization/TrialOnlyPrefixHydrator.tsx +++ /dev/null @@ -1,65 +0,0 @@ -import { getTrialCollections } from 'api/liveSpecsExt'; -import useTrialStorageOnly from 'hooks/useTrialStorageOnly'; -import { difference } from 'lodash'; -import { useEffect, useMemo } from 'react'; -import { useBinding_collections } from 'stores/Binding/hooks'; -import { useBindingStore } from 'stores/Binding/Store'; -import { useTrialMetadataStore } from 'stores/TrialMetadata/Store'; -import { BaseComponentProps } from 'types'; -import { hasLength, stripPathing } from 'utils/misc-utils'; -import { useShallow } from 'zustand/react/shallow'; - -export default function TrialOnlyPrefixHydrator({ - children, -}: BaseComponentProps) { - const bindingsHydrated = useBindingStore((state) => state.hydrated); - const setSourceBackfillRecommended = useBindingStore( - (state) => state.setSourceBackfillRecommended - ); - const collections = useBinding_collections(); - - const trialOnlyPrefixes = useTrialMetadataStore( - useShallow((state) => state.trialStorageOnly) - ); - - const getTrialOnlyPrefixes = useTrialStorageOnly(); - - const newPrefixes = useMemo( - () => - difference( - collections.map((collection) => stripPathing(collection, true)), - trialOnlyPrefixes - ), - [collections, trialOnlyPrefixes] - ); - - useEffect(() => { - if (bindingsHydrated) { - getTrialOnlyPrefixes(newPrefixes).then( - (trialPrefixes) => { - if (hasLength(trialPrefixes)) { - getTrialCollections(trialPrefixes).then( - (response) => { - if (response.error) { - return; - } - - setSourceBackfillRecommended(response.data); - }, - () => {} - ); - } - }, - () => {} - ); - } - }, [ - bindingsHydrated, - getTrialOnlyPrefixes, - newPrefixes, - setSourceBackfillRecommended, - ]); - - // eslint-disable-next-line react/jsx-no-useless-fragment - return <>{children}; -} diff --git a/src/hooks/useTrialCollections.ts b/src/hooks/useTrialCollections.ts new file mode 100644 index 000000000..b1d452fec --- /dev/null +++ b/src/hooks/useTrialCollections.ts @@ -0,0 +1,67 @@ +import { getTrialCollections } from 'api/liveSpecsExt'; +import { difference, uniq } from 'lodash'; +import { useCallback, useMemo } from 'react'; +import { logRocketEvent } from 'services/shared'; +import { CustomEvents } from 'services/types'; +import { useBinding_collections } from 'stores/Binding/hooks'; +import { useBindingStore } from 'stores/Binding/Store'; +import { useTrialMetadataStore } from 'stores/TrialMetadata/Store'; +import { hasLength, stripPathing } from 'utils/misc-utils'; +import { useShallow } from 'zustand/react/shallow'; +import useTrialStorageOnly from './useTrialStorageOnly'; + +export default function useTrialCollections() { + const setTrialOnlyStorage = useBindingStore( + (state) => state.setTrialOnlyStorage + ); + const collections = useBinding_collections(); + + const storedTrialPrefixes = useTrialMetadataStore( + useShallow((state) => state.trialStorageOnly) + ); + + const getTrialOnlyPrefixes = useTrialStorageOnly(); + + const existingPrefixes = useMemo( + () => + uniq( + collections.map((collection) => stripPathing(collection, true)) + ), + [collections] + ); + + return useCallback( + async (catalogNames?: string[]) => { + const targetPrefixes = catalogNames + ? uniq(catalogNames.map((name) => stripPathing(name, true))) + : existingPrefixes; + + const newPrefixes = difference(targetPrefixes, storedTrialPrefixes); + + if (hasLength(newPrefixes)) { + await getTrialOnlyPrefixes(newPrefixes); + } + + const { data, error } = await getTrialCollections(targetPrefixes); + + if (error) { + logRocketEvent(CustomEvents.TRIAL_STORAGE_COLLECTION_ERROR, { + prefixes: targetPrefixes, + error, + }); + + return []; + } + + setTrialOnlyStorage(data); + + return data; + }, + [ + existingPrefixes, + getTrialOnlyPrefixes, + setTrialOnlyStorage, + storedTrialPrefixes, + ] + ); +} diff --git a/src/services/types.ts b/src/services/types.ts index 927e6ff34..04efec374 100644 --- a/src/services/types.ts +++ b/src/services/types.ts @@ -63,6 +63,7 @@ export enum CustomEvents { SUPABASE_CALL_UNAUTHENTICATED = 'Supabase_Call_Unauthenticated', SWR_LOADING_SLOW = 'SWR_Loading_Slow', TRANSLATION_KEY_MISSING = 'Translation_Key_Missing', + TRIAL_STORAGE_COLLECTION_ERROR = 'TrialStorage:CollectionError', TRIAL_STORAGE_UNKNOWN = 'TrialStorage:Unknown', UPDATE_AVAILABLE = 'Update_Available', URL_FORMAT_ERROR = 'URLFormatError', diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index 2da16136c..ab68117f2 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -1010,7 +1010,7 @@ const getInitialState = ( ); }, - setSourceBackfillRecommended: (values) => { + setTrialOnlyStorage: (values) => { if (!hasLength(values)) { return; } diff --git a/src/stores/Binding/types.ts b/src/stores/Binding/types.ts index 117cbf9fa..a4c7dc74d 100644 --- a/src/stores/Binding/types.ts +++ b/src/stores/Binding/types.ts @@ -109,7 +109,7 @@ export interface BindingState backfillSupported: boolean; setBackfillSupported: (val: BindingState['backfillSupported']) => void; - setSourceBackfillRecommended: (values: TrialCollectionQuery[]) => void; + setTrialOnlyStorage: (values: TrialCollectionQuery[]) => void; // Control sourceCapture optional settings sourceCaptureTargetSchemaSupported: boolean; From 99d99a1eeeecf81b2dde54d7173f3d02cc5e9fcf Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Mon, 3 Feb 2025 13:37:01 -0500 Subject: [PATCH 05/30] Display alert when an old, trial-only collection added --- src/components/collection/ResourceConfig.tsx | 16 +++++++++++++++ .../Bindings/Backfill/BackfillButton.tsx | 12 +++++++++++ .../Bindings/UpdateResourceConfigButton.tsx | 20 ++++++++++++++++--- .../AddSourceCaptureToSpecButton.tsx | 13 ++++++++++-- src/hooks/useTrialCollections.ts | 13 +----------- src/stores/Binding/Store.ts | 8 +++++--- src/stores/Binding/types.ts | 4 +++- 7 files changed, 65 insertions(+), 21 deletions(-) diff --git a/src/components/collection/ResourceConfig.tsx b/src/components/collection/ResourceConfig.tsx index 2adae22da..e50897934 100644 --- a/src/components/collection/ResourceConfig.tsx +++ b/src/components/collection/ResourceConfig.tsx @@ -4,6 +4,7 @@ import AdvancedOptions from 'components/editor/Bindings/AdvancedOptions'; import Backfill from 'components/editor/Bindings/Backfill'; import FieldSelectionViewer from 'components/editor/Bindings/FieldSelection'; import { useEditorStore_queryResponse_draftedBindingIndex } from 'components/editor/Store/hooks'; +import TrialOnlyPrefixAlert from 'components/materialization/TrialOnlyPrefixAlert'; import ErrorBoundryWrapper from 'components/shared/ErrorBoundryWrapper'; import { useEntityType } from 'context/EntityContext'; import { FormattedMessage } from 'react-intl'; @@ -46,8 +47,23 @@ function ResourceConfig({ 'disable' ); + const collectionAdded = useBinding_resourceConfigOfMetaBindingProperty( + bindingUUID, + 'added' + ); + return ( <> + {entityType === 'materialization' ? ( + + + + ) : null} + state.setTrialOnlyStorage + ); + // Draft Editor Store const draftSpecs = useEditorStore_queryResponse_draftSpecs(); @@ -153,9 +157,16 @@ function BackfillButton({ : undefined; setBackfilledBindings(increment, targetBindingUUID); + evaluateTrialCollections( bindingMetadata.map(({ collection }) => collection) + ).then( + (response) => { + setTrialOnlyStorage(response); + }, + () => {} ); + setFormState({ status: FormStatus.UPDATED }); }, (error) => { @@ -179,6 +190,7 @@ function BackfillButton({ evaluateTrialCollections, setBackfilledBindings, setFormState, + setTrialOnlyStorage, updateBackfillCounter, ] ); diff --git a/src/components/editor/Bindings/UpdateResourceConfigButton.tsx b/src/components/editor/Bindings/UpdateResourceConfigButton.tsx index 8bd7da049..02065d4c5 100644 --- a/src/components/editor/Bindings/UpdateResourceConfigButton.tsx +++ b/src/components/editor/Bindings/UpdateResourceConfigButton.tsx @@ -1,6 +1,7 @@ import { Button } from '@mui/material'; import { AddCollectionDialogCTAProps } from 'components/shared/Entity/types'; import invariableStores from 'context/Zustand/invariableStores'; +import useTrialCollections from 'hooks/useTrialCollections'; import { FormattedMessage } from 'react-intl'; import { @@ -8,6 +9,7 @@ import { useBinding_prefillResourceConfigs, useBinding_setRestrictedDiscoveredCollections, } from 'stores/Binding/hooks'; +import { useBindingStore } from 'stores/Binding/Store'; import { hasLength } from 'utils/misc-utils'; import { useStore } from 'zustand'; @@ -20,6 +22,12 @@ function UpdateResourceConfigButton({ toggle }: AddCollectionDialogCTAProps) { } ); + const evaluateTrialCollections = useTrialCollections(); + + const setTrialOnlyStorage = useBindingStore( + (state) => state.setTrialOnlyStorage + ); + const prefillResourceConfigs = useBinding_prefillResourceConfigs(); const discoveredCollections = useBinding_discoveredCollections(); @@ -33,9 +41,15 @@ function UpdateResourceConfigButton({ toggle }: AddCollectionDialogCTAProps) { }; }); - prefillResourceConfigs( - value.map(({ name }) => name), - true + const collections = value.map(({ name }) => name); + + prefillResourceConfigs(collections, true, true); + + evaluateTrialCollections(collections).then( + (response) => { + setTrialOnlyStorage(response); + }, + () => {} ); if (value.length > 0 && hasLength(discoveredCollections)) { diff --git a/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx b/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx index 77ed6a009..677e87ac8 100644 --- a/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx +++ b/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx @@ -21,6 +21,9 @@ function AddSourceCaptureToSpecButton({ toggle }: AddCollectionDialogCTAProps) { const { existingSourceCapture, updateDraft } = useSourceCapture(); const evaluateTrialCollections = useTrialCollections(); + const setTrialOnlyStorage = useBindingStore( + (state) => state.setTrialOnlyStorage + ); const [ sourceCaptureDeltaUpdatesSupported, sourceCaptureTargetSchemaSupported, @@ -81,8 +84,14 @@ function AddSourceCaptureToSpecButton({ toggle }: AddCollectionDialogCTAProps) { setSourceCapture(updatedSourceCapture.capture); if (selectedRow?.writes_to) { - prefillResourceConfigs(selectedRow.writes_to, true); - evaluateTrialCollections(selectedRow.writes_to as string[]); + prefillResourceConfigs(selectedRow.writes_to, true, true); + + const trialCollectionResponse = + await evaluateTrialCollections( + selectedRow.writes_to as string[] + ); + + setTrialOnlyStorage(trialCollectionResponse); } } diff --git a/src/hooks/useTrialCollections.ts b/src/hooks/useTrialCollections.ts index b1d452fec..ab8d32b70 100644 --- a/src/hooks/useTrialCollections.ts +++ b/src/hooks/useTrialCollections.ts @@ -4,16 +4,12 @@ import { useCallback, useMemo } from 'react'; import { logRocketEvent } from 'services/shared'; import { CustomEvents } from 'services/types'; import { useBinding_collections } from 'stores/Binding/hooks'; -import { useBindingStore } from 'stores/Binding/Store'; import { useTrialMetadataStore } from 'stores/TrialMetadata/Store'; import { hasLength, stripPathing } from 'utils/misc-utils'; import { useShallow } from 'zustand/react/shallow'; import useTrialStorageOnly from './useTrialStorageOnly'; export default function useTrialCollections() { - const setTrialOnlyStorage = useBindingStore( - (state) => state.setTrialOnlyStorage - ); const collections = useBinding_collections(); const storedTrialPrefixes = useTrialMetadataStore( @@ -53,15 +49,8 @@ export default function useTrialCollections() { return []; } - setTrialOnlyStorage(data); - return data; }, - [ - existingPrefixes, - getTrialOnlyPrefixes, - setTrialOnlyStorage, - storedTrialPrefixes, - ] + [existingPrefixes, getTrialOnlyPrefixes, storedTrialPrefixes] ); } diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index ab68117f2..bcf637c08 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -519,7 +519,7 @@ const getInitialState = ( ); }, - prefillResourceConfigs: (targetCollections, disableOmit) => { + prefillResourceConfigs: (targetCollections, disableOmit, trackAddition) => { set( produce((state: BindingState) => { const collections = getCollectionNames(state.resourceConfigs); @@ -568,8 +568,9 @@ const getInitialState = ( state.resourceConfigs[bindingUUID] = { ...jsonFormDefaults, meta: { - collectionName, + added: trackAddition, bindingIndex: reducedBindingCount + index, + collectionName, }, }; }); @@ -1203,8 +1204,9 @@ const getInitialState = ( const evaluatedConfig: ResourceConfig = { ...value, meta: { - collectionName: targetCollection, + added: targetResourceConfig.meta.added, bindingIndex: targetResourceConfig.meta.bindingIndex, + collectionName: targetCollection, trialOnlyStorage: targetResourceConfig.meta.trialOnlyStorage, }, diff --git a/src/stores/Binding/types.ts b/src/stores/Binding/types.ts index a4c7dc74d..a16219b56 100644 --- a/src/stores/Binding/types.ts +++ b/src/stores/Binding/types.ts @@ -24,6 +24,7 @@ export interface ResourceConfig extends JsonFormsData { meta: { collectionName: string; bindingIndex: number; + added?: boolean; disable?: boolean; onIncompatibleSchemaChange?: string; previouslyDisabled?: boolean; // Used to store if the binding was disabled last time we loaded in bindings @@ -149,7 +150,8 @@ export interface BindingState // and bindings are added to the specification via the collection selector. prefillResourceConfigs: ( targetCollections: string[], - disableOmit?: boolean + disableOmit?: boolean, + trackAddition?: boolean ) => void; // The combination of resource config store actions, `updateResourceConfig` and From 1e038e5bf32047c47eaf63decdc226a91769b450 Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Mon, 3 Feb 2025 15:06:39 -0500 Subject: [PATCH 06/30] Reduce scope of useTrialCollection hook --- src/hooks/useTrialCollections.ts | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/hooks/useTrialCollections.ts b/src/hooks/useTrialCollections.ts index ab8d32b70..728cde473 100644 --- a/src/hooks/useTrialCollections.ts +++ b/src/hooks/useTrialCollections.ts @@ -1,38 +1,29 @@ import { getTrialCollections } from 'api/liveSpecsExt'; import { difference, uniq } from 'lodash'; -import { useCallback, useMemo } from 'react'; +import { useCallback } from 'react'; import { logRocketEvent } from 'services/shared'; import { CustomEvents } from 'services/types'; -import { useBinding_collections } from 'stores/Binding/hooks'; import { useTrialMetadataStore } from 'stores/TrialMetadata/Store'; import { hasLength, stripPathing } from 'utils/misc-utils'; import { useShallow } from 'zustand/react/shallow'; import useTrialStorageOnly from './useTrialStorageOnly'; export default function useTrialCollections() { - const collections = useBinding_collections(); - const storedTrialPrefixes = useTrialMetadataStore( useShallow((state) => state.trialStorageOnly) ); const getTrialOnlyPrefixes = useTrialStorageOnly(); - const existingPrefixes = useMemo( - () => - uniq( - collections.map((collection) => stripPathing(collection, true)) - ), - [collections] - ); - return useCallback( async (catalogNames?: string[]) => { const targetPrefixes = catalogNames ? uniq(catalogNames.map((name) => stripPathing(name, true))) - : existingPrefixes; + : []; - const newPrefixes = difference(targetPrefixes, storedTrialPrefixes); + const newPrefixes = hasLength(targetPrefixes) + ? difference(targetPrefixes, storedTrialPrefixes) + : []; if (hasLength(newPrefixes)) { await getTrialOnlyPrefixes(newPrefixes); @@ -51,6 +42,6 @@ export default function useTrialCollections() { return data; }, - [existingPrefixes, getTrialOnlyPrefixes, storedTrialPrefixes] + [getTrialOnlyPrefixes, storedTrialPrefixes] ); } From 98ec30e0707cc97720d9f712d3a7f756bba65d1f Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Wed, 5 Feb 2025 10:14:47 -0500 Subject: [PATCH 07/30] Correct backfill alert triggers --- .../Bindings/Backfill/SectionWrapper.tsx | 7 ++-- .../materialization/TrialOnlyPrefixAlert.tsx | 36 ++++++++++++++++--- src/components/materialization/shared.ts | 15 ++++++++ src/stores/Binding/Store.ts | 22 ++++++++---- src/stores/Binding/types.ts | 1 + 5 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 src/components/materialization/shared.ts diff --git a/src/components/editor/Bindings/Backfill/SectionWrapper.tsx b/src/components/editor/Bindings/Backfill/SectionWrapper.tsx index ca6824af5..7c77ace52 100644 --- a/src/components/editor/Bindings/Backfill/SectionWrapper.tsx +++ b/src/components/editor/Bindings/Backfill/SectionWrapper.tsx @@ -3,7 +3,7 @@ import TrialOnlyPrefixAlert from 'components/materialization/TrialOnlyPrefixAler import { useEntityType } from 'context/EntityContext'; import { useIntl } from 'react-intl'; import { useBinding_backfilledBindings } from 'stores/Binding/hooks'; -import { useBindingStore } from 'stores/Binding/Store'; +import { hasLength } from 'utils/misc-utils'; import { SectionWrapperProps } from './types'; export default function SectionWrapper({ @@ -15,9 +15,6 @@ export default function SectionWrapper({ const entityType = useEntityType(); - const backfillAllBindings = useBindingStore( - (state) => state.backfillAllBindings - ); const backfilledBindings = useBinding_backfilledBindings(); return ( @@ -36,7 +33,7 @@ export default function SectionWrapper({ triggered={ bindingUUID ? backfilledBindings.includes(bindingUUID) - : backfillAllBindings + : hasLength(backfilledBindings) } /> ) : null} diff --git a/src/components/materialization/TrialOnlyPrefixAlert.tsx b/src/components/materialization/TrialOnlyPrefixAlert.tsx index 399003d8c..f61f08eab 100644 --- a/src/components/materialization/TrialOnlyPrefixAlert.tsx +++ b/src/components/materialization/TrialOnlyPrefixAlert.tsx @@ -1,10 +1,13 @@ import { Typography } from '@mui/material'; import AlertBox from 'components/shared/AlertBox'; +import { useMemo } from 'react'; import { useIntl } from 'react-intl'; import { + useBinding_backfilledBindings, useBinding_resourceConfigOfMetaBindingProperty, useBinding_resourceConfigs, } from 'stores/Binding/hooks'; +import { isBeforeTrialInterval } from './shared'; import { TrialOnlyPrefixAlertProps } from './types'; export default function TrialOnlyPrefixAlert({ @@ -14,18 +17,43 @@ export default function TrialOnlyPrefixAlert({ }: TrialOnlyPrefixAlertProps) { const intl = useIntl(); + const backfilledBindings = useBinding_backfilledBindings(); const resourceConfigs = useBinding_resourceConfigs(); const trialOnlyStorage = useBinding_resourceConfigOfMetaBindingProperty( bindingUUID, 'trialOnlyStorage' ); + const updatedAt = useBinding_resourceConfigOfMetaBindingProperty( + bindingUUID, + 'updatedAt' + ); + + const hideBindingLevelAlert = useMemo( + () => + Boolean( + bindingUUID && + (!trialOnlyStorage || + (trialOnlyStorage && + !isBeforeTrialInterval( + typeof updatedAt === 'string' + ? updatedAt + : undefined + ))) + ), + [bindingUUID, trialOnlyStorage, updatedAt] + ); if ( - (bindingUUID && !trialOnlyStorage) || + hideBindingLevelAlert || (!bindingUUID && - Object.values(resourceConfigs).every( - (config) => !config.meta.trialOnlyStorage - )) || + backfilledBindings.every((uuid) => { + const config = resourceConfigs[uuid]; + + return ( + !config.meta.trialOnlyStorage || + !isBeforeTrialInterval(config.meta.updatedAt) + ); + })) || !triggered ) { return null; diff --git a/src/components/materialization/shared.ts b/src/components/materialization/shared.ts new file mode 100644 index 000000000..13899ede7 --- /dev/null +++ b/src/components/materialization/shared.ts @@ -0,0 +1,15 @@ +import { DateTime, Interval } from 'luxon'; + +export const isBeforeTrialInterval = (timestamp: string | undefined) => { + return ( + typeof timestamp === 'string' && + Interval.fromDateTimes( + DateTime.utc().minus({ days: 20 }), + DateTime.utc() + ).isAfter( + DateTime.fromISO(timestamp, { + zone: 'utc', + }) + ) + ); +}; diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index bcf637c08..5990546d6 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -1022,7 +1022,8 @@ const getInitialState = ( ({ catalog_name }) => catalog_name ); - const resourceMap: { [collection: string]: string[] } = {}; + const mappedBindingUUIDs: { [collection: string]: string[] } = + {}; Object.entries(state.resourceConfigs) .filter(([_uuid, config]) => @@ -1031,22 +1032,28 @@ const getInitialState = ( .forEach(([uuid, config]) => { const { collectionName } = config.meta; - if (Object.keys(resourceMap).includes(collectionName)) { - resourceMap[collectionName].push(uuid); + if ( + Object.keys(mappedBindingUUIDs).includes( + collectionName + ) + ) { + mappedBindingUUIDs[collectionName].push(uuid); } else { - resourceMap[collectionName] = [uuid]; + mappedBindingUUIDs[collectionName] = [uuid]; } }); - values.forEach(({ catalog_name }) => { - resourceMap[catalog_name].forEach((uuid) => { + values.forEach(({ catalog_name, updated_at }) => { + mappedBindingUUIDs[catalog_name].forEach((uuid) => { state.resourceConfigs[uuid].meta.trialOnlyStorage = true; + + state.resourceConfigs[uuid].meta.updatedAt = updated_at; }); }); }), false, - 'Source Backfill Recommended Set' + 'Trial Only Storage Set' ); }, @@ -1209,6 +1216,7 @@ const getInitialState = ( collectionName: targetCollection, trialOnlyStorage: targetResourceConfig.meta.trialOnlyStorage, + updatedAt: targetResourceConfig.meta.updatedAt, }, }; diff --git a/src/stores/Binding/types.ts b/src/stores/Binding/types.ts index a16219b56..5cf065143 100644 --- a/src/stores/Binding/types.ts +++ b/src/stores/Binding/types.ts @@ -29,6 +29,7 @@ export interface ResourceConfig extends JsonFormsData { onIncompatibleSchemaChange?: string; previouslyDisabled?: boolean; // Used to store if the binding was disabled last time we loaded in bindings trialOnlyStorage?: boolean; + updatedAt?: string; }; } From ff3ef0f4c56c5802f26bcfa537ed1e8fdfaee1b8 Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Wed, 5 Feb 2025 13:59:05 -0500 Subject: [PATCH 08/30] Rename and simplify setTrialOnlyStorage action --- .../Bindings/Backfill/BackfillButton.tsx | 8 ++--- .../Bindings/UpdateResourceConfigButton.tsx | 6 ++-- .../AddSourceCaptureToSpecButton.tsx | 6 ++-- src/stores/Binding/Store.ts | 29 ++----------------- src/stores/Binding/types.ts | 2 +- 5 files changed, 13 insertions(+), 38 deletions(-) diff --git a/src/components/editor/Bindings/Backfill/BackfillButton.tsx b/src/components/editor/Bindings/Backfill/BackfillButton.tsx index 7e5e43136..766a5f994 100644 --- a/src/components/editor/Bindings/Backfill/BackfillButton.tsx +++ b/src/components/editor/Bindings/Backfill/BackfillButton.tsx @@ -56,8 +56,8 @@ function BackfillButton({ const setBackfilledBindings = useBinding_setBackfilledBindings(); const backfillSupported = useBinding_backfillSupported(); - const setTrialOnlyStorage = useBindingStore( - (state) => state.setTrialOnlyStorage + const setCollectionMetadata = useBindingStore( + (state) => state.setCollectionMetadata ); // Draft Editor Store @@ -162,7 +162,7 @@ function BackfillButton({ bindingMetadata.map(({ collection }) => collection) ).then( (response) => { - setTrialOnlyStorage(response); + setCollectionMetadata(response); }, () => {} ); @@ -189,8 +189,8 @@ function BackfillButton({ evaluateServerDifferences, evaluateTrialCollections, setBackfilledBindings, + setCollectionMetadata, setFormState, - setTrialOnlyStorage, updateBackfillCounter, ] ); diff --git a/src/components/editor/Bindings/UpdateResourceConfigButton.tsx b/src/components/editor/Bindings/UpdateResourceConfigButton.tsx index 02065d4c5..95ed3abd2 100644 --- a/src/components/editor/Bindings/UpdateResourceConfigButton.tsx +++ b/src/components/editor/Bindings/UpdateResourceConfigButton.tsx @@ -24,8 +24,8 @@ function UpdateResourceConfigButton({ toggle }: AddCollectionDialogCTAProps) { const evaluateTrialCollections = useTrialCollections(); - const setTrialOnlyStorage = useBindingStore( - (state) => state.setTrialOnlyStorage + const setCollectionMetadata = useBindingStore( + (state) => state.setCollectionMetadata ); const prefillResourceConfigs = useBinding_prefillResourceConfigs(); @@ -47,7 +47,7 @@ function UpdateResourceConfigButton({ toggle }: AddCollectionDialogCTAProps) { evaluateTrialCollections(collections).then( (response) => { - setTrialOnlyStorage(response); + setCollectionMetadata(response); }, () => {} ); diff --git a/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx b/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx index 677e87ac8..c82a78bdf 100644 --- a/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx +++ b/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx @@ -21,8 +21,8 @@ function AddSourceCaptureToSpecButton({ toggle }: AddCollectionDialogCTAProps) { const { existingSourceCapture, updateDraft } = useSourceCapture(); const evaluateTrialCollections = useTrialCollections(); - const setTrialOnlyStorage = useBindingStore( - (state) => state.setTrialOnlyStorage + const setCollectionMetadata = useBindingStore( + (state) => state.setCollectionMetadata ); const [ sourceCaptureDeltaUpdatesSupported, @@ -91,7 +91,7 @@ function AddSourceCaptureToSpecButton({ toggle }: AddCollectionDialogCTAProps) { selectedRow.writes_to as string[] ); - setTrialOnlyStorage(trialCollectionResponse); + setCollectionMetadata(trialCollectionResponse); } } diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index 5990546d6..b9075028d 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -1011,40 +1011,15 @@ const getInitialState = ( ); }, - setTrialOnlyStorage: (values) => { + setCollectionMetadata: (values) => { if (!hasLength(values)) { return; } set( produce((state: BindingState) => { - const targetCollections = values.map( - ({ catalog_name }) => catalog_name - ); - - const mappedBindingUUIDs: { [collection: string]: string[] } = - {}; - - Object.entries(state.resourceConfigs) - .filter(([_uuid, config]) => - targetCollections.includes(config.meta.collectionName) - ) - .forEach(([uuid, config]) => { - const { collectionName } = config.meta; - - if ( - Object.keys(mappedBindingUUIDs).includes( - collectionName - ) - ) { - mappedBindingUUIDs[collectionName].push(uuid); - } else { - mappedBindingUUIDs[collectionName] = [uuid]; - } - }); - values.forEach(({ catalog_name, updated_at }) => { - mappedBindingUUIDs[catalog_name].forEach((uuid) => { + state.bindings[catalog_name].forEach((uuid) => { state.resourceConfigs[uuid].meta.trialOnlyStorage = true; diff --git a/src/stores/Binding/types.ts b/src/stores/Binding/types.ts index 5cf065143..26cd1af87 100644 --- a/src/stores/Binding/types.ts +++ b/src/stores/Binding/types.ts @@ -111,7 +111,7 @@ export interface BindingState backfillSupported: boolean; setBackfillSupported: (val: BindingState['backfillSupported']) => void; - setTrialOnlyStorage: (values: TrialCollectionQuery[]) => void; + setCollectionMetadata: (values: TrialCollectionQuery[]) => void; // Control sourceCapture optional settings sourceCaptureTargetSchemaSupported: boolean; From 28b01d7d5ce54678d2292e31ce1ebb1c18d7e63f Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Thu, 6 Feb 2025 10:34:13 -0500 Subject: [PATCH 09/30] Add soureBackfillRecommended flag to binding metadata --- src/components/collection/Config.tsx | 8 ++++- .../Bindings/Backfill/SectionWrapper.tsx | 12 +++++-- .../editor/Bindings/Row/ErrorIndicator.tsx | 19 ++++++++-- .../materialization/TrialOnlyPrefixAlert.tsx | 36 ++++--------------- src/stores/Binding/Store.ts | 18 +++++++--- src/stores/Binding/types.ts | 1 + 6 files changed, 56 insertions(+), 38 deletions(-) diff --git a/src/components/collection/Config.tsx b/src/components/collection/Config.tsx index 9f06ac10e..2fa7a19fc 100644 --- a/src/components/collection/Config.tsx +++ b/src/components/collection/Config.tsx @@ -14,6 +14,7 @@ import { useBinding_hydrationErrorsExist, useBinding_resourceConfigErrorsExist, } from 'stores/Binding/hooks'; +import { useBindingStore } from 'stores/Binding/Store'; import { useFormStateStore_messagePrefix } from 'stores/FormState/hooks'; interface Props { @@ -36,6 +37,11 @@ function CollectionConfig({ const resourceConfigErrorsExist = useBinding_resourceConfigErrorsExist(); const bindingErrorsExist = useBinding_bindingErrorsExist(); const fullSourceErrorsExist = useBinding_fullSourceErrorsExist(); + const sourceBackfillRecommended = useBindingStore((state) => + Object.values(state.resourceConfigs).some( + (config) => config.meta.sourceBackfillRecommended + ) + ); // Form State Store const messagePrefix = useFormStateStore_messagePrefix(); @@ -45,7 +51,7 @@ function CollectionConfig({ resourceConfigErrorsExist || fullSourceErrorsExist; - const hasWarnings = bindingErrorsExist; + const hasWarnings = bindingErrorsExist || sourceBackfillRecommended; return ( @@ -32,7 +40,7 @@ export default function SectionWrapper({ messageId={alertMessageId} triggered={ bindingUUID - ? backfilledBindings.includes(bindingUUID) + ? Boolean(bindingSourceBackfillRecommended) : hasLength(backfilledBindings) } /> diff --git a/src/components/editor/Bindings/Row/ErrorIndicator.tsx b/src/components/editor/Bindings/Row/ErrorIndicator.tsx index 1b37a9fd3..bdd1c876e 100644 --- a/src/components/editor/Bindings/Row/ErrorIndicator.tsx +++ b/src/components/editor/Bindings/Row/ErrorIndicator.tsx @@ -3,6 +3,7 @@ import { WarningCircle } from 'iconoir-react'; import { useBinding_fullSourceOfBindingProperty, useBinding_resourceConfigOfBindingProperty, + useBinding_resourceConfigOfMetaBindingProperty, } from 'stores/Binding/hooks'; interface Props { @@ -22,14 +23,28 @@ function BindingsSelectorErrorIndicator({ bindingUUID }: Props) { 'errors' ); - if (bindingErrors?.length > 0 || configErrors?.length > 0) { + const sourceBackfillRecommended = + useBinding_resourceConfigOfMetaBindingProperty( + bindingUUID, + 'sourceBackfillRecommended' + ); + + if ( + bindingErrors?.length > 0 || + configErrors?.length > 0 || + Boolean(sourceBackfillRecommended) + ) { return ( 0 || + configErrors?.length > 0 + ? theme.palette.error.main + : theme.palette.warning.main, }} /> diff --git a/src/components/materialization/TrialOnlyPrefixAlert.tsx b/src/components/materialization/TrialOnlyPrefixAlert.tsx index f61f08eab..84cb7b676 100644 --- a/src/components/materialization/TrialOnlyPrefixAlert.tsx +++ b/src/components/materialization/TrialOnlyPrefixAlert.tsx @@ -4,7 +4,6 @@ import { useMemo } from 'react'; import { useIntl } from 'react-intl'; import { useBinding_backfilledBindings, - useBinding_resourceConfigOfMetaBindingProperty, useBinding_resourceConfigs, } from 'stores/Binding/hooks'; import { isBeforeTrialInterval } from './shared'; @@ -19,33 +18,10 @@ export default function TrialOnlyPrefixAlert({ const backfilledBindings = useBinding_backfilledBindings(); const resourceConfigs = useBinding_resourceConfigs(); - const trialOnlyStorage = useBinding_resourceConfigOfMetaBindingProperty( - bindingUUID, - 'trialOnlyStorage' - ); - const updatedAt = useBinding_resourceConfigOfMetaBindingProperty( - bindingUUID, - 'updatedAt' - ); - const hideBindingLevelAlert = useMemo( + const hideTopLevelAlert = useMemo( () => - Boolean( - bindingUUID && - (!trialOnlyStorage || - (trialOnlyStorage && - !isBeforeTrialInterval( - typeof updatedAt === 'string' - ? updatedAt - : undefined - ))) - ), - [bindingUUID, trialOnlyStorage, updatedAt] - ); - - if ( - hideBindingLevelAlert || - (!bindingUUID && + !bindingUUID && backfilledBindings.every((uuid) => { const config = resourceConfigs[uuid]; @@ -53,9 +29,11 @@ export default function TrialOnlyPrefixAlert({ !config.meta.trialOnlyStorage || !isBeforeTrialInterval(config.meta.updatedAt) ); - })) || - !triggered - ) { + }), + [backfilledBindings, bindingUUID, resourceConfigs] + ); + + if (hideTopLevelAlert || !triggered) { return null; } diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index b9075028d..ae68cbd52 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -5,6 +5,7 @@ import { getLiveSpecsByLiveSpecId, getSchema_Resource, } from 'api/hydration'; +import { isBeforeTrialInterval } from 'components/materialization/shared'; import { GlobalSearchParams } from 'hooks/searchParams/useGlobalSearchParams'; import { LiveSpecsExtQuery } from 'hooks/useLiveSpecsExt'; import produce from 'immer'; @@ -1020,10 +1021,17 @@ const getInitialState = ( produce((state: BindingState) => { values.forEach(({ catalog_name, updated_at }) => { state.bindings[catalog_name].forEach((uuid) => { - state.resourceConfigs[uuid].meta.trialOnlyStorage = - true; - - state.resourceConfigs[uuid].meta.updatedAt = updated_at; + const triggered = + state.backfilledBindings.includes(uuid) || + state.resourceConfigs[uuid].meta.added; + + state.resourceConfigs[uuid].meta = { + ...state.resourceConfigs[uuid].meta, + sourceBackfillRecommended: + triggered && isBeforeTrialInterval(updated_at), + trialOnlyStorage: true, + updatedAt: updated_at, + }; }); }); }), @@ -1189,6 +1197,8 @@ const getInitialState = ( added: targetResourceConfig.meta.added, bindingIndex: targetResourceConfig.meta.bindingIndex, collectionName: targetCollection, + sourceBackfillRecommended: + targetResourceConfig.meta.sourceBackfillRecommended, trialOnlyStorage: targetResourceConfig.meta.trialOnlyStorage, updatedAt: targetResourceConfig.meta.updatedAt, diff --git a/src/stores/Binding/types.ts b/src/stores/Binding/types.ts index 26cd1af87..d37b39e59 100644 --- a/src/stores/Binding/types.ts +++ b/src/stores/Binding/types.ts @@ -28,6 +28,7 @@ export interface ResourceConfig extends JsonFormsData { disable?: boolean; onIncompatibleSchemaChange?: string; previouslyDisabled?: boolean; // Used to store if the binding was disabled last time we loaded in bindings + sourceBackfillRecommended?: boolean; trialOnlyStorage?: boolean; updatedAt?: string; }; From 880248b1bbda4104c55f4e050d085149989d1b32 Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Fri, 7 Feb 2025 10:08:51 -0500 Subject: [PATCH 10/30] Evaluate trial collections when binding store hydrated --- src/hooks/useTrialCollections.ts | 62 ++++++++++++++++++++------------ src/stores/Binding/Hydrator.tsx | 5 +++ src/stores/Binding/Store.ts | 12 +++++++ src/stores/Binding/types.ts | 1 + 4 files changed, 57 insertions(+), 23 deletions(-) diff --git a/src/hooks/useTrialCollections.ts b/src/hooks/useTrialCollections.ts index 728cde473..d448d4319 100644 --- a/src/hooks/useTrialCollections.ts +++ b/src/hooks/useTrialCollections.ts @@ -8,6 +8,39 @@ import { hasLength, stripPathing } from 'utils/misc-utils'; import { useShallow } from 'zustand/react/shallow'; import useTrialStorageOnly from './useTrialStorageOnly'; +// This function was created and exported so the binding store hydrator +// can use the same logic to evaluate trial collection as the core hook. +export const evaluateTrialCollections = async ( + catalogNames: string[] | undefined, + getTrialOnlyPrefixes: (prefixes: string[]) => Promise, + storedTrialPrefixes: string[] +) => { + const targetPrefixes = catalogNames + ? uniq(catalogNames.map((name) => stripPathing(name, true))) + : []; + + const newPrefixes = hasLength(targetPrefixes) + ? difference(targetPrefixes, storedTrialPrefixes) + : []; + + if (hasLength(newPrefixes)) { + await getTrialOnlyPrefixes(newPrefixes); + } + + const { data, error } = await getTrialCollections(targetPrefixes); + + if (error) { + logRocketEvent(CustomEvents.TRIAL_STORAGE_COLLECTION_ERROR, { + prefixes: targetPrefixes, + error, + }); + + return []; + } + + return data; +}; + export default function useTrialCollections() { const storedTrialPrefixes = useTrialMetadataStore( useShallow((state) => state.trialStorageOnly) @@ -17,30 +50,13 @@ export default function useTrialCollections() { return useCallback( async (catalogNames?: string[]) => { - const targetPrefixes = catalogNames - ? uniq(catalogNames.map((name) => stripPathing(name, true))) - : []; - - const newPrefixes = hasLength(targetPrefixes) - ? difference(targetPrefixes, storedTrialPrefixes) - : []; - - if (hasLength(newPrefixes)) { - await getTrialOnlyPrefixes(newPrefixes); - } - - const { data, error } = await getTrialCollections(targetPrefixes); - - if (error) { - logRocketEvent(CustomEvents.TRIAL_STORAGE_COLLECTION_ERROR, { - prefixes: targetPrefixes, - error, - }); - - return []; - } + const trialCollections = await evaluateTrialCollections( + catalogNames, + getTrialOnlyPrefixes, + storedTrialPrefixes + ); - return data; + return trialCollections; }, [getTrialOnlyPrefixes, storedTrialPrefixes] ); diff --git a/src/stores/Binding/Hydrator.tsx b/src/stores/Binding/Hydrator.tsx index fedee19a0..57dfd9af4 100644 --- a/src/stores/Binding/Hydrator.tsx +++ b/src/stores/Binding/Hydrator.tsx @@ -1,5 +1,6 @@ import { useEntityType } from 'context/EntityContext'; import { useEntityWorkflow, useEntityWorkflow_Editing } from 'context/Workflow'; +import useTrialStorageOnly from 'hooks/useTrialStorageOnly'; import { useEffect, useRef } from 'react'; import { logRocketConsole } from 'services/shared'; import { useDetailsFormStore } from 'stores/DetailsForm/Store'; @@ -21,6 +22,8 @@ export const BindingHydrator = ({ children }: BaseComponentProps) => { const workflow = useEntityWorkflow(); const editWorkflow = useEntityWorkflow_Editing(); + const getTrialOnlyPrefixes = useTrialStorageOnly(); + const connectorTagId = useDetailsFormStore( (state) => state.details.data.connectorImage.id ); @@ -44,6 +47,7 @@ export const BindingHydrator = ({ children }: BaseComponentProps) => { editWorkflow, entityType, connectorTagId, + getTrialOnlyPrefixes, rehydrating.current ) .then( @@ -77,6 +81,7 @@ export const BindingHydrator = ({ children }: BaseComponentProps) => { connectorTagId, editWorkflow, entityType, + getTrialOnlyPrefixes, hydrateState, setActive, setHydrated, diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index ae68cbd52..c947333bf 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -8,6 +8,7 @@ import { import { isBeforeTrialInterval } from 'components/materialization/shared'; import { GlobalSearchParams } from 'hooks/searchParams/useGlobalSearchParams'; import { LiveSpecsExtQuery } from 'hooks/useLiveSpecsExt'; +import { evaluateTrialCollections } from 'hooks/useTrialCollections'; import produce from 'immer'; import { difference, @@ -89,6 +90,7 @@ const hydrateSpecificationDependentState = async ( entityType: Entity, fallbackInterval: string | null, get: StoreApi['getState'], + getTrialOnlyPrefixes: (prefixes: string[]) => Promise, liveSpec: LiveSpecsExtQuery['spec'], searchParams: URLSearchParams ): Promise => { @@ -140,6 +142,14 @@ const hydrateSpecificationDependentState = async ( ); } + const trialCollections = await evaluateTrialCollections( + Object.keys(get().bindings), + getTrialOnlyPrefixes, + [] + ); + + get().setCollectionMetadata(trialCollections); + return null; }; @@ -356,6 +366,7 @@ const getInitialState = ( editWorkflow, entityType, connectorTagId, + getTrialOnlyPrefixes, rehydrating ) => { const searchParams = new URLSearchParams(window.location.search); @@ -400,6 +411,7 @@ const getInitialState = ( entityType, fallbackInterval, get, + getTrialOnlyPrefixes, liveSpecs[0].spec, searchParams ); diff --git a/src/stores/Binding/types.ts b/src/stores/Binding/types.ts index d37b39e59..97ce2a0f6 100644 --- a/src/stores/Binding/types.ts +++ b/src/stores/Binding/types.ts @@ -189,6 +189,7 @@ export interface BindingState editWorkflow: boolean, entityType: Entity, connectorTagId: string, + getTrialOnlyPrefixes: (prefixes: string[]) => Promise, rehydrating?: boolean ) => Promise; From 846e8242509f3c1bd27ca663bdc806892a0cbdcb Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Mon, 10 Feb 2025 13:22:33 -0500 Subject: [PATCH 11/30] Consolidate log rocket events --- src/hooks/useTrialCollections.ts | 5 ++++- src/hooks/useTrialStorageOnly.ts | 2 +- src/services/types.ts | 3 +-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/hooks/useTrialCollections.ts b/src/hooks/useTrialCollections.ts index d448d4319..098389a1f 100644 --- a/src/hooks/useTrialCollections.ts +++ b/src/hooks/useTrialCollections.ts @@ -23,6 +23,9 @@ export const evaluateTrialCollections = async ( ? difference(targetPrefixes, storedTrialPrefixes) : []; + // There is an implicit dependency on this function as it keeps the + // trial metadata store state in sync whenever trial collections need + // to be evaluated. if (hasLength(newPrefixes)) { await getTrialOnlyPrefixes(newPrefixes); } @@ -30,7 +33,7 @@ export const evaluateTrialCollections = async ( const { data, error } = await getTrialCollections(targetPrefixes); if (error) { - logRocketEvent(CustomEvents.TRIAL_STORAGE_COLLECTION_ERROR, { + logRocketEvent(CustomEvents.TRIAL_STORAGE, { prefixes: targetPrefixes, error, }); diff --git a/src/hooks/useTrialStorageOnly.ts b/src/hooks/useTrialStorageOnly.ts index 3febc0d70..ad9c0fdb1 100644 --- a/src/hooks/useTrialStorageOnly.ts +++ b/src/hooks/useTrialStorageOnly.ts @@ -19,7 +19,7 @@ const getTrialStorageOnlyPrefixes = async ( const { data, error } = await getStorageMappingStores(prefixes); if (error || !data) { - logRocketEvent(CustomEvents.TRIAL_STORAGE_UNKNOWN, { prefixes, error }); + logRocketEvent(CustomEvents.TRIAL_STORAGE, { prefixes, error }); return []; } diff --git a/src/services/types.ts b/src/services/types.ts index 04efec374..cdaeeed13 100644 --- a/src/services/types.ts +++ b/src/services/types.ts @@ -63,8 +63,7 @@ export enum CustomEvents { SUPABASE_CALL_UNAUTHENTICATED = 'Supabase_Call_Unauthenticated', SWR_LOADING_SLOW = 'SWR_Loading_Slow', TRANSLATION_KEY_MISSING = 'Translation_Key_Missing', - TRIAL_STORAGE_COLLECTION_ERROR = 'TrialStorage:CollectionError', - TRIAL_STORAGE_UNKNOWN = 'TrialStorage:Unknown', + TRIAL_STORAGE = 'TrialStorage', UPDATE_AVAILABLE = 'Update_Available', URL_FORMAT_ERROR = 'URLFormatError', } From 02b8b467913a8ebbde73fda431e3e16a9eb105dc Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Tue, 11 Feb 2025 09:52:50 -0500 Subject: [PATCH 12/30] Move and rename trial storage-related hooks --- .../editor/Bindings/Backfill/BackfillButton.tsx | 2 +- .../Bindings/UpdateResourceConfigButton.tsx | 2 +- .../AddSourceCaptureToSpecButton.tsx | 2 +- .../{ => trialStorage}/useTrialCollections.ts | 12 ++++++------ .../useTrialPrefixes.ts} | 16 ++++++---------- src/stores/Binding/Hydrator.tsx | 8 ++++---- src/stores/Binding/Store.ts | 2 +- 7 files changed, 20 insertions(+), 24 deletions(-) rename src/hooks/{ => trialStorage}/useTrialCollections.ts (85%) rename src/hooks/{useTrialStorageOnly.ts => trialStorage/useTrialPrefixes.ts} (79%) diff --git a/src/components/editor/Bindings/Backfill/BackfillButton.tsx b/src/components/editor/Bindings/Backfill/BackfillButton.tsx index 766a5f994..a4d318d5d 100644 --- a/src/components/editor/Bindings/Backfill/BackfillButton.tsx +++ b/src/components/editor/Bindings/Backfill/BackfillButton.tsx @@ -2,7 +2,7 @@ import { Box, Stack, Typography } from '@mui/material'; import BooleanToggleButton from 'components/shared/buttons/BooleanToggleButton'; import { BooleanString } from 'components/shared/buttons/types'; import { useEntityWorkflow } from 'context/Workflow'; -import useTrialCollections from 'hooks/useTrialCollections'; +import useTrialCollections from 'hooks/trialStorage/useTrialCollections'; import { useCallback, useMemo } from 'react'; import { useIntl } from 'react-intl'; import { diff --git a/src/components/editor/Bindings/UpdateResourceConfigButton.tsx b/src/components/editor/Bindings/UpdateResourceConfigButton.tsx index 95ed3abd2..3f80f8ea8 100644 --- a/src/components/editor/Bindings/UpdateResourceConfigButton.tsx +++ b/src/components/editor/Bindings/UpdateResourceConfigButton.tsx @@ -1,7 +1,7 @@ import { Button } from '@mui/material'; import { AddCollectionDialogCTAProps } from 'components/shared/Entity/types'; import invariableStores from 'context/Zustand/invariableStores'; -import useTrialCollections from 'hooks/useTrialCollections'; +import useTrialCollections from 'hooks/trialStorage/useTrialCollections'; import { FormattedMessage } from 'react-intl'; import { diff --git a/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx b/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx index c82a78bdf..db933f496 100644 --- a/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx +++ b/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx @@ -1,7 +1,7 @@ import { Button } from '@mui/material'; import { AddCollectionDialogCTAProps } from 'components/shared/Entity/types'; import invariableStores from 'context/Zustand/invariableStores'; -import useTrialCollections from 'hooks/useTrialCollections'; +import useTrialCollections from 'hooks/trialStorage/useTrialCollections'; import { FormattedMessage } from 'react-intl'; import { useBinding_prefillResourceConfigs } from 'stores/Binding/hooks'; import { useBindingStore } from 'stores/Binding/Store'; diff --git a/src/hooks/useTrialCollections.ts b/src/hooks/trialStorage/useTrialCollections.ts similarity index 85% rename from src/hooks/useTrialCollections.ts rename to src/hooks/trialStorage/useTrialCollections.ts index 098389a1f..4ca7e0b4d 100644 --- a/src/hooks/useTrialCollections.ts +++ b/src/hooks/trialStorage/useTrialCollections.ts @@ -6,13 +6,13 @@ import { CustomEvents } from 'services/types'; import { useTrialMetadataStore } from 'stores/TrialMetadata/Store'; import { hasLength, stripPathing } from 'utils/misc-utils'; import { useShallow } from 'zustand/react/shallow'; -import useTrialStorageOnly from './useTrialStorageOnly'; +import useTrialPrefixes from './useTrialPrefixes'; // This function was created and exported so the binding store hydrator // can use the same logic to evaluate trial collection as the core hook. export const evaluateTrialCollections = async ( catalogNames: string[] | undefined, - getTrialOnlyPrefixes: (prefixes: string[]) => Promise, + storeTrialPrefixes: (prefixes: string[]) => Promise, storedTrialPrefixes: string[] ) => { const targetPrefixes = catalogNames @@ -27,7 +27,7 @@ export const evaluateTrialCollections = async ( // trial metadata store state in sync whenever trial collections need // to be evaluated. if (hasLength(newPrefixes)) { - await getTrialOnlyPrefixes(newPrefixes); + await storeTrialPrefixes(newPrefixes); } const { data, error } = await getTrialCollections(targetPrefixes); @@ -49,18 +49,18 @@ export default function useTrialCollections() { useShallow((state) => state.trialStorageOnly) ); - const getTrialOnlyPrefixes = useTrialStorageOnly(); + const storeTrialOnlyPrefixes = useTrialPrefixes(); return useCallback( async (catalogNames?: string[]) => { const trialCollections = await evaluateTrialCollections( catalogNames, - getTrialOnlyPrefixes, + storeTrialOnlyPrefixes, storedTrialPrefixes ); return trialCollections; }, - [getTrialOnlyPrefixes, storedTrialPrefixes] + [storeTrialOnlyPrefixes, storedTrialPrefixes] ); } diff --git a/src/hooks/useTrialStorageOnly.ts b/src/hooks/trialStorage/useTrialPrefixes.ts similarity index 79% rename from src/hooks/useTrialStorageOnly.ts rename to src/hooks/trialStorage/useTrialPrefixes.ts index ad9c0fdb1..cc1e5079e 100644 --- a/src/hooks/useTrialStorageOnly.ts +++ b/src/hooks/trialStorage/useTrialPrefixes.ts @@ -13,9 +13,7 @@ const ESTUARY_TRIAL_STORAGE = { prefix: 'collection-data/', }; -const getTrialStorageOnlyPrefixes = async ( - prefixes: string[] -): Promise => { +const getTrialPrefixes = async (prefixes: string[]): Promise => { const { data, error } = await getStorageMappingStores(prefixes); if (error || !data) { @@ -33,10 +31,10 @@ const getTrialStorageOnlyPrefixes = async ( .map(({ catalog_prefix }) => catalog_prefix); }; -export default function useTrialStorageOnly() { +export default function useTrialPrefixes() { const objectRoles = useEntitiesStore_capabilities_adminable(); - const addTrialStorageOnly = useTrialMetadataStore( + const addTrialPrefix = useTrialMetadataStore( (state) => state.addTrialStorageOnly ); @@ -54,14 +52,12 @@ export default function useTrialStorageOnly() { return []; } - const trialPrefixes = await getTrialStorageOnlyPrefixes( - filteredPrefixes - ); + const trialPrefixes = await getTrialPrefixes(filteredPrefixes); - addTrialStorageOnly(trialPrefixes); + addTrialPrefix(trialPrefixes); return trialPrefixes; }, - [addTrialStorageOnly, objectRoles] + [addTrialPrefix, objectRoles] ); } diff --git a/src/stores/Binding/Hydrator.tsx b/src/stores/Binding/Hydrator.tsx index 57dfd9af4..894a3c9fd 100644 --- a/src/stores/Binding/Hydrator.tsx +++ b/src/stores/Binding/Hydrator.tsx @@ -1,6 +1,6 @@ import { useEntityType } from 'context/EntityContext'; import { useEntityWorkflow, useEntityWorkflow_Editing } from 'context/Workflow'; -import useTrialStorageOnly from 'hooks/useTrialStorageOnly'; +import useTrialPrefixes from 'hooks/trialStorage/useTrialPrefixes'; import { useEffect, useRef } from 'react'; import { logRocketConsole } from 'services/shared'; import { useDetailsFormStore } from 'stores/DetailsForm/Store'; @@ -22,7 +22,7 @@ export const BindingHydrator = ({ children }: BaseComponentProps) => { const workflow = useEntityWorkflow(); const editWorkflow = useEntityWorkflow_Editing(); - const getTrialOnlyPrefixes = useTrialStorageOnly(); + const getTrialPrefixes = useTrialPrefixes(); const connectorTagId = useDetailsFormStore( (state) => state.details.data.connectorImage.id @@ -47,7 +47,7 @@ export const BindingHydrator = ({ children }: BaseComponentProps) => { editWorkflow, entityType, connectorTagId, - getTrialOnlyPrefixes, + getTrialPrefixes, rehydrating.current ) .then( @@ -81,7 +81,7 @@ export const BindingHydrator = ({ children }: BaseComponentProps) => { connectorTagId, editWorkflow, entityType, - getTrialOnlyPrefixes, + getTrialPrefixes, hydrateState, setActive, setHydrated, diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index c947333bf..da09c41c6 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -7,8 +7,8 @@ import { } from 'api/hydration'; import { isBeforeTrialInterval } from 'components/materialization/shared'; import { GlobalSearchParams } from 'hooks/searchParams/useGlobalSearchParams'; +import { evaluateTrialCollections } from 'hooks/trialStorage/useTrialCollections'; import { LiveSpecsExtQuery } from 'hooks/useLiveSpecsExt'; -import { evaluateTrialCollections } from 'hooks/useTrialCollections'; import produce from 'immer'; import { difference, From 9ff4db801a37ff27877d8c4552ead4fcd920211c Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Tue, 11 Feb 2025 10:11:39 -0500 Subject: [PATCH 13/30] Remove trial metadata store --- src/hooks/trialStorage/useTrialCollections.ts | 37 +++------- src/hooks/trialStorage/useTrialPrefixes.ts | 9 +-- src/stores/Binding/Store.ts | 3 +- src/stores/TrialMetadata/Store.ts | 73 ------------------- 4 files changed, 14 insertions(+), 108 deletions(-) delete mode 100644 src/stores/TrialMetadata/Store.ts diff --git a/src/hooks/trialStorage/useTrialCollections.ts b/src/hooks/trialStorage/useTrialCollections.ts index 4ca7e0b4d..a78c2b59b 100644 --- a/src/hooks/trialStorage/useTrialCollections.ts +++ b/src/hooks/trialStorage/useTrialCollections.ts @@ -1,40 +1,32 @@ import { getTrialCollections } from 'api/liveSpecsExt'; -import { difference, uniq } from 'lodash'; +import { uniq } from 'lodash'; import { useCallback } from 'react'; import { logRocketEvent } from 'services/shared'; import { CustomEvents } from 'services/types'; -import { useTrialMetadataStore } from 'stores/TrialMetadata/Store'; import { hasLength, stripPathing } from 'utils/misc-utils'; -import { useShallow } from 'zustand/react/shallow'; import useTrialPrefixes from './useTrialPrefixes'; // This function was created and exported so the binding store hydrator // can use the same logic to evaluate trial collection as the core hook. export const evaluateTrialCollections = async ( catalogNames: string[] | undefined, - storeTrialPrefixes: (prefixes: string[]) => Promise, - storedTrialPrefixes: string[] + getTrialPrefixes: (prefixes: string[]) => Promise ) => { - const targetPrefixes = catalogNames + const prefixes = catalogNames ? uniq(catalogNames.map((name) => stripPathing(name, true))) : []; - const newPrefixes = hasLength(targetPrefixes) - ? difference(targetPrefixes, storedTrialPrefixes) - : []; - - // There is an implicit dependency on this function as it keeps the - // trial metadata store state in sync whenever trial collections need - // to be evaluated. - if (hasLength(newPrefixes)) { - await storeTrialPrefixes(newPrefixes); + if (!hasLength(prefixes)) { + return []; } - const { data, error } = await getTrialCollections(targetPrefixes); + const trialPrefixes = await getTrialPrefixes(prefixes); + + const { data, error } = await getTrialCollections(trialPrefixes); if (error) { logRocketEvent(CustomEvents.TRIAL_STORAGE, { - prefixes: targetPrefixes, + prefixes: trialPrefixes, error, }); @@ -45,22 +37,17 @@ export const evaluateTrialCollections = async ( }; export default function useTrialCollections() { - const storedTrialPrefixes = useTrialMetadataStore( - useShallow((state) => state.trialStorageOnly) - ); - - const storeTrialOnlyPrefixes = useTrialPrefixes(); + const getTrialPrefixes = useTrialPrefixes(); return useCallback( async (catalogNames?: string[]) => { const trialCollections = await evaluateTrialCollections( catalogNames, - storeTrialOnlyPrefixes, - storedTrialPrefixes + getTrialPrefixes ); return trialCollections; }, - [storeTrialOnlyPrefixes, storedTrialPrefixes] + [getTrialPrefixes] ); } diff --git a/src/hooks/trialStorage/useTrialPrefixes.ts b/src/hooks/trialStorage/useTrialPrefixes.ts index cc1e5079e..b5b63d7e2 100644 --- a/src/hooks/trialStorage/useTrialPrefixes.ts +++ b/src/hooks/trialStorage/useTrialPrefixes.ts @@ -4,7 +4,6 @@ import { useCallback } from 'react'; import { logRocketEvent } from 'services/shared'; import { CustomEvents } from 'services/types'; import { useEntitiesStore_capabilities_adminable } from 'stores/Entities/hooks'; -import { useTrialMetadataStore } from 'stores/TrialMetadata/Store'; import { hasLength } from 'utils/misc-utils'; const ESTUARY_TRIAL_STORAGE = { @@ -34,10 +33,6 @@ const getTrialPrefixes = async (prefixes: string[]): Promise => { export default function useTrialPrefixes() { const objectRoles = useEntitiesStore_capabilities_adminable(); - const addTrialPrefix = useTrialMetadataStore( - (state) => state.addTrialStorageOnly - ); - return useCallback( async (prefixes: string[]) => { if (!hasLength(prefixes)) { @@ -54,10 +49,8 @@ export default function useTrialPrefixes() { const trialPrefixes = await getTrialPrefixes(filteredPrefixes); - addTrialPrefix(trialPrefixes); - return trialPrefixes; }, - [addTrialPrefix, objectRoles] + [objectRoles] ); } diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index da09c41c6..8d7010c67 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -144,8 +144,7 @@ const hydrateSpecificationDependentState = async ( const trialCollections = await evaluateTrialCollections( Object.keys(get().bindings), - getTrialOnlyPrefixes, - [] + getTrialOnlyPrefixes ); get().setCollectionMetadata(trialCollections); diff --git a/src/stores/TrialMetadata/Store.ts b/src/stores/TrialMetadata/Store.ts deleted file mode 100644 index 01fa0e862..000000000 --- a/src/stores/TrialMetadata/Store.ts +++ /dev/null @@ -1,73 +0,0 @@ -import produce from 'immer'; -import { pull, union } from 'lodash'; -import { hasLength } from 'utils/misc-utils'; -import { devtoolsOptions } from 'utils/store-utils'; -import { create, StoreApi } from 'zustand'; -import { devtools, NamedSet } from 'zustand/middleware'; - -export interface TrialMetadataState { - addTrialStorageOnly: (value: string | string[]) => void; - removeTrialStorageOnly: (value?: string) => void; - trialStorageOnly: string[]; -} - -const getInitialStateData = (): Pick< - TrialMetadataState, - 'trialStorageOnly' -> => ({ - trialStorageOnly: [], -}); - -const getInitialState = ( - set: NamedSet, - _get: StoreApi['getState'] -): TrialMetadataState => ({ - ...getInitialStateData(), - - addTrialStorageOnly: (values) => { - set( - produce((state: TrialMetadataState) => { - if ( - typeof values === 'string' && - !state.trialStorageOnly.includes(values) - ) { - state.trialStorageOnly.push(values); - } - - if (typeof values !== 'string') { - state.trialStorageOnly = hasLength(values) - ? union(state.trialStorageOnly, values) - : []; - } - }), - false, - 'Prefixes with trial storage only added' - ); - }, - - removeTrialStorageOnly: (value) => { - set( - produce((state: TrialMetadataState) => { - if (value) { - state.trialStorageOnly = pull( - state.trialStorageOnly, - value - ); - - return; - } - - state.trialStorageOnly = []; - }), - false, - 'Prefixes with trial storage only removed' - ); - }, -}); - -export const useTrialMetadataStore = create()( - devtools( - (set, get) => getInitialState(set, get), - devtoolsOptions('trial-metadata') - ) -); From e9e950bcc42f512c1687766dd322605df5e6df5d Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Tue, 11 Feb 2025 10:29:47 -0500 Subject: [PATCH 14/30] Extend condition dictating whether the top-level backfill alert is hidden --- src/components/materialization/TrialOnlyPrefixAlert.tsx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/components/materialization/TrialOnlyPrefixAlert.tsx b/src/components/materialization/TrialOnlyPrefixAlert.tsx index 84cb7b676..2b5db13d9 100644 --- a/src/components/materialization/TrialOnlyPrefixAlert.tsx +++ b/src/components/materialization/TrialOnlyPrefixAlert.tsx @@ -23,11 +23,14 @@ export default function TrialOnlyPrefixAlert({ () => !bindingUUID && backfilledBindings.every((uuid) => { - const config = resourceConfigs[uuid]; + const config = Object.keys(resourceConfigs).includes(uuid) + ? resourceConfigs[uuid] + : undefined; return ( - !config.meta.trialOnlyStorage || - !isBeforeTrialInterval(config.meta.updatedAt) + config && + (!config.meta.trialOnlyStorage || + !isBeforeTrialInterval(config.meta.updatedAt)) ); }), [backfilledBindings, bindingUUID, resourceConfigs] From 45d035230985f9c34ac0736fffd96ef496e9dbbe Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Tue, 11 Feb 2025 13:42:25 -0500 Subject: [PATCH 15/30] Narrow trial collection query and update add trigger --- src/api/liveSpecsExt.ts | 17 ++++++++++++----- src/components/collection/ResourceConfig.tsx | 7 ++++++- src/hooks/trialStorage/useTrialCollections.ts | 7 +++++-- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/api/liveSpecsExt.ts b/src/api/liveSpecsExt.ts index 99c38e577..b581952ca 100644 --- a/src/api/liveSpecsExt.ts +++ b/src/api/liveSpecsExt.ts @@ -435,27 +435,34 @@ export interface TrialCollectionQuery { updated_at: string; } -const getTrialCollections = async (prefixes: string[]) => { +const getTrialCollections = async ( + trialPrefixes: string[], + catalogNames: string[] +) => { const limiter = pLimit(3); const promises: Promise>[] = []; let index = 0; + const trialCollections = catalogNames.filter((name) => + trialPrefixes.some((prefix) => name.startsWith(prefix)) + ); + const promiseGenerator = (idx: number) => { const trialThreshold = DateTime.utc().minus({ days: 20 }); - const prefixFilter = prefixes + const catalogNameFilter = trialCollections .slice(idx, idx + CHUNK_SIZE) - .map((prefix) => `catalog_name.like.${prefix}%`) + .map((name) => `catalog_name.eq.${name}`) .join(','); return supabaseClient .from(TABLES.LIVE_SPECS_EXT) .select('catalog_name,updated_at') - .or(prefixFilter) + .or(catalogNameFilter) .eq('spec_type', 'collection') .lt('updated_at', trialThreshold); }; - while (index < prefixes.length) { + while (index < trialCollections.length) { const prom = promiseGenerator(index); promises.push(limiter(() => prom)); diff --git a/src/components/collection/ResourceConfig.tsx b/src/components/collection/ResourceConfig.tsx index e50897934..710b65aec 100644 --- a/src/components/collection/ResourceConfig.tsx +++ b/src/components/collection/ResourceConfig.tsx @@ -47,6 +47,11 @@ function ResourceConfig({ 'disable' ); + const trialCollection = useBinding_resourceConfigOfMetaBindingProperty( + bindingUUID, + 'trialOnlyStorage' + ); + const collectionAdded = useBinding_resourceConfigOfMetaBindingProperty( bindingUUID, 'added' @@ -59,7 +64,7 @@ function ResourceConfig({ ) : null} diff --git a/src/hooks/trialStorage/useTrialCollections.ts b/src/hooks/trialStorage/useTrialCollections.ts index a78c2b59b..0e3af408f 100644 --- a/src/hooks/trialStorage/useTrialCollections.ts +++ b/src/hooks/trialStorage/useTrialCollections.ts @@ -16,13 +16,16 @@ export const evaluateTrialCollections = async ( ? uniq(catalogNames.map((name) => stripPathing(name, true))) : []; - if (!hasLength(prefixes)) { + if (!hasLength(prefixes) || !catalogNames) { return []; } const trialPrefixes = await getTrialPrefixes(prefixes); - const { data, error } = await getTrialCollections(trialPrefixes); + const { data, error } = await getTrialCollections( + trialPrefixes, + catalogNames + ); if (error) { logRocketEvent(CustomEvents.TRIAL_STORAGE, { From e51f8a6e0ff467a36fc46b7f1b1dcfd56a165c5d Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Wed, 12 Feb 2025 09:11:11 -0500 Subject: [PATCH 16/30] Store binding metadata in useGenerateCatalog --- .../materialization/useGenerateCatalog.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/components/materialization/useGenerateCatalog.ts b/src/components/materialization/useGenerateCatalog.ts index 2994646c9..4887b9335 100644 --- a/src/components/materialization/useGenerateCatalog.ts +++ b/src/components/materialization/useGenerateCatalog.ts @@ -14,6 +14,7 @@ import { } from 'components/editor/Store/hooks'; import useEntityWorkflowHelpers from 'components/shared/Entity/hooks/useEntityWorkflowHelpers'; import { useEntityWorkflow_Editing } from 'context/Workflow'; +import useTrialCollections from 'hooks/trialStorage/useTrialCollections'; import useEntityNameSuffix from 'hooks/useEntityNameSuffix'; import { useCallback } from 'react'; import { logRocketEvent } from 'services/shared'; @@ -57,6 +58,8 @@ function useGenerateCatalog() { const isEdit = useEntityWorkflow_Editing(); const { callFailed } = useEntityWorkflowHelpers(); + const getTrialCollections = useTrialCollections(); + // Details Form Store const detailsFormsErrorsExist = useDetailsFormStore( (state) => state.errorsExist @@ -126,6 +129,10 @@ function useGenerateCatalog() { (state) => state.onIncompatibleSchemaChange ); + const setCollectionMetadata = useBindingStore( + (state) => state.setCollectionMetadata + ); + // Source Capture Store const sourceCapture = useSourceCaptureStore_sourceCaptureDefinition(); @@ -290,6 +297,12 @@ function useGenerateCatalog() { true ); + const trialCollections = await getTrialCollections( + Object.keys(bindings) + ); + + setCollectionMetadata(trialCollections); + // Mutate the draft first so that we are not running // update _after_ the form is showing as "done" await mutateDraftSpecs(); @@ -343,6 +356,7 @@ function useGenerateCatalog() { endpointSchema, fullSourceConfigs, fullSourceErrorsExist, + getTrialCollections, imageConnectorId, imageConnectorTagId, persistedDraftId, @@ -356,6 +370,7 @@ function useGenerateCatalog() { serverEndpointConfigData, serverUpdateRequired, setCatalogName, + setCollectionMetadata, setDraftId, setDraftedEntityName, setEncryptedEndpointConfig, From 0bb43edd5fb3d2d9186ae83ece935cbedf78c8be Mon Sep 17 00:00:00 2001 From: Kiahna Tucker Date: Thu, 13 Feb 2025 09:47:10 -0500 Subject: [PATCH 17/30] Move collection metadata properties into independent state --- src/components/collection/Config.tsx | 4 +- src/components/collection/ResourceConfig.tsx | 14 +-- .../Bindings/Backfill/SectionWrapper.tsx | 45 ++++++--- .../editor/Bindings/Backfill/index.tsx | 4 +- .../editor/Bindings/Backfill/types.ts | 4 +- .../editor/Bindings/Row/ErrorIndicator.tsx | 21 ++-- src/components/editor/Bindings/Row/Name.tsx | 14 ++- src/components/editor/Bindings/Row/types.ts | 10 ++ .../Bindings/UpdateResourceConfigButton.tsx | 4 +- .../AddSourceCaptureToSpecButton.tsx | 4 +- .../materialization/TrialOnlyPrefixAlert.tsx | 29 +----- src/components/materialization/types.ts | 1 - .../materialization/useGenerateCatalog.ts | 15 --- src/stores/Binding/Store.ts | 96 ++++++++++--------- src/stores/Binding/hooks.ts | 31 +++++- src/stores/Binding/shared.ts | 17 ++++ src/stores/Binding/types.ts | 24 +++-- 17 files changed, 190 insertions(+), 147 deletions(-) create mode 100644 src/components/editor/Bindings/Row/types.ts diff --git a/src/components/collection/Config.tsx b/src/components/collection/Config.tsx index 2fa7a19fc..e60c0d019 100644 --- a/src/components/collection/Config.tsx +++ b/src/components/collection/Config.tsx @@ -38,8 +38,8 @@ function CollectionConfig({ const bindingErrorsExist = useBinding_bindingErrorsExist(); const fullSourceErrorsExist = useBinding_fullSourceErrorsExist(); const sourceBackfillRecommended = useBindingStore((state) => - Object.values(state.resourceConfigs).some( - (config) => config.meta.sourceBackfillRecommended + Object.values(state.collectionMetadata).some( + (meta) => meta.sourceBackfillRecommended ) ); diff --git a/src/components/collection/ResourceConfig.tsx b/src/components/collection/ResourceConfig.tsx index 710b65aec..185bf6cc4 100644 --- a/src/components/collection/ResourceConfig.tsx +++ b/src/components/collection/ResourceConfig.tsx @@ -9,6 +9,7 @@ import ErrorBoundryWrapper from 'components/shared/ErrorBoundryWrapper'; import { useEntityType } from 'context/EntityContext'; import { FormattedMessage } from 'react-intl'; import { + useBinding_collectionMetadataProperty, useBinding_currentBindingIndex, useBinding_hydrated, useBinding_resourceConfigOfMetaBindingProperty, @@ -47,13 +48,13 @@ function ResourceConfig({ 'disable' ); - const trialCollection = useBinding_resourceConfigOfMetaBindingProperty( - bindingUUID, - 'trialOnlyStorage' + const trialCollection = useBinding_collectionMetadataProperty( + collectionName, + 'trialStorage' ); - const collectionAdded = useBinding_resourceConfigOfMetaBindingProperty( - bindingUUID, + const collectionAdded = useBinding_collectionMetadataProperty( + collectionName, 'added' ); @@ -62,7 +63,6 @@ function ResourceConfig({ {entityType === 'materialization' ? ( @@ -93,7 +93,7 @@ function ResourceConfig({ diff --git a/src/components/editor/Bindings/Backfill/SectionWrapper.tsx b/src/components/editor/Bindings/Backfill/SectionWrapper.tsx index 367408d5a..cbe68b2a1 100644 --- a/src/components/editor/Bindings/Backfill/SectionWrapper.tsx +++ b/src/components/editor/Bindings/Backfill/SectionWrapper.tsx @@ -1,29 +1,47 @@ import { Box, Stack, Typography } from '@mui/material'; +import { isBeforeTrialInterval } from 'components/materialization/shared'; import TrialOnlyPrefixAlert from 'components/materialization/TrialOnlyPrefixAlert'; import { useEntityType } from 'context/EntityContext'; +import { useMemo } from 'react'; import { useIntl } from 'react-intl'; import { - useBinding_backfilledBindings, - useBinding_resourceConfigOfMetaBindingProperty, + useBinding_backfilledCollections, + useBinding_collectionMetadataProperty, } from 'stores/Binding/hooks'; -import { hasLength } from 'utils/misc-utils'; +import { useBindingStore } from 'stores/Binding/Store'; +import { useShallow } from 'zustand/react/shallow'; import { SectionWrapperProps } from './types'; export default function SectionWrapper({ alertMessageId, - bindingUUID, children, + collection, }: SectionWrapperProps) { const intl = useIntl(); const entityType = useEntityType(); - const backfilledBindings = useBinding_backfilledBindings(); - const bindingSourceBackfillRecommended = - useBinding_resourceConfigOfMetaBindingProperty( - bindingUUID, - 'sourceBackfillRecommended' - ); + const bindingLevelAlertTriggered = useBinding_collectionMetadataProperty( + collection, + 'sourceBackfillRecommended' + ); + const backfilledCollections = useBinding_backfilledCollections(); + const collectionMetadata = useBindingStore( + useShallow((state) => state.collectionMetadata) + ); + + const topLevelAlertTriggered = useMemo( + () => + !collection && + backfilledCollections.some((name) => { + return ( + Object.keys(collectionMetadata).includes(name) && + collectionMetadata[name].trialStorage && + isBeforeTrialInterval(collectionMetadata[name].updatedAt) + ); + }), + [backfilledCollections, collection, collectionMetadata] + ); return ( @@ -36,12 +54,11 @@ export default function SectionWrapper({ {entityType === 'materialization' ? ( ) : null} diff --git a/src/components/editor/Bindings/Backfill/index.tsx b/src/components/editor/Bindings/Backfill/index.tsx index 6bb58b596..f451d51ca 100644 --- a/src/components/editor/Bindings/Backfill/index.tsx +++ b/src/components/editor/Bindings/Backfill/index.tsx @@ -7,7 +7,7 @@ import { BackfillProps } from './types'; export default function Backfill({ bindingIndex, - bindingUUID, + collection, collectionEnabled, }: BackfillProps) { const entityType = useEntityType(); @@ -18,7 +18,7 @@ export default function Backfill({ return showBackfillButton ? ( 0 || diff --git a/src/components/editor/Bindings/Row/Name.tsx b/src/components/editor/Bindings/Row/Name.tsx index 9652b9195..109b64663 100644 --- a/src/components/editor/Bindings/Row/Name.tsx +++ b/src/components/editor/Bindings/Row/Name.tsx @@ -2,24 +2,22 @@ import { Button, Typography } from '@mui/material'; import { typographyTruncation } from 'context/Theme'; import { stripPathing } from 'utils/misc-utils'; import BindingsSelectorErrorIndicator from './ErrorIndicator'; - -interface RowProps { - bindingUUID: string; - collection: string; - shortenName?: boolean; -} +import { SelectorNameProps } from './types'; function BindingsSelectorName({ bindingUUID, collection, shortenName, -}: RowProps) { +}: SelectorNameProps) { return (