Skip to content

Commit

Permalink
Tracker Builder: Manage datasource config (#9284)
Browse files Browse the repository at this point in the history
* Tracker Builder: Manage datasource config

* Apply feedback
  • Loading branch information
PopDaph authored Dec 11, 2024
1 parent 93cfac7 commit d19da68
Show file tree
Hide file tree
Showing 11 changed files with 697 additions and 96 deletions.
4 changes: 2 additions & 2 deletions front/admin/db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ import {
DocumentTrackerChangeSuggestion,
TrackedDocument,
TrackerConfigurationModel,
TrackerDataSouceConfigurationModel,
TrackerDataSourceConfigurationModel,
TrackerGenerationModel,
} from "@app/lib/models/doc_tracker";
import { FeatureFlag } from "@app/lib/models/feature_flag";
Expand Down Expand Up @@ -129,7 +129,7 @@ async function main() {
await DocumentTrackerChangeSuggestion.sync({ alter: true });

await TrackerConfigurationModel.sync({ alter: true });
await TrackerDataSouceConfigurationModel.sync({ alter: true });
await TrackerDataSourceConfigurationModel.sync({ alter: true });
await TrackerGenerationModel.sync({ alter: true });

await Plan.sync({ alter: true });
Expand Down
3 changes: 2 additions & 1 deletion front/components/data_source_view/DataSourceViewSelector.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,8 @@ interface DataSourceViewsSelectorProps {
useCase:
| "spaceDatasourceManagement"
| "assistantBuilder"
| "transcriptsProcessing";
| "transcriptsProcessing"
| "trackerBuilder";
dataSourceViews: DataSourceViewType[];
allowedSpaces?: SpaceType[];
selectionConfigurations: DataSourceViewSelectionConfigurations;
Expand Down
170 changes: 127 additions & 43 deletions front/components/trackers/TrackerBuilder.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ import {
useSendNotification,
} from "@dust-tt/sparkle";
import type {
DataSourceViewSelectionConfiguration,
DataSourceViewType,
SpaceType,
SubscriptionType,
SupportedModel,
TrackerConfigurationStateType,
TrackerConfigurationType,
WorkspaceType,
} from "@dust-tt/types";
import {
Expand All @@ -28,54 +28,61 @@ import { useState } from "react";

import { AdvancedSettings } from "@app/components/assistant_builder/InstructionScreen";
import AppLayout from "@app/components/sparkle/AppLayout";
import { AppLayoutSimpleSaveCancelTitle } from "@app/components/sparkle/AppLayoutTitle";
import {
AppLayoutSimpleCloseTitle,
AppLayoutSimpleSaveCancelTitle,
} from "@app/components/sparkle/AppLayoutTitle";
import TrackerBuilderDataSourceModal from "@app/components/trackers/TrackerBuilderDataSourceModal";
import { isEmailValid } from "@app/lib/utils";

export const TrackerBuilder = ({
owner,
subscription,
globalSpace,
dataSourceViews,
trackerToEdit,
initialTrackerState,
initialTrackerId,
}: {
owner: WorkspaceType;
subscription: SubscriptionType;
globalSpace: SpaceType;
dataSourceViews: DataSourceViewType[];
trackerToEdit: TrackerConfigurationType | null;
initialTrackerState: TrackerConfigurationStateType | null;
initialTrackerId: string | null;
}) => {
const router = useRouter();
const sendNotification = useSendNotification();
const [isSubmitting, setIsSubmitting] = useState(false);

const [tracker, setTracker] = useState<TrackerConfigurationStateType>({
name: trackerToEdit?.name ?? null,
nameError: null,
description: trackerToEdit?.description ?? null,
descriptionError: null,
prompt: trackerToEdit?.prompt ?? null,
promptError: null,
frequency: trackerToEdit?.frequency ?? "daily",
frequencyError: null,
recipients: trackerToEdit?.recipients?.join(", ") ?? null,
recipientsError: null,
modelId:
trackerToEdit?.modelId ?? CLAUDE_3_5_SONNET_DEFAULT_MODEL_CONFIG.modelId,
providerId:
trackerToEdit?.providerId ??
CLAUDE_3_5_SONNET_DEFAULT_MODEL_CONFIG.providerId,
temperature: trackerToEdit?.temperature ?? 0.5,
});
const [edited, setEdited] = useState(false);
const [isSubmitting, setIsSubmitting] = useState(false);
const [showMaintainedDsModal, setShowMaintainedDsModal] = useState(false);
const [showWatchedDsModal, setShowWatchedDataSourcesModal] = useState(false);

void dataSourceViews; // todo: use this
const [tracker, setTracker] = useState<TrackerConfigurationStateType>(
initialTrackerState ?? {
name: null,
nameError: null,
description: null,
descriptionError: null,
prompt: null,
promptError: null,
frequency: "daily",
frequencyError: null,
recipients: "",
recipientsError: null,
modelId: CLAUDE_3_5_SONNET_DEFAULT_MODEL_CONFIG.modelId,
providerId: CLAUDE_3_5_SONNET_DEFAULT_MODEL_CONFIG.providerId,
temperature: 0.5,
maintainedDataSources: {},
watchedDataSources: {},
}
);

const extractEmails = (text: string): string[] => [
...new Set(text.split(/[\n,]+/).map((e) => e.trim())),
];

const onSubmit = async () => {
// Validate the form
setIsSubmitting(true);
const validateForm = () => {
let hasValidationError = false;
if (!tracker.name) {
setTracker((t) => ({
Expand Down Expand Up @@ -109,6 +116,33 @@ export const TrackerBuilder = ({
}));
hasValidationError = true;
}
return hasValidationError;
};

const dataSourceToPayload = (
{
dataSourceView,
selectedResources,
isSelectAll,
}: DataSourceViewSelectionConfiguration,
workspaceId: string
) => ({
dataSourceViewId: dataSourceView.sId,
workspaceId,
filter: {
parents: !isSelectAll
? {
in: selectedResources.map((r) => r.internalId),
not: [],
}
: null,
},
});

// todo use submit function.
const onSubmit = async () => {
setIsSubmitting(true);
const hasValidationError = validateForm();
if (hasValidationError) {
setIsSubmitting(false);
return;
Expand All @@ -117,8 +151,8 @@ export const TrackerBuilder = ({
let route = `/api/w/${owner.sId}/spaces/${globalSpace.sId}/trackers`;
let method = "POST";

if (trackerToEdit) {
route += `/${trackerToEdit.sId}`;
if (initialTrackerId) {
route += `/${initialTrackerId}`;
method = "PATCH";
}

Expand All @@ -133,6 +167,12 @@ export const TrackerBuilder = ({
temperature: tracker.temperature,
frequency: tracker.frequency,
recipients: tracker.recipients ? extractEmails(tracker.recipients) : [],
maintainedDataSources: Object.values(tracker.maintainedDataSources).map(
(ds) => dataSourceToPayload(ds, owner.sId)
),
watchedDataSources: Object.values(tracker.watchedDataSources).map(
(ds) => dataSourceToPayload(ds, owner.sId)
),
}),
headers: {
"Content-Type": "application/json",
Expand All @@ -143,7 +183,7 @@ export const TrackerBuilder = ({
if (!res.ok) {
const resJson = await res.json();
sendNotification({
title: trackerToEdit
title: initialTrackerId
? "Failed to update tracker"
: "Failed to create tracker",
description: resJson.error.message,
Expand All @@ -154,8 +194,8 @@ export const TrackerBuilder = ({
}

sendNotification({
title: trackerToEdit ? "Tracker updated" : "Tracker Created",
description: trackerToEdit
title: initialTrackerId ? "Tracker updated" : "Tracker Created",
description: initialTrackerId
? "Tracker updated successfully"
: "Tracker created successfully.",
type: "success",
Expand All @@ -170,18 +210,62 @@ export const TrackerBuilder = ({
subscription={subscription}
hideSidebar
isWideMode
pageTitle={trackerToEdit ? "Dust - Edit Tracker" : "Dust - New Tracker"}
pageTitle={
initialTrackerId ? "Dust - Edit Tracker" : "Dust - New Tracker"
}
titleChildren={
<AppLayoutSimpleSaveCancelTitle
title={trackerToEdit ? "Edit Tracker" : "New Tracker"}
onCancel={async () => {
await router.push(`/w/${owner.sId}/assistant/labs/trackers`);
}}
onSave={onSubmit}
isSaving={isSubmitting}
/>
!edited ? (
<AppLayoutSimpleCloseTitle
title={initialTrackerId ? "Edit Tracker" : "New Tracker"}
onClose={() => {
void router.push(`/w/${owner.sId}/assistant/labs/trackers`);
}}
/>
) : (
<AppLayoutSimpleSaveCancelTitle
title={initialTrackerId ? "Edit Tracker" : "New Tracker"}
onCancel={() => {
void router.push(`/w/${owner.sId}/assistant/labs/trackers`);
}}
onSave={onSubmit}
isSaving={isSubmitting}
/>
)
}
>
<TrackerBuilderDataSourceModal
isOpen={showMaintainedDsModal}
setOpen={(isOpen) => setShowMaintainedDsModal(isOpen)}
owner={owner}
onSave={async (dsConfigs) => {
setEdited(true);
setTracker((t) => ({
...t,
maintainedDataSources: dsConfigs,
}));
}}
dataSourceViews={dataSourceViews}
initialDataSourceConfigurations={tracker.maintainedDataSources}
allowedSpaces={[globalSpace]}
viewType="documents"
/>
<TrackerBuilderDataSourceModal
isOpen={showWatchedDsModal}
setOpen={(isOpen) => setShowWatchedDataSourcesModal(isOpen)}
owner={owner}
onSave={async (dsConfigs) => {
setEdited(true);
setTracker((t) => ({
...t,
watchedDataSources: dsConfigs,
}));
}}
dataSourceViews={dataSourceViews}
initialDataSourceConfigurations={tracker.watchedDataSources}
allowedSpaces={[globalSpace]}
viewType="documents"
/>

<div className="mx-auto flex w-full max-w-4xl flex-col gap-16 pb-12">
<div className="flex">
<div className="flex flex-grow" />
Expand Down Expand Up @@ -343,7 +427,7 @@ export const TrackerBuilder = ({
<Button
label="Select Documents"
onClick={() => {
alert("Select Documents");
setShowMaintainedDsModal(true);
}}
className="w-fit"
/>
Expand All @@ -359,7 +443,7 @@ export const TrackerBuilder = ({
<Button
label="Select Documents"
onClick={() => {
alert("Select Documents");
setShowWatchedDataSourcesModal(true);
}}
className="w-fit"
/>
Expand Down
97 changes: 97 additions & 0 deletions front/components/trackers/TrackerBuilderDataSourceModal.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
import { Modal } from "@dust-tt/sparkle";
import type {
ContentNodesViewType,
DataSourceViewSelectionConfigurations,
DataSourceViewType,
SpaceType,
WorkspaceType,
} from "@dust-tt/types";
import type { SetStateAction } from "react";
import { useCallback, useMemo, useState } from "react";

import { DataSourceViewsSelector } from "@app/components/data_source_view/DataSourceViewSelector";
import {
supportsDocumentsData,
supportsStructuredData,
} from "@app/lib/data_sources";

interface TrackerBuilderDataSourceModal {
initialDataSourceConfigurations: DataSourceViewSelectionConfigurations;
allowedSpaces: SpaceType[];
dataSourceViews: DataSourceViewType[];
isOpen: boolean;
onSave: (dsConfigs: DataSourceViewSelectionConfigurations) => void;
owner: WorkspaceType;
setOpen: (isOpen: boolean) => void;
viewType: ContentNodesViewType;
}

export default function TrackerBuilderDataSourceModal({
initialDataSourceConfigurations,
allowedSpaces,
dataSourceViews,
isOpen,
onSave,
owner,
setOpen,
viewType,
}: TrackerBuilderDataSourceModal) {
const [hasChanged, setHasChanged] = useState(false);

const [selectionConfigurations, setSelectionConfigurations] =
useState<DataSourceViewSelectionConfigurations>(
initialDataSourceConfigurations
);

const setSelectionConfigurationsCallback = useCallback(
(func: SetStateAction<DataSourceViewSelectionConfigurations>) => {
setHasChanged(true);
setSelectionConfigurations(func);
},
[setSelectionConfigurations]
);

const supportedDataSourceViewsForViewType = useMemo(
() =>
viewType === "documents"
? dataSourceViews.filter((dsv) => supportsDocumentsData(dsv.dataSource))
: dataSourceViews.filter((dsv) =>
supportsStructuredData(dsv.dataSource)
),
[dataSourceViews, viewType]
);

return (
<Modal
isOpen={isOpen}
onClose={() => {
setSelectionConfigurations(initialDataSourceConfigurations);
setOpen(false);
}}
onSave={() => {
onSave(selectionConfigurations);
setOpen(false);
}}
hasChanged={hasChanged}
variant="side-md"
title="Manage data sources selection"
className="flex flex-col overflow-hidden"
>
<div
id="dataSourceViewsSelector"
className="overflow-y-auto scrollbar-hide"
>
<DataSourceViewsSelector
useCase="trackerBuilder"
dataSourceViews={supportedDataSourceViewsForViewType}
allowedSpaces={allowedSpaces}
owner={owner}
selectionConfigurations={selectionConfigurations}
setSelectionConfigurations={setSelectionConfigurationsCallback}
viewType={viewType}
isRootSelectable={true}
/>
</div>
</Modal>
);
}
Loading

0 comments on commit d19da68

Please sign in to comment.