Skip to content

Commit

Permalink
enh: gpt-4-turbo rollout (#2512)
Browse files Browse the repository at this point in the history
* enh: gpt-4-turbo

* more clean-up

* migrations to move assistants to gpt-4-turbo

* Preserve the routing from 32k to 4

* nit
  • Loading branch information
spolu authored Nov 13, 2023
1 parent 8ca89f7 commit b1cbb82
Show file tree
Hide file tree
Showing 5 changed files with 85 additions and 53 deletions.
6 changes: 3 additions & 3 deletions front/components/assistant_builder/AssistantBuilder.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import {
CLAUDE_INSTANT_DEFAULT_MODEL_CONFIG,
getSupportedModelConfig,
GPT_3_5_TURBO_16K_MODEL_CONFIG,
GPT_4_32K_MODEL_CONFIG,
GPT_4_TURBO_MODEL_CONFIG,
SupportedModel,
} from "@app/lib/assistant";
import { CONNECTOR_CONFIGURATIONS } from "@app/lib/connector_providers";
Expand All @@ -63,7 +63,7 @@ import AssistantBuilderDustAppModal from "./AssistantBuilderDustAppModal";
import DustAppSelectionSection from "./DustAppSelectionSection";

const usedModelConfigs = [
GPT_4_32K_MODEL_CONFIG,
GPT_4_TURBO_MODEL_CONFIG,
GPT_3_5_TURBO_16K_MODEL_CONFIG,
CLAUDE_DEFAULT_MODEL_CONFIG,
CLAUDE_INSTANT_DEFAULT_MODEL_CONFIG,
Expand Down Expand Up @@ -219,7 +219,7 @@ export default function AssistantBuilder({
...DEFAULT_ASSISTANT_STATE,
generationSettings: {
...DEFAULT_ASSISTANT_STATE.generationSettings,
modelSettings: GPT_4_32K_MODEL_CONFIG,
modelSettings: GPT_4_TURBO_MODEL_CONFIG,
},
});

Expand Down
23 changes: 5 additions & 18 deletions front/lib/api/assistant/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import {
renderConversationForModel,
runGeneration,
} from "@app/lib/api/assistant/generation";
import { GPT_4_32K_MODEL_CONFIG, GPT_4_MODEL_CONFIG } from "@app/lib/assistant";
import { GPT_4_TURBO_MODEL_CONFIG } from "@app/lib/assistant";
import { Authenticator } from "@app/lib/auth";
import { Err, Ok, Result } from "@app/lib/result";
import logger from "@app/logger/logger";
Expand Down Expand Up @@ -60,9 +60,9 @@ export async function generateActionInputs(

const MIN_GENERATION_TOKENS = 2048;

let model: { providerId: string; modelId: string } = {
providerId: GPT_4_32K_MODEL_CONFIG.providerId,
modelId: GPT_4_32K_MODEL_CONFIG.modelId,
const model: { providerId: string; modelId: string } = {
providerId: GPT_4_TURBO_MODEL_CONFIG.providerId,
modelId: GPT_4_TURBO_MODEL_CONFIG.modelId,
};

// Turn the conversation into a digest that can be presented to the model.
Expand All @@ -71,26 +71,13 @@ export async function generateActionInputs(
model,
prompt,
allowedTokenCount:
GPT_4_32K_MODEL_CONFIG.contextSize - MIN_GENERATION_TOKENS,
GPT_4_TURBO_MODEL_CONFIG.contextSize - MIN_GENERATION_TOKENS,
});

if (modelConversationRes.isErr()) {
return modelConversationRes;
}

// If we use gpt-4-32k but tokens used is less than GPT_4_CONTEXT_SIZE-MIN_GENERATION_TOKENS then
// switch the model back to GPT_4 standard (8k context, cheaper).
if (
model.modelId === GPT_4_32K_MODEL_CONFIG.modelId &&
modelConversationRes.value.tokensUsed <
GPT_4_MODEL_CONFIG.contextSize - MIN_GENERATION_TOKENS
) {
model = {
providerId: GPT_4_MODEL_CONFIG.providerId,
modelId: GPT_4_MODEL_CONFIG.modelId,
};
}

const config = cloneBaseConfig(
DustProdActionRegistry["assistant-v2-inputs-generator"].config
);
Expand Down
42 changes: 11 additions & 31 deletions front/lib/api/assistant/global_agents.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,19 @@ import {
CLAUDE_DEFAULT_MODEL_CONFIG,
CLAUDE_INSTANT_DEFAULT_MODEL_CONFIG,
GPT_3_5_TURBO_16K_MODEL_CONFIG,
GPT_4_32K_MODEL_CONFIG,
GPT_4_TURBO_MODEL_CONFIG,
MISTRAL_7B_DEFAULT_MODEL_CONFIG,
} from "@app/lib/assistant";
import { GLOBAL_AGENTS_SID } from "@app/lib/assistant";
import { Authenticator, prodAPICredentialsForOwner } from "@app/lib/auth";
import { ConnectorProvider } from "@app/lib/connectors_api";
import { isDevelopmentOrDustWorkspace } from "@app/lib/development";
import { DustAPI } from "@app/lib/dust_api";
import { GlobalAgentSettings } from "@app/lib/models/assistant/agent";
import { FREE_TEST_PLAN_CODE } from "@app/lib/plans/plan_codes";
import logger from "@app/logger/logger";
import {
AgentConfigurationType,
GlobalAgentStatus,
} from "@app/types/assistant/agent";
import { PlanType } from "@app/types/plan";
import { WorkspaceType } from "@app/types/user";

class HelperAssistantPrompt {
private static instance: HelperAssistantPrompt;
Expand Down Expand Up @@ -88,8 +83,8 @@ async function _getHelperGlobalAgent(
throw new Error("Unexpected `auth` without `plan`.");
}
const model = {
providerId: GPT_4_32K_MODEL_CONFIG.providerId,
modelId: GPT_4_32K_MODEL_CONFIG.modelId,
providerId: GPT_4_TURBO_MODEL_CONFIG.providerId,
modelId: GPT_4_TURBO_MODEL_CONFIG.modelId,
};
return {
id: -1,
Expand Down Expand Up @@ -138,37 +133,22 @@ async function _getGPT35TurboGlobalAgent({
};
}

async function _getGPT4GlobalAgent({
owner,
plan,
}: {
owner: WorkspaceType;
plan: PlanType;
}): Promise<AgentConfigurationType> {
const isFreePlan = plan.code === FREE_TEST_PLAN_CODE;
const isDustOrDevWorkspace = isDevelopmentOrDustWorkspace(owner);
const shouldUseTurboModel = isFreePlan || isDustOrDevWorkspace;
async function _getGPT4GlobalAgent(): Promise<AgentConfigurationType> {
return {
id: -1,
sId: GLOBAL_AGENTS_SID.GPT4,
version: 0,
name: "gpt4",
description: shouldUseTurboModel
? "OpenAI's cost-effective and high throughput model (128K context)."
: "OpenAI's most powerful model (32k context).",
description: "OpenAI's most powerful and recent model (128k context).",
pictureUrl: "https://dust.tt/static/systemavatar/gpt4_avatar_full.png",
status: "active",
scope: "global",
generation: {
id: -1,
prompt: "",
model: {
providerId: shouldUseTurboModel
? GPT_4_TURBO_MODEL_CONFIG.providerId
: GPT_4_32K_MODEL_CONFIG.providerId,
modelId: shouldUseTurboModel
? GPT_4_TURBO_MODEL_CONFIG.modelId
: GPT_4_32K_MODEL_CONFIG.modelId,
providerId: GPT_4_TURBO_MODEL_CONFIG.providerId,
modelId: GPT_4_TURBO_MODEL_CONFIG.modelId,
},
temperature: 0.7,
},
Expand Down Expand Up @@ -339,8 +319,8 @@ async function _getManagedDataSourceAgent(
id: -1,
prompt,
model: {
providerId: GPT_4_32K_MODEL_CONFIG.providerId,
modelId: GPT_4_32K_MODEL_CONFIG.modelId,
providerId: GPT_4_TURBO_MODEL_CONFIG.providerId,
modelId: GPT_4_TURBO_MODEL_CONFIG.modelId,
},
temperature: 0.4,
},
Expand Down Expand Up @@ -514,8 +494,8 @@ async function _getDustGlobalAgent(
prompt:
"Assist the user based on the retrieved data from their workspace.",
model: {
providerId: GPT_4_32K_MODEL_CONFIG.providerId,
modelId: GPT_4_32K_MODEL_CONFIG.modelId,
providerId: GPT_4_TURBO_MODEL_CONFIG.providerId,
modelId: GPT_4_TURBO_MODEL_CONFIG.modelId,
},
temperature: 0.4,
},
Expand Down Expand Up @@ -568,7 +548,7 @@ export async function getGlobalAgent(
agentConfiguration = await _getGPT35TurboGlobalAgent({ settings });
break;
case GLOBAL_AGENTS_SID.GPT4:
agentConfiguration = await _getGPT4GlobalAgent({ owner, plan });
agentConfiguration = await _getGPT4GlobalAgent();
break;
case GLOBAL_AGENTS_SID.CLAUDE_INSTANT:
agentConfiguration = await _getClaudeInstantGlobalAgent({ settings });
Expand Down
2 changes: 1 addition & 1 deletion front/lib/assistant.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ export const GPT_4_MODEL_CONFIG = {
export const GPT_4_TURBO_MODEL_CONFIG = {
providerId: "openai",
modelId: GPT_4_TURBO_MODEL_ID,
displayName: "GPT 4 Turbo",
displayName: "GPT 4",
contextSize: 128000,
recommendedTopK: 32,
} as const;
Expand Down
65 changes: 65 additions & 0 deletions front/migrations/20231113_migrate_assistants_to_gpt4_turbo.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import {
AgentConfiguration,
AgentGenerationConfiguration,
Workspace,
} from "@app/lib/models";
import { Err } from "@app/lib/result";

const { LIVE, WORKSPACE } = process.env;

async function main() {
if (!WORKSPACE) {
throw new Err("WORKSPACE is required");
}
const wId = WORKSPACE;

console.log(`Updateing agents for workspace ${wId}...`);
const w = await Workspace.findOne({ where: { sId: wId } });
if (!w) {
throw new Error(`Workspace ${wId} not found`);
}

const agentConfigurations = await AgentConfiguration.findAll({
where: { workspaceId: w.id },
});

for (const c of agentConfigurations) {
if (!c.generationConfigurationId) {
console.log(
"Skipping agent (no generation configuration)",
c.sId,
c.name
);
continue;
}

const g = await AgentGenerationConfiguration.findOne({
where: { id: c.generationConfigurationId },
});

if (!g) {
throw new Error(
`Generation configuration ${c.generationConfigurationId} not found`
);
}

if (g.modelId === "gpt-4" || g.modelId === "gpt-4-32k") {
if (LIVE) {
await g.update({ modelId: "gpt-4-1106-preview" });
console.log("Updated", c.sId, c.name);
} else {
console.log("Would update", c.sId, c.name);
}
}
}
}

main()
.then(() => {
console.log("Done");
process.exit(0);
})
.catch((err) => {
console.error(err);
process.exit(1);
});

0 comments on commit b1cbb82

Please sign in to comment.