From e281427de3556416c328242cee69933a62662908 Mon Sep 17 00:00:00 2001 From: Rachel Shu Date: Sat, 6 Jul 2024 10:41:20 -0700 Subject: [PATCH 1/2] implement methods for calling SDXL and Stability Ultra api, implement generateImage() for image generation upon market creation --- backend/shared/src/helpers/stability-utils.ts | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 backend/shared/src/helpers/stability-utils.ts diff --git a/backend/shared/src/helpers/stability-utils.ts b/backend/shared/src/helpers/stability-utils.ts new file mode 100644 index 0000000000..ee6c329d47 --- /dev/null +++ b/backend/shared/src/helpers/stability-utils.ts @@ -0,0 +1,123 @@ +import { promptGPT4 } from "./openai-utils" + +const apiHost = 'https://api.stability.ai' +const STABILITY_API_KEY = process.env.STABILITY_API_KEY + +interface StabilityAPIV1 { + textPrompts: { text: string }[], + width: number, + height: number +} + +interface StabilityAPIV2 { + prompt: string, + negativePrompt: string, + aspectRatio: string, + outputFormat: string, +} + +const StabilitySDXL = async ({textPrompts, width, height}: StabilityAPIV1): Promise => { + + const body = JSON.stringify({ + text_prompts: textPrompts, + width: width, + height: height + }); + + try { + const response = await fetch(`${apiHost}/v1/generation/stable-diffusion-xl-1024-v1-0/text-to-image`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${STABILITY_API_KEY}`, + 'Accept': 'image/png' + }, + body: body + }) + + if (!response.ok) { + throw new Error(`HTTP Error ${response.status}`) + } + return await response.arrayBuffer() + } catch (err) { + console.error('Error generating SDXL image:', err) + return undefined + } +} + +const StabilityUltra = async ({prompt, aspectRatio, outputFormat}: StabilityAPIV2): Promise => { + + const formData = new FormData(); + formData.append('prompt', prompt); + formData.append('aspect_ratio', aspectRatio); + formData.append('output_format', outputFormat); + + try { + const response = await fetch(`${apiHost}/v2beta/stable-image/generate/ultra`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${STABILITY_API_KEY}`, + 'Accept': 'image/*', + }, + body: formData + }) + + if (!response.ok) { + console.log('Error generating Ultra image:', response) + throw new Error(`HTTP Error ${response.body}`) + } + return await response.arrayBuffer() + } catch (err) { + console.error('Error generating Ultra image:', err) + return undefined + } +} + +export const generateImage = async (marketDetails: string, useUltra: boolean = false): Promise => { + + const ultraPrompt = `Describe a vivid, cohesive wide-format image that encapsulates the essence of this question: ${marketDetails} + + Follow these guidelines to create an effective prompt for Stable Diffusion Ultra: + 1. Describe the color and relative position of objects and characters in the scene (bottom left, right third, etc). + 2. Describe the setting and atmosphere. Use adjectives to convey the mood. + 3. Focus on the question's general theme. Don't add details that distract from the main idea. The model will fill in the gaps. + 4. Describe the image using composition terms (focal point, leading lines, etc). + 5. Use short sentences. + + Caution: + * Steer clear of common prediction market symbols (hourglass, crystal ball, calendar, stock chart, etc). + * Limit use of text elements, as the model may not generate them accurately. + * Describe only what is visible in the image, not what is implied or symbolic, and avoid complex scenes. + + Craft your prompt concisely, at most 75 words.`; + + const sdxlPrompt = `Generate a vivid, detailed description illustrating this prediction market: ${marketDetails} + Follow these guidelines to create an effective prompt for Stable Diffusion 1.5: + * In the first sentence, choose a charismatic subject that represents the question. Use multiple descriptive adjectives for the subject. + * In the second sentence, choose a relevant background and describe it in detail, including lighting and mood. + * In the third sentence, specify an artistic style (e.g., oil painting, cartoon, photorealistic) appropriate to the tone. + + Caution: + * Steer clear of cliché prediction market symbols (crystal balls, calendars, hourglasses, etc.). + * Avoid text and numbers since the model will not generate it well. + * Describe only what is visible in the image, and happening in the present moment, not what is implied or what it symbolizes. + + SD 1.5 is an older model and gets confused with longer prompts, so you must not get fancy. You must return a prompt of no more than 50 words.` + + const imagePrompt = await promptGPT4(useUltra ? ultraPrompt : sdxlPrompt) as string + if (!imagePrompt) { + throw new Error('Did not receive image prompt') + } + + return useUltra ? StabilityUltra({ + prompt: `A hyperrealistic anime-inspired graphic poster.` + imagePrompt, + negativePrompt: `The image looks like caricature, stock photos, or video games.`, + aspectRatio: '16:9', + outputFormat: 'png' + }) + : StabilitySDXL({ + textPrompts: [{ text: imagePrompt }], + width: 1344, + height: 768, + }) +} \ No newline at end of file From d54e02f37383feff8a61b953bdfb865cf1a11295 Mon Sep 17 00:00:00 2001 From: Rachel Shu Date: Sat, 6 Jul 2024 10:42:33 -0700 Subject: [PATCH 2/2] rewrite uploadAndSetCoverImage to call new generateImage method from stability-utils.ts, rewrite uploadToStorage to take raw bytes rather than url --- backend/api/src/helpers/on-create-market.ts | 37 +++++++++++++-------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/backend/api/src/helpers/on-create-market.ts b/backend/api/src/helpers/on-create-market.ts index ac1875f771..ecfae72e4e 100644 --- a/backend/api/src/helpers/on-create-market.ts +++ b/backend/api/src/helpers/on-create-market.ts @@ -24,7 +24,8 @@ import { } from 'common/supabase/groups' import { HOUSE_LIQUIDITY_PROVIDER_ID } from 'common/antes' import { randomString } from 'common/util/random' -import { generateImage } from 'shared/helpers/openai-utils' +import { generateImage as generateMarketBanner } from 'shared/helpers/stability-utils' +import { MarketTierType } from 'common/tier' export const onCreateMarket = async ( contract: Contract, @@ -90,22 +91,35 @@ export const onCreateMarket = async ( ) } - await uploadAndSetCoverImage(pg, question, contract.id, creatorUsername) + let answerList: string[] = [] + if ('answers' in contract) { + answerList = contract.answers.map((a) => a.text) + } + + await uploadAndSetCoverImage(pg, question, desc, answerList, contract.id, creatorUsername, contract.marketTier ?? 'basic') } const uploadAndSetCoverImage = async ( pg: SupabaseDirectClient, question: string, + desc: JSONContent, + answerList: string[], contractId: string, - creatorUsername: string + creatorUsername: string, + marketTier: MarketTierType ) => { - const dalleImage = await generateImage(question) - if (!dalleImage) return - console.log('generated dalle image: ' + dalleImage) + const description = richTextToString(desc) as string + const formattedAnswers = "Possible answers:" + answerList.join(', ') + "\n\n" + const questionDescription = `Question: ${question}\n\nAdditional detail: ${description.slice(500)}\n\n${answerList.length != 0 ? formattedAnswers : ''}` + const highTier: boolean = ['plus', 'premium', 'crystal'].includes(marketTier) + + const bannerImage = await generateMarketBanner(questionDescription, highTier) + if (!bannerImage) return + + console.log('generated cover image: ' + bannerImage) - // Upload to firestore bucket. if we succeed, update the url. we do this because openAI deletes images after a month const coverImageUrl = await uploadToStorage( - dalleImage, + bannerImage, creatorUsername ).catch((err) => { console.error('Failed to load image', err) @@ -116,11 +130,8 @@ const uploadAndSetCoverImage = async ( await updateContract(pg, contractId, { coverImageUrl }) } -export const uploadToStorage = async (imgUrl: string, username: string) => { - const response = await fetch(imgUrl) - - const arrayBuffer = await response.arrayBuffer() - const inputBuffer = Buffer.from(arrayBuffer) +export const uploadToStorage = async (img: ArrayBuffer, username: string) => { + const inputBuffer = Buffer.from(img) const buffer = await sharp(inputBuffer) .toFormat('jpeg', { quality: 60 })