Skip to content

ai uf summary [experimental / poc - not meant for prod] #89958

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 57 additions & 1 deletion static/app/components/feedback/list/feedbackList.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import FeedbackListItem from 'sentry/components/feedback/list/feedbackListItem';
import useListItemCheckboxState from 'sentry/components/feedback/list/useListItemCheckboxState';
import useFeedbackQueryKeys from 'sentry/components/feedback/useFeedbackQueryKeys';
import LoadingIndicator from 'sentry/components/loadingIndicator';
import {IconHappy, IconMeh, IconSad} from 'sentry/icons';
import {t} from 'sentry/locale';
import {space} from 'sentry/styles/space';
import useFetchInfiniteListData from 'sentry/utils/api/useFetchInfiniteListData';
Expand All @@ -40,8 +41,33 @@ function NoFeedback({title, subtitle}: {subtitle: string; title: string}) {
);
}

export default function FeedbackList() {
interface FeedbackListProps {
feedbackSummary: {
error: Error | null;
keySentiments: Array<{
type: 'positive' | 'negative' | 'neutral';
value: string;
}>;
loading: boolean;
summary: string | null;
};
}

export default function FeedbackList({feedbackSummary}: FeedbackListProps) {
const {listQueryKey} = useFeedbackQueryKeys();
const {summary, keySentiments} = feedbackSummary;

const getSentimentIcon = (type: string) => {
switch (type) {
case 'positive':
return <IconHappy color="green400" />;
case 'negative':
return <IconSad color="red400" />;
default:
return <IconMeh color="yellow400" />;
}
};

const {
isFetchingNextPage,
isFetchingPreviousPage,
Expand Down Expand Up @@ -95,6 +121,18 @@ export default function FeedbackList() {

return (
<Fragment>
<Summary>
<SummaryHeader>{t('Feedback Summary')}</SummaryHeader>
<div>{summary}</div>
<div>
{keySentiments.map(sentiment => (
<Sentiment key={sentiment.value}>
{getSentimentIcon(sentiment.type)}
{sentiment.value}
</Sentiment>
))}
</div>
</Summary>
<FeedbackListHeader {...checkboxState} />
<FeedbackListItems>
<InfiniteLoader
Expand Down Expand Up @@ -152,6 +190,24 @@ export default function FeedbackList() {
);
}

const SummaryHeader = styled('div')`
font-weight: bold;
`;

const Summary = styled('div')`
padding: ${space(2)};
border-bottom: 1px solid ${p => p.theme.innerBorder};
display: flex;
flex-direction: column;
gap: ${space(2)};
`;

const Sentiment = styled('div')`
display: flex;
align-items: center;
gap: ${space(1)};
`;

const FeedbackListItems = styled('div')`
display: grid;
flex-grow: 1;
Expand Down
40 changes: 40 additions & 0 deletions static/app/components/feedback/list/useFeedbackMessages.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import type {FeedbackIssue} from 'sentry/utils/feedback/types';
import {useApiQuery} from 'sentry/utils/queryClient';
import {decodeList} from 'sentry/utils/queryString';
import useLocationQuery from 'sentry/utils/url/useLocationQuery';
import useOrganization from 'sentry/utils/useOrganization';

const FEEDBACK_STALE_TIME = 10 * 60 * 1000;

export default function useFeedbackMessages() {
const organization = useOrganization();
const queryView = useLocationQuery({
fields: {
limit: 10,
queryReferrer: 'feedback_list_page',
project: decodeList,
statsPeriod: '7d',
},
});

const {data, isPending, isError} = useApiQuery<FeedbackIssue[]>(
[
`/organizations/${organization.slug}/issues/`,
{
query: {
...queryView,
query: `issue.category:feedback status:unresolved`,
},
},
],
{staleTime: FEEDBACK_STALE_TIME}
);

if (isPending || isError) {
return [];
}

return data.map(feedback => {
return feedback.metadata.message;
});
}
161 changes: 161 additions & 0 deletions static/app/components/feedback/list/useFeedbackSummary.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
import {useEffect, useMemo, useRef, useState} from 'react';

import useFeedbackMessages from 'sentry/components/feedback/list/useFeedbackMessages';
import useOpenAIKey from 'sentry/components/feedback/list/useOpenAIKey';

export type Sentiment = {
type: 'positive' | 'negative' | 'neutral';
value: string;
};

const SUMMARY_REGEX = /Summary:(.*?)Key sentiments:/s;
const SENTIMENT_REGEX = /- (.*?):\s*(positive|negative|neutral)/gi;

async function getSentimentSummary({
messages,
apiKey,
}: {
apiKey: string;
messages: string[];
}) {
const inputText = messages.map(msg => `- ${msg}`).join('\n');
const prompt = `
You are an AI assistant that analyzes customer feedback. Below is a list of user messages.

${inputText}

Figure out the top 4 specific sentiments in the messages. Be concise but also specific in the summary.

The summary should be at most 2 sentences, and complete the sentence "Users say...".

After the summary, for each sentiment, also indicate if it is mostly positive or negative.

The output format should be:

Summary: <1-2 sentence summary>
Key sentiments:
- <sentiment>: positive/negative/neutral
- <sentiment>: positive/negative/neutral
- <sentiment>: positive/negative/neutral
- <sentiment>: positive/negative/neutral
`;

const response = await fetch('https://api.openai.com/v1/chat/completions', {
method: 'POST',
headers: {
Authorization: `Bearer ${apiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: 'gpt-3.5-turbo',
messages: [{role: 'user', content: prompt}],
temperature: 0.3,
}),
});

const data = await response.json();
return data.choices[0].message.content;
}

export default function useFeedbackSummary(): {
error: Error | null;
keySentiments: Sentiment[];
loading: boolean;
summary: string | null;
} {
const apiKey = useOpenAIKey();
const messages = useFeedbackMessages();

const [response, setResponse] = useState<string | null>(null);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<Error | null>(null);
const requestMadeRef = useRef(false);

const finalResultRef = useRef<{
keySentiments: Sentiment[];
summary: string | null;
}>({
summary: null,
keySentiments: [],
});

useEffect(() => {
if (!apiKey || !messages.length || requestMadeRef.current) {
return;
}

setLoading(true);
setError(null);
requestMadeRef.current = true;

getSentimentSummary({messages, apiKey})
.then(result => {
setResponse(result);
})
.catch(err => {
setError(
err instanceof Error ? err : new Error('Failed to get sentiment summary')
);
})
.finally(() => {
setLoading(false);
});
}, [apiKey, messages]);

const parsedResults = useMemo(() => {
if (!response) {
return finalResultRef.current;
}

let summaryText: string | null = null;
const parsedSentiments: Sentiment[] = [];
const summaryMatch = response.match(SUMMARY_REGEX);

if (summaryMatch?.[1]) {
summaryText = summaryMatch[1].trim();
}

SENTIMENT_REGEX.lastIndex = 0;
let match = SENTIMENT_REGEX.exec(response);
while (match !== null) {
if (match[1] && match[2]) {
const value = match[1].trim();
const type = match[2].toLowerCase() as 'positive' | 'negative' | 'neutral';
parsedSentiments.push({value, type});
}
match = SENTIMENT_REGEX.exec(response);
}

finalResultRef.current = {
summary: summaryText,
keySentiments: parsedSentiments,
};

return finalResultRef.current;
}, [response]);

if (loading) {
return {
summary: null,
keySentiments: [],
loading: true,
error: null,
};
}

if (error) {
return {
summary: null,
keySentiments: [],
loading: false,
error,
};
}

return {
summary: parsedResults.summary,
keySentiments: parsedResults.keySentiments,
loading: false,
error: null,
};
}
3 changes: 3 additions & 0 deletions static/app/components/feedback/list/useOpenAIKey.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
export default function useOpenAIKey() {
return '';
}
4 changes: 3 additions & 1 deletion static/app/views/feedback/feedbackListPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import FeedbackSearch from 'sentry/components/feedback/feedbackSearch';
import FeedbackSetupPanel from 'sentry/components/feedback/feedbackSetupPanel';
import FeedbackWhatsNewBanner from 'sentry/components/feedback/feedbackWhatsNewBanner';
import FeedbackList from 'sentry/components/feedback/list/feedbackList';
import useFeedbackSummary from 'sentry/components/feedback/list/useFeedbackSummary';
import useCurrentFeedbackId from 'sentry/components/feedback/useCurrentFeedbackId';
import useHaveSelectedProjectsSetupFeedback, {
useHaveSelectedProjectsSetupNewFeedback,
Expand Down Expand Up @@ -44,6 +45,7 @@ export default function FeedbackListPage() {
const pageFilters = usePageFilters();
const projects = useProjects();
const prefersStackedNav = usePrefersStackedNav();
const feedbackSummary = useFeedbackSummary();

const selectedProjects = projects.projects.filter(p =>
pageFilters.selection.projects.includes(Number(p.id))
Expand Down Expand Up @@ -84,7 +86,7 @@ export default function FeedbackListPage() {
{hasSetupOneFeedback || hasSlug ? (
<Fragment>
<Container style={{gridArea: 'list'}}>
<FeedbackList />
<FeedbackList feedbackSummary={feedbackSummary} />
</Container>
<SearchContainer>
<FeedbackSearch />
Expand Down
Loading