Skip to content

Commit

Permalink
Use appID for searching instead of datastoreId. Fixed alignment and f…
Browse files Browse the repository at this point in the history 
…ormatting
  • Loading branch information
Abhinav Rau committed Sep 19, 2024
1 parent 213b42e commit d7f56e3
Show file tree
Hide file tree
Showing 12 changed files with 160 additions and 79 deletions.
7 changes: 5 additions & 2 deletions src/common.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export const vertex_ai_search_configValues = [

["GCP PARAMETERS", ""],
["Vertex AI Search Project Number", "384473000457"],
["Vertex AI Search DataStore Name", "alphabet-pdfs_1695783402380"],
["Vertex AI Search App ID", "l300-arau_1695783344117"],
["Vertex AI Project ID", "argolis-arau"],
["Vertex AI Location", "us-central1"],
["", ""],
Expand All @@ -29,7 +29,10 @@ export const vertex_ai_search_configValues = [
["", ""],
["SUMMARY GENERATION SETTINGS", ""],
["summaryResultCount (1-5)", "2"], //summaryResultCount
["Preamble (Customized Summaries)", ""],
[
"Preamble (Customized Summaries)",
`You are an expert financial analyst. Only use the data returned from documents. All finance numbers must be reported in billions, millions or thousands. Be brief. Answer should be no more than 2 sentences please.`,
],
["Summarization Model", "gemini-1.0-pro-001/answer_gen/v1"],
[
"SummaryMatchingAdditionalPrompt",
Expand Down
2 changes: 1 addition & 1 deletion src/excel/excel_create_tables.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export async function createExcelTable(
excelTable.name = `${worksheetName}.${tableType}`;
excelTable.getRange().format.font.size = fontSize;

excelTable.getRange().format.wrapText = true;
//excelTable.getRange().format.wrapText = true;
excelTable.showFilterButton = false;

excelTable.getHeaderRowRange().values = [valuesArray[0]];
Expand Down
14 changes: 6 additions & 8 deletions src/excel/excel_search_runner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,9 @@ export class ExcelSearchRunner extends TaskRunner {
"Vertex AI Search Project Number",
)
][0],
vertexAISearchDataStoreName:
vertexAISearchAppId:
valueColumn.values[
findIndexByColumnsNameIn2DArray(
configColumn.values,
"Vertex AI Search DataStore Name",
)
findIndexByColumnsNameIn2DArray(configColumn.values, "Vertex AI Search App ID")
][0],
vertexAIProjectID:
valueColumn.values[
Expand Down Expand Up @@ -232,8 +229,9 @@ export class ExcelSearchRunner extends TaskRunner {
await this.processsAllRows(context, config, countRows, this.idColumn.values);

// autofit the content
currentWorksheet.getUsedRange().format.autofitColumns();
currentWorksheet.getUsedRange().format.autofitRows();
//currentWorksheet.getUsedRange().format.autofitColumns();
//currentWorksheet.getUsedRange().format.autofitRows();

await context.sync();
} catch (error) {
appendLog(`Caught Exception in executeSearchTests: ${error.message} `, error);
Expand Down Expand Up @@ -281,7 +279,7 @@ export class ExcelSearchRunner extends TaskRunner {
rowNum,
response_json,
).then(async (callsSoFar) => {
appendLog(`testCaseID: ${rowNum} Processed Search Summary.`);
appendLog(`testCaseID: ${rowNum} Finished Check Grounding.`);
});

this.searchTaskPromiseSet.add(checkGroundingPromise);
Expand Down
10 changes: 5 additions & 5 deletions src/excel/excel_search_tables.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@ export async function createVAIConfigTable() {
sheetTitleFontSize,
);

await makeRowBold("A3:B3");
await makeRowBold("A9:B9");
await makeRowBold("A17:B17");
await makeRowBold("A23:B23");
await makeRowBold("A26:B26");
await makeRowBold("A4:B4");
await makeRowBold("A10:B10");
await makeRowBold("A18:B18");
await makeRowBold("A24:B24");
await makeRowBold("A27:B27");
}

export async function createVAIDataTable() {
Expand Down
5 changes: 3 additions & 2 deletions src/excel/excel_summarization_runner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,9 @@ export class SummarizationRunner extends TaskRunner {
await this.processsAllRows(context, config, countRows, this.idColumn.values);

// autofit the content
currentWorksheet.getUsedRange().format.autofitColumns();
currentWorksheet.getUsedRange().format.autofitRows();
//currentWorksheet.getUsedRange().format.autofitColumns();
//currentWorksheet.getUsedRange().format.autofitRows();

await context.sync();
} catch (error) {
appendError(`Caught Exception in createSummarizationData `, error);
Expand Down
3 changes: 3 additions & 0 deletions src/excel/excel_summarization_tables.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ export async function createSummarizationEvalDataTable() {
sheet.getRange("E:E").format.columnWidth = 455;
sheet.getRange("F:F").format.columnWidth = 455;

sheet.getRange("G:G").format.columnWidth = 455;
sheet.getRange("G:G").format.columnWidth = 455;

sheet.getRange("E:E").format.wrapText = true;
sheet.getRange("F:F").format.wrapText = true;

Expand Down
6 changes: 4 additions & 2 deletions src/excel/excel_synthetic_qa_runner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,10 @@ export class SyntheticQARunner extends TaskRunner {
await this.processsAllRows(context, config, countRows, this.idColumn.values);

// autofit the content
currentWorksheet.getUsedRange().format.autofitColumns();
currentWorksheet.getUsedRange().format.autofitRows();
//currentWorksheet.getUsedRange().format.autofitColumns();
//currentWorksheet.getUsedRange().format.autofitRows();


await context.sync();
} catch (error) {
appendError(`Caught Exception in createSyntheticQAData `, error);
Expand Down
99 changes: 97 additions & 2 deletions src/vertex_ai.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export async function callVertexAISearch(id, query, config) {
const ignoreAdversarialQuery = config.ignoreAdversarialQuery;
const ignoreNonSummarySeekingQuery = config.ignoreNonSummarySeekingQuery;
const projectNumber = config.vertexAISearchProjectNumber;
const datastoreName = config.vertexAISearchDataStoreName;
const searchAppId = config.vertexAISearchAppId;

var data = {
query: query,
Expand All @@ -45,7 +45,7 @@ export async function callVertexAISearch(id, query, config) {
},
};

const url = `https://discoveryengine.googleapis.com/v1alpha/projects/${projectNumber}/locations/global/collections/default_collection/dataStores/${datastoreName}/servingConfigs/default_search:search`;
const url = `https://discoveryengine.googleapis.com/v1alpha/projects/${projectNumber}/locations/global/collections/default_collection/engines/${searchAppId}/servingConfigs/default_search:search`;

const { status, json_output } = await callVertexAI(url, token, data, id);

Expand All @@ -55,6 +55,8 @@ export async function callVertexAISearch(id, query, config) {
}

export async function calculateSimilarityUsingPalm2(id, sentence1, sentence2, config) {
appendLog(`testCaseID: ${id}: SummaryMatch Started `);

const token = config.accessToken;
const projectId = config.vertexAIProjectID;
const location = config.vertexAILocation;
Expand Down Expand Up @@ -225,6 +227,99 @@ export async function callCheckGrounding(config, answerCandidate, factsArray, id

return { id: id, status_code: status, output: json_output };
}

export async function createSearchEvalSampleQuerySetId(config, querySetId, querySetDisplayName) {
const token = config.accessToken;
const projectId = config.vertexAIProjectID;

var payload = {
displayName: `${querySetDisplayName}`,
};

const url = `https://discoveryengine.googleapis.com/v1beta/projects/${projectId}/locations/global/sampleQuerySets?sampleQuerySetId=${querySetId}`;

const { status, json_output } = await callVertexAI(url, token, payload, id);

appendLog(`createSearchEvalQuerySet: Finished Successfully.`);

return { status_code: status, output: json_output };
}

export async function createSearchEvalImportSampleQueryDataset(config, querySetId, queryEntry) {
const token = config.accessToken;
const projectId = config.vertexAIProjectID;

var payload = {
inlineSource: {
sampleQueries: [
{
queryEntry,
},
],
},
};

const url = `https://discoveryengine.googleapis.com/v1beta/projects/${projectId}/locations/global/sampleQuerySets/${querySetId}/sampleQueries:import`;

const { status, json_output } = await callVertexAI(url, token, payload);

appendLog(`createSearchEvalImportSampleQueryDataset: Finished Successfully.`);

return { status_code: status, output: json_output };
}

export async function createSearchEvalSubmitEvalJob(config, querySetId, searchAppID) {
const token = config.accessToken;
const projectId = config.vertexAIProjectID;

var payload = {
evaluationSpec: {
querySetSpec: {
sampleQuerySet: `projects/${projectId}/locations/global/sampleQuerySets/${querySetId}`,
},
searchRequest: {
servingConfig: `projects/${projectId}/locations/global/collections/default_collection/engines/${searchAppID}/servingConfigs/default_search`,
},
},
};

const url = `https://discoveryengine.googleapis.com/v1beta/projects/${projectId}/locations/global/evaluations`;

const { status, json_output } = await callVertexAI(url, token, payload);

appendLog(`createSearchEvalRunEval: Finished Successfully.`);

return { status_code: status, output: json_output };
}

export async function getSearchEvalResults(config, evaluationId, checkStatus) {
const token = config.accessToken;
const projectId = config.vertexAIProjectID;

var payload = {
evaluationSpec: {
querySetSpec: {
sampleQuerySet: `projects/${projectId}/locations/global/sampleQuerySets/${querySetId}`,
},
searchRequest: {
servingConfig: `projects/${projectId}/locations/global/collections/default_collection/engines/${searchAppID}/servingConfigs/default_search`,
},
},
};

var url = `https://discoveryengine.googleapis.com/v1beta/projects/${projectId}/locations/global/evaluations/${evaluationId}:listResults`;

if (checkStatus) {
url = `https://discoveryengine.googleapis.com/v1beta/projects/${projectId}/locations/global/evaluations/${evaluationId}`;
}

const { status, json_output } = await callVertexAI(url, token, payload);

appendLog(`getSearchEvalResults: Finished Successfully.`);

return { status_code: status, output: json_output };
}

export async function callVertexAI(url, token, data, id) {
try {
const response = await fetch(url, {
Expand Down
45 changes: 12 additions & 33 deletions taskpane.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,24 +66,14 @@ <h3>How to use </h3>
<p><button id="createSearchTables">Create Search Tables</button></p>

<p>
2. Use the "Synthetic Q&A" Tab to generate Questions and Answers from documents stored in GCS.
2. Use the "Synthetic Q&A" Tab to generate Questions and Answers
<b>OR</b>
If you have your own test cases then put them in a seperate sheet.
</p>
<p>
3. Reference the Questions and Answers in the "Query" and "Expected Summary" columns. And reference the expected link in the "Expected Link1" column.
</p>
<p>
4. Fill in the ID column for all rows you want the tests to run on.
If you have your own test cases copy into the test case table.
</p>
<p>
5. Fill in Config Parameters for your GCP environment in the Config Table.
Adjust the different config parameters like "Preamble" etc.
for your use case.
</p>
<p>
6. Click button below to run tests starting from first row.
</p>
3. Fill in Config Parameters for your GCP environment in the Config Table.
Adjust the <b>Premable</b> and other settings for your use case.
</p>
<p><b>
Note:</b> Tests will stop when it sees an empty ID value.
</p>
Expand All @@ -103,18 +93,13 @@ <h3>How to use </h3>
</p>
<p><button id="createGenQATables">Create Q&A Tables</button></p>
<p>
2. Fill in "FileUri" and "mimeType" columns you want questions and answers to be generated from. Valid mimeTypes for Gemini are <a href="https://firebase.google.com/docs/vertex-ai/input-file-requirements"> documented here.</a>
</p>
<p>
3. Fill in the ID column for all rows you want the tests to run on.
2. Fill in "FileUri" column with GCS doc link. <a href="https://firebase.google.com/docs/vertex-ai/input-file-requirements"> Valid file types.</a>
</p>
<p>
4. Fill in Config Parameters for your GCP environment in the Config Table.
Adjust the prompt and "System Instructions", "Prompt" and "Q&A Quality Prompt"
3. Fill in Config Parameters for your GCP environment in the Config Table.
Adjust the <b>Prompt</b> and <b>Q&A Quality Prompt</b>
for your use case.
</p>
<p>
5. Click button below to generate Q&As starting from first row.

</p>
<p><b>
Note:</b> Generation will stop when it sees an empty ID value.
Expand All @@ -137,18 +122,12 @@ <h3>How to use </h3>
<button id="createSummarizationTables">Create Summarization Tables</button>
</p>
<p>
2. Fill in "Text to Summarize" column with Text to summarize.
2. Fill in "Context" column with Text to summarize.
</p>
<p>
3. Fill in the ID column for all rows you want the rows you want summarized.
</p>
<p>
4. Fill in Config Parameters for your GCP environment in the Config Table.
Adjust the prompt and "System Instructions", "Prompt" and "Summarization Quality Prompt"
3. Fill in Config Parameters for your GCP environment in the Config Table.
Adjust the <b>Prompt</b> and <b>Summarization Quality Prompt</b>
for your use case.
</p>
<p>
5. Click button below to generate summarizes starting from first row.
</p>
<p><b>
Note:</b> Generation will stop when it sees an empty ID value.
Expand Down
2 changes: 1 addition & 1 deletion test/data/search/search_extractive_answer/test_vai_search_extractive_answer_request.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"ignoreAdversarialQuery": "true",
"ignoreNonSummarySeekingQuery": "true",
"modelPromptSpec": {
"preamble": ""
"preamble": "You are an expert financial analyst. Only use the data returned from documents. All finance numbers must be reported in billions, millions or thousands. Be brief. Answer should be no more than 2 sentences please."
},
"modelSpec": {
"version": "gemini-1.0-pro-001/answer_gen/v1"
Expand Down
11 changes: 5 additions & 6 deletions test/test_search_runner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ export var testCaseRows = vertex_ai_search_testTableHeader.concat([
"link2", // Actual Link 2
"link3", // Actual Link 3
],
[
[
"2", // ID
"What is Google's revenue for the year ending December 31, 2021", //Query
"Revenue is $2.2 billion", //Expected Summary
Expand Down Expand Up @@ -319,7 +319,7 @@ describe("When Search Run Tests is clicked ", () => {
// Get the config parameters from the config table
const config = await excelSearchRunner.getSearchConfig();

const url = `https://discoveryengine.googleapis.com/v1alpha/projects/${config.vertexAISearchProjectNumber}/locations/global/collections/default_collection/dataStores/${config.vertexAISearchDataStoreName}/servingConfigs/default_search:search`;
const url = `https://discoveryengine.googleapis.com/v1alpha/projects/${config.vertexAISearchProjectNumber}/locations/global/collections/default_collection/engines/${config.vertexAISearchAppId}/servingConfigs/default_search:search`;
// Prepare the request response mock the call to VertexAISearch
const { requestJson, expectedResponse } = mockDiscoveryEngineRequestResponse(
1,
Expand All @@ -338,7 +338,7 @@ describe("When Search Run Tests is clicked ", () => {

const grouding_url = `https://discoveryengine.googleapis.com/v1/projects/${config.vertexAIProjectID}/locations/global/groundingConfigs/default_grounding_config:check`;
// Prepare the request response mock the call to VertexAISearch
const { requestJson: grouding_requestJson, expectedResponse: grouding_expectedResponse } =
const { requestJson: grouding_requestJson, expectedResponse: grouding_expectedResponse } =
mockDiscoveryEngineRequestResponse(
1,
grouding_url,
Expand All @@ -364,7 +364,7 @@ describe("When Search Run Tests is clicked ", () => {
// check if vertex ai is called for summary match
const callsToSummaryMatch = fetchMock.calls().filter((call) => call[0] === summaryMatchUrl);
// Check if body is sent correctly to vertex ai search
expect(callsToSummaryMatch[0][1].body!==null).toBe(true);
expect(callsToSummaryMatch[0][1].body !== null).toBe(true);

// check if vertex ai is called for check grounding match
const callsToCheckGrounding = fetchMock.calls().filter((call) => call[0] === grouding_url);
Expand All @@ -373,7 +373,6 @@ describe("When Search Run Tests is clicked ", () => {

// Check if values get populated for each test case
for (let i = 1; i < testCaseRows.length; i++) {

// Match Actual Summary
const { cell: actual_summary_cell, col_index: actual_summary_col_index } =
getCellAndColumnIndexByName("Actual Summary", mockTestData, i);
Expand Down Expand Up @@ -467,7 +466,7 @@ describe("When Search Run Tests is clicked ", () => {
// Get the config parameters from the config table
const config = await excelSearchRunner.getSearchConfig();

const url = `https://discoveryengine.googleapis.com/v1alpha/projects/${config.vertexAISearchProjectNumber}/locations/global/collections/default_collection/dataStores/${config.vertexAISearchDataStoreName}/servingConfigs/default_search:search`;
const url = `https://discoveryengine.googleapis.com/v1alpha/projects/${config.vertexAISearchProjectNumber}/locations/global/collections/default_collection/engines/${config.vertexAISearchAppId}/servingConfigs/default_search:search`;

// Prepare the request response mock the call to VertexAISearch
const { requestJson, expectedResponse } = mockDiscoveryEngineRequestResponse(
Expand Down
Loading

0 comments on commit d7f56e3

Please sign in to comment.