From 9cc7575787252939239cc134ec453526ab7b593b Mon Sep 17 00:00:00 2001 From: fatwang2 Date: Sat, 8 Feb 2025 23:22:58 +0800 Subject: [PATCH 1/2] add search1api for a search and retrieve tool --- .env.local.example | 4 ++- docs/CONFIGURATION.md | 15 +++++++++++ lib/tools/retrieve.ts | 50 +++++++++++++++++++++++++++++++++--- lib/tools/search.ts | 60 ++++++++++++++++++++++++++++++++++++++----- lib/types/index.ts | 21 +++++++++++++++ 5 files changed, 140 insertions(+), 10 deletions(-) diff --git a/.env.local.example b/.env.local.example index fe96ae10..f088cae1 100644 --- a/.env.local.example +++ b/.env.local.example @@ -69,7 +69,7 @@ TAVILY_API_KEY=[YOUR_TAVILY_API_KEY] # Get your API key at: https://app.tavily. # Alternative Search Providers # Configure different search backends (default: Tavily) #------------------------------------------------------------------------------ -# SEARCH_API=searxng # Available options: tavily, searxng, exa +# SEARCH_API=searxng # Available options: tavily, searxng, exa, search1api # SearXNG Configuration (Required if SEARCH_API=searxng) # SEARXNG_API_URL=http://localhost:8080 # Replace with your local SearXNG API URL or docker http://searxng:8080 @@ -84,6 +84,8 @@ TAVILY_API_KEY=[YOUR_TAVILY_API_KEY] # Get your API key at: https://app.tavily. # SEARXNG_TIME_RANGE=None # SEARXNG_SAFESEARCH=0 +# SEARCH1API_API_KEY=[YOUR_SEARCH1API_API_KEY](Required if SEARCH_API=search1api, it supports retrieve tool too) + #------------------------------------------------------------------------------ # Additional Features # Enable extra functionality as needed diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 188403f2..6cd44460 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -40,6 +40,21 @@ LOCAL_REDIS_URL=redis://localhost:6379 ## Search Providers +### Search1API Configuration + +[Search1API](https://www.search1api.com) is a unified search API optimized for AI applications: + +- **Image Search**: Built-in support for image search results +- **Advanced Search**: Advanced search mode with full content crawling when you need +- **Retrieval Tool**: Specialized tool for extracting content from specific URLs + +Configure Search1API as your search and Retrieve tool: + +```bash +SEARCH_API=search1api +SEARCH1API_API_KEY=[YOUR_SEARCH1API_API_KEY] +``` + ### SearXNG Configuration SearXNG can be used as an alternative search backend with advanced search capabilities. diff --git a/lib/tools/retrieve.ts b/lib/tools/retrieve.ts index 1d1a9e9d..c37bc00e 100644 --- a/lib/tools/retrieve.ts +++ b/lib/tools/retrieve.ts @@ -76,15 +76,59 @@ async function fetchTavilyExtractData( } } +async function fetchSearch1APIData( + url: string +): Promise { + try { + const apiKey = process.env.SEARCH1API_API_KEY + if (!apiKey) { + console.error('Search1API API key is not set') + return null + } + + const response = await fetch('https://api.search1api.com/crawl', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ url }) + }) + + const json = await response.json() + if (!json.results) { + return null + } + + const content = json.results.content.slice(0, CONTENT_CHARACTER_LIMIT) + + return { + results: [ + { + title: json.results.title, + content, + url: json.results.link || url + } + ], + query: '', + images: [] + } + } catch (error) { + console.error('Search1API Crawl error:', error) + return null + } +} + export const retrieveTool = tool({ description: 'Retrieve content from the web', parameters: retrieveSchema, execute: async ({ url }) => { let results: SearchResultsType | null - // Use Jina if the API key is set, otherwise use Tavily - const useJina = process.env.JINA_API_KEY - if (useJina) { + // Use Search1API if SEARCH_API is set to search1api + if (process.env.SEARCH_API === 'search1api') { + results = await fetchSearch1APIData(url) + } else if (process.env.JINA_API_KEY) { results = await fetchJinaReaderData(url) } else { results = await fetchTavilyExtractData(url) diff --git a/lib/tools/search.ts b/lib/tools/search.ts index 1a1170ab..5b61b12f 100644 --- a/lib/tools/search.ts +++ b/lib/tools/search.ts @@ -25,7 +25,7 @@ export const searchTool = tool({ query.length < 5 ? query + ' '.repeat(5 - query.length) : query let searchResult: SearchResults const searchAPI = - (process.env.SEARCH_API as 'tavily' | 'exa' | 'searxng') || 'tavily' + (process.env.SEARCH_API as 'tavily' | 'exa' | 'searxng' | 'search1api') || 'tavily' const effectiveSearchDepth = searchAPI === 'searxng' && @@ -64,12 +64,14 @@ export const searchTool = tool({ ? tavilySearch : searchAPI === 'exa' ? exaSearch + : searchAPI === 'search1api' + ? search1apiSearch : searxngSearch)( - filledQuery, - max_results, - effectiveSearchDepth === 'advanced' ? 'advanced' : 'basic', - include_domains, - exclude_domains + filledQuery, + max_results, + effectiveSearchDepth === 'advanced' ? 'advanced' : 'basic', + include_domains, + exclude_domains ) } } catch (error) { @@ -279,3 +281,49 @@ async function searxngSearch( throw error } } + +async function search1apiSearch( + query: string, + maxResults: number = 10, + searchDepth: 'basic' | 'advanced' = 'basic', + includeDomains: string[] = [], + excludeDomains: string[] = [] +): Promise { + const apiKey = process.env.SEARCH1API_API_KEY + if (!apiKey) { + throw new Error('SEARCH1API_API_KEY is not set in the environment variables') + } + + const response = await fetch('https://api.search1api.com/search', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + query, + search_service: 'google', + image: true, + max_results: maxResults, + crawl_results: searchDepth === 'advanced' ? maxResults : 0 + }) + }) + + if (!response.ok) { + throw new Error( + `Search1API error: ${response.status} ${response.statusText}` + ) + } + + const data = await response.json() + return { + results: data.results.map((result: any) => ({ + title: result.title, + url: result.link, + content: result.content || result.snippet + })), + query, + images: data.images?.map((url: string) => sanitizeUrl(url)) || [], + number_of_results: data.results.length + } +} diff --git a/lib/types/index.ts b/lib/types/index.ts index 6ba9f7a7..349546c0 100644 --- a/lib/types/index.ts +++ b/lib/types/index.ts @@ -113,3 +113,24 @@ export type SearXNGSearchResults = { number_of_results?: number query: string } + +export interface Search1APIResult { + title: string + link: string + snippet: string + content: string +} + +export interface Search1APIResponse { + searchParameters: { + query: string + search_service: string + max_results: number + crawl_results: number + gl: string + hl: string + image: boolean + } + results: Search1APIResult[] + images: string[] +} From 0ebf7a76c14a719c4ef58ec32900f87bd5fcda7e Mon Sep 17 00:00:00 2001 From: fatwang2 Date: Sat, 8 Feb 2025 23:49:15 +0800 Subject: [PATCH 2/2] feat: add search1api as search and retrieve provider --- bun.lockb | Bin 303900 -> 303892 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/bun.lockb b/bun.lockb index c09aac26bc19602bd075db335bab1fcf2bdab4c2..b85ef0c38c5f5d9fa24661927b9ec36fa5d7e1aa 100755 GIT binary patch delta 3001 zcmXxm3tY}u9LMp~>8a$BkqQyja_i-i>ntJV)=YBu+IV4`VJ!F8YkQezHf)wKlf#7E z$566*m`gJuD!O09$VerHG0SYPeNLVI_4far@A?1F|M{QidCvcdz0`d3rREz)+pSs| zzB=44DO|Qn>%UD3VJC!W554{0CKi5mi4fn@&~1D8n=DrCPLr4;xV7H5ugB0F!gJtk{FHVk^(XU=t|9$uFSvEO73X%98of%!Q9Fwo@i+PMt=k@8V}N zeQ{T8H^-e-r3_&`XAXCxt3Xc$TZ+ltspC!di^)95PMYjI8GoWB2I#3}yI_uMMIC0c zRFkzPi-w+|i}mM9;i-;am98V(kX4cC8M$JPZ%cj8WLHh*MOJCDbd$9sdu*}{mCbNL znWoE|uFKSVyJj*U>KZc(*U9wE_+ppIa+G&EL*$yn{pc#@j`a-XnXChK3Yngbe3Nyg z-UfY_1!|!lqL42BL}x59qZgaQyO8ZA(^tM>GX1GdFvpds^9!z9aTPEv9 zmdN4$sy1DAw9*oHOjiK)5$MxOP1ck881#0Rj6cx}@j77BQ>`!F8)wlS_f6J^`ZV+y zWhU!OeH?m>2U57yot%}ShZSOh?P!Mdw6lirRSL#nxY5-DkNHeiqZ~I;gD0rKBV^(_ z?jQ@rsKW1fh-~B{4`rxCK1xx58_2<3i?E79iu)+SQ(VJ6l;8nw;WymIMWo^iEv;z@;XA}YFQuEI z*EzjT?!;bfgu(`FgimUhjHx3ViG1}JSJcYCW19>nixef8sP}T z7`%-)(H{Z0%wSIJ8{jp#Krenx;Q(heL^Cvk1&v{^re;asCOLeL^N@>d6`dtL9AYWA zt3z4RGprfwQggguQamZwS78j&Q;nnk%7~i&gjni?Ixmn2s5khe?=#i9%IoOD~@{ znBm^&i_iH~g`h1r?S=O6wy2povYS;uZ%*S3j^HTtgCQOXNW^jI$D<7&;X^FO0(^{x zScFI{#S*-aIhczNFdy^q9%f?}rb}M-Oo}M1K_CVo2m>()UEq&SXbpE{a--Lgja+2G zo_F1XLh1scX5~tsCi*e46LC1FVsho6nRnJA{4>s3jM5##5^p+A}obBeLnC+d$fbZEBc<} zPrSf0JjEZ_2mMUjfK8ZjV4 z`-R$*FAq2`B)fnZHKRaw91_AXfe1n{hTtLH7EZT=gn$Ej;xNr0aTNOL7%xh9BLkTZN*2n26QX!I!)cGeOKJ;~{UygP#X3Y|G~X5dG<}2$ z)SwzwxQB97f+uO+kC`$RQz-od^e>N8=wBOWun%jol_B-74gKrI4$UwaE!C4k8R&M9 zSDlC$=-&aleO5&jNe{O(G_@qe6>&AT1l!6Y*{`kho#*xKG~4;-GSuG5ZdyP${@EJW f%G6WNEl+R(wjx_`B7M#ceyVRTw^f|5?^W|3(PONO delta 3054 zcmXxm33N?I9LMqdCYxA-L=wanB&j%Jcd7`n*CDYlr{$ofD2@HJJ?d4fS`C6?s8kYb ztJXw3yT%%c)O$%JvDHoxg4$v`{r+Wo&->)g{ms0&_s-3mxyw%YCY|Hs5u$h*X{GzW6*hQ=marIRz1pZ!-;LtU&cEDr-l=EeH*bbU3kn(Pm9Wq&UvOOj{OvcBmf%c}qbknXT z*+R%j9w}=#!V2Qx1_v?}$4tXol<%7CxXEgh6`SmY$?A}mnCv8bK2}{kG}$Tb%4C=_ zIh4B&hBT8c%VhN_mzW_rO(w(C0Es3$ql2^P!fBd^P<4)5m7$S`Capqom`sKt$7ErY zS3zz$S5J{H@~GltHNte$d%kHNPPW5j1tyb!;dNwkHHA8zE<(oCNdlbK`F6e}6RQV&Di~k#E;Die~$VDE`A|C}P z#5tVD1zf}>{DaH5f~&ZO>$riNxP{y9mfb3%a0hpB55>5T2PnZqJi=qhJR$SKQ5?f@ zoWM!Q+;9q+_=p?)7{k#Koe+htkolrJdY~ugw?3rM3%wDI81zLy^v3`U#34F9jNM2; zB9f4d70~z@aae?S%)$grLM*0X24*4>jVkizHUHr5>^*hAGpeb5Ax~ft;;|S@@e7t= zIb@!dd37aLK_=4Gkje8m$b^}QB*>(=7BUgu;vRaj?~Oi)MhyBwCbs?tl-A`bH~5~J}6 zKE)V}#b+3Y@tAuXKp=8pxuMq?hNaj`j;Qy^Jfk3J2QoWSXIrNBt7($kxJ-tQuxh zo`u)cy}?_o;qfFQ3CUQCb!g9{k3?NGKtqHg4DWeaIxbK7k9k5ve}Vu6VlXEef}t3O zN7SEXU%2{i19i*Pc^pBVaR5qE27W@6B&+JOogmy zGcXfD)YU~j1fxC_UQ_oN|KbUr;u)SpR=@R#$5M<#38#O8W;WjE8WMyc6h(x0a2I>f zoO|@gHU8XVs=B(#%q*M3y;L3B>s>EZPR|d;;YxAyvUyk8RdzhraW&Pg GWyya*Pq&u<