diff --git a/.env.local.example b/.env.local.example index 2b8f4c0..70160c3 100644 --- a/.env.local.example +++ b/.env.local.example @@ -86,5 +86,7 @@ POSTGRES_PASSWORD= POSTGRES_DATABASE= #CLERK AUTHENTICATION -NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY= -CLERK_SECRET_KEY= \ No newline at end of file +NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY="" +CLERK_SECRET_KEY="" +NEXT_PUBLIC_CLERK_SIGN_IN_URL="/sign-in" +NEXT_PUBLIC_CLERK_SIGN_UP_URL="/sign-up" diff --git a/app/api/advanced-search/route.ts b/app/api/advanced-search/route.ts index 77021e7..65efe50 100644 --- a/app/api/advanced-search/route.ts +++ b/app/api/advanced-search/route.ts @@ -1,16 +1,17 @@ -import { NextResponse } from 'next/server'; -import http from 'http'; -import https from 'https'; -import { JSDOM, VirtualConsole } from 'jsdom'; +import { NextResponse } from 'next/server' +import http from 'http' +import https from 'https' +import { JSDOM, VirtualConsole } from 'jsdom' import { SearXNGSearchResults, SearXNGResponse, SearXNGResult, - SearchResultItem, -} from '@/lib/types'; -import { Agent } from 'http'; -import { Redis } from '@upstash/redis'; -import { createClient } from 'redis'; + SearchResultItem +} from '@/lib/types' +import { Agent } from 'http' +import { Redis } from '@upstash/redis' +import { createClient } from 'redis' +import { fetchUser } from '@/lib/prisma' /** * Maximum number of results to fetch from SearXNG. @@ -20,37 +21,37 @@ import { createClient } from 'redis'; const SEARXNG_MAX_RESULTS = Math.max( 10, Math.min(100, parseInt(process.env.SEARXNG_MAX_RESULTS || '50', 10)) -); +) -const CACHE_TTL = 3600; // Cache time-to-live in seconds (1 hour) -const CACHE_EXPIRATION_CHECK_INTERVAL = 3600000; // 1 hour in milliseconds +const CACHE_TTL = 3600 // Cache time-to-live in seconds (1 hour) +const CACHE_EXPIRATION_CHECK_INTERVAL = 3600000 // 1 hour in milliseconds -let redisClient: Redis | ReturnType | null = null; +let redisClient: Redis | ReturnType | null = null // Initialize Redis client based on environment variables async function initializeRedisClient() { - if (redisClient) return redisClient; + if (redisClient) return redisClient - const useLocalRedis = process.env.USE_LOCAL_REDIS === 'true'; + const useLocalRedis = process.env.USE_LOCAL_REDIS === 'true' if (useLocalRedis) { const localRedisUrl = - process.env.LOCAL_REDIS_URL || 'redis://localhost:6379'; - redisClient = createClient({ url: localRedisUrl }); - await redisClient.connect(); + process.env.LOCAL_REDIS_URL || 'redis://localhost:6379' + redisClient = createClient({ url: localRedisUrl }) + await redisClient.connect() } else { - const upstashRedisRestUrl = process.env.UPSTASH_REDIS_REST_URL; - const upstashRedisRestToken = process.env.UPSTASH_REDIS_REST_TOKEN; + const upstashRedisRestUrl = process.env.UPSTASH_REDIS_REST_URL + const upstashRedisRestToken = process.env.UPSTASH_REDIS_REST_TOKEN if (upstashRedisRestUrl && upstashRedisRestToken) { redisClient = new Redis({ url: upstashRedisRestUrl, - token: upstashRedisRestToken, - }); + token: upstashRedisRestToken + }) } } - return redisClient; + return redisClient } // Function to get cached results @@ -58,26 +59,26 @@ async function getCachedResults( cacheKey: string ): Promise { try { - const client = await initializeRedisClient(); - if (!client) return null; + const client = await initializeRedisClient() + if (!client) return null - let cachedData: string | null; + let cachedData: string | null if (client instanceof Redis) { - cachedData = await client.get(cacheKey); + cachedData = await client.get(cacheKey) } else { - cachedData = await client.get(cacheKey); + cachedData = await client.get(cacheKey) } if (cachedData) { - console.log(`Cache hit for key: ${cacheKey}`); - return JSON.parse(cachedData); + console.log(`Cache hit for key: ${cacheKey}`) + return JSON.parse(cachedData) } else { - console.log(`Cache miss for key: ${cacheKey}`); - return null; + console.log(`Cache miss for key: ${cacheKey}`) + return null } } catch (error) { - console.error('Redis cache error:', error); - return null; + console.error('Redis cache error:', error) + return null } } @@ -87,58 +88,65 @@ async function setCachedResults( results: SearXNGSearchResults ): Promise { try { - const client = await initializeRedisClient(); - if (!client) return; + const client = await initializeRedisClient() + if (!client) return - const serializedResults = JSON.stringify(results); + const serializedResults = JSON.stringify(results) if (client instanceof Redis) { - await client.set(cacheKey, serializedResults, { ex: CACHE_TTL }); + await client.set(cacheKey, serializedResults, { ex: CACHE_TTL }) } else { - await client.set(cacheKey, serializedResults, { EX: CACHE_TTL }); + await client.set(cacheKey, serializedResults, { EX: CACHE_TTL }) } - console.log(`Cached results for key: ${cacheKey}`); + console.log(`Cached results for key: ${cacheKey}`) } catch (error) { - console.error('Redis cache error:', error); + console.error('Redis cache error:', error) } } // Function to periodically clean up expired cache entries async function cleanupExpiredCache() { try { - const client = await initializeRedisClient(); - if (!client) return; + const client = await initializeRedisClient() + if (!client) return - const keys = await client.keys('search:*'); + const keys = await client.keys('search:*') for (const key of keys) { - const ttl = await client.ttl(key); + const ttl = await client.ttl(key) if (ttl <= 0) { - await client.del(key); - console.log(`Removed expired cache entry: ${key}`); + await client.del(key) + console.log(`Removed expired cache entry: ${key}`) } } } catch (error) { - console.error('Cache cleanup error:', error); + console.error('Cache cleanup error:', error) } } // Set up periodic cache cleanup -setInterval(cleanupExpiredCache, CACHE_EXPIRATION_CHECK_INTERVAL); +setInterval(cleanupExpiredCache, CACHE_EXPIRATION_CHECK_INTERVAL) export async function POST(request: Request) { + const user = await fetchUser() + if (!user) { + return new NextResponse(null, { + status: 403, + statusText: 'Unauthorized' + }) + } const { query, maxResults, searchDepth, includeDomains, excludeDomains } = - await request.json(); + await request.json() - const SEARXNG_DEFAULT_DEPTH = process.env.SEARXNG_DEFAULT_DEPTH || 'basic'; + const SEARXNG_DEFAULT_DEPTH = process.env.SEARXNG_DEFAULT_DEPTH || 'basic' try { const cacheKey = `search:${query}:${maxResults}:${searchDepth}:${ Array.isArray(includeDomains) ? includeDomains.join(',') : '' - }:${Array.isArray(excludeDomains) ? excludeDomains.join(',') : ''}`; + }:${Array.isArray(excludeDomains) ? excludeDomains.join(',') : ''}` // Try to get cached results - const cachedResults = await getCachedResults(cacheKey); + const cachedResults = await getCachedResults(cacheKey) if (cachedResults) { - return NextResponse.json(cachedResults); + return NextResponse.json(cachedResults) } // If not cached, perform the search @@ -148,14 +156,14 @@ export async function POST(request: Request) { searchDepth || SEARXNG_DEFAULT_DEPTH, Array.isArray(includeDomains) ? includeDomains : [], Array.isArray(excludeDomains) ? excludeDomains : [] - ); + ) // Cache the results - await setCachedResults(cacheKey, results); + await setCachedResults(cacheKey, results) - return NextResponse.json(results); + return NextResponse.json(results) } catch (error) { - console.error('Advanced search error:', error); + console.error('Advanced search error:', error) return NextResponse.json( { message: 'Internal Server Error', @@ -163,10 +171,10 @@ export async function POST(request: Request) { query: query, results: [], images: [], - number_of_results: 0, + number_of_results: 0 }, { status: 500 } - ); + ) } } @@ -177,126 +185,126 @@ async function advancedSearchXNGSearch( includeDomains: string[] = [], excludeDomains: string[] = [] ): Promise { - const apiUrl = process.env.SEARXNG_API_URL; + const apiUrl = process.env.SEARXNG_API_URL if (!apiUrl) { - throw new Error('SEARXNG_API_URL is not set in the environment variables'); + throw new Error('SEARXNG_API_URL is not set in the environment variables') } const SEARXNG_ENGINES = - process.env.SEARXNG_ENGINES || 'google,bing,duckduckgo,wikipedia'; - const SEARXNG_TIME_RANGE = process.env.SEARXNG_TIME_RANGE || 'None'; - const SEARXNG_SAFESEARCH = process.env.SEARXNG_SAFESEARCH || '0'; + process.env.SEARXNG_ENGINES || 'google,bing,duckduckgo,wikipedia' + const SEARXNG_TIME_RANGE = process.env.SEARXNG_TIME_RANGE || 'None' + const SEARXNG_SAFESEARCH = process.env.SEARXNG_SAFESEARCH || '0' const SEARXNG_CRAWL_MULTIPLIER = parseInt( process.env.SEARXNG_CRAWL_MULTIPLIER || '4', 10 - ); + ) try { - const url = new URL(`${apiUrl}/search`); - url.searchParams.append('q', query); - url.searchParams.append('format', 'json'); - url.searchParams.append('categories', 'general,images'); + const url = new URL(`${apiUrl}/search`) + url.searchParams.append('q', query) + url.searchParams.append('format', 'json') + url.searchParams.append('categories', 'general,images') // Add time_range if it's not 'None' if (SEARXNG_TIME_RANGE !== 'None') { - url.searchParams.append('time_range', SEARXNG_TIME_RANGE); + url.searchParams.append('time_range', SEARXNG_TIME_RANGE) } - url.searchParams.append('safesearch', SEARXNG_SAFESEARCH); - url.searchParams.append('engines', SEARXNG_ENGINES); + url.searchParams.append('safesearch', SEARXNG_SAFESEARCH) + url.searchParams.append('engines', SEARXNG_ENGINES) - const resultsPerPage = 10; - const pageno = Math.ceil(maxResults / resultsPerPage); - url.searchParams.append('pageno', String(pageno)); + const resultsPerPage = 10 + const pageno = Math.ceil(maxResults / resultsPerPage) + url.searchParams.append('pageno', String(pageno)) //console.log('SearXNG API URL:', url.toString()) // Log the full URL for debugging const data: | SearXNGResponse | { error: string; status: number; data: string } = - await fetchJsonWithRetry(url.toString(), 3); + await fetchJsonWithRetry(url.toString(), 3) if ('error' in data) { - console.error('Invalid response from SearXNG:', data); + console.error('Invalid response from SearXNG:', data) throw new Error( `Invalid response from SearXNG: ${data.error}. Status: ${data.status}. Data: ${data.data}` - ); + ) } if (!data || !Array.isArray(data.results)) { - console.error('Invalid response structure from SearXNG:', data); - throw new Error('Invalid response structure from SearXNG'); + console.error('Invalid response structure from SearXNG:', data) + throw new Error('Invalid response structure from SearXNG') } let generalResults = data.results.filter( (result: SearXNGResult) => result && !result.img_src - ); + ) // Apply domain filtering manually if (includeDomains.length > 0 || excludeDomains.length > 0) { - generalResults = generalResults.filter((result) => { - const domain = new URL(result.url).hostname; + generalResults = generalResults.filter(result => { + const domain = new URL(result.url).hostname return ( (includeDomains.length === 0 || - includeDomains.some((d) => domain.includes(d))) && + includeDomains.some(d => domain.includes(d))) && (excludeDomains.length === 0 || - !excludeDomains.some((d) => domain.includes(d))) - ); - }); + !excludeDomains.some(d => domain.includes(d))) + ) + }) } if (searchDepth === 'advanced') { const crawledResults = await Promise.all( generalResults .slice(0, maxResults * SEARXNG_CRAWL_MULTIPLIER) - .map((result) => crawlPage(result, query)) - ); + .map(result => crawlPage(result, query)) + ) generalResults = crawledResults - .filter((result) => result !== null && isQualityContent(result.content)) - .map((result) => result as SearXNGResult); + .filter(result => result !== null && isQualityContent(result.content)) + .map(result => result as SearXNGResult) - const MIN_RELEVANCE_SCORE = 10; + const MIN_RELEVANCE_SCORE = 10 generalResults = generalResults - .map((result) => ({ + .map(result => ({ ...result, - score: calculateRelevanceScore(result, query), + score: calculateRelevanceScore(result, query) })) - .filter((result) => result.score >= MIN_RELEVANCE_SCORE) + .filter(result => result.score >= MIN_RELEVANCE_SCORE) .sort((a, b) => b.score - a.score) - .slice(0, maxResults); + .slice(0, maxResults) } - generalResults = generalResults.slice(0, maxResults); + generalResults = generalResults.slice(0, maxResults) const imageResults = (data.results || []) .filter((result: SearXNGResult) => result && result.img_src) - .slice(0, maxResults); + .slice(0, maxResults) return { results: generalResults.map( (result: SearXNGResult): SearchResultItem => ({ title: result.title || '', url: result.url || '', - content: result.content || '', + content: result.content || '' }) ), query: data.query || query, images: imageResults .map((result: SearXNGResult) => { - const imgSrc = result.img_src || ''; - return imgSrc.startsWith('http') ? imgSrc : `${apiUrl}${imgSrc}`; + const imgSrc = result.img_src || '' + return imgSrc.startsWith('http') ? imgSrc : `${apiUrl}${imgSrc}` }) .filter(Boolean), - number_of_results: data.number_of_results || generalResults.length, - }; + number_of_results: data.number_of_results || generalResults.length + } } catch (error) { - console.error('SearchXNG API error:', error); + console.error('SearchXNG API error:', error) return { results: [], query: query, images: [], - number_of_results: 0, - }; + number_of_results: 0 + } } } @@ -305,94 +313,94 @@ async function crawlPage( query: string ): Promise { try { - const html = await fetchHtmlWithTimeout(result.url, 20000); + const html = await fetchHtmlWithTimeout(result.url, 20000) // virtual console to suppress JSDOM warnings - const virtualConsole = new VirtualConsole(); - virtualConsole.on('error', () => {}); - virtualConsole.on('warn', () => {}); + const virtualConsole = new VirtualConsole() + virtualConsole.on('error', () => {}) + virtualConsole.on('warn', () => {}) const dom = new JSDOM(html, { runScripts: 'outside-only', resources: 'usable', - virtualConsole, - }); - const document = dom.window.document; + virtualConsole + }) + const document = dom.window.document // Remove script, style, nav, header, and footer elements document .querySelectorAll('script, style, nav, header, footer') - .forEach((el: Element) => el.remove()); + .forEach((el: Element) => el.remove()) const mainContent = document.querySelector('main') || document.querySelector('article') || document.querySelector('.content') || document.querySelector('#content') || - document.body; + document.body if (mainContent) { // Prioritize specific content elements - const priorityElements = mainContent.querySelectorAll('h1, h2, h3, p'); + const priorityElements = mainContent.querySelectorAll('h1, h2, h3, p') let extractedText = Array.from(priorityElements) - .map((el) => el.textContent?.trim()) + .map(el => el.textContent?.trim()) .filter(Boolean) - .join('\n\n'); + .join('\n\n') // If not enough content, fall back to other elements if (extractedText.length < 500) { const contentElements = mainContent.querySelectorAll( 'h4, h5, h6, li, td, th, blockquote, pre, code' - ); + ) extractedText += '\n\n' + Array.from(contentElements) - .map((el) => el.textContent?.trim()) + .map(el => el.textContent?.trim()) .filter(Boolean) - .join('\n\n'); + .join('\n\n') } // Extract metadata const metaDescription = document .querySelector('meta[name="description"]') - ?.getAttribute('content') || ''; + ?.getAttribute('content') || '' const metaKeywords = document .querySelector('meta[name="keywords"]') - ?.getAttribute('content') || ''; + ?.getAttribute('content') || '' const ogTitle = document .querySelector('meta[property="og:title"]') - ?.getAttribute('content') || ''; + ?.getAttribute('content') || '' const ogDescription = document .querySelector('meta[property="og:description"]') - ?.getAttribute('content') || ''; + ?.getAttribute('content') || '' // Combine metadata with extracted text - extractedText = `${result.title}\n\n${ogTitle}\n\n${metaDescription}\n\n${ogDescription}\n\n${metaKeywords}\n\n${extractedText}`; + extractedText = `${result.title}\n\n${ogTitle}\n\n${metaDescription}\n\n${ogDescription}\n\n${metaKeywords}\n\n${extractedText}` // Limit the extracted text to 10000 characters - extractedText = extractedText.substring(0, 10000); + extractedText = extractedText.substring(0, 10000) // Highlight query terms in the content - result.content = highlightQueryTerms(extractedText, query); + result.content = highlightQueryTerms(extractedText, query) // Extract publication date - const publishedDate = extractPublicationDate(document); + const publishedDate = extractPublicationDate(document) if (publishedDate) { - result.publishedDate = publishedDate.toISOString(); + result.publishedDate = publishedDate.toISOString() } } - return result; + return result } catch (error) { - console.error(`Error crawling ${result.url}:`, error); + console.error(`Error crawling ${result.url}:`, error) return { ...result, - content: result.content || 'Content unavailable due to crawling error.', - }; + content: result.content || 'Content unavailable due to crawling error.' + } } } @@ -401,92 +409,92 @@ function highlightQueryTerms(content: string, query: string): string { const terms = query .toLowerCase() .split(/\s+/) - .filter((term) => term.length > 2) - .map((term) => term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')); // Escape special characters + .filter(term => term.length > 2) + .map(term => term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')) // Escape special characters - let highlightedContent = content; + let highlightedContent = content - terms.forEach((term) => { - const regex = new RegExp(`\\b${term}\\b`, 'gi'); + terms.forEach(term => { + const regex = new RegExp(`\\b${term}\\b`, 'gi') highlightedContent = highlightedContent.replace( regex, - (match) => `${match}` - ); - }); + match => `${match}` + ) + }) - return highlightedContent; + return highlightedContent } catch (error) { //console.error('Error in highlightQueryTerms:', error) - return content; // Return original content if highlighting fails + return content // Return original content if highlighting fails } } function calculateRelevanceScore(result: SearXNGResult, query: string): number { try { - const lowercaseContent = result.content.toLowerCase(); - const lowercaseQuery = query.toLowerCase(); + const lowercaseContent = result.content.toLowerCase() + const lowercaseQuery = query.toLowerCase() const queryWords = lowercaseQuery .split(/\s+/) - .filter((word) => word.length > 2) - .map((word) => word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')); // Escape special characters + .filter(word => word.length > 2) + .map(word => word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')) // Escape special characters - let score = 0; + let score = 0 // Check for exact phrase match if (lowercaseContent.includes(lowercaseQuery)) { - score += 30; + score += 30 } // Check for individual word matches - queryWords.forEach((word) => { - const regex = new RegExp(`\\b${word}\\b`, 'g'); - const wordCount = (lowercaseContent.match(regex) || []).length; - score += wordCount * 3; - }); + queryWords.forEach(word => { + const regex = new RegExp(`\\b${word}\\b`, 'g') + const wordCount = (lowercaseContent.match(regex) || []).length + score += wordCount * 3 + }) // Boost score for matches in the title - const lowercaseTitle = result.title.toLowerCase(); + const lowercaseTitle = result.title.toLowerCase() if (lowercaseTitle.includes(lowercaseQuery)) { - score += 20; + score += 20 } - queryWords.forEach((word) => { - const regex = new RegExp(`\\b${word}\\b`, 'g'); + queryWords.forEach(word => { + const regex = new RegExp(`\\b${word}\\b`, 'g') if (lowercaseTitle.match(regex)) { - score += 10; + score += 10 } - }); + }) // Boost score for recent content (if available) if (result.publishedDate) { - const publishDate = new Date(result.publishedDate); - const now = new Date(); + const publishDate = new Date(result.publishedDate) + const now = new Date() const daysSincePublished = - (now.getTime() - publishDate.getTime()) / (1000 * 3600 * 24); + (now.getTime() - publishDate.getTime()) / (1000 * 3600 * 24) if (daysSincePublished < 30) { - score += 15; + score += 15 } else if (daysSincePublished < 90) { - score += 10; + score += 10 } else if (daysSincePublished < 365) { - score += 5; + score += 5 } } // Penalize very short content if (result.content.length < 200) { - score -= 10; + score -= 10 } else if (result.content.length > 1000) { - score += 5; + score += 5 } // Boost score for content with more highlighted terms - const highlightCount = (result.content.match(//g) || []).length; - score += highlightCount * 2; + const highlightCount = (result.content.match(//g) || []).length + score += highlightCount * 2 - return score; + return score } catch (error) { //console.error('Error in calculateRelevanceScore:', error) - return 0; // Return 0 if scoring fails + return 0 // Return 0 if scoring fails } } @@ -497,80 +505,80 @@ function extractPublicationDate(document: Document): Date | null { 'meta[name="publication-date"]', 'meta[name="date"]', 'time[datetime]', - 'time[pubdate]', - ]; + 'time[pubdate]' + ] for (const selector of dateSelectors) { - const element = document.querySelector(selector); + const element = document.querySelector(selector) if (element) { const dateStr = element.getAttribute('content') || element.getAttribute('datetime') || - element.getAttribute('pubdate'); + element.getAttribute('pubdate') if (dateStr) { - const date = new Date(dateStr); + const date = new Date(dateStr) if (!isNaN(date.getTime())) { - return date; + return date } } } } - return null; + return null } -const httpAgent = new http.Agent({ keepAlive: true }); +const httpAgent = new http.Agent({ keepAlive: true }) const httpsAgent = new https.Agent({ keepAlive: true, - rejectUnauthorized: true, // change to false if you want to ignore SSL certificate errors + rejectUnauthorized: true // change to false if you want to ignore SSL certificate errors //but use this with caution. -}); +}) async function fetchJsonWithRetry(url: string, retries: number): Promise { for (let i = 0; i < retries; i++) { try { - return await fetchJson(url); + return await fetchJson(url) } catch (error) { - if (i === retries - 1) throw error; - await new Promise((resolve) => setTimeout(resolve, 1000 * (i + 1))); + if (i === retries - 1) throw error + await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1))) } } } function fetchJson(url: string): Promise { return new Promise((resolve, reject) => { - const protocol = url.startsWith('https:') ? https : http; - const agent = url.startsWith('https:') ? httpsAgent : httpAgent; - const request = protocol.get(url, { agent }, (res) => { - let data = ''; - res.on('data', (chunk) => { - data += chunk; - }); + const protocol = url.startsWith('https:') ? https : http + const agent = url.startsWith('https:') ? httpsAgent : httpAgent + const request = protocol.get(url, { agent }, res => { + let data = '' + res.on('data', chunk => { + data += chunk + }) res.on('end', () => { try { // Check if the response is JSON if (res.headers['content-type']?.includes('application/json')) { - resolve(JSON.parse(data)); + resolve(JSON.parse(data)) } else { // If not JSON, return an object with the raw data and status resolve({ error: 'Invalid JSON response', status: res.statusCode, - data: data.substring(0, 200), // Include first 200 characters of the response - }); + data: data.substring(0, 200) // Include first 200 characters of the response + }) } } catch (e) { - reject(e); + reject(e) } - }); - }); - request.on('error', reject); + }) + }) + request.on('error', reject) request.on('timeout', () => { - request.destroy(); - reject(new Error('Request timed out')); - }); - request.setTimeout(15000); // 15 second timeout - }); + request.destroy() + reject(new Error('Request timed out')) + }) + request.setTimeout(15000) // 15 second timeout + }) } async function fetchHtmlWithTimeout( @@ -580,20 +588,20 @@ async function fetchHtmlWithTimeout( try { return await Promise.race([ fetchHtml(url), - timeout(timeoutMs, `Fetching ${url} timed out after ${timeoutMs}ms`), - ]); + timeout(timeoutMs, `Fetching ${url} timed out after ${timeoutMs}ms`) + ]) } catch (error) { - console.error(`Error fetching ${url}:`, error); - const errorMessage = error instanceof Error ? error.message : String(error); - return `Error fetching content: ${errorMessage}`; + console.error(`Error fetching ${url}:`, error) + const errorMessage = error instanceof Error ? error.message : String(error) + return `Error fetching content: ${errorMessage}` } } function fetchHtml(url: string): Promise { return new Promise((resolve, reject) => { - const protocol = url.startsWith('https:') ? https : http; - const agent = url.startsWith('https:') ? httpsAgent : httpAgent; - const request = protocol.get(url, { agent }, (res) => { + const protocol = url.startsWith('https:') ? https : http + const agent = url.startsWith('https:') ? httpsAgent : httpAgent + const request = protocol.get(url, { agent }, res => { if ( res.statusCode && res.statusCode >= 300 && @@ -603,40 +611,40 @@ function fetchHtml(url: string): Promise { // Handle redirects fetchHtml(new URL(res.headers.location, url).toString()) .then(resolve) - .catch(reject); - return; + .catch(reject) + return } - let data = ''; - res.on('data', (chunk) => { - data += chunk; - }); - res.on('end', () => resolve(data)); - }); - request.on('error', (error) => { + let data = '' + res.on('data', chunk => { + data += chunk + }) + res.on('end', () => resolve(data)) + }) + request.on('error', error => { //console.error(`Error fetching ${url}:`, error) - reject(error); - }); + reject(error) + }) request.on('timeout', () => { - request.destroy(); + request.destroy() //reject(new Error(`Request timed out for ${url}`)) - resolve(''); - }); - request.setTimeout(10000); // 10 second timeout - }); + resolve('') + }) + request.setTimeout(10000) // 10 second timeout + }) } function timeout(ms: number, message: string): Promise { return new Promise((_, reject) => { setTimeout(() => { - reject(new Error(message)); - }, ms); - }); + reject(new Error(message)) + }, ms) + }) } function isQualityContent(text: string): boolean { - const words = text.split(/\s+/).length; - const sentences = text.split(/[.!?]+/).length; - const avgWordsPerSentence = words / sentences; + const words = text.split(/\s+/).length + const sentences = text.split(/[.!?]+/).length + const avgWordsPerSentence = words / sentences return ( words > 50 && @@ -645,5 +653,5 @@ function isQualityContent(text: string): boolean { avgWordsPerSentence < 30 && !text.includes('Content unavailable due to crawling error') && !text.includes('Error fetching content:') - ); + ) } diff --git a/app/api/sign-up/route.ts b/app/api/sign-up/route.ts new file mode 100644 index 0000000..5c9393a --- /dev/null +++ b/app/api/sign-up/route.ts @@ -0,0 +1,34 @@ +import { type NextRequest, NextResponse } from 'next/server'; +import { currentUser } from '@clerk/nextjs/server'; +import { prisma } from '@/lib/prisma'; + +export async function GET(request: NextRequest) { + const url = request.nextUrl.clone(); + url.pathname = '/'; + const user = await currentUser(); + if (!user) { + return NextResponse.redirect(url); + } + const prismaUser = await prisma.user.findUnique({ + where: { userId: user.id }, + }); + if (prismaUser) { + return NextResponse.redirect(url); + } + try { + await prisma.user.create({ + data: { + userId: user.id, + email: user.emailAddresses[0].emailAddress, + imageUrl: user.imageUrl, + password: 'defaultPassword', + role: 'user', + }, + }); + } catch (error) { + console.error(error); + } + return NextResponse.redirect(url); +} + +export const dynamic = 'force-dynamic'; diff --git a/app/layout.tsx b/app/layout.tsx index d508e7b..6a01c0f 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -1,5 +1,6 @@ import type { Metadata, Viewport } from 'next'; import { Inter as FontSans } from 'next/font/google'; +import { ClerkProvider } from '@clerk/nextjs'; import './globals.css'; import { cn } from '@/lib/utils'; import { ThemeProvider } from '@/components/theme-provider'; @@ -47,23 +48,25 @@ export default function RootLayout({ children: React.ReactNode; }>) { return ( - - - - -
- {children} - -