diff --git a/README.md b/README.md index 050b565..ec17f1c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Manages Nginx for reverse proxy to multiple LLMs, with TLS & Bearer Auth tokens. Deployed with docker. -- Aggregates multiple OpenAI-type LLM APIs (all routes must be prefixed with "/v1") +- Aggregates multiple OpenAI-type LLM APIs - Supports cloudflare domains - Uses Let's Encrypt for TLS certificates - Uses certbot for certificate issuance and renewal @@ -33,7 +33,7 @@ version: '3.6' services: llmp: - image: ghcr.io/j4ys0n/llm-proxy:1.5.0 + image: ghcr.io/j4ys0n/llm-proxy:1.5.1 container_name: llmp hostname: llmp restart: unless-stopped @@ -51,7 +51,7 @@ services: Here's what your `.env` file should look like: ```bash PORT=8080 # node.js listen port. right now nginx is hard coded, so don't change this. -TARGET_URLS=http://localhost:1234,http://192.168.1.100:1234|api-key-here # list of api endpoints (don't add /v1) +TARGET_URLS=http://localhost:1234,http://192.168.1.100:1234|api-key-here # list of api endpoints (/v1 is optional) JWT_SECRET=randomly_generated_secret # secret for JWT token generation, change this! AUTH_USERNAME=admin AUTH_PASSWORD=secure_password # super basic auth credentials for the admin interface diff --git a/docker-compose.yml b/docker-compose.yml index 9ab1fbc..904be54 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.6' services: llmp: - image: ghcr.io/j4ys0n/llm-proxy:1.5.0 + image: ghcr.io/j4ys0n/llm-proxy:1.5.1 container_name: llmp hostname: llmp restart: unless-stopped diff --git a/package.json b/package.json index fec05d3..57c96e1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "llm-proxy", - "version": "1.5.0", + "version": "1.5.1", "description": "Manages Nginx for reverse proxy to multiple LLMs, with TLS & Bearer Auth tokens", "main": "dist/index.js", "scripts": { diff --git a/src/controllers/llm.ts b/src/controllers/llm.ts index 0902b05..328de76 100644 --- a/src/controllers/llm.ts +++ b/src/controllers/llm.ts @@ -15,19 +15,43 @@ export interface ModelMap { const defaultContentType = 'application/json' +function getPath(url: string): { path: string, base: string, apiKey?: string } { + try { + const urlParts = url.split('|') + const apiKey = urlParts.length > 1 ? urlParts[1] : undefined + const urlObject = new URL(url) + return { + path: urlObject.pathname || '/v1', + base: urlObject.origin, + apiKey + } + } catch (error) { + // Return the input if it's already a path starting with '/' + if (url.startsWith('/')) return { path: url, base: 'http://localhost' } + // Return '/v1' for invalid URLs + return { path: '/v1', base: 'http://localhost' } + } +} + async function fetchModels(targetUrls: string[]): Promise { const tmp: ModelMap = {} for (const urlAndToken of targetUrls) { const [url, apiKey] = urlAndToken.split('|').map(s => s.trim()) - const reqHeaders: { [key: string]: string } = { + const { path, base } = getPath(url) + const headers: { [key: string]: string } = { accept: defaultContentType, 'Content-Type': defaultContentType } if (apiKey != null && apiKey !== '') { - reqHeaders['Authorization'] = `Bearer ${apiKey}` + headers['Authorization'] = `Bearer ${apiKey}` + } + const params = { + method: 'GET', + url: `${base}/${path}/models`, + headers } try { - const response = await axios.get(`${url}/v1/models`) + const response = await axios(params) const models = response.data.data || [] const hostId = extractDomainName(url) models.forEach((model: Model) => { @@ -85,26 +109,36 @@ export class LLMController { public async forwardPostRequest(req: Request, res: Response, next: NextFunction) { if ( req.method === 'POST' && - (req.path.startsWith('v1/') || req.path.startsWith('/v1/')) && + (req.path.startsWith('v1') || req.path.startsWith('/v1')) && req.body != null && req.body.model != null && this.targetUrls.length > 0 ) { const { model: modelId } = req.body - let targetUrl = this.targetUrls[0] // Default to first URL if no matching model found + const { base: firstBaseUrl, path: firstPath, apiKey: firstApiKey } = getPath(this.targetUrls[0]) + let targetUrl = firstBaseUrl // Default to first URL if no matching model found + let targetPath = firstPath + let targetApiKey = firstApiKey const hash = md5(modelId) if (modelId && this.modelCache[hash]) { - targetUrl = this.modelCache[hash].url + const { path, base, apiKey } = getPath(this.modelCache[hash].url) + targetUrl = base + targetPath = path + targetApiKey = apiKey } - const fullUrl = new URL(req.path, targetUrl).toString() + const reqPath = req.path.startsWith('/v1/') ? req.path.replace('/v1', targetPath) : `${targetPath}${req.path}` + const fullUrl = new URL(reqPath, targetUrl).toString() log('info', `Forwarding request to: ${fullUrl} -> ${modelId}`) - + const headers = { ...req.headers } + if (targetApiKey) { + headers['Authorization'] = `Bearer ${targetApiKey}` + } try { const axiosConfig: AxiosRequestConfig = { method: req.method, url: fullUrl, - headers: { ...req.headers }, + headers, data: req.body, responseType: 'stream' }