perf: 优化执行效率，重构自定义 rule 加载与处理规则

package.json

@@ -27,7 +27,6 @@
     "cookie",
     "env"
   ],
-  "packageManager": "[email protected]",
   "engines": {
     "node": ">=16"
   },

src/lib/getConfig.ts

@@ -11,7 +11,7 @@ import { basename, resolve } from 'node:path';
 import { homedir } from 'node:os';
 import { assign, color } from '@lzwme/fe-utils';
 import { logger } from './helper';
-import { rulesManage } from './rulesManage';
+import { rulesManager } from './rulesManager';
 import { Watcher } from './watch';
 import { loadW2Rules } from './w2RulesManage';
 
@@ -93,12 +93,12 @@ export function getConfig(useCache = true) {
       }
     });
 
-    const allRules = rulesManage.loadRules(config.ruleDirs, true);
+    const allRules = rulesManager.loadRules(config.ruleDirs, true);
     config.rules.forEach(d => {
       if (Array.isArray(d)) d.forEach(d => d.ruleId && allRules.set(d.ruleId, d));
       else if (d?.ruleId) allRules.set(d.ruleId, d);
     });
-    rulesManage.classifyRules([...allRules.values()], config, true);
+    rulesManager.classifyRules([...allRules.values()], config, true);
 
     loadW2Rules(config.whistleRules, 'clear');
 

src/lib/ruleHandler.ts

@@ -2,7 +2,7 @@
  * @Author: renxia
  * @Date: 2024-01-10 16:58:26
  * @LastEditors: renxia
- * @LastEditTime: 2024-03-05 15:33:27
+ * @LastEditTime: 2024-05-08 15:19:51
  * @Description: 基于 whistle 的 cookie 自动抓取插件
  */
 
@@ -23,7 +23,7 @@ type RuleHandlerOptions = {
   resBody?: Buffer;
 };
 
-function ruleMatcher({ rule, req }: RuleHandlerOptions) {
+export function ruleMatcher({ rule, req }: RuleHandlerOptions) {
   let errmsg = '';
 
   // method
@@ -87,7 +87,7 @@ export async function ruleHandler({ rule, req, res, reqBody, resBody }: RuleHand
     }
   }
 
-  const params: RuleHandlerParams = { req, reqBody: req._reqBody, resBody: res._resBody, headers, url, cookieObj, allCacheData: [], X };
+  const params: RuleHandlerParams = { req, reqBody: req._reqBody, resBody: res._resBody, headers, url, cookieObj, cacheData: [], X };
   if (resHeaders) params.resHeaders = resHeaders;
 
   if (rule.getCacheUid) {
@@ -115,12 +115,13 @@ export async function ruleHandler({ rule, req, res, reqBody, resBody }: RuleHand
 
       cacheData[uid] = { update: now, data: { uid, headers: req.originalReq.headers, data: uidData } };
 
-      params.allCacheData = [];
+      params.cacheData = [];
       const cacheDuration = 1000 * (Number(rule.cacheDuration || config.cacheDuration) || (rule.updateEnvValue ? 12 : 24 * 10) * 3600);
       for (const [key, value] of Object.entries(cacheData)) {
         if (cacheDuration && now - value.update > cacheDuration) delete cacheData[key];
-        else params.allCacheData.push(value.data);
+        else params.cacheData.push(value.data);
       }
+      params.allCacheData = params.cacheData;
 
       storage.setItem(rule.ruleId, cacheData);
     } else {

src/lib/rulesManage.ts → src/lib/rulesManager.ts

@@ -26,18 +26,13 @@ function ruleFormat(rule: RuleItem) {
   }
 
   if (!rule.on || !RuleOnList.includes(rule.on)) {
-    const ruleOn = rule.getCacheUid ? 'req-header' : 'res-body';
+    const ruleOn = rule.getCacheUid ? 'res-body' : 'req-header';
     if (rule.on) logger.warn(`[${rule.on}] 参数错误，自动设置为[${ruleOn}]！只能取值为: ${RuleOnList.join(', ')}`, rule.ruleId);
     rule.on = ruleOn;
   }
 
   if (!rule.desc) rule.desc = `${rule.ruleId}_${rule.url || rule.method}`;
 
-  if (!rule.mitm && typeof rule.url === 'string') {
-    const r = /^https:\/\/([a-z.*]+)/.exec(rule.url);
-    if (r) rule.mitm = [r[1]];
-  }
-
   if (rule.mitm) {
     if (!Array.isArray(rule.mitm)) rule.mitm = [rule.mitm];
     rule.mitm = rule.mitm.filter(d => d && (typeof d === 'string' || d instanceof RegExp));
@@ -151,7 +146,7 @@ function removeRule(ruleId?: string[], filepath?: string[]) {
   const ruleSet = new Set(ruleId ? ruleId : []);
   const fileSet = new Set(filepath ? filepath : []);
 
-  for (const [type, item] of Object.entries(rulesManage.rules)) {
+  for (const [type, item] of Object.entries(rulesManager.rules)) {
     for (const [ruleId, rule] of item) {
       if (ruleSet.has(ruleId) || fileSet.has(rule._source)) {
         item.delete(ruleId);
@@ -174,7 +169,7 @@ const onRuleFileChange: WatcherOnChange = (type, filepath) => {
   }
 };
 
-export const rulesManage = {
+export const rulesManager = {
   rules: RulesCache,
   ruleFormat,
   classifyRules,

src/lib/update.ts

@@ -2,7 +2,7 @@
  * @Author: renxia
  * @Date: 2024-01-11 13:38:34
  * @LastEditors: renxia
- * @LastEditTime: 2024-04-22 09:22:04
+ * @LastEditTime: 2024-05-07 09:34:05
  * @Description:
  */
 import fs from 'node:fs';
@@ -22,7 +22,7 @@ export async function updateToQlEnvConfig(envConfig: EnvConfig, updateEnvValue?:
   if (!(await ql.login())) return;
 
   if (Date.now() - updateCache.updateTime > 1000 * 60 * 60 * 1) updateCache.qlEnvList = [];
-  let { name, value, desc, sep } = envConfig;
+  let { name, value, desc, sep = '\n' } = envConfig;
   let item = updateCache.qlEnvList.find(d => d.name === name);
   if (!item) {
     updateCache.qlEnvList = await ql.getEnvList();
@@ -62,7 +62,8 @@ export async function updateToQlEnvConfig(envConfig: EnvConfig, updateEnvValue?:
   }
 
   const isSuccess = r.code === 200;
-  logger.info(`${item ? green('更新') : magenta('新增')}QL环境变量[${name}]`, isSuccess ? '成功' : r);
+  const count = params.value.includes(sep) ? params.value.trim().split(sep).length : 1;
+  logger.info(`${item ? green('更新') : magenta('新增')}QL环境变量[${green(name)}][${count}]`, isSuccess ? '成功' : r);
   if (isSuccess && item) item.value = value;
 
   return value;

src/lib/w2RulesManage.ts

@@ -2,7 +2,7 @@
  * @Author: renxia
  * @Date: 2024-01-22 14:00:13
  * @LastEditors: renxia
- * @LastEditTime: 2024-03-04 22:48:50
+ * @LastEditTime: 2024-05-08 13:59:33
  * @Description:
  */
 import { existsSync, readFileSync } from 'node:fs';
@@ -11,6 +11,7 @@ import { color } from '@lzwme/fe-utils';
 import { Watcher } from './watch';
 import { logger } from './helper';
 import type { WhistleRuleItem } from '../../typings';
+import { isMatch } from 'micromatch';
 
 type W2RuleMapItem = { rules: string[]; values?: Record<string, any>; config?: WhistleRuleItem };
 
@@ -121,11 +122,12 @@ export function loadW2Rules(whistleRules: WhistleRuleItem[], action: 'clear' | '
 
 const cache = {
   w2Rules: {
-    value: '',
+    value: {} as Record<string, any>,
     update: 0,
   },
 };
-export function getW2Rules() {
+
+export function getW2Rules(req: Whistle.PluginRequest) {
   const now = Date.now();
 
   // 30s 缓存
@@ -138,8 +140,28 @@ export function getW2Rules() {
       Object.assign(values, item.values);
     });
 
-    cache.w2Rules.value = JSON.stringify({ rules: [...rulesSet].join('\n').replace(/#.+\n?/gm, ''), values });
+    const rules = [...rulesSet]
+      .join('\n')
+      .split('\n')
+      .map(d => d.trim())
+      .filter(d => d && d.startsWith('#') && d.includes(' ') && d.includes('//'));
+    cache.w2Rules.value = { rules, values };
+  }
+
+  const { rules, values } = cache.w2Rules.value;
+  const { fullUrl } = req;
+
+  for (const line of rules) {
+    let isMatched = false;
+    let url: string | RegExp = line.split(' ')[0];
+    if ((url as string).startsWith('^http')) url = new RegExp(url);
+
+    if (url instanceof RegExp) isMatched = url.test(fullUrl);
+    else isMatched = fullUrl.includes(url) || isMatch(fullUrl, url);
+
+    // 性能考虑，暂只支持一个规则
+    if (isMatched) return { rules: [line], values };
   }
 
-  return cache.w2Rules.value;
+  return;
 }

src/rulesServer.ts

@@ -2,7 +2,7 @@
  * @Author: renxia
  * @Date: 2024-01-22 14:00:13
  * @LastEditors: renxia
- * @LastEditTime: 2024-03-01 16:28:09
+ * @LastEditTime: 2024-05-08 14:01:31
  * @Description:
  */
 import { handlerW2RuleFiles, getW2Rules } from './lib/w2RulesManage';
@@ -15,6 +15,7 @@ export function rulesServer(server: Whistle.PluginServer, options: Whistle.Plugi
       await handlerW2RuleFiles({ path: isUrl ? '' : rulePath, url: isUrl ? rulePath : '' });
     }
 
-    res.end(getW2Rules());
+    const rules = getW2Rules(req);
+    rules ? res.end(rules) : res.end();
   });
 }

src/server.ts

@@ -9,7 +9,7 @@ import { color } from '@lzwme/fe-utils';
 import { toQueryString } from '@lzwme/fe-utils/cjs/common/url';
 import { logger } from './lib/helper';
 import { ruleHandler } from './lib/ruleHandler';
-import { rulesManage } from './lib/rulesManage';
+import { rulesManager } from './lib/rulesManager';
 import * as util from './util/util';
 
 export default (server: Whistle.PluginServer, options: Whistle.PluginOptions) => {
@@ -21,7 +21,7 @@ export default (server: Whistle.PluginServer, options: Whistle.PluginOptions) =>
       if (util.isRemote(req)) return req.passThrough();
 
       const { method, headers, fullUrl: url } = req;
-      const reqHeaderRules = rulesManage.rules['req-header'];
+      const reqHeaderRules = rulesManager.rules['req-header'];
       logger.trace('[request]', color.cyan(method), color.gray(url));
 
       if (reqHeaderRules?.size > 0) {
@@ -40,7 +40,7 @@ export default (server: Whistle.PluginServer, options: Whistle.PluginOptions) =>
         async (body, next, ctx) => {
           reqBody = body;
 
-          const mockRules = rulesManage.rules['req-body'];
+          const mockRules = rulesManager.rules['req-body'];
           if (mockRules?.size > 0) {
             for (const rule of mockRules.values()) {
               try {
@@ -64,7 +64,7 @@ export default (server: Whistle.PluginServer, options: Whistle.PluginOptions) =>
           next({ body });
         },
         async (body, next, ctx) => {
-          const resBodyRules = rulesManage.rules['res-body'];
+          const resBodyRules = rulesManager.rules['res-body'];
 
           if (resBodyRules?.size > 0) {
             for (const rule of resBodyRules.values()) {

src/tunnelRulesServer.ts

@@ -1,18 +1,22 @@
-import { getConfig } from './lib/getConfig';
-import { rulesManage } from './lib/rulesManage';
 import { isMatch } from 'micromatch';
+import { getConfig } from './lib/getConfig';
+import { ruleMatcher } from './lib/ruleHandler';
+import { rulesManager } from './lib/rulesManager';
 
 function mitmMatch(req: Whistle.PluginRequest) {
-  if (rulesManage.rules['res-body'].size === 0) return;
+  if (rulesManager.rules['res-body'].size === 0) return;
 
   const host = (req.headers.host || new URL(req.originalReq.fullUrl).host).split(':')[0];
-  const resBodyRules = rulesManage.rules['res-body'].values();
+  const resBodyRules = rulesManager.rules['res-body'].values();
 
-  for (const item of resBodyRules) {
-    if (item.mitm) {
-      const ok = (item.mitm as (string | RegExp)[]).some(d => (d instanceof RegExp ? d.test(host) : isMatch(host, d)));
+  for (const rule of resBodyRules) {
+    if (rule.mitm) {
+      const ok = (rule.mitm as (string | RegExp)[]).some(d => (d instanceof RegExp ? d.test(host) : isMatch(host, d)));
       if (ok) return host;
     }
+
+    const msg = ruleMatcher({ rule, req: req as unknown as Whistle.PluginServerRequest, res: null });
+    if (!msg) return host;
   }
 }
 

typings/index.d.ts

@@ -2,7 +2,7 @@
  * @Author: renxia
  * @Date: 2024-01-11 16:53:50
  * @LastEditors: renxia
- * @LastEditTime: 2024-04-12 09:17:15
+ * @LastEditTime: 2024-05-08 15:19:58
  * @Description:
  */
 /// <reference path="global.d.ts" />
@@ -104,8 +104,6 @@ export interface RuleItem {
   /**
    * MITM 域名匹配配置。
    * 当 res-body 类型的规则命中时会主动启用 https 解析拦截(whistle 未启用 https 拦截时)。
-   * 若 url 参数以  https:// 开头，则从其提取 host 域名配置部分作为 mitm 默认值。
-   * 推荐配置该项以选择性的启用 https 拦截，以提升 whistle 代理性能与效率。
    */
   mitm?: string | RegExp | (string | RegExp)[];
   /** url 匹配规则 */
@@ -129,7 +127,7 @@ export interface RuleItem {
   /** <${type}>handler 简写。根据 type 类型自动识别 */
   handler?: (ctx: RuleHandlerParams) => PromiseMaybe<RuleHandlerResult>;
   // /** 规则处理并返回环境变量配置。可以数组的形式返回多个 */
-  // saveCookieHandler?: (ctx: RuleHandlerParams & { allCacheData: CacheData[] }) => PromiseMaybe<RuleHandlerResult | EnvConfig | EnvConfig[]>;
+  // saveCookieHandler?: (ctx: RuleHandlerParams & { cacheData: CacheData[] }) => PromiseMaybe<RuleHandlerResult | EnvConfig | EnvConfig[]>;
   // /** [mock] 接口模拟处理，返回需 mock 的结果。若返回为空则表示忽略 */
   // mockHandler?: (ctx: RuleHandlerParams) => PromiseMaybe<RuleHandlerResult<string | Buffer> | Buffer | string | object>;
   // /** [modify] 接收到请求返回数据后修改或保存数据的处理 */
@@ -164,7 +162,12 @@ export type RuleHandlerParams = {
   /** req.headers.cookie 格式化为的对象格式 */
   cookieObj: Record<string, string>;
   /** 当设置了 getCacheUid 时，返回同一规则缓存的所有数据(包含由 getCacheUid 格式化返回的 data 数据) */
-  allCacheData: CacheData[];
+  cacheData: CacheData[];
+  /**
+   * 同 cacheData
+   * @deprecated 将在后续版本废除，请使用 cacheData
+   */
+  allCacheData?: CacheData[];
   /** [on=req-body, res-body] 请求参数 body */
   reqBody?: Record<string, any> | Buffer;
   /** [on=res-body] 远程接口返回的 body */