diff --git a/.gitignore b/.gitignore index 22e58a6c8de..99046c51a0b 100644 --- a/.gitignore +++ b/.gitignore @@ -69,8 +69,10 @@ $RECYCLE.BIN/ tmp/ data/ geolite2/ +db-ip/ output/ geoip *.dat *.mmdb *.srs +*.mrs diff --git a/README.md b/README.md index c656b36d9f0..68213d7f620 100644 --- a/README.md +++ b/README.md @@ -461,6 +461,7 @@ These two concepts are notable: `input` and `output`. The `input` is the data so - **maxmindMMDB**：MaxMind GeoLite2 country mmdb 数据格式（`GeoLite2-Country.mmdb`） - **maxmindGeoLite2ASNCSV**：MaxMind GeoLite2 ASN CSV 数据格式（`GeoLite2-ASN-CSV.zip`） - **maxmindGeoLite2CountryCSV**：MaxMind GeoLite2 country CSV 数据格式（`GeoLite2-Country-CSV.zip`） +- **dbipCountryMMDB**：DB-IP country mmdb 数据格式（`dbip-country-lite.mmdb`） - **mihomoMRS**：mihomo MRS 数据格式（`geoip-cn.mrs`） - **singboxSRS**：sing-box SRS 数据格式（`geoip-cn.srs`） - **clashRuleSetClassical**：[classical 类型的 Clash RuleSet](https://github.com/Dreamacro/clash/wiki/premium-core-features#classical) @@ -474,6 +475,7 @@ These two concepts are notable: `input` and `output`. The `input` is the data so - **lookup**：从指定的列表中查找指定的 IP 或 CIDR - **v2rayGeoIPDat**：V2Ray GeoIP dat 数据格式（`geoip.dat`） - **maxmindMMDB**：MaxMind GeoLite2 country mmdb 数据格式（`GeoLite2-Country.mmdb`） +- **dbipCountryMMDB**：DB-IP country mmdb 数据格式（`dbip-country-lite.mmdb`） - **mihomoMRS**：mihomo MRS 数据格式（`geoip-cn.mrs`） - **singboxSRS**：sing-box SRS 数据格式（`geoip-cn.srs`） - **clashRuleSetClassical**：[classical 类型的 Clash RuleSet](https://github.com/Dreamacro/clash/wiki/premium-core-features#classical) @@ -527,6 +529,7 @@ All available input formats: - clashRuleSet (Convert ipcidr type of Clash RuleSet to other formats) - clashRuleSetClassical (Convert classical type of Clash RuleSet to other formats (just processing IP & CIDR lines)) - cutter (Remove data from previous steps) + - dbipCountryMMDB (Convert DB-IP country mmdb database to other formats) - json (Convert JSON data to other formats) - maxmindGeoLite2ASNCSV (Convert MaxMind GeoLite2 ASN CSV data to other formats) - maxmindGeoLite2CountryCSV (Convert MaxMind GeoLite2 country CSV data to other formats) @@ -543,6 +546,7 @@ All available input formats: All available output formats: - clashRuleSet (Convert data to ipcidr type of Clash RuleSet) - clashRuleSetClassical (Convert data to classical type of Clash RuleSet) + - dbipCountryMMDB (Convert data to DB-IP country mmdb database format) - lookup (Lookup specified IP or CIDR from various formats of data) - maxmindMMDB (Convert data to MaxMind mmdb database format) - mihomoMRS (Convert data to mihomo MRS format) diff --git a/configuration.md b/configuration.md index a0e5c3d5860..0fab7956221 100644 --- a/configuration.md +++ b/configuration.md @@ -18,6 +18,7 @@ - **clashRuleSet**：ipcidr 类型的 Clash RuleSet - **clashRuleSetClassical**：classical 类型的 Clash RuleSet - **cutter**：用于裁剪前置步骤中的数据 +- **dbipCountryMMDB**：DB-IP country mmdb 数据格式（`dbip-country-lite.mmdb`） - **json**：JSON 数据格式 - **maxmindGeoLite2ASNCSV**：MaxMind GeoLite2 ASN CSV 数据格式（`GeoLite2-ASN-CSV.zip`） - **maxmindGeoLite2CountryCSV**：MaxMind GeoLite2 country CSV 数据格式（`GeoLite2-Country-CSV.zip`） @@ -34,6 +35,7 @@ - **clashRuleSet**：ipcidr 类型的 Clash RuleSet - **clashRuleSetClassical**：classical 类型的 Clash RuleSet +- **dbipCountryMMDB**：DB-IP country mmdb 数据格式（`dbip-country-lite.mmdb`） - **lookup**：从指定的列表中查找指定的 IP 或 CIDR - **maxmindMMDB**：MaxMind GeoLite2 country mmdb 数据格式（`GeoLite2-Country.mmdb`） - **mihomoMRS**：mihomo MRS 数据格式（`geoip-cn.mrs`） @@ -166,6 +168,58 @@ } ``` +### **dbipCountryMMDB** + +- **type**：（必须）输入格式的名称 +- **action**：（必须）操作类型，值为 `add`（添加 IP 地址）或 `remove`（移除 IP 地址） +- **args**：（可选） + - **uri**：（可选）DB-IP country MMDB 格式文件路径，可为本地文件路径或远程 `http`、`https` 文件 URL。 + - **wantedList**：（可选）指定需要的类别/文件。 + - **onlyIPType**：（可选）只处理的 IP 地址类型，值为 `ipv4` 或 `ipv6`。 + +```jsonc +// 默认使用文件： +// ./db-ip/dbip-country-lite.mmdb +{ + "type": "dbipCountryMMDB", + "action": "add" // 添加 IP 地址 +} +``` + +```jsonc +{ + "type": "dbipCountryMMDB", + "action": "add", // 添加 IP 地址 + "args": { + "uri": "./db-ip/dbip-country-lite.mmdb" + } +} +``` + +```jsonc +{ + "type": "dbipCountryMMDB", + "action": "add", // 添加 IP 地址 + "args": { + "uri": "https://example.com/my.mmdb", + "wantedList": ["cn", "us", "jp"], // 只需要名为 cn、us、jp 的类别 + "onlyIPType": "ipv4" // 只添加 IPv4 地址 + } +} +``` + +```jsonc +{ + "type": "dbipCountryMMDB", + "action": "remove", // 添加 IP 地址 + "args": { + "uri": "https://example.com/my.mmdb", + "wantedList": ["cn", "us", "jp"], // 只移除名为 cn、us、jp 这三个类别的 IPv4 地址 + "onlyIPType": "ipv4" // 只移除 IPv4 地址 + } +} +``` + ### **json** - **type**：（必须）输入格式的名称 @@ -854,6 +908,103 @@ } ``` +### **dbipCountryMMDB** + +- **type**：（必须）输入格式的名称 +- **action**：（必须）操作类型，值必须为 `output` +- **args**：（可选） + - **outputName**：（可选）输出的文件名 + - **outputDir**：（可选）输出目录 + - **onlyIPType**：（可选）输出的 IP 地址类型，值为 `ipv4` 或 `ipv6` + - **wantedList**：（可选，数组）指定需要输出的类别 + - **excludedList**：（可选，数组）指定不需要输出的类别 + - **overwriteList**：（可选，数组）指定最后写入的类别（原因见👇） + +> 由于 DB-IP mmdb 文件格式的限制，当不同列表的 IP 或 CIDR 数据有交集或重复项时，后写入的列表的 IP 或 CIDR 数据会覆盖（overwrite）之前已写入的列表的数据。譬如，IP `1.1.1.1` 同属于列表 `AU` 和列表 `Cloudflare`。如果 `Cloudflare` 在 `AU` 之后写入，则 IP `1.1.1.1` 最终归属于列表 `Cloudflare`。 +> +> 为了确保某些指定的列表、被修改的列表一定囊括属于它的所有 IP 或 CIDR 数据，可在 output 输出格式为 `dbipCountryMMDB` 的配置中增加选项 `overwriteList`，该选项中指定的列表会在最后逐一写入，列表中最后一项优先级最高。若已设置选项 `wantedList`，则无需设置 `overwriteList`。`wantedList` 中指定的列表会在最后逐一写入，列表中最后一项优先级最高。 +> +> `wantedList`、`overwriteList`、`excludedList` 三者中，`excludedList` 优先级最高。即：若设置了选项 `excludedList`，最终不会输出存在于 `excludedList` 中的列表。 + +```jsonc +// 默认输出目录 ./output/db-ip +{ + "type": "dbipCountryMMDB", + "action": "output" +} +``` + +```jsonc +{ + "type": "dbipCountryMMDB", + "action": "output", + "args": { + "outputDir": "./output", // 输出文件到 output 目录 + "outputName": "Country-only-cn-private.mmdb", // 输出文件名为 Country-only-cn-private.mmdb + "wantedList": ["cn", "private"] // 只输出 cn、private 类别 + } +} +``` + +```jsonc +{ + "type": "dbipCountryMMDB", + "action": "output", + "args": { + "outputDir": "./output", // 输出文件到 output 目录 + "outputName": "Country-without-cn-private.mmdb", // 输出文件名为 Country-without-cn-private.mmdb + "excludedList": ["cn", "private"] // 不输出 cn、private 类别 + } +} +``` + +```jsonc +{ + "type": "dbipCountryMMDB", + "action": "output", + "args": { + "outputName": "Country.mmdb", // 输出文件名为 Country.mmdb + "overwriteList": ["cn", "google"] // 确保 cn、google 类别最后写入，且 google 比 cn 后写入 + } +} +``` + +```jsonc +{ + "type": "dbipCountryMMDB", + "action": "output", + "args": { + "outputName": "Country.mmdb", // 输出文件名为 Country.mmdb + "overwriteList": ["cn", "google"], // 确保 cn、google 类别最后写入，且 google 比 cn 后写入 + "onlyIPType": "ipv4" // 只输出 cn、private 类别的 IPv4 地址 + } +} +``` + +```jsonc +{ + "type": "dbipCountryMMDB", + "action": "output", + "args": { + "outputName": "Country.mmdb", // 输出文件名为 Country.mmdb + "excludedList": ["private"], // 最终不输出 private 类别 + "wantedList": ["private" ,"au", "cloudflare"] // 只输出 au、cloudflare 类别，并确保 cloudflare 比 au 后写入。但由于 private 存在于 excludedList 中，最终不输出 private 类别 + } +} +``` + +```jsonc +{ + "type": "dbipCountryMMDB", + "action": "output", + "args": { + "outputName": "Country.mmdb", // 输出文件名为 Country.mmdb + "excludedList": ["private"], // 最终不输出 private 类别 + "overwriteList": ["private" ,"cn", "google"] // 确保 cn、google 类别最后写入，且 google 比 cn 后写入。但由于 private 存在于 excludedList 中，最终不输出 private 类别 + } +} +``` + ### **lookup** - **type**：（必须）输入格式的名称 diff --git a/lookup.go b/lookup.go index 503eeed97ed..47ef55edde5 100644 --- a/lookup.go +++ b/lookup.go @@ -161,7 +161,7 @@ func getInputForLookup(format, name, uri, dir string) lib.InputConverter { switch strings.ToLower(format) { case strings.ToLower(maxmind.TypeMaxmindMMDBIn): - input = &maxmind.MaxmindMMDBIn{ + input = &maxmind.MMDBIn{ Type: maxmind.TypeMaxmindMMDBIn, Action: lib.ActionAdd, Description: maxmind.DescMaxmindMMDBIn, diff --git a/plugin/maxmind/common_in.go b/plugin/maxmind/common_in.go new file mode 100644 index 00000000000..e2a95d7052d --- /dev/null +++ b/plugin/maxmind/common_in.go @@ -0,0 +1,55 @@ +package maxmind + +import ( + "encoding/json" + "path/filepath" + "strings" + + "github.com/Loyalsoldier/geoip/lib" +) + +var ( + defaultGeoLite2MMDBFile = filepath.Join("./", "geolite2", "GeoLite2-Country.mmdb") + defaultDBIPCountryMMDBFile = filepath.Join("./", "db-ip", "dbip-country-lite.mmdb") +) + +func newMMDBIn(iType string, iDesc string, action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + var tmp struct { + URI string `json:"uri"` + Want []string `json:"wantedList"` + OnlyIPType lib.IPType `json:"onlyIPType"` + } + + if len(data) > 0 { + if err := json.Unmarshal(data, &tmp); err != nil { + return nil, err + } + } + + if tmp.URI == "" { + switch iType { + case TypeMaxmindMMDBIn: + tmp.URI = defaultGeoLite2MMDBFile + + case TypeDBIPCountryMMDBIn: + tmp.URI = defaultDBIPCountryMMDBFile + } + } + + // Filter want list + wantList := make(map[string]bool) + for _, want := range tmp.Want { + if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { + wantList[want] = true + } + } + + return &MMDBIn{ + Type: iType, + Action: action, + Description: iDesc, + URI: tmp.URI, + Want: wantList, + OnlyIPType: tmp.OnlyIPType, + }, nil +} diff --git a/plugin/maxmind/common_out.go b/plugin/maxmind/common_out.go new file mode 100644 index 00000000000..9874f58a01e --- /dev/null +++ b/plugin/maxmind/common_out.go @@ -0,0 +1,57 @@ +package maxmind + +import ( + "encoding/json" + "path/filepath" + + "github.com/Loyalsoldier/geoip/lib" +) + +var ( + defaultOutputName = "Country.mmdb" + defaultMaxmindOutputDir = filepath.Join("./", "output", "maxmind") + defaultDBIPOutputDir = filepath.Join("./", "output", "db-ip") +) + +func newMMDBOut(iType string, iDesc string, action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { + var tmp struct { + OutputName string `json:"outputName"` + OutputDir string `json:"outputDir"` + Want []string `json:"wantedList"` + Overwrite []string `json:"overwriteList"` + Exclude []string `json:"excludedList"` + OnlyIPType lib.IPType `json:"onlyIPType"` + } + + if len(data) > 0 { + if err := json.Unmarshal(data, &tmp); err != nil { + return nil, err + } + } + + if tmp.OutputName == "" { + tmp.OutputName = defaultOutputName + } + + if tmp.OutputDir == "" { + switch iType { + case TypeMaxmindMMDBOut: + tmp.OutputDir = defaultMaxmindOutputDir + + case TypeDBIPCountryMMDBOut: + tmp.OutputDir = defaultDBIPOutputDir + } + } + + return &MMDBOut{ + Type: iType, + Action: action, + Description: iDesc, + OutputName: tmp.OutputName, + OutputDir: tmp.OutputDir, + Want: tmp.Want, + Overwrite: tmp.Overwrite, + Exclude: tmp.Exclude, + OnlyIPType: tmp.OnlyIPType, + }, nil +} diff --git a/plugin/maxmind/dbip_mmdb_in.go b/plugin/maxmind/dbip_mmdb_in.go new file mode 100644 index 00000000000..b23e753c50f --- /dev/null +++ b/plugin/maxmind/dbip_mmdb_in.go @@ -0,0 +1,26 @@ +package maxmind + +import ( + "encoding/json" + + "github.com/Loyalsoldier/geoip/lib" +) + +/* +The types in this file extend the type `typeMaxmindMMDBIn`, +which make it possible to support more formats for the project. +*/ + +const ( + TypeDBIPCountryMMDBIn = "dbipCountryMMDB" + DescDBIPCountryMMDBIn = "Convert DB-IP country mmdb database to other formats" +) + +func init() { + lib.RegisterInputConfigCreator(TypeDBIPCountryMMDBIn, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { + return newMMDBIn(TypeDBIPCountryMMDBIn, DescDBIPCountryMMDBIn, action, data) + }) + lib.RegisterInputConverter(TypeDBIPCountryMMDBIn, &MMDBIn{ + Description: DescDBIPCountryMMDBIn, + }) +} diff --git a/plugin/maxmind/dbip_mmdb_out.go b/plugin/maxmind/dbip_mmdb_out.go new file mode 100644 index 00000000000..e60a3adf130 --- /dev/null +++ b/plugin/maxmind/dbip_mmdb_out.go @@ -0,0 +1,26 @@ +package maxmind + +import ( + "encoding/json" + + "github.com/Loyalsoldier/geoip/lib" +) + +/* +The types in this file extend the type `typeMaxmindMMDBOut`, +which make it possible to support more formats for the project. +*/ + +const ( + TypeDBIPCountryMMDBOut = "dbipCountryMMDB" + DescDBIPCountryMMDBOut = "Convert data to DB-IP country mmdb database format" +) + +func init() { + lib.RegisterOutputConfigCreator(TypeDBIPCountryMMDBOut, func(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { + return newMMDBOut(TypeDBIPCountryMMDBOut, DescDBIPCountryMMDBOut, action, data) + }) + lib.RegisterOutputConverter(TypeDBIPCountryMMDBOut, &MMDBOut{ + Description: DescDBIPCountryMMDBOut, + }) +} diff --git a/plugin/maxmind/mmdb_in.go b/plugin/maxmind/mmdb_in.go index 0a539dd15ed..0aab3917dbe 100644 --- a/plugin/maxmind/mmdb_in.go +++ b/plugin/maxmind/mmdb_in.go @@ -4,10 +4,10 @@ import ( "encoding/json" "fmt" "os" - "path/filepath" "strings" "github.com/Loyalsoldier/geoip/lib" + "github.com/oschwald/geoip2-golang" "github.com/oschwald/maxminddb-golang" ) @@ -16,55 +16,16 @@ const ( DescMaxmindMMDBIn = "Convert MaxMind mmdb database to other formats" ) -var ( - defaultMMDBFile = filepath.Join("./", "geolite2", "GeoLite2-Country.mmdb") -) - func init() { lib.RegisterInputConfigCreator(TypeMaxmindMMDBIn, func(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - return newMaxmindMMDBIn(action, data) + return newMMDBIn(TypeMaxmindMMDBIn, DescMaxmindMMDBIn, action, data) }) - lib.RegisterInputConverter(TypeMaxmindMMDBIn, &MaxmindMMDBIn{ + lib.RegisterInputConverter(TypeMaxmindMMDBIn, &MMDBIn{ Description: DescMaxmindMMDBIn, }) } -func newMaxmindMMDBIn(action lib.Action, data json.RawMessage) (lib.InputConverter, error) { - var tmp struct { - URI string `json:"uri"` - Want []string `json:"wantedList"` - OnlyIPType lib.IPType `json:"onlyIPType"` - } - - if len(data) > 0 { - if err := json.Unmarshal(data, &tmp); err != nil { - return nil, err - } - } - - if tmp.URI == "" { - tmp.URI = defaultMMDBFile - } - - // Filter want list - wantList := make(map[string]bool) - for _, want := range tmp.Want { - if want = strings.ToUpper(strings.TrimSpace(want)); want != "" { - wantList[want] = true - } - } - - return &MaxmindMMDBIn{ - Type: TypeMaxmindMMDBIn, - Action: action, - Description: DescMaxmindMMDBIn, - URI: tmp.URI, - Want: wantList, - OnlyIPType: tmp.OnlyIPType, - }, nil -} - -type MaxmindMMDBIn struct { +type MMDBIn struct { Type string Action lib.Action Description string @@ -73,19 +34,19 @@ type MaxmindMMDBIn struct { OnlyIPType lib.IPType } -func (m *MaxmindMMDBIn) GetType() string { +func (m *MMDBIn) GetType() string { return m.Type } -func (m *MaxmindMMDBIn) GetAction() lib.Action { +func (m *MMDBIn) GetAction() lib.Action { return m.Action } -func (m *MaxmindMMDBIn) GetDescription() string { +func (m *MMDBIn) GetDescription() string { return m.Description } -func (m *MaxmindMMDBIn) Input(container lib.Container) (lib.Container, error) { +func (m *MMDBIn) Input(container lib.Container) (lib.Container, error) { var content []byte var err error switch { @@ -134,7 +95,7 @@ func (m *MaxmindMMDBIn) Input(container lib.Container) (lib.Container, error) { return container, nil } -func (m *MaxmindMMDBIn) generateEntries(content []byte, entries map[string]*lib.Entry) error { +func (m *MMDBIn) generateEntries(content []byte, entries map[string]*lib.Entry) error { db, err := maxminddb.FromBytes(content) if err != nil { return err diff --git a/plugin/maxmind/mmdb_out.go b/plugin/maxmind/mmdb_out.go index 08e0c04c91c..cc0e4d7928a 100644 --- a/plugin/maxmind/mmdb_out.go +++ b/plugin/maxmind/mmdb_out.go @@ -19,57 +19,15 @@ const ( DescMaxmindMMDBOut = "Convert data to MaxMind mmdb database format" ) -var ( - defaultOutputName = "Country.mmdb" - defaultOutputDir = filepath.Join("./", "output", "maxmind") -) - func init() { lib.RegisterOutputConfigCreator(TypeMaxmindMMDBOut, func(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { - return newMMDBOut(action, data) + return newMMDBOut(TypeMaxmindMMDBOut, DescMaxmindMMDBOut, action, data) }) lib.RegisterOutputConverter(TypeMaxmindMMDBOut, &MMDBOut{ Description: DescMaxmindMMDBOut, }) } -func newMMDBOut(action lib.Action, data json.RawMessage) (lib.OutputConverter, error) { - var tmp struct { - OutputName string `json:"outputName"` - OutputDir string `json:"outputDir"` - Want []string `json:"wantedList"` - Overwrite []string `json:"overwriteList"` - Exclude []string `json:"excludedList"` - OnlyIPType lib.IPType `json:"onlyIPType"` - } - - if len(data) > 0 { - if err := json.Unmarshal(data, &tmp); err != nil { - return nil, err - } - } - - if tmp.OutputName == "" { - tmp.OutputName = defaultOutputName - } - - if tmp.OutputDir == "" { - tmp.OutputDir = defaultOutputDir - } - - return &MMDBOut{ - Type: TypeMaxmindMMDBOut, - Action: action, - Description: DescMaxmindMMDBOut, - OutputName: tmp.OutputName, - OutputDir: tmp.OutputDir, - Want: tmp.Want, - Overwrite: tmp.Overwrite, - Exclude: tmp.Exclude, - OnlyIPType: tmp.OnlyIPType, - }, nil -} - type MMDBOut struct { Type string Action lib.Action @@ -95,10 +53,23 @@ func (m *MMDBOut) GetDescription() string { } func (m *MMDBOut) Output(container lib.Container) error { + dbName := "" + dbDesc := "" + + switch m.Type { + case TypeMaxmindMMDBOut: + dbName = "GeoLite2-Country" + dbDesc = "Customized GeoLite2 Country database" + + case TypeDBIPCountryMMDBOut: + dbName = "DBIP-Country-Lite" + dbDesc = "Customized DB-IP Country Lite database" + } + writer, err := mmdbwriter.New( mmdbwriter.Options{ - DatabaseType: "GeoLite2-Country", - Description: map[string]string{"en": "Customized GeoLite2 Country database"}, + DatabaseType: dbName, + Description: map[string]string{"en": dbDesc}, RecordSize: 24, IncludeReservedNetworks: true, },