Fix ipset creation if there were no IPs in the lists; Add experimenta…

…l IPv6 support

Makefile

@@ -1,5 +1,5 @@
-PKG_VERSION:=1.0.0
-PKG_RELEASE:=7
+PKG_VERSION:=1.0.1
+PKG_RELEASE:=1
 PKG_FULLVERSION:=$(PKG_VERSION)-$(PKG_RELEASE)
 
 BINARY_NAME=keenetic-pbr

keenetic-pbr.example.conf

@@ -7,9 +7,10 @@ summarize = true
 [[ipset]]
 ipset_name = "vpn"
 flush_before_applying = true
+ip_version = 4
 
     [ipset.routing]
-    interface = "nwg1"
+    interface = "nwg0"
     fwmark = 1001
     table = 1001
     priority = 1001

lib/config.go

@@ -23,6 +23,7 @@ type GeneralConfig struct {
 
 type IpsetConfig struct {
 	IpsetName           string        `toml:"ipset_name"`
+	IpVersion           uint8         `toml:"ip_version"`
 	Routing             RoutingConfig `toml:"routing"`
 	FlushBeforeApplying bool          `toml:"flush_before_applying"`
 	List                []ListSource  `toml:"list"`
@@ -84,6 +85,13 @@ func (c *Config) validateConfig() error {
 		}
 		names[ipset.IpsetName] = true
 
+		if ipset.IpVersion != 6 {
+			if ipset.IpVersion != 4 && ipset.IpVersion != 0 {
+				return fmt.Errorf("unknown IP version %d, check your configuration", ipset.IpVersion)
+			}
+			ipset.IpVersion = 4
+		}
+
 		if ipset.Routing.Interface == "" {
 			return fmt.Errorf("interface cannot be empty, check your configuration")
 		}
@@ -124,19 +132,11 @@ func (c *Config) validateConfig() error {
 	return nil
 }
 
-func validateUnique(items []interface{}) error {
-	seen := make(map[interface{}]bool)
-	for _, item := range items {
-		if seen[item] {
-			return fmt.Errorf("duplicate item found: %v", item)
-		}
-		seen[item] = true
-	}
-	return nil
-}
-
-func GenRoutingConfig(c *Config) error {
+func GenRoutingConfig(c *Config, ipFamily uint8) error {
 	for _, ipset := range c.Ipset {
+		if ipset.IpVersion != ipFamily {
+			continue
+		}
 		fmt.Printf("%s %s %d %d %d\n", ipset.IpsetName, ipset.Routing.Interface, ipset.Routing.FwMark, ipset.Routing.IpRouteTable, ipset.Routing.IpRulePriority)
 	}
 	return nil

lib/ipset.go

@@ -2,33 +2,61 @@ package lib
 
 import (
 	"fmt"
+	"log"
 	"os/exec"
 )
 
-type IpsetManager struct{}
+// CreateIpset creates a new ipset with the given name and IP family (4 or 6)
+func CreateIpset(ipsetCommand string, ipset IpsetConfig) error {
+	// Determine IP family
+	family := "inet"
+	if ipset.IpVersion == 6 {
+		family = "inet6"
+	} else if ipset.IpVersion != 0 && ipset.IpVersion != 4 {
+		log.Printf("unknown IP version %d, assuming IPv4", ipset.IpVersion)
+	}
+
+	cmd := exec.Command(ipsetCommand, "create", ipset.IpsetName, "hash:net", "family", family, "-exist")
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("failed to create ipset %s (IPv%d): %v", ipset.IpsetName, ipset.IpVersion, err)
+	}
 
-func (im *IpsetManager) AddToIpset(ipsetCommand string, ipset IpsetConfig, networks []string) error {
+	return nil
+}
+
+// AddToIpset adds the given networks to the specified ipset
+func AddToIpset(ipsetCommand string, ipset IpsetConfig, networks []string) error {
 	cmd := exec.Command(ipsetCommand, "restore", "-exist")
 	stdin, err := cmd.StdinPipe()
 	if err != nil {
-		return fmt.Errorf("Failed to get stdin pipe: %v", err)
+		return fmt.Errorf("failed to get stdin pipe: %v", err)
 	}
 
 	go func() {
 		defer stdin.Close()
-		fmt.Fprintf(stdin, "create %s hash:net family inet\n", ipset.IpsetName)
-
+		// Write commands to stdin
 		if ipset.FlushBeforeApplying {
-			fmt.Fprintf(stdin, "flush %s\n", ipset.IpsetName)
+			if _, err := fmt.Fprintf(stdin, "flush %s\n", ipset.IpsetName); err != nil {
+				log.Printf("failed to flush ipset %s: %v", ipset.IpsetName, err)
+			}
 		}
 
+		errorCounter := 0
 		for _, network := range networks {
-			fmt.Fprintf(stdin, "add %s %s\n", ipset.IpsetName, network)
+			if _, err := fmt.Fprintf(stdin, "add %s %s\n", ipset.IpsetName, network); err != nil {
+				log.Printf("failed to add address %s to ipset %s: %v", network, ipset.IpsetName, err)
+				errorCounter++
+
+				if errorCounter > 10 {
+					log.Printf("too many errors, aborting import")
+					return
+				}
+			}
 		}
 	}()
 
 	if output, err := cmd.CombinedOutput(); err != nil {
-		return fmt.Errorf("Failed to add addresses to ipset %s: %v\n%s", ipset.IpsetName, err, output)
+		return fmt.Errorf("failed to add addresses to ipset %s: %v\n%s", ipset.IpsetName, err, output)
 	}
 
 	return nil

lib/router.go

@@ -4,10 +4,10 @@ import (
 	"bufio"
 	"fmt"
 	"log"
-	"net"
 	"os"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"strings"
 )
 
@@ -42,18 +42,18 @@ func ApplyLists(config *Config) error {
 		}
 	}
 
-	ipsetManager := &IpsetManager{}
-
 	for _, ipset := range config.Ipset {
 		var ipv4Networks []string
+		var ipv6Networks []string
 		var domains []string
 
 		// Process lists
 		for _, list := range ipset.List {
 			// Read and process list file
 			content, err := os.ReadFile(filepath.Join(listsDir, fmt.Sprintf("%s-%s.lst", ipset.IpsetName, list.ListName)))
 			if err != nil {
-				return fmt.Errorf("failed to read list file %s-%s: %v\n\nplease, run keenetic-pbr download", err)
+				log.Printf("failed to read list file %s-%s: %v\n\nplease, run \"keenetic-pbr download\"", err)
+				continue
 			}
 
 			scanner := bufio.NewScanner(strings.NewReader(string(content)))
@@ -65,27 +65,46 @@ func ApplyLists(config *Config) error {
 
 				if isDNSName(line) {
 					domains = append(domains, line)
-				} else if isIPv4(line) || IsCIDR(line) {
-					ipv4Networks = append(ipv4Networks, line)
+				} else if isIP(line) || isCIDR(line) {
+					// ipv4 contain dots, ipv6 contain colons
+					if strings.Contains(line, ".") {
+						ipv4Networks = append(ipv4Networks, line)
+					} else {
+						ipv6Networks = append(ipv6Networks, line)
+					}
 				}
 			}
 		}
 
-		if len(ipv4Networks) > 0 {
+		err := CreateIpset(config.General.IpsetPath, ipset)
+		if err != nil {
+			log.Printf("Could not create ipset '%s': %v", ipset.IpsetName, err)
+		}
+
+		// Filling ipv4 ipset
+		if ipset.IpVersion != 6 && len(ipv4Networks) > 0 {
 			// Summarize networks if requested
 			var ipsLen = len(ipv4Networks)
 			if config.General.Summarize {
-				ipv4Networks = NetworkSummarizer{}.SummarizeIPv4(ipv4Networks)
+				ipv4Networks = SummarizeIPv4(ipv4Networks)
 			}
 
 			// Apply networks to ipsets
 			if config.General.Summarize {
-				log.Printf("Filling ipset '%s' (%d items, %d after summarization)...", ipset.IpsetName, ipsLen, len(ipv4Networks))
+				log.Printf("Filling ipset '%s' (IPv4) (%d items, %d after summarization)...", ipset.IpsetName, ipsLen, len(ipv4Networks))
 			} else {
-				log.Printf("Filling ipset '%s' (%d items)...", ipset.IpsetName, ipsLen)
+				log.Printf("Filling ipset '%s' (IPv4) (%d items)...", ipset.IpsetName, ipsLen)
 			}
-			if err := ipsetManager.AddToIpset(config.General.IpsetPath, ipset, ipv4Networks); err != nil {
-				return err
+			if err := AddToIpset(config.General.IpsetPath, ipset, ipv4Networks); err != nil {
+				log.Printf("Could not fill ipset (IPv4) '%s': %v", ipset.IpsetName, err)
+			}
+		}
+
+		if ipset.IpVersion == 6 && len(ipv6Networks) > 0 {
+			// Apply networks to ipsets
+			log.Printf("Filling ipset '%s' (IPv6) (%d items)...", ipset.IpsetName, len(ipv6Networks))
+			if err := AddToIpset(config.General.IpsetPath, ipset, ipv6Networks); err != nil {
+				log.Printf("Could not fill ipset (IPv6) '%s': %v", ipset.IpsetName, err)
 			}
 		}
 
@@ -99,7 +118,8 @@ func ApplyLists(config *Config) error {
 			}
 			defer f.Close()
 
-			domains = removeDuplicateStr(domains)
+			slices.Sort(domains)
+			domains = slices.Compact(domains)
 
 			writer := bufio.NewWriter(f)
 			for _, domain := range domains {
@@ -114,46 +134,3 @@ func ApplyLists(config *Config) error {
 	log.Print("Configuration applied successfully")
 	return nil
 }
-
-// isDNSName will validate the given string as a DNS name
-func isDNSName(str string) bool {
-	if str == "" || len(strings.Replace(str, ".", "", -1)) > 255 {
-		// constraints already violated
-		return false
-	}
-	return !isIP(str) && rxDNSName.MatchString(str)
-}
-
-func isIP(str string) bool {
-	return net.ParseIP(str) != nil
-}
-
-// isIPv4 checks if the string is an IP version 4.
-func isIPv4(str string) bool {
-	ip := net.ParseIP(str)
-	return ip != nil && strings.Contains(str, ".")
-}
-
-// isIPv6 checks if the string is an IP version 6.
-func isIPv6(str string) bool {
-	ip := net.ParseIP(str)
-	return ip != nil && strings.Contains(str, ":")
-}
-
-// IsCIDR checks if the string is an valid CIDR notiation (IPV4 & IPV6)
-func IsCIDR(str string) bool {
-	_, _, err := net.ParseCIDR(str)
-	return err == nil
-}
-
-func removeDuplicateStr(strSlice []string) []string {
-	allKeys := make(map[string]bool)
-	list := []string{}
-	for _, item := range strSlice {
-		if _, value := allKeys[item]; !value {
-			allKeys[item] = true
-			list = append(list, item)
-		}
-	}
-	return list
-}

lib/summarizer.go

@@ -5,9 +5,7 @@ import (
 	"sort"
 )
 
-type NetworkSummarizer struct{}
-
-func (ns NetworkSummarizer) SummarizeIPv4(networks []string) []string {
+func SummarizeIPv4(networks []string) []string {
 	if len(networks) == 0 {
 		return nil
 	}

lib/validator.go

@@ -0,0 +1,51 @@
+/*
+The MIT License (MIT)
+
+Copyright (c) 2014-2020 Alex Saskevich
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+Credits: https://github.com/asaskevich/govalidator
+*/
+
+package lib
+
+import (
+	"net"
+	"strings"
+)
+
+// isDNSName will validate the given string as a DNS name
+func isDNSName(str string) bool {
+	if str == "" || len(strings.Replace(str, ".", "", -1)) > 255 {
+		// constraints already violated
+		return false
+	}
+	return !isIP(str) && rxDNSName.MatchString(str)
+}
+
+func isIP(str string) bool {
+	return net.ParseIP(str) != nil
+}
+
+// isCIDR checks if the string is an valid CIDR notiation (IPV4 & IPV6)
+func isCIDR(str string) bool {
+	_, _, err := net.ParseCIDR(str)
+	return err == nil
+}

main.go

@@ -27,6 +27,7 @@ type Config struct {
 type CLI struct {
 	configPath string
 	command    Command
+	ipFamily   uint8
 }
 
 func parseFlags() *CLI {
@@ -42,7 +43,8 @@ func parseFlags() *CLI {
 		fmt.Fprintf(os.Stderr, "Commands:\n")
 		fmt.Fprintf(os.Stderr, "  download                Download lists\n")
 		fmt.Fprintf(os.Stderr, "  apply                   Import lists to ipset and update dnsmasq lists\n")
-		fmt.Fprintf(os.Stderr, "  gen-routing-config      Gen configuration for routing scripts (ipset, iface_name, fwmark, table, priority)\n\n")
+		fmt.Fprintf(os.Stderr, "  gen-routing-config      Gen IPv4 configuration for routing scripts (ipset, iface_name, fwmark, table, priority)\n\n")
+		fmt.Fprintf(os.Stderr, "  gen-routing-config-ipv6 Gen IPv6 configuration for routing scripts (ipset, iface_name, fwmark, table, priority)\n\n")
 		fmt.Fprintf(os.Stderr, "Options:\n")
 		flag.PrintDefaults()
 	}
@@ -63,6 +65,10 @@ func parseFlags() *CLI {
 		cli.command = Apply
 	case "gen-routing-config":
 		cli.command = GenRoutingConfig
+		cli.ipFamily = 4
+	case "gen-routing-config-ipv6":
+		cli.command = GenRoutingConfig
+		cli.ipFamily = 6
 	default:
 		fmt.Fprintf(os.Stderr, "Unknown command: %s\n", args[0])
 		flag.Usage()
@@ -106,7 +112,7 @@ func main() {
 			log.Fatalf("Failed to apply configuration: %v", err)
 		}
 	case GenRoutingConfig:
-		if err := lib.GenRoutingConfig(config); err != nil {
+		if err := lib.GenRoutingConfig(config, cli.ipFamily); err != nil {
 			log.Fatalf("Failed to apply configuration: %v", err)
 		}
 	}