first commit

.gitignore

@@ -0,0 +1,4 @@
+test
+AppData
+*.exe
+WebWanderOff

Chatgpt Next.yaml

@@ -0,0 +1,5 @@
+name: ChatGPT Next
+listen_addr: "127.0.0.1:8101"
+default_server: "chat-gpt.chinatcc.com"
+default_scheme: "https"
+cache_root: "AppData"

README.md

@@ -0,0 +1,3 @@
+# WanderOff
+
+This program caches various static resources so that web apps can be used offline. Especially for web programs that require downloading large resources such as wasm and webgpu. 

core.go

@@ -0,0 +1,164 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"mime"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/gabriel-vasile/mimetype"
+)
+
+type CacheSystem struct {
+	ListenAddr       string
+	ThirdPartyPrefix string
+	DefaultServer    string
+	DefaultScheme    string
+	CacheRoot        string
+	OfflineDomains   []string
+}
+
+func (c *CacheSystem) Listen() {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", c.cacheProxyHandler)
+	log.Println("Listening on:", c.ListenAddr)
+	log.Fatal(http.ListenAndServe(c.ListenAddr, mux))
+}
+
+func (c *CacheSystem) cacheProxyHandler(w http.ResponseWriter, r *http.Request) {
+	// Remove '/' from the request URI to get the actual URL
+	targetURL := r.RequestURI[1:]
+	cachePath := ""
+	if strings.HasPrefix(targetURL, "https/") {
+		cachePath = c.CacheRoot + "/" + strings.Replace(targetURL, "https/", "", 1)
+		targetURL = strings.Replace(targetURL, "https/", "https://", 1)
+	} else if strings.HasPrefix(targetURL, "http/") {
+		cachePath = c.CacheRoot + "/" + strings.Replace(targetURL, "https/", "", 1)
+		targetURL = strings.Replace(targetURL, "http/", "http://", 1)
+	} else {
+		cachePath = c.CacheRoot + "/" + c.DefaultServer + "/" + targetURL
+		targetURL = c.DefaultScheme + "://" + c.DefaultServer + "/" + targetURL
+	}
+
+	// Determine the cache path
+
+	if strings.HasSuffix(cachePath, "/") {
+		cachePath = cachePath + "index"
+	}
+	log.Println("try cache:", cachePath)
+	cacheDir := path.Dir(cachePath)
+
+	// Check if the resource is already cached
+	if _, err := os.Stat(cachePath); err == nil {
+		// Serve the resource from the cache
+		log.Println("hit cache", cachePath)
+		c.serveFile(w, r, cachePath)
+		return
+	}
+
+	log.Println("Download static data from:", targetURL)
+	// The resource is not cached, make a request to the target URL
+	client := &http.Client{}
+	req, err := http.NewRequest(r.Method, targetURL, nil)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// Forward headers and body if method is POST
+	//req.Header = r.Header
+	if r.Method == "POST" {
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		req.Body = io.NopCloser(bytes.NewBuffer(body))
+	}
+
+	// Perform the request
+	resp, err := client.Do(req)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	defer resp.Body.Close()
+
+	isGzip := resp.Header.Get("Content-Encoding") == "gzip"
+
+	// Read the response
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// 如果经过 Gzip 压缩，则解压缩
+	if isGzip {
+		data, err = UncompressGzip(data)
+		if err != nil {
+			fmt.Println("解压缩错误:", err)
+			return
+		}
+	}
+
+	// Create the cache directory if it does not exist
+	if err := os.MkdirAll(cacheDir, os.ModePerm); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// Write the response to the cache
+	if err := os.WriteFile(cachePath, data, 0666); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	c.serveFile(w, r, cachePath)
+}
+
+func (c *CacheSystem) serveFile(w http.ResponseWriter, r *http.Request, name string) {
+	contentType := "application/octet-stream" // Default MIME type
+	// Check if the resource is already cached
+	if fileInfo, err := os.Stat(name); err == nil {
+		//print the content-type of cachePath
+		if mimeType := mime.TypeByExtension(path.Ext(fileInfo.Name())); mimeType != "" {
+			contentType = mimeType
+		} else if mimeType, err := mimetype.DetectFile(name); err == nil {
+			contentType = mimeType.String()
+		}
+
+		w.Header().Set("Content-Type", contentType)
+
+		if isTextMimeType(contentType) {
+			log.Println("It is text, replace")
+			w.Write(c.replaceUrlInText(name))
+			return
+		}
+
+		// Serve the resource from the cache
+		http.ServeFile(w, r, name)
+		return
+	}
+}
+
+func (c *CacheSystem) replaceUrlInText(file string) []byte {
+	// readAll data from file
+	// replace https://cdn.jsdelivr.net with http://127.0.0.1:8099/http/cdn.jsdelivr.net
+	//replaceRules =
+	data, err := os.ReadFile(file)
+	if err != nil {
+		log.Fatalf("unable to read file: %v", err)
+	}
+	for _, domain := range c.OfflineDomains {
+		target := fmt.Sprintf("http://%s/%s", c.ListenAddr, strings.Replace(domain, "://", "/", 1))
+		data = bytes.ReplaceAll(data, []byte(domain), []byte(target))
+	}
+
+	return data
+}

go.mod

@@ -0,0 +1,10 @@
+module github.com/linexjlin/WebWanderOff
+
+go 1.20
+
+require github.com/gabriel-vasile/mimetype v1.4.3
+
+require (
+	golang.org/x/net v0.17.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)

go.sum

@@ -0,0 +1,7 @@
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

inpaintweb.yaml

@@ -0,0 +1,8 @@
+name: inpaintweb
+listen_addr: "127.0.0.1:8099"
+default_server: "inpaintweb.lxfater.com"
+default_scheme: "https"
+cache_root: "AppData"
+offline_domains:
+  - "https://cdn.jsdelivr.net"
+  - "https://huggingface.co"

main.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/yaml.v2"
+)
+
+type Config struct {
+	Name           string   `yaml:"name"`
+	ListenAddr     string   `yaml:"listen_addr"`
+	DefaultServer  string   `yaml:"default_server"`
+	DefaultScheme  string   `yaml:"default_scheme"`
+	CacheRoot      string   `yaml:"cache_root"`
+	OfflineDomains []string `yaml:"offline_domains"`
+}
+
+func main() {
+	//set log format show line number
+	log.SetFlags(log.LstdFlags | log.Lshortfile)
+	// 获取当前目录下所有的 inpaint.yaml 配置文件名
+	files, err := filepath.Glob("*.yaml")
+	if err != nil {
+		fmt.Printf("failed to read config files: %v", err)
+		return
+	}
+
+	// 遍历每个文件，并读取配置内容
+	for _, file := range files {
+		// 读取 YAML 文件内容
+		content, err := os.ReadFile(file)
+		if err != nil {
+			fmt.Printf("failed to read file %s: %v\n", file, err)
+			continue
+		}
+
+		// 解析 YAML 文件数据到 Config 结构体实例
+		var conf Config
+		if err := yaml.Unmarshal(content, &conf); err != nil {
+			fmt.Printf("failed to unmarshal file %s: %v\n", file, err)
+			continue
+		}
+
+		// 打印该配置文件的内容
+		fmt.Printf("Configuration of %s:\n", file)
+		fmt.Printf("  Name: %s\n", conf.Name)
+		fmt.Printf("  ListenAddr: %s\n", conf.ListenAddr)
+		fmt.Printf("  DefaultServer: %s\n", conf.DefaultServer)
+		fmt.Printf("  DefaultScheme: %s\n", conf.DefaultScheme)
+		fmt.Printf("  CacheRoot: %s\n", conf.CacheRoot)
+		fmt.Printf("  OfflineDomains: %v\n", conf.OfflineDomains)
+
+		cs := CacheSystem{ListenAddr: conf.ListenAddr, DefaultServer: conf.DefaultServer, DefaultScheme: conf.DefaultScheme, CacheRoot: conf.CacheRoot, OfflineDomains: conf.OfflineDomains}
+		go cs.Listen()
+	}
+
+	select {}
+}

utils.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io/ioutil"
+	"strings"
+)
+
+// 解压缩 Gzip
+func UncompressGzip(data []byte) ([]byte, error) {
+	reader, err := gzip.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+
+	uncompressed, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	return uncompressed, nil
+}
+
+func isTextMimeType(mimeType string) bool {
+	textTypes := []string{
+		"text/",
+		"application/json",
+		"application/javascript",
+		"application/xml",
+		"application/xhtml+xml",
+	}
+
+	for _, prefix := range textTypes {
+		if strings.HasPrefix(mimeType, prefix) {
+			return true
+		}
+	}
+	return false
+}