From 1c5a6ab29c0ff372c3174251f563e2192182338a Mon Sep 17 00:00:00 2001 From: linexjlin Date: Wed, 6 Dec 2023 23:15:22 +0800 Subject: [PATCH] first commit --- .gitignore | 4 ++ Chatgpt Next.yaml | 5 ++ README.md | 3 + core.go | 164 ++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 10 +++ go.sum | 7 ++ inpaintweb.yaml | 8 +++ main.go | 61 +++++++++++++++++ utils.go | 41 ++++++++++++ 9 files changed, 303 insertions(+) create mode 100644 .gitignore create mode 100644 Chatgpt Next.yaml create mode 100644 README.md create mode 100644 core.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 inpaintweb.yaml create mode 100644 main.go create mode 100644 utils.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5c93fde --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +test +AppData +*.exe +WebWanderOff \ No newline at end of file diff --git a/Chatgpt Next.yaml b/Chatgpt Next.yaml new file mode 100644 index 0000000..8dac6b5 --- /dev/null +++ b/Chatgpt Next.yaml @@ -0,0 +1,5 @@ +name: ChatGPT Next +listen_addr: "127.0.0.1:8101" +default_server: "chat-gpt.chinatcc.com" +default_scheme: "https" +cache_root: "AppData" \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..239b67d --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# WanderOff + +This program caches various static resources so that web apps can be used offline. Especially for web programs that require downloading large resources such as wasm and webgpu. \ No newline at end of file diff --git a/core.go b/core.go new file mode 100644 index 0000000..2e10a4d --- /dev/null +++ b/core.go @@ -0,0 +1,164 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "log" + "mime" + "net/http" + "os" + "path" + "strings" + + "github.com/gabriel-vasile/mimetype" +) + +type CacheSystem struct { + ListenAddr string + ThirdPartyPrefix string + DefaultServer string + DefaultScheme string + CacheRoot string + OfflineDomains []string +} + +func (c *CacheSystem) Listen() { + mux := http.NewServeMux() + mux.HandleFunc("/", c.cacheProxyHandler) + log.Println("Listening on:", c.ListenAddr) + log.Fatal(http.ListenAndServe(c.ListenAddr, mux)) +} + +func (c *CacheSystem) cacheProxyHandler(w http.ResponseWriter, r *http.Request) { + // Remove '/' from the request URI to get the actual URL + targetURL := r.RequestURI[1:] + cachePath := "" + if strings.HasPrefix(targetURL, "https/") { + cachePath = c.CacheRoot + "/" + strings.Replace(targetURL, "https/", "", 1) + targetURL = strings.Replace(targetURL, "https/", "https://", 1) + } else if strings.HasPrefix(targetURL, "http/") { + cachePath = c.CacheRoot + "/" + strings.Replace(targetURL, "https/", "", 1) + targetURL = strings.Replace(targetURL, "http/", "http://", 1) + } else { + cachePath = c.CacheRoot + "/" + c.DefaultServer + "/" + targetURL + targetURL = c.DefaultScheme + "://" + c.DefaultServer + "/" + targetURL + } + + // Determine the cache path + + if strings.HasSuffix(cachePath, "/") { + cachePath = cachePath + "index" + } + log.Println("try cache:", cachePath) + cacheDir := path.Dir(cachePath) + + // Check if the resource is already cached + if _, err := os.Stat(cachePath); err == nil { + // Serve the resource from the cache + log.Println("hit cache", cachePath) + c.serveFile(w, r, cachePath) + return + } + + log.Println("Download static data from:", targetURL) + // The resource is not cached, make a request to the target URL + client := &http.Client{} + req, err := http.NewRequest(r.Method, targetURL, nil) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Forward headers and body if method is POST + //req.Header = r.Header + if r.Method == "POST" { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + req.Body = io.NopCloser(bytes.NewBuffer(body)) + } + + // Perform the request + resp, err := client.Do(req) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer resp.Body.Close() + + isGzip := resp.Header.Get("Content-Encoding") == "gzip" + + // Read the response + data, err := io.ReadAll(resp.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // 如果经过 Gzip 压缩，则解压缩 + if isGzip { + data, err = UncompressGzip(data) + if err != nil { + fmt.Println("解压缩错误:", err) + return + } + } + + // Create the cache directory if it does not exist + if err := os.MkdirAll(cacheDir, os.ModePerm); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Write the response to the cache + if err := os.WriteFile(cachePath, data, 0666); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + c.serveFile(w, r, cachePath) +} + +func (c *CacheSystem) serveFile(w http.ResponseWriter, r *http.Request, name string) { + contentType := "application/octet-stream" // Default MIME type + // Check if the resource is already cached + if fileInfo, err := os.Stat(name); err == nil { + //print the content-type of cachePath + if mimeType := mime.TypeByExtension(path.Ext(fileInfo.Name())); mimeType != "" { + contentType = mimeType + } else if mimeType, err := mimetype.DetectFile(name); err == nil { + contentType = mimeType.String() + } + + w.Header().Set("Content-Type", contentType) + + if isTextMimeType(contentType) { + log.Println("It is text, replace") + w.Write(c.replaceUrlInText(name)) + return + } + + // Serve the resource from the cache + http.ServeFile(w, r, name) + return + } +} + +func (c *CacheSystem) replaceUrlInText(file string) []byte { + // readAll data from file + // replace https://cdn.jsdelivr.net with http://127.0.0.1:8099/http/cdn.jsdelivr.net + //replaceRules = + data, err := os.ReadFile(file) + if err != nil { + log.Fatalf("unable to read file: %v", err) + } + for _, domain := range c.OfflineDomains { + target := fmt.Sprintf("http://%s/%s", c.ListenAddr, strings.Replace(domain, "://", "/", 1)) + data = bytes.ReplaceAll(data, []byte(domain), []byte(target)) + } + + return data +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ac90dbd --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/linexjlin/WebWanderOff + +go 1.20 + +require github.com/gabriel-vasile/mimetype v1.4.3 + +require ( + golang.org/x/net v0.17.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d5bec1a --- /dev/null +++ b/go.sum @@ -0,0 +1,7 @@ +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/inpaintweb.yaml b/inpaintweb.yaml new file mode 100644 index 0000000..c14fafe --- /dev/null +++ b/inpaintweb.yaml @@ -0,0 +1,8 @@ +name: inpaintweb +listen_addr: "127.0.0.1:8099" +default_server: "inpaintweb.lxfater.com" +default_scheme: "https" +cache_root: "AppData" +offline_domains: + - "https://cdn.jsdelivr.net" + - "https://huggingface.co" \ No newline at end of file diff --git a/main.go b/main.go new file mode 100644 index 0000000..ab5e070 --- /dev/null +++ b/main.go @@ -0,0 +1,61 @@ +package main + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "gopkg.in/yaml.v2" +) + +type Config struct { + Name string `yaml:"name"` + ListenAddr string `yaml:"listen_addr"` + DefaultServer string `yaml:"default_server"` + DefaultScheme string `yaml:"default_scheme"` + CacheRoot string `yaml:"cache_root"` + OfflineDomains []string `yaml:"offline_domains"` +} + +func main() { + //set log format show line number + log.SetFlags(log.LstdFlags | log.Lshortfile) + // 获取当前目录下所有的 inpaint.yaml 配置文件名 + files, err := filepath.Glob("*.yaml") + if err != nil { + fmt.Printf("failed to read config files: %v", err) + return + } + + // 遍历每个文件，并读取配置内容 + for _, file := range files { + // 读取 YAML 文件内容 + content, err := os.ReadFile(file) + if err != nil { + fmt.Printf("failed to read file %s: %v\n", file, err) + continue + } + + // 解析 YAML 文件数据到 Config 结构体实例 + var conf Config + if err := yaml.Unmarshal(content, &conf); err != nil { + fmt.Printf("failed to unmarshal file %s: %v\n", file, err) + continue + } + + // 打印该配置文件的内容 + fmt.Printf("Configuration of %s:\n", file) + fmt.Printf(" Name: %s\n", conf.Name) + fmt.Printf(" ListenAddr: %s\n", conf.ListenAddr) + fmt.Printf(" DefaultServer: %s\n", conf.DefaultServer) + fmt.Printf(" DefaultScheme: %s\n", conf.DefaultScheme) + fmt.Printf(" CacheRoot: %s\n", conf.CacheRoot) + fmt.Printf(" OfflineDomains: %v\n", conf.OfflineDomains) + + cs := CacheSystem{ListenAddr: conf.ListenAddr, DefaultServer: conf.DefaultServer, DefaultScheme: conf.DefaultScheme, CacheRoot: conf.CacheRoot, OfflineDomains: conf.OfflineDomains} + go cs.Listen() + } + + select {} +} diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..4a6fc17 --- /dev/null +++ b/utils.go @@ -0,0 +1,41 @@ +package main + +import ( + "bytes" + "compress/gzip" + "io/ioutil" + "strings" +) + +// 解压缩 Gzip +func UncompressGzip(data []byte) ([]byte, error) { + reader, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + defer reader.Close() + + uncompressed, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + + return uncompressed, nil +} + +func isTextMimeType(mimeType string) bool { + textTypes := []string{ + "text/", + "application/json", + "application/javascript", + "application/xml", + "application/xhtml+xml", + } + + for _, prefix := range textTypes { + if strings.HasPrefix(mimeType, prefix) { + return true + } + } + return false +}