embed dependencies

josStorer · May 29, 2023 · 6fc5a33 · 6fc5a33
1 parent d66c306
commit 6fc5a33
Show file tree

Hide file tree

Showing 8 changed files with 52 additions and 89 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,6 +11,7 @@ __pycache__
 /frontend/stats.html
 /frontend/package.json.md5
 /backend-python/get-pip.py
+/backend-python/.get-pip.py
 /py310
 *.zip
 /cmd-helper.bat

diff --git a/backend-golang/utils.go b/backend-golang/utils.go
@@ -3,8 +3,10 @@ package backend_golang
 import (
 	"archive/zip"
 	"bufio"
+	"embed"
 	"errors"
 	"io"
+	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -41,6 +43,34 @@ func Cmd(args ...string) (string, error) {
 	}
 }
 
+func CopyEmbed(efs embed.FS) error {
+	err := fs.WalkDir(efs, ".", func(path string, d fs.DirEntry, err error) error {
+		if d.IsDir() {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		content, err := efs.ReadFile(path)
+		if err != nil {
+			return err
+		}
+
+		err = os.MkdirAll(path[:strings.LastIndex(path, "/")], 0755)
+		if err != nil {
+			return err
+		}
+
+		err = os.WriteFile(path, content, 0644)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+	return err
+}
+
 func GetPython() (string, error) {
 	switch platform := runtime.GOOS; platform {
 	case "windows":

diff --git a/backend-python/rwkv_pip/20B_tokenizer.json → backend-python/rwkv_pip/.20B_tokenizer.json b/backend-python/rwkv_pip/20B_tokenizer.json → backend-python/rwkv_pip/.20B_tokenizer.json
diff --git a/...-python/rwkv_pip/rwkv_vocab_v20230424.txt → ...python/rwkv_pip/.rwkv_vocab_v20230424.txt b/...-python/rwkv_pip/rwkv_vocab_v20230424.txt → ...python/rwkv_pip/.rwkv_vocab_v20230424.txt
diff --git a/frontend/src/startup.ts b/frontend/src/startup.ts
@@ -1,26 +1,12 @@
 import commonStore from './stores/commonStore';
-import { FileExists, ReadJson } from '../wailsjs/go/backend_golang/App';
-import {
-  Cache,
-  checkUpdate,
-  deleteDynamicProgramFiles,
-  downloadProgramFiles,
-  LocalConfig,
-  refreshModels,
-  saveCache
-} from './utils';
+import { ReadJson } from '../wailsjs/go/backend_golang/App';
+import { Cache, checkUpdate, downloadProgramFiles, LocalConfig, refreshModels, saveCache } from './utils';
 import { getStatus } from './apis';
 import { EventsOn } from '../wailsjs/runtime';
 import { defaultModelConfigs } from './pages/Configs';
 
 export async function startup() {
-  FileExists('cache.json').then((exists) => {
-    if (exists)
-      downloadProgramFiles();
-    else {
-      deleteDynamicProgramFiles().then(downloadProgramFiles);
-    }
-  });
+  downloadProgramFiles();
   EventsOn('downloadList', (data) => {
     if (data)
       commonStore.setDownloadList(data);

diff --git a/frontend/src/utils/index.tsx b/frontend/src/utils/index.tsx
@@ -176,15 +176,16 @@ export function isSystemLightMode() {
 export function downloadProgramFiles() {
   manifest.programFiles.forEach(({ url, path }) => {
     FileExists(path).then(exists => {
-      if (!exists)
+      if (!exists && url)
         AddToDownloadList(path, url.replace('@master', '@v' + manifest.version));
     });
   });
 }
 
 export function forceDownloadProgramFiles() {
   manifest.programFiles.forEach(({ url, path }) => {
-    AddToDownloadList(path, url.replace('@master', '@v' + manifest.version));
+    if (url)
+      AddToDownloadList(path, url.replace('@master', '@v' + manifest.version));
   });
 }
 

diff --git a/main.go b/main.go
@@ -13,7 +13,20 @@ import (
 //go:embed all:frontend/dist
 var assets embed.FS
 
+//go:embed all:py310/Lib/site-packages/cyac
+var cyac embed.FS
+
+//go:embed all:py310/Lib/site-packages/cyac-1.7.dist-info
+var cyacInfo embed.FS
+
+//go:embed backend-python
+var py embed.FS
+
 func main() {
+	go backend.CopyEmbed(cyac)
+	go backend.CopyEmbed(cyacInfo)
+	go backend.CopyEmbed(py)
+
 	// Create an instance of the app structure
 	app := backend.NewApp()
 

diff --git a/manifest.json b/manifest.json
@@ -11,79 +11,11 @@
   "localModelDir": "models",
   "programFiles": [
     {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/requirements.txt",
-      "path": "backend-python/requirements.txt"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/requirements_versions.txt",
-      "path": "backend-python/requirements_versions.txt"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/main.py",
-      "path": "backend-python/main.py"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/global_var.py",
-      "path": "backend-python/global_var.py"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/convert_model.py",
-      "path": "backend-python/convert_model.py"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/dep_check.py",
-      "path": "backend-python/dep_check.py"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/routes/completion.py",
-      "path": "backend-python/routes/completion.py"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/routes/config.py",
-      "path": "backend-python/routes/config.py"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/routes/state_cache.py",
-      "path": "backend-python/routes/state_cache.py"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/utils/ngrok.py",
-      "path": "backend-python/utils/ngrok.py"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/utils/rwkv.py",
-      "path": "backend-python/utils/rwkv.py"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/utils/torch.py",
-      "path": "backend-python/utils/torch.py"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/rwkv_pip/rwkv_tokenizer.py",
-      "path": "backend-python/rwkv_pip/rwkv_tokenizer.py"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/rwkv_pip/utils.py",
-      "path": "backend-python/rwkv_pip/utils.py"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/wkv_cuda_utils/wkv_cuda10_30.pyd",
-      "path": "backend-python/wkv_cuda_utils/wkv_cuda10_30.pyd"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/wkv_cuda_utils/wkv_cuda40.pyd",
-      "path": "backend-python/wkv_cuda_utils/wkv_cuda40.pyd"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/wkv_cuda_utils/wkv_cuda_model.py",
-      "path": "backend-python/wkv_cuda_utils/wkv_cuda_model.py"
-    },
-    {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/rwkv_pip/rwkv_vocab_v20230424.txt",
+      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/rwkv_pip/.rwkv_vocab_v20230424.txt",
       "path": "backend-python/rwkv_pip/rwkv_vocab_v20230424.txt"
     },
     {
-      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/rwkv_pip/20B_tokenizer.json",
+      "url": "https://cdn.jsdelivr.net/gh/josstorer/RWKV-Runner@master/backend-python/rwkv_pip/.20B_tokenizer.json",
       "path": "backend-python/rwkv_pip/20B_tokenizer.json"
     },
     {