feat: uploader !!!

.gitignore

@@ -1,3 +1,4 @@
 testoutcome_*
 aixifan_config.json
-aixifan_downloads/
+aixifan_downloads/
+/aixifan

README.md

@@ -7,9 +7,9 @@
     - [x] m3u8 Downloader
     - [x] ffmpeg convert ts to mp4 (?)
     - [x] Clean up hlsdl tmp files
-- [ ] Uploader
+    - [ ] itemimage
     - [x] Check if item exists
-    - [ ] mapping IA S3 protocol
+- [ ] Uploader
     - [ ] modify metadata
 - [x] Version check
 - [ ] CLI

go.mod

@@ -1,20 +1,24 @@
 module github.com/saveweb/aixifan
 
-go 1.23.3
+go 1.23.4
 
 require (
 	github.com/canhlinh/hlsdl v0.0.0-20240124081241-e7eb3b498559
+	github.com/saveweb/go2internetarchive v0.1.3
 	github.com/tidwall/gjson v1.18.0
 	golang.org/x/net v0.32.0
 )
 
 require (
 	github.com/go-resty/resty/v2 v2.11.0 // indirect
 	github.com/grafov/m3u8 v0.12.0 // indirect
-	github.com/mattn/go-runewidth v0.0.15 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/schollz/progressbar/v3 v3.17.1 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/term v0.27.0 // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 )

go.sum

@@ -2,10 +2,13 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/canhlinh/hlsdl v0.0.0-20240124081241-e7eb3b498559 h1:hqwX4Qer4iuDCJQhII+Wr+PZdoGv+loawaiCcSCVKMU=
 github.com/canhlinh/hlsdl v0.0.0-20240124081241-e7eb3b498559/go.mod h1:RoxHXYtRu2f/z55VL4U4Go56JBgJLlqhD5/c8I7z+1c=
+github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
+github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -19,25 +22,37 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/saveweb/go2internetarchive v0.1.3 h1:mQMCE0SjSjDytof0Oi6r3ubF3uhuOzj9QVlmXNMgwgc=
+github.com/saveweb/go2internetarchive v0.1.3/go.mod h1:oWhhLOLOXugUbT/ZZ6PmRqZjTXKq4F9BhXDbXAqrucw=
+github.com/schollz/progressbar/v3 v3.17.1 h1:bI1MTaoQO+v5kzklBjYNRQLoVpe0zbyRZNK6DFkVC5U=
+github.com/schollz/progressbar/v3 v3.17.1/go.mod h1:RzqpnsPQNjUyIgdglUjRLgD7sVnxN1wpmBMV+UiEbL4=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
 github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
@@ -85,6 +100,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
+golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -103,3 +120,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk=
 gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -7,6 +7,7 @@ import (
 	"os"
 
 	"github.com/saveweb/aixifan/pkg/downloader"
+	"github.com/saveweb/aixifan/pkg/uploader"
 	"github.com/saveweb/aixifan/pkg/utils"
 )
 
@@ -32,6 +33,8 @@ func main() {
 	downSkipIACheck := downCmd.Bool("s", false, "Do not check if the item already exists on IA")
 
 	upCmd := flag.NewFlagSet("up", flag.ExitOnError)
+	upDougaId := upCmd.String("i", "", "Douga ID (int-string, NOT contain 'ac' or '_')")
+	upDelete := upCmd.Bool("d", false, "Delete the dougaDir after uploading successfully")
 
 	versionCmd := flag.NewFlagSet("version", flag.ExitOnError)
 
@@ -43,10 +46,10 @@ func main() {
 	switch os.Args[1] {
 	case "down":
 		downCmd.Parse(os.Args[2:])
-		os.Exit(downloader.Main(downCmd, downNoVersionCheck, downDougaId, downSkipIACheck))
+		os.Exit(downloader.Main(downCmd, *downNoVersionCheck, *downDougaId, *downSkipIACheck))
 	case "up":
 		upCmd.Parse(os.Args[2:])
-		panic("not implemented")
+		os.Exit(uploader.Main(upCmd, *upDougaId, *upDelete))
 	case "version":
 		versionCmd.Parse(os.Args[2:])
 		fmt.Println(utils.GetVersion().Version)

pkg/downloader/main.go → pkg/downloader/down.go

@@ -8,8 +8,12 @@ import (
 	"github.com/saveweb/aixifan/pkg/utils"
 )
 
-func Main(downCmd *flag.FlagSet, downNoVersionCheck *bool, downDougaId *string, downSkipIACheck *bool) int {
-	dougaId := *downDougaId
+func Main(downCmd *flag.FlagSet, downNoVersionCheck bool, downDougaId string, downSkipIACheck bool) int {
+	dougaId := downDougaId
+	if dougaId == "" {
+		downCmd.Usage()
+		return 2
+	}
 
 	// check new version
 	newVersionChan := make(chan bool, 1)
@@ -24,7 +28,7 @@ func Main(downCmd *flag.FlagSet, downNoVersionCheck *bool, downDougaId *string,
 		}
 	}(newVersionChan)
 	defer close(newVersionChan)
-	if !*downNoVersionCheck {
+	if !downNoVersionCheck {
 		go func(ch chan bool) {
 			slog.Info("Checking new version...")
 			newVersion, err := utils.NewVersionAvailable()
@@ -42,7 +46,7 @@ func Main(downCmd *flag.FlagSet, downNoVersionCheck *bool, downDougaId *string,
 	}
 
 	// Check IA Item (dedup)
-	if !*downSkipIACheck {
+	if !downSkipIACheck {
 		slog.Info("Checking IA item...")
 		identifier := utils.ToIdentifier(dougaId)
 		exists, err := utils.CheckIAItemExist(identifier)
@@ -58,11 +62,6 @@ func Main(downCmd *flag.FlagSet, downNoVersionCheck *bool, downDougaId *string,
 	}
 
 	// download
-	if dougaId == "" {
-		downCmd.Usage()
-		return 2
-	}
-
 	config, err := config.LoadOrNewConfig()
 	if err != nil {
 		panic(err)

pkg/downloader/downloader.go

@@ -2,8 +2,10 @@ package downloader
 
 import (
 	"fmt"
+	"io"
 	"log/slog"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"time"
@@ -125,13 +127,52 @@ func DownloadVideo(dougaDir, acid, part string) error {
 	return nil
 }
 
+func DownloadCover(dougaDir, part string) error {
+	info, err := extractor.GetPartInfo(part)
+	if err != nil {
+		return err
+	}
+	urlParsed, err := url.Parse(info.CoverUrl)
+	ext := path.Ext(urlParsed.Path)
+	coverFilepath := path.Join(dougaDir, "douga"+info.DougaId+"_itemimage"+ext)
+	slog.Info("Downloading cover", "coverFilepath", coverFilepath, "coverUrl", info.CoverUrl)
+
+	client := &http.Client{
+		Timeout: 15 * time.Second,
+	}
+	req, _ := http.NewRequest("GET", info.CoverUrl, nil)
+	req.Header.Set("User-Agent", utils.GetUA())
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if err := os.WriteFile(coverFilepath, body, 0644); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func Download(downloadsHomeDir string, dougaId string) error {
 	if !validateDougaId(dougaId) {
 		return fmt.Errorf("invalid dougaId")
 	}
 
-	client := &http.Client{Timeout: 15 * time.Second}
+	dougaDir := path.Join(downloadsHomeDir, dougaId)
+	if isAllDownloaded(dougaDir) {
+		slog.Info("Already downloaded", "dougaDir", dougaDir)
+		return nil
+	}
 
+	client := &http.Client{Timeout: 15 * time.Second}
 	parts, err := api.GetDougaAll(client, dougaId)
 	if err != nil {
 		return err
@@ -140,7 +181,6 @@ func Download(downloadsHomeDir string, dougaId string) error {
 		return fmt.Errorf("parts is empty")
 	}
 
-	dougaDir := path.Join(downloadsHomeDir, dougaId)
 	if err := os.MkdirAll(dougaDir, 0755); err != nil {
 		return err
 	}
@@ -151,11 +191,38 @@ func Download(downloadsHomeDir string, dougaId string) error {
 	}
 	slog.Info("DougaInfos saved", "dougaDir", dougaDir)
 
+	if err := DownloadCover(dougaDir, parts[0]); err != nil {
+		slog.Error("DownloadCover", "err", err)
+		return err
+	}
+	slog.Info("DownloadCover saved", "dougaDir", dougaDir)
+
 	for i, part := range parts {
 		acid := fmt.Sprintf("%s_%d", dougaId, i+1)
 		if err := DownloadVideo(dougaDir, acid, part); err != nil {
 			return err
 		}
 	}
+	if err := markAllDownloaded(dougaDir); err != nil {
+		return err
+	}
+
 	return Cleanup(dougaDir)
 }
+
+// check _alldownloaded.mark
+func isAllDownloaded(dougaDir string) bool {
+	// check if dougaDir/_alldownloaded.mark exists
+	_, err := os.Stat(path.Join(dougaDir, "_alldownloaded.mark"))
+	return err == nil
+}
+
+func markAllDownloaded(dougaDir string) error {
+	// create dougaDir/_alldownloaded.mark
+	f, err := os.Create(path.Join(dougaDir, "_alldownloaded.mark"))
+	if err != nil {
+		return fmt.Errorf("failed to create _alldownloaded.mark: %w", err)
+	}
+	defer f.Close()
+	return nil
+}

pkg/downloader/downloader_test.go

@@ -0,0 +1 @@
+package downloader

pkg/extractor/extractor.go

@@ -2,6 +2,7 @@ package extractor
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strings"
@@ -92,6 +93,36 @@ func GetTitles(part string) (dougaTitle string, partTitle string, err error) {
 	return dougaTitle, partTitle, nil
 }
 
+type Tag struct {
+	Name string `json:"name"`
+	Id   string `json:"id"`
+}
+
+type CurrentVideoInfo struct {
+	Id        string `json:"id"` // video id (类似 B 站的 cid)
+	PartTitle string `json:"title"`
+}
+
+type PartInfo struct {
+	CoverUrl         string           `json:"coverUrl"` // 封面
+	DougaId          string           `json:"dougaId"`
+	DougaTitle       string           `json:"title"`
+	CurrentVideoInfo CurrentVideoInfo `json:"currentVideoInfo"`
+	Description      string           `json:"description"`
+	User             Tag              `json:"user"` // 共用 Tag，反正字段一致
+	TagList          []Tag            `json:"tagList"`
+	CreateTime       string           `json:"createTime"`
+}
+
+func GetPartInfo(part string) (PartInfo, error) {
+	var info PartInfo
+	err := json.Unmarshal([]byte(part), &info)
+	if err != nil {
+		return info, err
+	}
+	return info, nil
+}
+
 // part: video info json
 func GetKsPlayJson(part string) (string, error) {
 	result := gjson.Get(part, "currentVideoInfo.ksPlayJson")