diff --git a/api/controller/viblo.go b/api/controller/viblo.go index 3071678..9eb76f0 100644 --- a/api/controller/viblo.go +++ b/api/controller/viblo.go @@ -30,7 +30,7 @@ func (v *vibloController) PublishNews(c echo.Context) error { if err := c.Bind(&req); err != nil { return err } - if err := v.bot.PostEditorChoice(req.ChannelID, req.Limit); err != nil { + if err := v.bot.PostTrending(req.ChannelID, req.Limit); err != nil { return c.JSON(500, dtos.Response{ Message: err.Error(), }) diff --git a/cmd/viblo/services/viblo.go b/cmd/viblo/services/viblo.go index 2813613..275ad22 100644 --- a/cmd/viblo/services/viblo.go +++ b/cmd/viblo/services/viblo.go @@ -7,6 +7,10 @@ import ( "github.com/bwmarrin/discordgo" "github.com/fu-js/discord-bot/cmd/viblo/dtos" "github.com/fu-js/discord-bot/pkg/utils/log" + "github.com/headzoo/surf/agent" + "github.com/headzoo/surf/browser" + "gopkg.in/headzoo/surf.v1" + "html" "net/http" "strings" "time" @@ -33,27 +37,37 @@ type VibloService interface { } type vibloService struct { + surf *browser.Browser } func NewVibloService() VibloService { - return &vibloService{} + b := surf.NewBrowser() + + return &vibloService{ + surf: b, + } } func (s *vibloService) GetEditorChoices(limit int) ([]dtos.VibloPost, error) { data := dtos.VibloPostResponse{} + tab := s.surf.NewTab() + tab.SetUserAgent(agent.Chrome()) - resp, err := http.Get(fmt.Sprintf("https://viblo.asia/api/posts/editors-choice?limit=%v", limit)) - if err != nil { - log.Zap.Errorw("error when call viblo editor choices api", "error", err) + tab.AddRequestHeader("Accept", "text/json") + tab.AddRequestHeader("Accept-Charset", "utf8") + + if err := tab.Open(fmt.Sprintf("https://viblo.asia/api/posts/editors-choice?limit=%v", limit)); err != nil { return nil, err } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - err := errors.New(fmt.Sprintf("http request with non ok status: %v", resp.StatusCode)) + if tab.StatusCode() != http.StatusOK { + err := errors.New(fmt.Sprintf("http request with non ok status: %v", tab.StatusCode())) log.Zap.Errorw("error when call viblo editor choices api", "error", err) return nil, err } - if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + time.Sleep(time.Second * 6) + body := tab.Body() + body = html.UnescapeString(body) + if err := json.Unmarshal([]byte(body), &data); err != nil { log.Zap.Errorw("error when decode viblo editor choices response", "error", err) return nil, err } @@ -63,18 +77,22 @@ func (s *vibloService) GetEditorChoices(limit int) ([]dtos.VibloPost, error) { func (s *vibloService) GetTrending(limit int) ([]dtos.VibloPost, error) { data := dtos.VibloPostResponse{} - resp, err := http.Get(fmt.Sprintf("https://viblo.asia/api/posts/trending?limit=%v", limit)) - if err != nil { - log.Zap.Errorw("error when call viblo trending api", "error", err) + tab := s.surf.NewTab() + tab.SetUserAgent(agent.Chrome()) + + if err := tab.Open(fmt.Sprintf("https://viblo.asia/api/posts/trending?limit=%v", limit)); err != nil { return nil, err } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - err := errors.New(fmt.Sprintf("http request with non ok status: %v", resp.StatusCode)) + + if tab.StatusCode() != http.StatusOK { + err := errors.New(fmt.Sprintf("http request with non ok status: %v", tab.StatusCode())) log.Zap.Errorw("error when call viblo trending api", "error", err) return nil, err } - if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + time.Sleep(time.Second * 6) + body := tab.Body() + body = html.UnescapeString(body) + if err := json.Unmarshal([]byte(body), &data); err != nil { log.Zap.Errorw("error when decode viblo trending response", "error", err) return nil, err } diff --git a/go.mod b/go.mod index 9de8d98..ca263e0 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,10 @@ require ( ) require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect github.com/gorilla/websocket v1.4.0 // indirect + github.com/headzoo/surf v1.0.1 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/labstack/gommon v0.3.1 // indirect github.com/mattn/go-colorable v0.1.12 // indirect @@ -23,7 +26,8 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect - golang.org/x/net v0.0.0-20210913180222-943fd674d43e // indirect + golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 // indirect golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect golang.org/x/text v0.3.7 // indirect + gopkg.in/headzoo/surf.v1 v1.0.1 // indirect ) diff --git a/go.sum b/go.sum index ce3b883..1068121 100644 --- a/go.sum +++ b/go.sum @@ -50,10 +50,14 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -227,6 +231,8 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/headzoo/surf v1.0.1 h1:wk3+LT8gjnCxEwfBJl6MhaNg154En5KjgmgzAG9uMS0= +github.com/headzoo/surf v1.0.1/go.mod h1:/bct0m/iMNEqpn520y01yoaWxsAEigGFPnvyR1ewR5M= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -458,6 +464,8 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210913180222-943fd674d43e h1:+b/22bPvDYt4NPDcy4xAGCmON713ONAWFeY3Z7I3tR8= golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -774,6 +782,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/headzoo/surf.v1 v1.0.1 h1:oDBy9b5NlTb2Hvl3hF8NN+Qy7ypC9/g5YDP85pPh13k= +gopkg.in/headzoo/surf.v1 v1.0.1/go.mod h1:T0BH8276y+OPL0E4tisxCFjBVIAKGbwdYU7AS7/EpQQ= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=