Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use suft #9

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

use suft #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
72a370e
Ãuse ssuft
qwerty22121998 Jan 15, 2022
90157c2
wait
qwerty22121998 Jan 15, 2022
75a46b7
agent
qwerty22121998 Jan 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/controller/viblo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func (v *vibloController) PublishNews(c echo.Context) error {
if err := c.Bind(&req); err != nil {
return err
}
if err := v.bot.PostEditorChoice(req.ChannelID, req.Limit); err != nil {
if err := v.bot.PostTrending(req.ChannelID, req.Limit); err != nil {
return c.JSON(500, dtos.Response{
Message: err.Error(),
})
Expand Down
48 changes: 33 additions & 15 deletions cmd/viblo/services/viblo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ import (
"github.com/bwmarrin/discordgo"
"github.com/fu-js/discord-bot/cmd/viblo/dtos"
"github.com/fu-js/discord-bot/pkg/utils/log"
"github.com/headzoo/surf/agent"
"github.com/headzoo/surf/browser"
"gopkg.in/headzoo/surf.v1"
"html"
"net/http"
"strings"
"time"
Expand All @@ -33,27 +37,37 @@ type VibloService interface {
}

type vibloService struct {
surf *browser.Browser
}

func NewVibloService() VibloService {
return &vibloService{}
b := surf.NewBrowser()

return &vibloService{
surf: b,
}
}

func (s *vibloService) GetEditorChoices(limit int) ([]dtos.VibloPost, error) {
data := dtos.VibloPostResponse{}
tab := s.surf.NewTab()
tab.SetUserAgent(agent.Chrome())

resp, err := http.Get(fmt.Sprintf("https://viblo.asia/api/posts/editors-choice?limit=%v", limit))
if err != nil {
log.Zap.Errorw("error when call viblo editor choices api", "error", err)
tab.AddRequestHeader("Accept", "text/json")
tab.AddRequestHeader("Accept-Charset", "utf8")

if err := tab.Open(fmt.Sprintf("https://viblo.asia/api/posts/editors-choice?limit=%v", limit)); err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
err := errors.New(fmt.Sprintf("http request with non ok status: %v", resp.StatusCode))
if tab.StatusCode() != http.StatusOK {
err := errors.New(fmt.Sprintf("http request with non ok status: %v", tab.StatusCode()))
log.Zap.Errorw("error when call viblo editor choices api", "error", err)
return nil, err
}
if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
time.Sleep(time.Second * 6)
body := tab.Body()
body = html.UnescapeString(body)
if err := json.Unmarshal([]byte(body), &data); err != nil {
log.Zap.Errorw("error when decode viblo editor choices response", "error", err)
return nil, err
}
Expand All @@ -63,18 +77,22 @@ func (s *vibloService) GetEditorChoices(limit int) ([]dtos.VibloPost, error) {
func (s *vibloService) GetTrending(limit int) ([]dtos.VibloPost, error) {
data := dtos.VibloPostResponse{}

resp, err := http.Get(fmt.Sprintf("https://viblo.asia/api/posts/trending?limit=%v", limit))
if err != nil {
log.Zap.Errorw("error when call viblo trending api", "error", err)
tab := s.surf.NewTab()
tab.SetUserAgent(agent.Chrome())

if err := tab.Open(fmt.Sprintf("https://viblo.asia/api/posts/trending?limit=%v", limit)); err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
err := errors.New(fmt.Sprintf("http request with non ok status: %v", resp.StatusCode))

if tab.StatusCode() != http.StatusOK {
err := errors.New(fmt.Sprintf("http request with non ok status: %v", tab.StatusCode()))
log.Zap.Errorw("error when call viblo trending api", "error", err)
return nil, err
}
if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
time.Sleep(time.Second * 6)
body := tab.Body()
body = html.UnescapeString(body)
if err := json.Unmarshal([]byte(body), &data); err != nil {
log.Zap.Errorw("error when decode viblo trending response", "error", err)
return nil, err
}
Expand Down
6 changes: 5 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@ require (
)

require (
github.com/PuerkitoBio/goquery v1.8.0 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/gorilla/websocket v1.4.0 // indirect
github.com/headzoo/surf v1.0.1 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/labstack/gommon v0.3.1 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
Expand All @@ -23,7 +26,8 @@ require (
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
golang.org/x/net v0.0.0-20210913180222-943fd674d43e // indirect
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 // indirect
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect
golang.org/x/text v0.3.7 // indirect
gopkg.in/headzoo/surf.v1 v1.0.1 // indirect
)
10 changes: 10 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,14 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
Expand Down Expand Up @@ -227,6 +231,8 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn
github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
github.com/headzoo/surf v1.0.1 h1:wk3+LT8gjnCxEwfBJl6MhaNg154En5KjgmgzAG9uMS0=
github.com/headzoo/surf v1.0.1/go.mod h1:/bct0m/iMNEqpn520y01yoaWxsAEigGFPnvyR1ewR5M=
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
Expand Down Expand Up @@ -458,6 +464,8 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210913180222-943fd674d43e h1:+b/22bPvDYt4NPDcy4xAGCmON713ONAWFeY3Z7I3tR8=
golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -774,6 +782,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/headzoo/surf.v1 v1.0.1 h1:oDBy9b5NlTb2Hvl3hF8NN+Qy7ypC9/g5YDP85pPh13k=
gopkg.in/headzoo/surf.v1 v1.0.1/go.mod h1:T0BH8276y+OPL0E4tisxCFjBVIAKGbwdYU7AS7/EpQQ=
gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
Expand Down