Skip to content

Commit

Permalink
fix: file renamed to url_name, missing posts fixed
Browse files Browse the repository at this point in the history
  • Loading branch information
@serenevoid
serenevoid committed May 16, 2023
1 parent e5d341b commit 23bb1a0
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 80 deletions.
101 changes: 53 additions & 48 deletions models/models.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@ package models
import (
"encoding/json"
"fmt"
"html/template"
"os"
"strings"
"html/template"
)

/*
Thread struct used to store the thread details
Thread struct used to store the thread details
*/
type thread struct {
No int `json:"no"`
Expand All @@ -18,64 +18,66 @@ type thread struct {
}

/*
Page struct used to store page details
Page struct used to store page details
*/
type page struct {
Page uint8 `json:"page"`
Threads []thread `json:"threads"`
}

/*
Post struct used to store post details
Post struct used to store post details
*/
type Post struct {
No int `json:"no"`
Now string `json:"now"`
Name string `json:"name"`
Sub string `json:"sub"`
Com string `json:"Com"`
Com_html template.HTML
Filename string `json:"filename"`
Ext string `json:"ext"`
W int `json:"w"`
H int `json:"h"`
Tn_w int `json:"tn_w"`
Tn_h int `json:"tn_h"`
Tim int64 `json:"tim"`
Time int64 `json:"time"`
Md5 string `json:"md5"`
Fsize int `json:"fsize"`
Size string
Resto int `json:"resto"`
Bumplimit int `json:"bumplimit"`
Imagelimit int `json:"imagelimit"`
Semantic_url string `json:"semantic_url"`
Replies int `json:"replies"`
Images int `json:"images"`
Unique_ips int `json:"unique_ips"`
No int `json:"no"`
Now string `json:"now"`
Name string `json:"name"`
Sub string `json:"sub"`
Com string `json:"Com"`
Com_html template.HTML
Filename string `json:"filename"`
File_ID string
IsMediaPresent bool
Ext string `json:"ext"`
W int `json:"w"`
H int `json:"h"`
Tn_w int `json:"tn_w"`
Tn_h int `json:"tn_h"`
Tim int64 `json:"tim"`
Time int64 `json:"time"`
Md5 string `json:"md5"`
Fsize int `json:"fsize"`
Size string
Resto int `json:"resto"`
Bumplimit int `json:"bumplimit"`
Imagelimit int `json:"imagelimit"`
Semantic_url string `json:"semantic_url"`
Replies int `json:"replies"`
Images int `json:"images"`
Unique_ips int `json:"unique_ips"`
}

/*
Post array type to store array of posts
Post array type to store array of posts
*/
type post_array struct {
Posts []Post `json:"posts"`
}

/*
File struct to store file data
File struct to store file data
*/
type File struct {
File_name string
Media_URL string
File_name string
Media_URL string
}

/*
Extract the thread IDs from the json byte array.
Extract the thread IDs from the json byte array.
@param []byte - the body of the response from http req
@param []byte - the body of the response from http req
@return ([]int, error) - (array of thread IDs, error)
@return ([]int, error) - (array of thread IDs, error)
*/
func Get_threads_from_json(body []byte) ([]int, error) {
var dat []page
Expand All @@ -100,11 +102,11 @@ func Get_threads_from_json(body []byte) ([]int, error) {
}

/*
Extract the thread IDs from the json byte array.
Extract the thread IDs from the json byte array.
@param []byte - the body of the response from http req
@param []byte - the body of the response from http req
@return ([]int, error) - (array of thread IDs, error)
@return ([]int, error) - (array of thread IDs, error)
*/
func Get_posts_from_json(body []byte) ([]Post, error) {
var media_list []Post
Expand All @@ -119,20 +121,23 @@ func Get_posts_from_json(body []byte) ([]Post, error) {
for i := 0; i < len(post_list); i++ {
post_content := post_list[i]
post_content.Size = ByteCountDecimal(post_content.Fsize)
post_content.Com_html = template.HTML(post_content.Com)
post_content.Com_html = template.HTML(post_content.Com)
post_content.IsMediaPresent = false
if post_content.Tim != 0 {
media_list = append(media_list, post_content)
post_content.File_ID = fmt.Sprint(post_content.Tim)
post_content.IsMediaPresent = true
}
media_list = append(media_list, post_content)
}
return media_list, nil
}

/*
Get all media URLs from the list of posts for downloading them.
Get all media URLs from the list of posts for downloading them.
@param (string, []Post) - (board and thread ID, array of all posts)
@param (string, []Post) - (board and thread ID, array of all posts)
@return ([]File, error) - (list of all files, error)
@return ([]File, error) - (list of all files, error)
*/
func Get_media_urls_from_posts(entry string, posts []Post) ([]File, error) {
var file_list []File
Expand All @@ -144,15 +149,15 @@ func Get_media_urls_from_posts(entry string, posts []Post) ([]File, error) {
var media_file File
if post.Filename != "" {
sep := string(os.PathSeparator)
media_file.File_name = "archive" + sep + board + sep + thread + sep + "media" + sep + post.Filename + post.Ext
media_file.File_name = "archive" + sep + board + sep + thread + sep + "media" + sep + post.File_ID + post.Ext
}
if post.Tim != 0 {
media_file.Media_URL = fmt.Sprintf("https://i.4cdn.org/%s/%s%s", board, fmt.Sprint(post.Tim), post.Ext)
}
var thumbnail_file File
if post.Filename != "" {
sep := string(os.PathSeparator)
thumbnail_file.File_name = "archive" + sep + board + sep + thread + sep + "thumbnails" + sep + post.Filename + "s.jpg"
thumbnail_file.File_name = "archive" + sep + board + sep + thread + sep + "thumbnails" + sep + fmt.Sprint(post.Tim) + "s.jpg"
}
if post.Tim != 0 {
thumbnail_file.Media_URL = fmt.Sprintf("https://i.4cdn.org/%s/%ss.jpg", board, fmt.Sprint(post.Tim))
Expand All @@ -164,11 +169,11 @@ func Get_media_urls_from_posts(entry string, posts []Post) ([]File, error) {
}

/*
Convert size from bytes to a readable format
Convert size from bytes to a readable format
@param (int) - (size byte int)
@param (int) - (size byte int)
@return (string) - (readable string format)
@return (string) - (readable string format)
*/
func ByteCountDecimal(b int) string {
const unit = 1000
Expand Down
50 changes: 26 additions & 24 deletions networking/networking.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ import (
)

/*
Get all response data from an http req.
Get all response data from an http req.
@param string - the URL to request data from
@param string - the URL to request data from
@return ([]byte, error) - (response body data, error)
@return ([]byte, error) - (response body data, error)
*/
func get_response(url string) ([]byte, error) {
client := &http.Client{
Expand All @@ -41,9 +41,9 @@ func get_response(url string) ([]byte, error) {
}
defer resp.Body.Close()

if resp.StatusCode != 200 {
return nil, nil
}
if resp.StatusCode != 200 {
return nil, nil
}

body, err := ioutil.ReadAll(resp.Body)
if err != nil {
Expand All @@ -54,11 +54,11 @@ func get_response(url string) ([]byte, error) {
}

/*
Get all posts from a thread
Get all posts from a thread
@param string - board and thread ID
@param string - board and thread ID
@return ([]Posts, error) - (list of posts present in thread, error)
@return ([]Posts, error) - (list of posts present in thread, error)
*/
func Get_posts_from_thread(entry string) ([]models.Post, error) {
entry_elements := strings.Split(entry, "_")
Expand All @@ -69,10 +69,10 @@ func Get_posts_from_thread(entry string) ([]models.Post, error) {
if err != nil {
return nil, fmt.Errorf("%s %s", err, entry)
}
if body == nil {
fmt.Println("Unable to reach thread ", entry)
return nil, nil
}
if body == nil {
fmt.Println("Unable to reach thread ", entry)
return nil, nil
}
posts, err := models.Get_posts_from_json(body)
if err != nil {
return nil, err
Expand All @@ -81,11 +81,11 @@ func Get_posts_from_thread(entry string) ([]models.Post, error) {
}

/*
Download specified file from URL.
Download specified file from URL.
@param string - URL of the file
@param string - URL of the file
@return ([]byte, error) - bytes recieved from the request
@return ([]byte, error) - bytes recieved from the request
*/
func downloadFile(URL string) ([]byte, error) {
response, err := http.Get(URL)
Expand All @@ -105,22 +105,24 @@ func downloadFile(URL string) ([]byte, error) {
}

/*
Download a files from a given list
Download a files from a given list
@param []File - list of all files to be downloaded
@param []File - list of all files to be downloaded
*/
func Download_media(file_list []models.File) {
var waiter sync.WaitGroup
waiter.Add(len(file_list))
for _, file := range file_list {
go func(file models.File, waiter *sync.WaitGroup) {
_, err := os.Stat(file.File_name)
if errors.Is(err, os.ErrNotExist) {
b, err := downloadFile(file.Media_URL)
if err != nil {
fmt.Println("Download of " + file.File_name + " failed")
if file.File_name != "" {
_, err := os.Stat(file.File_name)
if errors.Is(err, os.ErrNotExist) {
b, err := downloadFile(file.Media_URL)
if err != nil {
fmt.Println("Download of " + file.File_name + " failed")
}
ioutil.WriteFile(file.File_name, b, 0777)
}
ioutil.WriteFile(file.File_name, b, 0777)
}
waiter.Done()
}(file, &waiter)
Expand Down
22 changes: 14 additions & 8 deletions template.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,20 +45,24 @@
</script>
<style>
body {
background-color: #383838;
background-color: #000000;
overflow-x: hidden; /* Disable horizontal scroll */
}
a {
color: #4488bb
color: #324A5F
}
subject {
font-size: 16px;
}
#preview {
border-radius: 0.5em;
cursor: pointer;
}
div.postContainer {
max-width: 1000px;
background-color: #282828;
border: 2px solid #686868!important;
padding: 5px;
background-color: #0C1821;
border: 2px solid #1B2A41!important;
padding: 1em;
margin: 5px auto 5px auto;
color: #D6DAF0;
border-radius: 10px;
Expand Down Expand Up @@ -104,16 +108,18 @@
</a>
</span>
</div>
{{ if .IsMediaPresent }}
<div class="file">
<div class="fileText">
File:
<a title="{{ .Filename }}{{ .Ext }}" href="./media/{{ .Filename }}{{ .Ext }}" target="_blank">
{{ .Filename }}{{ .Ext }}
<a title="{{ .Filename }}{{ .Ext }}" href="./media/{{ .File_ID }}{{ .Ext }}" target="_blank">
{{ .File_ID }}{{ .Ext }}
</a>
({{ .Size }}, {{ .W }}x{{ .H }})
</div>
<img src="./thumbnails/{{ .Filename }}s.jpg" alt="{{ .Filename }}{{ .Ext }}" data-md5="{{ .Md5 }}" loading="lazy" onclick="open_overlay(this)">
<img id="preview" src="./thumbnails/{{ .File_ID }}s.jpg" alt="{{ .File_ID }}{{ .Ext }}" data-md5="{{ .Md5 }}" loading="lazy" onclick="open_overlay(this)">
</div>
{{ end }}
<blockquote>{{ .Com_html }}</blockquote>
</div>
{{end}}
Expand Down

0 comments on commit 23bb1a0

Please sign in to comment.