Merge branch 'iv-org:master' into master

.github/ISSUE_TEMPLATE/bug_report.md

@@ -10,8 +10,10 @@ assignees: ''
 <!--
   BEFORE TRYING TO REPORT A BUG:
  
-  * Read the FAQ!
-  * Use the search function to check if there is already an issue open for your problem!
+  * Read the FAQ: https://docs.invidious.io/faq/!
+  * Use the search function to check if there is already an issue open for your problem: https://github.com/search?q=repo%3Aiv-org%2Finvidious+replace+me+with+your+bug&type=issues!
+
+  MAKE SURE TO FOLLOW THE TWO STEPS ABOVE BEFORE REPORTING A BUG. A BUG THAT ALREADY EXIST WILL IMMEDIATELY CLOSED.
 
   If you want to suggest a new feature please use "Feature request" instead
   If you want to suggest an enhancement to an existing feature please use "Enhancement" instead

.github/workflows/ci.yml

@@ -54,7 +54,7 @@ jobs:
 
       - name: Install required APT packages
         run: |
-          sudo apt install -y libsqlite3-dev 
+          sudo apt install -y libsqlite3-dev
         shell: bash
 
       - name: Install Crystal
@@ -65,7 +65,9 @@ jobs:
       - name: Cache Shards
         uses: actions/cache@v3
         with:
-          path: ./lib
+          path: |
+            ./lib
+            ./bin
           key: shards-${{ hashFiles('shard.lock') }}
 
       - name: Install Shards
@@ -77,14 +79,6 @@ jobs:
       - name: Run tests
         run: crystal spec
 
-      - name: Run lint
-        run: |
-          if ! crystal tool format --check; then
-            crystal tool format
-            git diff
-            exit 1
-          fi
-
       - name: Build
         run: crystal build --warnings all --error-on-warnings --error-trace src/invidious.cr
 
@@ -130,8 +124,12 @@ jobs:
       - name: Test Docker
         run: while curl -Isf http://localhost:3000; do sleep 1; done
 
-  ameba_lint:
+  lint:
+
     runs-on: ubuntu-latest
+
+    continue-on-error: true
+
     steps:
       - uses: actions/checkout@v4
         with:
@@ -151,7 +149,18 @@ jobs:
           key: shards-${{ hashFiles('shard.lock') }}
 
       - name: Install Shards
-        run: shards install
+        run: |
+          if ! shards check; then
+            shards install
+          fi
+
+      - name: Check Crystal formatter compliance
+        run: |
+          if ! crystal tool format --check; then
+            crystal tool format
+            git diff
+            exit 1
+          fi
 
       - name: Run Ameba linter
         run: bin/ameba

.github/workflows/stale.yml

@@ -13,14 +13,11 @@ jobs:
     - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        days-before-stale: 365
-        days-before-pr-stale: 90 
-        days-before-close: 30
-        exempt-pr-labels: blocked,exempt-stale
+        days-before-stale: 730
+        days-before-pr-stale: -1 
+        days-before-close: 60
         stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.'
-        stale-pr-message: 'This pull request has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely abandoned or outdated. If you think this pull request is still relevant and applicable, you just have to post a comment and it will be unmarked.'
         stale-issue-label: "stale"
-        stale-pr-label: "stale"
         ascending: true
-        # Never mark feature requests/enhancements as stale
-        exempt-issue-labels: "feature-request,enhancement,exempt-stale"
+        # Exempt the following types of issues from being staled
+        exempt-issue-labels: "feature-request,enhancement,discussion,exempt-stale"

CHANGELOG.md

@@ -3,8 +3,98 @@
 ## vX.Y.0 (future)
 
 
+## v2.20241110.0
+
+### Wrap-up
+
+This release is most importantly here to fix to the annoying "Youtube API returned error 400"
+error that prevented all channel pages from loading.
+
+If you're updating from the previous release, it provides no improvements on the ability to play
+videos. If updating from a commit in-between release, it removes the "Please sign in" error caused
+by a previous attempt at restoring video playback on large instances.
+
+In the preferences, a new option allows for control of video preload. When enabled, this option
+tells the browser to load the video as soon as the page is loaded (this used to be the default).
+When disabled, the video starts loading only when the "play" button is pressed.
+
+New interface languages available: Bulgarian, Welsh and Lombard
+
+New dependency required: `tzdata`.
+
+An HTTP proxy can be configured directly in Invidious, if needed. \
+**NOTE:** In that case, it is recommended to comment out `force_resolve`.
+
+
+### New features & important changes
+
+#### For users
+
+* Channels: Fix "Youtube API returned error 400" error preventing channel pages from loading
+* Channels: Shorts can now be sorted by "newest", "oldest" and "popular"
+* Preferences: Addition of the new "preload" option
+* New interface languages available: Bulgarian, Welsh and Lombard
+* Added "Filipino (auto-generated)" to the list of caption languages available
+* Lots of new translations from Weblate
+
+#### For instance owners
+
+* Allow the configuration of an HTTP proxy to talk to Youtube
+* Invidious tries to reconnect to `inv_sig_helper` if the socket is closed
+* The instance list is downloaded in the background to improve redirection speed
+* New `colorize_logs` option makes each log level a different color
+
+#### For developpers
+
+* `/api/v1/channels/{id}/shorts` now supports the `sort-by` parameter with the following values:
+  `newest`, `oldest` and `popular`
+* Older `/api/v1/channels/xyz/{id}` (tab name before UCID) were removed
+* API/Search: New video metadata available: `isNew`, `is4k`, `is8k`, `isVr180`, `isVr360`,
+  `is3d` and `hasCaptions`
+
+### Bugs fixed
+
+#### User-side
+
+* Channels: The second page of shorts now loads as expected
+* Channels: Fixed intermittent empty "playlists" tab
+* Search: Fixed `youtu.be` URLs not being properly redirected to the watch page
+* Fixed `DB::MappingException` error on the subscriptions feed (due to missing `tzdata` in docker)
+* Switching to another instance is much faster
+* Fixed an "invalid byte sequence" error when subscribing to a playlist
+* Videos: Playback URLs were sometimes broken when cached and `inv_sig_helper` was used
+
+#### For instance owners
+
+* Fix `force_resolve` being ignored in some cases
+
+#### API
+
+* API/Videos: Fixed `live_now` and `premiere_timestamp` sometimes not having the right values
+
+
 ### Full list of pull requests merged since the last release (newest first)
 
+* API: Add "sort_by" parameter to channels/shorts endpoint ([#5071], thanks @iBicha)
+* Docker: Install tzdata in Dockerfile ([#5070], by @SamantazFox)
+* Videos: Stop using TVHTML5_SIMPLY_EMBEDDED_PLAYER ([#5063], thanks @unixfox)
+* Routing: Deprecate old channel API routes ([#5045], by @SamantazFox)
+* Videos: use WEB client instead of WEB CREATOR ([#4984], thanks @unixfox)
+* Parsers: Fix parsing live_now and premiere_timestamp ([#4934], thanks @absidue)
+* Stale bot updates ([#5060], thanks @syeopite)
+* Channels: Fix "Youtube API returned error 400" ([#5059], by @SamantazFox)
+* Channels: Fix for live videos ([#5027], thanks @iBicha)
+* Locales: Add Bulgarian, Welsh and Lombard to the list ([#5046], by @SamantazFox)
+* Shards: Update database dependencies ([#5034], by @SamantazFox)
+* Logger: Add color support for different log levels ([#4931], thanks @Fijxu)
+* Fix named arg syntax when passing force_resolve ([#4754], thanks @syeopite)
+* Use make_client instead of calling HTTP::Client ([#4709], thanks @syeopite)
+* Add "Filipino (auto-generated)" to the list of caption languages ([#4995], by @SamantazFox)
+* Makefile: Add MT option to enable the 'preview_mt' flag ([#4993], by @SamantazFox)
+* SigHelper: Reconnect to signature helper ([#4991], thanks @Fijxu)
+* Fix player menus hiding onHover ready ([#4750], thanks @giacomocerquone)
+* Use connection pools when requesting images from YouTube ([#4326], thanks @syeopite)
+* Add support for using Invidious through a HTTP Proxy ([#4270], thanks @syeopite)
 * Search: Fix 'youtu.be' URLs in sanitizer ([#4894], by @SamantazFox)
 * Ameba: Disable Style/RedundantNext rule ([#4888], thanks @syeopite)
 * Playlists: Fix 'invalid byte sequence' error when subscribing ([#4887], thanks @DmitrySandalov)
@@ -22,7 +112,12 @@
 
 [#4122]: https://github.com/iv-org/invidious/pull/4122
 [#4193]: https://github.com/iv-org/invidious/pull/4193
+[#4270]: https://github.com/iv-org/invidious/pull/4270
+[#4326]: https://github.com/iv-org/invidious/pull/4326
 [#4652]: https://github.com/iv-org/invidious/pull/4652
+[#4709]: https://github.com/iv-org/invidious/pull/4709
+[#4750]: https://github.com/iv-org/invidious/pull/4750
+[#4754]: https://github.com/iv-org/invidious/pull/4754
 [#4850]: https://github.com/iv-org/invidious/pull/4850
 [#4862]: https://github.com/iv-org/invidious/pull/4862
 [#4863]: https://github.com/iv-org/invidious/pull/4863
@@ -32,7 +127,22 @@
 [#4923]: https://github.com/iv-org/invidious/pull/4923
 [#4928]: https://github.com/iv-org/invidious/pull/4928
 [#4930]: https://github.com/iv-org/invidious/pull/4930
+[#4931]: https://github.com/iv-org/invidious/pull/4931
+[#4934]: https://github.com/iv-org/invidious/pull/4934
 [#4942]: https://github.com/iv-org/invidious/pull/4942
+[#4984]: https://github.com/iv-org/invidious/pull/4984
+[#4991]: https://github.com/iv-org/invidious/pull/4991
+[#4993]: https://github.com/iv-org/invidious/pull/4993
+[#4995]: https://github.com/iv-org/invidious/pull/4995
+[#5027]: https://github.com/iv-org/invidious/pull/5027
+[#5034]: https://github.com/iv-org/invidious/pull/5034
+[#5045]: https://github.com/iv-org/invidious/pull/5045
+[#5046]: https://github.com/iv-org/invidious/pull/5046
+[#5059]: https://github.com/iv-org/invidious/pull/5059
+[#5060]: https://github.com/iv-org/invidious/pull/5060
+[#5063]: https://github.com/iv-org/invidious/pull/5063
+[#5070]: https://github.com/iv-org/invidious/pull/5070
+[#5071]: https://github.com/iv-org/invidious/pull/5071
 
 
 ## v2.20240825.2 (2024-08-26)

Makefile

@@ -7,6 +7,11 @@ STATIC   := 0
 
 NO_DBG_SYMBOLS := 0
 
+# Enable multi-threading.
+# Warning: Experimental feature!!
+# invidious is not stable when MT is enabled.
+MT := 0
+
 
 FLAGS ?=
 
@@ -19,6 +24,10 @@ ifeq ($(STATIC), 1)
   FLAGS += --static
 endif
 
+ifeq ($(MT), 1)
+  FLAGS += -Dpreview_mt
+endif
+
 
 ifeq ($(NO_DBG_SYMBOLS), 1)
   FLAGS += --no-debug

assets/css/player.css

@@ -68,6 +68,7 @@
 
 .video-js.player-style-youtube .vjs-menu-button-popup .vjs-menu {
   margin-bottom: 2em;
+  padding-top: 2em
 }
 
 .video-js.player-style-youtube .vjs-progress-control .vjs-progress-holder, .video-js.player-style-youtube .vjs-progress-control {height: 5px;

config/config.example.yml

@@ -173,6 +173,17 @@ https_only: false
 ##
 #force_resolve:
 
+##
+## Configuration for using a HTTP proxy
+##
+## If unset, then no HTTP proxy will be used.
+## 
+http_proxy:
+  user:
+  password:
+  host:
+  port:
+
 
 ##
 ## Use Innertube's transcripts API instead of timedtext for closed captions
@@ -222,6 +233,17 @@ https_only: false
 ##
 #log_level: Info
 
+##
+## Enables colors in logs. Useful for debugging purposes
+## This is overridden if "-k" or "--colorize"
+## are passed on the command line.
+## Colors are also disabled if the environment variable
+## NO_COLOR is present and has any value
+##
+## Accepted values: true, false
+## Default: true
+##
+#colorize_logs: false
 
 # -----------------------------
 #  Features

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM crystallang/crystal:1.12.1-alpine AS builder
+FROM crystallang/crystal:1.12.2-alpine AS builder
 
 RUN apk add --no-cache sqlite-static yaml-static
 
@@ -32,8 +32,8 @@ RUN if [[ "${release}" == 1 ]] ; then \
         --link-flags "-lxml2 -llzma"; \
     fi
 
-FROM alpine:3.18
-RUN apk add --no-cache rsvg-convert ttf-opensans tini
+FROM alpine:3.20
+RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata
 WORKDIR /invidious
 RUN addgroup -g 1000 -S invidious && \
     adduser -u 1000 -S invidious -G invidious

docker/Dockerfile.arm64

@@ -1,5 +1,6 @@
-FROM alpine:3.19 AS builder
-RUN apk add --no-cache 'crystal=1.10.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-static zlib-static openssl-libs-static openssl-dev musl-dev xz-static
+FROM alpine:3.20 AS builder
+RUN apk add --no-cache 'crystal=1.12.2-r0' shards sqlite-static yaml-static yaml-dev libxml2-static \
+       zlib-static openssl-libs-static openssl-dev musl-dev xz-static
 
 ARG release
 
@@ -32,8 +33,8 @@ RUN if [[ "${release}" == 1 ]] ; then \
         --link-flags "-lxml2 -llzma"; \
     fi
 
-FROM alpine:3.18
-RUN apk add --no-cache rsvg-convert ttf-opensans tini
+FROM alpine:3.20
+RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata
 WORKDIR /invidious
 RUN addgroup -g 1000 -S invidious && \
     adduser -u 1000 -S invidious -G invidious

locales/en-US.json

@@ -286,6 +286,7 @@
     "Esperanto": "Esperanto",
     "Estonian": "Estonian",
     "Filipino": "Filipino",
+    "Filipino (auto-generated)": "Filipino (auto-generated)",
     "Finnish": "Finnish",
     "French": "French",
     "French (auto-generated)": "French (auto-generated)",

shard.lock

@@ -10,16 +10,20 @@ shards:
 
   backtracer:
     git: https://github.com/sija/backtracer.cr.git
-    version: 1.2.1
+    version: 1.2.2
 
   db:
     git: https://github.com/crystal-lang/crystal-db.git
-    version: 0.10.1
+    version: 0.13.1
 
   exception_page:
     git: https://github.com/crystal-loot/exception_page.git
     version: 0.2.2
 
+  http_proxy:
+    git: https://github.com/mamantoha/http_proxy.git
+    version: 0.10.3
+
   kemal:
     git: https://github.com/kemalcr/kemal.git
     version: 1.1.2
@@ -30,7 +34,7 @@ shards:
 
   pg:
     git: https://github.com/will/crystal-pg.git
-    version: 0.24.0
+    version: 0.28.0
 
   protodec:
     git: https://github.com/iv-org/protodec.git
@@ -42,9 +46,9 @@ shards:
 
   spectator:
     git: https://github.com/icy-arctic-fox/spectator.git
-    version: 0.10.4
+    version: 0.10.6
 
   sqlite3:
     git: https://github.com/crystal-lang/crystal-sqlite3.git
-    version: 0.18.0
+    version: 0.21.0