diff --git a/CHANGELOG.md b/CHANGELOG.md
index e8f679e1..b8889f27 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,8 @@
 
 - Fix signature has expired error if payload is a string [#555](https://github.com/jwt/ruby-jwt/pull/555) - [@GobinathAL](https://github.com/GobinathAL).
 - Fix key base equality and spaceship operators [#569](https://github.com/jwt/ruby-jwt/pull/569) - [@magneland](https://github.com/magneland).
+- Drop dependency to base64 gem [#578](https://github.com/jwt/ruby-jwt/pull/578) - [@anakinj](https://github.com/anakinj).
+- Deprecation warning for decoding content not compliant with RFC 4648 [#578](https://github.com/jwt/ruby-jwt/pull/578) - [@anakinj](https://github.com/anakinj).
 - Your contribution here
 
 ## [v2.7.1](https://github.com/jwt/ruby-jwt/tree/v2.8.0) (2023-06-09)
diff --git a/lib/jwt/base64.rb b/lib/jwt/base64.rb
index e69808b1..c2f43915 100644
--- a/lib/jwt/base64.rb
+++ b/lib/jwt/base64.rb
@@ -1,18 +1,32 @@
 # frozen_string_literal: true
 
-require 'base64'
-
 module JWT
-  # Base64 helpers
+  # Base64 encoding and decoding
   class Base64
     class << self
+      # Encode a string with URL-safe Base64 complying with RFC 4648 (not padded).
       def url_encode(str)
-        ::Base64.encode64(str).tr('+/', '-_').gsub(/[\n=]/, '')
+        encoded = [str].pack('m0')
+        encoded.chomp!('==') || encoded.chomp!('=')
+        encoded.tr!('+/', '-_')
+        encoded
       end
 
+      # Decode a string with URL-safe Base64 complying with RFC 4648.
+      # Deprecated support for RFC 2045 remains for now. ("All line breaks or other characters not found in Table 1 must be ignored by decoding software")
       def url_decode(str)
-        str += '=' * (4 - str.length.modulo(4))
-        ::Base64.decode64(str.tr('-_', '+/'))
+        if !str.end_with?('=') && str.length % 4 != 0
+          str = str.ljust((str.length + 3) & ~3, '=')
+          str.tr!('-_', '+/')
+        else
+          str = str.tr('-_', '+/')
+        end
+        str.unpack1('m0')
+      rescue ArgumentError => e
+        raise unless e.message == 'invalid base64'
+
+        warn('[DEPRECATION] Invalid base64 input detected, could be because of invalid padding, trailing whitespaces or newline chars. Graceful handling of invalid input will be dropped in the next major version of ruby-jwt')
+        str.unpack1('m')
       end
     end
   end