retry with different url params when --bypass-cdn-image-compression…

… is enabled

wikiteam3/dumpgenerator/dump/image/image.py

@@ -1,6 +1,5 @@
 import datetime
 import os
-import random
 import re
 import shutil
 import sys
@@ -87,7 +86,7 @@ def modify_params(params: Optional[Dict] = None) -> Dict:
                 # bypass Cloudflare Polish (image optimization)
                 # <https://developers.cloudflare.com/images/polish/>
                 params["_wiki_t"] = int(time.time()*1000)
-                params[f"_wiki_{random.randint(10,99)}_"] = "random"
+                params["_wikiteam3_nocdn"] = "init_req" # this value will be changed on hard retry
 
             return params
 

wikiteam3/utils/monkey_patch.py

@@ -1,6 +1,5 @@
 import os
 import ssl
-import sys
 import time
 from typing import Optional
 import warnings
@@ -89,7 +88,7 @@ class SessionMonkeyPatch:
     Monkey patch `requests.Session.send`
     """
     hijacked = False
-    def __init__(self,*, session: requests.Session, config: Config,
+    def __init__(self,*, session: requests.Session, config: Optional[Config]=None,
                  add_delay: bool=False, delay_msg: Optional[str]=None,
                  hard_retries: int=0,
                  free_timeout_connections: bool=True, vaild_lft_sec: int=60 * 3
@@ -127,7 +126,7 @@ def hijack(self):
         # Monkey patch `requests.Session.send`
         self.old_send_method = self.session.send
 
-        def new_send(request, **kwargs):
+        def new_send(request: requests.PreparedRequest, **kwargs):
             hard_retries_left = self.hard_retries + 1
             if hard_retries_left <= 0:
                 raise ValueError('hard_retries must be positive')
@@ -149,6 +148,17 @@ def new_send(request, **kwargs):
                         raise
 
                     print('Hard retry... (%d), due to: %s' % (hard_retries_left, e))
+
+                    # if --bypass-cdn-image-compression is enabled, retry with different url
+                    assert isinstance(request.url, str)
+                    if '_wikiteam3_nocdn=' in request.url:
+                        request.url = request.url.replace('_wikiteam3_nocdn=init_req', f'_wikiteam3_nocdn=retry_{hard_retries_left}')
+                        request.url = request.url.replace(
+                            f'_wikiteam3_nocdn=retry_{hard_retries_left + 1}',
+                            f'_wikiteam3_nocdn=retry_{hard_retries_left}'
+                            )
+                        print('--bypass-cdn-image-compression: change url to', request.url, 'on hard retry...')
+
                     time.sleep(3)
 
         self.session.send = new_send # type: ignore