Merge pull request #429 from MortezaBashsiz/py-progressbar

added progress bar and improved logging

.gitignore

@@ -10,6 +10,7 @@ python/__pycache__/
 python/*/__pycache__/
 python/.vscode/
 python/*/log/
+python/.xray-configs
 .tmp
 golang/CFScanner
 windows/*.rsuser

python/README.md

@@ -212,6 +212,9 @@ Contributors names and contact info
   * Fixed a bug in custom config template
 * 1.1.0 
   * Added random sampling
+* 1.2.0
+  * Added progress bar
+  * Improved logging
 
 [python]: https://img.shields.io/badge/-Python-3776AB?logo=python&logoColor=white
-[version]: https://img.shields.io/badge/Version-1.1.0-blue
+[version]: https://img.shields.io/badge/Version-1.2.0-blue

python/args/parser.py

@@ -1,9 +1,9 @@
 import argparse
 
-from report.clog import CLogger
 from report.print import color_text
+from rich.console import Console
 
-logger = CLogger("args")
+console = Console()
 
 
 def _title(text):
@@ -217,11 +217,12 @@ def formatter(prog): return argparse.HelpFormatter(
             parsed_args.sample_size = float(parsed_args.sample_size)
         elif parsed_args.sample_size >= 1:
             if parsed_args.sample_size % 1 > 0.000001:
-                logger.warn(
-                    f"Sample size rounded to integer value: {round(parsed_args.sample_size)}"
+                console.log(
+                    f"[yellow]Sample size rounded to integer value: {round(parsed_args.sample_size)}[/yellow]"
                 )
             parsed_args.sample_size = round(parsed_args.sample_size)
         else:
-            raise ValueError(color_text("Sample size must be a positive number.", rgb=(255, 0, 0)))
+            raise ValueError(color_text(
+                "Sample size must be a positive number.", rgb=(255, 0, 0)))
 
     return parsed_args

python/args/testconfig.py

@@ -2,14 +2,13 @@
 import json
 import os
 
-from report.clog import CLogger
 from utils.exceptions import BinaryNotFoundError, TemplateReadError
 from utils.os import detect_system
 from utils.requests import download_file
 from xray import templates
 from xray.binary import download_binary
 
-logger = CLogger("testconfig")
+from utils.exceptions import *
 
 PATH = os.path.dirname(os.path.abspath(__file__))
 PARENT_PATH = os.path.dirname(PATH)
@@ -47,10 +46,10 @@ def from_args(cls, args: argparse.Namespace):
                     (file_content["path"].lstrip("/"))
 
         if args.template_path is None:
-            test_config.custom_template = False # user did not provide a custom template
+            test_config.custom_template = False  # user did not provide a custom template
             test_config.proxy_config_template = templates.vmess_ws_tls
         else:
-            test_config.custom_template = True # user provided a custom template 
+            test_config.custom_template = True  # user provided a custom template
             try:
                 with open(args.template_path, "r") as infile:
                     test_config.proxy_config_template = infile.read()
@@ -78,7 +77,7 @@ def from_args(cls, args: argparse.Namespace):
         test_config.max_ul_latency = args.max_ul_latency
         test_config.n_tries = args.n_tries
         test_config.novpn = args.no_vpn
-        
+
         test_config.sample_size = args.sample_size
 
         system_info = detect_system()
@@ -92,9 +91,12 @@ def from_args(cls, args: argparse.Namespace):
                 )
             test_config.binpath = args.binpath
         else:
-            test_config.binpath = download_binary(
-                system_info=system_info,
-                bin_dir=PARENT_PATH
-            )
+            try:
+                test_config.binpath = download_binary(
+                    system_info=system_info,
+                    bin_dir=PARENT_PATH
+                )
+            except Exception as e:
+                raise BinaryDownloadError(str(e))
 
         return test_config

python/cfscanner.py

@@ -8,96 +8,176 @@
 
 from args.parser import parse_args
 from args.testconfig import TestConfig
-from report.clog import CLogger
-from report.print import print_ok
+from rich import print as rprint
+from rich.console import Console
+from rich.progress import Progress
 from speedtest.conduct import test_ip
 from speedtest.tools import mean_jitter
 from subnets import cidr_to_ip_list, get_num_ips_in_cidr, read_cidrs
-from utils.exceptions import BinaryNotFoundError, TemplateReadError
+from utils.exceptions import *
 from utils.os import create_dir
 
-log = CLogger("cfscanner-python")
+console = Console()
 
 SCRIPTDIR = os.path.dirname(os.path.realpath(__file__))
-CONFIGDIR = f"{SCRIPTDIR}/../config"
-RESULTDIR = f"{SCRIPTDIR}/../result"
+CONFIGDIR = f"{SCRIPTDIR}/.xray-configs"
+RESULTDIR = f"{SCRIPTDIR}/result"
 START_DT_STR = datetime.now().strftime(r"%Y%m%d_%H%M%S")
 INTERIM_RESULTS_PATH = os.path.join(RESULTDIR, f'{START_DT_STR}_result.csv')
 
 
 if __name__ == "__main__":
+    console = Console()
+
     args = parse_args()
 
     if not args.no_vpn:
-        create_dir(CONFIGDIR)
+        with console.status(f"[green]Creating config dir \"{CONFIGDIR}\"[/green]"):
+            try:
+                create_dir(CONFIGDIR)
+            except Exception as e:
+                console.log("[red]Could not created config directory[/red]")
+                exit(1)
+        console.log(f"[blue]Config directory created \"{CONFIGDIR}\"[/blue]")
         configFilePath = args.config_path
 
-    create_dir(RESULTDIR)
+    with console.status(f"[green]Creating results directory \"{RESULTDIR}\"[/green]"):
+        try:
+            create_dir(RESULTDIR)
+        except Exception as e:
+            console.log("[red]Could not create results directory[/red]")
+    console.log(f"[blue]Results directory created \"{RESULTDIR}\"[/blue]")
 
     # create empty result file
-    with open(INTERIM_RESULTS_PATH, "w") as empty_file:
-        titles = [
-            "ip", "avg_download_speed", "avg_upload_speed",
-            "avg_download_latency", "avg_upload_latency",
-            "avg_download_jitter", "avg_upload_jitter"
-        ]
-        titles += [f"download_speed_{i+1}" for i in range(args.n_tries)]
-        titles += [f"upload_speed_{i+1}" for i in range(args.n_tries)]
-        titles += [f"download_latency_{i+1}" for i in range(args.n_tries)]
-        titles += [f"upload_latency_{i+1}" for i in range(args.n_tries)]
-        empty_file.write(",".join(titles) + "\n")
+    with console.status(f"[green]Creating empty result file {INTERIM_RESULTS_PATH}[/green]"):
+        try:
+            with open(INTERIM_RESULTS_PATH, "w") as empty_file:
+                titles = [
+                    "ip", "avg_download_speed", "avg_upload_speed",
+                    "avg_download_latency", "avg_upload_latency",
+                    "avg_download_jitter", "avg_upload_jitter"
+                ]
+                titles += [f"download_speed_{i+1}" for i in range(args.n_tries)]
+                titles += [f"upload_speed_{i+1}" for i in range(args.n_tries)]
+                titles += [f"download_latency_{i+1}" for i in range(args.n_tries)]
+                titles += [f"upload_latency_{i+1}" for i in range(args.n_tries)]
+                empty_file.write(",".join(titles) + "\n")
+        except Exception as e:
+            console.log(f"[red]Could not create empty result file:\n\"{INTERIM_RESULTS_PATH}\"[/red]")
 
     threadsCount = args.threads
 
     if args.subnets:
-        cidr_list = read_cidrs(args.subnets)
+        with console.status("[green]Reading subnets from \"{args.subnets}\"[/green]"):
+            try:
+                cidr_list = read_cidrs(args.subnets)
+            except SubnetsReadError as e:
+                console.log(f"[red]Could not read subnets. {e}[/red]")
+                exit(1)
+            except Exception as e:
+                console.log(f"Unknown error in reading subnets: {e}")
+                exit(1)
+        console.log(
+            f"[blue]Subnets successfully read from \"{args.subnets}\"[/blue]")
     else:
-        cidr_list = read_cidrs(
-            "https://raw.githubusercontent.com/MortezaBashsiz/CFScanner/main/bash/cf.local.iplist"
+        subnets_default_address = "https://raw.githubusercontent.com/MortezaBashsiz/CFScanner/main/config/cf.local.iplist"
+        console.log(
+            f"[blue]Subnets not provided. Default address will be used:\n\"{subnets_default_address}\"[/blue]"
         )
-
+        with console.status(f"[green]Retrieving subnets from \"{subnets_default_address}\"[/green]"):
+            try:
+                cidr_list = read_cidrs(
+                    "https://raw.githubusercontent.com/MortezaBashsiz/CFScanner/main/config/cf.local.iplist"
+                )
+            except SubnetsReadError as e:
+                console.log(f"[red]Could not read subnets. {e}[/red]")
+                exit(1)
+            except Exception as e:
+                console.log(f"Unknown error in reading subnets: {e}")
+                exit(1) 
     try:
         test_config = TestConfig.from_args(args)
     except TemplateReadError:
-        log.error("Could not read template from file.")
+        console.log(
+            f"[red]Could not read template from file \"{args.template_path}\"[/red]")
         exit(1)
     except BinaryNotFoundError:
-        log.error("Could not find xray/v2ray binary.", args.binpath)
+        console.log(
+            f"[red]Could not find xray/v2ray binary from path \"{args.binpath}\"[/red]")
         exit(1)
     except Exception as e:
-        log.error("Unknown error while reading template.")
-        log.exception(e)
+        console.print_exception()
         exit(1)
 
-    n_total_ips = sum(get_num_ips_in_cidr(cidr, sample_size=test_config.sample_size) for cidr in cidr_list)
-    log.info(f"Starting to scan {n_total_ips} ips...")
-
-    big_ip_list = [ip for cidr in cidr_list for ip in cidr_to_ip_list(cidr, sample_size=test_config.sample_size)]
-
-    with multiprocessing.Pool(processes=threadsCount) as pool:
-        for res in pool.imap(partial(test_ip, test_config=test_config, config_dir=CONFIGDIR), big_ip_list):
-            if res:
-                down_mean_jitter = mean_jitter(res["download"]["latency"])
-                up_mean_jitter = mean_jitter(
-                    res["upload"]["latency"]) if test_config.do_upload_test else -1
-                mean_down_speed = statistics.mean(res["download"]["speed"])
-                mean_up_speed = statistics.mean(
-                    res["upload"]["speed"]) if test_config.do_upload_test else -1
-                mean_down_latency = statistics.mean(res["download"]["latency"])
-                mean_up_latency = statistics.mean(
-                    res["upload"]["latency"]) if test_config.do_upload_test else -1
-
-                print_ok(scan_result=res)
-
-                with open(INTERIM_RESULTS_PATH, "a") as outfile:
-                    res_parts = [
-                        res["ip"], mean_down_speed, mean_up_speed,
-                        mean_down_latency, mean_up_latency,
-                        down_mean_jitter, up_mean_jitter
-                    ]
-                    res_parts += res["download"]["speed"]
-                    res_parts += res["upload"]["speed"]
-                    res_parts += res["download"]["latency"]
-                    res_parts += res["upload"]["latency"]
-
-                    outfile.write(",".join(map(str, res_parts)) + "\n")
+    n_total_ips = sum(get_num_ips_in_cidr(
+        cidr,
+        sample_size=test_config.sample_size
+    ) for cidr in cidr_list)
+    console.log(f"[blue]Starting to scan {n_total_ips} ips...[/blue]")
+
+    cidr_ip_lists = [
+        cidr_to_ip_list(
+            cidr,
+            sample_size=test_config.sample_size)
+        for cidr in cidr_list
+    ]
+    big_ip_list = [(ip, cidr) for cidr, ip_list in zip(
+        cidr_list, cidr_ip_lists) for ip in ip_list]
+
+    cidr_scanned_ips = {cidr: 0 for cidr in cidr_list}
+
+    cidr_prog_tasks = dict()
+
+    with Progress() as progress:
+        all_ips_task = progress.add_task(
+            f"all subnets - {n_total_ips} ips", total=n_total_ips)
+
+        with multiprocessing.Pool(processes=threadsCount) as pool:
+            try:
+                for res in pool.imap(partial(test_ip, test_config=test_config, config_dir=CONFIGDIR), big_ip_list):
+                    progress.update(all_ips_task, advance=1)
+                    if cidr_scanned_ips[res.cidr] == 0:
+                        n_ips_cidr = get_num_ips_in_cidr(
+                            res.cidr, sample_size=test_config.sample_size)
+                        cidr_prog_tasks[res.cidr] = progress.add_task(
+                            f"{res.cidr} - {n_ips_cidr} ips", total=n_ips_cidr)
+                    progress.update(cidr_prog_tasks[res.cidr], advance=1)
+
+                    if res.is_ok:
+                        down_mean_jitter = mean_jitter(
+                            res.result["download"]["latency"])
+                        up_mean_jitter = mean_jitter(
+                            res.result["upload"]["latency"]) if test_config.do_upload_test else -1
+                        mean_down_speed = statistics.mean(
+                            res.result["download"]["speed"])
+                        mean_up_speed = statistics.mean(
+                            res.result["upload"]["speed"]) if test_config.do_upload_test else -1
+                        mean_down_latency = statistics.mean(
+                            res.result["download"]["latency"])
+                        mean_up_latency = statistics.mean(
+                            res.result["upload"]["latency"]) if test_config.do_upload_test else -1
+
+                        rprint(res.message)
+
+                        with open(INTERIM_RESULTS_PATH, "a") as outfile:
+                            res_parts = [
+                                res.ip, mean_down_speed, mean_up_speed,
+                                mean_down_latency, mean_up_latency,
+                                down_mean_jitter, up_mean_jitter
+                            ]
+                            res_parts += res.result["download"]["speed"]
+                            res_parts += res.result["upload"]["speed"]
+                            res_parts += res.result["download"]["latency"]
+                            res_parts += res.result["upload"]["latency"]
+
+                            outfile.write(",".join(map(str, res_parts)) + "\n")
+                    else:
+                        rprint(res.message)
+
+                    cidr_scanned_ips[res.cidr] += 1
+                    if cidr_scanned_ips[res.cidr] == get_num_ips_in_cidr(res.cidr, sample_size=test_config.sample_size):
+                        progress.remove_task(cidr_prog_tasks[res.cidr])
+            except StartProxyServiceError as e:
+                progress.stop()
+                console.log(f"[red]{e}[/red]")
+                pool.terminate()