Fix line ending handling in reverse_readfile/readline across OS, and not skipping empty lines

.github/workflows/test.yml

@@ -6,6 +6,7 @@ jobs:
   build:
     strategy:
       max-parallel: 20
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-14, windows-latest]
         python-version: ["3.9", "3.x"]
@@ -29,6 +30,6 @@ jobs:
         run: pytest --cov=monty --cov-report html:coverage_reports tests
 
       - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

benchmark/benchmark.py

@@ -0,0 +1,175 @@
+"""Utility script for monty reverse reader speed benchmark.
+- File of various sizes.
+- Find the last line, 75 % line and 50 % line, and compare with reading from forwards.
+"""
+
+from __future__ import annotations
+
+import os
+import platform
+import sys
+import time
+
+from monty.io import reverse_readfile, reverse_readline
+
+# Test config
+FILE_SIZES_MB = (1, 10, 100, 500, 1000, 5000)
+
+
+def create_test_file(file_path, target_size_mb):
+    """Creates a text file with lines until the target size is reached."""
+    target_size = target_size_mb * 1024 * 1024  # Convert MB to bytes
+    line_number = 1
+
+    start = time.perf_counter()
+
+    # Create a list of lines and concatenate them at the end
+    lines = []
+    total_bytes_written = 0
+
+    while total_bytes_written < target_size:
+        line = f"This is line number {line_number}\n"
+        line_bytes = line.encode("utf-8")
+
+        if total_bytes_written + len(line_bytes) > target_size:
+            break
+
+        lines.append(line)
+        total_bytes_written += len(line_bytes)
+        line_number += 1
+
+    with open(file_path, "wb") as f:
+        f.write("".join(lines).encode("utf-8"))
+
+    last_time = time.perf_counter() - start
+
+    total_lines = line_number - 1
+    print(
+        f"Test file of size {target_size_mb} MB created with {total_lines} lines, time used {last_time:.2f} seconds."
+    )
+    return total_lines
+
+
+def print_separator(title: str):
+    print(f"{title}".center(80, "="))
+    print("")
+
+
+def test_builtin_readline(file_path, total_lines):
+    """Test built-in readline function."""
+    start = time.perf_counter()
+    with open(file_path, "r") as f:
+        for _ in range(total_lines):
+            _last_line = f.readline()
+    last_time = time.perf_counter() - start
+
+    line_75_idx = int(0.75 * total_lines)
+    line_50_idx = int(0.5 * total_lines)
+
+    start = time.perf_counter()
+    with open(file_path, "r") as f:
+        for _ in range(line_75_idx + 1):
+            _line_75 = f.readline()
+    time_75 = time.perf_counter() - start
+
+    start = time.perf_counter()
+    with open(file_path, "r") as f:
+        for _ in range(line_50_idx + 1):
+            _line_50 = f.readline()
+    time_50 = time.perf_counter() - start
+
+    print_separator("Built-in readline")
+    print(f"Last line {total_lines} read, time taken: {last_time:.8f} s.")
+    print(f"75% line {line_75_idx} read, time taken: {time_75:.8f} s.")
+    print(f"50% line {line_50_idx} read, time taken: {time_50:.8f} s.")
+    print_separator("End of Built-in readline")
+
+    return last_time, time_75, time_50
+
+
+def test_reverse_readline(file_path, total_lines):
+    """Test reverse_readline function."""
+    start = time.perf_counter()
+    with open(file_path, "r") as f:
+        _last_line = next(reverse_readline(f))
+    last_time = time.perf_counter() - start
+
+    line_75_idx = int(0.75 * total_lines)
+    line_50_idx = int(0.5 * total_lines)
+
+    start = time.perf_counter()
+    with open(file_path, "r") as f:
+        for idx, _line in enumerate(reverse_readline(f), 1):
+            if idx == total_lines - line_75_idx:
+                break
+    time_75 = time.perf_counter() - start
+
+    start = time.perf_counter()
+    with open(file_path, "r") as f:
+        for idx, _line in enumerate(reverse_readline(f), 1):
+            if idx == total_lines - line_50_idx:
+                break
+    time_50 = time.perf_counter() - start
+
+    print_separator("reverse_readline")
+    print(f"Last line {total_lines} read, time taken: {last_time:.8f} s.")
+    print(f"75% line {line_75_idx} read, time taken: {time_75:.8f} s.")
+    print(f"50% line {line_50_idx} read, time taken: {time_50:.8f} s.")
+    print_separator("End of reverse_readline")
+
+    return last_time, time_75, time_50
+
+
+def test_reverse_readfile(file_path, total_lines):
+    """Test reverse_readfile function."""
+    start = time.perf_counter()
+    _last_line = next(reverse_readfile(file_path))
+    last_time = time.perf_counter() - start
+
+    line_75_idx = int(0.75 * total_lines)
+    line_50_idx = int(0.5 * total_lines)
+
+    start = time.perf_counter()
+    for idx, _line in enumerate(reverse_readfile(file_path), 1):
+        if idx == total_lines - line_75_idx:
+            break
+    time_75 = time.perf_counter() - start
+
+    start = time.perf_counter()
+    for idx, _line in enumerate(reverse_readfile(file_path), 1):
+        if idx == total_lines - line_50_idx:
+            break
+    time_50 = time.perf_counter() - start
+
+    print_separator("reverse_readfile")
+    print(f"Last line {total_lines} read, time taken: {last_time:.8f} s.")
+    print(f"75% line {line_75_idx} read, time taken: {time_75:.8f} s.")
+    print(f"50% line {line_50_idx} read, time taken: {time_50:.8f} s.")
+    print_separator("End of reverse_readfile")
+
+    return last_time, time_75, time_50
+
+
+def run_benchmark(file_size_mb):
+    """Run benchmark for all test functions."""
+    print_separator(f"Benchmarking file size: {file_size_mb} MB")
+
+    test_file = f"test_file_{file_size_mb}MB.txt"
+    total_lines = create_test_file(test_file, file_size_mb)
+
+    test_builtin_readline(test_file, total_lines)
+    test_reverse_readline(test_file, total_lines)
+    test_reverse_readfile(test_file, total_lines)
+
+    os.remove(test_file)
+
+
+if __name__ == "__main__":
+    # Show OS info
+    os_info = platform.platform()
+    python_version = sys.version.split()[0]
+    print(f"\nRunning on OS: {os_info}, Python {python_version}")
+
+    # Run benchmark for each file size
+    for file_size_mb in FILE_SIZES_MB:
+        run_benchmark(file_size_mb)

benchmark/develop-ubuntu2204.txt

@@ -0,0 +1,145 @@
+
+Running on OS: Linux-5.15.153.1-microsoft-standard-WSL2-x86_64-with-glibc2.35, Python 3.12.5
+==========================Benchmarking file size: 1 MB==========================
+
+Test file of size 1 MB created with 40757 lines, time used 0.01 seconds.
+===============================Built-in readline================================
+
+Last line 40757 read, time taken: 0.00299960 s.
+75% line 30567 read, time taken: 0.00225260 s.
+50% line 20378 read, time taken: 0.00147340 s.
+============================End of Built-in readline============================
+
+================================reverse_readline================================
+
+Last line 40757 read, time taken: 0.00215450 s.
+75% line 30567 read, time taken: 0.00299860 s.
+50% line 20378 read, time taken: 0.00347230 s.
+============================End of reverse_readline=============================
+
+================================reverse_readfile================================
+
+Last line 40757 read, time taken: 0.00007200 s.
+75% line 30567 read, time taken: 0.00393440 s.
+50% line 20378 read, time taken: 0.00788680 s.
+============================End of reverse_readfile=============================
+
+=========================Benchmarking file size: 10 MB==========================
+
+Test file of size 10 MB created with 392476 lines, time used 0.11 seconds.
+===============================Built-in readline================================
+
+Last line 392476 read, time taken: 0.02891790 s.
+75% line 294357 read, time taken: 0.02203730 s.
+50% line 196238 read, time taken: 0.01453720 s.
+============================End of Built-in readline============================
+
+================================reverse_readline================================
+
+Last line 392476 read, time taken: 0.00010690 s.
+75% line 294357 read, time taken: 0.08467931 s.
+50% line 196238 read, time taken: 0.16308102 s.
+============================End of reverse_readline=============================
+
+================================reverse_readfile================================
+
+Last line 392476 read, time taken: 0.00007110 s.
+75% line 294357 read, time taken: 0.03886231 s.
+50% line 196238 read, time taken: 0.07676581 s.
+============================End of reverse_readfile=============================
+
+=========================Benchmarking file size: 100 MB=========================
+
+Test file of size 100 MB created with 3784596 lines, time used 1.11 seconds.
+===============================Built-in readline================================
+
+Last line 3784596 read, time taken: 0.28840513 s.
+75% line 2838447 read, time taken: 0.22593426 s.
+50% line 1892298 read, time taken: 0.14234112 s.
+============================End of Built-in readline============================
+
+================================reverse_readline================================
+
+Last line 3784596 read, time taken: 0.00010290 s.
+75% line 2838447 read, time taken: 0.80831332 s.
+50% line 1892298 read, time taken: 1.62029043 s.
+============================End of reverse_readline=============================
+
+================================reverse_readfile================================
+
+Last line 3784596 read, time taken: 0.00007960 s.
+75% line 2838447 read, time taken: 0.35162211 s.
+50% line 1892298 read, time taken: 0.70727881 s.
+============================End of reverse_readfile=============================
+
+=========================Benchmarking file size: 500 MB=========================
+
+Test file of size 500 MB created with 18462038 lines, time used 5.24 seconds.
+===============================Built-in readline================================
+
+Last line 18462038 read, time taken: 1.38365400 s.
+75% line 13846528 read, time taken: 1.11287734 s.
+50% line 9231019 read, time taken: 0.69148150 s.
+============================End of Built-in readline============================
+
+================================reverse_readline================================
+
+Last line 18462038 read, time taken: 0.00009210 s.
+75% line 13846528 read, time taken: 3.98585572 s.
+50% line 9231019 read, time taken: 7.93171154 s.
+============================End of reverse_readline=============================
+
+================================reverse_readfile================================
+
+Last line 18462038 read, time taken: 0.00007860 s.
+75% line 13846528 read, time taken: 1.75920299 s.
+50% line 9231019 read, time taken: 3.45924340 s.
+============================End of reverse_readfile=============================
+
+========================Benchmarking file size: 1000 MB=========================
+
+Test file of size 1000 MB created with 36540934 lines, time used 10.33 seconds.
+===============================Built-in readline================================
+
+Last line 36540934 read, time taken: 2.79645362 s.
+75% line 27405700 read, time taken: 2.20113669 s.
+50% line 18270467 read, time taken: 1.37504352 s.
+============================End of Built-in readline============================
+
+================================reverse_readline================================
+
+Last line 36540934 read, time taken: 0.00009140 s.
+75% line 27405700 read, time taken: 8.09242921 s.
+50% line 18270467 read, time taken: 15.43773309 s.
+============================End of reverse_readline=============================
+
+================================reverse_readfile================================
+
+Last line 36540934 read, time taken: 0.00010630 s.
+75% line 27405700 read, time taken: 3.53976401 s.
+50% line 18270467 read, time taken: 7.18230309 s.
+============================End of reverse_readfile=============================
+
+========================Benchmarking file size: 5000 MB=========================
+
+Test file of size 5000 MB created with 178466370 lines, time used 59.52 seconds.
+===============================Built-in readline================================
+
+Last line 178466370 read, time taken: 14.19376238 s.
+75% line 133849777 read, time taken: 10.26362936 s.
+50% line 89233185 read, time taken: 6.69335968 s.
+============================End of Built-in readline============================
+
+================================reverse_readline================================
+
+Last line 178466370 read, time taken: 0.00255530 s.
+75% line 133849777 read, time taken: 39.30825986 s.
+50% line 89233185 read, time taken: 76.32949390 s.
+============================End of reverse_readline=============================
+
+================================reverse_readfile================================
+
+Last line 178466370 read, time taken: 0.00073260 s.
+75% line 133849777 read, time taken: 18.99764580 s.
+50% line 89233185 read, time taken: 37.87064222 s.
+============================End of reverse_readfile=============================