pycorrectness

#!/usr/bin/env python3
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

import collections
import io
import lzma
import os
from pathlib import Path

from tabulate import tabulate

import orjson

dirname = os.path.join(os.path.dirname(__file__), "data")

LIBRARIES = ["orjson", "ujson", "rapidjson", "simplejson", "json"]


def read_fixture_bytes(filename, subdir=None):
    if subdir is None:
        parts = (dirname, filename)
    else:
        parts = (dirname, subdir, filename)
    path = Path(*parts)
    if path.suffix == ".xz":
        contents = lzma.decompress(path.read_bytes())
    else:
        contents = path.read_bytes()
    return contents


PARSING = {
    filename: read_fixture_bytes(filename, "parsing")
    for filename in os.listdir("data/parsing")
}

JSONCHECKER = {
    filename: read_fixture_bytes(filename, "jsonchecker")
    for filename in os.listdir("data/jsonchecker")
}


RESULTS = collections.defaultdict(dict)


def read_fixture(filename, subdir=None):
    if not filename in BYTES_CACHE:
        BYTES_CACHE[filename] = read_fixture_bytes(filename, subdir)
    return BYTES_CACHE[filename]


def test_passed(library, fixture):
    passed = []
    try:
        passed.append(library.loads(fixture) == orjson.loads(fixture))
        passed.append(
            library.loads(fixture.decode("utf-8"))
            == orjson.loads(fixture.decode("utf-8"))
        )
    except Exception:
        passed.append(False)
    return all(passed)


def test_failed(library, fixture):
    rejected_as_bytes = False
    try:
        library.loads(fixture)
    except Exception:
        rejected_as_bytes = True

    rejected_as_str = False
    try:
        library.loads(fixture.decode("utf-8"))
    except Exception:
        rejected_as_str = True
    return rejected_as_bytes and rejected_as_str


MISTAKEN_PASSES = {key: 0 for key in LIBRARIES}

MISTAKEN_FAILS = {key: 0 for key in LIBRARIES}

PASS_WHITELIST = ("fail01.json", "fail18.json")


def should_pass(filename):
    return (
        filename.startswith("y_")
        or filename.startswith("pass")
        or filename in PASS_WHITELIST
    )


def should_fail(filename):
    return (
        filename.startswith("n_")
        or filename.startswith("i_string")
        or filename.startswith("i_object")
        or filename.startswith("fail")
    ) and not filename in PASS_WHITELIST


for libname in LIBRARIES:
    library = __import__(libname)
    for fixture_set in (PARSING, JSONCHECKER):
        for filename, fixture in fixture_set.items():
            if should_pass(filename):
                res = test_passed(library, fixture)
                RESULTS[filename][libname] = res
                if not res:
                    MISTAKEN_PASSES[libname] += 1

            elif should_fail(filename):
                res = test_failed(library, fixture)
                RESULTS[filename][libname] = res
                if not res:
                    MISTAKEN_FAILS[libname] += 1
            elif filename.startswith("i_"):
                continue
            else:
                raise NotImplementedError

FILENAMES = sorted(list(PARSING.keys()) + list(JSONCHECKER.keys()))


tab_results = []
for filename in FILENAMES:
    entry = [
        filename,
    ]
    for libname in LIBRARIES:
        try:
            entry.append("ok" if RESULTS[filename][libname] else "fail")
        except KeyError:
            continue
    tab_results.append(entry)

buf = io.StringIO()
buf.write(tabulate(tab_results, ["Fixture"] + LIBRARIES, tablefmt="github"))
buf.write("\n")
print(buf.getvalue())

failure_results = [
    [libname, MISTAKEN_FAILS[libname], MISTAKEN_PASSES[libname]]
    for libname in LIBRARIES
]

buf = io.StringIO()
buf.write(
    tabulate(
        failure_results,
        [
            "Library",
            "Invalid JSON documents not rejected",
            "Valid JSON documents not deserialized",
        ],
        tablefmt="github",
    )
)
buf.write("\n")
print(buf.getvalue())

num_results = len([each for each in tab_results if len(each) > 1])

print(f"{num_results} documents tested")