Merge branch 'topic/bbannier/py39' [skip CI]

.pre-commit-config.yaml

@@ -37,7 +37,7 @@ repos:
     stages: ["pre-commit"]
 
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: 'v19.1.3'
+  rev: 'v19.1.4'
   hooks:
   - id: clang-format
     types_or: ["c", "c++"]
@@ -93,16 +93,10 @@ repos:
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.7.2
+  rev: v0.8.2
   hooks:
   - id: ruff
     args: [ --fix ]
   - id: ruff-format
 
-- repo: https://github.com/asottile/pyupgrade
-  rev: v3.19.0
-  hooks:
-  - id: pyupgrade
-    args: ["--py37-plus"]
-
 exclude: 3rdparty/|doc/(autogen|.*examples)/|/Baseline/|(\.svg$)|(\.dat$)

CHANGES

@@ -1,3 +1,17 @@
+1.12.0-dev.223 | 2024-12-09 11:54:03 +0100
+
+  * Fix ruff 'ISC' lints. (Benjamin Bannier, Corelight)
+
+  * Fix ruff 'I' lints. (Benjamin Bannier, Corelight)
+
+  * Fix ruff 'C4' lints. (Benjamin Bannier, Corelight)
+
+  * Migrate Python linting to ruff. (Benjamin Bannier, Corelight)
+
+  * Bump pre-commit hooks. (Benjamin Bannier, Corelight)
+
+  * Upgrade Python to 3.9 with pyupgrade. (Benjamin Bannier, Corelight)
+
 1.12.0-dev.216 | 2024-12-06 10:34:31 +0100
 
   * Introduce `deprecated` helper function in Spicy validator. (Benjamin Bannier, Corelight)

VERSION

@@ -1 +1 @@
-1.12.0-dev.216
+1.12.0-dev.223

doc/conf.py

@@ -13,8 +13,8 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
-import sys
 import subprocess
+import sys
 
 sys.path.insert(0, os.path.abspath("scripts"))
 
@@ -54,7 +54,7 @@
 
 # Extlinks extension
 extlinks = {
-    "repo": ("https://github.com/zeek/spicy/blob/v%s/%%s" % release, "#%s"),
+    "repo": (f"https://github.com/zeek/spicy/blob/v{release}/%s", "#%s"),
     "issue": ("https://github.com/zeek/spicy/issues/%s", "#%s"),
     "pr": ("https://github.com/zeek/spicy/pulls/%s", "#%s"),
     "zeek": ("https://docs.zeek.org/en/master/%s", "%s"),
@@ -72,16 +72,15 @@
         "%s",
     ),
     "package-release-tgz": (
-        "https://github.com/zeek/spicy/releases/download/v%s/spicy_%%s.tar.gz"
-        % release,
+        f"https://github.com/zeek/spicy/releases/download/v{release}/spicy_%s.tar.gz",
         "%s",
     ),
     "package-release-rpm": (
-        "https://github.com/zeek/spicy/releases/download/v%s/spicy_%%s.rpm" % release,
+        f"https://github.com/zeek/spicy/releases/download/v{release}/spicy_%s.rpm",
         "%s",
     ),
     "package-release-deb": (
-        "https://github.com/zeek/spicy/releases/download/v%s/spicy_%%s.deb" % release,
+        f"https://github.com/zeek/spicy/releases/download/v{release}/spicy_%s.deb",
         "%s",
     ),
 }

doc/scripts/spicy-doc-to-rst

@@ -8,13 +8,12 @@ import argparse
 import copy
 import filecmp
 import json
-import os.path
 import os
+import os.path
 import re
 import sys
 import textwrap
-
-from typing import Dict, List, Pattern, Set
+from re import Pattern
 
 
 def fatalError(message: str):
@@ -58,10 +57,7 @@ def binary(token):
 
 
 Operators = {
-    "Add": lambda op: "add <sp> {}[{}]".format(
-        op.operands[0].rst(in_operator=True),
-        op.operands[1].rst(in_operator=True, markup=False),
-    ),
+    "Add": lambda op: f"add <sp> {op.operands[0].rst(in_operator=True)}[{op.operands[1].rst(in_operator=True, markup=False)}]",
     "Begin": call("begin"),
     "BitAnd": binary("&"),
     "BitOr": binary("|"),
@@ -75,13 +71,8 @@ Operators = {
         TypedType.sub("\\1", op.operands[1].rst(in_operator=True, markup=False)),
         op.operands[0].rst(in_operator=True, markup=False),
     ),
-    "CustomAssign": lambda op: "{} = {}".format(
-        op.operands[0].rst(in_operator=True), op.operands[1].rst(in_operator=True)
-    ),
-    "Delete": lambda op: "delete <sp> {}[{}]".format(
-        op.operands[0].rst(in_operator=True),
-        op.operands[1].rst(in_operator=True, markup=False),
-    ),
+    "CustomAssign": lambda op: f"{op.operands[0].rst(in_operator=True)} = {op.operands[1].rst(in_operator=True)}",
+    "Delete": lambda op: f"delete <sp> {op.operands[0].rst(in_operator=True)}[{op.operands[1].rst(in_operator=True, markup=False)}]",
     "Deref": unary("*"),
     "DecrPostfix": unary("", "--"),
     "DecrPrefix": unary("++"),
@@ -98,15 +89,8 @@ Operators = {
     "HasMember": binary("?."),
     "TryMember": binary(".?"),
     "Member": binary("."),
-    "Index": lambda op: "{}[{}]".format(
-        op.operands[0].rst(in_operator=True),
-        op.operands[1].rst(in_operator=True, markup=False),
-    ),
-    "IndexAssign": lambda op: "{}[{}] = {}".format(
-        op.operands[0].rst(in_operator=True),
-        op.operands[1].rst(in_operator=True, markup=False),
-        op.operands[2].rst(in_operator=True, markup=False),
-    ),
+    "Index": lambda op: f"{op.operands[0].rst(in_operator=True)}[{op.operands[1].rst(in_operator=True, markup=False)}]",
+    "IndexAssign": lambda op: f"{op.operands[0].rst(in_operator=True)}[{op.operands[1].rst(in_operator=True, markup=False)}] = {op.operands[2].rst(in_operator=True, markup=False)}",
     "IncrPostfix": unary("", "++"),
     "IncrPrefix": unary("++"),
     "LogicalAnd": binary("&&"),
@@ -121,10 +105,7 @@ Operators = {
     "Pack": keyword("pack"),
     "Power": binary("**"),
     "Unpack": keyword("unpack"),
-    "Unset": lambda op: "unset <sp> {}.{}".format(
-        op.operands[0].rst(in_operator=True),
-        op.operands[1].rst(in_operator=True, markup=False),
-    ),
+    "Unset": lambda op: f"unset <sp> {op.operands[0].rst(in_operator=True)}.{op.operands[1].rst(in_operator=True, markup=False)}",
     "SignNeg": unary("-"),
     "Size": unary("|", "|"),
     "ShiftLeft": binary("<<"),
@@ -233,18 +214,15 @@ class Operator:
             sig = Operators[self.kind](self)
         except KeyError:
             print(
-                "error: " f"operator {self.kind} not supported by spicy-doc-to-rst yet",
+                f"error: operator {self.kind} not supported by spicy-doc-to-rst yet",
                 file=sys.stderr,
             )
             sys.exit(1)
 
         result = fmtType(self.rtype)
-        return ".. spicy:operator:: " "{ns}::{kind} {result} {sig}\n\n{doc}".format(
-            ns=self.namespace,
-            kind=self.kind,
-            result=result,
-            sig=sig,
-            doc=fmtDoc(self.doc),
+        return (
+            ".. spicy:operator:: "
+            f"{self.namespace}::{self.kind} {result} {sig}\n\n{fmtDoc(self.doc)}"
         )
 
     def __lt__(self, other):
@@ -278,15 +256,7 @@ class Method:
         result = fmtType(self.rtype)
         sig = (
             ".. spicy:method:: "
-            "{ns}::{id} {self} {id} {const} {result} ({args})\n\n{doc}".format(
-                ns=self.namespace,
-                result=result,
-                self=self_,
-                const=const,
-                id=self.id,
-                args=args,
-                doc=fmtDoc(self.doc),
-            )
+            f"{self.namespace}::{self.id} {self_} {self.id} {const} {result} ({args})\n\n{fmtDoc(self.doc)}"
         )
         return sig
 
@@ -365,8 +335,8 @@ try:
 except ValueError as e:
     fatalError(f"cannot parse input: {e}")
 
-operators: Dict[str, List[Operator]] = {}
-methods: Dict[str, List[Method]] = {}
+operators: dict[str, list[Operator]] = {}
+methods: dict[str, list[Method]] = {}
 
 for op in meta:
     if op["kind"] == "MemberCall":
@@ -416,7 +386,7 @@ for ns in sorted(keys):
     # example is the vector's index operators for constant and non-constant
     # instances, respectively. Other duplications are coming from joining
     # namespaces for integers.
-    already_recorded: Set[str] = set()
+    already_recorded: set[str] = set()
 
     def print_unique(out, s):
         if s not in already_recorded:

doc/scripts/spicy-pygments.py

@@ -1,6 +1,6 @@
 # Copyright (c) 2020-2023 by the Zeek Project. See LICENSE for details.
 
-from pygments.lexer import RegexLexer, include, words, bygroups
+from pygments.lexer import RegexLexer, bygroups, include, words
 from pygments.token import (
     Comment,
     Keyword,

doc/scripts/spicy.py

@@ -5,18 +5,18 @@
 """
 
 import os.path
+import subprocess
+
 from docutils import nodes
 from docutils.parsers.rst import directives
-from sphinx.util.nodes import make_refnode, logging
-from sphinx.util.console import darkgreen, red
-from sphinx.roles import XRefRole
-from sphinx.locale import _
-from sphinx.domains import Domain, ObjType
-from sphinx.directives.code import CodeBlock, LiteralInclude
+from sphinx import addnodes, version_info
 from sphinx.directives import ObjectDescription
-from sphinx import version_info
-from sphinx import addnodes
-import subprocess
+from sphinx.directives.code import CodeBlock, LiteralInclude
+from sphinx.domains import Domain, ObjType
+from sphinx.locale import _
+from sphinx.roles import XRefRole
+from sphinx.util.console import darkgreen, red
+from sphinx.util.nodes import logging, make_refnode
 
 
 def setup(Sphinx):
@@ -50,9 +50,11 @@ def add_target_and_index(self, name, sig, signode):
             if key in objects:
                 self.env.warn(
                     self.env.docname,
-                    f"duplicate description of {self.objtype} {name}, "
-                    + "other instance in "
-                    + self.env.doc2path(objects[key]),
+                    (
+                        f"duplicate description of {self.objtype} {name}, ",
+                        "other instance in ",
+                        self.env.doc2path(objects[key]),
+                    ),
                     self.lineno,
                 )
             objects[key] = self.env.docname
@@ -285,7 +287,7 @@ def run(self):
             old = ""
 
         if text != old:
-            self.message("updating %s" % darkgreen(self.file[0]))
+            self.message(f"updating {darkgreen(self.file[0])}")
             f = open(self.file[1], "w")
             f.write(
                 "# Automatically generated; edit in Sphinx source code, not here.\n"
@@ -344,14 +346,11 @@ def __init__(self, *args, **kwargs):
             if "prefix" not in options:
                 self.prefix = None
 
-        self.content_hash = (
-            "# Automatically generated; do not edit. -- <HASH> %s/%s/%s"
-            % (self.exec_, self.show_as, self.expect_failure)
-        )
+        self.content_hash = f"# Automatically generated; do not edit. -- <HASH> {self.exec_}/{self.show_as}/{self.expect_failure}"
 
         source_orig = args[1][0]
         file = "_" + source_orig
-        index = "_%s" % args[1][1] if len(args[1]) > 1 else ""
+        index = f"_{args[1][1]}" if len(args[1]) > 1 else ""
         output = f"examples/{file}.output{index}"
         args = list(args)
         args[1] = [output]
@@ -360,7 +359,7 @@ def __init__(self, *args, **kwargs):
         super(LiteralInclude, self).__init__(*args, **kwargs)
 
         source = self.env.relfn2path(os.path.join("examples/", file))[0]
-        self.update(source_orig, source, source + ".output%s" % index, self.exec_)
+        self.update(source_orig, source, source + f".output{index}", self.exec_)
 
     def run(self):
         literal = LiteralInclude.run(self)
@@ -389,9 +388,7 @@ def update(self, source_orig, source, destination, cmd):
         # Abort if that's not the case.
         if "CI" in os.environ:
             self.error(
-                "error during CI: {} is not up to date in repository".format(
-                    destination
-                )
+                f"error during CI: {destination} is not up to date in repository"
             )
             return
 
@@ -406,7 +403,7 @@ def update(self, source_orig, source, destination, cmd):
             one_cmd = one_cmd.strip()
 
             one_cmd = one_cmd.replace("%INPUT", source)
-            self.message("executing %s" % darkgreen(one_cmd))
+            self.message(f"executing {darkgreen(one_cmd)}")
 
             try:
                 output = subprocess.check_output(
@@ -437,7 +434,7 @@ def update(self, source_orig, source, destination, cmd):
                     out.write(b"\n")
 
                 if show_as:
-                    one_cmd = "# %s\n" % show_as[0].strip()
+                    one_cmd = f"# {show_as[0].strip()}\n"
                     one_cmd = one_cmd.replace(
                         "%INPUT", self.show_with if self.show_with else source_orig
                     )

ruff.toml

@@ -0,0 +1,16 @@
+# Copyright (c) 2020-2023 by the Zeek Project. See LICENSE for details.
+
+target-version = "py39"
+
+exclude = [
+    "3rdparty",
+]
+
+[lint]
+select = [
+    "C4",
+    "F",
+    "I",
+    "ISC",
+    "UP",
+]

tests/Scripts/license-header.py

@@ -1,15 +1,13 @@
 #!/usr/bin/env python3
 
-import sys
 import re
+import sys
 
 for f in sys.argv[1:]:
     with open(f) as input:
         if not any(
-            map(
-                lambda x: re.match(r".*Copyright.*by\ the\ Zeek\ Project", x),
-                input.readlines(),
-            )
+            re.match(r".*Copyright.*by\ the\ Zeek\ Project", x)
+            for x in input.readlines()
         ):
             print(f"{f} does not seem to contain a license header")
             sys.exit(1)