Extend the input formats recognized by grammarinator-parse

Until now, grammarinator-parse expected to receive a list of files
as inputs to be processed. From now on, it also accepts directories,
which will be recursively traversed. Furthermore, file patterns
are also supported to easily filter and parse tests by patterns.

docs/guide/population.rst

@@ -30,8 +30,10 @@ variations and explore different test cases.
    :replace: "parse.py/grammarinator-parse"
 
 The usage of the ``grammarinator-parse`` utility is generally straightforward.
-It takes a set of inputs (``--input``) and processes them with the specified
-grammars (``FILE``). The start rule, which determines the root of every tree
+It takes a set of inputs and processes them with the specified grammars
+(``FILE``). Inputs can be listed as files or directories (using ``--input``), or
+specified with file patterns (using ``--glob``). The listed directories are
+traversed recursively. The start rule, which determines the root of every tree
 in the population, can be defined using the ``--rule`` argument. After the
 parsing is completed and the tree is created, various
 :doc:`transformers <transformers>` (``--transformer``) can be applied to

grammarinator/cli.py

@@ -5,6 +5,7 @@
 # This file may not be copied, modified, or distributed except
 # according to those terms.
 
+import glob
 import logging
 import os
 
@@ -59,3 +60,16 @@ def process_tree_format_argument(args):
     tree_format = tree_formats[args.tree_format]
     args.tree_extension = tree_format['extension']
     args.tree_codec = tree_format['codec_class']()
+
+
+def iter_files(args):
+    for fn in args.input:
+        if os.path.isdir(fn):
+            for dirpath, _, filenames in os.walk(fn):
+                for f in filenames:
+                    yield os.path.join(dirpath, f)
+        else:
+            yield fn
+
+    for pattern in args.glob:
+        yield from glob.glob(pattern)

grammarinator/parse.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018-2023 Renata Hodovan, Akos Kiss.
+# Copyright (c) 2018-2024 Renata Hodovan, Akos Kiss.
 #
 # Licensed under the BSD 3-Clause License
 # <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
@@ -14,7 +14,7 @@
 from antlerinator import add_antlr_argument, process_antlr_argument
 from inators.arg import add_log_level_argument, add_sys_path_argument, add_sys_recursion_limit_argument, add_version_argument, process_log_level_argument, process_sys_path_argument, process_sys_recursion_limit_argument
 
-from .cli import add_disable_cleanup_argument, add_encoding_argument, add_encoding_errors_argument, add_tree_format_argument, add_jobs_argument, import_list, init_logging, logger, process_tree_format_argument
+from .cli import add_disable_cleanup_argument, add_encoding_argument, add_encoding_errors_argument, add_tree_format_argument, add_jobs_argument, import_list, init_logging, iter_files, logger, process_tree_format_argument
 from .pkgdata import __version__
 from .runtime import RuleSize
 from .tool import DefaultPopulation, ParserTool
@@ -40,8 +40,10 @@ def execute():
                             """)
     parser.add_argument('grammar', metavar='FILE', nargs='+',
                         help='ANTLR grammar files describing the expected format of input to parse.')
-    parser.add_argument('-i', '--input', metavar='FILE', nargs='+', required=True,
-                        help='input files to process.')
+    parser.add_argument('-i', '--input', metavar='FILE', nargs='+',
+                        help='input files or directories to process.')
+    parser.add_argument('--glob', metavar='PATTERN', nargs='+',
+                        help='wildcard patterns for input files to process (supported wildcards: ?, *, **, [])')
     parser.add_argument('-r', '--rule', metavar='NAME',
                         help='name of the rule to start parsing with (default: first parser rule).')
     parser.add_argument('-t', '--transformer', metavar='NAME', action='append', default=[],
@@ -81,10 +83,10 @@ def execute():
                     population=DefaultPopulation(args.out, args.tree_extension, codec=args.tree_codec), max_depth=args.max_depth, cleanup=args.cleanup, encoding=args.encoding, errors=args.encoding_errors) as parser:
         if args.jobs > 1:
             with Pool(args.jobs) as pool:
-                for _ in pool.imap_unordered(parser.parse, args.input):
+                for _ in pool.imap_unordered(parser.parse, iter_files(args)):
                     pass
         else:
-            for fn in args.input:
+            for fn in iter_files(args):
                 parser.parse(fn)