From 206030468bd63be59322fd8ae2e0dfade22c8775 Mon Sep 17 00:00:00 2001
From: Tom Close <tom.g.close@gmail.com>
Date: Fri, 20 Sep 2024 15:17:03 +1000
Subject: [PATCH 1/3] added TypedCollection base class of TypedDirectory and
 TypedSet

---
 fileformats/core/__init__.py                |  3 +-
 fileformats/core/collection.py              | 77 +++++++++++++++++++++
 fileformats/core/fileset.py                 | 42 +----------
 fileformats/core/mock.py                    | 53 ++++++++++++++
 fileformats/generic/directory.py            | 47 +++----------
 fileformats/generic/generate_sample_data.py |  7 +-
 fileformats/generic/set.py                  | 27 +++-----
 fileformats/generic/tests/test_directory.py |  2 +-
 8 files changed, 158 insertions(+), 100 deletions(-)
 create mode 100644 fileformats/core/collection.py
 create mode 100644 fileformats/core/mock.py

diff --git a/fileformats/core/__init__.py b/fileformats/core/__init__.py
index fb91fff..fcd84c0 100644
--- a/fileformats/core/__init__.py
+++ b/fileformats/core/__init__.py
@@ -1,7 +1,8 @@
 from ._version import __version__
 from .classifier import Classifier
 from .datatype import DataType
-from .fileset import FileSet, MockMixin
+from .mock import MockMixin
+from .fileset import FileSet
 from .field import Field
 from .identification import (
     to_mime,
diff --git a/fileformats/core/collection.py b/fileformats/core/collection.py
new file mode 100644
index 0000000..207867b
--- /dev/null
+++ b/fileformats/core/collection.py
@@ -0,0 +1,77 @@
+import typing as ty
+from pathlib import Path
+from abc import ABCMeta, abstractproperty
+from fileformats.core import FileSet, validated_property, mtime_cached_property
+from fileformats.core.decorators import classproperty
+from fileformats.core.exceptions import FormatDefinitionError, FormatMismatchError
+
+
+class TypedCollection(FileSet, metaclass=ABCMeta):
+    """Base class for collections of files-sets of specific types either in a directory
+    or a collection of file paths"""
+
+    content_types: ty.Tuple[
+        ty.Union[ty.Type[FileSet], ty.Type[ty.Optional[FileSet]]], ...
+    ] = ()
+
+    @abstractproperty
+    def content_fspaths(self) -> ty.Iterable[Path]:
+        ...  # noqa: E704
+
+    @mtime_cached_property
+    def contents(self) -> ty.List[FileSet]:
+        contnts = []
+        for content_type in self.potential_content_types:
+            assert content_type
+            for p in self.content_fspaths:
+                try:
+                    contnts.append(content_type([p], **self._load_kwargs))
+                except FormatMismatchError:
+                    continue
+        return contnts
+
+    @validated_property
+    def _validate_required_content_types(self) -> None:
+        not_found = set(self.required_content_types)
+        if not not_found:
+            return
+        for fspath in self.content_fspaths:
+            for content_type in list(not_found):
+                if content_type.matches(fspath):
+                    not_found.remove(content_type)
+                    if not not_found:
+                        return
+        assert not_found
+        raise FormatMismatchError(
+            f"Did not find the required content types, {not_found}, in {self}"
+        )
+
+    @classproperty
+    def potential_content_types(cls) -> ty.Tuple[ty.Type[FileSet], ...]:
+        content_types: ty.List[ty.Type[FileSet]] = []
+        for content_type in cls.content_types:  # type: ignore[assignment]
+            if ty.get_origin(content_type) is ty.Union:
+                args = ty.get_args(content_type)
+                if not len(args) == 2 and None not in args:
+                    raise FormatDefinitionError(
+                        "Only Optional types are allowed in content_type definitions, "
+                        f"not {content_type}"
+                    )
+                content_types.append(args[0] if args[0] is not None else args[1])
+            else:
+                content_types.append(content_type)  # type: ignore[arg-type]
+        return tuple(content_types)
+
+    @classproperty
+    def required_content_types(cls) -> ty.Tuple[ty.Type[FileSet], ...]:
+        content_types: ty.List[ty.Type[FileSet]] = []
+        for content_type in cls.content_types:  # type: ignore[assignment]
+            if ty.get_origin(content_type) is None:
+                content_types.append(content_type)  # type: ignore[arg-type]
+        return tuple(content_types)
+
+    @classproperty
+    def unconstrained(cls) -> bool:
+        """Whether the file-format is unconstrained by extension, magic number or another
+        constraint"""
+        return super().unconstrained and not cls.content_types
diff --git a/fileformats/core/fileset.py b/fileformats/core/fileset.py
index b0a5ce9..f29ce46 100644
--- a/fileformats/core/fileset.py
+++ b/fileformats/core/fileset.py
@@ -42,6 +42,7 @@
 from .datatype import DataType
 from .extras import extra
 from .fs_mount_identifier import FsMountIdentifier
+from .mock import MockMixin
 
 if ty.TYPE_CHECKING:
     from pydra.engine.task import TaskBase
@@ -1742,44 +1743,3 @@ def _new_copy_path(
     _formats_by_name: ty.Optional[ty.Dict[str, ty.Set[ty.Type["FileSet"]]]] = None
     _required_props: ty.Optional[ty.Tuple[str, ...]] = None
     _valid_class: ty.Optional[bool] = None
-
-
-class MockMixin:
-    """Strips out validation methods of a class, allowing it to be mocked in a way that
-    still satisfies type-checking"""
-
-    def __init__(
-        self,
-        fspaths: FspathsInputType,
-        metadata: ty.Union[ty.Dict[str, ty.Any], bool, None] = False,
-    ):
-        self.fspaths = fspaths_converter(fspaths)
-        self._metadata = metadata
-
-    @classproperty
-    def type_name(cls) -> str:
-        return cls.mocked.type_name
-
-    def __bytes_repr__(self, cache: ty.Dict[str, ty.Any]) -> ty.Iterable[bytes]:
-        yield from (str(fspath).encode() for fspath in self.fspaths)
-
-    @classproperty
-    def mocked(cls) -> FileSet:
-        """The "true" class that the mocked class is based on"""
-        return next(c for c in cls.__mro__ if not issubclass(c, MockMixin))  # type: ignore[no-any-return, attr-defined]
-
-    @classproperty
-    def namespace(cls) -> str:
-        """The "namespace" the format belongs to under the "fileformats" umbrella
-        namespace"""
-        mro: ty.Tuple[ty.Type] = cls.__mro__  # type: ignore
-        for base in mro:
-            if issubclass(base, MockMixin):
-                continue
-            try:
-                return base.namespace  # type: ignore
-            except FormatDefinitionError:
-                pass
-        raise FormatDefinitionError(
-            f"None of of the bases classes of {cls} ({mro}) have a valid namespace"
-        )
diff --git a/fileformats/core/mock.py b/fileformats/core/mock.py
new file mode 100644
index 0000000..dca8ca8
--- /dev/null
+++ b/fileformats/core/mock.py
@@ -0,0 +1,53 @@
+import typing as ty
+from .utils import (
+    fspaths_converter,
+)
+from .decorators import classproperty
+from .typing import FspathsInputType
+from .exceptions import (
+    FormatDefinitionError,
+)
+
+if ty.TYPE_CHECKING:
+    from .fileset import FileSet
+
+
+class MockMixin:
+    """Strips out validation methods of a class, allowing it to be mocked in a way that
+    still satisfies type-checking"""
+
+    def __init__(
+        self,
+        fspaths: FspathsInputType,
+        metadata: ty.Union[ty.Dict[str, ty.Any], bool, None] = False,
+    ):
+        self.fspaths = fspaths_converter(fspaths)
+        self._metadata = metadata
+
+    @classproperty
+    def type_name(cls) -> str:
+        return cls.mocked.type_name
+
+    def __bytes_repr__(self, cache: ty.Dict[str, ty.Any]) -> ty.Iterable[bytes]:
+        yield from (str(fspath).encode() for fspath in self.fspaths)
+
+    @classproperty
+    def mocked(cls) -> "FileSet":
+        """The "true" class that the mocked class is based on"""
+        return next(c for c in cls.__mro__ if not issubclass(c, MockMixin))  # type: ignore[no-any-return, attr-defined]
+
+    @classproperty
+    def namespace(cls) -> str:
+        """The "namespace" the format belongs to under the "fileformats" umbrella
+        namespace"""
+        mro: ty.Tuple[ty.Type] = cls.__mro__  # type: ignore
+        for base in mro:
+            if issubclass(base, MockMixin):
+                continue
+            try:
+                return base.namespace  # type: ignore
+            except FormatDefinitionError:
+                pass
+        raise FormatDefinitionError(
+            f"None of of the bases classes of {cls} ({mro}) have a valid namespace"
+        )
diff --git a/fileformats/generic/directory.py b/fileformats/generic/directory.py
index 2a46453..2be2b43 100644
--- a/fileformats/generic/directory.py
+++ b/fileformats/generic/directory.py
@@ -3,13 +3,13 @@
 from fileformats.core.exceptions import FormatMismatchError
 from fileformats.core.decorators import (
     validated_property,
-    classproperty,
     mtime_cached_property,
 )
 from .fsobject import FsObject
 from fileformats.core.fileset import FileSet, FILE_CHUNK_LEN_DEFAULT
 from fileformats.core.mixin import WithClassifiers
 from fileformats.core.typing import CryptoMethod
+from fileformats.core.collection import TypedCollection
 from .file import File
 
 
@@ -67,6 +67,10 @@ def hash_files(
             ignore_hidden_dirs=ignore_hidden_dirs,
         )
 
+    @property
+    def content_fspaths(self) -> ty.Iterable[Path]:
+        return self.fspath.iterdir()
+
     # Duck-type Path methods
 
     def __div__(self, other: ty.Union[str, Path]) -> Path:
@@ -85,7 +89,7 @@ def iterdir(self) -> ty.Iterator[Path]:
         return self.fspath.iterdir()
 
 
-class TypedDirectory(Directory):
+class TypedDirectory(TypedCollection, Directory):  # type: ignore[misc]
     """Directory that must contain a specific set of content types. Only files that match
     the content types will be considered as contents in the `contents` property.
 
@@ -95,43 +99,12 @@ class TypedDirectory(Directory):
         the content types that are expected to be found within the directory
     """
 
-    @mtime_cached_property
-    def contents(self) -> ty.List[FileSet]:
-        contnts = []
-        for content_type in self.content_types:
-            assert content_type
-            for p in self.fspath.iterdir():
-                try:
-                    contnts.append(content_type([p], **self._load_kwargs))
-                except FormatMismatchError:
-                    continue
-        return contnts
-
-    @classproperty
-    def unconstrained(cls) -> bool:
-        """Whether the file-format is unconstrained by extension, magic number or another
-        constraint"""
-        return super().unconstrained and not cls.content_types
-
-    @validated_property
-    def _validate_contents(self) -> None:
-        if not self.content_types:
-            return
-        not_found = set(self.content_types)
-        for fspath in self.fspath.iterdir():
-            for content_type in list(not_found):
-                if content_type.matches(fspath):
-                    not_found.remove(content_type)
-                    if not not_found:
-                        return
-        assert not_found
-        raise FormatMismatchError(
-            f"Did not find the required content types, {not_found}, within the "
-            f"directory {self.fspath} of {self}"
-        )
+    @property
+    def content_fspaths(self) -> ty.Iterable[Path]:
+        return self.fspath.iterdir()
 
 
-class DirectoryOf(WithClassifiers, TypedDirectory):
+class DirectoryOf(WithClassifiers, TypedDirectory):  # type: ignore[misc]
     """Generic directory classified by the formats of its contents"""
 
     # WithClassifiers-required class attrs
diff --git a/fileformats/generic/generate_sample_data.py b/fileformats/generic/generate_sample_data.py
index d09d2f5..15fb987 100644
--- a/fileformats/generic/generate_sample_data.py
+++ b/fileformats/generic/generate_sample_data.py
@@ -69,7 +69,7 @@ def directory_containing_generate_sample_data(
 ) -> ty.List[Path]:
     a_dir = generator.generate_fspath(TypedDirectory)
     a_dir.mkdir()
-    for tp in directory.content_types:
+    for tp in directory.potential_content_types:
         tp.sample_data(generator.child(dest_dir=a_dir))
     return [a_dir]
 
@@ -81,6 +81,9 @@ def set_of_sample_data(
 ) -> ty.List[Path]:
     return list(
         itertools.chain(
-            *(tp.sample_data(generator.child()) for tp in set_of.content_types)
+            *(
+                tp.sample_data(generator.child())
+                for tp in set_of.potential_content_types
+            )
         )
     )
diff --git a/fileformats/generic/set.py b/fileformats/generic/set.py
index 286330f..adc34cc 100644
--- a/fileformats/generic/set.py
+++ b/fileformats/generic/set.py
@@ -1,17 +1,20 @@
 import typing as ty
-from fileformats.core import FileSet, validated_property
-from functools import cached_property
+from pathlib import Path
+from fileformats.core import FileSet
 from fileformats.core.mixin import WithClassifiers
+from fileformats.core.collection import TypedCollection
 
 
-class TypedSet(FileSet):
+class TypedSet(TypedCollection):
     """List of specific file types (similar to the contents of a directory but not
     enclosed in one)"""
 
-    content_types: ty.Tuple[ty.Type[FileSet], ...] = ()
-
     MAX_REPR_PATHS = 3
 
+    @property
+    def content_fspaths(self) -> ty.Iterable[Path]:
+        return self.fspaths
+
     def __repr__(self) -> str:
         paths_repr = (
             "'"
@@ -22,20 +25,8 @@ def __repr__(self) -> str:
             paths_repr += ", ..."
         return f"{self.type_name}({paths_repr})"
 
-    @cached_property
-    def contents(self) -> ty.List[FileSet]:  # type: ignore[override]
-        contnts = []
-        for content_type in self.content_types:
-            for p in self.fspaths:
-                contnts.append(content_type([p], **self._load_kwargs))
-        return contnts
-
-    @validated_property
-    def _validate_contents(self) -> None:
-        self.contents
-
 
-class SetOf(WithClassifiers, TypedSet):
+class SetOf(WithClassifiers, TypedSet):  # type: ignore[misc]
     # WithClassifiers-required class attrs
     classifiers_attr_name = "content_types"
     allowed_classifiers = (FileSet,)
diff --git a/fileformats/generic/tests/test_directory.py b/fileformats/generic/tests/test_directory.py
index e512cbe..008b292 100644
--- a/fileformats/generic/tests/test_directory.py
+++ b/fileformats/generic/tests/test_directory.py
@@ -6,7 +6,7 @@ def test_sample_directory():
     assert isinstance(Directory.sample(), Directory)
 
 
-def test_sample_directory_containing():
+def test_sample_directory_of():
     sample = DirectoryOf[MyFormatGz].sample()
     assert isinstance(sample, DirectoryOf[MyFormatGz])
     assert all(isinstance(c, MyFormatGz) for c in sample.contents)

From 8b0c5fc6b60cbfcc89f3736ee358b926fc3ccd08 Mon Sep 17 00:00:00 2001
From: Tom Close <tom.g.close@gmail.com>
Date: Fri, 20 Sep 2024 16:08:32 +1000
Subject: [PATCH 2/3] tested out optional content types for TypedSet and
 TypedDirectory

---
 fileformats/core/collection.py              | 14 ++-----
 fileformats/core/mixin.py                   | 30 ++++++++++----
 fileformats/core/utils.py                   | 37 +++++++++++++++++
 fileformats/generic/directory.py            |  1 +
 fileformats/generic/set.py                  | 16 +++++++-
 fileformats/generic/tests/test_directory.py | 45 ++++++++++++++++++++-
 6 files changed, 122 insertions(+), 21 deletions(-)

diff --git a/fileformats/core/collection.py b/fileformats/core/collection.py
index 207867b..2fe6f3f 100644
--- a/fileformats/core/collection.py
+++ b/fileformats/core/collection.py
@@ -3,7 +3,8 @@
 from abc import ABCMeta, abstractproperty
 from fileformats.core import FileSet, validated_property, mtime_cached_property
 from fileformats.core.decorators import classproperty
-from fileformats.core.exceptions import FormatDefinitionError, FormatMismatchError
+from fileformats.core.exceptions import FormatMismatchError
+from fileformats.core.utils import get_optional_type
 
 
 class TypedCollection(FileSet, metaclass=ABCMeta):
@@ -50,16 +51,7 @@ def _validate_required_content_types(self) -> None:
     def potential_content_types(cls) -> ty.Tuple[ty.Type[FileSet], ...]:
         content_types: ty.List[ty.Type[FileSet]] = []
         for content_type in cls.content_types:  # type: ignore[assignment]
-            if ty.get_origin(content_type) is ty.Union:
-                args = ty.get_args(content_type)
-                if not len(args) == 2 and None not in args:
-                    raise FormatDefinitionError(
-                        "Only Optional types are allowed in content_type definitions, "
-                        f"not {content_type}"
-                    )
-                content_types.append(args[0] if args[0] is not None else args[1])
-            else:
-                content_types.append(content_type)  # type: ignore[arg-type]
+            content_types.append(get_optional_type(content_type))  # type: ignore[arg-type]
         return tuple(content_types)
 
     @classproperty
diff --git a/fileformats/core/mixin.py b/fileformats/core/mixin.py
index 3b4fc58..1df6863 100644
--- a/fileformats/core/mixin.py
+++ b/fileformats/core/mixin.py
@@ -4,7 +4,7 @@
 import logging
 from .datatype import DataType
 import fileformats.core
-from .utils import describe_task, matching_source
+from .utils import describe_task, matching_source, get_optional_type
 from .decorators import validated_property, classproperty
 from .identification import to_mime_format_name
 from .converter_helpers import SubtypeVar, ConverterSpec
@@ -292,6 +292,7 @@ def my_func(file: MyFormatWithClassifiers[Integer]):
     # Default values for class attrs
     multiple_classifiers = True
     allowed_classifiers: ty.Optional[ty.Tuple[ty.Type[Classifier], ...]] = None
+    allow_optional_classifiers = False
     exclusive_classifiers: ty.Tuple[ty.Type[Classifier], ...] = ()
     ordered_classifiers = False
     generically_classifiable = False
@@ -320,7 +321,9 @@ def wildcard_classifiers(
     ) -> ty.FrozenSet[ty.Type[SubtypeVar]]:
         if classifiers is None:
             classifiers = cls.classifiers if cls.is_classified else ()
-        return frozenset(t for t in classifiers if issubclass(t, SubtypeVar))
+        return frozenset(
+            t for t in classifiers if issubclass(get_optional_type(t), SubtypeVar)  # type: ignore[misc]
+        )
 
     @classmethod
     def non_wildcard_classifiers(
@@ -329,7 +332,9 @@ def non_wildcard_classifiers(
         if classifiers is None:
             classifiers = cls.classifiers if cls.is_classified else ()
         assert classifiers is not None
-        return frozenset(q for q in classifiers if not issubclass(q, SubtypeVar))
+        return frozenset(
+            q for q in classifiers if not issubclass(get_optional_type(q), SubtypeVar)
+        )
 
     @classmethod
     def __class_getitem__(
@@ -341,11 +346,15 @@ def __class_getitem__(
             classifiers_tuple = tuple(classifiers)
         else:
             classifiers_tuple = (classifiers,)
+        classifiers_to_check = tuple(
+            get_optional_type(c, cls.allow_optional_classifiers)
+            for c in classifiers_tuple
+        )
 
         if cls.allowed_classifiers:
             not_allowed = [
                 q
-                for q in classifiers_tuple
+                for q in classifiers_to_check
                 if not any(issubclass(q, t) for t in cls.allowed_classifiers)
             ]
             if not_allowed:
@@ -357,15 +366,17 @@ def __class_getitem__(
         if cls.multiple_classifiers:
             if not cls.ordered_classifiers:
                 # Check for duplicate classifiers in the multiple list
-                if len(classifiers_tuple) > 1:
+                if len(classifiers_to_check) > 1:
                     # Sort the classifiers into categories and ensure that there aren't more
                     # than one type for each category. Otherwise, if the classifier doesn't
                     # belong to a category, check to see that there aren't multiple sub-classes
                     # in the classifier set
                     repetitions: ty.Dict[
                         ty.Type[Classifier], ty.List[ty.Type[Classifier]]
-                    ] = {c: [] for c in cls.exclusive_classifiers + classifiers_tuple}
-                    for classifier in classifiers_tuple:
+                    ] = {
+                        c: [] for c in cls.exclusive_classifiers + classifiers_to_check
+                    }
+                    for classifier in classifiers_to_check:
                         for exc_classifier in repetitions:
                             if issubclass(classifier, exc_classifier):
                                 repetitions[exc_classifier].append(classifier)
@@ -381,7 +392,10 @@ def __class_getitem__(
                             )
                         )
                 classifiers_tuple = tuple(
-                    sorted(set(classifiers_tuple), key=lambda x: x.__name__)
+                    sorted(
+                        set(classifiers_tuple),
+                        key=lambda x: get_optional_type(x).__name__,
+                    )
                 )
         else:
             if len(classifiers_tuple) > 1:
diff --git a/fileformats/core/utils.py b/fileformats/core/utils.py
index 24ec888..9d9f6af 100644
--- a/fileformats/core/utils.py
+++ b/fileformats/core/utils.py
@@ -11,6 +11,7 @@
 from contextlib import contextmanager
 from .typing import FspathsInputType
 import fileformats.core
+from fileformats.core.exceptions import FormatDefinitionError
 
 if ty.TYPE_CHECKING:
     import pydra.engine.core
@@ -228,3 +229,39 @@ def import_extras_module(klass: ty.Type["fileformats.core.DataType"]) -> ExtrasM
     else:
         extras_imported = True
     return ExtrasModule(extras_imported, extras_pkg, extras_pypi)
+
+
+TypeType = ty.TypeVar("TypeType", bound=ty.Type[ty.Any])
+
+
+def get_optional_type(
+    type_: ty.Union[TypeType, ty.Type[ty.Optional[TypeType]]], allowed: bool = True
+) -> TypeType:
+    """Checks if a type is an Optional type
+
+    Parameters
+    ----------
+    type_ : ty.Type
+        the type to check
+    allowed : bool
+        whether Optional types are allowed or not
+
+    Returns
+    -------
+    bool
+        whether the type is an Optional type or not
+    """
+    if ty.get_origin(type_) is None:
+        return type_  # type: ignore[return-value]
+    if not allowed:
+        raise FormatDefinitionError(
+            f"Optional types are not allowed in content_type definitions ({type_}) "
+            "in this context"
+        )
+    args = ty.get_args(type_)
+    if len(args) != 2 and None in ty.get_args(type_):
+        raise FormatDefinitionError(
+            "Only Optional types are allowed in content_type definitions, "
+            f"not {type_}"
+        )
+    return args[0] if args[0] is not None else args[1]  # type: ignore[no-any-return]
diff --git a/fileformats/generic/directory.py b/fileformats/generic/directory.py
index 2be2b43..6f09fa9 100644
--- a/fileformats/generic/directory.py
+++ b/fileformats/generic/directory.py
@@ -110,4 +110,5 @@ class DirectoryOf(WithClassifiers, TypedDirectory):  # type: ignore[misc]
     # WithClassifiers-required class attrs
     classifiers_attr_name = "content_types"
     allowed_classifiers = (FileSet,)
+    allow_optional_classifiers = True
     generically_classifiable = True
diff --git a/fileformats/generic/set.py b/fileformats/generic/set.py
index adc34cc..2f5ccec 100644
--- a/fileformats/generic/set.py
+++ b/fileformats/generic/set.py
@@ -1,8 +1,10 @@
 import typing as ty
+import itertools
 from pathlib import Path
-from fileformats.core import FileSet
+from fileformats.core import FileSet, validated_property
 from fileformats.core.mixin import WithClassifiers
 from fileformats.core.collection import TypedCollection
+from fileformats.core.exceptions import FormatMismatchError
 
 
 class TypedSet(TypedCollection):
@@ -25,9 +27,21 @@ def __repr__(self) -> str:
             paths_repr += ", ..."
         return f"{self.type_name}({paths_repr})"
 
+    @validated_property
+    def _all_paths_used(self) -> None:
+        all_contents_paths = set(itertools.chain(*(c.fspaths for c in self.contents)))
+        missing = self.fspaths - all_contents_paths
+        if missing:
+            contents_str = "\n".join(repr(c) for c in self.contents)
+            raise FormatMismatchError(
+                f"Paths {[str(p) for p in missing]} are not used by any of the "
+                f"contents of {self.type_name}:\n{contents_str}"
+            )
+
 
 class SetOf(WithClassifiers, TypedSet):  # type: ignore[misc]
     # WithClassifiers-required class attrs
     classifiers_attr_name = "content_types"
     allowed_classifiers = (FileSet,)
+    allow_optional_classifiers = True
     generically_classifiable = True
diff --git a/fileformats/generic/tests/test_directory.py b/fileformats/generic/tests/test_directory.py
index 008b292..4cf2916 100644
--- a/fileformats/generic/tests/test_directory.py
+++ b/fileformats/generic/tests/test_directory.py
@@ -1,5 +1,8 @@
+import typing as ty
+import pytest
 from fileformats.generic import Directory, DirectoryOf, SetOf
-from fileformats.testing import MyFormatGz
+from fileformats.testing import MyFormatGz, YourFormat, EncodedText
+from fileformats.core.exceptions import FormatMismatchError
 
 
 def test_sample_directory():
@@ -16,3 +19,43 @@ def test_sample_set_of():
     sample = SetOf[MyFormatGz].sample()
     assert isinstance(sample, SetOf)
     assert all(isinstance(c, MyFormatGz) for c in sample.contents)
+
+
+def test_directory_optional_contents(tmp_path):
+    my_format = MyFormatGz.sample(dest_dir=tmp_path)
+    sample_dir = DirectoryOf[MyFormatGz](tmp_path)
+    EncodedText.sample(dest_dir=tmp_path)
+    assert sample_dir.contents == [my_format]
+
+    with pytest.raises(
+        FormatMismatchError, match="Did not find the required content types"
+    ):
+        DirectoryOf[MyFormatGz, YourFormat](sample_dir)
+
+    optional_dir = DirectoryOf[MyFormatGz, ty.Optional[YourFormat]](sample_dir)
+    assert optional_dir.contents == [my_format]
+    your_format = YourFormat.sample(dest_dir=tmp_path)
+    optional_dir = DirectoryOf[MyFormatGz, ty.Optional[YourFormat]](sample_dir)
+    assert optional_dir.contents == [my_format, your_format]
+    required_dir = DirectoryOf[MyFormatGz, YourFormat](sample_dir)
+    assert required_dir.contents == [my_format, your_format]
+
+
+def test_set_optional_contents():
+    my_format = MyFormatGz.sample()
+    your_format = YourFormat.sample()
+
+    sample_set = SetOf[MyFormatGz, YourFormat](my_format, your_format)
+    assert sample_set.contents == [my_format, your_format]
+    with pytest.raises(
+        FormatMismatchError, match="are not used by any of the contents of "
+    ):
+        SetOf[MyFormatGz](my_format, your_format)
+    with pytest.raises(
+        FormatMismatchError, match="Did not find the required content types"
+    ):
+        SetOf[MyFormatGz, YourFormat](my_format)
+    sample_set = SetOf[MyFormatGz, ty.Optional[YourFormat]](my_format)
+    assert sample_set.contents == [my_format]
+    sample_set = SetOf[MyFormatGz, ty.Optional[YourFormat]](my_format, your_format)
+    assert sample_set.contents == [my_format, your_format]

From 233d07936f1a4b5d44694dce2f8207f89b9f3ab3 Mon Sep 17 00:00:00 2001
From: Tom Close <tom.g.close@gmail.com>
Date: Fri, 20 Sep 2024 16:16:14 +1000
Subject: [PATCH 3/3] fixed up bug

---
 fileformats/core/mixin.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fileformats/core/mixin.py b/fileformats/core/mixin.py
index 1df6863..e296afc 100644
--- a/fileformats/core/mixin.py
+++ b/fileformats/core/mixin.py
@@ -442,7 +442,9 @@ def __class_getitem__(
             class_attrs[cls.classifiers_attr_name] = (
                 classifiers_tuple if cls.multiple_classifiers else classifiers_tuple[0]
             )
-            classifier_names = [t.__name__ for t in classifiers_tuple]
+            classifier_names = [
+                get_optional_type(t).__name__ for t in classifiers_tuple
+            ]
             if not cls.ordered_classifiers:
                 classifier_names.sort()
             classified = type(