From 1d49c4242ed82f79b9436edcadae1b29e297da65 Mon Sep 17 00:00:00 2001 From: David Souther Date: Wed, 28 Feb 2024 14:47:38 -0500 Subject: [PATCH 1/9] DocGen languages() -> List[str] --- aws_doc_sdk_examples_tools/doc_gen.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/aws_doc_sdk_examples_tools/doc_gen.py b/aws_doc_sdk_examples_tools/doc_gen.py index 68809f6..853dec3 100644 --- a/aws_doc_sdk_examples_tools/doc_gen.py +++ b/aws_doc_sdk_examples_tools/doc_gen.py @@ -43,6 +43,13 @@ def collect_snippets(self, snippets_root: Optional[Path]): self.snippets = snippets self.errors.extend(errs) + def languages(self) -> Set[str]: + languages: Set[str] = set() + for sdk_name, sdk in self.sdks.items(): + for version in sdk.versions: + languages += f"{sdk_name}:{version.version}" + return languages + @classmethod def from_root(cls, root: Path) -> "DocGen": errors = MetadataErrors() From bf692ab7513f35585725162143f2bee24d3c5c18 Mon Sep 17 00:00:00 2001 From: David Souther Date: Wed, 28 Feb 2024 14:47:53 -0500 Subject: [PATCH 2/9] py.typed --- aws_doc_sdk_examples_tools/py.typed | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 aws_doc_sdk_examples_tools/py.typed diff --git a/aws_doc_sdk_examples_tools/py.typed b/aws_doc_sdk_examples_tools/py.typed new file mode 100644 index 0000000..e69de29 From 0ba4af84faefc2e9c7f277ce2b70f9c57bc2e238 Mon Sep 17 00:00:00 2001 From: David Souther Date: Mon, 4 Mar 2024 10:24:11 -0500 Subject: [PATCH 3/9] Working on correct project structure for internal and external tooling --- MANIFEST.in | 2 +- .../config}/.yamllint.yaml | 0 .../config}/curated_example_schema.yaml | 0 .../config}/example_schema.yaml | 0 .../config}/example_strict_schema.yaml | 0 .../config}/sdks.yaml | 0 .../config}/sdks_schema.yaml | 0 .../config}/services.yaml | 0 .../config}/services_schema.yaml | 0 aws_doc_sdk_examples_tools/doc_gen.py | 67 ++++++++++++++++--- aws_doc_sdk_examples_tools/metadata.py | 8 ++- .../metadata_validator.py | 9 ++- aws_doc_sdk_examples_tools/snippets.py | 20 +++--- setup.py | 1 + 14 files changed, 78 insertions(+), 29 deletions(-) rename {config => aws_doc_sdk_examples_tools/config}/.yamllint.yaml (100%) rename {config => aws_doc_sdk_examples_tools/config}/curated_example_schema.yaml (100%) rename {config => aws_doc_sdk_examples_tools/config}/example_schema.yaml (100%) rename {config => aws_doc_sdk_examples_tools/config}/example_strict_schema.yaml (100%) rename {config => aws_doc_sdk_examples_tools/config}/sdks.yaml (100%) rename {config => aws_doc_sdk_examples_tools/config}/sdks_schema.yaml (100%) rename {config => aws_doc_sdk_examples_tools/config}/services.yaml (100%) rename {config => aws_doc_sdk_examples_tools/config}/services_schema.yaml (100%) diff --git a/MANIFEST.in b/MANIFEST.in index a029c65..f867763 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -graft config +include aws_doc_sdk_examples_tools/config/*.yaml diff --git a/config/.yamllint.yaml b/aws_doc_sdk_examples_tools/config/.yamllint.yaml similarity index 100% rename from config/.yamllint.yaml rename to aws_doc_sdk_examples_tools/config/.yamllint.yaml diff --git a/config/curated_example_schema.yaml b/aws_doc_sdk_examples_tools/config/curated_example_schema.yaml similarity index 100% rename from config/curated_example_schema.yaml rename to aws_doc_sdk_examples_tools/config/curated_example_schema.yaml diff --git a/config/example_schema.yaml b/aws_doc_sdk_examples_tools/config/example_schema.yaml similarity index 100% rename from config/example_schema.yaml rename to aws_doc_sdk_examples_tools/config/example_schema.yaml diff --git a/config/example_strict_schema.yaml b/aws_doc_sdk_examples_tools/config/example_strict_schema.yaml similarity index 100% rename from config/example_strict_schema.yaml rename to aws_doc_sdk_examples_tools/config/example_strict_schema.yaml diff --git a/config/sdks.yaml b/aws_doc_sdk_examples_tools/config/sdks.yaml similarity index 100% rename from config/sdks.yaml rename to aws_doc_sdk_examples_tools/config/sdks.yaml diff --git a/config/sdks_schema.yaml b/aws_doc_sdk_examples_tools/config/sdks_schema.yaml similarity index 100% rename from config/sdks_schema.yaml rename to aws_doc_sdk_examples_tools/config/sdks_schema.yaml diff --git a/config/services.yaml b/aws_doc_sdk_examples_tools/config/services.yaml similarity index 100% rename from config/services.yaml rename to aws_doc_sdk_examples_tools/config/services.yaml diff --git a/config/services_schema.yaml b/aws_doc_sdk_examples_tools/config/services_schema.yaml similarity index 100% rename from config/services_schema.yaml rename to aws_doc_sdk_examples_tools/config/services_schema.yaml diff --git a/aws_doc_sdk_examples_tools/doc_gen.py b/aws_doc_sdk_examples_tools/doc_gen.py index 853dec3..fad4e34 100644 --- a/aws_doc_sdk_examples_tools/doc_gen.py +++ b/aws_doc_sdk_examples_tools/doc_gen.py @@ -10,7 +10,7 @@ # from os import glob from aws_doc_sdk_examples_tools.metadata import Example, parse as parse_examples -from aws_doc_sdk_examples_tools.metadata_errors import MetadataErrors +from aws_doc_sdk_examples_tools.metadata_errors import MetadataErrors, MetadataError from aws_doc_sdk_examples_tools.metadata_validator import validate_metadata from aws_doc_sdk_examples_tools.project_validator import ( check_files, @@ -25,6 +25,11 @@ ) +@dataclass +class DocGenMergeWarning(MetadataError): + pass + + @dataclass class DocGen: root: Path @@ -36,10 +41,16 @@ class DocGen: examples: List[Example] = field(default_factory=list) cross_blocks: Set[str] = field(default_factory=set) - def collect_snippets(self, snippets_root: Optional[Path]): + def collect_snippets( + self, snippets_root: Optional[Path] = None, prefix: Optional[str] = None + ): if snippets_root is None: snippets_root = self.root.parent.parent - snippets, errs = collect_snippets(snippets_root) + if prefix is not None: + prefix = f"{prefix}_" + if prefix is None: + prefix = "" + snippets, errs = collect_snippets(snippets_root, prefix) self.snippets = snippets self.errors.extend(errs) @@ -47,25 +58,59 @@ def languages(self) -> Set[str]: languages: Set[str] = set() for sdk_name, sdk in self.sdks.items(): for version in sdk.versions: - languages += f"{sdk_name}:{version.version}" + languages.add(f"{sdk_name}:{version.version}") return languages + def merge(self, other: "DocGen") -> MetadataErrors: + """Merge fiends from other into self, prioritizing self fields""" + warnings = MetadataErrors() + for name, sdk in other.sdks.items(): + if name not in self.sdks: + self.sdks[name] = sdk + else: + warnings.append( + DocGenMergeWarning( + file=str(other.root), id=f"conflict in sdk {name}" + ) + ) + for name, service in other.services.items(): + if name not in self.services: + self.services[name] = service + warnings.append( + DocGenMergeWarning( + file=str(other.root), id=f"conflict in service {name}" + ) + ) + for name, snippet in other.snippets.items(): + if name not in self.snippets: + self.snippets[name] = snippet + warnings.append( + DocGenMergeWarning( + file=str(other.root), id=f"conflict in snippet {name}" + ) + ) + + self.snippet_files.update(other.snippet_files) + self.cross_blocks.update(other.cross_blocks) + self.examples += other.examples + + return warnings + @classmethod - def from_root(cls, root: Path) -> "DocGen": + def from_root(cls, root: Path, config: Optional[Path] = None) -> "DocGen": errors = MetadataErrors() metadata = root / ".doc_gen/metadata" - with open( - Path(__file__).parent.parent / "config" / "sdks.yaml", encoding="utf-8" - ) as file: + if config is None: + config = Path(__file__).parent / "config" + + with open(config / "sdks.yaml", encoding="utf-8") as file: meta = yaml.safe_load(file) sdks, errs = parse_sdks("sdks.yaml", meta) errors.extend(errs) - with open( - Path(__file__).parent.parent / "config" / "services.yaml", encoding="utf-8" - ) as file: + with open(config / "services.yaml", encoding="utf-8") as file: meta = yaml.safe_load(file) services, service_errors = parse_services("services.yaml", meta) errors.extend(service_errors) diff --git a/aws_doc_sdk_examples_tools/metadata.py b/aws_doc_sdk_examples_tools/metadata.py index a099e8d..78056cc 100755 --- a/aws_doc_sdk_examples_tools/metadata.py +++ b/aws_doc_sdk_examples_tools/metadata.py @@ -275,12 +275,14 @@ def parse_services( if name not in known_services: errors.append(metadata_errors.UnknownService(service=name)) else: - service: Dict[str, None] | None = yaml.get(name) + service: Dict[str, None] | Set[str] | None = yaml.get(name) # While .get replaces missing with {}, `sqs: ` in yaml parses a literal `None` + if isinstance(service, dict): + service = set(service.keys()) if service is None: - service = {} + service = set() # Make a copy of the dict - services[name] = [*service.keys()] + services[name] = [*service] return services diff --git a/aws_doc_sdk_examples_tools/metadata_validator.py b/aws_doc_sdk_examples_tools/metadata_validator.py index 232520d..62445bd 100755 --- a/aws_doc_sdk_examples_tools/metadata_validator.py +++ b/aws_doc_sdk_examples_tools/metadata_validator.py @@ -185,12 +185,11 @@ def validate_files( def validate_metadata(doc_gen_root: Path, errors: MetadataErrors) -> MetadataErrors: - with open(Path(__file__).parent.parent / "config" / "sdks.yaml") as sdks_file: + config = Path(__file__).parent / "config" + with open(config / "sdks.yaml") as sdks_file: sdks_yaml: Dict[str, Any] = yaml.safe_load(sdks_file) - with open( - Path(__file__).parent.parent / "config" / "services.yaml" - ) as services_file: + with open(config / "services.yaml") as services_file: services_yaml = yaml.safe_load(services_file) SdkVersion.sdks = sdks_yaml @@ -207,7 +206,7 @@ def validate_metadata(doc_gen_root: Path, errors: MetadataErrors) -> MetadataErr validators[BlockContent.tag] = BlockContent validators[String.tag] = StringExtension - schema_root = Path(__file__).parent.parent / "config" + schema_root = Path(__file__).parent / "config" to_validate = [ # (schema, metadata_glob) diff --git a/aws_doc_sdk_examples_tools/snippets.py b/aws_doc_sdk_examples_tools/snippets.py index 27e9a91..f451f01 100644 --- a/aws_doc_sdk_examples_tools/snippets.py +++ b/aws_doc_sdk_examples_tools/snippets.py @@ -81,21 +81,21 @@ def message(self): return f" unicode error: {str(self.err)}" -def _tag_from_line(token: str, line: str) -> str: +def _tag_from_line(token: str, line: str, prefix: str) -> str: tag_start = line.find(token) + len(token) tag_end = line.find("]", tag_start) - return line[tag_start:tag_end].strip() + return prefix + line[tag_start:tag_end].strip() def parse_snippets( - lines: List[str], file: Path + lines: List[str], file: Path, prefix: str ) -> Tuple[Dict[str, Snippet], MetadataErrors]: snippets: Dict[str, Snippet] = {} errors = MetadataErrors() open_tags: Set[str] = set() for line_idx, line in enumerate(lines): if SNIPPET_START in line: - tag = _tag_from_line(SNIPPET_START, line) + tag = _tag_from_line(SNIPPET_START, line, prefix) if tag in snippets: errors.append( DuplicateSnippetStartError(file=str(file), line=line_idx, tag=tag) @@ -110,7 +110,7 @@ def parse_snippets( ) open_tags.add(tag) elif SNIPPET_END in line: - tag = _tag_from_line(SNIPPET_END, line) + tag = _tag_from_line(SNIPPET_END, line, prefix) if tag not in snippets: errors.append( MissingSnippetStartError(file=str(file), line=line_idx, tag=tag) @@ -135,23 +135,25 @@ def parse_snippets( return snippets, errors -def find_snippets(file: Path) -> Tuple[Dict[str, Snippet], MetadataErrors]: +def find_snippets(file: Path, prefix: str) -> Tuple[Dict[str, Snippet], MetadataErrors]: errors = MetadataErrors() snippets: Dict[str, Snippet] = {} with open(file, encoding="utf-8") as snippet_file: try: - snippets, errs = parse_snippets(snippet_file.readlines(), file) + snippets, errs = parse_snippets(snippet_file.readlines(), file, prefix) errors.extend(errs) except UnicodeDecodeError as err: errors.append(MetadataUnicodeError(file=str(file), err=err)) return snippets, errors -def collect_snippets(root: Path) -> Tuple[Dict[str, Snippet], MetadataErrors]: +def collect_snippets( + root: Path, prefix: str = "" +) -> Tuple[Dict[str, Snippet], MetadataErrors]: snippets: Dict[str, Snippet] = {} errors = MetadataErrors() for file in get_files(root, validator_config.skip): - snips, errs = find_snippets(file) + snips, errs = find_snippets(file, prefix) snippets.update(snips) errors.extend(errs) return snippets, errors diff --git a/setup.py b/setup.py index 382fe9f..fd8196e 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ name="aws_doc_sdk_examples_tools", version="0.0.1", packages=["aws_doc_sdk_examples_tools"], + package_data={"aws_doc_sdk_examples_tools": ["config/*.yaml"]}, install_requires=[ "pathspec==0.11.2", "PyYAML==6.0.1", From cc1bbca0eee648488feebedf73c123274b084d12 Mon Sep 17 00:00:00 2001 From: David Souther Date: Mon, 4 Mar 2024 12:55:57 -0500 Subject: [PATCH 4/9] use . local imports --- aws_doc_sdk_examples_tools/doc_gen.py | 14 +++++++------- aws_doc_sdk_examples_tools/metadata.py | 8 ++++---- aws_doc_sdk_examples_tools/metadata_test.py | 8 ++++---- aws_doc_sdk_examples_tools/metadata_validator.py | 2 +- aws_doc_sdk_examples_tools/project_validator.py | 6 +++--- .../project_validator_test.py | 2 +- aws_doc_sdk_examples_tools/sdks.py | 2 +- aws_doc_sdk_examples_tools/sdks_test.py | 2 +- aws_doc_sdk_examples_tools/services.py | 2 +- aws_doc_sdk_examples_tools/services_test.py | 2 +- aws_doc_sdk_examples_tools/snippets.py | 6 +++--- aws_doc_sdk_examples_tools/spdx.py | 2 +- aws_doc_sdk_examples_tools/validate.py | 2 +- 13 files changed, 29 insertions(+), 29 deletions(-) diff --git a/aws_doc_sdk_examples_tools/doc_gen.py b/aws_doc_sdk_examples_tools/doc_gen.py index fad4e34..bf9bd5c 100644 --- a/aws_doc_sdk_examples_tools/doc_gen.py +++ b/aws_doc_sdk_examples_tools/doc_gen.py @@ -9,16 +9,16 @@ # from os import glob -from aws_doc_sdk_examples_tools.metadata import Example, parse as parse_examples -from aws_doc_sdk_examples_tools.metadata_errors import MetadataErrors, MetadataError -from aws_doc_sdk_examples_tools.metadata_validator import validate_metadata -from aws_doc_sdk_examples_tools.project_validator import ( +from .metadata import Example, parse as parse_examples +from .metadata_errors import MetadataErrors, MetadataError +from .metadata_validator import validate_metadata +from .project_validator import ( check_files, verify_sample_files, ) -from aws_doc_sdk_examples_tools.sdks import Sdk, parse as parse_sdks -from aws_doc_sdk_examples_tools.services import Service, parse as parse_services -from aws_doc_sdk_examples_tools.snippets import ( +from .sdks import Sdk, parse as parse_sdks +from .services import Service, parse as parse_services +from .snippets import ( Snippet, collect_snippets, validate_snippets, diff --git a/aws_doc_sdk_examples_tools/metadata.py b/aws_doc_sdk_examples_tools/metadata.py index 78056cc..5d88f1d 100755 --- a/aws_doc_sdk_examples_tools/metadata.py +++ b/aws_doc_sdk_examples_tools/metadata.py @@ -9,14 +9,14 @@ from os.path import splitext from aws_doc_sdk_examples_tools import metadata_errors -from aws_doc_sdk_examples_tools.metadata_errors import ( +from .metadata_errors import ( MetadataErrors, MetadataParseError, DuplicateItemException, ) -from aws_doc_sdk_examples_tools.metadata_validator import StringExtension -from aws_doc_sdk_examples_tools.services import Service -from aws_doc_sdk_examples_tools.sdks import Sdk +from .metadata_validator import StringExtension +from .services import Service +from .sdks import Sdk @dataclass diff --git a/aws_doc_sdk_examples_tools/metadata_test.py b/aws_doc_sdk_examples_tools/metadata_test.py index dd70219..6073d71 100644 --- a/aws_doc_sdk_examples_tools/metadata_test.py +++ b/aws_doc_sdk_examples_tools/metadata_test.py @@ -11,7 +11,7 @@ from typing import List, Set, Tuple from aws_doc_sdk_examples_tools import metadata_errors -from aws_doc_sdk_examples_tools.metadata import ( +from .metadata import ( parse, Example, Url, @@ -20,9 +20,9 @@ Excerpt, idFormat, ) -from aws_doc_sdk_examples_tools.doc_gen import DocGen -from aws_doc_sdk_examples_tools.sdks import Sdk -from aws_doc_sdk_examples_tools.services import Service, ServiceExpanded +from .doc_gen import DocGen +from .sdks import Sdk +from .services import Service, ServiceExpanded def load( diff --git a/aws_doc_sdk_examples_tools/metadata_validator.py b/aws_doc_sdk_examples_tools/metadata_validator.py index 62445bd..ac0598f 100755 --- a/aws_doc_sdk_examples_tools/metadata_validator.py +++ b/aws_doc_sdk_examples_tools/metadata_validator.py @@ -21,7 +21,7 @@ from yamale import YamaleError # type: ignore from yamale.validators import DefaultValidators, Validator, String # type: ignore -from aws_doc_sdk_examples_tools.metadata_errors import ( +from .metadata_errors import ( MetadataErrors, MetadataParseError, ) diff --git a/aws_doc_sdk_examples_tools/project_validator.py b/aws_doc_sdk_examples_tools/project_validator.py index 994be35..277cf3b 100644 --- a/aws_doc_sdk_examples_tools/project_validator.py +++ b/aws_doc_sdk_examples_tools/project_validator.py @@ -27,14 +27,14 @@ from pathlib import Path from typing import List -from aws_doc_sdk_examples_tools.file_utils import get_files -from aws_doc_sdk_examples_tools.metadata_errors import ( +from .file_utils import get_files +from .metadata_errors import ( MetadataErrors, MetadataError, MetadataParseError, DuplicateItemException, ) -from aws_doc_sdk_examples_tools.spdx import verify_spdx +from .spdx import verify_spdx from aws_doc_sdk_examples_tools import validator_config logger = logging.getLogger(__name__) diff --git a/aws_doc_sdk_examples_tools/project_validator_test.py b/aws_doc_sdk_examples_tools/project_validator_test.py index a42e490..7b30987 100644 --- a/aws_doc_sdk_examples_tools/project_validator_test.py +++ b/aws_doc_sdk_examples_tools/project_validator_test.py @@ -10,7 +10,7 @@ from typing import List from aws_doc_sdk_examples_tools import project_validator -from aws_doc_sdk_examples_tools.metadata_errors import MetadataErrors +from .metadata_errors import MetadataErrors @pytest.mark.parametrize( diff --git a/aws_doc_sdk_examples_tools/sdks.py b/aws_doc_sdk_examples_tools/sdks.py index 2618f82..ff689ec 100644 --- a/aws_doc_sdk_examples_tools/sdks.py +++ b/aws_doc_sdk_examples_tools/sdks.py @@ -7,7 +7,7 @@ from dataclasses import dataclass, field from aws_doc_sdk_examples_tools import metadata_errors -from aws_doc_sdk_examples_tools.metadata_errors import ( +from .metadata_errors import ( MetadataErrors, MetadataParseError, check_mapping, diff --git a/aws_doc_sdk_examples_tools/sdks_test.py b/aws_doc_sdk_examples_tools/sdks_test.py index ae76464..4717c32 100644 --- a/aws_doc_sdk_examples_tools/sdks_test.py +++ b/aws_doc_sdk_examples_tools/sdks_test.py @@ -11,7 +11,7 @@ from typing import Dict, Tuple from aws_doc_sdk_examples_tools import metadata_errors -from aws_doc_sdk_examples_tools.sdks import ( +from .sdks import ( parse, Sdk, SdkVersion, diff --git a/aws_doc_sdk_examples_tools/services.py b/aws_doc_sdk_examples_tools/services.py index d82d460..b905501 100644 --- a/aws_doc_sdk_examples_tools/services.py +++ b/aws_doc_sdk_examples_tools/services.py @@ -6,7 +6,7 @@ from typing import Any, Dict, Optional, Set, Union from dataclasses import dataclass, field from aws_doc_sdk_examples_tools import metadata_errors -from aws_doc_sdk_examples_tools.metadata_errors import MetadataErrors, check_mapping +from .metadata_errors import MetadataErrors, check_mapping @dataclass diff --git a/aws_doc_sdk_examples_tools/services_test.py b/aws_doc_sdk_examples_tools/services_test.py index 47f7173..0080125 100644 --- a/aws_doc_sdk_examples_tools/services_test.py +++ b/aws_doc_sdk_examples_tools/services_test.py @@ -7,7 +7,7 @@ import yaml from aws_doc_sdk_examples_tools import metadata_errors -from aws_doc_sdk_examples_tools.services import ( +from .services import ( parse, Service, ServiceGuide, diff --git a/aws_doc_sdk_examples_tools/snippets.py b/aws_doc_sdk_examples_tools/snippets.py index f451f01..4249c88 100644 --- a/aws_doc_sdk_examples_tools/snippets.py +++ b/aws_doc_sdk_examples_tools/snippets.py @@ -9,9 +9,9 @@ from aws_doc_sdk_examples_tools import validator_config -from aws_doc_sdk_examples_tools.file_utils import get_files, clear -from aws_doc_sdk_examples_tools.metadata import Example -from aws_doc_sdk_examples_tools.metadata_errors import MetadataErrors, MetadataError +from .file_utils import get_files, clear +from .metadata import Example +from .metadata_errors import MetadataErrors, MetadataError SNIPPET_START = "snippet-start:[" SNIPPET_END = "snippet-end:[" diff --git a/aws_doc_sdk_examples_tools/spdx.py b/aws_doc_sdk_examples_tools/spdx.py index 7c36e34..d8e91c9 100644 --- a/aws_doc_sdk_examples_tools/spdx.py +++ b/aws_doc_sdk_examples_tools/spdx.py @@ -8,7 +8,7 @@ import re from sys import argv -from aws_doc_sdk_examples_tools.metadata_errors import MetadataError, MetadataErrors +from .metadata_errors import MetadataError, MetadataErrors from aws_doc_sdk_examples_tools import validator_config diff --git a/aws_doc_sdk_examples_tools/validate.py b/aws_doc_sdk_examples_tools/validate.py index fe28ad8..54b132c 100755 --- a/aws_doc_sdk_examples_tools/validate.py +++ b/aws_doc_sdk_examples_tools/validate.py @@ -5,7 +5,7 @@ from pathlib import Path from sys import exit -from aws_doc_sdk_examples_tools.doc_gen import DocGen +from .doc_gen import DocGen def main(): From 07bc69f13b940e00a07659e572e28bb0e6be64dc Mon Sep 17 00:00:00 2001 From: David Souther Date: Tue, 5 Mar 2024 16:04:31 -0500 Subject: [PATCH 5/9] Handle merging multiple doc_gens --- aws_doc_sdk_examples_tools/doc_gen.py | 53 +++++++++++++-------- aws_doc_sdk_examples_tools/snippets.py | 7 ++- aws_doc_sdk_examples_tools/snippets_test.py | 2 +- 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/aws_doc_sdk_examples_tools/doc_gen.py b/aws_doc_sdk_examples_tools/doc_gen.py index bf9bd5c..3d6c8eb 100644 --- a/aws_doc_sdk_examples_tools/doc_gen.py +++ b/aws_doc_sdk_examples_tools/doc_gen.py @@ -44,12 +44,12 @@ class DocGen: def collect_snippets( self, snippets_root: Optional[Path] = None, prefix: Optional[str] = None ): - if snippets_root is None: - snippets_root = self.root.parent.parent if prefix is not None: prefix = f"{prefix}_" if prefix is None: prefix = "" + if snippets_root is None: + snippets_root = self.root snippets, errs = collect_snippets(snippets_root, prefix) self.snippets = snippets self.errors.extend(errs) @@ -97,9 +97,23 @@ def merge(self, other: "DocGen") -> MetadataErrors: return warnings @classmethod - def from_root(cls, root: Path, config: Optional[Path] = None) -> "DocGen": - errors = MetadataErrors() + def empty(cls) -> "DocGen": + return DocGen(root=Path("/"), errors=MetadataErrors()) + + def clone(self) -> "DocGen": + return DocGen( + root=self.root, + errors=MetadataErrors(), + sdks={**self.sdks}, + services={**self.services}, + snippets={}, + snippet_files=set(), + cross_blocks=set(), + examples=[], + ) + def for_root(self, root: Path, config: Optional[Path] = None) -> "DocGen": + self.root = root metadata = root / ".doc_gen/metadata" if config is None: @@ -108,38 +122,39 @@ def from_root(cls, root: Path, config: Optional[Path] = None) -> "DocGen": with open(config / "sdks.yaml", encoding="utf-8") as file: meta = yaml.safe_load(file) sdks, errs = parse_sdks("sdks.yaml", meta) - errors.extend(errs) + self.errors.extend(errs) with open(config / "services.yaml", encoding="utf-8") as file: meta = yaml.safe_load(file) services, service_errors = parse_services("services.yaml", meta) - errors.extend(service_errors) + self.errors.extend(service_errors) cross = set( [path.name for path in (metadata.parent / "cross-content").glob("*.xml")] ) - doc_gen = cls( - root=root, - sdks=sdks, - services=services, - errors=errors, - cross_blocks=cross, - ) + self.root = root + self.sdks = sdks + self.services = services + self.cross_blocks = cross for path in metadata.glob("*_metadata.yaml"): with open(path) as file: ex, errs = parse_examples( path.name, yaml.safe_load(file), - doc_gen.sdks, - doc_gen.services, - doc_gen.cross_blocks, + self.sdks, + self.services, + self.cross_blocks, ) - doc_gen.examples.extend(ex) - errors.extend(errs) + self.examples.extend(ex) + self.errors.extend(errs) + + return self - return doc_gen + @classmethod + def from_root(cls, root: Path, config: Optional[Path] = None) -> "DocGen": + return DocGen.empty().for_root(root, config) def validate(self, check_spdx: bool): check_files(self.root, self.errors, check_spdx) diff --git a/aws_doc_sdk_examples_tools/snippets.py b/aws_doc_sdk_examples_tools/snippets.py index 4249c88..fc428cf 100644 --- a/aws_doc_sdk_examples_tools/snippets.py +++ b/aws_doc_sdk_examples_tools/snippets.py @@ -230,11 +230,11 @@ def validate_snippets( snippet_files.add(snippet_file) -def write_snippets(root: Path, snippets: Dict[str, Snippet]): +def write_snippets(root: Path, snippets: Dict[str, Snippet], check: bool = False): errors = MetadataErrors() for tag in snippets: name = root / f"{tag}.txt" - if name.exists(): + if check and name.exists(): errors.append(SnippetAlreadyWritten(file=str(name))) else: try: @@ -248,8 +248,7 @@ def write_snippets(root: Path, snippets: Dict[str, Snippet]): def write_snippet_file(folder: Path, snippet_file: Path): name = str(snippet_file).replace("/", ".") dest = folder / f"{name}.txt" - if not dest.exists(): - copyfile(folder / snippet_file, dest) + copyfile(folder / snippet_file, dest) def main(): diff --git a/aws_doc_sdk_examples_tools/snippets_test.py b/aws_doc_sdk_examples_tools/snippets_test.py index 79c242b..178ffb9 100644 --- a/aws_doc_sdk_examples_tools/snippets_test.py +++ b/aws_doc_sdk_examples_tools/snippets_test.py @@ -62,6 +62,6 @@ def test_verify_snippet_start_end(file_contents: str, expected_error_count: int): """Test that various kinds of mismatched snippet-start and -end tags are counted correctly as errors.""" - _, errors = snippets.parse_snippets(file_contents.split("\n"), Path("test")) + _, errors = snippets.parse_snippets(file_contents.split("\n"), Path("test"), "") error_count = len(errors) assert error_count == expected_error_count From 9897b15e81021508e83a763c3960d2083fc8e156 Mon Sep 17 00:00:00 2001 From: David Souther Date: Wed, 6 Mar 2024 10:25:44 -0500 Subject: [PATCH 6/9] Collect snippet_files immediately, then validate they exist later --- aws_doc_sdk_examples_tools/doc_gen.py | 10 +++++-- aws_doc_sdk_examples_tools/snippets.py | 37 +++++++++++++------------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/aws_doc_sdk_examples_tools/doc_gen.py b/aws_doc_sdk_examples_tools/doc_gen.py index 3d6c8eb..eb4c665 100644 --- a/aws_doc_sdk_examples_tools/doc_gen.py +++ b/aws_doc_sdk_examples_tools/doc_gen.py @@ -140,15 +140,21 @@ def for_root(self, root: Path, config: Optional[Path] = None) -> "DocGen": for path in metadata.glob("*_metadata.yaml"): with open(path) as file: - ex, errs = parse_examples( + examples, errs = parse_examples( path.name, yaml.safe_load(file), self.sdks, self.services, self.cross_blocks, ) - self.examples.extend(ex) + self.examples.extend(examples) self.errors.extend(errs) + for example in examples: + for lang in example.languages: + language = example.languages[lang] + for version in language.versions: + for excerpt in version.excerpts: + self.snippet_files.update(excerpt.snippet_files) return self diff --git a/aws_doc_sdk_examples_tools/snippets.py b/aws_doc_sdk_examples_tools/snippets.py index fc428cf..7236264 100644 --- a/aws_doc_sdk_examples_tools/snippets.py +++ b/aws_doc_sdk_examples_tools/snippets.py @@ -209,25 +209,24 @@ def validate_snippets( tag=snippet_tag, ) ) - for snippet_file in excerpt.snippet_files: - if not (root / snippet_file).exists(): - # Ensure all snippet_files exist - errors.append( - MissingSnippetFile( - file=example.file, - snippet_file=snippet_file, - id=f"{lang}:{version.sdk_version}", - ) - ) - if re.search(win_unsafe_re, str(snippet_file)): - errors.append( - WindowsUnsafeSnippetFile( - file=example.file, - snippet_file=snippet_file, - id=f"{lang}:{version.sdk_version}", - ) - ) - snippet_files.add(snippet_file) + for snippet_file in excerpt.snippet_files: + if not (root / snippet_file).exists(): + # Ensure all snippet_files exist + errors.append( + MissingSnippetFile( + file=example.file, + snippet_file=snippet_file, + id=f"{lang}:{version.sdk_version}", + ) + ) + if re.search(win_unsafe_re, str(snippet_file)): + errors.append( + WindowsUnsafeSnippetFile( + file=example.file, + snippet_file=snippet_file, + id=f"{lang}:{version.sdk_version}", + ) + ) def write_snippets(root: Path, snippets: Dict[str, Snippet], check: bool = False): From 87ef7022ddf5146a4ae730efcf7ef07c5e627206 Mon Sep 17 00:00:00 2001 From: David Souther Date: Thu, 7 Mar 2024 09:14:58 -0500 Subject: [PATCH 7/9] Merge logic for review; need tests --- aws_doc_sdk_examples_tools/doc_gen.py | 20 +++++++--- aws_doc_sdk_examples_tools/metadata.py | 52 ++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/aws_doc_sdk_examples_tools/doc_gen.py b/aws_doc_sdk_examples_tools/doc_gen.py index eb4c665..416aef2 100644 --- a/aws_doc_sdk_examples_tools/doc_gen.py +++ b/aws_doc_sdk_examples_tools/doc_gen.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, List, Optional, Set +from typing import Dict, Iterable, Optional, Set # from os import glob @@ -38,7 +38,7 @@ class DocGen: services: Dict[str, Service] = field(default_factory=dict) snippets: Dict[str, Snippet] = field(default_factory=dict) snippet_files: Set[str] = field(default_factory=set) - examples: List[Example] = field(default_factory=list) + examples: Dict[str, Example] = field(default_factory=dict) cross_blocks: Set[str] = field(default_factory=set) def collect_snippets( @@ -92,10 +92,18 @@ def merge(self, other: "DocGen") -> MetadataErrors: self.snippet_files.update(other.snippet_files) self.cross_blocks.update(other.cross_blocks) - self.examples += other.examples + self.extend_examples(other.examples.values()) return warnings + def extend_examples(self, examples: Iterable[Example]): + for example in examples: + id = example.id + if id in self.examples: + self.examples[id].merge(example, self.errors) + else: + self.examples[id] = example + @classmethod def empty(cls) -> "DocGen": return DocGen(root=Path("/"), errors=MetadataErrors()) @@ -109,7 +117,7 @@ def clone(self) -> "DocGen": snippets={}, snippet_files=set(), cross_blocks=set(), - examples=[], + examples={}, ) def for_root(self, root: Path, config: Optional[Path] = None) -> "DocGen": @@ -147,7 +155,7 @@ def for_root(self, root: Path, config: Optional[Path] = None) -> "DocGen": self.services, self.cross_blocks, ) - self.examples.extend(examples) + self.extend_examples(examples) self.errors.extend(errs) for example in examples: for lang in example.languages: @@ -167,7 +175,7 @@ def validate(self, check_spdx: bool): verify_sample_files(self.root, self.errors) validate_metadata(self.root, self.errors) validate_snippets( - self.examples, + [*self.examples.values()], self.snippets, self.snippet_files, self.errors, diff --git a/aws_doc_sdk_examples_tools/metadata.py b/aws_doc_sdk_examples_tools/metadata.py index 5d88f1d..13fd51b 100755 --- a/aws_doc_sdk_examples_tools/metadata.py +++ b/aws_doc_sdk_examples_tools/metadata.py @@ -10,6 +10,7 @@ from aws_doc_sdk_examples_tools import metadata_errors from .metadata_errors import ( + MetadataError, MetadataErrors, MetadataParseError, DuplicateItemException, @@ -143,6 +144,22 @@ class Language: name: str versions: List[Version] + def merge(self, other: "Language", errors: MetadataErrors): + """Add new versions from `other`""" + # TODO Error for mismatched names? + if self.name != other.name: + return + for other_version in other.versions: + self_version = filter( + lambda v: v.sdk_version == other_version.sdk_version, self.versions + ) + if self_version is None: + self.versions.append(other_version) + # Merge down to the SDK Version level, so later guides can add new + # excerpts to existing examples, but don't try to merge the excerpts + # within the language. If a tributary or writer feels they need to + # modify an excerpt, they should go modify the excerpt directly. + @classmethod def from_yaml( cls, @@ -177,6 +194,12 @@ def from_yaml( return cls(name, versions), errors +@dataclass +class ExampleMergeMismatchedId(MetadataError): + other_id: str = "" + other_file: str = "" + + @dataclass class Example: id: str @@ -199,6 +222,35 @@ class Example: synopsis_list: List[str] = field(default_factory=list) source_key: Optional[str] = field(default=None) + def merge(self, other: Example, errors: MetadataErrors): + """Combine `other` Example into self example. + + Merge down to the SDK Version level, so later guides can add new excerpts to existing examples, but don't try to merge the excerpts within the language. + If a tributary or writer feels they need to modify an excerpt, they should go modify the excerpt directly. + + Keep title, title_abbrev, synopsis, guide_topic, category, service_main, synopsis_list, and source_key from source (typically awsdocs/aws-doc-sdk-examples). + !NOTE: This means `merge` is NOT associative! + + Add error if IDs are not the same and return early. + """ + if self.id != other.id: + errors.append( + ExampleMergeMismatchedId( + id=self.id, other_id=other.id, file=self.file, other_file=other.file + ) + ) + return + + for service, actions in other.services.items(): + if service not in self.services: + self.services[service] = actions + + for name, language in other.languages.items(): + if name not in self.languages: + self.languages[name] = language + else: + self.languages[name].merge(language, errors) + @classmethod def from_yaml( cls, From 4ca27def42b18437084bd16a645eac860b4bc28e Mon Sep 17 00:00:00 2001 From: David Souther Date: Thu, 7 Mar 2024 13:03:45 -0500 Subject: [PATCH 8/9] First DocGen merge test --- aws_doc_sdk_examples_tools/doc_gen.py | 2 +- aws_doc_sdk_examples_tools/metadata_errors.py | 3 ++ aws_doc_sdk_examples_tools/metadata_test.py | 37 ++++++++++++++++++- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/aws_doc_sdk_examples_tools/doc_gen.py b/aws_doc_sdk_examples_tools/doc_gen.py index 416aef2..c83fa7d 100644 --- a/aws_doc_sdk_examples_tools/doc_gen.py +++ b/aws_doc_sdk_examples_tools/doc_gen.py @@ -62,7 +62,7 @@ def languages(self) -> Set[str]: return languages def merge(self, other: "DocGen") -> MetadataErrors: - """Merge fiends from other into self, prioritizing self fields""" + """Merge fields from other into self, prioritizing self fields""" warnings = MetadataErrors() for name, sdk in other.sdks.items(): if name not in self.sdks: diff --git a/aws_doc_sdk_examples_tools/metadata_errors.py b/aws_doc_sdk_examples_tools/metadata_errors.py index 982fb76..9a40f6e 100644 --- a/aws_doc_sdk_examples_tools/metadata_errors.py +++ b/aws_doc_sdk_examples_tools/metadata_errors.py @@ -101,6 +101,9 @@ def __str__(self) -> str: errs = "\n".join([f"\t{err}" for err in self._errors]) return f"ExampleErrors with {len(self)} errors:\n{errs}" + def __eq__(self, __value: object) -> bool: + return isinstance(__value, MetadataErrors) and self._errors == __value._errors + @dataclass class MissingServiceBody(MetadataParseError): diff --git a/aws_doc_sdk_examples_tools/metadata_test.py b/aws_doc_sdk_examples_tools/metadata_test.py index 6073d71..2c9dbbc 100644 --- a/aws_doc_sdk_examples_tools/metadata_test.py +++ b/aws_doc_sdk_examples_tools/metadata_test.py @@ -10,7 +10,8 @@ from pathlib import Path from typing import List, Set, Tuple -from aws_doc_sdk_examples_tools import metadata_errors +from . import metadata_errors +from .metadata_errors import MetadataErrors from .metadata import ( parse, Example, @@ -461,5 +462,39 @@ def test_idFormat(): assert not idFormat("test", TEST_SERVICES) +@pytest.mark.parametrize( + ["a", "b", "d"], + [ + ( + DocGen( + root=Path("/a"), + errors=MetadataErrors(), + sdks={ + "a": Sdk(name="a", guide="guide_a", property="a_prop", versions=[]) + }, + ), + DocGen( + root=Path("/b"), + errors=MetadataErrors(), + sdks={ + "b": Sdk(name="b", guide="guide_b", property="b_prop", versions=[]) + }, + ), + DocGen( + root=Path("/a"), + errors=MetadataErrors(), + sdks={ + "a": Sdk(name="a", guide="guide_a", property="a_prop", versions=[]), + "b": Sdk(name="b", guide="guide_b", property="b_prop", versions=[]), + }, + ), + ) + ], +) +def test_merge(a: DocGen, b: DocGen, d: DocGen): + a.merge(b) + assert a == d + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From 34f1f4e63d1db47e7081c4e0f37a682a1fb98823 Mon Sep 17 00:00:00 2001 From: David Souther Date: Mon, 11 Mar 2024 11:25:01 -0400 Subject: [PATCH 9/9] Code review comments --- aws_doc_sdk_examples_tools/doc_gen.py | 4 +++ aws_doc_sdk_examples_tools/metadata.py | 35 ++++++++++--------- aws_doc_sdk_examples_tools/metadata_test.py | 14 ++++---- .../project_validator.py | 1 - aws_doc_sdk_examples_tools/sdks.py | 10 ++++++ aws_doc_sdk_examples_tools/services.py | 3 ++ aws_doc_sdk_examples_tools/spdx.py | 2 ++ 7 files changed, 44 insertions(+), 25 deletions(-) diff --git a/aws_doc_sdk_examples_tools/doc_gen.py b/aws_doc_sdk_examples_tools/doc_gen.py index c83fa7d..33052f5 100644 --- a/aws_doc_sdk_examples_tools/doc_gen.py +++ b/aws_doc_sdk_examples_tools/doc_gen.py @@ -171,6 +171,10 @@ def from_root(cls, root: Path, config: Optional[Path] = None) -> "DocGen": return DocGen.empty().for_root(root, config) def validate(self, check_spdx: bool): + for sdk in self.sdks.values(): + sdk.validate(self.errors) + for service in self.services.values(): + service.validate(self.errors) check_files(self.root, self.errors, check_spdx) verify_sample_files(self.root, self.errors) validate_metadata(self.root, self.errors) diff --git a/aws_doc_sdk_examples_tools/metadata.py b/aws_doc_sdk_examples_tools/metadata.py index 13fd51b..75fd4a7 100755 --- a/aws_doc_sdk_examples_tools/metadata.py +++ b/aws_doc_sdk_examples_tools/metadata.py @@ -65,10 +65,10 @@ class Version: excerpts: List[Excerpt] = field(default_factory=list) # Link to the source code for this example. TODO rename. github: Optional[str] = field(default=None) - add_services: Dict[str, List[str]] = field(default_factory=dict) + add_services: Dict[str, Set[str]] = field(default_factory=dict) # Deprecated. Replace with guide_topic list. sdkguide: Optional[str] = field(default=None) - # Link to additional topic places. TODO: Overwritten by aws-doc-sdk-example when merging. + # Link to additional topic places. more_info: List[Url] = field(default_factory=list) @classmethod @@ -218,7 +218,7 @@ class Example: # TODO document service_main and services. Not to be used by tributaries. Part of Cross Service. # List of services used by the examples. Lines up with those in services.yaml. service_main: Optional[str] = field(default=None) - services: Dict[str, List[str]] = field(default_factory=dict) + services: Dict[str, Set[str]] = field(default_factory=dict) synopsis_list: List[str] = field(default_factory=list) source_key: Optional[str] = field(default=None) @@ -264,16 +264,20 @@ def from_yaml( title = get_with_valid_entities("title", yaml, errors) title_abbrev = get_with_valid_entities("title_abbrev", yaml, errors) synopsis = get_with_valid_entities("synopsis", yaml, errors, opt=True) + synopsis_list = [str(syn) for syn in yaml.get("synopsis_list", [])] - category = yaml.get("category", "") source_key = yaml.get("source_key") - parsed_services = parse_services(yaml.get("services", {}), errors, services) - synopsis_list = [str(syn) for syn in yaml.get("synopsis_list", [])] guide_topic = Url.from_yaml(yaml.get("guide_topic")) if isinstance(guide_topic, MetadataParseError): errors.append(guide_topic) guide_topic = None + parsed_services = parse_services(yaml.get("services", {}), errors, services) + category = yaml.get("category") + if category is None or category == "": + category = "Api" if len(parsed_services) == 1 else "Cross" + is_action = category == "Api" + service_main = yaml.get("service_main", None) if service_main is not None and service_main not in services: try: @@ -281,11 +285,6 @@ def from_yaml( except DuplicateItemException: pass - if category == "": - category = "Api" if len(parsed_services) == 1 else "Cross" - - is_action = category == "Api" - yaml_languages = yaml.get("languages") languages: Dict[str, Language] = {} if yaml_languages is None: @@ -319,22 +318,24 @@ def from_yaml( def parse_services( yaml: Any, errors: MetadataErrors, known_services: Dict[str, Service] -) -> Dict[str, List[str]]: +) -> Dict[str, Set[str]]: if yaml is None: return {} - services: Dict[str, List[str]] = {} + services: Dict[str, Set[str]] = {} for name in yaml: if name not in known_services: errors.append(metadata_errors.UnknownService(service=name)) else: service: Dict[str, None] | Set[str] | None = yaml.get(name) # While .get replaces missing with {}, `sqs: ` in yaml parses a literal `None` - if isinstance(service, dict): - service = set(service.keys()) if service is None: service = set() - # Make a copy of the dict - services[name] = [*service] + if isinstance(service, dict): + service = set(service.keys()) + if isinstance(service, set): + # Make a copy of the set for ourselves + service = set(service) + services[name] = set(service) return services diff --git a/aws_doc_sdk_examples_tools/metadata_test.py b/aws_doc_sdk_examples_tools/metadata_test.py index 2c9dbbc..b31f190 100644 --- a/aws_doc_sdk_examples_tools/metadata_test.py +++ b/aws_doc_sdk_examples_tools/metadata_test.py @@ -145,9 +145,9 @@ def test_parse(): title_abbrev="Deleting a topic", synopsis="Shows how to delete an &SNS; topic.", services={ - "sns": ["Operation1", "Operation2"], - "ses": ["Operation1", "Operation2"], - "sqs": [], + "sns": set(["Operation1", "Operation2"]), + "ses": set(["Operation1", "Operation2"]), + "sqs": set(), }, languages={"C++": language}, ) @@ -186,7 +186,7 @@ def test_parse_cross(): title="Delete Topic", title_abbrev="delete topic", synopsis="", - services={"sns": []}, + services={"sns": set()}, languages={"Java": language}, ) assert actual[0] == example @@ -225,7 +225,7 @@ def test_parse_curated(): title_abbrev="AutoGluon Tabular with SageMaker Pipelines", source_key="amazon-sagemaker-examples", languages={"Java": language}, - services={"s3": []}, + services={"s3": set()}, synopsis="use AutoGluon with SageMaker Pipelines.", ) @@ -258,7 +258,7 @@ def test_verify_load_successful(): sdk_version=3, github=None, block_content=None, - add_services={"s3": []}, + add_services={"s3": set()}, excerpts=[ Excerpt( description="Descriptive", @@ -313,7 +313,7 @@ def test_verify_load_successful(): category="Usage", service_main=None, languages=languages, - services={"sns": [], "sqs": []}, + services={"sns": set(), "sqs": set()}, ) assert actual[0] == example diff --git a/aws_doc_sdk_examples_tools/project_validator.py b/aws_doc_sdk_examples_tools/project_validator.py index 277cf3b..9c6a315 100644 --- a/aws_doc_sdk_examples_tools/project_validator.py +++ b/aws_doc_sdk_examples_tools/project_validator.py @@ -58,7 +58,6 @@ def check_files(root: Path, errors: MetadataErrors, do_check_spdx: bool): verify_no_deny_list_words(file_contents, file_path, errors) verify_no_secret_keys(file_contents, file_path, errors) - verify_no_secret_keys(file_contents, file_path, errors) if do_check_spdx: verify_spdx(file_contents, file_path, errors) diff --git a/aws_doc_sdk_examples_tools/sdks.py b/aws_doc_sdk_examples_tools/sdks.py index ff689ec..66867a9 100644 --- a/aws_doc_sdk_examples_tools/sdks.py +++ b/aws_doc_sdk_examples_tools/sdks.py @@ -123,6 +123,12 @@ def from_yaml( ) +@dataclass +class SdkWithNoVersionsError(metadata_errors.MetadataError): + def message(self): + return "SDK has no versions" + + @dataclass class Sdk: name: str @@ -130,6 +136,10 @@ class Sdk: guide: str property: str + def validate(self, errors: MetadataErrors): + if len(self.versions) == 0: + errors.append(SdkWithNoVersionsError(id=self.name)) + @classmethod def from_yaml(cls, name: str, yaml: Dict[str, Any]) -> tuple[Sdk, MetadataErrors]: errors = MetadataErrors() diff --git a/aws_doc_sdk_examples_tools/services.py b/aws_doc_sdk_examples_tools/services.py index b905501..77f880d 100644 --- a/aws_doc_sdk_examples_tools/services.py +++ b/aws_doc_sdk_examples_tools/services.py @@ -35,6 +35,9 @@ class Service: guide: Optional[ServiceGuide] = field(default=None) tags: Dict[str, Set[str]] = field(default_factory=dict) + def validate(self, errors: MetadataErrors): + pass + @classmethod def from_yaml( cls, name: str, yaml: Dict[str, Any] diff --git a/aws_doc_sdk_examples_tools/spdx.py b/aws_doc_sdk_examples_tools/spdx.py index d8e91c9..1532c73 100644 --- a/aws_doc_sdk_examples_tools/spdx.py +++ b/aws_doc_sdk_examples_tools/spdx.py @@ -49,6 +49,8 @@ def verify_spdx(file_contents: str, file_location: Path, errors: MetadataErrors) if file_location.suffix in validator_config.IGNORE_SPDX_SUFFIXES: return lines = file_contents.splitlines() + if len(lines) == 0: + return if skip_first_line(lines): lines = lines[1:] if len(lines) < 2: