diff --git a/MANIFEST.in b/MANIFEST.in index a029c65..f867763 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -graft config +include aws_doc_sdk_examples_tools/config/*.yaml diff --git a/config/.yamllint.yaml b/aws_doc_sdk_examples_tools/config/.yamllint.yaml similarity index 100% rename from config/.yamllint.yaml rename to aws_doc_sdk_examples_tools/config/.yamllint.yaml diff --git a/config/curated_example_schema.yaml b/aws_doc_sdk_examples_tools/config/curated_example_schema.yaml similarity index 100% rename from config/curated_example_schema.yaml rename to aws_doc_sdk_examples_tools/config/curated_example_schema.yaml diff --git a/config/example_schema.yaml b/aws_doc_sdk_examples_tools/config/example_schema.yaml similarity index 100% rename from config/example_schema.yaml rename to aws_doc_sdk_examples_tools/config/example_schema.yaml diff --git a/config/example_strict_schema.yaml b/aws_doc_sdk_examples_tools/config/example_strict_schema.yaml similarity index 100% rename from config/example_strict_schema.yaml rename to aws_doc_sdk_examples_tools/config/example_strict_schema.yaml diff --git a/config/sdks.yaml b/aws_doc_sdk_examples_tools/config/sdks.yaml similarity index 100% rename from config/sdks.yaml rename to aws_doc_sdk_examples_tools/config/sdks.yaml diff --git a/config/sdks_schema.yaml b/aws_doc_sdk_examples_tools/config/sdks_schema.yaml similarity index 100% rename from config/sdks_schema.yaml rename to aws_doc_sdk_examples_tools/config/sdks_schema.yaml diff --git a/config/services.yaml b/aws_doc_sdk_examples_tools/config/services.yaml similarity index 100% rename from config/services.yaml rename to aws_doc_sdk_examples_tools/config/services.yaml diff --git a/config/services_schema.yaml b/aws_doc_sdk_examples_tools/config/services_schema.yaml similarity index 100% rename from config/services_schema.yaml rename to aws_doc_sdk_examples_tools/config/services_schema.yaml diff --git a/aws_doc_sdk_examples_tools/doc_gen.py b/aws_doc_sdk_examples_tools/doc_gen.py index 68809f6..33052f5 100644 --- a/aws_doc_sdk_examples_tools/doc_gen.py +++ b/aws_doc_sdk_examples_tools/doc_gen.py @@ -5,26 +5,31 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, List, Optional, Set +from typing import Dict, Iterable, Optional, Set # from os import glob -from aws_doc_sdk_examples_tools.metadata import Example, parse as parse_examples -from aws_doc_sdk_examples_tools.metadata_errors import MetadataErrors -from aws_doc_sdk_examples_tools.metadata_validator import validate_metadata -from aws_doc_sdk_examples_tools.project_validator import ( +from .metadata import Example, parse as parse_examples +from .metadata_errors import MetadataErrors, MetadataError +from .metadata_validator import validate_metadata +from .project_validator import ( check_files, verify_sample_files, ) -from aws_doc_sdk_examples_tools.sdks import Sdk, parse as parse_sdks -from aws_doc_sdk_examples_tools.services import Service, parse as parse_services -from aws_doc_sdk_examples_tools.snippets import ( +from .sdks import Sdk, parse as parse_sdks +from .services import Service, parse as parse_services +from .snippets import ( Snippet, collect_snippets, validate_snippets, ) +@dataclass +class DocGenMergeWarning(MetadataError): + pass + + @dataclass class DocGen: root: Path @@ -33,68 +38,148 @@ class DocGen: services: Dict[str, Service] = field(default_factory=dict) snippets: Dict[str, Snippet] = field(default_factory=dict) snippet_files: Set[str] = field(default_factory=set) - examples: List[Example] = field(default_factory=list) + examples: Dict[str, Example] = field(default_factory=dict) cross_blocks: Set[str] = field(default_factory=set) - def collect_snippets(self, snippets_root: Optional[Path]): + def collect_snippets( + self, snippets_root: Optional[Path] = None, prefix: Optional[str] = None + ): + if prefix is not None: + prefix = f"{prefix}_" + if prefix is None: + prefix = "" if snippets_root is None: - snippets_root = self.root.parent.parent - snippets, errs = collect_snippets(snippets_root) + snippets_root = self.root + snippets, errs = collect_snippets(snippets_root, prefix) self.snippets = snippets self.errors.extend(errs) + def languages(self) -> Set[str]: + languages: Set[str] = set() + for sdk_name, sdk in self.sdks.items(): + for version in sdk.versions: + languages.add(f"{sdk_name}:{version.version}") + return languages + + def merge(self, other: "DocGen") -> MetadataErrors: + """Merge fields from other into self, prioritizing self fields""" + warnings = MetadataErrors() + for name, sdk in other.sdks.items(): + if name not in self.sdks: + self.sdks[name] = sdk + else: + warnings.append( + DocGenMergeWarning( + file=str(other.root), id=f"conflict in sdk {name}" + ) + ) + for name, service in other.services.items(): + if name not in self.services: + self.services[name] = service + warnings.append( + DocGenMergeWarning( + file=str(other.root), id=f"conflict in service {name}" + ) + ) + for name, snippet in other.snippets.items(): + if name not in self.snippets: + self.snippets[name] = snippet + warnings.append( + DocGenMergeWarning( + file=str(other.root), id=f"conflict in snippet {name}" + ) + ) + + self.snippet_files.update(other.snippet_files) + self.cross_blocks.update(other.cross_blocks) + self.extend_examples(other.examples.values()) + + return warnings + + def extend_examples(self, examples: Iterable[Example]): + for example in examples: + id = example.id + if id in self.examples: + self.examples[id].merge(example, self.errors) + else: + self.examples[id] = example + @classmethod - def from_root(cls, root: Path) -> "DocGen": - errors = MetadataErrors() + def empty(cls) -> "DocGen": + return DocGen(root=Path("/"), errors=MetadataErrors()) + + def clone(self) -> "DocGen": + return DocGen( + root=self.root, + errors=MetadataErrors(), + sdks={**self.sdks}, + services={**self.services}, + snippets={}, + snippet_files=set(), + cross_blocks=set(), + examples={}, + ) + def for_root(self, root: Path, config: Optional[Path] = None) -> "DocGen": + self.root = root metadata = root / ".doc_gen/metadata" - with open( - Path(__file__).parent.parent / "config" / "sdks.yaml", encoding="utf-8" - ) as file: + if config is None: + config = Path(__file__).parent / "config" + + with open(config / "sdks.yaml", encoding="utf-8") as file: meta = yaml.safe_load(file) sdks, errs = parse_sdks("sdks.yaml", meta) - errors.extend(errs) + self.errors.extend(errs) - with open( - Path(__file__).parent.parent / "config" / "services.yaml", encoding="utf-8" - ) as file: + with open(config / "services.yaml", encoding="utf-8") as file: meta = yaml.safe_load(file) services, service_errors = parse_services("services.yaml", meta) - errors.extend(service_errors) + self.errors.extend(service_errors) cross = set( [path.name for path in (metadata.parent / "cross-content").glob("*.xml")] ) - doc_gen = cls( - root=root, - sdks=sdks, - services=services, - errors=errors, - cross_blocks=cross, - ) + self.root = root + self.sdks = sdks + self.services = services + self.cross_blocks = cross for path in metadata.glob("*_metadata.yaml"): with open(path) as file: - ex, errs = parse_examples( + examples, errs = parse_examples( path.name, yaml.safe_load(file), - doc_gen.sdks, - doc_gen.services, - doc_gen.cross_blocks, + self.sdks, + self.services, + self.cross_blocks, ) - doc_gen.examples.extend(ex) - errors.extend(errs) + self.extend_examples(examples) + self.errors.extend(errs) + for example in examples: + for lang in example.languages: + language = example.languages[lang] + for version in language.versions: + for excerpt in version.excerpts: + self.snippet_files.update(excerpt.snippet_files) + + return self - return doc_gen + @classmethod + def from_root(cls, root: Path, config: Optional[Path] = None) -> "DocGen": + return DocGen.empty().for_root(root, config) def validate(self, check_spdx: bool): + for sdk in self.sdks.values(): + sdk.validate(self.errors) + for service in self.services.values(): + service.validate(self.errors) check_files(self.root, self.errors, check_spdx) verify_sample_files(self.root, self.errors) validate_metadata(self.root, self.errors) validate_snippets( - self.examples, + [*self.examples.values()], self.snippets, self.snippet_files, self.errors, diff --git a/aws_doc_sdk_examples_tools/metadata.py b/aws_doc_sdk_examples_tools/metadata.py index a099e8d..75fd4a7 100755 --- a/aws_doc_sdk_examples_tools/metadata.py +++ b/aws_doc_sdk_examples_tools/metadata.py @@ -9,14 +9,15 @@ from os.path import splitext from aws_doc_sdk_examples_tools import metadata_errors -from aws_doc_sdk_examples_tools.metadata_errors import ( +from .metadata_errors import ( + MetadataError, MetadataErrors, MetadataParseError, DuplicateItemException, ) -from aws_doc_sdk_examples_tools.metadata_validator import StringExtension -from aws_doc_sdk_examples_tools.services import Service -from aws_doc_sdk_examples_tools.sdks import Sdk +from .metadata_validator import StringExtension +from .services import Service +from .sdks import Sdk @dataclass @@ -64,10 +65,10 @@ class Version: excerpts: List[Excerpt] = field(default_factory=list) # Link to the source code for this example. TODO rename. github: Optional[str] = field(default=None) - add_services: Dict[str, List[str]] = field(default_factory=dict) + add_services: Dict[str, Set[str]] = field(default_factory=dict) # Deprecated. Replace with guide_topic list. sdkguide: Optional[str] = field(default=None) - # Link to additional topic places. TODO: Overwritten by aws-doc-sdk-example when merging. + # Link to additional topic places. more_info: List[Url] = field(default_factory=list) @classmethod @@ -143,6 +144,22 @@ class Language: name: str versions: List[Version] + def merge(self, other: "Language", errors: MetadataErrors): + """Add new versions from `other`""" + # TODO Error for mismatched names? + if self.name != other.name: + return + for other_version in other.versions: + self_version = filter( + lambda v: v.sdk_version == other_version.sdk_version, self.versions + ) + if self_version is None: + self.versions.append(other_version) + # Merge down to the SDK Version level, so later guides can add new + # excerpts to existing examples, but don't try to merge the excerpts + # within the language. If a tributary or writer feels they need to + # modify an excerpt, they should go modify the excerpt directly. + @classmethod def from_yaml( cls, @@ -177,6 +194,12 @@ def from_yaml( return cls(name, versions), errors +@dataclass +class ExampleMergeMismatchedId(MetadataError): + other_id: str = "" + other_file: str = "" + + @dataclass class Example: id: str @@ -195,10 +218,39 @@ class Example: # TODO document service_main and services. Not to be used by tributaries. Part of Cross Service. # List of services used by the examples. Lines up with those in services.yaml. service_main: Optional[str] = field(default=None) - services: Dict[str, List[str]] = field(default_factory=dict) + services: Dict[str, Set[str]] = field(default_factory=dict) synopsis_list: List[str] = field(default_factory=list) source_key: Optional[str] = field(default=None) + def merge(self, other: Example, errors: MetadataErrors): + """Combine `other` Example into self example. + + Merge down to the SDK Version level, so later guides can add new excerpts to existing examples, but don't try to merge the excerpts within the language. + If a tributary or writer feels they need to modify an excerpt, they should go modify the excerpt directly. + + Keep title, title_abbrev, synopsis, guide_topic, category, service_main, synopsis_list, and source_key from source (typically awsdocs/aws-doc-sdk-examples). + !NOTE: This means `merge` is NOT associative! + + Add error if IDs are not the same and return early. + """ + if self.id != other.id: + errors.append( + ExampleMergeMismatchedId( + id=self.id, other_id=other.id, file=self.file, other_file=other.file + ) + ) + return + + for service, actions in other.services.items(): + if service not in self.services: + self.services[service] = actions + + for name, language in other.languages.items(): + if name not in self.languages: + self.languages[name] = language + else: + self.languages[name].merge(language, errors) + @classmethod def from_yaml( cls, @@ -212,16 +264,20 @@ def from_yaml( title = get_with_valid_entities("title", yaml, errors) title_abbrev = get_with_valid_entities("title_abbrev", yaml, errors) synopsis = get_with_valid_entities("synopsis", yaml, errors, opt=True) + synopsis_list = [str(syn) for syn in yaml.get("synopsis_list", [])] - category = yaml.get("category", "") source_key = yaml.get("source_key") - parsed_services = parse_services(yaml.get("services", {}), errors, services) - synopsis_list = [str(syn) for syn in yaml.get("synopsis_list", [])] guide_topic = Url.from_yaml(yaml.get("guide_topic")) if isinstance(guide_topic, MetadataParseError): errors.append(guide_topic) guide_topic = None + parsed_services = parse_services(yaml.get("services", {}), errors, services) + category = yaml.get("category") + if category is None or category == "": + category = "Api" if len(parsed_services) == 1 else "Cross" + is_action = category == "Api" + service_main = yaml.get("service_main", None) if service_main is not None and service_main not in services: try: @@ -229,11 +285,6 @@ def from_yaml( except DuplicateItemException: pass - if category == "": - category = "Api" if len(parsed_services) == 1 else "Cross" - - is_action = category == "Api" - yaml_languages = yaml.get("languages") languages: Dict[str, Language] = {} if yaml_languages is None: @@ -267,20 +318,24 @@ def from_yaml( def parse_services( yaml: Any, errors: MetadataErrors, known_services: Dict[str, Service] -) -> Dict[str, List[str]]: +) -> Dict[str, Set[str]]: if yaml is None: return {} - services: Dict[str, List[str]] = {} + services: Dict[str, Set[str]] = {} for name in yaml: if name not in known_services: errors.append(metadata_errors.UnknownService(service=name)) else: - service: Dict[str, None] | None = yaml.get(name) + service: Dict[str, None] | Set[str] | None = yaml.get(name) # While .get replaces missing with {}, `sqs: ` in yaml parses a literal `None` if service is None: - service = {} - # Make a copy of the dict - services[name] = [*service.keys()] + service = set() + if isinstance(service, dict): + service = set(service.keys()) + if isinstance(service, set): + # Make a copy of the set for ourselves + service = set(service) + services[name] = set(service) return services diff --git a/aws_doc_sdk_examples_tools/metadata_errors.py b/aws_doc_sdk_examples_tools/metadata_errors.py index 982fb76..9a40f6e 100644 --- a/aws_doc_sdk_examples_tools/metadata_errors.py +++ b/aws_doc_sdk_examples_tools/metadata_errors.py @@ -101,6 +101,9 @@ def __str__(self) -> str: errs = "\n".join([f"\t{err}" for err in self._errors]) return f"ExampleErrors with {len(self)} errors:\n{errs}" + def __eq__(self, __value: object) -> bool: + return isinstance(__value, MetadataErrors) and self._errors == __value._errors + @dataclass class MissingServiceBody(MetadataParseError): diff --git a/aws_doc_sdk_examples_tools/metadata_test.py b/aws_doc_sdk_examples_tools/metadata_test.py index dd70219..b31f190 100644 --- a/aws_doc_sdk_examples_tools/metadata_test.py +++ b/aws_doc_sdk_examples_tools/metadata_test.py @@ -10,8 +10,9 @@ from pathlib import Path from typing import List, Set, Tuple -from aws_doc_sdk_examples_tools import metadata_errors -from aws_doc_sdk_examples_tools.metadata import ( +from . import metadata_errors +from .metadata_errors import MetadataErrors +from .metadata import ( parse, Example, Url, @@ -20,9 +21,9 @@ Excerpt, idFormat, ) -from aws_doc_sdk_examples_tools.doc_gen import DocGen -from aws_doc_sdk_examples_tools.sdks import Sdk -from aws_doc_sdk_examples_tools.services import Service, ServiceExpanded +from .doc_gen import DocGen +from .sdks import Sdk +from .services import Service, ServiceExpanded def load( @@ -144,9 +145,9 @@ def test_parse(): title_abbrev="Deleting a topic", synopsis="Shows how to delete an &SNS; topic.", services={ - "sns": ["Operation1", "Operation2"], - "ses": ["Operation1", "Operation2"], - "sqs": [], + "sns": set(["Operation1", "Operation2"]), + "ses": set(["Operation1", "Operation2"]), + "sqs": set(), }, languages={"C++": language}, ) @@ -185,7 +186,7 @@ def test_parse_cross(): title="Delete Topic", title_abbrev="delete topic", synopsis="", - services={"sns": []}, + services={"sns": set()}, languages={"Java": language}, ) assert actual[0] == example @@ -224,7 +225,7 @@ def test_parse_curated(): title_abbrev="AutoGluon Tabular with SageMaker Pipelines", source_key="amazon-sagemaker-examples", languages={"Java": language}, - services={"s3": []}, + services={"s3": set()}, synopsis="use AutoGluon with SageMaker Pipelines.", ) @@ -257,7 +258,7 @@ def test_verify_load_successful(): sdk_version=3, github=None, block_content=None, - add_services={"s3": []}, + add_services={"s3": set()}, excerpts=[ Excerpt( description="Descriptive", @@ -312,7 +313,7 @@ def test_verify_load_successful(): category="Usage", service_main=None, languages=languages, - services={"sns": [], "sqs": []}, + services={"sns": set(), "sqs": set()}, ) assert actual[0] == example @@ -461,5 +462,39 @@ def test_idFormat(): assert not idFormat("test", TEST_SERVICES) +@pytest.mark.parametrize( + ["a", "b", "d"], + [ + ( + DocGen( + root=Path("/a"), + errors=MetadataErrors(), + sdks={ + "a": Sdk(name="a", guide="guide_a", property="a_prop", versions=[]) + }, + ), + DocGen( + root=Path("/b"), + errors=MetadataErrors(), + sdks={ + "b": Sdk(name="b", guide="guide_b", property="b_prop", versions=[]) + }, + ), + DocGen( + root=Path("/a"), + errors=MetadataErrors(), + sdks={ + "a": Sdk(name="a", guide="guide_a", property="a_prop", versions=[]), + "b": Sdk(name="b", guide="guide_b", property="b_prop", versions=[]), + }, + ), + ) + ], +) +def test_merge(a: DocGen, b: DocGen, d: DocGen): + a.merge(b) + assert a == d + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/aws_doc_sdk_examples_tools/metadata_validator.py b/aws_doc_sdk_examples_tools/metadata_validator.py index 232520d..ac0598f 100755 --- a/aws_doc_sdk_examples_tools/metadata_validator.py +++ b/aws_doc_sdk_examples_tools/metadata_validator.py @@ -21,7 +21,7 @@ from yamale import YamaleError # type: ignore from yamale.validators import DefaultValidators, Validator, String # type: ignore -from aws_doc_sdk_examples_tools.metadata_errors import ( +from .metadata_errors import ( MetadataErrors, MetadataParseError, ) @@ -185,12 +185,11 @@ def validate_files( def validate_metadata(doc_gen_root: Path, errors: MetadataErrors) -> MetadataErrors: - with open(Path(__file__).parent.parent / "config" / "sdks.yaml") as sdks_file: + config = Path(__file__).parent / "config" + with open(config / "sdks.yaml") as sdks_file: sdks_yaml: Dict[str, Any] = yaml.safe_load(sdks_file) - with open( - Path(__file__).parent.parent / "config" / "services.yaml" - ) as services_file: + with open(config / "services.yaml") as services_file: services_yaml = yaml.safe_load(services_file) SdkVersion.sdks = sdks_yaml @@ -207,7 +206,7 @@ def validate_metadata(doc_gen_root: Path, errors: MetadataErrors) -> MetadataErr validators[BlockContent.tag] = BlockContent validators[String.tag] = StringExtension - schema_root = Path(__file__).parent.parent / "config" + schema_root = Path(__file__).parent / "config" to_validate = [ # (schema, metadata_glob) diff --git a/aws_doc_sdk_examples_tools/project_validator.py b/aws_doc_sdk_examples_tools/project_validator.py index 994be35..9c6a315 100644 --- a/aws_doc_sdk_examples_tools/project_validator.py +++ b/aws_doc_sdk_examples_tools/project_validator.py @@ -27,14 +27,14 @@ from pathlib import Path from typing import List -from aws_doc_sdk_examples_tools.file_utils import get_files -from aws_doc_sdk_examples_tools.metadata_errors import ( +from .file_utils import get_files +from .metadata_errors import ( MetadataErrors, MetadataError, MetadataParseError, DuplicateItemException, ) -from aws_doc_sdk_examples_tools.spdx import verify_spdx +from .spdx import verify_spdx from aws_doc_sdk_examples_tools import validator_config logger = logging.getLogger(__name__) @@ -58,7 +58,6 @@ def check_files(root: Path, errors: MetadataErrors, do_check_spdx: bool): verify_no_deny_list_words(file_contents, file_path, errors) verify_no_secret_keys(file_contents, file_path, errors) - verify_no_secret_keys(file_contents, file_path, errors) if do_check_spdx: verify_spdx(file_contents, file_path, errors) diff --git a/aws_doc_sdk_examples_tools/project_validator_test.py b/aws_doc_sdk_examples_tools/project_validator_test.py index a42e490..7b30987 100644 --- a/aws_doc_sdk_examples_tools/project_validator_test.py +++ b/aws_doc_sdk_examples_tools/project_validator_test.py @@ -10,7 +10,7 @@ from typing import List from aws_doc_sdk_examples_tools import project_validator -from aws_doc_sdk_examples_tools.metadata_errors import MetadataErrors +from .metadata_errors import MetadataErrors @pytest.mark.parametrize( diff --git a/aws_doc_sdk_examples_tools/py.typed b/aws_doc_sdk_examples_tools/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/aws_doc_sdk_examples_tools/sdks.py b/aws_doc_sdk_examples_tools/sdks.py index 2618f82..66867a9 100644 --- a/aws_doc_sdk_examples_tools/sdks.py +++ b/aws_doc_sdk_examples_tools/sdks.py @@ -7,7 +7,7 @@ from dataclasses import dataclass, field from aws_doc_sdk_examples_tools import metadata_errors -from aws_doc_sdk_examples_tools.metadata_errors import ( +from .metadata_errors import ( MetadataErrors, MetadataParseError, check_mapping, @@ -123,6 +123,12 @@ def from_yaml( ) +@dataclass +class SdkWithNoVersionsError(metadata_errors.MetadataError): + def message(self): + return "SDK has no versions" + + @dataclass class Sdk: name: str @@ -130,6 +136,10 @@ class Sdk: guide: str property: str + def validate(self, errors: MetadataErrors): + if len(self.versions) == 0: + errors.append(SdkWithNoVersionsError(id=self.name)) + @classmethod def from_yaml(cls, name: str, yaml: Dict[str, Any]) -> tuple[Sdk, MetadataErrors]: errors = MetadataErrors() diff --git a/aws_doc_sdk_examples_tools/sdks_test.py b/aws_doc_sdk_examples_tools/sdks_test.py index ae76464..4717c32 100644 --- a/aws_doc_sdk_examples_tools/sdks_test.py +++ b/aws_doc_sdk_examples_tools/sdks_test.py @@ -11,7 +11,7 @@ from typing import Dict, Tuple from aws_doc_sdk_examples_tools import metadata_errors -from aws_doc_sdk_examples_tools.sdks import ( +from .sdks import ( parse, Sdk, SdkVersion, diff --git a/aws_doc_sdk_examples_tools/services.py b/aws_doc_sdk_examples_tools/services.py index d82d460..77f880d 100644 --- a/aws_doc_sdk_examples_tools/services.py +++ b/aws_doc_sdk_examples_tools/services.py @@ -6,7 +6,7 @@ from typing import Any, Dict, Optional, Set, Union from dataclasses import dataclass, field from aws_doc_sdk_examples_tools import metadata_errors -from aws_doc_sdk_examples_tools.metadata_errors import MetadataErrors, check_mapping +from .metadata_errors import MetadataErrors, check_mapping @dataclass @@ -35,6 +35,9 @@ class Service: guide: Optional[ServiceGuide] = field(default=None) tags: Dict[str, Set[str]] = field(default_factory=dict) + def validate(self, errors: MetadataErrors): + pass + @classmethod def from_yaml( cls, name: str, yaml: Dict[str, Any] diff --git a/aws_doc_sdk_examples_tools/services_test.py b/aws_doc_sdk_examples_tools/services_test.py index 47f7173..0080125 100644 --- a/aws_doc_sdk_examples_tools/services_test.py +++ b/aws_doc_sdk_examples_tools/services_test.py @@ -7,7 +7,7 @@ import yaml from aws_doc_sdk_examples_tools import metadata_errors -from aws_doc_sdk_examples_tools.services import ( +from .services import ( parse, Service, ServiceGuide, diff --git a/aws_doc_sdk_examples_tools/snippets.py b/aws_doc_sdk_examples_tools/snippets.py index 27e9a91..7236264 100644 --- a/aws_doc_sdk_examples_tools/snippets.py +++ b/aws_doc_sdk_examples_tools/snippets.py @@ -9,9 +9,9 @@ from aws_doc_sdk_examples_tools import validator_config -from aws_doc_sdk_examples_tools.file_utils import get_files, clear -from aws_doc_sdk_examples_tools.metadata import Example -from aws_doc_sdk_examples_tools.metadata_errors import MetadataErrors, MetadataError +from .file_utils import get_files, clear +from .metadata import Example +from .metadata_errors import MetadataErrors, MetadataError SNIPPET_START = "snippet-start:[" SNIPPET_END = "snippet-end:[" @@ -81,21 +81,21 @@ def message(self): return f" unicode error: {str(self.err)}" -def _tag_from_line(token: str, line: str) -> str: +def _tag_from_line(token: str, line: str, prefix: str) -> str: tag_start = line.find(token) + len(token) tag_end = line.find("]", tag_start) - return line[tag_start:tag_end].strip() + return prefix + line[tag_start:tag_end].strip() def parse_snippets( - lines: List[str], file: Path + lines: List[str], file: Path, prefix: str ) -> Tuple[Dict[str, Snippet], MetadataErrors]: snippets: Dict[str, Snippet] = {} errors = MetadataErrors() open_tags: Set[str] = set() for line_idx, line in enumerate(lines): if SNIPPET_START in line: - tag = _tag_from_line(SNIPPET_START, line) + tag = _tag_from_line(SNIPPET_START, line, prefix) if tag in snippets: errors.append( DuplicateSnippetStartError(file=str(file), line=line_idx, tag=tag) @@ -110,7 +110,7 @@ def parse_snippets( ) open_tags.add(tag) elif SNIPPET_END in line: - tag = _tag_from_line(SNIPPET_END, line) + tag = _tag_from_line(SNIPPET_END, line, prefix) if tag not in snippets: errors.append( MissingSnippetStartError(file=str(file), line=line_idx, tag=tag) @@ -135,23 +135,25 @@ def parse_snippets( return snippets, errors -def find_snippets(file: Path) -> Tuple[Dict[str, Snippet], MetadataErrors]: +def find_snippets(file: Path, prefix: str) -> Tuple[Dict[str, Snippet], MetadataErrors]: errors = MetadataErrors() snippets: Dict[str, Snippet] = {} with open(file, encoding="utf-8") as snippet_file: try: - snippets, errs = parse_snippets(snippet_file.readlines(), file) + snippets, errs = parse_snippets(snippet_file.readlines(), file, prefix) errors.extend(errs) except UnicodeDecodeError as err: errors.append(MetadataUnicodeError(file=str(file), err=err)) return snippets, errors -def collect_snippets(root: Path) -> Tuple[Dict[str, Snippet], MetadataErrors]: +def collect_snippets( + root: Path, prefix: str = "" +) -> Tuple[Dict[str, Snippet], MetadataErrors]: snippets: Dict[str, Snippet] = {} errors = MetadataErrors() for file in get_files(root, validator_config.skip): - snips, errs = find_snippets(file) + snips, errs = find_snippets(file, prefix) snippets.update(snips) errors.extend(errs) return snippets, errors @@ -207,32 +209,31 @@ def validate_snippets( tag=snippet_tag, ) ) - for snippet_file in excerpt.snippet_files: - if not (root / snippet_file).exists(): - # Ensure all snippet_files exist - errors.append( - MissingSnippetFile( - file=example.file, - snippet_file=snippet_file, - id=f"{lang}:{version.sdk_version}", - ) - ) - if re.search(win_unsafe_re, str(snippet_file)): - errors.append( - WindowsUnsafeSnippetFile( - file=example.file, - snippet_file=snippet_file, - id=f"{lang}:{version.sdk_version}", - ) - ) - snippet_files.add(snippet_file) + for snippet_file in excerpt.snippet_files: + if not (root / snippet_file).exists(): + # Ensure all snippet_files exist + errors.append( + MissingSnippetFile( + file=example.file, + snippet_file=snippet_file, + id=f"{lang}:{version.sdk_version}", + ) + ) + if re.search(win_unsafe_re, str(snippet_file)): + errors.append( + WindowsUnsafeSnippetFile( + file=example.file, + snippet_file=snippet_file, + id=f"{lang}:{version.sdk_version}", + ) + ) -def write_snippets(root: Path, snippets: Dict[str, Snippet]): +def write_snippets(root: Path, snippets: Dict[str, Snippet], check: bool = False): errors = MetadataErrors() for tag in snippets: name = root / f"{tag}.txt" - if name.exists(): + if check and name.exists(): errors.append(SnippetAlreadyWritten(file=str(name))) else: try: @@ -246,8 +247,7 @@ def write_snippets(root: Path, snippets: Dict[str, Snippet]): def write_snippet_file(folder: Path, snippet_file: Path): name = str(snippet_file).replace("/", ".") dest = folder / f"{name}.txt" - if not dest.exists(): - copyfile(folder / snippet_file, dest) + copyfile(folder / snippet_file, dest) def main(): diff --git a/aws_doc_sdk_examples_tools/snippets_test.py b/aws_doc_sdk_examples_tools/snippets_test.py index 79c242b..178ffb9 100644 --- a/aws_doc_sdk_examples_tools/snippets_test.py +++ b/aws_doc_sdk_examples_tools/snippets_test.py @@ -62,6 +62,6 @@ def test_verify_snippet_start_end(file_contents: str, expected_error_count: int): """Test that various kinds of mismatched snippet-start and -end tags are counted correctly as errors.""" - _, errors = snippets.parse_snippets(file_contents.split("\n"), Path("test")) + _, errors = snippets.parse_snippets(file_contents.split("\n"), Path("test"), "") error_count = len(errors) assert error_count == expected_error_count diff --git a/aws_doc_sdk_examples_tools/spdx.py b/aws_doc_sdk_examples_tools/spdx.py index 7c36e34..1532c73 100644 --- a/aws_doc_sdk_examples_tools/spdx.py +++ b/aws_doc_sdk_examples_tools/spdx.py @@ -8,7 +8,7 @@ import re from sys import argv -from aws_doc_sdk_examples_tools.metadata_errors import MetadataError, MetadataErrors +from .metadata_errors import MetadataError, MetadataErrors from aws_doc_sdk_examples_tools import validator_config @@ -49,6 +49,8 @@ def verify_spdx(file_contents: str, file_location: Path, errors: MetadataErrors) if file_location.suffix in validator_config.IGNORE_SPDX_SUFFIXES: return lines = file_contents.splitlines() + if len(lines) == 0: + return if skip_first_line(lines): lines = lines[1:] if len(lines) < 2: diff --git a/aws_doc_sdk_examples_tools/validate.py b/aws_doc_sdk_examples_tools/validate.py index fe28ad8..54b132c 100755 --- a/aws_doc_sdk_examples_tools/validate.py +++ b/aws_doc_sdk_examples_tools/validate.py @@ -5,7 +5,7 @@ from pathlib import Path from sys import exit -from aws_doc_sdk_examples_tools.doc_gen import DocGen +from .doc_gen import DocGen def main(): diff --git a/setup.py b/setup.py index 382fe9f..fd8196e 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ name="aws_doc_sdk_examples_tools", version="0.0.1", packages=["aws_doc_sdk_examples_tools"], + package_data={"aws_doc_sdk_examples_tools": ["config/*.yaml"]}, install_requires=[ "pathspec==0.11.2", "PyYAML==6.0.1",