Merge pull request #8 from awsdocs/feat/project_structure

Feat/project structure

MANIFEST.in

@@ -1,2 +1,2 @@
-graft config
+include aws_doc_sdk_examples_tools/config/*.yaml
 

aws_doc_sdk_examples_tools/doc_gen.py

@@ -5,26 +5,31 @@
 
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Dict, List, Optional, Set
+from typing import Dict, Iterable, Optional, Set
 
 # from os import glob
 
-from aws_doc_sdk_examples_tools.metadata import Example, parse as parse_examples
-from aws_doc_sdk_examples_tools.metadata_errors import MetadataErrors
-from aws_doc_sdk_examples_tools.metadata_validator import validate_metadata
-from aws_doc_sdk_examples_tools.project_validator import (
+from .metadata import Example, parse as parse_examples
+from .metadata_errors import MetadataErrors, MetadataError
+from .metadata_validator import validate_metadata
+from .project_validator import (
     check_files,
     verify_sample_files,
 )
-from aws_doc_sdk_examples_tools.sdks import Sdk, parse as parse_sdks
-from aws_doc_sdk_examples_tools.services import Service, parse as parse_services
-from aws_doc_sdk_examples_tools.snippets import (
+from .sdks import Sdk, parse as parse_sdks
+from .services import Service, parse as parse_services
+from .snippets import (
     Snippet,
     collect_snippets,
     validate_snippets,
 )
 
 
+@dataclass
+class DocGenMergeWarning(MetadataError):
+    pass
+
+
 @dataclass
 class DocGen:
     root: Path
@@ -33,68 +38,148 @@ class DocGen:
     services: Dict[str, Service] = field(default_factory=dict)
     snippets: Dict[str, Snippet] = field(default_factory=dict)
     snippet_files: Set[str] = field(default_factory=set)
-    examples: List[Example] = field(default_factory=list)
+    examples: Dict[str, Example] = field(default_factory=dict)
     cross_blocks: Set[str] = field(default_factory=set)
 
-    def collect_snippets(self, snippets_root: Optional[Path]):
+    def collect_snippets(
+        self, snippets_root: Optional[Path] = None, prefix: Optional[str] = None
+    ):
+        if prefix is not None:
+            prefix = f"{prefix}_"
+        if prefix is None:
+            prefix = ""
         if snippets_root is None:
-            snippets_root = self.root.parent.parent
-        snippets, errs = collect_snippets(snippets_root)
+            snippets_root = self.root
+        snippets, errs = collect_snippets(snippets_root, prefix)
         self.snippets = snippets
         self.errors.extend(errs)
 
+    def languages(self) -> Set[str]:
+        languages: Set[str] = set()
+        for sdk_name, sdk in self.sdks.items():
+            for version in sdk.versions:
+                languages.add(f"{sdk_name}:{version.version}")
+        return languages
+
+    def merge(self, other: "DocGen") -> MetadataErrors:
+        """Merge fields from other into self, prioritizing self fields"""
+        warnings = MetadataErrors()
+        for name, sdk in other.sdks.items():
+            if name not in self.sdks:
+                self.sdks[name] = sdk
+            else:
+                warnings.append(
+                    DocGenMergeWarning(
+                        file=str(other.root), id=f"conflict in sdk {name}"
+                    )
+                )
+        for name, service in other.services.items():
+            if name not in self.services:
+                self.services[name] = service
+                warnings.append(
+                    DocGenMergeWarning(
+                        file=str(other.root), id=f"conflict in service {name}"
+                    )
+                )
+        for name, snippet in other.snippets.items():
+            if name not in self.snippets:
+                self.snippets[name] = snippet
+                warnings.append(
+                    DocGenMergeWarning(
+                        file=str(other.root), id=f"conflict in snippet {name}"
+                    )
+                )
+
+        self.snippet_files.update(other.snippet_files)
+        self.cross_blocks.update(other.cross_blocks)
+        self.extend_examples(other.examples.values())
+
+        return warnings
+
+    def extend_examples(self, examples: Iterable[Example]):
+        for example in examples:
+            id = example.id
+            if id in self.examples:
+                self.examples[id].merge(example, self.errors)
+            else:
+                self.examples[id] = example
+
     @classmethod
-    def from_root(cls, root: Path) -> "DocGen":
-        errors = MetadataErrors()
+    def empty(cls) -> "DocGen":
+        return DocGen(root=Path("/"), errors=MetadataErrors())
+
+    def clone(self) -> "DocGen":
+        return DocGen(
+            root=self.root,
+            errors=MetadataErrors(),
+            sdks={**self.sdks},
+            services={**self.services},
+            snippets={},
+            snippet_files=set(),
+            cross_blocks=set(),
+            examples={},
+        )
 
+    def for_root(self, root: Path, config: Optional[Path] = None) -> "DocGen":
+        self.root = root
         metadata = root / ".doc_gen/metadata"
 
-        with open(
-            Path(__file__).parent.parent / "config" / "sdks.yaml", encoding="utf-8"
-        ) as file:
+        if config is None:
+            config = Path(__file__).parent / "config"
+
+        with open(config / "sdks.yaml", encoding="utf-8") as file:
             meta = yaml.safe_load(file)
             sdks, errs = parse_sdks("sdks.yaml", meta)
-            errors.extend(errs)
+            self.errors.extend(errs)
 
-        with open(
-            Path(__file__).parent.parent / "config" / "services.yaml", encoding="utf-8"
-        ) as file:
+        with open(config / "services.yaml", encoding="utf-8") as file:
             meta = yaml.safe_load(file)
             services, service_errors = parse_services("services.yaml", meta)
-            errors.extend(service_errors)
+            self.errors.extend(service_errors)
 
         cross = set(
             [path.name for path in (metadata.parent / "cross-content").glob("*.xml")]
         )
 
-        doc_gen = cls(
-            root=root,
-            sdks=sdks,
-            services=services,
-            errors=errors,
-            cross_blocks=cross,
-        )
+        self.root = root
+        self.sdks = sdks
+        self.services = services
+        self.cross_blocks = cross
 
         for path in metadata.glob("*_metadata.yaml"):
             with open(path) as file:
-                ex, errs = parse_examples(
+                examples, errs = parse_examples(
                     path.name,
                     yaml.safe_load(file),
-                    doc_gen.sdks,
-                    doc_gen.services,
-                    doc_gen.cross_blocks,
+                    self.sdks,
+                    self.services,
+                    self.cross_blocks,
                 )
-                doc_gen.examples.extend(ex)
-                errors.extend(errs)
+                self.extend_examples(examples)
+                self.errors.extend(errs)
+                for example in examples:
+                    for lang in example.languages:
+                        language = example.languages[lang]
+                        for version in language.versions:
+                            for excerpt in version.excerpts:
+                                self.snippet_files.update(excerpt.snippet_files)
+
+        return self
 
-        return doc_gen
+    @classmethod
+    def from_root(cls, root: Path, config: Optional[Path] = None) -> "DocGen":
+        return DocGen.empty().for_root(root, config)
 
     def validate(self, check_spdx: bool):
+        for sdk in self.sdks.values():
+            sdk.validate(self.errors)
+        for service in self.services.values():
+            service.validate(self.errors)
         check_files(self.root, self.errors, check_spdx)
         verify_sample_files(self.root, self.errors)
         validate_metadata(self.root, self.errors)
         validate_snippets(
-            self.examples,
+            [*self.examples.values()],
             self.snippets,
             self.snippet_files,
             self.errors,

aws_doc_sdk_examples_tools/metadata.py

@@ -9,14 +9,15 @@
 from os.path import splitext
 
 from aws_doc_sdk_examples_tools import metadata_errors
-from aws_doc_sdk_examples_tools.metadata_errors import (
+from .metadata_errors import (
+    MetadataError,
     MetadataErrors,
     MetadataParseError,
     DuplicateItemException,
 )
-from aws_doc_sdk_examples_tools.metadata_validator import StringExtension
-from aws_doc_sdk_examples_tools.services import Service
-from aws_doc_sdk_examples_tools.sdks import Sdk
+from .metadata_validator import StringExtension
+from .services import Service
+from .sdks import Sdk
 
 
 @dataclass
@@ -64,10 +65,10 @@ class Version:
     excerpts: List[Excerpt] = field(default_factory=list)
     # Link to the source code for this example. TODO rename.
     github: Optional[str] = field(default=None)
-    add_services: Dict[str, List[str]] = field(default_factory=dict)
+    add_services: Dict[str, Set[str]] = field(default_factory=dict)
     # Deprecated. Replace with guide_topic list.
     sdkguide: Optional[str] = field(default=None)
-    # Link to additional topic places. TODO: Overwritten by aws-doc-sdk-example when merging.
+    # Link to additional topic places.
     more_info: List[Url] = field(default_factory=list)
 
     @classmethod
@@ -143,6 +144,22 @@ class Language:
     name: str
     versions: List[Version]
 
+    def merge(self, other: "Language", errors: MetadataErrors):
+        """Add new versions from `other`"""
+        # TODO Error for mismatched names?
+        if self.name != other.name:
+            return
+        for other_version in other.versions:
+            self_version = filter(
+                lambda v: v.sdk_version == other_version.sdk_version, self.versions
+            )
+            if self_version is None:
+                self.versions.append(other_version)
+            # Merge down to the SDK Version level, so later guides can add new
+            # excerpts to existing examples, but don't try to merge the excerpts
+            # within the language. If a tributary or writer feels they need to
+            # modify an excerpt, they should go modify the excerpt directly.
+
     @classmethod
     def from_yaml(
         cls,
@@ -177,6 +194,12 @@ def from_yaml(
         return cls(name, versions), errors
 
 
+@dataclass
+class ExampleMergeMismatchedId(MetadataError):
+    other_id: str = ""
+    other_file: str = ""
+
+
 @dataclass
 class Example:
     id: str
@@ -195,10 +218,39 @@ class Example:
     # TODO document service_main and services. Not to be used by tributaries. Part of Cross Service.
     # List of services used by the examples. Lines up with those in services.yaml.
     service_main: Optional[str] = field(default=None)
-    services: Dict[str, List[str]] = field(default_factory=dict)
+    services: Dict[str, Set[str]] = field(default_factory=dict)
     synopsis_list: List[str] = field(default_factory=list)
     source_key: Optional[str] = field(default=None)
 
+    def merge(self, other: Example, errors: MetadataErrors):
+        """Combine `other` Example into self example.
+
+        Merge down to the SDK Version level, so later guides can add new excerpts to existing examples, but don't try to merge the excerpts within the language.
+        If a tributary or writer feels they need to modify an excerpt, they should go modify the excerpt directly.
+
+        Keep title, title_abbrev, synopsis, guide_topic, category, service_main, synopsis_list, and source_key from source (typically awsdocs/aws-doc-sdk-examples).
+        !NOTE: This means `merge` is NOT associative!
+
+        Add error if IDs are not the same and return early.
+        """
+        if self.id != other.id:
+            errors.append(
+                ExampleMergeMismatchedId(
+                    id=self.id, other_id=other.id, file=self.file, other_file=other.file
+                )
+            )
+            return
+
+        for service, actions in other.services.items():
+            if service not in self.services:
+                self.services[service] = actions
+
+        for name, language in other.languages.items():
+            if name not in self.languages:
+                self.languages[name] = language
+            else:
+                self.languages[name].merge(language, errors)
+
     @classmethod
     def from_yaml(
         cls,
@@ -212,28 +264,27 @@ def from_yaml(
         title = get_with_valid_entities("title", yaml, errors)
         title_abbrev = get_with_valid_entities("title_abbrev", yaml, errors)
         synopsis = get_with_valid_entities("synopsis", yaml, errors, opt=True)
+        synopsis_list = [str(syn) for syn in yaml.get("synopsis_list", [])]
 
-        category = yaml.get("category", "")
         source_key = yaml.get("source_key")
-        parsed_services = parse_services(yaml.get("services", {}), errors, services)
-        synopsis_list = [str(syn) for syn in yaml.get("synopsis_list", [])]
         guide_topic = Url.from_yaml(yaml.get("guide_topic"))
         if isinstance(guide_topic, MetadataParseError):
             errors.append(guide_topic)
             guide_topic = None
 
+        parsed_services = parse_services(yaml.get("services", {}), errors, services)
+        category = yaml.get("category")
+        if category is None or category == "":
+            category = "Api" if len(parsed_services) == 1 else "Cross"
+        is_action = category == "Api"
+
         service_main = yaml.get("service_main", None)
         if service_main is not None and service_main not in services:
             try:
                 errors.append(metadata_errors.UnknownService(service=service_main))
             except DuplicateItemException:
                 pass
 
-        if category == "":
-            category = "Api" if len(parsed_services) == 1 else "Cross"
-
-        is_action = category == "Api"
-
         yaml_languages = yaml.get("languages")
         languages: Dict[str, Language] = {}
         if yaml_languages is None:
@@ -267,20 +318,24 @@ def from_yaml(
 
 def parse_services(
     yaml: Any, errors: MetadataErrors, known_services: Dict[str, Service]
-) -> Dict[str, List[str]]:
+) -> Dict[str, Set[str]]:
     if yaml is None:
         return {}
-    services: Dict[str, List[str]] = {}
+    services: Dict[str, Set[str]] = {}
     for name in yaml:
         if name not in known_services:
             errors.append(metadata_errors.UnknownService(service=name))
         else:
-            service: Dict[str, None] | None = yaml.get(name)
+            service: Dict[str, None] | Set[str] | None = yaml.get(name)
             # While .get replaces missing with {}, `sqs: ` in yaml parses a literal `None`
             if service is None:
-                service = {}
-            # Make a copy of the dict
-            services[name] = [*service.keys()]
+                service = set()
+            if isinstance(service, dict):
+                service = set(service.keys())
+            if isinstance(service, set):
+                # Make a copy of the set for ourselves
+                service = set(service)
+            services[name] = set(service)
     return services