diff --git a/src/docs-assembler/AssembleSources.cs b/src/docs-assembler/AssembleSources.cs index 759354ef..e747d47c 100644 --- a/src/docs-assembler/AssembleSources.cs +++ b/src/docs-assembler/AssembleSources.cs @@ -33,6 +33,7 @@ public record TocConfigurationMapping public class AssembleSources { + public AssembleContext AssembleContext { get; } public FrozenDictionary AssembleSets { get; } public FrozenDictionary TocTopLevelMappings { get; } @@ -51,16 +52,17 @@ public static async Task AssembleAsync(AssembleContext context, return sources; } - private AssembleSources(AssembleContext context, Checkout[] checkouts) + private AssembleSources(AssembleContext assembleContext, Checkout[] checkouts) { - TocTopLevelMappings = GetConfiguredSources(context); + AssembleContext = assembleContext; + TocTopLevelMappings = GetConfiguredSources(assembleContext); - var crossLinkFetcher = new AssemblerCrossLinkFetcher(NullLoggerFactory.Instance, context.Configuration); - UriResolver = new PublishEnvironmentUriResolver(TocTopLevelMappings, context.Environment); + var crossLinkFetcher = new AssemblerCrossLinkFetcher(NullLoggerFactory.Instance, assembleContext.Configuration); + UriResolver = new PublishEnvironmentUriResolver(TocTopLevelMappings, assembleContext.Environment); var crossLinkResolver = new CrossLinkResolver(crossLinkFetcher, UriResolver); AssembleSets = checkouts .Where(c => !c.Repository.Skip) - .Select(c => new AssemblerDocumentationSet(NullLoggerFactory.Instance, context, c, crossLinkResolver, TreeCollector)) + .Select(c => new AssemblerDocumentationSet(NullLoggerFactory.Instance, assembleContext, c, crossLinkResolver, TreeCollector)) .ToDictionary(s => s.Checkout.Repository.Name, s => s) .ToFrozenDictionary(); @@ -85,7 +87,7 @@ private AssembleSources(AssembleContext context, Checkout[] checkouts) var file = tocFiles.FirstOrDefault(f => f.Exists); if (file is null) { - context.Collector.EmitWarning(context.ConfigurationPath.FullName, $"Unable to find toc file in {tocDirectory}"); + assembleContext.Collector.EmitWarning(assembleContext.ConfigurationPath.FullName, $"Unable to find toc file in {tocDirectory}"); file = tocFiles.First(); } diff --git a/src/docs-assembler/Cli/RepositoryCommands.cs b/src/docs-assembler/Cli/RepositoryCommands.cs index 58943ab8..5505da1b 100644 --- a/src/docs-assembler/Cli/RepositoryCommands.cs +++ b/src/docs-assembler/Cli/RepositoryCommands.cs @@ -88,8 +88,8 @@ public async Task BuildAll( var navigation = new GlobalNavigation(assembleSources, navigationFile); - var pathProvider = new GlobalNavigationPathProvider(assembleSources, assembleContext); - var htmlWriter = new GlobalNavigationHtmlWriter(assembleContext, navigation, assembleSources); + var pathProvider = new GlobalNavigationPathProvider(navigationFile, assembleSources, assembleContext); + var htmlWriter = new GlobalNavigationHtmlWriter(navigationFile, assembleContext, navigation, assembleSources); var builder = new AssemblerBuilder(logger, assembleContext, htmlWriter, pathProvider); await builder.BuildAllAsync(assembleSources.AssembleSets, ctx); diff --git a/src/docs-assembler/Navigation/GlobalNavigationFile.cs b/src/docs-assembler/Navigation/GlobalNavigationFile.cs index 8bb62811..ac4637b6 100644 --- a/src/docs-assembler/Navigation/GlobalNavigationFile.cs +++ b/src/docs-assembler/Navigation/GlobalNavigationFile.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System.Collections.Frozen; +using System.IO.Abstractions; using Documentation.Assembler.Configuration; using Elastic.Markdown.IO.Configuration; using Elastic.Markdown.IO.Navigation; @@ -10,18 +11,23 @@ namespace Documentation.Assembler.Navigation; -public record GlobalNavigationFile +public record GlobalNavigationFile : ITableOfContentsScope { private readonly AssembleContext _context; private readonly AssembleSources _assembleSources; public IReadOnlyCollection TableOfContents { get; } + public IReadOnlyCollection Phantoms { get; } + + public IDirectoryInfo ScopeDirectory { get; } public GlobalNavigationFile(AssembleContext context, AssembleSources assembleSources) { _context = context; _assembleSources = assembleSources; - TableOfContents = Deserialize(); + TableOfContents = Deserialize("toc"); + Phantoms = Deserialize("phantoms"); + ScopeDirectory = _context.NavigationPath.Directory!; } public void EmitWarning(string message) => @@ -31,18 +37,15 @@ public void EmitError(string message) => _context.Collector.EmitWarning(_context.NavigationPath.FullName, message); - private IReadOnlyCollection Deserialize() + private IReadOnlyCollection Deserialize(string key) { var reader = new YamlStreamReader(_context.NavigationPath, _context.Collector); try { foreach (var entry in reader.Read()) { - switch (entry.Key) - { - case "toc": - return ReadChildren(reader, entry.Entry, null, 0); - } + if (entry.Key == key) + return ReadChildren(key, reader, entry.Entry, null, 0); } } catch (Exception e) @@ -54,7 +57,8 @@ private IReadOnlyCollection Deserialize() return []; } - private IReadOnlyCollection ReadChildren(YamlStreamReader reader, KeyValuePair entry, string? parent, int depth) + private IReadOnlyCollection ReadChildren(string key, YamlStreamReader reader, KeyValuePair entry, string? parent, + int depth) { var entries = new List(); if (entry.Key is not YamlScalarNode { Value: not null } scalarKey) @@ -71,7 +75,11 @@ private IReadOnlyCollection ReadChildren(YamlStreamReader reader, foreach (var tocEntry in sequence.Children.OfType()) { - var child = ReadChild(reader, tocEntry, parent, depth); + + var child = + key == "toc" + ? ReadTocDefinition(reader, tocEntry, parent, depth) + : ReadPhantomDefinition(reader, tocEntry); if (child is not null) entries.Add(child); } @@ -79,7 +87,27 @@ private IReadOnlyCollection ReadChildren(YamlStreamReader reader, return entries; } - private TocReference? ReadChild(YamlStreamReader reader, YamlMappingNode tocEntry, string? parent, int depth) + private TocReference? ReadPhantomDefinition(YamlStreamReader reader, YamlMappingNode tocEntry) + { + foreach (var entry in tocEntry.Children) + { + var key = ((YamlScalarNode)entry.Key).Value; + switch (key) + { + case "toc": + var source = reader.ReadString(entry); + if (source != null && !source.Contains("://")) + source = ContentSourceMoniker.CreateString(NarrativeRepository.RepositoryName, source); + var sourceUri = new Uri(source!); + var tocReference = new TocReference(sourceUri, this, "", true, []); + return tocReference; + } + } + + return null; + } + + private TocReference? ReadTocDefinition(YamlStreamReader reader, YamlMappingNode tocEntry, string? parent, int depth) { string? repository = null; string? source = null; @@ -139,9 +167,6 @@ private IReadOnlyCollection ReadChildren(YamlStreamReader reader, } var navigationItems = new List(); - //TODO not needed - //var tocChildren = mapping.TableOfContentsConfiguration.TableOfContents; - //navigationItems.AddRange(tocChildren); foreach (var entry in tocEntry.Children) { @@ -155,7 +180,7 @@ private IReadOnlyCollection ReadChildren(YamlStreamReader reader, continue; } - var children = ReadChildren(reader, entry, parent, depth + 1); + var children = ReadChildren("toc", reader, entry, parent, depth + 1); navigationItems.AddRange(children); break; } @@ -166,4 +191,5 @@ private IReadOnlyCollection ReadChildren(YamlStreamReader reader, var tocReference = new TocReference(sourceUri, mapping.TableOfContentsConfiguration, path, true, navigationItems); return tocReference; } + } diff --git a/src/docs-assembler/Navigation/GlobalNavigationHtmlWriter.cs b/src/docs-assembler/Navigation/GlobalNavigationHtmlWriter.cs index ef3f945a..f591e51b 100644 --- a/src/docs-assembler/Navigation/GlobalNavigationHtmlWriter.cs +++ b/src/docs-assembler/Navigation/GlobalNavigationHtmlWriter.cs @@ -3,53 +3,50 @@ // See the LICENSE file in the project root for more information using System.Collections.Concurrent; +using System.Collections.Immutable; using Elastic.Markdown.IO.Configuration; using Elastic.Markdown.IO.Navigation; using Elastic.Markdown.Slices; namespace Documentation.Assembler.Navigation; -public class GlobalNavigationHtmlWriter(AssembleContext assembleContext, GlobalNavigation navigation, AssembleSources assembleSources) : INavigationHtmlWriter +public class GlobalNavigationHtmlWriter( + GlobalNavigationFile navigationFile, + AssembleContext assembleContext, + GlobalNavigation globalNavigation, + AssembleSources assembleSources) : INavigationHtmlWriter { private readonly AssembleContext _assembleContext = assembleContext; private readonly ConcurrentDictionary _renderedNavigationCache = []; - private (DocumentationGroup, Uri) GetRealNavigationRoot(INavigation navigation) - { - if (navigation is not DocumentationGroup group) - throw new InvalidOperationException($"Expected a {nameof(DocumentationGroup)}"); + private ImmutableHashSet Phantoms { get; } = [.. navigationFile.Phantoms.Select(p => p.Source)]; - - if (group.NavigationRoot is TableOfContentsTree tree) + private (DocumentationGroup, Uri) GetRealNavigationRoot(TableOfContentsTree tree) + { + if (!assembleSources.TocTopLevelMappings.TryGetValue(tree.Source, out var topLevelUri)) { - if (!assembleSources.TocTopLevelMappings.TryGetValue(tree.Source, out var topLevelUri)) - { - _assembleContext.Collector.EmitWarning(_assembleContext.NavigationPath.FullName, $"Could not find a top level mapping for {tree.Source}"); - return (tree, tree.Source); - } - - if (!assembleSources.TreeCollector.TryGetTableOfContentsTree(topLevelUri.TopLevelSource, out var topLevel)) - { - _assembleContext.Collector.EmitWarning(_assembleContext.NavigationPath.FullName, $"Could not find a toc tree for {topLevelUri.TopLevelSource}"); - return (tree, tree.Source); - - } - return (topLevel, topLevelUri.TopLevelSource); - + _assembleContext.Collector.EmitWarning(_assembleContext.NavigationPath.FullName, $"Could not find a top level mapping for {tree.Source}"); + return (tree, tree.Source); } - else if (group.NavigationRoot is DocumentationGroup) + + if (!assembleSources.TreeCollector.TryGetTableOfContentsTree(topLevelUri.TopLevelSource, out var topLevel)) { - var source = group.FolderName == "reference/index.md" - ? new Uri("docs-content://reference/") - : throw new InvalidOperationException($"{group.FolderName} is not a valid navigation root"); - return (group, source); + _assembleContext.Collector.EmitWarning(_assembleContext.NavigationPath.FullName, $"Could not find a toc tree for {topLevelUri.TopLevelSource}"); + return (tree, tree.Source); + } - throw new InvalidOperationException($"Unknown navigation root {group.NavigationRoot}"); + return (topLevel, topLevelUri.TopLevelSource); } public async Task RenderNavigation(INavigation currentRootNavigation, Cancel ctx = default) { - var (navigation, source) = GetRealNavigationRoot(currentRootNavigation); + if (currentRootNavigation is not TableOfContentsTree tree) + throw new InvalidOperationException($"Expected a {nameof(DocumentationGroup)}"); + + if (Phantoms.Contains(tree.Source)) + return string.Empty; + + var (navigation, source) = GetRealNavigationRoot(tree); if (_renderedNavigationCache.TryGetValue(source, out var value)) return value; @@ -64,17 +61,13 @@ public async Task RenderNavigation(INavigation currentRootNavigation, Ca var model = CreateNavigationModel(navigation); value = await ((INavigationHtmlWriter)this).Render(model, ctx); _renderedNavigationCache[source] = value; - if (source == new Uri("docs-content://extend")) - { - } - return value; } private NavigationViewModel CreateNavigationModel(DocumentationGroup group) { - var topLevelItems = navigation.TopLevelItems; + var topLevelItems = globalNavigation.TopLevelItems; return new NavigationViewModel { Title = group.Index?.NavigationTitle ?? "Docs", diff --git a/src/docs-assembler/Navigation/GlobalNavigationPathProvider.cs b/src/docs-assembler/Navigation/GlobalNavigationPathProvider.cs index 91af1c64..c9e2db75 100644 --- a/src/docs-assembler/Navigation/GlobalNavigationPathProvider.cs +++ b/src/docs-assembler/Navigation/GlobalNavigationPathProvider.cs @@ -2,20 +2,12 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information -using System.Collections.Frozen; using System.Collections.Immutable; -using System.Diagnostics.CodeAnalysis; using System.IO.Abstractions; -using Documentation.Assembler.Building; -using Documentation.Assembler.Configuration; -using Documentation.Assembler.Sourcing; using Elastic.Markdown; -using Elastic.Markdown.CrossLinks; using Elastic.Markdown.Diagnostics; using Elastic.Markdown.IO; using Elastic.Markdown.IO.Configuration; -using Elastic.Markdown.IO.Navigation; -using Microsoft.Extensions.Logging.Abstractions; namespace Documentation.Assembler.Navigation; @@ -25,8 +17,9 @@ public record GlobalNavigationPathProvider : IDocumentationFileOutputProvider private readonly AssembleContext _context; private ImmutableSortedSet TableOfContentsPrefixes { get; } + private ImmutableSortedSet PhantomPrefixes { get; } - public GlobalNavigationPathProvider(AssembleSources assembleSources, AssembleContext context) + public GlobalNavigationPathProvider(GlobalNavigationFile navigationFile, AssembleSources assembleSources, AssembleContext context) { _assembleSources = assembleSources; _context = context; @@ -36,45 +29,14 @@ public GlobalNavigationPathProvider(AssembleSources assembleSources, AssembleCon .Select(v => v.Source.ToString()) .OrderByDescending(v => v.Length) ]; - } - - /* - public IFileInfo? LocateDocSetYaml(Uri crossLinkUri) - { - if (!TryGetCheckout(crossLinkUri, out var checkout)) - return null; - - var tocDirectory = _readFs.DirectoryInfo.New(Path.Combine(checkout.Directory.FullName, crossLinkUri.Host, crossLinkUri.AbsolutePath.TrimStart('/'))); - if (!tocDirectory.Exists) - { - _context.Collector.EmitError(_context.NavigationPath, $"Unable to find toc directory: {tocDirectory.FullName}"); - return null; - } - - var docsetYaml = _readFs.FileInfo.New(Path.Combine(tocDirectory.FullName, "docset.yml")); - var tocYaml = _readFs.FileInfo.New(Path.Combine(tocDirectory.FullName, "toc.yml")); - if (!docsetYaml.Exists && !tocYaml.Exists) - { - _context.Collector.EmitError(_context.NavigationPath, $"Unable to find docset.yml or toc.yml in: {tocDirectory.FullName}"); - return null; - } - return docsetYaml.Exists ? docsetYaml : tocYaml; + PhantomPrefixes = [..navigationFile.Phantoms + .Select(p => p.Source.ToString()) + .OrderByDescending(v => v.Length) + .ToArray() + ]; } - public bool TryGetCheckout(Uri crossLinkUri, [NotNullWhen(true)] out Checkout? checkout) - { - if (_checkoutsLookup.TryGetValue(crossLinkUri.Scheme, out checkout)) - return true; - - _context.Collector.EmitError(_context.ConfigurationPath, - !_repoConfigLookup.TryGetValue(crossLinkUri.Scheme, out _) - ? $"Repository: '{crossLinkUri.Scheme}' is not defined in assembler.yml" - : $"Unable to find checkout for repository: {crossLinkUri.Scheme}" - ); - return false; - }*/ - public IFileInfo? OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath) { if (relativePath.StartsWith("_static/", StringComparison.Ordinal)) @@ -124,6 +86,12 @@ public bool TryGetCheckout(Uri crossLinkUri, [NotNullWhen(true)] out Checkout? c if (relativePath.EndsWith(".asciidoc", StringComparison.Ordinal)) return null; + foreach (var prefix in PhantomPrefixes) + { + if (lookup.StartsWith(prefix, StringComparison.Ordinal)) + return null; + } + var fallBack = fs.Path.Combine(outputDirectory.FullName, "_failed", repositoryName, relativePath); _context.Collector.EmitError(_context.NavigationPath, $"No toc for output path: '{lookup}' falling back to: '{fallBack}'"); return fs.FileInfo.New(fallBack); diff --git a/src/docs-assembler/assembler.yml b/src/docs-assembler/assembler.yml index cd1823c5..3898acbd 100644 --- a/src/docs-assembler/assembler.yml +++ b/src/docs-assembler/assembler.yml @@ -34,12 +34,12 @@ references: cloud-on-k8s: cloud: current: master - curator: - skip: true - current: master - ecctl: - current: master - skip: true + # curator: + # skip: true + # current: master + # ecctl: + # current: master + # skip: true ecs-dotnet: ecs-logging-go-logrus: ecs-logging-go-zap: @@ -60,18 +60,18 @@ references: elasticsearch-php: elasticsearch-py: # https://github.com/elastic/elasticsearch-rs ci checks not running? - elasticsearch-rs: - skip: true + # elasticsearch-rs: + # skip: true elasticsearch-ruby: elasticsearch: go-elasticsearch: integrations: kibana: - logstash-docs: + logstash-docs-md: skip: true logstash: search-ui: integration-docs: - security-docs: - # wait for move to https://github.com/elastic/detection-rules/pull/4507/files - skip: true + # security-docs: + # # wait for move to https://github.com/elastic/detection-rules/pull/4507/files + # skip: true diff --git a/src/docs-assembler/navigation.yml b/src/docs-assembler/navigation.yml index d932fd11..d286c064 100644 --- a/src/docs-assembler/navigation.yml +++ b/src/docs-assembler/navigation.yml @@ -6,6 +6,11 @@ # 📝 = no PR started yet ############################################# +# Use sparingly, makes assembler aware of toc container folders +# That are not linked in the global toc but all the children toc they define are. +phantoms: + - toc: elasticsearch://reference + toc: ############# # NARRATIVE # @@ -131,29 +136,14 @@ toc: # Children include the entire AsciiDoc book # Elasticsearch and index management - # ✅ https://github.com/elastic/docs-content/blob/main/reference/elasticsearch/toc.yml - - toc: docs-content://reference/elasticsearch + # ✅ https://github.com/elastic/elasticsearch/blob/main/docs/reference/elasticsearch/toc.yml + - toc: elasticsearch://reference/elasticsearch path_prefix: reference/elasticsearch + # Children include: Configuration, JVM settings, Roles, + # Elasticsearch privileges, Index settings, Index lifecycle actions, + # REST APIs, Mapping, Elasticsearch audit events, Command line tools children: - # 📝 TO DO: Delete this item - # This should be replaced by: - # * `elasticsearch://reference/elasticsearch` - # * `elasticsearch://reference/community-contributed` - # * `elasticsearch://reference/elasticsearch-plugins` - # * `elasticsearch://reference/query-languages` - # * `elasticsearch://reference/scripting-languages` - # TODO MARTIJN duplicate path_prefix - - toc: elasticsearch://reference - path_prefix: reference/elasticsearch2 - - # ✅ https://github.com/elastic/elasticsearch/blob/main/docs/reference/elasticsearch/toc.yml - #- toc: elasticsearch://reference/elasticsearch - # path_prefix: reference/elasticsearch - # Children include: Configuration, JVM settings, Roles, - # Elasticsearch privileges, Index settings, Index lifecycle actions, - # REST APIs, Mapping, Elasticsearch audit events, Command line tools - # Curator # ✅ https://github.com/elastic/curator/blob/master/docs/reference/toc.yml # TODO MARTIJN: needs to run once more on master for the link index @@ -258,7 +248,7 @@ toc: # APM agents # ✅ https://github.com/elastic/docs-content/blob/main/reference/apm-agents/toc.yml - - toc: docs-content://reference/ingestion-tools/apm/agents + - toc: docs-content://reference/apm-agents path_prefix: reference/apm/agents children: # APM Android agent @@ -329,11 +319,6 @@ toc: # Filebeat, Heartbeat, Metricbeat, Packetbeat, Winlogbeat, # Elastic logging plugin for Docker - # 📝 TO DO: Delete this item - # This should be replaced by `elasticsearch://reference/enrich-processor` below - - toc: elasticsearch://reference/ingestion-tools/enrich-processor - path_prefix: reference/elasticsearch/enrich-processor - # Processor reference # ✅ https://github.com/elastic/elasticsearch/blob/main/docs/reference/enrich-processor/toc.yml - toc: elasticsearch://reference/enrich-processor @@ -370,11 +355,6 @@ toc: path_prefix: reference/aws-forwarder # Children include the entire AsciiDoc book - # 📝 TO DO: Delete this item - # This should be replaced by `elasticsearch://reference/search-connectors` below - - toc: elasticsearch://reference/ingestion-tools/search-connectors - path_prefix: reference/elasticsearch/search-connectors - # Search connectors # ✅ https://github.com/elastic/elasticsearch/blob/main/docs/reference/search-connectors/toc.yml - toc: elasticsearch://reference/search-connectors @@ -417,13 +397,13 @@ toc: # ECS # ✅ https://github.com/elastic/docs-content/blob/main/reference/ecs/toc.yml - toc: docs-content://reference/ecs - path_prefix: reference/ecs + path_prefix: reference/ecs/overview children: # ECS reference # ✅ https://github.com/elastic/ecs/blob/main/docs/reference/toc.yml # TODO MARTIJN conflicting path_prefix - toc: ecs://reference - path_prefix: reference/ecs/ref + path_prefix: reference/ecs # Children include the entire AsciiDoc book # ECS logging libraries @@ -493,12 +473,6 @@ toc: # Metrics reference, Canvas function reference children: - # 📝 TO DO: Delete this item - # This should be replaced by `elasticsearch://reference/text-analysis` and - # `elasticsearch://reference/aggregations` below. - - toc: elasticsearch://reference/data-analysis - path_prefix: reference/elasticsearch/data-analysis - # ✅ https://github.com/elastic/elasticsearch/blob/main/docs/reference/text-analysis/toc.yml - toc: elasticsearch://reference/text-analysis path_prefix: reference/text-analysis @@ -509,14 +483,14 @@ toc: # Cloud # ✅ https://github.com/elastic/docs-content/blob/main/reference/cloud/toc.yml - toc: docs-content://reference/cloud - path_prefix: reference/cloud + path_prefix: reference/cloud/overview children: # Elastic Cloud Enterprise # Elastic Cloud Hosted # ✅ https://github.com/elastic/cloud/blob/master/docs/reference/toc.yml # TODO Martijn: conflicting path_prefix - toc: cloud://reference - path_prefix: reference/cloud/ref + path_prefix: reference/cloud # Children include the entire AsciiDoc book # (minus pages moved to docs-content)