From b30d3e2db72639aadd249a0d4a65caed60892f00 Mon Sep 17 00:00:00 2001 From: Ying Hua Date: Fri, 18 Mar 2022 13:07:01 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=9B=B3=EF=B8=8F=F0=9F=9B=B3=EF=B8=8F?= =?UTF-8?q?=F0=9F=9B=B3=EF=B8=8F=20(#7937)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * document for expected regression test diffs (#7897) * document for expected regression test diffs * fix spelling * address comments * Remove Dependency Lock design spec documentation (#7919) * Add credential auto renew documentation (#7918) * Update credential.md * Update credential.md * Update docs/designs/credential.md Co-authored-by: Ying Hua Co-authored-by: Ying Hua * ci: disable Azure pipeline for design doc-only changes (#7920) * Bump Microsoft.DocAsCode.MAML2Yaml.Lib from 1.1.1719.2 to 1.1.1878.4 (#7905) Bumps Microsoft.DocAsCode.MAML2Yaml.Lib from 1.1.1719.2 to 1.1.1878.4. --- updated-dependencies: - dependency-name: Microsoft.DocAsCode.MAML2Yaml.Lib dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump Microsoft.AspNetCore.TestHost from 6.0.1 to 6.0.2 (#7879) Bumps [Microsoft.AspNetCore.TestHost](https://github.com/dotnet/aspnetcore) from 6.0.1 to 6.0.2. - [Release notes](https://github.com/dotnet/aspnetcore/releases) - [Changelog](https://github.com/dotnet/aspnetcore/blob/main/docs/ReleasePlanning.md) - [Commits](https://github.com/dotnet/aspnetcore/compare/v6.0.1...v6.0.2) --- updated-dependencies: - dependency-name: Microsoft.AspNetCore.TestHost dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump coverlet.msbuild from 3.1.0 to 3.1.2 (#7874) Bumps [coverlet.msbuild](https://github.com/coverlet-coverage/coverlet) from 3.1.0 to 3.1.2. - [Release notes](https://github.com/coverlet-coverage/coverlet/releases) - [Commits](https://github.com/coverlet-coverage/coverlet/commits) --- updated-dependencies: - dependency-name: coverlet.msbuild dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump Microsoft.VisualStudio.Threading.Analyzers from 17.0.64 to 17.1.46 (#7906) Bumps [Microsoft.VisualStudio.Threading.Analyzers](https://github.com/microsoft/vs-threading) from 17.0.64 to 17.1.46. - [Release notes](https://github.com/microsoft/vs-threading/releases) - [Commits](https://github.com/microsoft/vs-threading/commits) --- updated-dependencies: - dependency-name: Microsoft.VisualStudio.Threading.Analyzers dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * try fix build failure on eus (#7929) * CommitBuildTimeProvider read latest state file before save (#7928) * CommitBuildTimeProvider read latest state file before save * Lock Save() to avoid race condition * fallback _buildTime to statice variable * Add repo info log * Bump Microsoft.Graph from 4.18.0 to 4.20.0 (#7931) Bumps [Microsoft.Graph](https://github.com/microsoftgraph/msgraph-sdk-dotnet) from 4.18.0 to 4.20.0. - [Release notes](https://github.com/microsoftgraph/msgraph-sdk-dotnet/releases) - [Commits](https://github.com/microsoftgraph/msgraph-sdk-dotnet/compare/4.18.0...4.20.0) --- updated-dependencies: - dependency-name: Microsoft.Graph dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump Microsoft.AspNetCore.TestHost from 6.0.2 to 6.0.3 (#7930) Bumps [Microsoft.AspNetCore.TestHost](https://github.com/dotnet/aspnetcore) from 6.0.2 to 6.0.3. - [Release notes](https://github.com/dotnet/aspnetcore/releases) - [Changelog](https://github.com/dotnet/aspnetcore/blob/main/docs/ReleasePlanning.md) - [Commits](https://github.com/dotnet/aspnetcore/compare/v6.0.2...v6.0.3) --- updated-dependencies: - dependency-name: Microsoft.AspNetCore.TestHost dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * revert DefaultAzureCredential change, upgrade package (#7933) * Bump Microsoft.DocsLearning.Hierarchy.DataContract (#7935) Bumps Microsoft.DocsLearning.Hierarchy.DataContract from 1.0.22046.1 to 1.0.22073.1. --- updated-dependencies: - dependency-name: Microsoft.DocsLearning.Hierarchy.DataContract dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ying Hua * Bump Microsoft.NET.Test.Sdk from 17.0.0 to 17.1.0 (#7936) Bumps [Microsoft.NET.Test.Sdk](https://github.com/microsoft/vstest) from 17.0.0 to 17.1.0. - [Release notes](https://github.com/microsoft/vstest/releases) - [Commits](https://github.com/microsoft/vstest/compare/v17.0.0...v17.1.0) --- updated-dependencies: - dependency-name: Microsoft.NET.Test.Sdk dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shichao Zhang Co-authored-by: ZhihongLi Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Renze Yu --- .github/actions/spelling/expect.txt | 2 + azure-pipelines-lgtm.yml | 3 + azure-pipelines-v3.yml | 10 +- docs/designs/credential.md | 8 +- docs/designs/restore-file-and-git.md | 128 ------------------ docs/developer-guide.md | 18 +++ .../Microsoft.Docs.Build.Specialized.csproj | 4 +- src/docfx/docfx.csproj | 8 +- src/docfx/lib/git/CommitBuildTimeProvider.cs | 54 +++++--- ...rosoft.Docs.MarkdigExtensions.Tests.csproj | 4 +- test/docfx.SpecTest/docfx.SpecTest.csproj | 4 +- test/docfx.Test/docfx.Test.csproj | 4 +- 12 files changed, 86 insertions(+), 161 deletions(-) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 8dbde95c196..57269392729 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -149,6 +149,8 @@ dfn dhs diagnosics dic +diff +diffs dirname disallowlists displayname diff --git a/azure-pipelines-lgtm.yml b/azure-pipelines-lgtm.yml index 3baaf608ebf..70585e7ca33 100644 --- a/azure-pipelines-lgtm.yml +++ b/azure-pipelines-lgtm.yml @@ -5,6 +5,9 @@ trigger: include: - v3 - v3-release + paths: + exclude: + - docs/designs/* pr: none variables: diff --git a/azure-pipelines-v3.yml b/azure-pipelines-v3.yml index 60408f52933..dfb2d002291 100644 --- a/azure-pipelines-v3.yml +++ b/azure-pipelines-v3.yml @@ -4,8 +4,16 @@ trigger: include: - v3 - v3-release + paths: + exclude: + - docs/designs/* pr: -- v3 + branches: + include: + - v3 + paths: + exclude: + - docs/designs/* variables: - group: devresourcekeyvault diff --git a/docs/designs/credential.md b/docs/designs/credential.md index 7f315cc03b0..92a6f5a8477 100644 --- a/docs/designs/credential.md +++ b/docs/designs/credential.md @@ -32,6 +32,11 @@ http: We can also put other required headers here. e.g. Azure Blob requires the `x-ms-blob-type` header when `PUT` a resource. +### Auto Renew +Sometimes the credential could be expired during DocFX building. +The credential handler acts as HTTP request middleware intercepts the failed HTTP response with 401 code and auto renew the credentials with max 3 retries. + + ## How to authorize other requests There is other types of authorization, like: @@ -56,4 +61,5 @@ Here is the steps: gitHub: authToken: ... ``` -3. Build system pass the secrets to DocFX to build docsets using them. \ No newline at end of file +3. Build system pass the secrets to DocFX to build docsets using them. + diff --git a/docs/designs/restore-file-and-git.md b/docs/designs/restore-file-and-git.md index 19ae4eb8de4..26d1d2da898 100644 --- a/docs/designs/restore-file-and-git.md +++ b/docs/designs/restore-file-and-git.md @@ -125,131 +125,3 @@ We are using [Shared/Exclusive Lock](https://www.ibm.com/support/knowledgecenter - During restore, we acquire an index available from pool with `exclusive lock`. >NOTE: Shared and Exclusive Lock need to be cross process, we are not going to discuss about that details here - -## Dependency lock - -There are some cases you want to keep using the **same version** of dependency files/repository between different `build`s. - -For example, if there are multiple contributors for your content, and there is a CI process to check everyone's change before merging their PRs, you truly want everyone including CI to use the same versions of the dependency files/repositories for builds, to provide valid checking result and reduce influences of different versions of dependencies. - -Docfx uses `.lock.json` as `dependency lock`. Whenever you run docfx `restore`, docfx generates or updates your `.lock.json`, which will look something like this: - -```json -{ - "git":{ - "https://github.com/docfx-code-sample#master":{ - "commit": "fd6968dec9a9a39aec1845232466fe35fca520da" - }, - "https://github.com/docfx-code-sample#live":{ - "commit": "a82cb2a51f6925b2fa87275218b01678c5aadfe1", - "git":{ - "https://github.com/azure-code-sample#live":{ - "commit": "c9a9a39aec184523246678c5aadfea82cb2" - } - }, - "downloads":{ - "https:///some-blob-service/build-history.json":{ - "hash": "d9a9a39aec184523246678c5aadfea82cb2" - } - } - } - }, - "downloads":{ - "https:///some-blob-service/github-user-cache.json":{ - "hash": "e9a9a39aec184523246678c5aadfea82cb2" - } - } -} -``` - -- `git` is the version mappings of dependency repositories -- `downloads` is the version mappings of dependency files -- `commit` is `commit id` of the dependency repository -- `hash` is the sha1 hash of file content - -This file describes an `exact`, and more importantly `reproducible` `dependencies` trees. Once it's present, any future `restore` will base its work off this file, instead of recalculating`. - -The presence of the `.lock.json` changes the `restore` behavior: - -- The dependency files/repository described by this `.lock.json` is reproduced, which means reproducing the structure/version described in the file. -- Any **missing dependencies** are restored in the usual fashion, calculated from `docfx.yml`. - -> Note: currently we only support `git` in `.lock.json` which means the `downloads` will be restored to latest version always, limited by the service provides these downloads. - -> Note: The file is always sorted, so you can safely check-in this file to our content repository. - -## Using locked dependencies - -You can setup your `.lock.json` file(local file or a URL) in the configuration like - -```yml -dependencyLock: https://some-blob-service/repo/branch/.lock.json -``` - -or - -```yml -dependencyLock: .lock.json -``` - -Whenever you run docfx `restore`, docfx will: - -- [**GET**] **Try** to get the `.lock.json` file from configuration, - - if it's present, any future dependencies restore will base its work off this file. - - if it's not set, restore the dependencies in the usual fashion -- [**UPDATE**] Once finished the restore, docfx will **try** to create/update the `.lock.json` file defined in configuration - - If it's present, generate new/update existing `.lock.json` based on the input `.lock.json` + `docfx.yml` - - If ir's not set, nothing happened - -For example, run docfx `restore` with below `docfx.yml` and `.lock.json` - -`docfx.yml`: -```yml -dependencies: - docfx-sample-code: https://github.com/dotnet/docfx-sample-code#master - azure-sample-code: https://github.com/microsoft/azure-sample-code#live -dependencyLock: .lock.json -``` - -`.lock.json`: -```json -{ - "git":{ - "https://github.com/dotnet/docfx-sample-code#master":{ - "commit": "19a9a39aec184523246678c5aadfea82cb2" - } - } -} -``` - -The output for `restored repositories` and `.lock.json` would be like: - -`restored repositories`: -```text -`%DOCFX_APPDATA_PATH%` - | - git - | - github.com+dotnet+docfx-sample-code+5e455995 - | - .git - | - master-19a9a39aec184523246678c5aadfea82cb2 - | - github.com+microsoft+azure-sample-code+5e4589s5 - | - .git - | - master-29a9a39aec184523246678c5aadfea82cb2 -``` -`.lock.json`: -```json -{ - "git":{ - "https://github.com/dotnet/docfx-sample-code#master":{ - "commit": "19a9a39aec184523246678c5aadfea82cb2" - }, - "https://github.com/microsoft/azure-sample-code#live":{ - "commit": "29a9a39aec184523246678c5aadfea82cb2" - } - } -} -``` -For above example: - -- Even there is new commits in `https://github.com/dotnet/docfx-sample-code` `master` branch, but with the `.lock.json` file, we still using the `19a9a39aec184523246678c5aadfea82cb2` commit. -- Since the `https://github.com/microsoft/azure-sample-code#live` is not defined in existing `.lock.json`, `restore` will check-out latest commit of live branch of this repository to one work tree and add it to `.lock.json`. - diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 633968bec74..fa17bd0f50a 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -229,3 +229,21 @@ Convention | Use case | Example `{XXX}Map`, `{XXX}Builder` | Builds an **immutable** `{XXX}Map` from a **mutable** `{XXX}Builder` | `DependencyMap`, `DependencyMapBuilder` `{XXX}Provider` | Groups **instance** helper methods for **Get** or **Resolve** | `MonikerProvider` `{XXX}Utility` | Groups **static** helper methods | `GitUtility` + + +## Regression Test Expected Diffs + +- Sometimes the contributors list may change. You may check against corresponding GitHub pages to double confirm whether the changes are expected or not. + - There will be a new contributor when a new contributor edits the article. + - A contributor will be deleted if her/his public email is disabled. If the new contributor list becomes empty, the whole contributor list will disappear. + - Display name of contributors may change when they change their display name. +- When new validation rules added, they may have the corresponding effects in the diff. +- If the content of articles changed, there will be diffs about content change as well as "word_count", .publish.json and .dependencymap.json, etc. +- Some metadata changes such as "update_at" are expected. + - update_at + - update_at_date_time + - _op_article_date_quotedISO8601 + +If you are confused about some diffs and finally understand they are expected, please add them here. + + diff --git a/src/Microsoft.Docs.Build.Specialized/Microsoft.Docs.Build.Specialized.csproj b/src/Microsoft.Docs.Build.Specialized/Microsoft.Docs.Build.Specialized.csproj index 96e200a1d86..43257db161f 100644 --- a/src/Microsoft.Docs.Build.Specialized/Microsoft.Docs.Build.Specialized.csproj +++ b/src/Microsoft.Docs.Build.Specialized/Microsoft.Docs.Build.Specialized.csproj @@ -7,8 +7,8 @@ - - + + diff --git a/src/docfx/docfx.csproj b/src/docfx/docfx.csproj index b32b572627a..f4d601e8896 100644 --- a/src/docfx/docfx.csproj +++ b/src/docfx/docfx.csproj @@ -12,7 +12,7 @@ - + @@ -23,9 +23,9 @@ - + - + @@ -41,7 +41,7 @@ - + diff --git a/src/docfx/lib/git/CommitBuildTimeProvider.cs b/src/docfx/lib/git/CommitBuildTimeProvider.cs index ae1f0552aba..57105c6ef5b 100644 --- a/src/docfx/lib/git/CommitBuildTimeProvider.cs +++ b/src/docfx/lib/git/CommitBuildTimeProvider.cs @@ -5,27 +5,23 @@ namespace Microsoft.Docs.Build; internal class CommitBuildTimeProvider { + private static readonly object s_lock = new(); + private static readonly DateTime s_now = DateTime.UtcNow; + private readonly DateTime _buildTime; private readonly Repository _repo; private readonly Config _config; private readonly string _commitBuildTimePath; - private readonly IReadOnlyDictionary _buildTimeByCommit; + private IReadOnlyDictionary _buildTimeByCommit; public CommitBuildTimeProvider(Config config, Repository repo) { _repo = repo; _config = config; _commitBuildTimePath = AppData.BuildHistoryStatePath; - _buildTime = config.BuildTime ?? DateTime.UtcNow; - - var exists = File.Exists(_commitBuildTimePath); - Log.Write($"{(exists ? "Using" : "Missing")} git commit build time cache file: '{_commitBuildTimePath}'"); - - var commitBuildTime = exists - ? ProcessUtility.ReadJsonFile(_commitBuildTimePath) - : new CommitBuildTime(); + _buildTime = config.BuildTime ?? s_now; - _buildTimeByCommit = commitBuildTime.Commits.ToDictionary(item => item.Sha, item => item.BuiltAt); + _buildTimeByCommit = ReadLatestCacheIfAny(); } public DateTime GetCommitBuildTime(string commitId) @@ -38,22 +34,42 @@ public void Save() return; } - using (PerfScope.Start($"Saving commit build time for {_repo.Commit}")) + lock (s_lock) { - var commits = _buildTimeByCommit.Select(item => new CommitBuildTimeItem { Sha = item.Key, BuiltAt = item.Value }).ToList(); - - // TODO: retrieve git log from `GitCommitProvider` since it should already be there. - foreach (var diffCommit in GitUtility.GetCommits(_repo.Path, _repo.Commit)) + using (PerfScope.Start($"Saving commit build time for {_repo.Path} {_repo.Commit}")) { - if (!_buildTimeByCommit.ContainsKey(diffCommit)) + _buildTimeByCommit = ReadLatestCacheIfAny(); + var commits = _buildTimeByCommit.Select(item => new CommitBuildTimeItem { Sha = item.Key, BuiltAt = item.Value }).ToList(); + + // TODO: retrieve git log from `GitCommitProvider` since it should already be there. + foreach (var diffCommit in GitUtility.GetCommits(_repo.Path, _repo.Commit)) { - commits.Add(new CommitBuildTimeItem { Sha = diffCommit, BuiltAt = _buildTime }); + if (!_buildTimeByCommit.ContainsKey(diffCommit)) + { + commits.Add(new CommitBuildTimeItem { Sha = diffCommit, BuiltAt = _buildTime }); + } } + + Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(_commitBuildTimePath)) ?? "."); + + ProcessUtility.WriteJsonFile(_commitBuildTimePath, new CommitBuildTime { Commits = commits }); } + } + } - Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(_commitBuildTimePath)) ?? "."); + private IReadOnlyDictionary ReadLatestCacheIfAny() + { + var exists = File.Exists(_commitBuildTimePath); + Log.Write($"{(exists ? "Using" : "Missing")} git commit build time cache file: '{_commitBuildTimePath}'"); - ProcessUtility.WriteJsonFile(_commitBuildTimePath, new CommitBuildTime { Commits = commits }); + if (exists) + { + var commitBuildTime = ProcessUtility.ReadJsonFile(_commitBuildTimePath); + return commitBuildTime.Commits.ToDictionary(item => item.Sha, item => item.BuiltAt); + } + else + { + return _buildTimeByCommit ?? new Dictionary(); } } } diff --git a/test/Microsoft.Docs.MarkdigExtensions.Tests/Microsoft.Docs.MarkdigExtensions.Tests.csproj b/test/Microsoft.Docs.MarkdigExtensions.Tests/Microsoft.Docs.MarkdigExtensions.Tests.csproj index 41e1a00acd7..cb217c45fad 100644 --- a/test/Microsoft.Docs.MarkdigExtensions.Tests/Microsoft.Docs.MarkdigExtensions.Tests.csproj +++ b/test/Microsoft.Docs.MarkdigExtensions.Tests/Microsoft.Docs.MarkdigExtensions.Tests.csproj @@ -6,8 +6,8 @@ - - + + diff --git a/test/docfx.SpecTest/docfx.SpecTest.csproj b/test/docfx.SpecTest/docfx.SpecTest.csproj index 45620067bfc..b673f015d9c 100644 --- a/test/docfx.SpecTest/docfx.SpecTest.csproj +++ b/test/docfx.SpecTest/docfx.SpecTest.csproj @@ -9,8 +9,8 @@ - - + + diff --git a/test/docfx.Test/docfx.Test.csproj b/test/docfx.Test/docfx.Test.csproj index 4318a9c2006..a2672b2ed0f 100644 --- a/test/docfx.Test/docfx.Test.csproj +++ b/test/docfx.Test/docfx.Test.csproj @@ -8,8 +8,8 @@ - - + +