diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge index 9c6f5aefd6de..f32eb7213b94 100755 --- a/.ci/generate-buildkite-pipeline-premerge +++ b/.ci/generate-buildkite-pipeline-premerge @@ -107,7 +107,7 @@ function add-dependencies() { done ;; compiler-rt|libc|openmp) - echo clang + echo clang lld ;; flang|lldb) for p in llvm clang; do diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh index f0577d1069d5..1e7b2d2a36c2 100755 --- a/.ci/monolithic-linux.sh +++ b/.ci/monolithic-linux.sh @@ -45,7 +45,7 @@ cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \ -D LLVM_ENABLE_ASSERTIONS=ON \ -D LLVM_BUILD_EXAMPLES=ON \ -D COMPILER_RT_BUILD_LIBFUZZER=OFF \ - -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \ + -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \ -D LLVM_ENABLE_LLD=ON \ -D CMAKE_CXX_FLAGS=-gmlt \ -D BOLT_CLANG_EXE=/usr/bin/clang \ diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh index 7ac806a0b399..a704e855f011 100755 --- a/.ci/monolithic-windows.sh +++ b/.ci/monolithic-windows.sh @@ -45,7 +45,7 @@ cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \ -D LLVM_ENABLE_ASSERTIONS=ON \ -D LLVM_BUILD_EXAMPLES=ON \ -D COMPILER_RT_BUILD_LIBFUZZER=OFF \ - -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml" \ + -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \ -D COMPILER_RT_BUILD_ORC=OFF \ -D CMAKE_C_COMPILER_LAUNCHER=sccache \ -D CMAKE_CXX_COMPILER_LAUNCHER=sccache \ diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000000..8c1dfd39b82c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + groups: + github-actions: + patterns: + - "*" + - package-ecosystem: "pip" + directory: "/llvm/docs" + schedule: + interval: "monthly" + groups: + llvm-docs-requirements: + patterns: + - "*" diff --git a/.github/new-prs-labeler.yml b/.github/new-prs-labeler.yml index e4ae2b7c839f..b42bcb1887fb 100644 --- a/.github/new-prs-labeler.yml +++ b/.github/new-prs-labeler.yml @@ -308,6 +308,9 @@ clang-tidy: - clang-tools-extra/docs/clang-tidy/** - clang-tools-extra/test/clang-tidy/** +clang-tools-extra: + - clang-tools-extra/** + tools:llvm-mca: - llvm/tools/llvm-mca/** - llvm/include/llvm/MCA/** @@ -591,13 +594,16 @@ mlgo: - llvm/include/llvm/Analysis/*Runner.h - llvm/unittests/Analysis/ML* - llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp + - llvm/lib/Analysis/TrainingLogger.cpp - llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h + - llvm/include/llvm/Analysis/Utils/TrainingLogger.h - llvm/test/Analysis/FunctionPropertiesAnalysis/* - llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp - llvm/test/Transforms/inline/ML/** - llvm/lib/CodeGen/ML* - llvm/unittests/CodeGen/ML* - llvm/test/CodeGen/MLRegAlloc/** + - llvm/utils/mlgo-utils/* tools:llvm-exegesis: - llvm/tools/llvm-exegesis/** diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 2781d9cc0e63..ce34d2337e9c 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1 +1 @@ -Github action workflows should be stored in this directrory. +Github action workflows should be stored in this directory. diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml new file mode 100644 index 000000000000..ad3d50d4d578 --- /dev/null +++ b/.github/workflows/build-ci-container.yml @@ -0,0 +1,60 @@ + +name: Build CI Container + +permissions: + contents: read + +on: + push: + branches: + - main + paths: + - .github/workflows/build-ci-container.yml + - '.github/workflows/containers/github-action-ci/**' + pull_request: + branches: + - main + paths: + - .github/workflows/build-ci-container.yml + - '.github/workflows/containers/github-action-ci/**' + +jobs: + build-ci-container: + if: github.repository_owner == 'llvm' + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - name: Write Variables + id: vars + run: | + tag=`date +%s` + container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/ci-ubuntu-22.04" + echo "container-name=$container_name" >> $GITHUB_OUTPUT + echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT + + - name: Checkout LLVM + uses: actions/checkout@v4 + with: + sparse-checkout: .github/workflows/containers/github-action-ci/ + + - name: Build Container + working-directory: ./.github/workflows/containers/github-action-ci/ + run: | + podman build -t ${{ steps.vars.outputs.container-name-tag }} . + podman tag ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}:latest + + - name: Test Container + run: | + for image in ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}; do + podman run --rm -it $image /usr/bin/bash -x -c 'printf '\''#include \nint main(int argc, char **argv) { std::cout << "Hello\\n"; }'\'' | clang++ -x c++ - && ./a.out | grep Hello' + done + + - name: Push Container + if: github.event_name == 'push' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io + podman push ${{ steps.vars.outputs.container-name-tag }} + podman push ${{ steps.vars.outputs.container-name }}:latest diff --git a/.github/workflows/containers/github-action-ci/Dockerfile b/.github/workflows/containers/github-action-ci/Dockerfile new file mode 100644 index 000000000000..d91a7ad3a9d0 --- /dev/null +++ b/.github/workflows/containers/github-action-ci/Dockerfile @@ -0,0 +1,48 @@ +FROM docker.io/library/ubuntu:22.04 as base +ENV LLVM_SYSROOT=/opt/llvm/ + +FROM base as toolchain +ENV LLVM_MAJOR=17 +ENV LLVM_VERSION=${LLVM_MAJOR}.0.6 +ENV LLVM_DIRNAME=clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-22.04 +ENV LLVM_FILENAME=${LLVM_DIRNAME}.tar.xz + +RUN apt-get update && \ + apt-get install -y \ + curl \ + xz-utils + +RUN mkdir -p $LLVM_SYSROOT/bin/ $LLVM_SYSROOT/lib/ + +RUN curl -O -L https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/$LLVM_FILENAME + +RUN tar -C $LLVM_SYSROOT --strip-components=1 -xJf $LLVM_FILENAME \ + $LLVM_DIRNAME/bin/clang \ + $LLVM_DIRNAME/bin/clang++ \ + $LLVM_DIRNAME/bin/clang-cl \ + $LLVM_DIRNAME/bin/clang-$LLVM_MAJOR \ + $LLVM_DIRNAME/bin/lld \ + $LLVM_DIRNAME/bin/ld.lld \ + $LLVM_DIRNAME/lib/clang/ + + +FROM base + +COPY --from=toolchain $LLVM_SYSROOT $LLVM_SYSROOT + +# Need to install curl for hendrikmuhs/ccache-action +# Need nodejs for some of the GitHub actions. +# Need perl-modules for clang analyzer tests. +RUN apt-get update && \ + apt-get install -y \ + binutils \ + cmake \ + curl \ + libstdc++-11-dev \ + ninja-build \ + nodejs \ + perl-modules \ + python3-psutil + +ENV LLVM_SYSROOT=$LLVM_SYSROOT +ENV PATH=${LLVM_SYSROOT}/bin:${PATH} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9c695e714edd..5936cdc46a90 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -138,7 +138,7 @@ jobs: - name: Build libcxx docs if: steps.docs-changed-subprojects.outputs.libcxx_any_changed == 'true' run: | - cmake -B libcxx-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RUNTIMES="libcxxabi;libcxx" -DLLVM_ENABLE_SPHINX=ON ./runtimes + cmake -B libcxx-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RUNTIMES="libcxxabi;libcxx;libunwind" -DLLVM_ENABLE_SPHINX=ON ./runtimes TZ=UTC ninja -C libcxx-build docs-libcxx-html - name: Build libc docs if: steps.docs-changed-subprojects.outputs.libc_any_changed == 'true' @@ -167,4 +167,3 @@ jobs: run: | cmake -B flang-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;mlir;flang" -DLLVM_ENABLE_SPHINX=ON -DSPHINX_WARNINGS_AS_ERRORS=OFF ./llvm TZ=UTC ninja -C flang-build docs-flang-html docs-flang-man - diff --git a/.github/workflows/issue-subscriber.yml b/.github/workflows/issue-subscriber.yml index 9a9c8f9c6516..ef6cd0674e80 100644 --- a/.github/workflows/issue-subscriber.yml +++ b/.github/workflows/issue-subscriber.yml @@ -13,19 +13,24 @@ jobs: runs-on: ubuntu-latest if: github.repository == 'llvm/llvm-project' steps: + - name: Checkout Automation Script + uses: actions/checkout@v4 + with: + sparse-checkout: llvm/utils/git/ + ref: main + - name: Setup Automation Script + working-directory: ./llvm/utils/git/ run: | - curl -O -L --fail https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/"$GITHUB_SHA"/llvm/utils/git/github-automation.py - curl -O -L --fail https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/"$GITHUB_SHA"/llvm/utils/git/requirements.txt - chmod a+x github-automation.py pip install -r requirements.txt - name: Update watchers + working-directory: ./llvm/utils/git/ # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable env: LABEL_NAME: ${{ github.event.label.name }} run: | - ./github-automation.py \ + python3 ./github-automation.py \ --token '${{ secrets.ISSUE_SUBSCRIBER_TOKEN }}' \ issue-subscriber \ --issue-number '${{ github.event.issue.number }}' \ diff --git a/.github/workflows/libclang-python-tests.yml b/.github/workflows/libclang-python-tests.yml new file mode 100644 index 000000000000..497f6ca5c554 --- /dev/null +++ b/.github/workflows/libclang-python-tests.yml @@ -0,0 +1,47 @@ +name: Libclang Python Binding Tests + +permissions: + contents: read + +on: + workflow_dispatch: + push: + branches: + - 'main' + paths: + - 'clang/bindings/python/**' + - 'clang/tools/libclang/**' + - 'clang/CMakeList.txt' + - '.github/workflows/libclang-python-tests.yml' + - '.github/workflows/llvm-project-tests.yml' + pull_request: + paths: + - 'clang/bindings/python/**' + - 'clang/tools/libclang/**' + - 'clang/CMakeList.txt' + - '.github/workflows/libclang-python-tests.yml' + - '.github/workflows/llvm-project-tests.yml' + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + check-clang-python: + # Build libclang and then run the libclang Python binding's unit tests. + name: Build and run Python unit tests + if: github.repository == 'llvm/llvm-project' + strategy: + fail-fast: false + matrix: + python-version: ["3.7", "3.11"] + uses: ./.github/workflows/llvm-project-tests.yml + with: + build_target: check-clang-python + projects: clang + # There is an issue running on "windows-2019". + # See https://github.com/llvm/llvm-project/issues/76601#issuecomment-1873049082. + os_list: '["ubuntu-latest"]' + python_version: ${{ matrix.python-version }} diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml index 25e8c8c1ef21..5727b956dc6d 100644 --- a/.github/workflows/libcxx-build-and-test.yaml +++ b/.github/workflows/libcxx-build-and-test.yaml @@ -23,8 +23,8 @@ on: - 'cmake/**' - '.github/workflows/libcxx-build-and-test.yaml' schedule: - # Run nightly at 8 AM UTC (or roughly 3 AM eastern) - - cron: '0 3 * * *' + # Run nightly at 08:00 UTC (aka 00:00 Pacific, aka 03:00 Eastern) + - cron: '0 8 * * *' permissions: contents: read # Default everything to read-only @@ -35,7 +35,6 @@ concurrency: env: - CMAKE: "/opt/bin/cmake" # LLVM POST-BRANCH bump version # LLVM POST-BRANCH add compiler test for ToT - 1, e.g. "Clang 17" # LLVM RELEASE bump remove compiler ToT - 3, e.g. "Clang 15" @@ -161,32 +160,26 @@ jobs: 'generic-no-unicode', 'generic-no-wide-characters', 'generic-no-rtti', + 'generic-optimized-speed', 'generic-static', - 'generic-with_llvm_unwinder', # TODO Find a better place for the benchmark and bootstrapping builds to live. They're either very expensive # or don't provide much value since the benchmark run results are too noise on the bots. 'benchmarks', 'bootstrapping-build' ] machine: [ 'libcxx-runners-8-set' ] - std_modules: [ 'OFF' ] include: - config: 'generic-cxx26' machine: libcxx-runners-8-set - std_modules: 'ON' - config: 'generic-asan' machine: libcxx-runners-8-set - std_modules: 'OFF' - config: 'generic-tsan' machine: libcxx-runners-8-set - std_modules: 'OFF' - config: 'generic-ubsan' machine: libcxx-runners-8-set - std_modules: 'OFF' # Use a larger machine for MSAN to avoid timeout and memory allocation issues. - config: 'generic-msan' machine: libcxx-runners-8-set - std_modules: 'OFF' runs-on: ${{ matrix.machine }} steps: - uses: actions/checkout@v4 @@ -196,7 +189,6 @@ jobs: CC: clang-18 CXX: clang++-18 ENABLE_CLANG_TIDY: "OFF" - ENABLE_STD_MODULES: ${{ matrix.std_modules }} - uses: actions/upload-artifact@v3 if: always() with: @@ -207,4 +199,3 @@ jobs: **/CMakeError.log **/CMakeOutput.log **/crash_diagnostics/* - diff --git a/.github/workflows/llvm-project-tests.yml b/.github/workflows/llvm-project-tests.yml index ba696515076c..3c0be7d7d4c1 100644 --- a/.github/workflows/llvm-project-tests.yml +++ b/.github/workflows/llvm-project-tests.yml @@ -18,6 +18,10 @@ on: os_list: required: false default: '["ubuntu-latest", "windows-2019", "macOS-11"]' + python_version: + required: false + type: string + default: '3.11' workflow_call: inputs: cache-key: @@ -46,6 +50,11 @@ on: # https://github.com/actions/virtual-environments/issues/5900 default: '["ubuntu-latest", "windows-2019", "macOS-11"]' + python_version: + required: false + type: string + default: '3.11' + concurrency: # Skip intermediate builds: always. # Cancel intermediate builds: only if it is a pull request build. @@ -75,7 +84,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: ${{ inputs.python_version }} - name: Install Ninja uses: llvm/actions/install-ninja@main # actions/checkout deletes any existing files in the new git directory, diff --git a/.github/workflows/new-prs.yml b/.github/workflows/new-prs.yml index 23fab598fc77..a60f82ce35d1 100644 --- a/.github/workflows/new-prs.yml +++ b/.github/workflows/new-prs.yml @@ -34,16 +34,21 @@ jobs: (github.event.pull_request.author_association != 'MEMBER') && (github.event.pull_request.author_association != 'OWNER') steps: + - name: Checkout Automation Script + uses: actions/checkout@v4 + with: + sparse-checkout: llvm/utils/git/ + ref: main + - name: Setup Automation Script + working-directory: ./llvm/utils/git/ run: | - curl -O -L --fail https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/github-automation.py - curl -O -L --fail https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/requirements.txt - chmod a+x github-automation.py pip install -r requirements.txt - name: Greet Author + working-directory: ./llvm/utils/git/ run: | - ./github-automation.py \ + python3 ./github-automation.py \ --token '${{ secrets.GITHUB_TOKEN }}' \ pr-greeter \ --issue-number "${{ github.event.pull_request.number }}" diff --git a/.github/workflows/pr-subscriber.yml b/.github/workflows/pr-subscriber.yml index 99f9be771588..3952493bb698 100644 --- a/.github/workflows/pr-subscriber.yml +++ b/.github/workflows/pr-subscriber.yml @@ -13,16 +13,21 @@ jobs: runs-on: ubuntu-latest if: github.repository == 'llvm/llvm-project' steps: + - name: Checkout Automation Script + uses: actions/checkout@v4 + with: + sparse-checkout: llvm/utils/git/ + ref: main + - name: Setup Automation Script + working-directory: ./llvm/utils/git/ run: | - curl -O -L --fail https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/github-automation.py - curl -O -L --fail https://raw.githubusercontent.com/"$GITHUB_REPOSITORY"/main/llvm/utils/git/requirements.txt - chmod a+x github-automation.py pip install -r requirements.txt - name: Update watchers + working-directory: ./llvm/utils/git/ run: | - ./github-automation.py \ + python3 ./github-automation.py \ --token '${{ secrets.ISSUE_SUBSCRIBER_TOKEN }}' \ pr-subscriber \ --issue-number "${{ github.event.number }}" \ diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 4e3eaff97a87..ebf6fa41898d 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -1,20 +1,29 @@ name: Release Binaries on: - push: - tags: - - 'llvmorg-*' workflow_dispatch: inputs: + release-version: + description: 'Release Version' + required: true + type: string upload: description: 'Upload binaries to the release page' required: true - default: true + default: false type: boolean - tag: - description: 'Tag to build' + + workflow_call: + inputs: + release-version: + description: 'Release Version' required: true type: string + upload: + description: 'Upload binaries to the release page' + required: true + default: false + type: boolean schedule: # * is a special character in YAML so you have to quote this string - cron: '0 8 1 * *' @@ -26,21 +35,26 @@ jobs: prepare: name: Prepare to build binaries runs-on: ubuntu-22.04 - if: github.repository == 'llvm/llvm-project' outputs: - release-version: ${{ steps.validate-tag.outputs.release-version }} - flags: ${{ steps.validate-tag.outputs.flags }} - build-dir: ${{ steps.validate-tag.outputs.build-dir }} - rc-flags: ${{ steps.validate-tag.outputs.rc-flags }} - ref: ${{ steps.validate-tag.outputs.ref }} - upload: ${{ steps.validate-tag.outputs.upload }} + release-version: ${{ steps.vars.outputs.release-version }} + flags: ${{ steps.vars.outputs.flags }} + build-dir: ${{ steps.vars.outputs.build-dir }} + rc-flags: ${{ steps.vars.outputs.rc-flags }} + ref: ${{ steps.vars.outputs.ref }} + upload: ${{ steps.vars.outputs.upload }} steps: - name: Checkout LLVM - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Check Permissions + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} check-permissions - - name: Validate and parse tag - id: validate-tag + - name: Collect Variables + id: vars # In order for the test-release.sh script to run correctly, the LLVM # source needs to be at the following location relative to the build dir: # | X.Y.Z-rcN | ./rcN/llvm-project @@ -61,9 +75,9 @@ jobs: if [ -n "${{ inputs.upload }}" ]; then upload="${{ inputs.upload }}" else - upload="true" + upload="false" fi - bash .github/workflows/set-release-binary-outputs.sh "${{ github.actor }}" "$tag" "$upload" + bash .github/workflows/set-release-binary-outputs.sh "$tag" "$upload" # Try to get around the 6 hour timeout by first running a job to fill # the build cache. @@ -77,15 +91,15 @@ jobs: - ubuntu-22.04 steps: - name: Checkout LLVM - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: ref: ${{ needs.prepare.outputs.ref }} - name: Install Ninja - uses: llvm/actions/install-ninja@main + uses: llvm/actions/install-ninja@22e9f909d35b50bd1181709564bfe816eaeaae81 # main - name: Setup sccache - uses: hendrikmuhs/ccache-action@v1 + uses: hendrikmuhs/ccache-action@ca3acd2731eef11f1572ccb126356c2f9298d35e # v1.2.9 with: max-size: 250M key: sccache-${{ matrix.os }}-release @@ -93,8 +107,8 @@ jobs: - name: Build Clang run: | - cmake -G Ninja -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_BUILD_TYPE=Release -DCMAKE_ENABLE_ASSERTIONS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DLLVM_ENABLE_PROJECTS=clang -S llvm -B build - ninja -v -C build + cmake -G Ninja -C clang/cmake/caches/Release.cmake -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_POSITION_INDEPENDENT_CODE=ON -S llvm -B build + ninja -v -C build clang build-binaries: @@ -119,13 +133,13 @@ jobs: steps: - name: Checkout LLVM - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: ref: ${{ needs.prepare.outputs.ref }} path: ${{ needs.prepare.outputs.build-dir }}/llvm-project - name: Setup sccache - uses: hendrikmuhs/ccache-action@v1 + uses: hendrikmuhs/ccache-action@ca3acd2731eef11f1572ccb126356c2f9298d35e # v1.2.9 with: max-size: 250M key: sccache-${{ matrix.target.os }}-release @@ -152,6 +166,7 @@ jobs: -triple ${{ matrix.target.triple }} \ -use-ninja \ -no-checkout \ + -use-cmake-cache \ -no-test-suite \ -configure-flags "-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache" diff --git a/.github/workflows/release-documentation.yml b/.github/workflows/release-documentation.yml new file mode 100644 index 000000000000..64572906988b --- /dev/null +++ b/.github/workflows/release-documentation.yml @@ -0,0 +1,88 @@ +name: Release Documentation + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + release-version: + description: 'Release Version' + required: true + type: string + upload: + description: 'Upload documentation' + required: false + type: boolean + + workflow_call: + inputs: + release-version: + description: 'Release Version' + required: true + type: string + upload: + description: 'Upload documentation' + required: false + type: boolean + +jobs: + release-documentation: + name: Build and Upload Release Documentation + runs-on: ubuntu-latest + env: + upload: ${{ inputs.upload && !contains(inputs.release-version, 'rc') }} + steps: + - name: Checkout LLVM + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Setup Python env + uses: actions/setup-python@v4 + with: + cache: 'pip' + cache-dependency-path: './llvm/docs/requirements.txt' + + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + graphviz \ + python3-github \ + ninja-build \ + texlive-font-utils + pip3 install --user -r ./llvm/docs/requirements.txt + + - name: Build Documentation + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ./llvm/utils/release/build-docs.sh -release "${{ inputs.release-version }}" -no-doxygen + + - name: Create Release Notes Artifact + uses: actions/upload-artifact@v3 + with: + name: release-notes + path: docs-build/html-export/ + + - name: Clone www-releases + if: env.upload + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: ${{ github.repository_owner }}/www-releases + ref: main + fetch-depth: 0 + path: www-releases + + - name: Upload Release Notes + if: env.upload + env: + WWW_RELEASES_TOKEN: ${{ secrets.WWW_RELEASES_TOKEN }} + run: | + mkdir -p ../www-releases/${{ inputs.release-version }} + mv ./docs-build/html-export/* ../www-releases/${{ inputs.release-version }} + cd ../www-releases + git add ${{ inputs.release-version }} + git config user.email "llvmbot@llvm.org" + git config user.name "llvmbot" + git commit -a -m "Add ${{ inputs.release-version }} documentation" + git push "https://$WWW_RELEASES_TOKEN@github.com/${{ github.repository_owner }}/www-releases" main:main diff --git a/.github/workflows/release-doxygen.yml b/.github/workflows/release-doxygen.yml new file mode 100644 index 000000000000..5e322849a1d0 --- /dev/null +++ b/.github/workflows/release-doxygen.yml @@ -0,0 +1,67 @@ +name: Release Doxygen + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + release-version: + description: 'Release Version' + required: true + type: string + upload: + description: 'Upload documentation' + required: false + type: boolean + + workflow_call: + inputs: + release-version: + description: 'Release Version' + required: true + type: string + upload: + description: 'Upload documentation' + required: false + type: boolean + +jobs: + release-doxygen: + name: Build and Upload Release Doxygen + runs-on: ubuntu-latest + permissions: + contents: write + env: + upload: ${{ inputs.upload && !contains(inputs.release-version, 'rc') }} + steps: + - name: Checkout LLVM + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Setup Python env + uses: actions/setup-python@v4 + with: + cache: 'pip' + cache-dependency-path: './llvm/docs/requirements.txt' + + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + doxygen \ + graphviz \ + python3-github \ + ninja-build \ + texlive-font-utils + pip3 install --user -r ./llvm/docs/requirements.txt + + - name: Build Doxygen + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ./llvm/utils/release/build-docs.sh -release "${{ inputs.release-version }}" -no-sphinx + + - name: Upload Doxygen + if: env.upload + run: | + ./llvm/utils/release/github-upload-release.py --token "$GITHUB_TOKEN" --release "${{ inputs.release-version }}" --user "${{ github.actor }}" upload --files ./*doxygen*.tar.xz diff --git a/.github/workflows/release-lit.yml b/.github/workflows/release-lit.yml new file mode 100644 index 000000000000..36b0b6edd518 --- /dev/null +++ b/.github/workflows/release-lit.yml @@ -0,0 +1,74 @@ +name: Release Lit + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + release-version: + description: 'Release Version' + required: true + type: string + + workflow_call: + inputs: + release-version: + description: 'Release Version' + required: true + type: string + +jobs: + release-lit: + name: Release Lit + runs-on: ubuntu-latest + steps: + - name: Checkout LLVM + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: "llvmorg-${{ inputs.release-version }}" + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y python3-setuptools python3-psutil python3-github + + - name: Check Permissions + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} check-permissions + + - name: Setup Cpp + uses: aminya/setup-cpp@v1 + with: + compiler: llvm-16.0.6 + cmake: true + ninja: true + + - name: Test lit + run: | + mkdir build && cd build + export FILECHECK_OPTS='-dump-input-filter=all -vv -color' + cmake ../llvm -DCMAKE_BUILD_TYPE=Release -G Ninja + ninja -v -j $(nproc) check-lit + + - name: Package lit + run: | + cd llvm/utils/lit + # Remove 'dev' suffix from lit version. + sed -i 's/ + "dev"//g' lit/__init__.py + python3 setup.py sdist + + - name: Upload lit to test.pypi.org + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.LLVM_LIT_TEST_PYPI_API_TOKEN }} + repository-url: https://test.pypi.org/legacy/ + packages-dir: llvm/utils/lit/dist/ + + - name: Upload lit to pypi.org + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.LLVM_LIT_PYPI_API_TOKEN }} + packages-dir: llvm/utils/lit/dist/ diff --git a/.github/workflows/release-tasks.yml b/.github/workflows/release-tasks.yml index 85b720e323d1..f2a831ad3577 100644 --- a/.github/workflows/release-tasks.yml +++ b/.github/workflows/release-tasks.yml @@ -1,7 +1,7 @@ name: Release Task permissions: - contents: read + contents: write on: push: @@ -10,112 +10,70 @@ on: - 'llvmorg-*' jobs: - release-tasks: - permissions: - contents: write # To upload assets to release. + validate-tag: + name: Validate Tag runs-on: ubuntu-latest if: github.repository == 'llvm/llvm-project' + outputs: + release-version: ${{ steps.validate-tag.outputs.release-version }} steps: - name: Validate Tag id: validate-tag run: | - test "${{ github.actor }}" = "tstellar" || test "${{ github.actor }}" = "tru" echo "${{ github.ref_name }}" | grep -e '^llvmorg-[0-9]\+\.[0-9]\+\.[0-9]\+\(-rc[0-9]\+\)\?$' release_version=$(echo "${{ github.ref_name }}" | sed 's/llvmorg-//g') echo "release-version=$release_version" >> "$GITHUB_OUTPUT" - - name: Checkout LLVM - uses: actions/checkout@v4 - - - name: Install Dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - doxygen \ - graphviz \ - python3-github \ - ninja-build \ - texlive-font-utils - pip3 install --user -r ./llvm/docs/requirements.txt - - - name: Create Release - run: | - ./llvm/utils/release/./github-upload-release.py --token ${{ github.token }} --release ${{ steps.validate-tag.outputs.release-version }} create - - - name: Build Documentation - run: | - ./llvm/utils/release/build-docs.sh -release ${{ steps.validate-tag.outputs.release-version }} - ./llvm/utils/release/github-upload-release.py --token ${{ github.token }} --release ${{ steps.validate-tag.outputs.release-version }} upload --files ./*doxygen*.tar.xz - - - name: Create Release Notes Artifact - uses: actions/upload-artifact@v3 - with: - name: release-notes - path: docs-build/html-export/ - - - name: Clone www-releases - if: ${{ !contains(steps.validate-tag.outputs.release-version, 'rc') }} - uses: actions/checkout@v4 - with: - repository: ${{ github.repository_owner }}/www-releases - ref: main - fetch-depth: 0 - path: www-releases - - - name: Upload Release Notes - if: ${{ !contains(steps.validate-tag.outputs.release-version, 'rc') }} - run: | - mkdir -p ../www-releases/${{ steps.validate-tag.outputs.release-version }} - mv ./docs-build/html-export/* ../www-releases/${{ steps.validate-tag.outputs.release-version }} - cd ../www-releases - git add ${{ steps.validate-tag.outputs.release-version }} - git config user.email "llvmbot@llvm.org" - git config user.name "llvmbot" - git commit -a -m "Add ${{ steps.validate-tag.outputs.release-version }} documentation" - git push https://${{ secrets.WWW_RELEASES_TOKEN }}@github.com/${{ github.repository_owner }}/www-releases main:main - - release-lit: + release-create: + name: Create a New Release runs-on: ubuntu-latest - if: github.repository == 'llvm/llvm-project' + needs: validate-tag steps: - - name: Checkout LLVM - uses: actions/checkout@v4 - - - name: Setup Cpp - uses: aminya/setup-cpp@v1 - with: - compiler: llvm-16.0.6 - cmake: true - ninja: true - - - name: Install dependencies + - name: Install Dependencies run: | sudo apt-get update - sudo apt-get install -y python3-setuptools python3-psutil + sudo apt-get install python3-github - - name: Test lit - run: | - mkdir build && cd build - export FILECHECK_OPTS='-dump-input-filter=all -vv -color' - cmake ../llvm -DCMAKE_BUILD_TYPE=Release -G Ninja - ninja -v -j $(nproc) check-lit + - name: Checkout LLVM + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Package lit + - name: Create Release + env: + GITHUB_TOKEN: ${{ github.token }} run: | - cd llvm/utils/lit - # Remove 'dev' suffix from lit version. - sed -i 's/ + "dev"//g' lit/__init__.py - python3 setup.py sdist - - - name: Upload lit to test.pypi.org - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.LLVM_LIT_TEST_PYPI_API_TOKEN }} - repository-url: https://test.pypi.org/legacy/ - packages-dir: llvm/utils/lit/dist/ + ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --release ${{ needs.validate-tag.outputs.release-version }} --user ${{ github.actor }} create + release-documentation: + name: Build and Upload Release Documentation + needs: + - validate-tag + uses: ./.github/workflows/release-documentation.yml + with: + release-version: ${{ needs.validate-tag.outputs.release-version }} + upload: true + + release-doxygen: + name: Build and Upload Release Doxygen + needs: + - validate-tag + - release-create + uses: ./.github/workflows/release-doxygen.yml + with: + release-version: ${{ needs.validate-tag.outputs.release-version }} + upload: true - - name: Upload lit to pypi.org - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.LLVM_LIT_PYPI_API_TOKEN }} - packages-dir: llvm/utils/lit/dist/ + release-lit: + name: Release Lit + needs: validate-tag + uses: ./.github/workflows/release-lit.yml + with: + release-version: ${{ needs.validate-tag.outputs.release-version }} + + release-binaries: + name: Build Release Binaries + needs: + - validate-tag + - release-create + uses: ./.github/workflows/release-binaries.yml + with: + release-version: ${{ needs.validate-tag.outputs.release-version }} + upload: true diff --git a/.github/workflows/set-release-binary-outputs.sh b/.github/workflows/set-release-binary-outputs.sh index 9bc459a24e80..59470cf83ba7 100644 --- a/.github/workflows/set-release-binary-outputs.sh +++ b/.github/workflows/set-release-binary-outputs.sh @@ -8,14 +8,8 @@ if [ -z "$GITHUB_OUTPUT" ]; then echo "Writing output variables to $GITHUB_OUTPUT" fi -github_user=$1 -tag=$2 -upload=$3 - -if [[ "$github_user" != "tstellar" && "$github_user" != "tru" ]]; then - echo "ERROR: User not allowed: $github_user" - exit 1 -fi +tag=$1 +upload=$2 if echo $tag | grep -e '^[0-9a-f]\+$'; then # This is a plain commit. diff --git a/README.md b/README.md index 0a948b66e06f..a37bc6b20dcb 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ Modifications (c) Copyright 2022-2024 Advanced Micro Devices, Inc. or its affili # The LLVM Compiler Infrastructure [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/llvm/llvm-project/badge)](https://securityscorecards.dev/viewer/?uri=github.com/llvm/llvm-project) +[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8273/badge)](https://www.bestpractices.dev/projects/8273) +[![libc++](https://github.com/llvm/llvm-project/actions/workflows/libcxx-build-and-test.yaml/badge.svg?branch=main&event=schedule)](https://github.com/llvm/llvm-project/actions/workflows/libcxx-build-and-test.yaml?query=event%3Aschedule) Welcome to the LLVM project! diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md new file mode 100644 index 000000000000..0a2c878ef4ae --- /dev/null +++ b/bolt/docs/BAT.md @@ -0,0 +1,97 @@ +# BOLT Address Translation (BAT) +# Purpose +A regular profile collection for BOLT involves collecting samples from +unoptimized binary. BOLT Address Translation allows collecting profile +from BOLT-optimized binary and using it for optimizing the input (pre-BOLT) +binary. + +# Overview +BOLT Address Translation is an extra section (`.note.bolt_bat`) inserted by BOLT +into the output binary containing translation tables and split functions linkage +information. This information enables mapping the profile back from optimized +binary onto the original binary. + +# Usage +`--enable-bat` flag controls the generation of BAT section. Sampled profile +needs to be passed along with the optimized binary containing BAT section to +`perf2bolt` which reads BAT section and produces fdata profile for the original +binary. Note that YAML profile generation is not supported since BAT doesn't +contain the metadata for input functions. + +# Internals +## Section contents +The section is organized as follows: +- Hot functions table + - Address translation tables +- Cold functions table + +## Construction and parsing +BAT section is created from `BoltAddressTranslation` class which captures +address translation information provided by BOLT linker. It is then encoded as a +note section in the output binary. + +During profile conversion when BAT-enabled binary is passed to perf2bolt, +`BoltAddressTranslation` class is populated from BAT section. The class is then +queried by `DataAggregator` during sample processing to reconstruct addresses/ +offsets in the input binary. + +## Encoding format +The encoding is specified in +[BoltAddressTranslation.h](/bolt/include/bolt/Profile/BoltAddressTranslation.h) +and [BoltAddressTranslation.cpp](/bolt/lib/Profile/BoltAddressTranslation.cpp). + +### Layout +The general layout is as follows: +``` +Hot functions table header +|------------------| +| Function entry | +| |--------------| | +| | OutOff InOff | | +| |--------------| | +~~~~~~~~~~~~~~~~~~~~ + +Cold functions table header +|------------------| +| Function entry | +| |--------------| | +| | OutOff InOff | | +| |--------------| | +~~~~~~~~~~~~~~~~~~~~ +``` + +### Functions table +Hot and cold functions tables share the encoding except difference marked below. +Header: +| Entry | Encoding | Description | +| ------ | ----- | ----------- | +| `NumFuncs` | ULEB128 | Number of functions in the functions table | + +The header is followed by Functions table with `NumFuncs` entries. +Output binary addresses are delta encoded, meaning that only the difference with +the last previous output address is stored. Addresses implicitly start at zero. +Output addresses are continuous through function start addresses and function +internal offsets, and between hot and cold fragments, to better spread deltas +and save space. + +Hot indices are delta encoded, implicitly starting at zero. +| Entry | Encoding | Description | +| ------ | ------| ----------- | +| `Address` | Continuous, Delta, ULEB128 | Function address in the output binary | +| `HotIndex` | Delta, ULEB128 | Cold functions only: index of corresponding hot function in hot functions table | +| `NumEntries` | ULEB128 | Number of address translation entries for a function | + +Function header is followed by `NumEntries` pairs of offsets for current +function. + +### Address translation table +Delta encoding means that only the difference with the previous corresponding +entry is encoded. Input offsets implicitly start at zero. +| Entry | Encoding | Description | +| ------ | ------| ----------- | +| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | +| `InputOffset` | Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | + +`BRANCHENTRY` bit denotes whether a given offset pair is a control flow source +(branch or call instruction). If not set, it signifies a control flow target +(basic block offset). diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index 07e4b283211c..f6bd61bc8898 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -11,6 +11,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Support/DataExtractor.h" #include #include #include @@ -118,6 +119,17 @@ class BoltAddressTranslation { void writeEntriesForBB(MapTy &Map, const BinaryBasicBlock &BB, uint64_t FuncAddress); + /// Write the serialized address translation table for a function. + template + void writeMaps(std::map &Maps, uint64_t &PrevAddress, + raw_ostream &OS); + + /// Read the serialized address translation table for a function. + /// Return a parse error if failed. + template + void parseMaps(std::vector &HotFuncs, uint64_t &PrevAddress, + DataExtractor &DE, uint64_t &Offset, Error &Err); + std::map Maps; /// Links outlined cold bocks to their original function @@ -125,7 +137,7 @@ class BoltAddressTranslation { /// Identifies the address of a control-flow changing instructions in a /// translation map entry - const static uint32_t BRANCHENTRY = 0x80000000; + const static uint32_t BRANCHENTRY = 0x1; }; } // namespace bolt diff --git a/bolt/lib/Passes/CacheMetrics.cpp b/bolt/lib/Passes/CacheMetrics.cpp index ba3a2a5f685f..f6708997a433 100644 --- a/bolt/lib/Passes/CacheMetrics.cpp +++ b/bolt/lib/Passes/CacheMetrics.cpp @@ -14,23 +14,20 @@ #include "bolt/Passes/CacheMetrics.h" #include "bolt/Core/BinaryBasicBlock.h" #include "bolt/Core/BinaryFunction.h" -#include "llvm/Support/CommandLine.h" #include using namespace llvm; using namespace bolt; -namespace opts { - -extern cl::OptionCategory BoltOptCategory; - -extern cl::opt ITLBPageSize; -extern cl::opt ITLBEntries; - -} // namespace opts - namespace { +/// The following constants are used to estimate the number of i-TLB cache +/// misses for a given code layout. Empirically the values result in high +/// correlations between the estimations and the perf measurements. +/// The constants do not affect the code layout algorithms. +constexpr unsigned ITLBPageSize = 4096; +constexpr unsigned ITLBEntries = 16; + /// Initialize and return a position map for binary basic blocks void extractBasicBlockInfo( const std::vector &BinaryFunctions, @@ -133,9 +130,6 @@ double expectedCacheHitRatio( const std::vector &BinaryFunctions, const std::unordered_map &BBAddr, const std::unordered_map &BBSize) { - - const double PageSize = opts::ITLBPageSize; - const uint64_t CacheEntries = opts::ITLBEntries; std::unordered_map Calls = extractFunctionCalls(BinaryFunctions); // Compute 'hotness' of the functions @@ -155,7 +149,8 @@ double expectedCacheHitRatio( for (BinaryFunction *BF : BinaryFunctions) { if (BF->getLayout().block_empty()) continue; - double Page = BBAddr.at(BF->getLayout().block_front()) / PageSize; + const uint64_t Page = + BBAddr.at(BF->getLayout().block_front()) / ITLBPageSize; PageSamples[Page] += FunctionSamples.at(BF); } @@ -166,15 +161,17 @@ double expectedCacheHitRatio( if (BF->getLayout().block_empty() || FunctionSamples.at(BF) == 0.0) continue; double Samples = FunctionSamples.at(BF); - double Page = BBAddr.at(BF->getLayout().block_front()) / PageSize; + const uint64_t Page = + BBAddr.at(BF->getLayout().block_front()) / ITLBPageSize; // The probability that the page is not present in the cache - double MissProb = pow(1.0 - PageSamples[Page] / TotalSamples, CacheEntries); + const double MissProb = + pow(1.0 - PageSamples[Page] / TotalSamples, ITLBEntries); // Processing all callers of the function for (std::pair Pair : Calls[BF]) { BinaryFunction *SrcFunction = Pair.first; - double SrcPage = - BBAddr.at(SrcFunction->getLayout().block_front()) / PageSize; + const uint64_t SrcPage = + BBAddr.at(SrcFunction->getLayout().block_front()) / ITLBPageSize; // Is this a 'long' or a 'short' call? if (Page != SrcPage) { // This is a miss diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index e004309e0e21..d3c33d6e6bc7 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -8,8 +8,9 @@ #include "bolt/Profile/BoltAddressTranslation.h" #include "bolt/Core/BinaryFunction.h" -#include "llvm/Support/DataExtractor.h" #include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/LEB128.h" #define DEBUG_TYPE "bolt-bat" @@ -44,7 +45,7 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, // and this deleted block will both share the same output address (the same // key), and we need to map back. We choose here to privilege the successor by // allowing it to overwrite the previously inserted key in the map. - Map[BBOutputOffset] = BBInputOffset; + Map[BBOutputOffset] = BBInputOffset << 1; const auto &IOAddressMap = BB.getFunction()->getBinaryContext().getIOAddressMap(); @@ -61,8 +62,8 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << " Key: " << Twine::utohexstr(OutputOffset) << " Val: " << Twine::utohexstr(InputOffset) << " (branch)\n"); - Map.insert( - std::pair(OutputOffset, InputOffset | BRANCHENTRY)); + Map.insert(std::pair(OutputOffset, + (InputOffset << 1) | BRANCHENTRY)); } } @@ -101,36 +102,53 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { } } - const uint32_t NumFuncs = Maps.size(); - OS.write(reinterpret_cast(&NumFuncs), 4); - LLVM_DEBUG(dbgs() << "Writing " << NumFuncs << " functions for BAT.\n"); + // Output addresses are delta-encoded + uint64_t PrevAddress = 0; + writeMaps(Maps, PrevAddress, OS); + writeMaps(Maps, PrevAddress, OS); + + outs() << "BOLT-INFO: Wrote " << Maps.size() << " BAT maps\n"; +} + +template +void BoltAddressTranslation::writeMaps(std::map &Maps, + uint64_t &PrevAddress, raw_ostream &OS) { + const uint32_t NumFuncs = + llvm::count_if(llvm::make_first_range(Maps), [&](const uint64_t Address) { + return Cold == ColdPartSource.count(Address); + }); + encodeULEB128(NumFuncs, OS); + LLVM_DEBUG(dbgs() << "Writing " << NumFuncs << (Cold ? " cold" : "") + << " functions for BAT.\n"); + size_t PrevIndex = 0; for (auto &MapEntry : Maps) { const uint64_t Address = MapEntry.first; + // Only process cold fragments in cold mode, and vice versa. + if (Cold != ColdPartSource.count(Address)) + continue; MapTy &Map = MapEntry.second; const uint32_t NumEntries = Map.size(); LLVM_DEBUG(dbgs() << "Writing " << NumEntries << " entries for 0x" << Twine::utohexstr(Address) << ".\n"); - OS.write(reinterpret_cast(&Address), 8); - OS.write(reinterpret_cast(&NumEntries), 4); + encodeULEB128(Address - PrevAddress, OS); + PrevAddress = Address; + if (Cold) { + size_t HotIndex = + std::distance(ColdPartSource.begin(), ColdPartSource.find(Address)); + encodeULEB128(HotIndex - PrevIndex, OS); + PrevIndex = HotIndex; + } + encodeULEB128(NumEntries, OS); + uint64_t InOffset = 0; + // Output and Input addresses and delta-encoded for (std::pair &KeyVal : Map) { - OS.write(reinterpret_cast(&KeyVal.first), 4); - OS.write(reinterpret_cast(&KeyVal.second), 4); + const uint64_t OutputAddress = KeyVal.first + Address; + encodeULEB128(OutputAddress - PrevAddress, OS); + PrevAddress = OutputAddress; + encodeSLEB128(KeyVal.second - InOffset, OS); + InOffset = KeyVal.second; } } - const uint32_t NumColdEntries = ColdPartSource.size(); - LLVM_DEBUG(dbgs() << "Writing " << NumColdEntries - << " cold part mappings.\n"); - OS.write(reinterpret_cast(&NumColdEntries), 4); - for (std::pair &ColdEntry : ColdPartSource) { - OS.write(reinterpret_cast(&ColdEntry.first), 8); - OS.write(reinterpret_cast(&ColdEntry.second), 8); - LLVM_DEBUG(dbgs() << " " << Twine::utohexstr(ColdEntry.first) << " -> " - << Twine::utohexstr(ColdEntry.second) << "\n"); - } - - outs() << "BOLT-INFO: Wrote " << Maps.size() << " BAT maps\n"; - outs() << "BOLT-INFO: Wrote " << NumColdEntries - << " BAT cold-to-hot entries\n"; } std::error_code BoltAddressTranslation::parse(StringRef Buf) { @@ -152,53 +170,54 @@ std::error_code BoltAddressTranslation::parse(StringRef Buf) { if (Name.substr(0, 4) != "BOLT") return make_error_code(llvm::errc::io_error); - if (Buf.size() - Offset < 4) - return make_error_code(llvm::errc::io_error); + Error Err(Error::success()); + std::vector HotFuncs; + uint64_t PrevAddress = 0; + parseMaps(HotFuncs, PrevAddress, DE, Offset, Err); + parseMaps(HotFuncs, PrevAddress, DE, Offset, Err); + outs() << "BOLT-INFO: Parsed " << Maps.size() << " BAT entries\n"; + return errorToErrorCode(std::move(Err)); +} - const uint32_t NumFunctions = DE.getU32(&Offset); - LLVM_DEBUG(dbgs() << "Parsing " << NumFunctions << " functions\n"); +template +void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, + uint64_t &PrevAddress, DataExtractor &DE, + uint64_t &Offset, Error &Err) { + const uint32_t NumFunctions = DE.getULEB128(&Offset, &Err); + LLVM_DEBUG(dbgs() << "Parsing " << NumFunctions << (Cold ? " cold" : "") + << " functions\n"); + size_t HotIndex = 0; for (uint32_t I = 0; I < NumFunctions; ++I) { - if (Buf.size() - Offset < 12) - return make_error_code(llvm::errc::io_error); - - const uint64_t Address = DE.getU64(&Offset); - const uint32_t NumEntries = DE.getU32(&Offset); + const uint64_t Address = PrevAddress + DE.getULEB128(&Offset, &Err); + PrevAddress = Address; + if (Cold) { + HotIndex += DE.getULEB128(&Offset, &Err); + ColdPartSource.emplace(Address, HotFuncs[HotIndex]); + } else { + HotFuncs.push_back(Address); + } + const uint32_t NumEntries = DE.getULEB128(&Offset, &Err); MapTy Map; LLVM_DEBUG(dbgs() << "Parsing " << NumEntries << " entries for 0x" << Twine::utohexstr(Address) << "\n"); - if (Buf.size() - Offset < 8 * NumEntries) - return make_error_code(llvm::errc::io_error); + uint64_t InputOffset = 0; for (uint32_t J = 0; J < NumEntries; ++J) { - const uint32_t OutputAddr = DE.getU32(&Offset); - const uint32_t InputAddr = DE.getU32(&Offset); - Map.insert(std::pair(OutputAddr, InputAddr)); - LLVM_DEBUG(dbgs() << Twine::utohexstr(OutputAddr) << " -> " - << Twine::utohexstr(InputAddr) << "\n"); + const uint64_t OutputDelta = DE.getULEB128(&Offset, &Err); + const uint64_t OutputAddress = PrevAddress + OutputDelta; + const uint64_t OutputOffset = OutputAddress - Address; + PrevAddress = OutputAddress; + const int64_t InputDelta = DE.getSLEB128(&Offset, &Err); + InputOffset += InputDelta; + Map.insert(std::pair(OutputOffset, InputOffset)); + LLVM_DEBUG( + dbgs() << formatv("{0:x} -> {1:x} ({2}/{3}b -> {4}/{5}b), {6:x}\n", + OutputOffset, InputOffset, OutputDelta, + encodeULEB128(OutputDelta, nulls()), InputDelta, + encodeSLEB128(InputDelta, nulls()), OutputAddress)); } Maps.insert(std::pair(Address, Map)); } - - if (Buf.size() - Offset < 4) - return make_error_code(llvm::errc::io_error); - - const uint32_t NumColdEntries = DE.getU32(&Offset); - LLVM_DEBUG(dbgs() << "Parsing " << NumColdEntries << " cold part mappings\n"); - for (uint32_t I = 0; I < NumColdEntries; ++I) { - if (Buf.size() - Offset < 16) - return make_error_code(llvm::errc::io_error); - const uint32_t ColdAddress = DE.getU64(&Offset); - const uint32_t HotAddress = DE.getU64(&Offset); - ColdPartSource.insert( - std::pair(ColdAddress, HotAddress)); - LLVM_DEBUG(dbgs() << Twine::utohexstr(ColdAddress) << " -> " - << Twine::utohexstr(HotAddress) << "\n"); - } - outs() << "BOLT-INFO: Parsed " << Maps.size() << " BAT entries\n"; - outs() << "BOLT-INFO: Parsed " << NumColdEntries - << " BAT cold-to-hot entries\n"; - - return std::error_code(); } void BoltAddressTranslation::dump(raw_ostream &OS) { @@ -209,7 +228,7 @@ void BoltAddressTranslation::dump(raw_ostream &OS) { OS << "BB mappings:\n"; for (const auto &Entry : MapEntry.second) { const bool IsBranch = Entry.second & BRANCHENTRY; - const uint32_t Val = Entry.second & ~BRANCHENTRY; + const uint32_t Val = Entry.second >> 1; // dropping BRANCHENTRY bit OS << "0x" << Twine::utohexstr(Entry.first) << " -> " << "0x" << Twine::utohexstr(Val); if (IsBranch) @@ -244,7 +263,7 @@ uint64_t BoltAddressTranslation::translate(uint64_t FuncAddress, --KeyVal; - const uint32_t Val = KeyVal->second & ~BRANCHENTRY; + const uint32_t Val = KeyVal->second >> 1; // dropping BRANCHENTRY bit // Branch source addresses are translated to the first instruction of the // source BB to avoid accounting for modifications BOLT may have made in the // BB regarding deletion/addition of instructions. diff --git a/bolt/lib/Rewrite/CMakeLists.txt b/bolt/lib/Rewrite/CMakeLists.txt index b0e2b7f46bef..6890f52e2b28 100644 --- a/bolt/lib/Rewrite/CMakeLists.txt +++ b/bolt/lib/Rewrite/CMakeLists.txt @@ -6,6 +6,7 @@ set(LLVM_LINK_COMPONENTS Object Support DWARFLinker + DWARFLinkerClassic AsmPrinter TargetParser ) diff --git a/bolt/lib/Rewrite/DWARFRewriter.cpp b/bolt/lib/Rewrite/DWARFRewriter.cpp index 05fb3e8fafe2..8e20306925fe 100644 --- a/bolt/lib/Rewrite/DWARFRewriter.cpp +++ b/bolt/lib/Rewrite/DWARFRewriter.cpp @@ -21,7 +21,7 @@ #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/DIE.h" -#include "llvm/DWARFLinker/DWARFStreamer.h" +#include "llvm/DWARFLinker/Classic/DWARFStreamer.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h" #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h" @@ -178,6 +178,9 @@ translateInputToOutputLocationList(const BinaryFunction &BF, return MergedLL; } +using namespace dwarf_linker; +using namespace dwarf_linker::classic; + namespace llvm { namespace bolt { /// Emits debug information into .debug_info or .debug_types section. @@ -278,10 +281,10 @@ class DIEStreamer : public DwarfStreamer { public: DIEStreamer(DIEBuilder *DIEBldr, DWARFRewriter &Rewriter, - DWARFLinker::OutputFileType OutFileType, + DWARFLinkerBase::OutputFileType OutFileType, raw_pwrite_stream &OutFile, std::function Translator, - DWARFLinker::messageHandler Warning) + DWARFLinkerBase::MessageHandlerTy Warning) : DwarfStreamer(OutFileType, OutFile, Translator, Warning), DIEBldr(DIEBldr), Rewriter(Rewriter){}; @@ -457,7 +460,7 @@ createDIEStreamer(const Triple &TheTriple, raw_pwrite_stream &OutFile, DWARFRewriter &Rewriter) { std::unique_ptr Streamer = std::make_unique( - &DIEBldr, Rewriter, llvm::DWARFLinker::OutputFileType::Object, OutFile, + &DIEBldr, Rewriter, DWARFLinkerBase::OutputFileType::Object, OutFile, [](StringRef Input) -> StringRef { return Input; }, [&](const Twine &Warning, StringRef Context, const DWARFDie *) {}); Error Err = Streamer->init(TheTriple, Swift5ReflectionSegmentName); diff --git a/bolt/lib/Rewrite/JITLinkLinker.cpp b/bolt/lib/Rewrite/JITLinkLinker.cpp index 10a3b6a0407f..66e129bf1d05 100644 --- a/bolt/lib/Rewrite/JITLinkLinker.cpp +++ b/bolt/lib/Rewrite/JITLinkLinker.cpp @@ -173,7 +173,8 @@ struct JITLinkLinker::Context : jitlink::JITLinkContext { void notifyFinalized( jitlink::JITLinkMemoryManager::FinalizedAlloc Alloc) override { - Linker.Allocs.push_back(std::move(Alloc)); + if (Alloc) + Linker.Allocs.push_back(std::move(Alloc)); ++Linker.MM->ObjectsLoaded; } }; diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index a95b1650753c..f5a8a5b71687 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -4112,6 +4112,7 @@ void RewriteInstance::encodeBATSection() { copyByteArray(BoltInfo), BoltInfo.size(), /*Alignment=*/1, /*IsReadOnly=*/true, ELF::SHT_NOTE); + outs() << "BOLT-INFO: BAT section size (bytes): " << BoltInfo.size() << '\n'; } template diff --git a/bolt/test/RISCV/relax.s b/bolt/test/RISCV/relax.s index bf9287e1d0d8..ec390ea76b5c 100644 --- a/bolt/test/RISCV/relax.s +++ b/bolt/test/RISCV/relax.s @@ -6,7 +6,7 @@ // CHECK: Binary Function "_start" after building cfg { // CHECK: jal ra, near_f -// CHECK-NEXT: auipc ra, far_f@plt +// CHECK-NEXT: auipc ra, far_f // CHECK-NEXT: jalr ra, 0xc(ra) // CHECK-NEXT: j near_f diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test index f68a8f7e9bcb..430b4cb00731 100644 --- a/bolt/test/X86/bolt-address-translation.test +++ b/bolt/test/X86/bolt-address-translation.test @@ -36,7 +36,7 @@ # # CHECK: BOLT: 3 out of 7 functions were overwritten. # CHECK: BOLT-INFO: Wrote 6 BAT maps -# CHECK: BOLT-INFO: Wrote 3 BAT cold-to-hot entries +# CHECK: BOLT-INFO: BAT section size (bytes): 404 # # usqrt mappings (hot part). We match against any key (left side containing # the bolted binary offsets) because BOLT may change where it puts instructions diff --git a/bolt/test/X86/merge-fdata-bat-mode.test b/bolt/test/X86/merge-fdata-bat-mode.test index ea3e4c3e4afc..41738e196b5d 100644 --- a/bolt/test/X86/merge-fdata-bat-mode.test +++ b/bolt/test/X86/merge-fdata-bat-mode.test @@ -4,6 +4,5 @@ RUN: merge-fdata %S/Inputs/bat_profile_1.fdata \ RUN: %S/Inputs/bat_profile_2.fdata \ RUN: | FileCheck %s --check-prefix=CHECK-FDATA +CHECK-FDATA: boltedcollection CHECK-FDATA: 1 main 451 1 SolveCubic 0 0 302 - - diff --git a/bolt/test/X86/merge-fdata-nobat-mode.test b/bolt/test/X86/merge-fdata-nobat-mode.test new file mode 100644 index 000000000000..870d9f880e28 --- /dev/null +++ b/bolt/test/X86/merge-fdata-nobat-mode.test @@ -0,0 +1,6 @@ +# Check that merge-fdata tool doesn't spuriously print boltedcollection + +RUN: merge-fdata %S/Inputs/blarge.fdata %S/Inputs/blarge.fdata \ +RUN: | FileCheck %s --check-prefix=CHECK-FDATA + +CHECK-FDATA-NOT: boltedcollection diff --git a/bolt/tools/bat-dump/bat-dump.cpp b/bolt/tools/bat-dump/bat-dump.cpp index fa53c5b122f8..71efe008d409 100644 --- a/bolt/tools/bat-dump/bat-dump.cpp +++ b/bolt/tools/bat-dump/bat-dump.cpp @@ -75,7 +75,7 @@ static std::string GetExecutablePath(const char *Argv0) { if (llvm::ErrorOr P = llvm::sys::findProgramByName(ExecutablePath)) ExecutablePath = *P; - return std::string(ExecutablePath.str()); + return std::string(ExecutablePath); } void dumpBATFor(llvm::object::ELFObjectFileBase *InputFile) { diff --git a/bolt/tools/driver/llvm-bolt.cpp b/bolt/tools/driver/llvm-bolt.cpp index 5a3af6a44b52..cc215a5256d2 100644 --- a/bolt/tools/driver/llvm-bolt.cpp +++ b/bolt/tools/driver/llvm-bolt.cpp @@ -174,7 +174,7 @@ static std::string GetExecutablePath(const char *Argv0) { if (llvm::ErrorOr P = llvm::sys::findProgramByName(ExecutablePath)) ExecutablePath = *P; - return std::string(ExecutablePath.str()); + return std::string(ExecutablePath); } int main(int argc, char **argv) { diff --git a/bolt/tools/heatmap/heatmap.cpp b/bolt/tools/heatmap/heatmap.cpp index f5204fa8ec60..9b190dd288b2 100644 --- a/bolt/tools/heatmap/heatmap.cpp +++ b/bolt/tools/heatmap/heatmap.cpp @@ -46,7 +46,7 @@ static std::string GetExecutablePath(const char *Argv0) { if (llvm::ErrorOr P = llvm::sys::findProgramByName(ExecutablePath)) ExecutablePath = *P; - return std::string(ExecutablePath.str()); + return std::string(ExecutablePath); } int main(int argc, char **argv) { diff --git a/bolt/tools/merge-fdata/merge-fdata.cpp b/bolt/tools/merge-fdata/merge-fdata.cpp index e21aeba73aca..104991dabc5f 100644 --- a/bolt/tools/merge-fdata/merge-fdata.cpp +++ b/bolt/tools/merge-fdata/merge-fdata.cpp @@ -329,7 +329,7 @@ void mergeLegacyProfiles(const SmallVectorImpl &Filenames) { MergedProfile.insert_or_assign(Key, Count); } - if (BoltedCollection) + if (BoltedCollection.value_or(false)) output() << "boltedcollection\n"; for (const auto &[Key, Value] : MergedProfile) output() << Key << " " << Value << "\n"; diff --git a/clang-tools-extra/clang-apply-replacements/include/clang-apply-replacements/Tooling/ApplyReplacements.h b/clang-tools-extra/clang-apply-replacements/include/clang-apply-replacements/Tooling/ApplyReplacements.h index 1f0d87371914..7a8408bcdff8 100644 --- a/clang-tools-extra/clang-apply-replacements/include/clang-apply-replacements/Tooling/ApplyReplacements.h +++ b/clang-tools-extra/clang-apply-replacements/include/clang-apply-replacements/Tooling/ApplyReplacements.h @@ -51,19 +51,27 @@ using FileToChangesMap = /// Directories starting with '.' are ignored during traversal. /// /// \param[in] Directory Directory to begin search for serialized -/// TranslationUnitReplacements. +/// TranslationUnitReplacements or TranslationUnitDiagnostics. /// \param[out] TUs Collection of all found and deserialized /// TranslationUnitReplacements or TranslationUnitDiagnostics. -/// \param[out] TUFiles Collection of all TranslationUnitReplacement files -/// found in \c Directory. +/// \param[out] TUFiles Collection of all TranslationUnitReplacement or +/// TranslationUnitDiagnostics files found in \c Directory. /// \param[in] Diagnostics DiagnosticsEngine used for error output. /// /// \returns An error_code indicating success or failure in navigating the /// directory structure. +template +std::error_code collectReplacementsFromDirectory( + const llvm::StringRef Directory, TranslationUnits &TUs, + TUReplacementFiles &TUFiles, + clang::DiagnosticsEngine &Diagnostics) = delete; + +template <> std::error_code collectReplacementsFromDirectory( const llvm::StringRef Directory, TUReplacements &TUs, TUReplacementFiles &TUFiles, clang::DiagnosticsEngine &Diagnostics); +template <> std::error_code collectReplacementsFromDirectory( const llvm::StringRef Directory, TUDiagnostics &TUs, TUReplacementFiles &TUFiles, clang::DiagnosticsEngine &Diagnostics); diff --git a/clang-tools-extra/clang-apply-replacements/lib/Tooling/ApplyReplacements.cpp b/clang-tools-extra/clang-apply-replacements/lib/Tooling/ApplyReplacements.cpp index b490780f4852..87ed1b8797cb 100644 --- a/clang-tools-extra/clang-apply-replacements/lib/Tooling/ApplyReplacements.cpp +++ b/clang-tools-extra/clang-apply-replacements/lib/Tooling/ApplyReplacements.cpp @@ -38,8 +38,10 @@ static void eatDiagnostics(const SMDiagnostic &, void *) {} namespace clang { namespace replace { -std::error_code collectReplacementsFromDirectory( - const llvm::StringRef Directory, TUReplacements &TUs, +namespace detail { +template +static std::error_code collectReplacementsFromDirectory( + const llvm::StringRef Directory, TranslationUnits &TUs, TUReplacementFiles &TUFiles, clang::DiagnosticsEngine &Diagnostics) { using namespace llvm::sys::fs; using namespace llvm::sys::path; @@ -68,7 +70,7 @@ std::error_code collectReplacementsFromDirectory( } yaml::Input YIn(Out.get()->getBuffer(), nullptr, &eatDiagnostics); - tooling::TranslationUnitReplacements TU; + typename TranslationUnits::value_type TU; YIn >> TU; if (YIn.error()) { // File doesn't appear to be a header change description. Ignore it. @@ -81,49 +83,22 @@ std::error_code collectReplacementsFromDirectory( return ErrorCode; } +} // namespace detail +template <> std::error_code collectReplacementsFromDirectory( - const llvm::StringRef Directory, TUDiagnostics &TUs, + const llvm::StringRef Directory, TUReplacements &TUs, TUReplacementFiles &TUFiles, clang::DiagnosticsEngine &Diagnostics) { - using namespace llvm::sys::fs; - using namespace llvm::sys::path; - - std::error_code ErrorCode; - - for (recursive_directory_iterator I(Directory, ErrorCode), E; - I != E && !ErrorCode; I.increment(ErrorCode)) { - if (filename(I->path())[0] == '.') { - // Indicate not to descend into directories beginning with '.' - I.no_push(); - continue; - } - - if (extension(I->path()) != ".yaml") - continue; - - TUFiles.push_back(I->path()); - - ErrorOr> Out = - MemoryBuffer::getFile(I->path()); - if (std::error_code BufferError = Out.getError()) { - errs() << "Error reading " << I->path() << ": " << BufferError.message() - << "\n"; - continue; - } - - yaml::Input YIn(Out.get()->getBuffer(), nullptr, &eatDiagnostics); - tooling::TranslationUnitDiagnostics TU; - YIn >> TU; - if (YIn.error()) { - // File doesn't appear to be a header change description. Ignore it. - continue; - } - - // Only keep files that properly parse. - TUs.push_back(TU); - } + return detail::collectReplacementsFromDirectory(Directory, TUs, TUFiles, + Diagnostics); +} - return ErrorCode; +template <> +std::error_code collectReplacementsFromDirectory( + const llvm::StringRef Directory, TUDiagnostics &TUs, + TUReplacementFiles &TUFiles, clang::DiagnosticsEngine &Diagnostics) { + return detail::collectReplacementsFromDirectory(Directory, TUs, TUFiles, + Diagnostics); } /// Extract replacements from collected TranslationUnitReplacements and diff --git a/clang-tools-extra/clang-doc/HTMLGenerator.cpp b/clang-tools-extra/clang-doc/HTMLGenerator.cpp index d2466857e511..f6355ccf4f84 100644 --- a/clang-tools-extra/clang-doc/HTMLGenerator.cpp +++ b/clang-tools-extra/clang-doc/HTMLGenerator.cpp @@ -457,7 +457,7 @@ writeFileDefinition(const Location &L, Node->Children.emplace_back(std::make_unique(" of file ")); auto LocFileNode = std::make_unique( HTMLTag::TAG_A, llvm::sys::path::filename(FileURL)); - LocFileNode->Attributes.emplace_back("href", std::string(FileURL.str())); + LocFileNode->Attributes.emplace_back("href", std::string(FileURL)); Node->Children.emplace_back(std::move(LocFileNode)); return Node; } diff --git a/clang-tools-extra/clang-include-fixer/tool/clang-include-fixer.el b/clang-tools-extra/clang-include-fixer/tool/clang-include-fixer.el index 272f282c47f5..f3a949f8c1b5 100644 --- a/clang-tools-extra/clang-include-fixer/tool/clang-include-fixer.el +++ b/clang-tools-extra/clang-include-fixer/tool/clang-include-fixer.el @@ -1,5 +1,6 @@ ;;; clang-include-fixer.el --- Emacs integration of the clang include fixer -*- lexical-binding: t; -*- +;; Version: 0.1.0 ;; Keywords: tools, c ;; Package-Requires: ((cl-lib "0.5") (json "1.2") (let-alist "1.0.4")) diff --git a/clang-tools-extra/clang-query/QueryParser.cpp b/clang-tools-extra/clang-query/QueryParser.cpp index 41933625a4fa..162acc1a598d 100644 --- a/clang-tools-extra/clang-query/QueryParser.cpp +++ b/clang-tools-extra/clang-query/QueryParser.cpp @@ -28,10 +28,8 @@ namespace query { // is found before End, return StringRef(). Begin is adjusted to exclude the // lexed region. StringRef QueryParser::lexWord() { - Line = Line.drop_while([](char c) { - // Don't trim newlines. - return StringRef(" \t\v\f\r").contains(c); - }); + // Don't trim newlines. + Line = Line.ltrim(" \t\v\f\r"); if (Line.empty()) // Even though the Line is empty, it contains a pointer and @@ -152,8 +150,7 @@ QueryRef QueryParser::parseSetTraversalKind(TraversalKind QuerySession::*Var) { QueryRef QueryParser::endQuery(QueryRef Q) { StringRef Extra = Line; - StringRef ExtraTrimmed = Extra.drop_while( - [](char c) { return StringRef(" \t\v\f\r").contains(c); }); + StringRef ExtraTrimmed = Extra.ltrim(" \t\v\f\r"); if ((!ExtraTrimmed.empty() && ExtraTrimmed[0] == '\n') || (ExtraTrimmed.size() >= 2 && ExtraTrimmed[0] == '\r' && diff --git a/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp b/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp index e414ac8c7705..5ecd4fb19131 100644 --- a/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp +++ b/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp @@ -100,7 +100,7 @@ ExpandModularHeadersPPCallbacks::ExpandModularHeadersPPCallbacks( /*OwnsHeaderSearch=*/false); PP->Initialize(Compiler.getTarget(), Compiler.getAuxTarget()); InitializePreprocessor(*PP, *PO, Compiler.getPCHContainerReader(), - Compiler.getFrontendOpts()); + Compiler.getFrontendOpts(), Compiler.getCodeGenOpts()); ApplyHeaderSearchOptions(*HeaderInfo, *HSO, LangOpts, Compiler.getTarget().getTriple()); } diff --git a/clang-tools-extra/clang-tidy/GlobList.cpp b/clang-tools-extra/clang-tidy/GlobList.cpp index 694db35106fd..dfe3f7c505b1 100644 --- a/clang-tools-extra/clang-tidy/GlobList.cpp +++ b/clang-tools-extra/clang-tidy/GlobList.cpp @@ -16,11 +16,7 @@ namespace clang::tidy { // from the GlobList. static bool consumeNegativeIndicator(StringRef &GlobList) { GlobList = GlobList.trim(); - if (GlobList.starts_with("-")) { - GlobList = GlobList.substr(1); - return true; - } - return false; + return GlobList.consume_front("-"); } // Converts first glob from the comma-separated list of globs to Regex and diff --git a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp index 435cb1e3fbcf..a8a23b045f80 100644 --- a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp @@ -17,6 +17,7 @@ #include "BoolPointerImplicitConversionCheck.h" #include "BranchCloneCheck.h" #include "CastingThroughVoidCheck.h" +#include "ChainedComparisonCheck.h" #include "ComparePointerToMemberVirtualFunctionCheck.h" #include "CopyConstructorInitCheck.h" #include "DanglingHandleCheck.h" @@ -108,6 +109,8 @@ class BugproneModule : public ClangTidyModule { CheckFactories.registerCheck("bugprone-branch-clone"); CheckFactories.registerCheck( "bugprone-casting-through-void"); + CheckFactories.registerCheck( + "bugprone-chained-comparison"); CheckFactories.registerCheck( "bugprone-compare-pointer-to-member-virtual-function"); CheckFactories.registerCheck( diff --git a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt index 70e7fbc7ec0c..1cd6fb207d76 100644 --- a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt +++ b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt @@ -12,6 +12,7 @@ add_clang_library(clangTidyBugproneModule BranchCloneCheck.cpp BugproneTidyModule.cpp CastingThroughVoidCheck.cpp + ChainedComparisonCheck.cpp ComparePointerToMemberVirtualFunctionCheck.cpp CopyConstructorInitCheck.cpp DanglingHandleCheck.cpp diff --git a/clang-tools-extra/clang-tidy/bugprone/ChainedComparisonCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ChainedComparisonCheck.cpp new file mode 100644 index 000000000000..c6bb5bdf5ce9 --- /dev/null +++ b/clang-tools-extra/clang-tidy/bugprone/ChainedComparisonCheck.cpp @@ -0,0 +1,150 @@ +//===--- ChainedComparisonCheck.cpp - clang-tidy --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ChainedComparisonCheck.h" +#include "clang/AST/ASTContext.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" +#include + +using namespace clang::ast_matchers; + +namespace clang::tidy::bugprone { +static bool isExprAComparisonOperator(const Expr *E) { + if (const auto *Op = dyn_cast_or_null(E->IgnoreImplicit())) + return Op->isComparisonOp(); + if (const auto *Op = + dyn_cast_or_null(E->IgnoreImplicit())) + return Op->isComparisonOp(); + return false; +} + +namespace { +AST_MATCHER(BinaryOperator, + hasBinaryOperatorAChildComparisonOperatorWithoutParen) { + return isExprAComparisonOperator(Node.getLHS()) || + isExprAComparisonOperator(Node.getRHS()); +} + +AST_MATCHER(CXXOperatorCallExpr, + hasCppOperatorAChildComparisonOperatorWithoutParen) { + return std::any_of(Node.arg_begin(), Node.arg_end(), + isExprAComparisonOperator); +} + +struct ChainedComparisonData { + llvm::SmallString<256U> Name; + llvm::SmallVector Operands; + + explicit ChainedComparisonData(const Expr *Op) { extract(Op); } + +private: + void add(const Expr *Operand); + void add(llvm::StringRef Opcode); + void extract(const Expr *Op); + void extract(const BinaryOperator *Op); + void extract(const CXXOperatorCallExpr *Op); +}; + +void ChainedComparisonData::add(const Expr *Operand) { + if (!Name.empty()) + Name += ' '; + Name += 'v'; + Name += std::to_string(Operands.size()); + Operands.push_back(Operand); +} + +void ChainedComparisonData::add(llvm::StringRef Opcode) { + Name += ' '; + Name += Opcode; +} + +void ChainedComparisonData::extract(const BinaryOperator *Op) { + const Expr *LHS = Op->getLHS()->IgnoreImplicit(); + if (isExprAComparisonOperator(LHS)) + extract(LHS); + else + add(LHS); + + add(Op->getOpcodeStr()); + + const Expr *RHS = Op->getRHS()->IgnoreImplicit(); + if (isExprAComparisonOperator(RHS)) + extract(RHS); + else + add(RHS); +} + +void ChainedComparisonData::extract(const CXXOperatorCallExpr *Op) { + const Expr *FirstArg = Op->getArg(0U)->IgnoreImplicit(); + if (isExprAComparisonOperator(FirstArg)) + extract(FirstArg); + else + add(FirstArg); + + add(getOperatorSpelling(Op->getOperator())); + + const Expr *SecondArg = Op->getArg(1U)->IgnoreImplicit(); + if (isExprAComparisonOperator(SecondArg)) + extract(SecondArg); + else + add(SecondArg); +} + +void ChainedComparisonData::extract(const Expr *Op) { + if (!Op) + return; + + if (const auto *BinaryOp = dyn_cast(Op)) { + extract(BinaryOp); + return; + } + + if (const auto *OverloadedOp = dyn_cast(Op)) { + if (OverloadedOp->getNumArgs() == 2U) + extract(OverloadedOp); + } +} + +} // namespace + +void ChainedComparisonCheck::registerMatchers(MatchFinder *Finder) { + const auto OperatorMatcher = expr(anyOf( + binaryOperator(isComparisonOperator(), + hasBinaryOperatorAChildComparisonOperatorWithoutParen()), + cxxOperatorCallExpr( + isComparisonOperator(), + hasCppOperatorAChildComparisonOperatorWithoutParen()))); + + Finder->addMatcher( + expr(OperatorMatcher, unless(hasParent(OperatorMatcher))).bind("op"), + this); +} + +void ChainedComparisonCheck::check(const MatchFinder::MatchResult &Result) { + const auto *MatchedOperator = Result.Nodes.getNodeAs("op"); + + ChainedComparisonData Data(MatchedOperator); + if (Data.Operands.empty()) + return; + + diag(MatchedOperator->getBeginLoc(), + "chained comparison '%0' may generate unintended results, use " + "parentheses to specify order of evaluation or a logical operator to " + "separate comparison expressions") + << llvm::StringRef(Data.Name).trim() << MatchedOperator->getSourceRange(); + + for (std::size_t Index = 0U; Index < Data.Operands.size(); ++Index) { + diag(Data.Operands[Index]->getBeginLoc(), "operand 'v%0' is here", + DiagnosticIDs::Note) + << Index << Data.Operands[Index]->getSourceRange(); + } +} + +} // namespace clang::tidy::bugprone diff --git a/clang-tools-extra/clang-tidy/bugprone/ChainedComparisonCheck.h b/clang-tools-extra/clang-tidy/bugprone/ChainedComparisonCheck.h new file mode 100644 index 000000000000..a914149a42e6 --- /dev/null +++ b/clang-tools-extra/clang-tidy/bugprone/ChainedComparisonCheck.h @@ -0,0 +1,34 @@ +//===--- ChainedComparisonCheck.h - clang-tidy ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_CHAINEDCOMPARISONCHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_CHAINEDCOMPARISONCHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::bugprone { + +/// Check detects chained comparison operators that can lead to unintended +/// behavior or logical errors. +/// +/// For the user-facing documentation see: +/// http://clang.llvm.org/extra/clang-tidy/checks/bugprone/chained-comparison.html +class ChainedComparisonCheck : public ClangTidyCheck { +public: + ChainedComparisonCheck(StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context) {} + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; + std::optional getCheckTraversalKind() const override { + return TK_IgnoreUnlessSpelledInSource; + } +}; + +} // namespace clang::tidy::bugprone + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_CHAINEDCOMPARISONCHECK_H diff --git a/clang-tools-extra/clang-tidy/bugprone/EmptyCatchCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/EmptyCatchCheck.cpp index 865c88391b0b..ff5b885aa95f 100644 --- a/clang-tools-extra/clang-tidy/bugprone/EmptyCatchCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/EmptyCatchCheck.cpp @@ -46,7 +46,7 @@ AST_MATCHER_P(CompoundStmt, hasAnyTextFromList, std::vector, StringRef Text = Lexer::getSourceText( CharSourceRange::getTokenRange(Node.getSourceRange()), SM, Context.getLangOpts()); - return std::any_of(List.begin(), List.end(), [&](const StringRef &Str) { + return llvm::any_of(List, [&](const StringRef &Str) { return Text.contains_insensitive(Str); }); } diff --git a/clang-tools-extra/clang-tidy/bugprone/ExceptionEscapeCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ExceptionEscapeCheck.cpp index 90bf523ffb00..620a57194acb 100644 --- a/clang-tools-extra/clang-tidy/bugprone/ExceptionEscapeCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/ExceptionEscapeCheck.cpp @@ -28,6 +28,10 @@ AST_MATCHER(FunctionDecl, isExplicitThrow) { Node.getExceptionSpecSourceRange().isValid(); } +AST_MATCHER(FunctionDecl, hasAtLeastOneParameter) { + return Node.getNumParams() > 0; +} + } // namespace ExceptionEscapeCheck::ExceptionEscapeCheck(StringRef Name, @@ -58,14 +62,16 @@ void ExceptionEscapeCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { void ExceptionEscapeCheck::registerMatchers(MatchFinder *Finder) { Finder->addMatcher( - functionDecl(isDefinition(), - anyOf(isNoThrow(), - allOf(anyOf(cxxDestructorDecl(), - cxxConstructorDecl(isMoveConstructor()), - cxxMethodDecl(isMoveAssignmentOperator()), - isMain(), hasName("swap")), - unless(isExplicitThrow())), - isEnabled(FunctionsThatShouldNotThrow))) + functionDecl( + isDefinition(), + anyOf(isNoThrow(), + allOf(anyOf(cxxDestructorDecl(), + cxxConstructorDecl(isMoveConstructor()), + cxxMethodDecl(isMoveAssignmentOperator()), isMain(), + allOf(hasAnyName("swap", "iter_swap", "iter_move"), + hasAtLeastOneParameter())), + unless(isExplicitThrow())), + isEnabled(FunctionsThatShouldNotThrow))) .bind("thrower"), this); } diff --git a/clang-tools-extra/clang-tidy/bugprone/ImplicitWideningOfMultiplicationResultCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ImplicitWideningOfMultiplicationResultCheck.cpp index 95a7c521eabb..6f22f02f3018 100644 --- a/clang-tools-extra/clang-tidy/bugprone/ImplicitWideningOfMultiplicationResultCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/ImplicitWideningOfMultiplicationResultCheck.cpp @@ -9,6 +9,7 @@ #include "ImplicitWideningOfMultiplicationResultCheck.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/Lex/Lexer.h" #include using namespace clang::ast_matchers; @@ -99,15 +100,16 @@ void ImplicitWideningOfMultiplicationResultCheck::handleImplicitCastExpr( "make conversion explicit to silence this warning", DiagnosticIDs::Note) << E->getSourceRange(); - + const SourceLocation EndLoc = Lexer::getLocForEndOfToken( + E->getEndLoc(), 0, *Result->SourceManager, getLangOpts()); if (ShouldUseCXXStaticCast) Diag << FixItHint::CreateInsertion( E->getBeginLoc(), "static_cast<" + Ty.getAsString() + ">(") - << FixItHint::CreateInsertion(E->getEndLoc(), ")"); + << FixItHint::CreateInsertion(EndLoc, ")"); else Diag << FixItHint::CreateInsertion(E->getBeginLoc(), "(" + Ty.getAsString() + ")(") - << FixItHint::CreateInsertion(E->getEndLoc(), ")"); + << FixItHint::CreateInsertion(EndLoc, ")"); Diag << includeStddefHeader(E->getBeginLoc()); } @@ -137,7 +139,11 @@ void ImplicitWideningOfMultiplicationResultCheck::handleImplicitCastExpr( Diag << FixItHint::CreateInsertion(LHS->getBeginLoc(), "static_cast<" + WideExprTy.getAsString() + ">(") - << FixItHint::CreateInsertion(LHS->getEndLoc(), ")"); + << FixItHint::CreateInsertion( + Lexer::getLocForEndOfToken(LHS->getEndLoc(), 0, + *Result->SourceManager, + getLangOpts()), + ")"); else Diag << FixItHint::CreateInsertion(LHS->getBeginLoc(), "(" + WideExprTy.getAsString() + ")"); @@ -206,16 +212,17 @@ void ImplicitWideningOfMultiplicationResultCheck::handlePointerOffsetting( "make conversion explicit to silence this warning", DiagnosticIDs::Note) << IndexExpr->getSourceRange(); - + const SourceLocation EndLoc = Lexer::getLocForEndOfToken( + IndexExpr->getEndLoc(), 0, *Result->SourceManager, getLangOpts()); if (ShouldUseCXXStaticCast) Diag << FixItHint::CreateInsertion( IndexExpr->getBeginLoc(), (Twine("static_cast<") + TyAsString + ">(").str()) - << FixItHint::CreateInsertion(IndexExpr->getEndLoc(), ")"); + << FixItHint::CreateInsertion(EndLoc, ")"); else Diag << FixItHint::CreateInsertion(IndexExpr->getBeginLoc(), (Twine("(") + TyAsString + ")(").str()) - << FixItHint::CreateInsertion(IndexExpr->getEndLoc(), ")"); + << FixItHint::CreateInsertion(EndLoc, ")"); Diag << includeStddefHeader(IndexExpr->getBeginLoc()); } @@ -229,7 +236,11 @@ void ImplicitWideningOfMultiplicationResultCheck::handlePointerOffsetting( Diag << FixItHint::CreateInsertion( LHS->getBeginLoc(), (Twine("static_cast<") + TyAsString + ">(").str()) - << FixItHint::CreateInsertion(LHS->getEndLoc(), ")"); + << FixItHint::CreateInsertion( + Lexer::getLocForEndOfToken(IndexExpr->getEndLoc(), 0, + *Result->SourceManager, + getLangOpts()), + ")"); else Diag << FixItHint::CreateInsertion(LHS->getBeginLoc(), (Twine("(") + TyAsString + ")").str()); diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp index 0b85ea19735e..370de12999ac 100644 --- a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp +++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp @@ -53,16 +53,72 @@ AST_MATCHER(ParmVarDecl, isTemplateTypeParameter) { FuncTemplate->getTemplateParameters()->getDepth(); } +AST_MATCHER_P(NamedDecl, hasSameNameAsBoundNode, std::string, BindingID) { + IdentifierInfo *II = Node.getIdentifier(); + if (nullptr == II) + return false; + StringRef Name = II->getName(); + + return Builder->removeBindings( + [this, Name](const ast_matchers::internal::BoundNodesMap &Nodes) { + const DynTypedNode &BN = Nodes.getNode(this->BindingID); + if (const auto *ND = BN.get()) { + if (!isa(ND)) + return true; + return ND->getName() != Name; + } + return true; + }); +} + +AST_MATCHER_P(LambdaCapture, hasCaptureKind, LambdaCaptureKind, Kind) { + return Node.getCaptureKind() == Kind; +} + +AST_MATCHER_P(LambdaExpr, hasCaptureDefaultKind, LambdaCaptureDefault, Kind) { + return Node.getCaptureDefault() == Kind; +} + } // namespace void MissingStdForwardCheck::registerMatchers(MatchFinder *Finder) { + auto RefToParmImplicit = allOf( + equalsBoundNode("var"), hasInitializer(ignoringParenImpCasts( + declRefExpr(to(equalsBoundNode("param")))))); + auto RefToParm = capturesVar( + varDecl(anyOf(hasSameNameAsBoundNode("param"), RefToParmImplicit))); + auto HasRefToParm = hasAnyCapture(RefToParm); + + auto CaptureInRef = + allOf(hasCaptureDefaultKind(LambdaCaptureDefault::LCD_ByRef), + unless(hasAnyCapture( + capturesVar(varDecl(hasSameNameAsBoundNode("param")))))); + auto CaptureInCopy = allOf( + hasCaptureDefaultKind(LambdaCaptureDefault::LCD_ByCopy), HasRefToParm); + auto CaptureByRefExplicit = hasAnyCapture( + allOf(hasCaptureKind(LambdaCaptureKind::LCK_ByRef), RefToParm)); + + auto CapturedInBody = + lambdaExpr(anyOf(CaptureInRef, CaptureInCopy, CaptureByRefExplicit)); + auto CapturedInCaptureList = hasAnyCapture(capturesVar( + varDecl(hasInitializer(ignoringParenImpCasts(equalsBoundNode("call")))))); + + auto CapturedInLambda = hasDeclContext(cxxRecordDecl( + isLambda(), + hasParent(lambdaExpr(forCallable(equalsBoundNode("func")), + anyOf(CapturedInCaptureList, CapturedInBody))))); + auto ToParam = hasAnyParameter(parmVarDecl(equalsBoundNode("param"))); auto ForwardCallMatcher = callExpr( - forCallable(equalsBoundNode("func")), argumentCountIs(1), + callExpr().bind("call"), argumentCountIs(1), + hasArgument( + 0, declRefExpr(to( + varDecl(optionally(equalsBoundNode("param"))).bind("var")))), + forCallable(anyOf(equalsBoundNode("func"), CapturedInLambda)), callee(unresolvedLookupExpr(hasAnyDeclaration( namedDecl(hasUnderlyingDecl(hasName("::std::forward")))))), - hasArgument(0, declRefExpr(to(equalsBoundNode("param"))).bind("ref")), + unless(anyOf(hasAncestor(typeLoc()), hasAncestor(expr(hasUnevaluatedContext()))))); diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.cpp index 23d730337152..f79a67819bb8 100644 --- a/clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.cpp +++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.cpp @@ -261,9 +261,9 @@ void PreferMemberInitializerCheck::check( SmallString<128> Insertion( {UseAssignment ? " = " : "{", - Lexer::getSourceText( - CharSourceRange(InitValue->getSourceRange(), true), - *Result.SourceManager, getLangOpts()), + Lexer::getSourceText(Result.SourceManager->getExpansionRange( + InitValue->getSourceRange()), + *Result.SourceManager, getLangOpts()), UseAssignment ? "" : "}"}); Diag << FixItHint::CreateInsertion(FieldEnd, Insertion) @@ -346,7 +346,7 @@ void PreferMemberInitializerCheck::check( if (InvalidFix) continue; StringRef NewInit = Lexer::getSourceText( - CharSourceRange(InitValue->getSourceRange(), true), + Result.SourceManager->getExpansionRange(InitValue->getSourceRange()), *Result.SourceManager, getLangOpts()); if (HasInitAlready) { if (InsertPos.isValid()) diff --git a/clang-tools-extra/clang-tidy/fuchsia/MultipleInheritanceCheck.cpp b/clang-tools-extra/clang-tidy/fuchsia/MultipleInheritanceCheck.cpp index 8aecd3ba27b2..b5ce23ae8fed 100644 --- a/clang-tools-extra/clang-tidy/fuchsia/MultipleInheritanceCheck.cpp +++ b/clang-tools-extra/clang-tidy/fuchsia/MultipleInheritanceCheck.cpp @@ -53,7 +53,7 @@ bool MultipleInheritanceCheck::isCurrentClassInterface( // Interfaces should have exclusively pure methods. return llvm::none_of(Node->methods(), [](const CXXMethodDecl *M) { - return M->isUserProvided() && !M->isPure() && !M->isStatic(); + return M->isUserProvided() && !M->isPureVirtual() && !M->isStatic(); }); } @@ -103,8 +103,8 @@ void MultipleInheritanceCheck::check(const MatchFinder::MatchResult &Result) { const auto *Base = cast(Ty->getDecl()->getDefinition()); if (!isInterface(Base)) NumConcrete++; } - - // Check virtual bases to see if there is more than one concrete + + // Check virtual bases to see if there is more than one concrete // non-virtual base. for (const auto &V : D->vbases()) { const auto *Ty = V.getType()->getAs(); diff --git a/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp b/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp index 995961b07748..3109bbb3724c 100644 --- a/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp +++ b/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp @@ -23,14 +23,16 @@ void AvoidCStyleCastsCheck::registerMatchers( // Filter out (EnumType)IntegerLiteral construct, which is generated // for non-type template arguments of enum types. // FIXME: Remove this once this is fixed in the AST. - unless(hasParent(substNonTypeTemplateParmExpr())), - // Avoid matches in template instantiations. - unless(isInTemplateInstantiation())) + unless(hasParent(substNonTypeTemplateParmExpr()))) .bind("cast"), this); + Finder->addMatcher( - cxxFunctionalCastExpr(unless(hasDescendant(cxxConstructExpr())), - unless(hasDescendant(initListExpr()))) + cxxFunctionalCastExpr( + hasDestinationType(hasCanonicalType(anyOf( + builtinType(), references(qualType()), pointsTo(qualType())))), + unless( + hasSourceExpression(anyOf(cxxConstructExpr(), initListExpr())))) .bind("cast"), this); } diff --git a/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.h b/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.h index 485640d23028..4267b896b699 100644 --- a/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.h +++ b/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.h @@ -31,6 +31,9 @@ class AvoidCStyleCastsCheck : public ClangTidyCheck { : ClangTidyCheck(Name, Context) {} void registerMatchers(ast_matchers::MatchFinder *Finder) override; void check(const ast_matchers::MatchFinder::MatchResult &Result) override; + std::optional getCheckTraversalKind() const override { + return TK_IgnoreUnlessSpelledInSource; + } }; } // namespace clang::tidy::google::readability diff --git a/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.cpp b/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.cpp index bb4e1de8cc4b..ef511e9108f2 100644 --- a/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.cpp +++ b/clang-tools-extra/clang-tidy/google/IntegerTypesCheck.cpp @@ -36,6 +36,12 @@ static Token getTokenAtLoc(SourceLocation Loc, return Tok; } +namespace { +AST_MATCHER(FunctionDecl, isUserDefineLiteral) { + return Node.getLiteralIdentifier() != nullptr; +} +} // namespace + namespace tidy::google::runtime { IntegerTypesCheck::IntegerTypesCheck(StringRef Name, ClangTidyContext *Context) @@ -56,11 +62,14 @@ void IntegerTypesCheck::registerMatchers(MatchFinder *Finder) { // http://google.github.io/styleguide/cppguide.html#64-bit_Portability // "Where possible, avoid passing arguments of types specified by // bitwidth typedefs to printf-based APIs." - Finder->addMatcher(typeLoc(loc(isInteger()), - unless(hasAncestor(callExpr( - callee(functionDecl(hasAttr(attr::Format))))))) - .bind("tl"), - this); + Finder->addMatcher( + typeLoc(loc(isInteger()), + unless(anyOf(hasAncestor(callExpr( + callee(functionDecl(hasAttr(attr::Format))))), + hasParent(parmVarDecl(hasAncestor( + functionDecl(isUserDefineLiteral()))))))) + .bind("tl"), + this); IdentTable = std::make_unique(getLangOpts()); } diff --git a/clang-tools-extra/clang-tidy/misc/CoroutineHostileRAIICheck.cpp b/clang-tools-extra/clang-tidy/misc/CoroutineHostileRAIICheck.cpp index a0e8700b0522..360335b86c64 100644 --- a/clang-tools-extra/clang-tidy/misc/CoroutineHostileRAIICheck.cpp +++ b/clang-tools-extra/clang-tidy/misc/CoroutineHostileRAIICheck.cpp @@ -56,7 +56,7 @@ AST_MATCHER_P(Stmt, forEachPrevStmt, ast_matchers::internal::Matcher, // Matches the expression awaited by the `co_await`. AST_MATCHER_P(CoawaitExpr, awaitable, ast_matchers::internal::Matcher, InnerMatcher) { - if (Expr *E = Node.getCommonExpr()) + if (Expr *E = Node.getOperand()) return InnerMatcher.matches(*E, Finder, Builder); return false; } diff --git a/clang-tools-extra/clang-tidy/misc/StaticAssertCheck.cpp b/clang-tools-extra/clang-tidy/misc/StaticAssertCheck.cpp index 77763a9f429f..35536b47700a 100644 --- a/clang-tools-extra/clang-tidy/misc/StaticAssertCheck.cpp +++ b/clang-tools-extra/clang-tidy/misc/StaticAssertCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "StaticAssertCheck.h" +#include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/AST/Expr.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -45,13 +46,20 @@ void StaticAssertCheck::registerMatchers(MatchFinder *Finder) { IsAlwaysFalse); auto NonConstexprFunctionCall = callExpr(hasDeclaration(functionDecl(unless(isConstexpr())))); + auto NonConstexprVariableReference = + declRefExpr(to(varDecl(unless(isConstexpr()))), + unless(hasAncestor(expr(matchers::hasUnevaluatedContext()))), + unless(hasAncestor(typeLoc()))); + + auto NonConstexprCode = + expr(anyOf(NonConstexprFunctionCall, NonConstexprVariableReference)); auto AssertCondition = expr( anyOf(expr(ignoringParenCasts(anyOf( AssertExprRoot, unaryOperator(hasUnaryOperand( ignoringParenCasts(AssertExprRoot)))))), anything()), - unless(findAll(NonConstexprFunctionCall))) + unless(NonConstexprCode), unless(hasDescendant(NonConstexprCode))) .bind("condition"); auto Condition = anyOf(ignoringParenImpCasts(callExpr( diff --git a/clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.cpp b/clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.cpp index 051375263e53..59e487bab311 100644 --- a/clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.cpp +++ b/clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.cpp @@ -117,8 +117,10 @@ void UnusedUsingDeclsCheck::check(const MatchFinder::MatchResult &Result) { /*SkipTrailingWhitespaceAndNewLine=*/true)); for (const auto *UsingShadow : Using->shadows()) { const auto *TargetDecl = UsingShadow->getTargetDecl()->getCanonicalDecl(); - if (shouldCheckDecl(TargetDecl)) + if (shouldCheckDecl(TargetDecl)) { Context.UsingTargetDecls.insert(TargetDecl); + UsingTargetDeclsCache.insert(TargetDecl); + } } if (!Context.UsingTargetDecls.empty()) Contexts.push_back(Context); @@ -201,13 +203,16 @@ void UnusedUsingDeclsCheck::check(const MatchFinder::MatchResult &Result) { void UnusedUsingDeclsCheck::removeFromFoundDecls(const Decl *D) { if (!D) return; + const Decl *CanonicalDecl = D->getCanonicalDecl(); + if (!UsingTargetDeclsCache.contains(CanonicalDecl)) + return; // FIXME: Currently, we don't handle the using-decls being used in different // scopes (such as different namespaces, different functions). Instead of // giving an incorrect message, we mark all of them as used. - // - // FIXME: Use a more efficient way to find a matching context. for (auto &Context : Contexts) { - if (Context.UsingTargetDecls.contains(D->getCanonicalDecl())) + if (Context.IsUsed) + continue; + if (Context.UsingTargetDecls.contains(CanonicalDecl)) Context.IsUsed = true; } } @@ -224,6 +229,7 @@ void UnusedUsingDeclsCheck::onEndOfTranslationUnit() { } } Contexts.clear(); + UsingTargetDeclsCache.clear(); } } // namespace clang::tidy::misc diff --git a/clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.h b/clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.h index fa2a8799d098..7bdaf12e8aec 100644 --- a/clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.h +++ b/clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.h @@ -26,6 +26,9 @@ class UnusedUsingDeclsCheck : public ClangTidyCheck { void registerMatchers(ast_matchers::MatchFinder *Finder) override; void check(const ast_matchers::MatchFinder::MatchResult &Result) override; void onEndOfTranslationUnit() override; + bool isLanguageVersionSupported(const LangOptions &LangOpts) const override { + return LangOpts.CPlusPlus; + } private: void removeFromFoundDecls(const Decl *D); @@ -46,6 +49,7 @@ class UnusedUsingDeclsCheck : public ClangTidyCheck { }; std::vector Contexts; + llvm::SmallPtrSet UsingTargetDeclsCache; StringRef RawStringHeaderFileExtensions; FileExtensionsSet HeaderFileExtensions; diff --git a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp index 8beaa62c78ba..f0791da143ad 100644 --- a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp @@ -942,11 +942,15 @@ bool LoopConvertCheck::isConvertible(ASTContext *Context, CanonicalInitVarType->isPointerType()) { // If the initializer and the variable are both pointers check if the // un-qualified pointee types match, otherwise we don't use auto. - if (!Context->hasSameUnqualifiedType( - CanonicalBeginType->getPointeeType(), - CanonicalInitVarType->getPointeeType())) - return false; + return Context->hasSameUnqualifiedType( + CanonicalBeginType->getPointeeType(), + CanonicalInitVarType->getPointeeType()); } + + if (CanonicalBeginType->isBuiltinType() || + CanonicalInitVarType->isBuiltinType()) + return false; + } else if (FixerKind == LFK_PseudoArray) { if (const auto *EndCall = Nodes.getNodeAs(EndCallName)) { // This call is required to obtain the container. diff --git a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp index 616e57efa76d..d1d7e9dcfa9c 100644 --- a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp @@ -323,8 +323,7 @@ bool MakeSmartPtrCheck::replaceNew(DiagnosticBuilder &Diag, return false; }; switch (New->getInitializationStyle()) { - case CXXNewInitializationStyle::None: - case CXXNewInitializationStyle::Implicit: { + case CXXNewInitializationStyle::None: { if (ArraySizeExpr.empty()) { Diag << FixItHint::CreateRemoval(SourceRange(NewStart, NewEnd)); } else { @@ -335,7 +334,7 @@ bool MakeSmartPtrCheck::replaceNew(DiagnosticBuilder &Diag, } break; } - case CXXNewInitializationStyle::Call: { + case CXXNewInitializationStyle::Parens: { // FIXME: Add fixes for constructors with parameters that can be created // with a C++11 braced-init-list (e.g. std::vector, std::map). // Unlike ordinal cases, braced list can not be deduced in @@ -372,7 +371,7 @@ bool MakeSmartPtrCheck::replaceNew(DiagnosticBuilder &Diag, } break; } - case CXXNewInitializationStyle::List: { + case CXXNewInitializationStyle::Braces: { // Range of the substring that we do not want to remove. SourceRange InitRange; if (const auto *NewConstruct = New->getConstructExpr()) { diff --git a/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp index 7af30e688b6a..aec67808846b 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp @@ -8,10 +8,12 @@ #include "UseAutoCheck.h" #include "clang/AST/ASTContext.h" +#include "clang/AST/TypeLoc.h" #include "clang/ASTMatchers/ASTMatchFinder.h" #include "clang/ASTMatchers/ASTMatchers.h" #include "clang/Basic/CharInfo.h" #include "clang/Tooling/FixIt.h" +#include "llvm/ADT/STLExtras.h" using namespace clang; using namespace clang::ast_matchers; @@ -333,6 +335,25 @@ void UseAutoCheck::replaceIterators(const DeclStmt *D, ASTContext *Context) { << FixItHint::CreateReplacement(Range, "auto"); } +static void ignoreTypeLocClasses( + TypeLoc &Loc, + std::initializer_list const &LocClasses) { + while (llvm::is_contained(LocClasses, Loc.getTypeLocClass())) + Loc = Loc.getNextTypeLoc(); +} + +static bool isMutliLevelPointerToTypeLocClasses( + TypeLoc Loc, + std::initializer_list const &LocClasses) { + ignoreTypeLocClasses(Loc, {TypeLoc::Paren, TypeLoc::Qualified}); + TypeLoc::TypeLocClass TLC = Loc.getTypeLocClass(); + if (TLC != TypeLoc::Pointer && TLC != TypeLoc::MemberPointer) + return false; + ignoreTypeLocClasses(Loc, {TypeLoc::Paren, TypeLoc::Qualified, + TypeLoc::Pointer, TypeLoc::MemberPointer}); + return llvm::is_contained(LocClasses, Loc.getTypeLocClass()); +} + void UseAutoCheck::replaceExpr( const DeclStmt *D, ASTContext *Context, llvm::function_ref GetType, StringRef Message) { @@ -342,6 +363,10 @@ void UseAutoCheck::replaceExpr( return; const QualType FirstDeclType = FirstDecl->getType().getCanonicalType(); + TypeSourceInfo *TSI = FirstDecl->getTypeSourceInfo(); + + if (TSI == nullptr) + return; std::vector StarRemovals; for (const auto *Dec : D->decls()) { @@ -383,17 +408,11 @@ void UseAutoCheck::replaceExpr( // is the same as the initializer, just more CV-qualified. However, TypeLoc // information is not reliable where CV qualifiers are concerned so we can't // do anything about this case for now. - TypeLoc Loc = FirstDecl->getTypeSourceInfo()->getTypeLoc(); - if (!RemoveStars) { - while (Loc.getTypeLocClass() == TypeLoc::Pointer || - Loc.getTypeLocClass() == TypeLoc::Qualified) - Loc = Loc.getNextTypeLoc(); - } - while (Loc.getTypeLocClass() == TypeLoc::LValueReference || - Loc.getTypeLocClass() == TypeLoc::RValueReference || - Loc.getTypeLocClass() == TypeLoc::Qualified) { - Loc = Loc.getNextTypeLoc(); - } + TypeLoc Loc = TSI->getTypeLoc(); + if (!RemoveStars) + ignoreTypeLocClasses(Loc, {TypeLoc::Pointer, TypeLoc::Qualified}); + ignoreTypeLocClasses(Loc, {TypeLoc::LValueReference, TypeLoc::RValueReference, + TypeLoc::Qualified}); SourceRange Range(Loc.getSourceRange()); if (MinTypeNameLength != 0 && @@ -405,12 +424,19 @@ void UseAutoCheck::replaceExpr( auto Diag = diag(Range.getBegin(), Message); + bool ShouldReplenishVariableName = isMutliLevelPointerToTypeLocClasses( + TSI->getTypeLoc(), {TypeLoc::FunctionProto, TypeLoc::ConstantArray}); + // Space after 'auto' to handle cases where the '*' in the pointer type is // next to the identifier. This avoids changing 'int *p' into 'autop'. - // FIXME: This doesn't work for function pointers because the variable name - // is inside the type. - Diag << FixItHint::CreateReplacement(Range, RemoveStars ? "auto " : "auto") - << StarRemovals; + llvm::StringRef Auto = ShouldReplenishVariableName + ? (RemoveStars ? "auto " : "auto *") + : (RemoveStars ? "auto " : "auto"); + std::string ReplenishedVariableName = + ShouldReplenishVariableName ? FirstDecl->getQualifiedNameAsString() : ""; + std::string Replacement = + (Auto + llvm::StringRef{ReplenishedVariableName}).str(); + Diag << FixItHint::CreateReplacement(Range, Replacement) << StarRemovals; } void UseAutoCheck::check(const MatchFinder::MatchResult &Result) { diff --git a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp index 4438f0b22063..430455a38f39 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp @@ -13,6 +13,10 @@ using namespace clang::ast_matchers; namespace clang::tidy::modernize { namespace { +AST_MATCHER_P(InitListExpr, initCountLeq, unsigned, N) { + return Node.getNumInits() <= N; +} + // Identical to hasAnyName, except it does not take template specifiers into // account. This is used to match the functions names as in // DefaultEmplacyFunctions below without caring about the template types of the @@ -205,11 +209,12 @@ void UseEmplaceCheck::registerMatchers(MatchFinder *Finder) { auto HasConstructExpr = has(ignoringImplicit(SoughtConstructExpr)); // allow for T{} to be replaced, even if no CTOR is declared - auto HasConstructInitListExpr = has(initListExpr(anyOf( - allOf(has(SoughtConstructExpr), - has(cxxConstructExpr(argumentCountIs(0)))), - has(cxxBindTemporaryExpr(has(SoughtConstructExpr), - has(cxxConstructExpr(argumentCountIs(0)))))))); + auto HasConstructInitListExpr = has(initListExpr( + initCountLeq(1), anyOf(allOf(has(SoughtConstructExpr), + has(cxxConstructExpr(argumentCountIs(0)))), + has(cxxBindTemporaryExpr( + has(SoughtConstructExpr), + has(cxxConstructExpr(argumentCountIs(0)))))))); auto HasBracedInitListExpr = anyOf(has(cxxBindTemporaryExpr(HasConstructInitListExpr)), HasConstructInitListExpr); diff --git a/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.cpp index 059a0af60d3e..9561cc71183d 100644 --- a/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.cpp +++ b/clang-tools-extra/clang-tidy/modernize/UseEqualsDeleteCheck.cpp @@ -17,11 +17,12 @@ namespace clang::tidy::modernize { namespace { AST_MATCHER(FunctionDecl, hasAnyDefinition) { - if (Node.hasBody() || Node.isPure() || Node.isDefaulted() || Node.isDeleted()) + if (Node.hasBody() || Node.isPureVirtual() || Node.isDefaulted() || + Node.isDeleted()) return true; if (const FunctionDecl *Definition = Node.getDefinition()) - if (Definition->hasBody() || Definition->isPure() || + if (Definition->hasBody() || Definition->isPureVirtual() || Definition->isDefaulted() || Definition->isDeleted()) return true; diff --git a/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.cpp b/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.cpp index 5bfa6fb0d02d..655e480ffa62 100644 --- a/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.cpp +++ b/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.cpp @@ -97,6 +97,15 @@ bool ForRangeCopyCheck::handleConstValueCopy(const VarDecl &LoopVar, return true; } +static bool isReferenced(const VarDecl &LoopVar, const Stmt &Stmt, + ASTContext &Context) { + const auto IsLoopVar = varDecl(equalsNode(&LoopVar)); + return !match(stmt(hasDescendant(declRefExpr(to(valueDecl(anyOf( + IsLoopVar, bindingDecl(forDecomposition(IsLoopVar)))))))), + Stmt, Context) + .empty(); +} + bool ForRangeCopyCheck::handleCopyIsOnlyConstReferenced( const VarDecl &LoopVar, const CXXForRangeStmt &ForRange, ASTContext &Context) { @@ -113,9 +122,7 @@ bool ForRangeCopyCheck::handleCopyIsOnlyConstReferenced( // compiler warning which can't be suppressed. // Since this case is very rare, it is safe to ignore it. if (!ExprMutationAnalyzer(*ForRange.getBody(), Context).isMutated(&LoopVar) && - !utils::decl_ref_expr::allDeclRefExprs(LoopVar, *ForRange.getBody(), - Context) - .empty()) { + isReferenced(LoopVar, *ForRange.getBody(), Context)) { auto Diag = diag( LoopVar.getLocation(), "loop variable is copied but only used as const reference; consider " diff --git a/clang-tools-extra/clang-tidy/performance/NoexceptSwapCheck.cpp b/clang-tools-extra/clang-tidy/performance/NoexceptSwapCheck.cpp index 25a58af74f7e..e7cba6e54e86 100644 --- a/clang-tools-extra/clang-tidy/performance/NoexceptSwapCheck.cpp +++ b/clang-tools-extra/clang-tidy/performance/NoexceptSwapCheck.cpp @@ -39,7 +39,8 @@ void NoexceptSwapCheck::registerMatchers(MatchFinder *Finder) { .bind("type"))))), hasParameter(1, hasType(qualType(hasCanonicalType( qualType(equalsBoundNode("type"))))))); - Finder->addMatcher(functionDecl(unless(isDeleted()), hasName("swap"), + Finder->addMatcher(functionDecl(unless(isDeleted()), + hasAnyName("swap", "iter_swap"), anyOf(MethodMatcher, FunctionMatcher)) .bind(BindFuncDeclName), this); diff --git a/clang-tools-extra/clang-tidy/readability/AvoidNestedConditionalOperatorCheck.cpp b/clang-tools-extra/clang-tidy/readability/AvoidNestedConditionalOperatorCheck.cpp new file mode 100644 index 000000000000..1b62f54d5557 --- /dev/null +++ b/clang-tools-extra/clang-tidy/readability/AvoidNestedConditionalOperatorCheck.cpp @@ -0,0 +1,52 @@ +//===--- AvoidNestedConditionalOperatorCheck.cpp - clang-tidy -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "AvoidNestedConditionalOperatorCheck.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/ASTMatchers/ASTMatchers.h" +#include "clang/Basic/DiagnosticIDs.h" + +using namespace clang::ast_matchers; + +namespace clang::tidy::readability { + +void AvoidNestedConditionalOperatorCheck::registerMatchers( + MatchFinder *Finder) { + Finder->addMatcher( + conditionalOperator( + anyOf( + hasCondition(ignoringParenCasts( + conditionalOperator().bind("nested-conditional-operator"))), + hasTrueExpression(ignoringParenCasts( + conditionalOperator().bind("nested-conditional-operator"))), + hasFalseExpression(ignoringParenCasts( + conditionalOperator().bind("nested-conditional-operator"))))) + .bind("conditional-operator"), + this); +} + +void AvoidNestedConditionalOperatorCheck::check( + const MatchFinder::MatchResult &Result) { + const auto *CO = + Result.Nodes.getNodeAs("conditional-operator"); + const auto *NCO = Result.Nodes.getNodeAs( + "nested-conditional-operator"); + assert(CO); + assert(NCO); + + if (CO->getBeginLoc().isMacroID() || NCO->getBeginLoc().isMacroID()) + return; + + diag(NCO->getBeginLoc(), + "conditional operator is used as sub-expression of parent conditional " + "operator, refrain from using nested conditional operators"); + diag(CO->getBeginLoc(), "parent conditional operator here", + DiagnosticIDs::Note); +} + +} // namespace clang::tidy::readability diff --git a/clang-tools-extra/clang-tidy/readability/AvoidNestedConditionalOperatorCheck.h b/clang-tools-extra/clang-tidy/readability/AvoidNestedConditionalOperatorCheck.h new file mode 100644 index 000000000000..9010156de6ce --- /dev/null +++ b/clang-tools-extra/clang-tidy/readability/AvoidNestedConditionalOperatorCheck.h @@ -0,0 +1,33 @@ +//===--- AvoidNestedConditionalOperatorCheck.h - clang-tidy -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_AVOID_NESTED_CONDITIONAL_OPERATOR_CHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_AVOID_NESTED_CONDITIONAL_OPERATOR_CHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::readability { + +/// Identifies instances of nested conditional operators in the code. +/// +/// For the user-facing documentation see: +/// http://clang.llvm.org/extra/clang-tidy/checks/readability/avoid-nested-conditional-operator.html +class AvoidNestedConditionalOperatorCheck : public ClangTidyCheck { +public: + AvoidNestedConditionalOperatorCheck(StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context) {} + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; + std::optional getCheckTraversalKind() const override { + return TK_IgnoreUnlessSpelledInSource; + } +}; + +} // namespace clang::tidy::readability + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_AVOID_NESTED_CONDITIONAL_OPERATOR_CHECK_H diff --git a/clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.cpp b/clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.cpp new file mode 100644 index 000000000000..e3400f614fa5 --- /dev/null +++ b/clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.cpp @@ -0,0 +1,57 @@ +//===--- AvoidReturnWithVoidValueCheck.cpp - clang-tidy -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "AvoidReturnWithVoidValueCheck.h" +#include "clang/AST/Stmt.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/ASTMatchers/ASTMatchers.h" + +using namespace clang::ast_matchers; + +namespace clang::tidy::readability { + +static constexpr auto IgnoreMacrosName = "IgnoreMacros"; +static constexpr auto IgnoreMacrosDefault = true; + +static constexpr auto StrictModeName = "StrictMode"; +static constexpr auto StrictModeDefault = true; + +AvoidReturnWithVoidValueCheck::AvoidReturnWithVoidValueCheck( + StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context), + IgnoreMacros( + Options.getLocalOrGlobal(IgnoreMacrosName, IgnoreMacrosDefault)), + StrictMode(Options.getLocalOrGlobal(StrictModeName, StrictModeDefault)) {} + +void AvoidReturnWithVoidValueCheck::registerMatchers(MatchFinder *Finder) { + Finder->addMatcher( + returnStmt( + hasReturnValue(allOf(hasType(voidType()), unless(initListExpr()))), + optionally(hasParent(compoundStmt().bind("compound_parent")))) + .bind("void_return"), + this); +} + +void AvoidReturnWithVoidValueCheck::check( + const MatchFinder::MatchResult &Result) { + const auto *VoidReturn = Result.Nodes.getNodeAs("void_return"); + if (IgnoreMacros && VoidReturn->getBeginLoc().isMacroID()) + return; + if (!StrictMode && !Result.Nodes.getNodeAs("compound_parent")) + return; + diag(VoidReturn->getBeginLoc(), "return statement within a void function " + "should not have a specified return value"); +} + +void AvoidReturnWithVoidValueCheck::storeOptions( + ClangTidyOptions::OptionMap &Opts) { + Options.store(Opts, IgnoreMacrosName, IgnoreMacros); + Options.store(Opts, StrictModeName, StrictMode); +} + +} // namespace clang::tidy::readability diff --git a/clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.h b/clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.h new file mode 100644 index 000000000000..f8148db43cd9 --- /dev/null +++ b/clang-tools-extra/clang-tidy/readability/AvoidReturnWithVoidValueCheck.h @@ -0,0 +1,44 @@ +//===--- AvoidReturnWithVoidValueCheck.h - clang-tidy -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_AVOIDRETURNWITHVOIDVALUECHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_AVOIDRETURNWITHVOIDVALUECHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::readability { + +/// Finds return statements with `void` values used within functions with `void` +/// result types. +/// +/// For the user-facing documentation see: +/// http://clang.llvm.org/extra/clang-tidy/checks/readability/avoid-return-with-void-value.html +class AvoidReturnWithVoidValueCheck : public ClangTidyCheck { +public: + AvoidReturnWithVoidValueCheck(StringRef Name, ClangTidyContext *Context); + + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; + +private: + std::optional getCheckTraversalKind() const override { + return TK_IgnoreUnlessSpelledInSource; + } + bool isLanguageVersionSupported(const LangOptions &LangOpts) const override { + return LangOpts.CPlusPlus; + } + void storeOptions(ClangTidyOptions::OptionMap &Opts) override; + +private: + bool IgnoreMacros; + bool StrictMode; +}; + +} // namespace clang::tidy::readability + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_AVOIDRETURNWITHVOIDVALUECHECK_H diff --git a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt index 5452c2d48a46..a6c8cbd8eb44 100644 --- a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt +++ b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt @@ -5,6 +5,8 @@ set(LLVM_LINK_COMPONENTS add_clang_library(clangTidyReadabilityModule AvoidConstParamsInDecls.cpp + AvoidNestedConditionalOperatorCheck.cpp + AvoidReturnWithVoidValueCheck.cpp AvoidUnconditionalPreprocessorIfCheck.cpp BracesAroundStatementsCheck.cpp ConstReturnTypeCheck.cpp @@ -20,6 +22,7 @@ add_clang_library(clangTidyReadabilityModule IdentifierLengthCheck.cpp IdentifierNamingCheck.cpp ImplicitBoolConversionCheck.cpp + RedundantInlineSpecifierCheck.cpp InconsistentDeclarationParameterNameCheck.cpp IsolateDeclarationCheck.cpp MagicNumbersCheck.cpp @@ -33,6 +36,7 @@ add_clang_library(clangTidyReadabilityModule QualifiedAutoCheck.cpp ReadabilityTidyModule.cpp RedundantAccessSpecifiersCheck.cpp + RedundantCastingCheck.cpp RedundantControlFlowCheck.cpp RedundantDeclarationCheck.cpp RedundantFunctionPtrDereferenceCheck.cpp diff --git a/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp b/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp index f0357fb49eff..19307b4cdcbe 100644 --- a/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp @@ -291,10 +291,14 @@ void ContainerSizeEmptyCheck::check(const MatchFinder::MatchResult &Result) { OpCode == BinaryOperatorKind::BO_NE)) return; - // Always true, no warnings for that. - if ((OpCode == BinaryOperatorKind::BO_GE && Value == 0 && ContainerIsLHS) || - (OpCode == BinaryOperatorKind::BO_LE && Value == 0 && !ContainerIsLHS)) - return; + // Always true/false, no warnings for that. + if (Value == 0) { + if ((OpCode == BinaryOperatorKind::BO_GT && !ContainerIsLHS) || + (OpCode == BinaryOperatorKind::BO_LT && ContainerIsLHS) || + (OpCode == BinaryOperatorKind::BO_LE && !ContainerIsLHS) || + (OpCode == BinaryOperatorKind::BO_GE && ContainerIsLHS)) + return; + } // Do not warn for size > 1, 1 < size, size <= 1, 1 >= size. if (Value == 1) { @@ -306,12 +310,32 @@ void ContainerSizeEmptyCheck::check(const MatchFinder::MatchResult &Result) { return; } + // Do not warn for size < 1, 1 > size, size <= 0, 0 >= size for non signed + // types + if ((OpCode == BinaryOperatorKind::BO_GT && Value == 1 && + !ContainerIsLHS) || + (OpCode == BinaryOperatorKind::BO_LT && Value == 1 && ContainerIsLHS) || + (OpCode == BinaryOperatorKind::BO_GE && Value == 0 && + !ContainerIsLHS) || + (OpCode == BinaryOperatorKind::BO_LE && Value == 0 && ContainerIsLHS)) { + const Expr *Container = ContainerIsLHS + ? BinaryOp->getLHS()->IgnoreImpCasts() + : BinaryOp->getRHS()->IgnoreImpCasts(); + if (Container->getType() + .getCanonicalType() + .getNonReferenceType() + ->isSignedIntegerType()) + return; + } + if (OpCode == BinaryOperatorKind::BO_NE && Value == 0) Negation = true; + if ((OpCode == BinaryOperatorKind::BO_GT || OpCode == BinaryOperatorKind::BO_GE) && ContainerIsLHS) Negation = true; + if ((OpCode == BinaryOperatorKind::BO_LT || OpCode == BinaryOperatorKind::BO_LE) && !ContainerIsLHS) diff --git a/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp b/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp index f0fca30de3b3..11706ffb5b7d 100644 --- a/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "ImplicitBoolConversionCheck.h" +#include "../utils/FixItHintUtils.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" #include "clang/Lex/Lexer.h" @@ -61,31 +62,6 @@ bool isUnaryLogicalNotOperator(const Stmt *Statement) { return UnaryOperatorExpr && UnaryOperatorExpr->getOpcode() == UO_LNot; } -bool areParensNeededForOverloadedOperator(OverloadedOperatorKind OperatorKind) { - switch (OperatorKind) { - case OO_New: - case OO_Delete: // Fall-through on purpose. - case OO_Array_New: - case OO_Array_Delete: - case OO_ArrowStar: - case OO_Arrow: - case OO_Call: - case OO_Subscript: - return false; - - default: - return true; - } -} - -bool areParensNeededForStatement(const Stmt *Statement) { - if (const auto *OperatorCall = dyn_cast(Statement)) { - return areParensNeededForOverloadedOperator(OperatorCall->getOperator()); - } - - return isa(Statement) || isa(Statement); -} - void fixGenericExprCastToBool(DiagnosticBuilder &Diag, const ImplicitCastExpr *Cast, const Stmt *Parent, ASTContext &Context) { @@ -105,9 +81,10 @@ void fixGenericExprCastToBool(DiagnosticBuilder &Diag, const Expr *SubExpr = Cast->getSubExpr(); - bool NeedInnerParens = areParensNeededForStatement(SubExpr); + bool NeedInnerParens = + SubExpr != nullptr && utils::fixit::areParensNeededForStatement(*SubExpr); bool NeedOuterParens = - Parent != nullptr && areParensNeededForStatement(Parent); + Parent != nullptr && utils::fixit::areParensNeededForStatement(*Parent); std::string StartLocInsertion; @@ -274,8 +251,7 @@ void ImplicitBoolConversionCheck::registerMatchers(MatchFinder *Finder) { allOf(anyOf(hasCastKind(CK_NullToPointer), hasCastKind(CK_NullToMemberPointer)), hasSourceExpression(cxxBoolLiteral()))), - hasSourceExpression(expr(hasType(booleanType()))), - unless(ExceptionCases)); + hasSourceExpression(expr(hasType(booleanType())))); auto BoolXor = binaryOperator(hasOperatorName("^"), hasLHS(ImplicitCastFromBool), hasRHS(ImplicitCastFromBool)); @@ -315,7 +291,7 @@ void ImplicitBoolConversionCheck::registerMatchers(MatchFinder *Finder) { traverse( TK_AsIs, implicitCastExpr( - ImplicitCastFromBool, + ImplicitCastFromBool, unless(ExceptionCases), // Exclude comparisons of bools, as they are always cast to // integers in such context: // bool_expr_a == bool_expr_b @@ -365,7 +341,7 @@ void ImplicitBoolConversionCheck::handleCastToBool(const ImplicitCastExpr *Cast, return; } - auto Diag = diag(Cast->getBeginLoc(), "implicit conversion %0 -> bool") + auto Diag = diag(Cast->getBeginLoc(), "implicit conversion %0 -> 'bool'") << Cast->getSubExpr()->getType(); StringRef EquivalentLiteral = @@ -382,7 +358,7 @@ void ImplicitBoolConversionCheck::handleCastFromBool( ASTContext &Context) { QualType DestType = NextImplicitCast ? NextImplicitCast->getType() : Cast->getType(); - auto Diag = diag(Cast->getBeginLoc(), "implicit conversion bool -> %0") + auto Diag = diag(Cast->getBeginLoc(), "implicit conversion 'bool' -> %0") << DestType; if (const auto *BoolLiteral = diff --git a/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp b/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp index b8e6e6414320..87b299bf1ef1 100644 --- a/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp +++ b/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp @@ -10,6 +10,8 @@ #include "../ClangTidyModule.h" #include "../ClangTidyModuleRegistry.h" #include "AvoidConstParamsInDecls.h" +#include "AvoidNestedConditionalOperatorCheck.h" +#include "AvoidReturnWithVoidValueCheck.h" #include "AvoidUnconditionalPreprocessorIfCheck.h" #include "BracesAroundStatementsCheck.h" #include "ConstReturnTypeCheck.h" @@ -36,9 +38,11 @@ #include "OperatorsRepresentationCheck.h" #include "QualifiedAutoCheck.h" #include "RedundantAccessSpecifiersCheck.h" +#include "RedundantCastingCheck.h" #include "RedundantControlFlowCheck.h" #include "RedundantDeclarationCheck.h" #include "RedundantFunctionPtrDereferenceCheck.h" +#include "RedundantInlineSpecifierCheck.h" #include "RedundantMemberInitCheck.h" #include "RedundantPreprocessorCheck.h" #include "RedundantSmartptrGetCheck.h" @@ -63,6 +67,10 @@ class ReadabilityModule : public ClangTidyModule { void addCheckFactories(ClangTidyCheckFactories &CheckFactories) override { CheckFactories.registerCheck( "readability-avoid-const-params-in-decls"); + CheckFactories.registerCheck( + "readability-avoid-nested-conditional-operator"); + CheckFactories.registerCheck( + "readability-avoid-return-with-void-value"); CheckFactories.registerCheck( "readability-avoid-unconditional-preprocessor-if"); CheckFactories.registerCheck( @@ -93,6 +101,8 @@ class ReadabilityModule : public ClangTidyModule { "readability-identifier-naming"); CheckFactories.registerCheck( "readability-implicit-bool-conversion"); + CheckFactories.registerCheck( + "readability-redundant-inline-specifier"); CheckFactories.registerCheck( "readability-inconsistent-declaration-parameter-name"); CheckFactories.registerCheck( @@ -111,6 +121,8 @@ class ReadabilityModule : public ClangTidyModule { "readability-qualified-auto"); CheckFactories.registerCheck( "readability-redundant-access-specifiers"); + CheckFactories.registerCheck( + "readability-redundant-casting"); CheckFactories.registerCheck( "readability-redundant-function-ptr-dereference"); CheckFactories.registerCheck( diff --git a/clang-tools-extra/clang-tidy/readability/RedundantCastingCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantCastingCheck.cpp new file mode 100644 index 000000000000..b9ff0e81cbc5 --- /dev/null +++ b/clang-tools-extra/clang-tidy/readability/RedundantCastingCheck.cpp @@ -0,0 +1,252 @@ +//===--- RedundantCastingCheck.cpp - clang-tidy ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "RedundantCastingCheck.h" +#include "../utils/FixItHintUtils.h" +#include "clang/AST/ASTContext.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/Lex/Lexer.h" + +using namespace clang::ast_matchers; + +namespace clang::tidy::readability { + +static bool areTypesEqual(QualType S, QualType D) { + if (S == D) + return true; + + const auto *TS = S->getAs(); + const auto *TD = D->getAs(); + if (TS != TD) + return false; + + QualType PtrS = S->getPointeeType(); + QualType PtrD = D->getPointeeType(); + + if (!PtrS.isNull() && !PtrD.isNull()) + return areTypesEqual(PtrS, PtrD); + + const DeducedType *DT = S->getContainedDeducedType(); + if (DT && DT->isDeduced()) + return D == DT->getDeducedType(); + + return false; +} + +static bool areTypesEqual(QualType TypeS, QualType TypeD, + bool IgnoreTypeAliases) { + const QualType CTypeS = TypeS.getCanonicalType(); + const QualType CTypeD = TypeD.getCanonicalType(); + if (CTypeS != CTypeD) + return false; + + return IgnoreTypeAliases || areTypesEqual(TypeS.getLocalUnqualifiedType(), + TypeD.getLocalUnqualifiedType()); +} + +static bool areBinaryOperatorOperandsTypesEqualToOperatorResultType( + const Expr *E, bool IgnoreTypeAliases) { + if (!E) + return true; + const Expr *WithoutImplicitAndParen = E->IgnoreParenImpCasts(); + if (!WithoutImplicitAndParen) + return true; + if (const auto *B = dyn_cast(WithoutImplicitAndParen)) { + const QualType Type = WithoutImplicitAndParen->getType(); + if (Type.isNull()) + return true; + + const QualType NonReferenceType = Type.getNonReferenceType(); + const QualType LHSType = B->getLHS()->IgnoreImplicit()->getType(); + if (LHSType.isNull() || !areTypesEqual(LHSType.getNonReferenceType(), + NonReferenceType, IgnoreTypeAliases)) + return false; + const QualType RHSType = B->getRHS()->IgnoreImplicit()->getType(); + if (RHSType.isNull() || !areTypesEqual(RHSType.getNonReferenceType(), + NonReferenceType, IgnoreTypeAliases)) + return false; + } + return true; +} + +static const Decl *getSourceExprDecl(const Expr *SourceExpr) { + const Expr *CleanSourceExpr = SourceExpr->IgnoreParenImpCasts(); + if (const auto *E = dyn_cast(CleanSourceExpr)) { + return E->getDecl(); + } + + if (const auto *E = dyn_cast(CleanSourceExpr)) { + return E->getCalleeDecl(); + } + + if (const auto *E = dyn_cast(CleanSourceExpr)) { + return E->getMemberDecl(); + } + return nullptr; +} + +RedundantCastingCheck::RedundantCastingCheck(StringRef Name, + ClangTidyContext *Context) + : ClangTidyCheck(Name, Context), + IgnoreMacros(Options.getLocalOrGlobal("IgnoreMacros", true)), + IgnoreTypeAliases(Options.getLocalOrGlobal("IgnoreTypeAliases", false)) {} + +void RedundantCastingCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { + Options.store(Opts, "IgnoreMacros", IgnoreMacros); + Options.store(Opts, "IgnoreTypeAliases", IgnoreTypeAliases); +} + +void RedundantCastingCheck::registerMatchers(MatchFinder *Finder) { + auto SimpleType = qualType(hasCanonicalType( + qualType(anyOf(builtinType(), references(builtinType()), + references(pointsTo(qualType())), pointsTo(qualType()))))); + + auto BitfieldMemberExpr = memberExpr(member(fieldDecl(isBitField()))); + + Finder->addMatcher( + explicitCastExpr( + unless(hasCastKind(CK_ConstructorConversion)), + unless(hasCastKind(CK_UserDefinedConversion)), + unless(cxxFunctionalCastExpr(hasDestinationType(unless(SimpleType)))), + + hasDestinationType(qualType().bind("dstType")), + hasSourceExpression(anyOf( + expr(unless(initListExpr()), unless(BitfieldMemberExpr), + hasType(qualType().bind("srcType"))) + .bind("source"), + initListExpr(unless(hasInit(1, expr())), + hasInit(0, expr(unless(BitfieldMemberExpr), + hasType(qualType().bind("srcType"))) + .bind("source")))))) + .bind("cast"), + this); +} + +void RedundantCastingCheck::check(const MatchFinder::MatchResult &Result) { + const auto *SourceExpr = Result.Nodes.getNodeAs("source"); + auto TypeD = *Result.Nodes.getNodeAs("dstType"); + + if (SourceExpr->getValueKind() == VK_LValue && + TypeD.getCanonicalType()->isRValueReferenceType()) + return; + + const auto TypeS = + Result.Nodes.getNodeAs("srcType")->getNonReferenceType(); + TypeD = TypeD.getNonReferenceType(); + + if (!areTypesEqual(TypeS, TypeD, IgnoreTypeAliases)) + return; + + if (!areBinaryOperatorOperandsTypesEqualToOperatorResultType( + SourceExpr, IgnoreTypeAliases)) + return; + + const auto *CastExpr = Result.Nodes.getNodeAs("cast"); + if (IgnoreMacros && + (CastExpr->getBeginLoc().isMacroID() || + CastExpr->getEndLoc().isMacroID() || CastExpr->getExprLoc().isMacroID())) + return; + + { + auto Diag = diag(CastExpr->getExprLoc(), + "redundant explicit casting to the same type %0 as the " + "sub-expression, remove this casting"); + Diag << TypeD; + + const SourceManager &SM = *Result.SourceManager; + const SourceLocation SourceExprBegin = + SM.getExpansionLoc(SourceExpr->getBeginLoc()); + const SourceLocation SourceExprEnd = + SM.getExpansionLoc(SourceExpr->getEndLoc()); + + if (SourceExprBegin != CastExpr->getBeginLoc()) + Diag << FixItHint::CreateRemoval(SourceRange( + CastExpr->getBeginLoc(), SourceExprBegin.getLocWithOffset(-1))); + + const SourceLocation NextToken = Lexer::getLocForEndOfToken( + SourceExprEnd, 0U, SM, Result.Context->getLangOpts()); + + if (SourceExprEnd != CastExpr->getEndLoc()) { + Diag << FixItHint::CreateRemoval( + SourceRange(NextToken, CastExpr->getEndLoc())); + } + + if (utils::fixit::areParensNeededForStatement(*SourceExpr)) { + Diag << FixItHint::CreateInsertion(SourceExprBegin, "(") + << FixItHint::CreateInsertion(NextToken, ")"); + } + } + + const auto *SourceExprDecl = getSourceExprDecl(SourceExpr); + if (!SourceExprDecl) + return; + + if (const auto *D = dyn_cast(SourceExprDecl)) { + diag(D->getLocation(), + "source type originates from the invocation of this constructor", + DiagnosticIDs::Note); + return; + } + + if (const auto *D = dyn_cast(SourceExprDecl)) { + diag(D->getLocation(), + "source type originates from the invocation of this " + "%select{function|method}0", + DiagnosticIDs::Note) + << isa(D) << D->getReturnTypeSourceRange(); + return; + } + + if (const auto *D = dyn_cast(SourceExprDecl)) { + diag(D->getLocation(), + "source type originates from referencing this member", + DiagnosticIDs::Note) + << SourceRange(D->getTypeSpecStartLoc(), D->getTypeSpecEndLoc()); + return; + } + + if (const auto *D = dyn_cast(SourceExprDecl)) { + diag(D->getLocation(), + "source type originates from referencing this parameter", + DiagnosticIDs::Note) + << SourceRange(D->getTypeSpecStartLoc(), D->getTypeSpecEndLoc()); + return; + } + + if (const auto *D = dyn_cast(SourceExprDecl)) { + diag(D->getLocation(), + "source type originates from referencing this variable", + DiagnosticIDs::Note) + << SourceRange(D->getTypeSpecStartLoc(), D->getTypeSpecEndLoc()); + return; + } + + if (const auto *D = dyn_cast(SourceExprDecl)) { + diag(D->getLocation(), + "source type originates from referencing this enum constant", + DiagnosticIDs::Note); + return; + } + + if (const auto *D = dyn_cast(SourceExprDecl)) { + diag(D->getLocation(), + "source type originates from referencing this bound variable", + DiagnosticIDs::Note); + return; + } + + if (const auto *D = dyn_cast(SourceExprDecl)) { + diag(D->getLocation(), + "source type originates from referencing this non-type template " + "parameter", + DiagnosticIDs::Note); + return; + } +} + +} // namespace clang::tidy::readability diff --git a/clang-tools-extra/clang-tidy/readability/RedundantCastingCheck.h b/clang-tools-extra/clang-tidy/readability/RedundantCastingCheck.h new file mode 100644 index 000000000000..fdcfede05d43 --- /dev/null +++ b/clang-tools-extra/clang-tidy/readability/RedundantCastingCheck.h @@ -0,0 +1,38 @@ +//===--- RedundantCastingCheck.h - clang-tidy -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REDUNDANTCASTINGCHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REDUNDANTCASTINGCHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::readability { + +/// Detects explicit type casting operations that involve the same source and +/// destination types, and subsequently recommend their removal. +/// +/// For the user-facing documentation see: +/// http://clang.llvm.org/extra/clang-tidy/checks/readability/redundant-casting.html +class RedundantCastingCheck : public ClangTidyCheck { +public: + RedundantCastingCheck(StringRef Name, ClangTidyContext *Context); + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; + void storeOptions(ClangTidyOptions::OptionMap &Opts) override; + std::optional getCheckTraversalKind() const override { + return TK_IgnoreUnlessSpelledInSource; + } + +private: + const bool IgnoreMacros; + const bool IgnoreTypeAliases; +}; + +} // namespace clang::tidy::readability + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REDUNDANTCASTINGCHECK_H diff --git a/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp new file mode 100644 index 000000000000..0e8d17d44424 --- /dev/null +++ b/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp @@ -0,0 +1,132 @@ +//===--- RedundantInlineSpecifierCheck.cpp - clang-tidy--------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "RedundantInlineSpecifierCheck.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/Decl.h" +#include "clang/AST/DeclCXX.h" +#include "clang/AST/DeclTemplate.h" +#include "clang/AST/ExprCXX.h" +#include "clang/ASTMatchers/ASTMatchers.h" +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/SourceLocation.h" +#include "clang/Basic/SourceManager.h" +#include "clang/Lex/Token.h" + +#include "../utils/LexerUtils.h" + +using namespace clang::ast_matchers; + +namespace clang::tidy::readability { + +namespace { +AST_POLYMORPHIC_MATCHER(isInlineSpecified, + AST_POLYMORPHIC_SUPPORTED_TYPES(FunctionDecl, + VarDecl)) { + if (const auto *FD = dyn_cast(&Node)) + return FD->isInlineSpecified(); + if (const auto *VD = dyn_cast(&Node)) + return VD->isInlineSpecified(); + llvm_unreachable("Not a valid polymorphic type"); +} + +AST_POLYMORPHIC_MATCHER_P(isInternalLinkage, + AST_POLYMORPHIC_SUPPORTED_TYPES(FunctionDecl, + VarDecl), + bool, strictMode) { + if (!strictMode) + return false; + if (const auto *FD = dyn_cast(&Node)) + return FD->getStorageClass() == SC_Static || FD->isInAnonymousNamespace(); + if (const auto *VD = dyn_cast(&Node)) + return VD->isInAnonymousNamespace(); + llvm_unreachable("Not a valid polymorphic type"); +} +} // namespace + +static SourceLocation getInlineTokenLocation(SourceRange RangeLocation, + const SourceManager &Sources, + const LangOptions &LangOpts) { + SourceLocation Loc = RangeLocation.getBegin(); + if (Loc.isMacroID()) + return {}; + + Token FirstToken; + Lexer::getRawToken(Loc, FirstToken, Sources, LangOpts, true); + std::optional CurrentToken = FirstToken; + while (CurrentToken && CurrentToken->getLocation() < RangeLocation.getEnd() && + CurrentToken->isNot(tok::eof)) { + if (CurrentToken->is(tok::raw_identifier) && + CurrentToken->getRawIdentifier() == "inline") + return CurrentToken->getLocation(); + + CurrentToken = + Lexer::findNextToken(CurrentToken->getLocation(), Sources, LangOpts); + } + return {}; +} + +void RedundantInlineSpecifierCheck::registerMatchers(MatchFinder *Finder) { + Finder->addMatcher( + functionDecl(isInlineSpecified(), + anyOf(isConstexpr(), isDeleted(), isDefaulted(), + isInternalLinkage(StrictMode), + allOf(isDefinition(), hasAncestor(recordDecl())))) + .bind("fun_decl"), + this); + + if (StrictMode) + Finder->addMatcher( + functionTemplateDecl( + has(functionDecl(allOf(isInlineSpecified(), isDefinition())))) + .bind("templ_decl"), + this); + + if (getLangOpts().CPlusPlus17) { + Finder->addMatcher( + varDecl(isInlineSpecified(), + anyOf(isInternalLinkage(StrictMode), + allOf(isConstexpr(), hasAncestor(recordDecl())))) + .bind("var_decl"), + this); + } +} + +template +void RedundantInlineSpecifierCheck::handleMatchedDecl( + const T *MatchedDecl, const SourceManager &Sources, + const MatchFinder::MatchResult &Result, StringRef Message) { + SourceLocation Loc = getInlineTokenLocation( + MatchedDecl->getSourceRange(), Sources, Result.Context->getLangOpts()); + if (Loc.isValid()) + diag(Loc, Message) << MatchedDecl << FixItHint::CreateRemoval(Loc); +} + +void RedundantInlineSpecifierCheck::check( + const MatchFinder::MatchResult &Result) { + const SourceManager &Sources = *Result.SourceManager; + + if (const auto *MatchedDecl = + Result.Nodes.getNodeAs("fun_decl")) { + handleMatchedDecl( + MatchedDecl, Sources, Result, + "function %0 has inline specifier but is implicitly inlined"); + } else if (const auto *MatchedDecl = + Result.Nodes.getNodeAs("var_decl")) { + handleMatchedDecl( + MatchedDecl, Sources, Result, + "variable %0 has inline specifier but is implicitly inlined"); + } else if (const auto *MatchedDecl = + Result.Nodes.getNodeAs("templ_decl")) { + handleMatchedDecl( + MatchedDecl, Sources, Result, + "function %0 has inline specifier but is implicitly inlined"); + } +} + +} // namespace clang::tidy::readability diff --git a/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.h b/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.h new file mode 100644 index 000000000000..cc0bfa9685d0 --- /dev/null +++ b/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.h @@ -0,0 +1,42 @@ +//===--- RedundantInlineSpecifierCheck.h - clang-tidy ------------*-C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REDUNDANTINLINESPECIFIERCHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REDUNDANTINLINESPECIFIERCHECK_H + +#include "../ClangTidyCheck.h" + +namespace clang::tidy::readability { + +/// Detects redundant ``inline`` specifiers on function and variable +/// declarations. +/// +/// For the user-facing documentation see: +/// http://clang.llvm.org/extra/clang-tidy/checks/readability/redundant-inline-specifier.html +class RedundantInlineSpecifierCheck : public ClangTidyCheck { +public: + RedundantInlineSpecifierCheck(StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context), + StrictMode(Options.getLocalOrGlobal("StrictMode", false)) {} + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; + std::optional getCheckTraversalKind() const override { + return TK_IgnoreUnlessSpelledInSource; + } + +private: + template + void handleMatchedDecl(const T *MatchedDecl, const SourceManager &Sources, + const ast_matchers::MatchFinder::MatchResult &Result, + StringRef Message); + const bool StrictMode; +}; + +} // namespace clang::tidy::readability + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_REDUNDANTINLINESPECIFIERCHECK_H diff --git a/clang-tools-extra/clang-tidy/readability/RedundantMemberInitCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantMemberInitCheck.cpp index b5d407773bb7..015347ee9294 100644 --- a/clang-tools-extra/clang-tidy/readability/RedundantMemberInitCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/RedundantMemberInitCheck.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "RedundantMemberInitCheck.h" +#include "../utils/LexerUtils.h" #include "../utils/Matchers.h" #include "clang/AST/ASTContext.h" #include "clang/ASTMatchers/ASTMatchFinder.h" @@ -18,33 +19,64 @@ using namespace clang::tidy::matchers; namespace clang::tidy::readability { +static SourceRange +getFullInitRangeInclWhitespaces(SourceRange Range, const SourceManager &SM, + const LangOptions &LangOpts) { + const Token PrevToken = + utils::lexer::getPreviousToken(Range.getBegin(), SM, LangOpts, false); + if (PrevToken.is(tok::unknown)) + return Range; + + if (PrevToken.isNot(tok::equal)) + return {PrevToken.getEndLoc(), Range.getEnd()}; + + return getFullInitRangeInclWhitespaces( + {PrevToken.getLocation(), Range.getEnd()}, SM, LangOpts); +} + void RedundantMemberInitCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { Options.store(Opts, "IgnoreBaseInCopyConstructors", IgnoreBaseInCopyConstructors); } void RedundantMemberInitCheck::registerMatchers(MatchFinder *Finder) { + auto ConstructorMatcher = + cxxConstructExpr(argumentCountIs(0), + hasDeclaration(cxxConstructorDecl(ofClass(cxxRecordDecl( + unless(isTriviallyDefaultConstructible())))))) + .bind("construct"); + Finder->addMatcher( cxxConstructorDecl( unless(isDelegatingConstructor()), ofClass(unless(isUnion())), forEachConstructorInitializer( - cxxCtorInitializer( - withInitializer( - cxxConstructExpr( - hasDeclaration( - cxxConstructorDecl(ofClass(cxxRecordDecl( - unless(isTriviallyDefaultConstructible())))))) - .bind("construct")), - unless(forField(hasType(isConstQualified()))), - unless(forField(hasParent(recordDecl(isUnion()))))) + cxxCtorInitializer(withInitializer(ConstructorMatcher), + unless(forField(fieldDecl( + anyOf(hasType(isConstQualified()), + hasParent(recordDecl(isUnion()))))))) .bind("init"))) .bind("constructor"), this); + + Finder->addMatcher(fieldDecl(hasInClassInitializer(ConstructorMatcher), + unless(hasParent(recordDecl(isUnion())))) + .bind("field"), + this); } void RedundantMemberInitCheck::check(const MatchFinder::MatchResult &Result) { - const auto *Init = Result.Nodes.getNodeAs("init"); const auto *Construct = Result.Nodes.getNodeAs("construct"); + + if (const auto *Field = Result.Nodes.getNodeAs("field")) { + const Expr *Init = Field->getInClassInitializer(); + diag(Construct->getExprLoc(), "initializer for member %0 is redundant") + << Field + << FixItHint::CreateRemoval(getFullInitRangeInclWhitespaces( + Init->getSourceRange(), *Result.SourceManager, getLangOpts())); + return; + } + + const auto *Init = Result.Nodes.getNodeAs("init"); const auto *ConstructorDecl = Result.Nodes.getNodeAs("constructor"); @@ -52,18 +84,15 @@ void RedundantMemberInitCheck::check(const MatchFinder::MatchResult &Result) { Init->isBaseInitializer()) return; - if (Construct->getNumArgs() == 0 || - Construct->getArg(0)->isDefaultArgument()) { - if (Init->isAnyMemberInitializer()) { - diag(Init->getSourceLocation(), "initializer for member %0 is redundant") - << Init->getAnyMember() - << FixItHint::CreateRemoval(Init->getSourceRange()); - } else { - diag(Init->getSourceLocation(), - "initializer for base class %0 is redundant") - << Construct->getType() - << FixItHint::CreateRemoval(Init->getSourceRange()); - } + if (Init->isAnyMemberInitializer()) { + diag(Init->getSourceLocation(), "initializer for member %0 is redundant") + << Init->getAnyMember() + << FixItHint::CreateRemoval(Init->getSourceRange()); + } else { + diag(Init->getSourceLocation(), + "initializer for base class %0 is redundant") + << Construct->getType() + << FixItHint::CreateRemoval(Init->getSourceRange()); } } diff --git a/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.cpp b/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.cpp index 26d9287f0704..edb67614bd55 100644 --- a/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.cpp @@ -277,7 +277,13 @@ class SimplifyBooleanExprCheck::Visitor : public RecursiveASTVisitor { } bool dataTraverseStmtPre(Stmt *S) { - if (S && !shouldIgnore(S)) + if (!S) { + return true; + } + if (Check->IgnoreMacros && S->getBeginLoc().isMacroID()) { + return false; + } + if (!shouldIgnore(S)) StmtStack.push_back(S); return true; } @@ -583,6 +589,7 @@ class SimplifyBooleanExprCheck::Visitor : public RecursiveASTVisitor { SimplifyBooleanExprCheck::SimplifyBooleanExprCheck(StringRef Name, ClangTidyContext *Context) : ClangTidyCheck(Name, Context), + IgnoreMacros(Options.get("IgnoreMacros", false)), ChainedConditionalReturn(Options.get("ChainedConditionalReturn", false)), ChainedConditionalAssignment( Options.get("ChainedConditionalAssignment", false)), @@ -671,6 +678,7 @@ void SimplifyBooleanExprCheck::reportBinOp(const ASTContext &Context, } void SimplifyBooleanExprCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { + Options.store(Opts, "IgnoreMacros", IgnoreMacros); Options.store(Opts, "ChainedConditionalReturn", ChainedConditionalReturn); Options.store(Opts, "ChainedConditionalAssignment", ChainedConditionalAssignment); diff --git a/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.h b/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.h index c4dad24ec399..ccc6f3d879fc 100644 --- a/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.h +++ b/clang-tools-extra/clang-tidy/readability/SimplifyBooleanExprCheck.h @@ -64,6 +64,7 @@ class SimplifyBooleanExprCheck : public ClangTidyCheck { StringRef Description, SourceRange ReplacementRange, StringRef Replacement); + const bool IgnoreMacros; const bool ChainedConditionalReturn; const bool ChainedConditionalAssignment; const bool SimplifyDeMorgan; diff --git a/clang-tools-extra/clang-tidy/utils/FixItHintUtils.cpp b/clang-tools-extra/clang-tidy/utils/FixItHintUtils.cpp index 226dd60b5bf5..bbdd4326b0ba 100644 --- a/clang-tools-extra/clang-tidy/utils/FixItHintUtils.cpp +++ b/clang-tools-extra/clang-tidy/utils/FixItHintUtils.cpp @@ -224,6 +224,40 @@ std::optional addQualifierToVarDecl(const VarDecl &Var, return std::nullopt; } +bool areParensNeededForStatement(const Stmt &Node) { + if (isa(&Node)) + return false; + + if (isa(&Node) || isa(&Node)) + return true; + + if (isa(&Node) || + isa(&Node)) + return true; + + if (const auto *Op = dyn_cast(&Node)) { + switch (Op->getOperator()) { + case OO_PlusPlus: + [[fallthrough]]; + case OO_MinusMinus: + return Op->getNumArgs() != 2; + case OO_Call: + [[fallthrough]]; + case OO_Subscript: + [[fallthrough]]; + case OO_Arrow: + return false; + default: + return true; + }; + } + + if (isa(&Node)) + return true; + + return false; +} + // Return true if expr needs to be put in parens when it is an argument of a // prefix unary operator, e.g. when it is a binary or ternary operator // syntactically. diff --git a/clang-tools-extra/clang-tidy/utils/FixItHintUtils.h b/clang-tools-extra/clang-tidy/utils/FixItHintUtils.h index 15894c4bf5cf..2b96b2b2ce60 100644 --- a/clang-tools-extra/clang-tidy/utils/FixItHintUtils.h +++ b/clang-tools-extra/clang-tidy/utils/FixItHintUtils.h @@ -47,6 +47,11 @@ addQualifierToVarDecl(const VarDecl &Var, const ASTContext &Context, // \brief Format a pointer to an expression std::string formatDereference(const Expr &ExprNode, const ASTContext &Context); + +// \brief Checks whatever a expression require extra () to be always used in +// safe way in any other expression. +bool areParensNeededForStatement(const Stmt &Node); + } // namespace clang::tidy::utils::fixit #endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_UTILS_FIXITHINTUTILS_H diff --git a/clang-tools-extra/clang-tidy/utils/HeaderGuard.cpp b/clang-tools-extra/clang-tidy/utils/HeaderGuard.cpp index 80580bc9888a..3540c496515b 100644 --- a/clang-tools-extra/clang-tidy/utils/HeaderGuard.cpp +++ b/clang-tools-extra/clang-tidy/utils/HeaderGuard.cpp @@ -281,7 +281,7 @@ void HeaderGuardCheck::registerPPCallbacks(const SourceManager &SM, std::string HeaderGuardCheck::sanitizeHeaderGuard(StringRef Guard) { // Only reserved identifiers are allowed to start with an '_'. - return Guard.drop_while([](char C) { return C == '_'; }).str(); + return Guard.ltrim('_').str(); } bool HeaderGuardCheck::shouldSuggestEndifComment(StringRef FileName) { diff --git a/clang-tools-extra/clangd/AST.cpp b/clang-tools-extra/clangd/AST.cpp index ae79eb21de94..3e374cf2a897 100644 --- a/clang-tools-extra/clangd/AST.cpp +++ b/clang-tools-extra/clangd/AST.cpp @@ -757,7 +757,7 @@ const TemplateTypeParmType *getFunctionPackType(const FunctionDecl *Callee) { // Returns the template parameter pack type that this parameter was expanded // from (if in the Args... or Args&... or Args&&... form), if this is the case, // nullptr otherwise. -const TemplateTypeParmType *getUnderylingPackType(const ParmVarDecl *Param) { +const TemplateTypeParmType *getUnderlyingPackType(const ParmVarDecl *Param) { const auto *PlainType = Param->getType().getTypePtr(); if (auto *RT = dyn_cast(PlainType)) PlainType = RT->getPointeeTypeAsWritten().getTypePtr(); @@ -793,8 +793,8 @@ class ForwardingCallVisitor : public RecursiveASTVisitor { public: ForwardingCallVisitor(ArrayRef Parameters) - : Parameters{Parameters}, PackType{getUnderylingPackType( - Parameters.front())} {} + : Parameters{Parameters}, + PackType{getUnderlyingPackType(Parameters.front())} {} bool VisitCallExpr(CallExpr *E) { auto *Callee = getCalleeDeclOrUniqueOverload(E); @@ -859,7 +859,7 @@ class ForwardingCallVisitor if (const auto *TTPT = getFunctionPackType(Callee)) { // In this case: Separate the parameters into head, pack and tail auto IsExpandedPack = [&](const ParmVarDecl *P) { - return getUnderylingPackType(P) == TTPT; + return getUnderlyingPackType(P) == TTPT; }; ForwardingInfo FI; FI.Head = MatchingParams.take_until(IsExpandedPack); @@ -964,7 +964,7 @@ resolveForwardingParameters(const FunctionDecl *D, unsigned MaxDepth) { if (const auto *TTPT = getFunctionPackType(D)) { // Split the parameters into head, pack and tail auto IsExpandedPack = [TTPT](const ParmVarDecl *P) { - return getUnderylingPackType(P) == TTPT; + return getUnderlyingPackType(P) == TTPT; }; ArrayRef Head = Parameters.take_until(IsExpandedPack); ArrayRef Pack = @@ -1016,7 +1016,7 @@ resolveForwardingParameters(const FunctionDecl *D, unsigned MaxDepth) { } bool isExpandedFromParameterPack(const ParmVarDecl *D) { - return getUnderylingPackType(D) != nullptr; + return getUnderlyingPackType(D) != nullptr; } } // namespace clangd diff --git a/clang-tools-extra/clangd/CompileCommands.cpp b/clang-tools-extra/clangd/CompileCommands.cpp index f4e8e7e74a3b..166f17e9b3f7 100644 --- a/clang-tools-extra/clangd/CompileCommands.cpp +++ b/clang-tools-extra/clangd/CompileCommands.cpp @@ -313,13 +313,15 @@ void CommandMangler::operator()(tooling::CompileCommand &Command, tooling::addTargetAndModeForProgramName(Cmd, Cmd.front()); - // Check whether the flag exists, either as -flag or -flag=* - auto Has = [&](llvm::StringRef Flag) { - for (llvm::StringRef Arg : Cmd) { - if (Arg.consume_front(Flag) && (Arg.empty() || Arg[0] == '=')) - return true; - } - return false; + // Check whether the flag exists in the command. + auto HasExact = [&](llvm::StringRef Flag) { + return llvm::any_of(Cmd, [&](llvm::StringRef Arg) { return Arg == Flag; }); + }; + + // Check whether the flag appears in the command as a prefix. + auto HasPrefix = [&](llvm::StringRef Flag) { + return llvm::any_of( + Cmd, [&](llvm::StringRef Arg) { return Arg.starts_with(Flag); }); }; llvm::erase_if(Cmd, [](llvm::StringRef Elem) { @@ -327,12 +329,13 @@ void CommandMangler::operator()(tooling::CompileCommand &Command, }); std::vector ToAppend; - if (ResourceDir && !Has("-resource-dir")) + if (ResourceDir && !HasExact("-resource-dir") && !HasPrefix("-resource-dir=")) ToAppend.push_back(("-resource-dir=" + *ResourceDir)); // Don't set `-isysroot` if it is already set or if `--sysroot` is set. // `--sysroot` is a superset of the `-isysroot` argument. - if (Sysroot && !Has("-isysroot") && !Has("--sysroot")) { + if (Sysroot && !HasPrefix("-isysroot") && !HasExact("--sysroot") && + !HasPrefix("--sysroot=")) { ToAppend.push_back("-isysroot"); ToAppend.push_back(*Sysroot); } @@ -343,7 +346,7 @@ void CommandMangler::operator()(tooling::CompileCommand &Command, } if (!Cmd.empty()) { - bool FollowSymlink = !Has("-no-canonical-prefixes"); + bool FollowSymlink = !HasExact("-no-canonical-prefixes"); Cmd.front() = (FollowSymlink ? ResolvedDrivers : ResolvedDriversNoFollow) .get(Cmd.front(), [&, this] { diff --git a/clang-tools-extra/clangd/DumpAST.cpp b/clang-tools-extra/clangd/DumpAST.cpp index b0cec65c39fa..9a525efb938e 100644 --- a/clang-tools-extra/clangd/DumpAST.cpp +++ b/clang-tools-extra/clangd/DumpAST.cpp @@ -143,6 +143,7 @@ class DumpVisitor : public RecursiveASTVisitor { TEMPLATE_ARGUMENT_KIND(Declaration); TEMPLATE_ARGUMENT_KIND(Template); TEMPLATE_ARGUMENT_KIND(TemplateExpansion); + TEMPLATE_ARGUMENT_KIND(StructuralValue); #undef TEMPLATE_ARGUMENT_KIND } llvm_unreachable("Unhandled ArgKind enum"); diff --git a/clang-tools-extra/clangd/FindTarget.cpp b/clang-tools-extra/clangd/FindTarget.cpp index 839cf6332fe8..e702c6b3537a 100644 --- a/clang-tools-extra/clangd/FindTarget.cpp +++ b/clang-tools-extra/clangd/FindTarget.cpp @@ -443,9 +443,16 @@ struct TargetFinder { Outer.add(TST->getAliasedType(), Flags | Rel::Underlying); // Don't *traverse* the alias, which would result in traversing the // template of the underlying type. - Outer.report( - TST->getTemplateName().getAsTemplateDecl()->getTemplatedDecl(), - Flags | Rel::Alias | Rel::TemplatePattern); + + TemplateDecl *TD = TST->getTemplateName().getAsTemplateDecl(); + // Builtin templates e.g. __make_integer_seq, __type_pack_element + // are such that they don't have alias *decls*. Even then, we still + // traverse their desugared *types* so that instantiated decls are + // collected. + if (llvm::isa(TD)) + return; + Outer.report(TD->getTemplatedDecl(), + Flags | Rel::Alias | Rel::TemplatePattern); } // specializations of template template parameters aren't instantiated // into decls, so they must refer to the parameter itself. @@ -1032,6 +1039,7 @@ class ExplicitReferenceCollector case TemplateArgument::Pack: case TemplateArgument::Type: case TemplateArgument::Expression: + case TemplateArgument::StructuralValue: break; // Handled by VisitType and VisitExpression. }; return RecursiveASTVisitor::TraverseTemplateArgumentLoc(A); diff --git a/clang-tools-extra/clangd/Hover.cpp b/clang-tools-extra/clangd/Hover.cpp index 82323fe16c82..06b949bc4a2b 100644 --- a/clang-tools-extra/clangd/Hover.cpp +++ b/clang-tools-extra/clangd/Hover.cpp @@ -1194,7 +1194,7 @@ void maybeAddSymbolProviders(ParsedAST &AST, HoverInfo &HI, const SourceManager &SM = AST.getSourceManager(); llvm::SmallVector RankedProviders = - include_cleaner::headersForSymbol(Sym, SM, AST.getPragmaIncludes().get()); + include_cleaner::headersForSymbol(Sym, SM, &AST.getPragmaIncludes()); if (RankedProviders.empty()) return; @@ -1254,7 +1254,7 @@ void maybeAddUsedSymbols(ParsedAST &AST, HoverInfo &HI, const Inclusion &Inc) { llvm::DenseSet UsedSymbols; include_cleaner::walkUsed( AST.getLocalTopLevelDecls(), collectMacroReferences(AST), - AST.getPragmaIncludes().get(), AST.getPreprocessor(), + &AST.getPragmaIncludes(), AST.getPreprocessor(), [&](const include_cleaner::SymbolReference &Ref, llvm::ArrayRef Providers) { if (Ref.RT != include_cleaner::RefType::Explicit || diff --git a/clang-tools-extra/clangd/IncludeCleaner.cpp b/clang-tools-extra/clangd/IncludeCleaner.cpp index 2f34c9493492..f86a121340f7 100644 --- a/clang-tools-extra/clangd/IncludeCleaner.cpp +++ b/clang-tools-extra/clangd/IncludeCleaner.cpp @@ -48,7 +48,6 @@ #include #include #include -#include #include #include #include @@ -237,18 +236,6 @@ removeAllUnusedIncludes(llvm::ArrayRef UnusedIncludes) { Diag.Fixes.front().Edits.begin(), Diag.Fixes.front().Edits.end()); } - - // TODO(hokein): emit a suitable text for the label. - ChangeAnnotation Annotation = {/*label=*/"", - /*needsConfirmation=*/true, - /*description=*/""}; - static const ChangeAnnotationIdentifier RemoveAllUnusedID = - "RemoveAllUnusedIncludes"; - for (unsigned I = 0; I < RemoveAll.Edits.size(); ++I) { - ChangeAnnotationIdentifier ID = RemoveAllUnusedID + std::to_string(I); - RemoveAll.Edits[I].annotationId = ID; - RemoveAll.Annotations.push_back({ID, Annotation}); - } return RemoveAll; } @@ -268,20 +255,8 @@ addAllMissingIncludes(llvm::ArrayRef MissingIncludeDiags) { Edits.try_emplace(Edit.newText, Edit); } } - // FIXME(hokein): emit used symbol reference in the annotation. - ChangeAnnotation Annotation = {/*label=*/"", - /*needsConfirmation=*/true, - /*description=*/""}; - static const ChangeAnnotationIdentifier AddAllMissingID = - "AddAllMissingIncludes"; - unsigned I = 0; - for (auto &It : Edits) { - ChangeAnnotationIdentifier ID = AddAllMissingID + std::to_string(I++); + for (auto &It : Edits) AddAllMissing.Edits.push_back(std::move(It.second)); - AddAllMissing.Edits.back().annotationId = ID; - - AddAllMissing.Annotations.push_back({ID, Annotation}); - } return AddAllMissing; } Fix fixAll(const Fix &RemoveAllUnused, const Fix &AddAllMissing) { @@ -292,11 +267,6 @@ Fix fixAll(const Fix &RemoveAllUnused, const Fix &AddAllMissing) { FixAll.Edits.push_back(F); for (const auto &F : AddAllMissing.Edits) FixAll.Edits.push_back(F); - - for (const auto &A : RemoveAllUnused.Annotations) - FixAll.Annotations.push_back(A); - for (const auto &A : AddAllMissing.Annotations) - FixAll.Annotations.push_back(A); return FixAll; } @@ -311,7 +281,7 @@ getUnused(ParsedAST &AST, auto IncludeID = static_cast(*MFI.HeaderID); if (ReferencedFiles.contains(IncludeID)) continue; - if (!mayConsiderUnused(MFI, AST, AST.getPragmaIncludes().get())) { + if (!mayConsiderUnused(MFI, AST, &AST.getPragmaIncludes())) { dlog("{0} was not used, but is not eligible to be diagnosed as unused", MFI.Written); continue; @@ -403,7 +373,7 @@ IncludeCleanerFindings computeIncludeCleanerFindings(ParsedAST &AST) { .getBuiltinDir(); include_cleaner::walkUsed( AST.getLocalTopLevelDecls(), /*MacroRefs=*/Macros, - AST.getPragmaIncludes().get(), AST.getPreprocessor(), + &AST.getPragmaIncludes(), AST.getPreprocessor(), [&](const include_cleaner::SymbolReference &Ref, llvm::ArrayRef Providers) { bool Satisfied = false; diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp index d91ce7283ece..14a91797f4d2 100644 --- a/clang-tools-extra/clangd/ParsedAST.cpp +++ b/clang-tools-extra/clangd/ParsedAST.cpp @@ -653,6 +653,7 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs, } IncludeStructure Includes; + include_cleaner::PragmaIncludes PI; // If we are using a preamble, copy existing includes. if (Preamble) { Includes = Preamble->Includes; @@ -660,11 +661,15 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs, // Replay the preamble includes so that clang-tidy checks can see them. ReplayPreamble::attach(Patch->preambleIncludes(), *Clang, Patch->modifiedBounds()); + PI = *Preamble->Pragmas; } // Important: collectIncludeStructure is registered *after* ReplayPreamble! // Otherwise we would collect the replayed includes again... // (We can't *just* use the replayed includes, they don't have Resolved path). Includes.collect(*Clang); + // Same for pragma-includes, we're already inheriting preamble includes, so we + // should only receive callbacks for non-preamble mainfile includes. + PI.record(*Clang); // Copy over the macros in the preamble region of the main file, and combine // with non-preamble macros below. MainFileMacros Macros; @@ -735,7 +740,7 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs, ParsedAST Result(Filename, Inputs.Version, std::move(Preamble), std::move(Clang), std::move(Action), std::move(Tokens), std::move(Macros), std::move(Marks), std::move(ParsedDecls), - std::move(Diags), std::move(Includes)); + std::move(Diags), std::move(Includes), std::move(PI)); llvm::move(getIncludeCleanerDiags(Result, Inputs.Contents), std::back_inserter(Result.Diags)); return std::move(Result); @@ -828,23 +833,21 @@ ParsedAST::ParsedAST(PathRef TUPath, llvm::StringRef Version, syntax::TokenBuffer Tokens, MainFileMacros Macros, std::vector Marks, std::vector LocalTopLevelDecls, - std::vector Diags, IncludeStructure Includes) + std::vector Diags, IncludeStructure Includes, + include_cleaner::PragmaIncludes PI) : TUPath(TUPath), Version(Version), Preamble(std::move(Preamble)), Clang(std::move(Clang)), Action(std::move(Action)), Tokens(std::move(Tokens)), Macros(std::move(Macros)), Marks(std::move(Marks)), Diags(std::move(Diags)), LocalTopLevelDecls(std::move(LocalTopLevelDecls)), - Includes(std::move(Includes)) { - Resolver = std::make_unique(getASTContext()); + Includes(std::move(Includes)), PI(std::move(PI)), + Resolver(std::make_unique(getASTContext())) { assert(this->Clang); assert(this->Action); } -std::shared_ptr -ParsedAST::getPragmaIncludes() const { - if (!Preamble) - return nullptr; - return Preamble->Pragmas; +const include_cleaner::PragmaIncludes &ParsedAST::getPragmaIncludes() const { + return PI; } std::optional ParsedAST::preambleVersion() const { diff --git a/clang-tools-extra/clangd/ParsedAST.h b/clang-tools-extra/clangd/ParsedAST.h index c68fdba6bd26..63e564bd68a7 100644 --- a/clang-tools-extra/clangd/ParsedAST.h +++ b/clang-tools-extra/clangd/ParsedAST.h @@ -103,10 +103,8 @@ class ParsedAST { /// Tokens recorded while parsing the main file. /// (!) does not have tokens from the preamble. const syntax::TokenBuffer &getTokens() const { return Tokens; } - /// Returns the PramaIncludes from the preamble. - /// Might be null if AST is built without a preamble. - std::shared_ptr - getPragmaIncludes() const; + /// Returns the PramaIncludes for preamble + main file includes. + const include_cleaner::PragmaIncludes &getPragmaIncludes() const; /// Returns the version of the ParseInputs this AST was built from. llvm::StringRef version() const { return Version; } @@ -129,7 +127,7 @@ class ParsedAST { std::unique_ptr Action, syntax::TokenBuffer Tokens, MainFileMacros Macros, std::vector Marks, std::vector LocalTopLevelDecls, std::vector Diags, - IncludeStructure Includes); + IncludeStructure Includes, include_cleaner::PragmaIncludes PI); Path TUPath; std::string Version; // In-memory preambles must outlive the AST, it is important that this member @@ -159,6 +157,7 @@ class ParsedAST { // top-level decls from the preamble. std::vector LocalTopLevelDecls; IncludeStructure Includes; + include_cleaner::PragmaIncludes PI; std::unique_ptr Resolver; }; diff --git a/clang-tools-extra/clangd/Selection.cpp b/clang-tools-extra/clangd/Selection.cpp index 8c6d5750ecef..277cb8769a1b 100644 --- a/clang-tools-extra/clangd/Selection.cpp +++ b/clang-tools-extra/clangd/Selection.cpp @@ -1113,6 +1113,9 @@ const DeclContext &SelectionTree::Node::getDeclContext() const { return *DC; return *Current->getLexicalDeclContext(); } + if (const auto *LE = CurrentNode->ASTNode.get()) + if (CurrentNode != this) + return *LE->getCallOperator(); } llvm_unreachable("A tree must always be rooted at TranslationUnitDecl."); } diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp index 37939d36425a..ee3772e3d380 100644 --- a/clang-tools-extra/clangd/SemanticHighlighting.cpp +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -265,7 +265,7 @@ bool isStatic(const Decl *D) { bool isAbstract(const Decl *D) { if (const auto *CMD = llvm::dyn_cast(D)) - return CMD->isPure(); + return CMD->isPureVirtual(); if (const auto *CRD = llvm::dyn_cast(D)) return CRD->hasDefinition() && CRD->isAbstract(); return false; diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp index f55d2a562395..250d571eea19 100644 --- a/clang-tools-extra/clangd/XRefs.cpp +++ b/clang-tools-extra/clangd/XRefs.cpp @@ -427,7 +427,7 @@ locateASTReferent(SourceLocation CurLoc, const syntax::Token *TouchedIdentifier, // Special case: virtual void ^method() = 0: jump to all overrides. // FIXME: extend it to ^virtual, unfortunately, virtual location is not // saved in the AST. - if (CMD->isPure()) { + if (CMD->isPureVirtual()) { if (TouchedIdentifier && SM.getSpellingLoc(CMD->getLocation()) == TouchedIdentifier->location()) { VirtualMethods.insert(getSymbolID(CMD)); @@ -1339,7 +1339,7 @@ maybeFindIncludeReferences(ParsedAST &AST, Position Pos, auto Converted = convertIncludes(AST); include_cleaner::walkUsed( AST.getLocalTopLevelDecls(), collectMacroReferences(AST), - AST.getPragmaIncludes().get(), AST.getPreprocessor(), + &AST.getPragmaIncludes(), AST.getPreprocessor(), [&](const include_cleaner::SymbolReference &Ref, llvm::ArrayRef Providers) { if (Ref.RT != include_cleaner::RefType::Explicit || diff --git a/clang-tools-extra/clangd/index/FileIndex.cpp b/clang-tools-extra/clangd/index/FileIndex.cpp index 5052c661cfc5..eb9562d2b6bf 100644 --- a/clang-tools-extra/clangd/index/FileIndex.cpp +++ b/clang-tools-extra/clangd/index/FileIndex.cpp @@ -223,7 +223,7 @@ FileShardedIndex::getShard(llvm::StringRef Uri) const { SlabTuple indexMainDecls(ParsedAST &AST) { return indexSymbols( AST.getASTContext(), AST.getPreprocessor(), AST.getLocalTopLevelDecls(), - &AST.getMacros(), *AST.getPragmaIncludes(), + &AST.getMacros(), AST.getPragmaIncludes(), /*IsIndexMainAST=*/true, AST.version(), /*CollectMainFileRefs=*/true); } diff --git a/clang-tools-extra/clangd/index/SymbolCollector.cpp b/clang-tools-extra/clangd/index/SymbolCollector.cpp index 7ef4b15febad..bf838e53f2a2 100644 --- a/clang-tools-extra/clangd/index/SymbolCollector.cpp +++ b/clang-tools-extra/clangd/index/SymbolCollector.cpp @@ -821,7 +821,8 @@ void SymbolCollector::setIncludeLocation(const Symbol &S, SourceLocation DefLoc, // Use the expansion location to get the #include header since this is // where the symbol is exposed. - IncludeFiles[S.ID] = SM.getDecomposedExpansionLoc(DefLoc).first; + if (FileID FID = SM.getDecomposedExpansionLoc(DefLoc).first; FID.isValid()) + IncludeFiles[S.ID] = FID; // We update providers for a symbol with each occurence, as SymbolCollector // might run while parsing, rather than at the end of a translation unit. @@ -879,16 +880,15 @@ void SymbolCollector::finish() { const Symbol *S = Symbols.find(SID); if (!S) continue; - assert(IncludeFiles.contains(SID)); - const auto FID = IncludeFiles.at(SID); + FileID FID = IncludeFiles.lookup(SID); // Determine if the FID is #include'd or #import'ed. Symbol::IncludeDirective Directives = Symbol::Invalid; auto CollectDirectives = shouldCollectIncludePath(S->SymInfo.Kind); if ((CollectDirectives & Symbol::Include) != 0) Directives |= Symbol::Include; // Only allow #import for symbols from ObjC-like files. - if ((CollectDirectives & Symbol::Import) != 0) { + if ((CollectDirectives & Symbol::Import) != 0 && FID.isValid()) { auto [It, Inserted] = FileToContainsImportsOrObjC.try_emplace(FID); if (Inserted) It->second = FilesWithObjCConstructs.contains(FID) || diff --git a/clang-tools-extra/clangd/test/GH75115.test b/clang-tools-extra/clangd/test/GH75115.test new file mode 100644 index 000000000000..dc86f5b9a6ce --- /dev/null +++ b/clang-tools-extra/clangd/test/GH75115.test @@ -0,0 +1,15 @@ +// RUN: rm -rf %t.dir && mkdir -p %t.dir +// RUN: echo '[{"directory": "%/t.dir", "command": "clang --target=x86_64-pc-windows-msvc -x c GH75115.test", "file": "GH75115.test"}]' > %t.dir/compile_commands.json +// RUN: clangd -enable-config=0 --compile-commands-dir=%t.dir -check=%s 2>&1 | FileCheck -strict-whitespace %s + +// CHECK: Building preamble... +// CHECK-NEXT: Built preamble +// CHECK-NEXT: Indexing headers... +// CHECK-NEXT: Building AST... +// CHECK-NEXT: Indexing AST... +// CHECK-NEXT: Building inlay hints +// CHECK-NEXT: semantic highlighting +// CHECK-NEXT: Testing features at each token +// CHECK-NEXT: All checks completed, 0 errors + +#define assert diff --git a/clang-tools-extra/clangd/test/include-cleaner-batch-fix.test b/clang-tools-extra/clangd/test/include-cleaner-batch-fix.test index af7cdba5b05f..07ebe1009a78 100644 --- a/clang-tools-extra/clangd/test/include-cleaner-batch-fix.test +++ b/clang-tools-extra/clangd/test/include-cleaner-batch-fix.test @@ -157,21 +157,10 @@ # CHECK-NEXT: { # CHECK-NEXT: "arguments": [ # CHECK-NEXT: { -# CHECK-NEXT: "changeAnnotations": { -# CHECK-NEXT: "AddAllMissingIncludes0": { -# CHECK-NEXT: "label": "", -# CHECK-NEXT: "needsConfirmation": true -# CHECK-NEXT: }, -# CHECK-NEXT: "AddAllMissingIncludes1": { -# CHECK-NEXT: "label": "", -# CHECK-NEXT: "needsConfirmation": true -# CHECK-NEXT: } -# CHECK-NEXT: }, # CHECK-NEXT: "documentChanges": [ # CHECK-NEXT: { # CHECK-NEXT: "edits": [ # CHECK-NEXT: { -# CHECK-NEXT: "annotationId": "AddAllMissingIncludes0", # CHECK-NEXT: "newText": "#include {{.*}}bar.h{{.*}}", # CHECK-NEXT: "range": { # CHECK-NEXT: "end": { @@ -185,7 +174,6 @@ # CHECK-NEXT: } # CHECK-NEXT: }, # CHECK-NEXT: { -# CHECK-NEXT: "annotationId": "AddAllMissingIncludes1", # CHECK-NEXT: "newText": "#include {{.*}}foo.h{{.*}}", # CHECK-NEXT: "range": { # CHECK-NEXT: "end": { @@ -213,29 +201,10 @@ # CHECK-NEXT: { # CHECK-NEXT: "arguments": [ # CHECK-NEXT: { -# CHECK-NEXT: "changeAnnotations": { -# CHECK-NEXT: "AddAllMissingIncludes0": { -# CHECK-NEXT: "label": "", -# CHECK-NEXT: "needsConfirmation": true -# CHECK-NEXT: }, -# CHECK-NEXT: "AddAllMissingIncludes1": { -# CHECK-NEXT: "label": "", -# CHECK-NEXT: "needsConfirmation": true -# CHECK-NEXT: }, -# CHECK-NEXT: "RemoveAllUnusedIncludes0": { -# CHECK-NEXT: "label": "", -# CHECK-NEXT: "needsConfirmation": true -# CHECK-NEXT: }, -# CHECK-NEXT: "RemoveAllUnusedIncludes1": { -# CHECK-NEXT: "label": "", -# CHECK-NEXT: "needsConfirmation": true -# CHECK-NEXT: } -# CHECK-NEXT: }, # CHECK-NEXT: "documentChanges": [ # CHECK-NEXT: { # CHECK-NEXT: "edits": [ # CHECK-NEXT: { -# CHECK-NEXT: "annotationId": "RemoveAllUnusedIncludes0", # CHECK-NEXT: "newText": "", # CHECK-NEXT: "range": { # CHECK-NEXT: "end": { @@ -249,7 +218,6 @@ # CHECK-NEXT: } # CHECK-NEXT: }, # CHECK-NEXT: { -# CHECK-NEXT: "annotationId": "RemoveAllUnusedIncludes1", # CHECK-NEXT: "newText": "", # CHECK-NEXT: "range": { # CHECK-NEXT: "end": { @@ -263,7 +231,6 @@ # CHECK-NEXT: } # CHECK-NEXT: }, # CHECK-NEXT: { -# CHECK-NEXT: "annotationId": "AddAllMissingIncludes0", # CHECK-NEXT: "newText": "#include {{.*}}bar.h{{.*}}", # CHECK-NEXT: "range": { # CHECK-NEXT: "end": { @@ -277,7 +244,6 @@ # CHECK-NEXT: } # CHECK-NEXT: }, # CHECK-NEXT: { -# CHECK-NEXT: "annotationId": "AddAllMissingIncludes1", # CHECK-NEXT: "newText": "#include {{.*}}foo.h{{.*}}", # CHECK-NEXT: "range": { # CHECK-NEXT: "end": { @@ -342,21 +308,10 @@ # CHECK-NEXT: { # CHECK-NEXT: "arguments": [ # CHECK-NEXT: { -# CHECK-NEXT: "changeAnnotations": { -# CHECK-NEXT: "RemoveAllUnusedIncludes0": { -# CHECK-NEXT: "label": "", -# CHECK-NEXT: "needsConfirmation": true -# CHECK-NEXT: }, -# CHECK-NEXT: "RemoveAllUnusedIncludes1": { -# CHECK-NEXT: "label": "", -# CHECK-NEXT: "needsConfirmation": true -# CHECK-NEXT: } -# CHECK-NEXT: }, # CHECK-NEXT: "documentChanges": [ # CHECK-NEXT: { # CHECK-NEXT: "edits": [ # CHECK-NEXT: { -# CHECK-NEXT: "annotationId": "RemoveAllUnusedIncludes0", # CHECK-NEXT: "newText": "", # CHECK-NEXT: "range": { # CHECK-NEXT: "end": { @@ -370,7 +325,6 @@ # CHECK-NEXT: } # CHECK-NEXT: }, # CHECK-NEXT: { -# CHECK-NEXT: "annotationId": "RemoveAllUnusedIncludes1", # CHECK-NEXT: "newText": "", # CHECK-NEXT: "range": { # CHECK-NEXT: "end": { @@ -398,29 +352,10 @@ # CHECK-NEXT: { # CHECK-NEXT: "arguments": [ # CHECK-NEXT: { -# CHECK-NEXT: "changeAnnotations": { -# CHECK-NEXT: "AddAllMissingIncludes0": { -# CHECK-NEXT: "label": "", -# CHECK-NEXT: "needsConfirmation": true -# CHECK-NEXT: }, -# CHECK-NEXT: "AddAllMissingIncludes1": { -# CHECK-NEXT: "label": "", -# CHECK-NEXT: "needsConfirmation": true -# CHECK-NEXT: }, -# CHECK-NEXT: "RemoveAllUnusedIncludes0": { -# CHECK-NEXT: "label": "", -# CHECK-NEXT: "needsConfirmation": true -# CHECK-NEXT: }, -# CHECK-NEXT: "RemoveAllUnusedIncludes1": { -# CHECK-NEXT: "label": "", -# CHECK-NEXT: "needsConfirmation": true -# CHECK-NEXT: } -# CHECK-NEXT: }, # CHECK-NEXT: "documentChanges": [ # CHECK-NEXT: { # CHECK-NEXT: "edits": [ # CHECK-NEXT: { -# CHECK-NEXT: "annotationId": "RemoveAllUnusedIncludes0", # CHECK-NEXT: "newText": "", # CHECK-NEXT: "range": { # CHECK-NEXT: "end": { @@ -434,7 +369,6 @@ # CHECK-NEXT: } # CHECK-NEXT: }, # CHECK-NEXT: { -# CHECK-NEXT: "annotationId": "RemoveAllUnusedIncludes1", # CHECK-NEXT: "newText": "", # CHECK-NEXT: "range": { # CHECK-NEXT: "end": { @@ -448,7 +382,6 @@ # CHECK-NEXT: } # CHECK-NEXT: }, # CHECK-NEXT: { -# CHECK-NEXT: "annotationId": "AddAllMissingIncludes0", # CHECK-NEXT: "newText": "#include {{.*}}bar.h{{.*}}", # CHECK-NEXT: "range": { # CHECK-NEXT: "end": { @@ -462,7 +395,6 @@ # CHECK-NEXT: } # CHECK-NEXT: }, # CHECK-NEXT: { -# CHECK-NEXT: "annotationId": "AddAllMissingIncludes1", # CHECK-NEXT: "newText": "#include {{.*}}foo.h{{.*}}", # CHECK-NEXT: "range": { # CHECK-NEXT: "end": { diff --git a/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp b/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp index 772177b60b5e..b64dd4acad4c 100644 --- a/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp +++ b/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp @@ -370,9 +370,8 @@ TEST(ArgStripperTest, OrderDependent) { } TEST(PrintArgvTest, All) { - std::vector Args = { - "one", "two", "thr ee", "f\"o\"ur", "fi\\ve", "$" - }; + std::vector Args = {"one", "two", "thr ee", + "f\"o\"ur", "fi\\ve", "$"}; const char *Expected = R"(one two "thr ee" "f\"o\"ur" "fi\\ve" $)"; EXPECT_EQ(Expected, printArgv(Args)); } @@ -450,6 +449,82 @@ TEST(CommandMangler, PathsAsPositional) { Mangler(Cmd, "a.cc"); EXPECT_THAT(Cmd.CommandLine, Contains("foo")); } + +TEST(CommandMangler, RespectsOriginalResourceDir) { + auto Mangler = CommandMangler::forTests(); + Mangler.ResourceDir = testPath("fake/resources"); + + { + tooling::CompileCommand Cmd; + Cmd.CommandLine = {"clang++", "-resource-dir", testPath("true/resources"), + "foo.cc"}; + Mangler(Cmd, "foo.cc"); + EXPECT_THAT(llvm::join(Cmd.CommandLine, " "), + HasSubstr("-resource-dir " + testPath("true/resources"))); + EXPECT_THAT(llvm::join(Cmd.CommandLine, " "), + Not(HasSubstr(testPath("fake/resources")))); + } + + { + tooling::CompileCommand Cmd; + Cmd.CommandLine = {"clang++", "-resource-dir=" + testPath("true/resources"), + "foo.cc"}; + Mangler(Cmd, "foo.cc"); + EXPECT_THAT(llvm::join(Cmd.CommandLine, " "), + HasSubstr("-resource-dir=" + testPath("true/resources"))); + EXPECT_THAT(llvm::join(Cmd.CommandLine, " "), + Not(HasSubstr(testPath("fake/resources")))); + } +} + +TEST(CommandMangler, RespectsOriginalSysroot) { + auto Mangler = CommandMangler::forTests(); + Mangler.Sysroot = testPath("fake/sysroot"); + + { + tooling::CompileCommand Cmd; + Cmd.CommandLine = {"clang++", "-isysroot", testPath("true/sysroot"), + "foo.cc"}; + Mangler(Cmd, "foo.cc"); + EXPECT_THAT(llvm::join(Cmd.CommandLine, " "), + HasSubstr("-isysroot " + testPath("true/sysroot"))); + EXPECT_THAT(llvm::join(Cmd.CommandLine, " "), + Not(HasSubstr(testPath("fake/sysroot")))); + } + + { + tooling::CompileCommand Cmd; + Cmd.CommandLine = {"clang++", "-isysroot" + testPath("true/sysroot"), + "foo.cc"}; + Mangler(Cmd, "foo.cc"); + EXPECT_THAT(llvm::join(Cmd.CommandLine, " "), + HasSubstr("-isysroot" + testPath("true/sysroot"))); + EXPECT_THAT(llvm::join(Cmd.CommandLine, " "), + Not(HasSubstr(testPath("fake/sysroot")))); + } + + { + tooling::CompileCommand Cmd; + Cmd.CommandLine = {"clang++", "--sysroot", testPath("true/sysroot"), + "foo.cc"}; + Mangler(Cmd, "foo.cc"); + EXPECT_THAT(llvm::join(Cmd.CommandLine, " "), + HasSubstr("--sysroot " + testPath("true/sysroot"))); + EXPECT_THAT(llvm::join(Cmd.CommandLine, " "), + Not(HasSubstr(testPath("fake/sysroot")))); + } + + { + tooling::CompileCommand Cmd; + Cmd.CommandLine = {"clang++", "--sysroot=" + testPath("true/sysroot"), + "foo.cc"}; + Mangler(Cmd, "foo.cc"); + EXPECT_THAT(llvm::join(Cmd.CommandLine, " "), + HasSubstr("--sysroot=" + testPath("true/sysroot"))); + EXPECT_THAT(llvm::join(Cmd.CommandLine, " "), + Not(HasSubstr(testPath("fake/sysroot")))); + } +} } // namespace } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp index 37643e5afa23..f302dcf5f09d 100644 --- a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp +++ b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp @@ -420,6 +420,62 @@ TEST(DiagnosticTest, MakeUnique) { "no matching constructor for initialization of 'S'"))); } +TEST(DiagnosticTest, CoroutineInHeader) { + StringRef CoroutineH = R"cpp( +namespace std { +template +struct coroutine_traits { using promise_type = typename Ret::promise_type; }; + +template +struct coroutine_handle { + static coroutine_handle from_address(void *) noexcept; + static coroutine_handle from_promise(Promise &promise); + constexpr void* address() const noexcept; +}; +template <> +struct coroutine_handle { + template + coroutine_handle(coroutine_handle) noexcept; + static coroutine_handle from_address(void *); + constexpr void* address() const noexcept; +}; + +struct awaitable { + bool await_ready() noexcept { return false; } + void await_suspend(coroutine_handle<>) noexcept {} + void await_resume() noexcept {} +}; +} // namespace std + )cpp"; + + StringRef Header = R"cpp( +#include "coroutine.h" +template struct [[clang::coro_return_type]] Gen { + struct promise_type { + Gen get_return_object() { + return {}; + } + std::awaitable initial_suspend(); + std::awaitable final_suspend() noexcept; + void unhandled_exception(); + void return_value(T t); + }; +}; + +Gen foo_coro(int b) { co_return b; } + )cpp"; + Annotations Main(R"cpp( +// error-ok +#include "header.hpp" +Gen $[[bar_coro]](int b) { return foo_coro(b); } + )cpp"); + TestTU TU = TestTU::withCode(Main.code()); + TU.AdditionalFiles["coroutine.h"] = std::string(CoroutineH); + TU.AdditionalFiles["header.hpp"] = std::string(Header); + TU.ExtraArgs.push_back("--std=c++20"); + EXPECT_THAT(TU.build().getDiagnostics(), ElementsAre(hasRange(Main.range()))); +} + TEST(DiagnosticTest, MakeShared) { // We usually miss diagnostics from header functions as we don't parse them. // std::make_shared is only parsed when --parse-forwarding-functions is set diff --git a/clang-tools-extra/clangd/unittests/DumpASTTests.cpp b/clang-tools-extra/clangd/unittests/DumpASTTests.cpp index d1b8f21b82c6..304682118c87 100644 --- a/clang-tools-extra/clangd/unittests/DumpASTTests.cpp +++ b/clang-tools-extra/clangd/unittests/DumpASTTests.cpp @@ -186,6 +186,17 @@ TEST(DumpASTTests, Arcana) { EXPECT_THAT(Node.children.front().arcana, testing::StartsWith("QualType ")); } +TEST(DumpASTTests, UnbalancedBraces) { + // Test that we don't crash while trying to compute a source range for the + // node whose ending brace is missing, and also that the source range is + // not empty. + Annotations Case("/*error-ok*/ $func[[int main() {]]"); + ParsedAST AST = TestTU::withCode(Case.code()).build(); + auto Node = dumpAST(DynTypedNode::create(findDecl(AST, "main")), + AST.getTokens(), AST.getASTContext()); + ASSERT_EQ(Node.range, Case.range("func")); +} + } // namespace } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp index cf30b388d234..9f713564b2c0 100644 --- a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp +++ b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp @@ -176,7 +176,7 @@ void update(FileIndex &M, llvm::StringRef Basename, llvm::StringRef Code) { auto AST = File.build(); M.updatePreamble(testPath(File.Filename), /*Version=*/"null", AST.getASTContext(), AST.getPreprocessor(), - *AST.getPragmaIncludes()); + AST.getPragmaIncludes()); } TEST(FileIndexTest, CustomizedURIScheme) { @@ -254,7 +254,7 @@ TEST(FileIndexTest, IWYUPragmaExport) { auto AST = File.build(); M.updatePreamble(testPath(File.Filename), /*Version=*/"null", AST.getASTContext(), AST.getPreprocessor(), - *AST.getPragmaIncludes()); + AST.getPragmaIncludes()); auto Symbols = runFuzzyFind(M, ""); EXPECT_THAT( @@ -446,7 +446,7 @@ TEST(FileIndexTest, Relations) { FileIndex Index; Index.updatePreamble(testPath(TU.Filename), /*Version=*/"null", AST.getASTContext(), AST.getPreprocessor(), - *AST.getPragmaIncludes()); + AST.getPragmaIncludes()); SymbolID A = findSymbol(TU.headerSymbols(), "A").ID; uint32_t Results = 0; RelationsRequest Req; @@ -567,7 +567,7 @@ TEST(FileIndexTest, StalePreambleSymbolsDeleted) { auto AST = File.build(); M.updatePreamble(testPath(File.Filename), /*Version=*/"null", AST.getASTContext(), AST.getPreprocessor(), - *AST.getPragmaIncludes()); + AST.getPragmaIncludes()); EXPECT_THAT(runFuzzyFind(M, ""), UnorderedElementsAre(qName("a"))); File.Filename = "f2.cpp"; @@ -575,7 +575,7 @@ TEST(FileIndexTest, StalePreambleSymbolsDeleted) { AST = File.build(); M.updatePreamble(testPath(File.Filename), /*Version=*/"null", AST.getASTContext(), AST.getPreprocessor(), - *AST.getPragmaIncludes()); + AST.getPragmaIncludes()); EXPECT_THAT(runFuzzyFind(M, ""), UnorderedElementsAre(qName("b"))); } @@ -720,7 +720,7 @@ TEST(FileIndexTest, Profile) { auto AST = TestTU::withHeaderCode("int a;").build(); FI.updateMain(FileName, AST); FI.updatePreamble(FileName, "v1", AST.getASTContext(), AST.getPreprocessor(), - *AST.getPragmaIncludes()); + AST.getPragmaIncludes()); llvm::BumpPtrAllocator Alloc; MemoryTree MT(&Alloc); diff --git a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp index fbd10c4a47a7..29cff68cf03b 100644 --- a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp +++ b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp @@ -709,6 +709,33 @@ TEST_F(TargetDeclTest, TypeAliasTemplate) { Rel::Alias | Rel::TemplatePattern}); } +TEST_F(TargetDeclTest, BuiltinTemplates) { + Code = R"cpp( + template struct integer_sequence {}; + [[__make_integer_seq]] X; + )cpp"; + EXPECT_DECLS( + "TemplateSpecializationTypeLoc", + {"struct integer_sequence", Rel::TemplatePattern | Rel::Underlying}, + {"template<> struct integer_sequence>", + Rel::TemplateInstantiation | Rel::Underlying}); + + // Dependent context. + Code = R"cpp( + template struct integer_sequence; + + template + using make_integer_sequence = [[__make_integer_seq]]; + )cpp"; + EXPECT_DECLS("TemplateSpecializationTypeLoc"); + + Code = R"cpp( + template + using type_pack_element = [[__type_pack_element]]; + )cpp"; + EXPECT_DECLS("TemplateSpecializationTypeLoc"); +} + TEST_F(TargetDeclTest, MemberOfTemplate) { Code = R"cpp( template struct Foo { diff --git a/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp b/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp index 5a6524dec2f0..7ed4a9103e1f 100644 --- a/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp +++ b/clang-tools-extra/clangd/unittests/IncludeCleanerTests.cpp @@ -316,8 +316,10 @@ TEST(IncludeCleaner, IWYUPragmas) { #include "public.h" void bar() { foo(); } + #include "keep_main_file.h" // IWYU pragma: keep )cpp"; TU.AdditionalFiles["behind_keep.h"] = guard(""); + TU.AdditionalFiles["keep_main_file.h"] = guard(""); TU.AdditionalFiles["exported.h"] = guard(""); TU.AdditionalFiles["public.h"] = guard("#include \"private.h\""); TU.AdditionalFiles["private.h"] = guard(R"cpp( diff --git a/clang-tools-extra/clangd/unittests/SelectionTests.cpp b/clang-tools-extra/clangd/unittests/SelectionTests.cpp index 4c019a1524f3..754e8c287c51 100644 --- a/clang-tools-extra/clangd/unittests/SelectionTests.cpp +++ b/clang-tools-extra/clangd/unittests/SelectionTests.cpp @@ -880,6 +880,19 @@ TEST(SelectionTest, DeclContextIsLexical) { } } +TEST(SelectionTest, DeclContextLambda) { + llvm::Annotations Test(R"cpp( + void foo(); + auto lambda = [] { + return $1^foo(); + }; + )cpp"); + auto AST = TestTU::withCode(Test.code()).build(); + auto ST = SelectionTree::createRight(AST.getASTContext(), AST.getTokens(), + Test.point("1"), Test.point("1")); + EXPECT_TRUE(ST.commonAncestor()->getDeclContext().isFunctionOrMethod()); +} + } // namespace } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/unittests/TestTU.cpp b/clang-tools-extra/clangd/unittests/TestTU.cpp index e65ae825b416..1f02c04125b1 100644 --- a/clang-tools-extra/clangd/unittests/TestTU.cpp +++ b/clang-tools-extra/clangd/unittests/TestTU.cpp @@ -164,7 +164,7 @@ SymbolSlab TestTU::headerSymbols() const { auto AST = build(); return std::get<0>(indexHeaderSymbols( /*Version=*/"null", AST.getASTContext(), AST.getPreprocessor(), - *AST.getPragmaIncludes())); + AST.getPragmaIncludes())); } RefSlab TestTU::headerRefs() const { @@ -177,7 +177,7 @@ std::unique_ptr TestTU::index() const { auto Idx = std::make_unique(); Idx->updatePreamble(testPath(Filename), /*Version=*/"null", AST.getASTContext(), AST.getPreprocessor(), - *AST.getPragmaIncludes()); + AST.getPragmaIncludes()); Idx->updateMain(testPath(Filename), AST); return std::move(Idx); } diff --git a/clang-tools-extra/clangd/unittests/tweaks/AddUsingTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/AddUsingTests.cpp index 1fd2487378d7..c2dd8e1bb8ee 100644 --- a/clang-tools-extra/clangd/unittests/tweaks/AddUsingTests.cpp +++ b/clang-tools-extra/clangd/unittests/tweaks/AddUsingTests.cpp @@ -309,6 +309,29 @@ namespace foo { void fun(); } void foo::fun() { ff(); })cpp"}, + // Inside a lambda. + { + R"cpp( +namespace NS { +void unrelated(); +void foo(); +} + +auto L = [] { + using NS::unrelated; + NS::f^oo(); +};)cpp", + R"cpp( +namespace NS { +void unrelated(); +void foo(); +} + +auto L = [] { + using NS::foo;using NS::unrelated; + foo(); +};)cpp", + }, // If all other using are fully qualified, add :: {R"cpp( #include "test.hpp" diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 571808a51596..5758b5acbc0b 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -119,7 +119,7 @@ Improvements to clang-tidy - Improved `--dump-config` to print check options in alphabetical order. -- Improved :program:`clang-tidy-diff.py` script. +- Improved :program:`clang-tidy-diff.py` script. * Return exit code `1` if any :program:`clang-tidy` subprocess exits with a non-zero code or if exporting fixes fails. @@ -143,6 +143,12 @@ New checks Detects unsafe or redundant two-step casting operations involving ``void*``. +- New :doc:`bugprone-chained-comparison + ` check. + + Check detects chained comparison operators that can lead to unintended + behavior or logical errors. + - New :doc:`bugprone-compare-pointer-to-member-virtual-function ` check. @@ -224,6 +230,28 @@ New checks Recommends the smallest possible underlying type for an ``enum`` or ``enum`` class based on the range of its enumerators. +- New :doc:`readability-avoid-nested-conditional-operator + ` check. + + Identifies instances of nested conditional operators in the code. + +- New :doc:`readability-avoid-return-with-void-value + ` check. + + Finds return statements with ``void`` values used within functions with + ``void`` result types. + +- New :doc:`readability-redundant-casting + ` check. + + Detects explicit type casting operations that involve the same source and + destination types, and subsequently recommend their removal. + +- New :doc:`readability-redundant-inline-specifier + ` check. + + Detects redundant ``inline`` specifiers on function and variable declarations. + - New :doc:`readability-reference-to-constructed-temporary ` check. @@ -254,6 +282,14 @@ Changes in existing checks casting during type conversions at variable initialization, now with improved compatibility for C++17 and later versions. +- Improved :doc:`bugprone-exception-escape + ` check by extending the default + check function names to include ``iter_swap`` and ``iter_move``. + +- Improved :doc:`bugprone-implicit-widening-of-multiplication-result + ` check + to correctly emit fixes. + - Improved :doc:`bugprone-lambda-function-name ` check by adding option `IgnoreMacros` to ignore warnings in macros. @@ -295,6 +331,10 @@ Changes in existing checks coroutine functions and increase issue detection for cases involving type aliases with references. +- Improved :doc:`cppcoreguidelines-missing-std-forward + ` check to + address false positives in the capture list and body of lambdas. + - Improved :doc:`cppcoreguidelines-narrowing-conversions ` check by extending the `IgnoreConversionFromTypes` option to include types without a @@ -303,7 +343,8 @@ Changes in existing checks - Improved :doc:`cppcoreguidelines-prefer-member-initializer ` check to ignore delegate constructors and ignore re-assignment for reference or when - initialization depend on field that is initialized before. + initialization depend on field that is initialized before. Additionally, it + now provides valid fixes for member variables initialized with macros. - Improved :doc:`cppcoreguidelines-pro-bounds-array-to-pointer-decay ` check @@ -336,6 +377,13 @@ Changes in existing checks to ignore unused parameters when they are marked as unused and parameters of deleted functions and constructors. +- Improved :doc:`google-readability-casting + ` check to ignore constructor + calls disguised as functional casts. + +- Improved :doc:`google-runtime-int ` + check to ignore false positives on user defined-literals. + - Improved :doc:`llvm-namespace-comment ` check to provide fixes for ``inline`` namespaces in the same format as :program:`clang-format`. @@ -358,7 +406,9 @@ Changes in existing checks ` check to avoid false positive when using pointer to member function. Additionally, the check no longer emits a diagnostic when a variable that is not type-dependent is an operand of a - type-dependent binary operator. + type-dependent binary operator. Improved performance of the check through + optimizations. The check no longer emits a diagnostic for non-parameter-pack + variables in C++17 fold expressions. - Improved :doc:`misc-include-cleaner ` check by adding option @@ -370,9 +420,13 @@ Changes in existing checks ` check to ignore false-positives in unevaluated context (e.g., ``decltype``). +- Improved :doc:`misc-static-assert + ` check to ignore false-positives when + referring to non-``constexpr`` variables in non-unevaluated context. + - Improved :doc:`misc-unused-using-decls ` check to avoid false positive when - using in elaborated type. + using in elaborated type and only check C++ files. - Improved :doc:`modernize-avoid-bind ` check to @@ -382,7 +436,8 @@ Changes in existing checks - Improved :doc:`modernize-loop-convert ` to support for-loops with iterators initialized by free functions like ``begin``, ``end``, or ``size`` - and avoid crash for array of dependent array. + and avoid crash for array of dependent array and non-dereferenceable builtin + types used as iterators. - Improved :doc:`modernize-make-shared ` check to support @@ -394,6 +449,14 @@ Changes in existing checks false-positives when constructing the container with ``count`` copies of elements with value ``value``. +- Improved :doc:`modernize-use-auto + ` to avoid create incorrect fix hints + for pointer to array type and pointer to function type. + +- Improved :doc:`modernize-use-emplace + ` to not replace aggregates that + ``emplace`` cannot construct with aggregate initialization. + - Improved :doc:`modernize-use-equals-delete ` check to ignore false-positives when special member function is actually used or implicit. @@ -408,38 +471,45 @@ Changes in existing checks - Improved :doc:`modernize-use-using ` check to fix function pointer and - forward declared ``typedef`` correctly. Added option `IgnoreExternC` to ignore ``typedef`` - declaration in ``extern "C"`` scope. + forward declared ``typedef`` correctly. Added option `IgnoreExternC` to ignore + ``typedef`` declaration in ``extern "C"`` scope. - Improved :doc:`performance-faster-string-find ` check to properly escape single quotes. +- Improved :doc:`performance-for-range-copy + ` check to handle cases where + the loop variable is a structured binding. + - Improved :doc:`performance-noexcept-move-constructor ` to better handle - conditional noexcept expressions, eliminating false-positives. + conditional ``noexcept`` expressions, eliminating false-positives. - Improved :doc:`performance-noexcept-swap ` check to enforce a stricter match with the swap function signature and better handling of condition - noexcept expressions, eliminating false-positives. + ``noexcept`` expressions, eliminating false-positives. ``iter_swap`` function + name is checked by default. - Improved :doc:`readability-braces-around-statements ` check to ignore false-positive for ``if constexpr`` in lambda expression. - Improved :doc:`readability-avoid-const-params-in-decls - ` diagnositics to - highlight the const location + ` diagnostics to + highlight the ``const`` location - Improved :doc:`readability-container-contains ` to correctly handle - interger literals with suffixes in fix-its. + integer literals with suffixes in fix-its. - Improved :doc:`readability-container-size-empty ` check to detect comparison between string and empty string literals and support - ``length()`` method as an alternative to ``size()``. + ``length()`` method as an alternative to ``size()``. Resolved false positives + tied to negative values from size-like methods, and one triggered by size + checks below zero. - Improved :doc:`readability-function-size ` check configuration to use @@ -457,13 +527,14 @@ Changes in existing checks ``camel_Snake_Case`` now detect more invalid identifier names. Fields in anonymous records (i.e. anonymous structs and unions) now can be checked with the naming rules associated with their enclosing scopes rather than the naming - rules of public struct/union members. + rules of public ``struct``/``union`` members. - Improved :doc:`readability-implicit-bool-conversion ` check to take do-while loops into account for the `AllowIntegerConditions` and `AllowPointerConditions` options. It also now provides more consistent - suggestions when parentheses are added to the return value. + suggestions when parentheses are added to the return value or expressions. + It also ignores false-positives for comparison containing bool bitfield. - Improved :doc:`readability-misleading-indentation ` check to ignore @@ -473,6 +544,15 @@ Changes in existing checks ` check to ignore false-positives in initializer list of record. +- Improved :doc:`readability-redundant-member-init + ` check to now also + detect redundant in-class initializers. + +- Improved :doc:`readability-simplify-boolean-expr + ` check by adding the + new option `IgnoreMacros` that allows to ignore boolean expressions originating + from expanded macros. + - Improved :doc:`readability-simplify-subscript-expr ` check by extending the default value of the `Types` option to include ``std::span``. diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/chained-comparison.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/chained-comparison.rst new file mode 100644 index 000000000000..45b069a8e29d --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/chained-comparison.rst @@ -0,0 +1,73 @@ +.. title:: clang-tidy - bugprone-chained-comparison + +bugprone-chained-comparison +=========================== + +Check detects chained comparison operators that can lead to unintended +behavior or logical errors. + +Chained comparisons are expressions that use multiple comparison operators +to compare three or more values. For example, the expression ``a < b < c`` +compares the values of ``a``, ``b``, and ``c``. However, this expression does +not evaluate as ``(a < b) && (b < c)``, which is probably what the developer +intended. Instead, it evaluates as ``(a < b) < c``, which may produce +unintended results, especially when the types of ``a``, ``b``, and ``c`` are +different. + +To avoid such errors, the check will issue a warning when a chained +comparison operator is detected, suggesting to use parentheses to specify +the order of evaluation or to use a logical operator to separate comparison +expressions. + +Consider the following examples: + +.. code-block:: c++ + + int a = 2, b = 6, c = 4; + if (a < b < c) { + // This block will be executed + } + + +In this example, the developer intended to check if ``a`` is less than ``b`` +and ``b`` is less than ``c``. However, the expression ``a < b < c`` is +equivalent to ``(a < b) < c``. Since ``a < b`` is ``true``, the expression +``(a < b) < c`` is evaluated as ``1 < c``, which is equivalent to ``true < c`` +and is invalid in this case as ``b < c`` is ``false``. + +Even that above issue could be detected as comparison of ``int`` to ``bool``, +there is more dangerous example: + +.. code-block:: c++ + + bool a = false, b = false, c = true; + if (a == b == c) { + // This block will be executed + } + +In this example, the developer intended to check if ``a``, ``b``, and ``c`` are +all equal. However, the expression ``a == b == c`` is evaluated as +``(a == b) == c``. Since ``a == b`` is true, the expression ``(a == b) == c`` +is evaluated as ``true == c``, which is equivalent to ``true == true``. +This comparison yields ``true``, even though ``a`` and ``b`` are ``false``, and +are not equal to ``c``. + +To avoid this issue, the developer can use a logical operator to separate the +comparison expressions, like this: + +.. code-block:: c++ + + if (a == b && b == c) { + // This block will not be executed + } + + +Alternatively, use of parentheses in the comparison expressions can make the +developer's intention more explicit and help avoid misunderstanding. + +.. code-block:: c++ + + if ((a == b) == c) { + // This block will be executed + } + diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/exception-escape.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/exception-escape.rst index e6aa8e001492..182fade7f47a 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/exception-escape.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/exception-escape.rst @@ -11,6 +11,8 @@ should not. The functions which should not throw exceptions are the following: * Move assignment operators * The ``main()`` functions * ``swap()`` functions +* ``iter_swap()`` functions +* ``iter_move()`` functions * Functions marked with ``throw()`` or ``noexcept`` * Other functions given as option diff --git a/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/optin.core.EnumCastOutOfRange.rst b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/optin.core.EnumCastOutOfRange.rst new file mode 100644 index 000000000000..99dd1adf38f6 --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/optin.core.EnumCastOutOfRange.rst @@ -0,0 +1,13 @@ +.. title:: clang-tidy - clang-analyzer-optin.core.EnumCastOutOfRange +.. meta:: + :http-equiv=refresh: 5;URL=https://clang.llvm.org/docs/analyzer/checkers.html#optin-core-enumcastoutofrange + +clang-analyzer-optin.core.EnumCastOutOfRange +============================================ + +Check integer to enumeration casts for out of range values. + +The `clang-analyzer-optin.core.EnumCastOutOfRange` check is an alias, please see +`Clang Static Analyzer Available Checkers +`_ +for more information. diff --git a/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/security.cert.env.InvalidPtr.rst b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/security.cert.env.InvalidPtr.rst new file mode 100644 index 000000000000..8986fa0e684f --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/security.cert.env.InvalidPtr.rst @@ -0,0 +1,13 @@ +.. title:: clang-tidy - clang-analyzer-security.cert.env.InvalidPtr +.. meta:: + :http-equiv=refresh: 5;URL=https://clang.llvm.org/docs/analyzer/checkers.html#security-cert-env-invalidptr + +clang-analyzer-security.cert.env.InvalidPtr +=========================================== + +Finds usages of possibly invalidated pointers. + +The `clang-analyzer-security.cert.env.InvalidPtr` check is an alias, please see +`Clang Static Analyzer Available Checkers +`_ +for more information. diff --git a/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/unix.Errno.rst b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/unix.Errno.rst new file mode 100644 index 000000000000..67a2d05811dc --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/unix.Errno.rst @@ -0,0 +1,13 @@ +.. title:: clang-tidy - clang-analyzer-unix.Errno +.. meta:: + :http-equiv=refresh: 5;URL=https://clang.llvm.org/docs/analyzer/checkers.html#unix-errno + +clang-analyzer-unix.Errno +========================= + +Check for improper use of 'errno'. + +The `clang-analyzer-unix.Errno` check is an alias, please see +`Clang Static Analyzer Available Checkers +`_ +for more information. diff --git a/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/unix.StdCLibraryFunctions.rst b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/unix.StdCLibraryFunctions.rst new file mode 100644 index 000000000000..17906732c86f --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/clang-analyzer/unix.StdCLibraryFunctions.rst @@ -0,0 +1,14 @@ +.. title:: clang-tidy - clang-analyzer-unix.StdCLibraryFunctions +.. meta:: + :http-equiv=refresh: 5;URL=https://clang.llvm.org/docs/analyzer/checkers.html#unix-stdclibraryfunctions + +clang-analyzer-unix.StdCLibraryFunctions +======================================== + +Check for invalid arguments of C standard library functions, and apply relations +between arguments and return value. + +The `clang-analyzer-unix.StdCLibraryFunctions` check is an alias, please see +`Clang Static Analyzer Available Checkers +`_ +for more information. diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst index 39d8b490d927..f773e80b562e 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/list.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst @@ -83,6 +83,7 @@ Clang-Tidy Checks :doc:`bugprone-bool-pointer-implicit-conversion `, "Yes" :doc:`bugprone-branch-clone `, :doc:`bugprone-casting-through-void `, + :doc:`bugprone-chained-comparison `, :doc:`bugprone-compare-pointer-to-member-virtual-function `, :doc:`bugprone-copy-constructor-init `, "Yes" :doc:`bugprone-dangling-handle `, @@ -337,6 +338,8 @@ Clang-Tidy Checks :doc:`portability-simd-intrinsics `, :doc:`portability-std-allocator-const `, :doc:`readability-avoid-const-params-in-decls `, "Yes" + :doc:`readability-avoid-nested-conditional-operator `, + :doc:`readability-avoid-return-with-void-value `, :doc:`readability-avoid-unconditional-preprocessor-if `, :doc:`readability-braces-around-statements `, "Yes" :doc:`readability-const-return-type `, "Yes" @@ -363,9 +366,11 @@ Clang-Tidy Checks :doc:`readability-operators-representation `, "Yes" :doc:`readability-qualified-auto `, "Yes" :doc:`readability-redundant-access-specifiers `, "Yes" + :doc:`readability-redundant-casting `, "Yes" :doc:`readability-redundant-control-flow `, "Yes" :doc:`readability-redundant-declaration `, "Yes" :doc:`readability-redundant-function-ptr-dereference `, "Yes" + :doc:`readability-redundant-inline-specifier `, "Yes" :doc:`readability-redundant-member-init `, "Yes" :doc:`readability-redundant-preprocessor `, :doc:`readability-redundant-smartptr-get `, "Yes" @@ -440,6 +445,7 @@ Clang-Tidy Checks :doc:`clang-analyzer-nullability.NullableDereferenced `, `Clang Static Analyzer nullability.NullableDereferenced `_, :doc:`clang-analyzer-nullability.NullablePassedToNonnull `, `Clang Static Analyzer nullability.NullablePassedToNonnull `_, :doc:`clang-analyzer-nullability.NullableReturnedFromNonnull `, `Clang Static Analyzer nullability.NullableReturnedFromNonnull `_, + :doc:`clang-analyzer-optin.core.EnumCastOutOfRange `, `Clang Static Analyzer optin.core.EnumCastOutOfRange `_, :doc:`clang-analyzer-optin.cplusplus.UninitializedObject `, `Clang Static Analyzer optin.cplusplus.UninitializedObject `_, :doc:`clang-analyzer-optin.cplusplus.VirtualCall `, `Clang Static Analyzer optin.cplusplus.VirtualCall `_, :doc:`clang-analyzer-optin.mpi.MPI-Checker `, `Clang Static Analyzer optin.mpi.MPI-Checker `_, @@ -479,6 +485,7 @@ Clang-Tidy Checks :doc:`clang-analyzer-osx.coreFoundation.containers.OutOfBounds `, `Clang Static Analyzer osx.coreFoundation.containers.OutOfBounds `_, :doc:`clang-analyzer-osx.coreFoundation.containers.PointerSizedValues `, `Clang Static Analyzer osx.coreFoundation.containers.PointerSizedValues `_, :doc:`clang-analyzer-security.FloatLoopCounter `, `Clang Static Analyzer security.FloatLoopCounter `_, + :doc:`clang-analyzer-security.cert.env.InvalidPtr `, `Clang Static Analyzer security.cert.env.InvalidPtr `_, :doc:`clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling `, `Clang Static Analyzer security.insecureAPI.DeprecatedOrUnsafeBufferHandling `_, :doc:`clang-analyzer-security.insecureAPI.UncheckedReturn `, `Clang Static Analyzer security.insecureAPI.UncheckedReturn `_, :doc:`clang-analyzer-security.insecureAPI.bcmp `, `Clang Static Analyzer security.insecureAPI.bcmp `_, @@ -493,9 +500,11 @@ Clang-Tidy Checks :doc:`clang-analyzer-security.insecureAPI.strcpy `, `Clang Static Analyzer security.insecureAPI.strcpy `_, :doc:`clang-analyzer-security.insecureAPI.vfork `, `Clang Static Analyzer security.insecureAPI.vfork `_, :doc:`clang-analyzer-unix.API `, `Clang Static Analyzer unix.API `_, + :doc:`clang-analyzer-unix.Errno `, `Clang Static Analyzer unix.Errno `_, :doc:`clang-analyzer-unix.Malloc `, `Clang Static Analyzer unix.Malloc `_, :doc:`clang-analyzer-unix.MallocSizeof `, `Clang Static Analyzer unix.MallocSizeof `_, :doc:`clang-analyzer-unix.MismatchedDeallocator `, `Clang Static Analyzer unix.MismatchedDeallocator `_, + :doc:`clang-analyzer-unix.StdCLibraryFunctions `, `Clang Static Analyzer unix.StdCLibraryFunctions `_, :doc:`clang-analyzer-unix.Vfork `, `Clang Static Analyzer unix.Vfork `_, :doc:`clang-analyzer-unix.cstring.BadSizeArg `, `Clang Static Analyzer unix.cstring.BadSizeArg `_, :doc:`clang-analyzer-unix.cstring.NullArg `, `Clang Static Analyzer unix.cstring.NullArg `_, diff --git a/clang-tools-extra/docs/clang-tidy/checks/misc/const-correctness.rst b/clang-tools-extra/docs/clang-tidy/checks/misc/const-correctness.rst index b5a59fa691da..86fba6c7e4f7 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/misc/const-correctness.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/misc/const-correctness.rst @@ -8,7 +8,7 @@ This check implements detection of local variables which could be declared as coding guidelines, such as: `ES.25 `_ from the C++ Core Guidelines and `AUTOSAR C++14 Rule A7-1-1 (6.7.1 Specifiers) -`_. +`_. Please note that this check's analysis is type-based only. Variables that are not modified but used to create a non-const handle that might escape the scope are not diagnosed diff --git a/clang-tools-extra/docs/clang-tidy/checks/misc/unconventional-assign-operator.rst b/clang-tools-extra/docs/clang-tidy/checks/misc/unconventional-assign-operator.rst index 49e3fd5b6ee4..3b4b65a5cb68 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/misc/unconventional-assign-operator.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/misc/unconventional-assign-operator.rst @@ -13,3 +13,6 @@ types and definitions with good return type but wrong ``return`` statements. type (e.g. ``int``). * Private and deleted operators are ignored. * The operator must always return ``*this``. + +This check implements `AUTOSAR C++14 Rule A13-2-1 +`_. diff --git a/clang-tools-extra/docs/clang-tidy/checks/performance/noexcept-swap.rst b/clang-tools-extra/docs/clang-tidy/checks/performance/noexcept-swap.rst index fa4007618ba0..2901d721706c 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/performance/noexcept-swap.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/performance/noexcept-swap.rst @@ -3,11 +3,11 @@ performance-noexcept-swap ========================= -The check flags user-defined swap functions not marked with ``noexcept`` or +The check flags user-defined swap and iter_swap functions not marked with ``noexcept`` or marked with ``noexcept(expr)`` where ``expr`` evaluates to ``false`` (but is not a ``false`` literal itself). -When a swap function is marked as ``noexcept``, it assures the compiler that +When a swap or iter_swap function is marked as ``noexcept``, it assures the compiler that no exceptions will be thrown during the swapping of two objects, which allows the compiler to perform certain optimizations such as omitting exception handling code. diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/avoid-nested-conditional-operator.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/avoid-nested-conditional-operator.rst new file mode 100644 index 000000000000..44b74283292c --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/readability/avoid-nested-conditional-operator.rst @@ -0,0 +1,21 @@ +.. title:: clang-tidy - readability-avoid-nested-conditional-operator + +readability-avoid-nested-conditional-operator +============================================= + +Identifies instances of nested conditional operators in the code. + +Nested conditional operators, also known as ternary operators, can contribute +to reduced code readability and comprehension. So they should be split as +several statements and stored the intermediate results in temporary variable. + +Examples: + +.. code-block:: c++ + + int NestInConditional = (condition1 ? true1 : false1) ? true2 : false2; + int NestInTrue = condition1 ? (condition2 ? true1 : false1) : false2; + int NestInFalse = condition1 ? true1 : condition2 ? true2 : false1; + +This check implements part of `AUTOSAR C++14 Rule A5-16-1 +`_. diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/avoid-return-with-void-value.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/avoid-return-with-void-value.rst new file mode 100644 index 000000000000..d802f9be829c --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/readability/avoid-return-with-void-value.rst @@ -0,0 +1,51 @@ +.. title:: clang-tidy - readability-avoid-return-with-void-value + +readability-avoid-return-with-void-value +======================================== + +Finds return statements with ``void`` values used within functions with +``void`` result types. + +A function with a ``void`` return type is intended to perform a task without +producing a return value. Return statements with expressions could lead +to confusion and may miscommunicate the function's intended behavior. + +Example: + +.. code-block:: + + void g(); + void f() { + // ... + return g(); + } + +In a long function body, the ``return`` statement suggests that the function +returns a value. However, ``return g();`` is a combination of two statements +that should be written as + +.. code-block:: + + g(); + return; + +to make clear that ``g()`` is called and immediately afterwards the function +returns (nothing). + +In C, the same issue is detected by the compiler if the ``-Wpedantic`` mode +is enabled. + +Options +------- + +.. option:: IgnoreMacros + + The value `false` specifies that return statements expanded + from macros are not checked. The default value is `true`. + +.. option:: StrictMode + + The value `false` specifies that a direct return statement shall + be excluded from the analysis if it is the only statement not + contained in a block like ``if (cond) return g();``. The default + value is `true`. diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/redundant-casting.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/redundant-casting.rst new file mode 100644 index 000000000000..23eaa225f03a --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/readability/redundant-casting.rst @@ -0,0 +1,37 @@ +.. title:: clang-tidy - readability-redundant-casting + +readability-redundant-casting +============================= + +Detects explicit type casting operations that involve the same source and +destination types, and subsequently recommend their removal. Covers a range of +explicit casting operations, including ``static_cast``, ``const_cast``, C-style +casts, and ``reinterpret_cast``. Its primary objective is to enhance code +readability and maintainability by eliminating unnecessary type casting. + +.. code-block:: c++ + + int value = 42; + int result = static_cast(value); + +In this example, the ``static_cast(value)`` is redundant, as it performs +a cast from an ``int`` to another ``int``. + +Casting operations involving constructor conversions, user-defined conversions, +functional casts, type-dependent casts, casts between distinct type aliases that +refer to the same underlying type, as well as bitfield-related casts and casts +directly from lvalue to rvalue, are all disregarded by the check. + +Options +------- + +.. option:: IgnoreMacros + + If set to `true`, the check will not give warnings inside macros. Default + is `true`. + +.. option:: IgnoreTypeAliases + + When set to `false`, the check will consider type aliases, and when set to + `true`, it will resolve all type aliases and operate on the underlying + types. Default is `false`. diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/redundant-inline-specifier.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/redundant-inline-specifier.rst new file mode 100644 index 000000000000..eee324cddab4 --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/readability/redundant-inline-specifier.rst @@ -0,0 +1,32 @@ +.. title:: clang-tidy - readability-redundant-inline-specifier + +readability-redundant-inline-specifier +====================================== + +Detects redundant ``inline`` specifiers on function and variable declarations. + +Examples: + +.. code-block:: c++ + + constexpr inline void f() {} + +In the example above the keyword ``inline`` is redundant since constexpr +functions are implicitly inlined + +.. code-block:: c++ + + class MyClass { + inline void myMethod() {} + }; + +In the example above the keyword ``inline`` is redundant since member functions +defined entirely inside a class/struct/union definition are implicitly inlined. + +Options +------- + +.. option:: StrictMode + + If set to `true`, the check will also flag functions and variables that + already have internal linkage as redundant. diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/redundant-member-init.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/redundant-member-init.rst index b2f86c4429cc..c26aaf73b16f 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/readability/redundant-member-init.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/readability/redundant-member-init.rst @@ -11,13 +11,14 @@ Example .. code-block:: c++ - // Explicitly initializing the member s is unnecessary. + // Explicitly initializing the member s and v is unnecessary. class Foo { public: Foo() : s() {} private: std::string s; + std::vector v {}; }; Options diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/simplify-boolean-expr.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/simplify-boolean-expr.rst index 18ab84b26a25..3d00d5b043a6 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/readability/simplify-boolean-expr.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/readability/simplify-boolean-expr.rst @@ -82,6 +82,11 @@ Examples: Options ------- +.. option:: IgnoreMacros + + If `true`, ignore boolean expressions originating from expanded macros. + Default is `false`. + .. option:: ChainedConditionalReturn If `true`, conditional boolean return statements at the end of an @@ -99,8 +104,8 @@ Options .. option:: SimplifyDeMorganRelaxed - If `true`, :option:`SimplifyDeMorgan` will also transform negated - conjunctions and disjunctions where there is no negation on either operand. + If `true`, :option:`SimplifyDeMorgan` will also transform negated + conjunctions and disjunctions where there is no negation on either operand. This option has no effect if :option:`SimplifyDeMorgan` is `false`. Default is `false`. diff --git a/clang-tools-extra/include-cleaner/include/clang-include-cleaner/Record.h b/clang-tools-extra/include-cleaner/include/clang-include-cleaner/Record.h index 2e0b462ce16d..2dcb5ea2555c 100644 --- a/clang-tools-extra/include-cleaner/include/clang-include-cleaner/Record.h +++ b/clang-tools-extra/include-cleaner/include/clang-include-cleaner/Record.h @@ -113,7 +113,8 @@ class PragmaIncludes { llvm::DenseSet ShouldKeep; /// Owns the strings. - llvm::BumpPtrAllocator Arena; + /// Each record() pushes a new one, while keeping all the old strings alive. + std::vector> Arena; // FIXME: add support for clang use_instead pragma }; diff --git a/clang-tools-extra/include-cleaner/lib/Record.cpp b/clang-tools-extra/include-cleaner/lib/Record.cpp index bd726cff12a9..c93c56adf650 100644 --- a/clang-tools-extra/include-cleaner/lib/Record.cpp +++ b/clang-tools-extra/include-cleaner/lib/Record.cpp @@ -178,7 +178,8 @@ class PragmaIncludes::RecordPragma : public PPCallbacks, public CommentHandler { : RecordPragma(CI.getPreprocessor(), Out) {} RecordPragma(const Preprocessor &P, PragmaIncludes *Out) : SM(P.getSourceManager()), HeaderInfo(P.getHeaderSearchInfo()), Out(Out), - UniqueStrings(Arena) {} + Arena(std::make_shared()), + UniqueStrings(*Arena) {} void FileChanged(SourceLocation Loc, FileChangeReason Reason, SrcMgr::CharacteristicKind FileType, @@ -204,7 +205,7 @@ class PragmaIncludes::RecordPragma : public PPCallbacks, public CommentHandler { std::unique(It.getSecond().begin(), It.getSecond().end()), It.getSecond().end()); } - Out->Arena = std::move(Arena); + Out->Arena.emplace_back(std::move(Arena)); } void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok, @@ -336,7 +337,7 @@ class PragmaIncludes::RecordPragma : public PPCallbacks, public CommentHandler { const SourceManager &SM; const HeaderSearch &HeaderInfo; PragmaIncludes *Out; - llvm::BumpPtrAllocator Arena; + std::shared_ptr Arena; /// Intern table for strings. Contents are on the arena. llvm::StringSaver UniqueStrings; diff --git a/clang-tools-extra/include-cleaner/test/tool.cpp b/clang-tools-extra/include-cleaner/test/tool.cpp index bd90679875ac..de47b2a3f377 100644 --- a/clang-tools-extra/include-cleaner/test/tool.cpp +++ b/clang-tools-extra/include-cleaner/test/tool.cpp @@ -22,6 +22,10 @@ int x = foo(); // IGNORE2-NOT: - "foobar.h" // IGNORE2: + "foo.h" +// RUN: clang-include-cleaner -print=changes %s --only-headers="foo\.h" -- -I%S/Inputs/ | FileCheck --match-full-lines --allow-empty --check-prefix=ONLY %s +// ONLY-NOT: - "foobar.h" +// ONLY: + "foo.h" + // RUN: clang-include-cleaner -print %s -- -I%S/Inputs/ | FileCheck --match-full-lines --check-prefix=PRINT %s // PRINT: #include "foo.h" // PRINT-NOT: {{^}}#include "foobar.h"{{$}} diff --git a/clang-tools-extra/include-cleaner/tool/IncludeCleaner.cpp b/clang-tools-extra/include-cleaner/tool/IncludeCleaner.cpp index 30aaee29b9a3..132ad2541150 100644 --- a/clang-tools-extra/include-cleaner/tool/IncludeCleaner.cpp +++ b/clang-tools-extra/include-cleaner/tool/IncludeCleaner.cpp @@ -16,10 +16,10 @@ #include "clang/Tooling/Tooling.h" #include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Regex.h" #include "llvm/Support/Signals.h" #include "llvm/Support/raw_ostream.h" @@ -57,6 +57,14 @@ cl::opt HTMLReportPath{ cl::cat(IncludeCleaner), }; +cl::opt OnlyHeaders{ + "only-headers", + cl::desc("A comma-separated list of regexes to match against suffix of a " + "header. Only headers that match will be analyzed."), + cl::init(""), + cl::cat(IncludeCleaner), +}; + cl::opt IgnoreHeaders{ "ignore-headers", cl::desc("A comma-separated list of regexes to match against suffix of a " @@ -110,14 +118,16 @@ format::FormatStyle getStyle(llvm::StringRef Filename) { class Action : public clang::ASTFrontendAction { public: - Action(llvm::function_ref HeaderFilter) - : HeaderFilter(HeaderFilter){}; + Action(llvm::function_ref HeaderFilter, + llvm::StringMap &EditedFiles) + : HeaderFilter(HeaderFilter), EditedFiles(EditedFiles) {} private: RecordedAST AST; RecordedPP PP; PragmaIncludes PI; llvm::function_ref HeaderFilter; + llvm::StringMap &EditedFiles; bool BeginInvocation(CompilerInstance &CI) override { // We only perform include-cleaner analysis. So we disable diagnostics that @@ -181,17 +191,8 @@ class Action : public clang::ASTFrontendAction { } } - if (Edit && (!Results.Missing.empty() || !Results.Unused.empty())) { - if (auto Err = llvm::writeToOutput( - Path, [&](llvm::raw_ostream &OS) -> llvm::Error { - OS << Final; - return llvm::Error::success(); - })) { - llvm::errs() << "Failed to apply edits to " << Path << ": " - << toString(std::move(Err)) << "\n"; - ++Errors; - } - } + if (!Results.Missing.empty() || !Results.Unused.empty()) + EditedFiles.try_emplace(Path, Final); } void writeHTML() { @@ -215,18 +216,25 @@ class ActionFactory : public tooling::FrontendActionFactory { : HeaderFilter(HeaderFilter) {} std::unique_ptr create() override { - return std::make_unique(HeaderFilter); + return std::make_unique(HeaderFilter, EditedFiles); + } + + const llvm::StringMap &editedFiles() const { + return EditedFiles; } private: llvm::function_ref HeaderFilter; + // Map from file name to final code with the include edits applied. + llvm::StringMap EditedFiles; }; -std::function headerFilter() { +// Compiles a regex list into a function that return true if any match a header. +// Prints and returns nullptr if any regexes are invalid. +std::function matchesAny(llvm::StringRef RegexFlag) { auto FilterRegs = std::make_shared>(); - llvm::SmallVector Headers; - llvm::StringRef(IgnoreHeaders).split(Headers, ',', -1, /*KeepEmpty=*/false); + RegexFlag.split(Headers, ',', -1, /*KeepEmpty=*/false); for (auto HeaderPattern : Headers) { std::string AnchoredPattern = "(" + HeaderPattern.str() + ")$"; llvm::Regex CompiledRegex(AnchoredPattern); @@ -247,6 +255,21 @@ std::function headerFilter() { }; } +std::function headerFilter() { + auto OnlyMatches = matchesAny(OnlyHeaders); + auto IgnoreMatches = matchesAny(IgnoreHeaders); + if (!OnlyMatches || !IgnoreMatches) + return nullptr; + + return [OnlyMatches, IgnoreMatches](llvm::StringRef Header) { + if (OnlyHeaders.getNumOccurrences() && !OnlyMatches(Header)) + return true; + if (IgnoreHeaders.getNumOccurrences() && IgnoreMatches(Header)) + return true; + return false; + }; +} + } // namespace } // namespace include_cleaner } // namespace clang @@ -274,21 +297,26 @@ int main(int argc, const char **argv) { clang::tooling::ClangTool Tool(OptionsParser->getCompilations(), OptionsParser->getSourcePathList()); - std::vector> Buffers; - for (const auto &File : OptionsParser->getSourcePathList()) { - auto Content = llvm::MemoryBuffer::getFile(File); - if (!Content) { - llvm::errs() << "Error: can't read file '" << File - << "': " << Content.getError().message() << "\n"; - return 1; - } - Buffers.push_back(std::move(Content.get())); - Tool.mapVirtualFile(File, Buffers.back()->getBuffer()); - } auto HeaderFilter = headerFilter(); if (!HeaderFilter) return 1; // error already reported. ActionFactory Factory(HeaderFilter); - return Tool.run(&Factory) || Errors != 0; + auto ErrorCode = Tool.run(&Factory); + if (Edit) { + for (const auto &NameAndContent : Factory.editedFiles()) { + llvm::StringRef FileName = NameAndContent.first(); + const std::string &FinalCode = NameAndContent.second; + if (auto Err = llvm::writeToOutput( + FileName, [&](llvm::raw_ostream &OS) -> llvm::Error { + OS << FinalCode; + return llvm::Error::success(); + })) { + llvm::errs() << "Failed to apply edits to " << FileName << ": " + << toString(std::move(Err)) << "\n"; + ++Errors; + } + } + } + return ErrorCode || Errors != 0; } diff --git a/clang-tools-extra/include-cleaner/unittests/RecordTest.cpp b/clang-tools-extra/include-cleaner/unittests/RecordTest.cpp index 0f2ded5f1834..d1f7590b2226 100644 --- a/clang-tools-extra/include-cleaner/unittests/RecordTest.cpp +++ b/clang-tools-extra/include-cleaner/unittests/RecordTest.cpp @@ -588,5 +588,30 @@ TEST_F(PragmaIncludeTest, OutlivesFMAndSM) { EXPECT_THAT(PI.getExporters(Private2FE.get(), FM), testing::ElementsAre(llvm::cantFail(FM.getFileRef("public.h")))); } + +TEST_F(PragmaIncludeTest, CanRecordManyTimes) { + Inputs.Code = R"cpp( + #include "public.h" + )cpp"; + Inputs.ExtraFiles["public.h"] = R"cpp( + #include "private.h" + )cpp"; + Inputs.ExtraFiles["private.h"] = R"cpp( + // IWYU pragma: private, include "public.h" + )cpp"; + + TestAST Processed = build(); + auto &FM = Processed.fileManager(); + auto PrivateFE = FM.getFile("private.h"); + llvm::StringRef Public = PI.getPublic(PrivateFE.get()); + EXPECT_EQ(Public, "\"public.h\""); + + // This build populates same PI during build, but this time we don't have + // any IWYU pragmas. Make sure strings from previous recordings are still + // alive. + Inputs.Code = ""; + build(); + EXPECT_EQ(Public, "\"public.h\""); +} } // namespace } // namespace clang::include_cleaner diff --git a/clang-tools-extra/modularize/ModularizeUtilities.cpp b/clang-tools-extra/modularize/ModularizeUtilities.cpp index 25de9c9dd05a..d25bb09b8cad 100644 --- a/clang-tools-extra/modularize/ModularizeUtilities.cpp +++ b/clang-tools-extra/modularize/ModularizeUtilities.cpp @@ -195,7 +195,7 @@ std::error_code ModularizeUtilities::loadSingleHeaderListsAndDependencies( // Get canonical form. HeaderFileName = getCanonicalPath(HeaderFileName); // Save the resulting header file path and dependencies. - HeaderFileNames.push_back(std::string(HeaderFileName.str())); + HeaderFileNames.push_back(std::string(HeaderFileName)); Dependencies[HeaderFileName.str()] = Dependents; } return std::error_code(); @@ -248,7 +248,7 @@ std::error_code ModularizeUtilities::loadProblemHeaderList( // Get canonical form. HeaderFileName = getCanonicalPath(HeaderFileName); // Save the resulting header file path. - ProblemFileNames.push_back(std::string(HeaderFileName.str())); + ProblemFileNames.push_back(std::string(HeaderFileName)); } return std::error_code(); } diff --git a/clang-tools-extra/test/clang-doc/templates.cpp b/clang-tools-extra/test/clang-doc/templates.cpp index eb7f4599629f..2e04a77ac9e6 100644 --- a/clang-tools-extra/test/clang-doc/templates.cpp +++ b/clang-tools-extra/test/clang-doc/templates.cpp @@ -7,7 +7,7 @@ // RUN: rm -rf %t template -void function(T x) {} +void function(T x) {} template<> void function(bool x) {} diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/chained-comparison.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/chained-comparison.c new file mode 100644 index 000000000000..53c6139dcfbc --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/chained-comparison.c @@ -0,0 +1,76 @@ +// RUN: %check_clang_tidy %s bugprone-chained-comparison %t + +void badly_chained_1(int x, int y, int z) +{ + int result = x < y < z; +} +// CHECK-MESSAGES: :[[@LINE-2]]:18: warning: chained comparison 'v0 < v1 < v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_2(int x, int y, int z) +{ + int result = x <= y <= z; +} +// CHECK-MESSAGES: :[[@LINE-2]]:18: warning: chained comparison 'v0 <= v1 <= v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_3(int x, int y, int z) +{ + int result = x > y > z; +} +// CHECK-MESSAGES: :[[@LINE-2]]:18: warning: chained comparison 'v0 > v1 > v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_4(int x, int y, int z) +{ + int result = x >= y >= z; +} +// CHECK-MESSAGES: :[[@LINE-2]]:18: warning: chained comparison 'v0 >= v1 >= v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_5(int x, int y, int z) +{ + int result = x == y != z; +} +// CHECK-MESSAGES: :[[@LINE-2]]:18: warning: chained comparison 'v0 == v1 != v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_6(int x, int y, int z) +{ + int result = x != y == z; +} +// CHECK-MESSAGES: :[[@LINE-2]]:18: warning: chained comparison 'v0 != v1 == v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_multiple(int a, int b, int c, int d, int e, int f, int g, int h) +{ + int result = a == b == c == d == e == f == g == h; +} +// CHECK-MESSAGES: :[[@LINE-2]]:18: warning: chained comparison 'v0 == v1 == v2 == v3 == v4 == v5 == v6 == v7' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_limit(int v[29]) +{ +// CHECK-MESSAGES: :[[@LINE+1]]:18: warning: chained comparison 'v0 == v1 == v2 == v3 == v4 == v5 == v6 == v7 == v8 == v9 == v10 == v11 == v12 == v13 == v14 == v15 == v16 == v17 == v18 == v19 == v20 == v21 == v22 == v23 == v24 == v25 == v26 == v27 == v28' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + int result = v[0] == v[1] == v[2] == v[3] == v[4] == v[5] == v[6] == v[7] == + v[8] == v[9] == v[10] == v[11] == v[12] == v[13] == v[14] == + v[15] == v[16] == v[17] == v[18] == v[19] == v[20] == v[21] == + v[22] == v[23] == v[24] == v[25] == v[26] == v[27] == v[28]; + +} + +void badly_chained_parens2(int x, int y, int z, int t, int a, int b) +{ + int result = (x < y) < (z && t) > (a == b); +} +// CHECK-MESSAGES: :[[@LINE-2]]:18: warning: chained comparison 'v0 < v1 > v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_inner(int x, int y, int z, int t, int u) +{ + int result = x && y < z < t && u; +} +// CHECK-MESSAGES: :[[@LINE-2]]:23: warning: chained comparison 'v0 < v1 < v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void properly_chained_1(int x, int y, int z) +{ + int result = x < y && y < z; +} + +void properly_chained_2(int x, int y, int z) +{ + int result = (x < y) == z; +} + diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/chained-comparison.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/chained-comparison.cpp new file mode 100644 index 000000000000..8a664aa205ac --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/chained-comparison.cpp @@ -0,0 +1,91 @@ +// RUN: %check_clang_tidy -std=c++98-or-later %s bugprone-chained-comparison %t + +void badly_chained_1(int x, int y, int z) +{ + bool result = x < y < z; +} +// CHECK-MESSAGES: :[[@LINE-2]]:19: warning: chained comparison 'v0 < v1 < v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_2(int x, int y, int z) +{ + bool result = x <= y <= z; +} +// CHECK-MESSAGES: :[[@LINE-2]]:19: warning: chained comparison 'v0 <= v1 <= v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_3(int x, int y, int z) +{ + bool result = x > y > z; +} +// CHECK-MESSAGES: :[[@LINE-2]]:19: warning: chained comparison 'v0 > v1 > v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_4(int x, int y, int z) +{ + bool result = x >= y >= z; +} +// CHECK-MESSAGES: :[[@LINE-2]]:19: warning: chained comparison 'v0 >= v1 >= v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_5(int x, int y, int z) +{ + bool result = x == y != z; +} +// CHECK-MESSAGES: :[[@LINE-2]]:19: warning: chained comparison 'v0 == v1 != v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_6(bool x, bool y, bool z) +{ + bool result = x != y == z; +} +// CHECK-MESSAGES: :[[@LINE-2]]:19: warning: chained comparison 'v0 != v1 == v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_multiple(bool a, bool b, bool c, bool d, bool e, bool f, bool g, bool h) +{ + bool result = a == b == c == d == e == f == g == h; +} +// CHECK-MESSAGES: :[[@LINE-2]]:19: warning: chained comparison 'v0 == v1 == v2 == v3 == v4 == v5 == v6 == v7' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_limit(bool v[29]) +{ +// CHECK-MESSAGES: :[[@LINE+1]]:19: warning: chained comparison 'v0 == v1 == v2 == v3 == v4 == v5 == v6 == v7 == v8 == v9 == v10 == v11 == v12 == v13 == v14 == v15 == v16 == v17 == v18 == v19 == v20 == v21 == v22 == v23 == v24 == v25 == v26 == v27 == v28' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + bool result = v[0] == v[1] == v[2] == v[3] == v[4] == v[5] == v[6] == v[7] == + v[8] == v[9] == v[10] == v[11] == v[12] == v[13] == v[14] == + v[15] == v[16] == v[17] == v[18] == v[19] == v[20] == v[21] == + v[22] == v[23] == v[24] == v[25] == v[26] == v[27] == v[28]; + +} + +void badly_chained_parens2(int x, int y, int z, int t, int a, int b) +{ + bool result = (x < y) < (z && t) > (a == b); +} +// CHECK-MESSAGES: :[[@LINE-2]]:19: warning: chained comparison 'v0 < v1 > v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void badly_chained_inner(int x, int y, int z, int t, int u) +{ + bool result = x && y < z < t && u; +} +// CHECK-MESSAGES: :[[@LINE-2]]:24: warning: chained comparison 'v0 < v1 < v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +void properly_chained_1(int x, int y, int z) +{ + bool result = x < y && y < z; +} + +void properly_chained_2(int x, int y, bool z) +{ + bool result = (x < y) == z; +} + +struct Value { + bool operator<(const Value&) const; +}; + +bool operator==(bool, const Value&); + +bool badWithCppOperator(Value a, Value b, Value c) { + return a < b == c; +} +// CHECK-MESSAGES: :[[@LINE-2]]:12: warning: chained comparison 'v0 < v1 == v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] + +bool mixedBinaryAndCpp(Value a, Value b, bool c) { + return a < b == c; +} +// CHECK-MESSAGES: :[[@LINE-2]]:12: warning: chained comparison 'v0 < v1 == v2' may generate unintended results, use parentheses to specify order of evaluation or a logical operator to separate comparison expressions [bugprone-chained-comparison] diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/exception-escape.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/exception-escape.cpp index 4a7149e81ce7..f5e74df1621c 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/exception-escape.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/exception-escape.cpp @@ -586,6 +586,16 @@ void swap(int&, int&) { throw 1; } +void iter_swap(int&, int&) { + // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: an exception may be thrown in function 'iter_swap' which should not throw exceptions + throw 1; +} + +void iter_move(int&) { + // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: an exception may be thrown in function 'iter_move' which should not throw exceptions + throw 1; +} + namespace std { class bad_alloc {}; } diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/implicit-widening-of-multiplication-result-array-subscript-expression.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/implicit-widening-of-multiplication-result-array-subscript-expression.cpp index 2881341c8785..5b432cb26629 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/implicit-widening-of-multiplication-result-array-subscript-expression.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/implicit-widening-of-multiplication-result-array-subscript-expression.cpp @@ -18,7 +18,7 @@ char *t0(char *base, int a, int b) { // CHECK-NOTES-CXX: static_cast( ) // CHECK-NOTES-ALL: :[[@LINE-5]]:16: note: perform multiplication in a wider type // CHECK-NOTES-C: (ptrdiff_t) - // CHECK-NOTES-CXX: static_cast() + // CHECK-NOTES-CXX: static_cast( ) } void *t1(char *base, int a, int b) { return &((a * b)[base]); @@ -35,7 +35,7 @@ char *t2(char *base, unsigned int a, int b) { // CHECK-NOTES-CXX: static_cast( ) // CHECK-NOTES-ALL: :[[@LINE-5]]:16: note: perform multiplication in a wider type // CHECK-NOTES-C: (size_t) - // CHECK-NOTES-CXX: static_cast() + // CHECK-NOTES-CXX: static_cast( ) } char *t3(char *base, int a, unsigned int b) { diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/implicit-widening-of-multiplication-result-int.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/implicit-widening-of-multiplication-result-int.cpp index cf13a7ea9fbd..8619a4ee8655 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/implicit-widening-of-multiplication-result-int.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/implicit-widening-of-multiplication-result-int.cpp @@ -18,7 +18,7 @@ long t0(int a, int b) { // CHECK-NOTES-CXX: static_cast( ) // CHECK-NOTES-ALL: :[[@LINE-5]]:10: note: perform multiplication in a wider type // CHECK-NOTES-C: (long) - // CHECK-NOTES-CXX: static_cast() + // CHECK-NOTES-CXX: static_cast( ) } unsigned long t1(int a, int b) { return a * b; @@ -28,7 +28,7 @@ unsigned long t1(int a, int b) { // CHECK-NOTES-CXX: static_cast( ) // CHECK-NOTES-ALL: :[[@LINE-5]]:10: note: perform multiplication in a wider type // CHECK-NOTES-C: (long) - // CHECK-NOTES-CXX: static_cast() + // CHECK-NOTES-CXX: static_cast( ) } long t2(unsigned int a, int b) { @@ -39,7 +39,7 @@ long t2(unsigned int a, int b) { // CHECK-NOTES-CXX: static_cast( ) // CHECK-NOTES-ALL: :[[@LINE-5]]:10: note: perform multiplication in a wider type // CHECK-NOTES-C: (unsigned long) - // CHECK-NOTES-CXX: static_cast() + // CHECK-NOTES-CXX: static_cast( ) } unsigned long t3(unsigned int a, int b) { return a * b; @@ -49,7 +49,7 @@ unsigned long t3(unsigned int a, int b) { // CHECK-NOTES-CXX: static_cast( ) // CHECK-NOTES-ALL: :[[@LINE-5]]:10: note: perform multiplication in a wider type // CHECK-NOTES-C: (unsigned long) - // CHECK-NOTES-CXX: static_cast() + // CHECK-NOTES-CXX: static_cast( ) } long t4(int a, unsigned int b) { diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/implicit-widening-of-multiplication-result-pointer-offset.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/implicit-widening-of-multiplication-result-pointer-offset.cpp index c812885507e8..5ba6a5639849 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/implicit-widening-of-multiplication-result-pointer-offset.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/implicit-widening-of-multiplication-result-pointer-offset.cpp @@ -18,7 +18,7 @@ char *t0(char *base, int a, int b) { // CHECK-NOTES-CXX: static_cast( ) // CHECK-NOTES-ALL: :[[@LINE-5]]:17: note: perform multiplication in a wider type // CHECK-NOTES-C: (ptrdiff_t) - // CHECK-NOTES-CXX: static_cast() + // CHECK-NOTES-CXX: static_cast( ) } char *t1(char *base, int a, int b) { return a * b + base; @@ -35,7 +35,7 @@ char *t2(char *base, unsigned int a, int b) { // CHECK-NOTES-CXX: static_cast( ) // CHECK-NOTES-ALL: :[[@LINE-5]]:17: note: perform multiplication in a wider type // CHECK-NOTES-C: (size_t) - // CHECK-NOTES-CXX: static_cast() + // CHECK-NOTES-CXX: static_cast( ) } char *t3(char *base, int a, unsigned int b) { diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward.cpp index b9720db272e4..443f338ba204 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward.cpp @@ -90,9 +90,9 @@ void lambda_value_capture(T&& t) { } template -void lambda_value_reference(T&& t) { - // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: forwarding reference parameter 't' is never forwarded inside the function body [cppcoreguidelines-missing-std-forward] - [&]() { T other = std::forward(t); }; +void lambda_value_capture_copy(T&& t) { + // CHECK-MESSAGES: :[[@LINE-1]]:36: warning: forwarding reference parameter 't' is never forwarded inside the function body [cppcoreguidelines-missing-std-forward] + [&,t]() { T other = std::forward(t); }; } } // namespace positive_cases @@ -147,4 +147,29 @@ class AClass { T data; }; +template +void lambda_value_reference(T&& t) { + [&]() { T other = std::forward(t); }; +} + +template +void lambda_value_reference_capture_list_ref_1(T&& t) { + [=, &t] { T other = std::forward(t); }; +} + +template +void lambda_value_reference_capture_list_ref_2(T&& t) { + [&t] { T other = std::forward(t); }; +} + +template +void lambda_value_reference_capture_list(T&& t) { + [t = std::forward(t)] { t(); }; +} + +template +void lambda_value_reference_auxiliary_var(T&& t) { + [&x = t]() { T other = std::forward(x); }; +} + } // namespace negative_cases diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/prefer-member-initializer.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/prefer-member-initializer.cpp index b5603dea316d..8086caa2aa6f 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/prefer-member-initializer.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/prefer-member-initializer.cpp @@ -586,3 +586,33 @@ struct ReassignmentAfterUnsafetyAssignment { } int m_i; }; + +namespace PR70189 { +#define RGB(r,g,b) ((unsigned long)(((unsigned char)(r)|((unsigned short)((unsigned char)(g))<<8))|(((unsigned long)(unsigned char)(b))<<16))) +#define INVALID_HANDLE_VALUE ((void*)(unsigned long long)-1) +#define SIMPLE 12 + +class Foo { +public: + Foo() { +// CHECK-FIXES: Foo() : m_color(RGB(255, 128, 0)), m_handle(INVALID_HANDLE_VALUE), m_myval(SIMPLE) { + m_color = RGB(255, 128, 0); +// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: 'm_color' should be initialized in a member initializer of the constructor [cppcoreguidelines-prefer-member-initializer] +// CHECK-FIXES: {{^\ *$}} + m_handle = INVALID_HANDLE_VALUE; +// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: 'm_handle' should be initialized in a member initializer of the constructor [cppcoreguidelines-prefer-member-initializer] +// CHECK-FIXES: {{^\ *$}} + m_myval = SIMPLE; +// CHECK-MESSAGES: :[[@LINE-1]]:5: warning: 'm_myval' should be initialized in a member initializer of the constructor [cppcoreguidelines-prefer-member-initializer] +// CHECK-FIXES: {{^\ *$}} + } +private: + unsigned long m_color; + void* m_handle; + int m_myval; +}; + +#undef SIMPLE +#undef INVALID_HANDLE_VALUE +#undef RGB +} diff --git a/clang-tools-extra/test/clang-tidy/checkers/google/readability-casting.cpp b/clang-tools-extra/test/clang-tidy/checkers/google/readability-casting.cpp index e25463cf451b..bc53b54264af 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/google/readability-casting.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/google/readability-casting.cpp @@ -1,4 +1,4 @@ -// RUN: %check_clang_tidy -std=c++11-or-later %s google-readability-casting %t +// RUN: %check_clang_tidy -std=c++11-or-later %s google-readability-casting %t -- -- -fexceptions bool g() { return false; } @@ -322,17 +322,10 @@ void conversions() { } template -T functional_cast_template_used_by_class(float i) { +T functional_cast_template(float i) { return T(i); } -template -T functional_cast_template_used_by_int(float i) { - return T(i); - // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: C-style casts are discouraged; use static_cast - // CHECK-FIXES: return static_cast(i); -} - struct S2 { S2(float); }; @@ -356,8 +349,8 @@ void functional_casts() { auto s = S(str); // Functional casts in template functions - functional_cast_template_used_by_class(x); - functional_cast_template_used_by_int(x); + functional_cast_template(x); + functional_cast_template(x); // New expressions are not functional casts auto w = new int(x); @@ -366,4 +359,6 @@ void functional_casts() { S2 t = T(x); // OK, constructor call S2 u = U(x); // NOK, it's a reinterpret_cast in disguise // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: C-style casts are discouraged; use static_cast/const_cast/reinterpret_cast + + throw S2(5.0f); } diff --git a/clang-tools-extra/test/clang-tidy/checkers/google/runtime-int.cpp b/clang-tools-extra/test/clang-tidy/checkers/google/runtime-int.cpp index 4bb739876b7f..88f0b519e9cb 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/google/runtime-int.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/google/runtime-int.cpp @@ -59,11 +59,13 @@ void qux() { // CHECK-MESSAGES: [[@LINE-1]]:3: warning: consider replacing 'short' with 'int16' } -// FIXME: This shouldn't warn, as UD-literal operators require one of a handful -// of types as an argument. struct some_value {}; -constexpr some_value operator"" _some_literal(unsigned long long int i); -// CHECK-MESSAGES: [[@LINE-1]]:47: warning: consider replacing 'unsigned long long' +constexpr some_value operator"" _some_literal(unsigned long long int i) +{ + short j; + // CHECK-MESSAGES: [[@LINE-1]]:3: warning: consider replacing 'short' with 'int16' + return some_value(); +} struct A { A& operator=(const A&); }; class B { A a[0]; }; diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp index 794578ceeeba..9da468128743 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp @@ -30,3 +30,31 @@ namespace gh57297{ struct Stream { }; template void f() { T t; Stream x; x << t; } } // namespace gh57297 + +namespace gh70323{ +// A fold expression may contain the checked variable as it's initializer. +// We don't know if the operator modifies that variable because the +// operator is type dependent due to the parameter pack. + +struct Stream {}; +template +void concatenate1(Args... args) +{ + Stream stream; + (stream << ... << args); +} + +template +void concatenate2(Args... args) +{ + Stream stream; + (args << ... << stream); +} + +template +void concatenate3(Args... args) +{ + Stream stream; + (..., (stream << args)); +} +} // namespace gh70323 diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/coroutine-hostile-raii.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/coroutine-hostile-raii.cpp index 55a7e4b8f295..c23c355dac1b 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/misc/coroutine-hostile-raii.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/misc/coroutine-hostile-raii.cpp @@ -1,7 +1,7 @@ // RUN: %check_clang_tidy -std=c++20 %s misc-coroutine-hostile-raii %t \ // RUN: -config="{CheckOptions: {\ // RUN: misc-coroutine-hostile-raii.RAIITypesList: 'my::Mutex; ::my::other::Mutex', \ -// RUN: misc-coroutine-hostile-raii.AllowedAwaitablesList: 'safe::awaitable; ::my::other::awaitable' \ +// RUN: misc-coroutine-hostile-raii.AllowedAwaitablesList: 'safe::awaitable; ::transformable::awaitable' \ // RUN: }}" namespace std { @@ -136,6 +136,9 @@ ReturnObject scopedLockableTest() { absl::Mutex no_warning_5; } +// ================================================================================ +// Safe awaitable +// ================================================================================ namespace safe { struct awaitable { bool await_ready() noexcept { return false; } @@ -150,6 +153,32 @@ ReturnObject RAIISafeSuspendTest() { co_await other{}; } +// ================================================================================ +// Safe transformable awaitable +// ================================================================================ +struct transformable { struct awaitable{}; }; +using alias_transformable_awaitable = transformable::awaitable; +struct UseTransformAwaitable { + struct promise_type { + UseTransformAwaitable get_return_object() { return {}; } + std::suspend_always initial_suspend() { return {}; } + std::suspend_always final_suspend() noexcept { return {}; } + void unhandled_exception() {} + std::suspend_always await_transform(transformable::awaitable) { return {}; } + }; +}; + +auto retAwaitable() { return transformable::awaitable{}; } +UseTransformAwaitable RAIISafeSuspendTest2() { + absl::Mutex a; + co_await retAwaitable(); + co_await transformable::awaitable{}; + co_await alias_transformable_awaitable{}; +} + +// ================================================================================ +// Lambdas +// ================================================================================ void lambda() { absl::Mutex no_warning; auto lambda = []() -> ReturnObject { @@ -164,6 +193,9 @@ void lambda() { absl::Mutex no_warning_2; } +// ================================================================================ +// Denylisted RAII +// ================================================================================ template ReturnObject raii_in_template(){ T a; diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.cpp index da4ab9baa394..16c6ba60c192 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.cpp @@ -20,6 +20,48 @@ void print(...); #define my_macro() assert(0 == 1) // CHECK-FIXES: #define my_macro() assert(0 == 1) +namespace PR24066 { + +void referenceMember() { + struct { + int A; + int B; + } S; + assert(&S.B - &S.A == 1); +} + +const int X = 1; +void referenceVariable() { + assert(X > 0); +} + + +constexpr int Y = 1; +void referenceConstexprVariable() { + assert(Y > 0); + // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be replaced by static_assert() [misc-static-assert] + // CHECK-FIXES-CXX11: {{^ }}static_assert(Y > 0, ""); + // CHECK-FIXES-CXX17: {{^ }}static_assert(Y > 0); +} + +void useInSizeOf() { + char a = 0; + assert(sizeof(a) == 1U); + // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be replaced by static_assert() [misc-static-assert] + // CHECK-FIXES-CXX11: {{^ }}static_assert(sizeof(a) == 1U, ""); + // CHECK-FIXES-CXX17: {{^ }}static_assert(sizeof(a) == 1U); +} + +void useInDecltype() { + char a = 0; + assert(static_cast(256) == 0); + // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: found assert() that could be replaced by static_assert() [misc-static-assert] + // CHECK-FIXES-CXX11: {{^ }}static_assert(static_cast(256) == 0, ""); + // CHECK-FIXES-CXX17: {{^ }}static_assert(static_cast(256) == 0); +} + +} + constexpr bool myfunc(int a, int b) { return a * b == 0; } typedef __SIZE_TYPE__ size_t; diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/loop-convert-basic.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/loop-convert-basic.cpp index e2b9336d620f..c29fbc9f9b23 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/modernize/loop-convert-basic.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/loop-convert-basic.cpp @@ -954,3 +954,16 @@ void dependenceArrayTest() { } } // namespace PseudoArray + +namespace PR78381 { + struct blocked_range { + int begin() const; + int end() const; + }; + + void test() { + blocked_range r; + for (auto i = r.begin(); i!=r.end(); ++i) { + } + } +} diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-auto-for-pointer.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-auto-for-pointer.cpp new file mode 100644 index 000000000000..8a3e0bab26c1 --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-auto-for-pointer.cpp @@ -0,0 +1,29 @@ +// RUN: %check_clang_tidy -check-suffix=REMOVE %s modernize-use-auto %t -- \ +// RUN: -config="{CheckOptions: {modernize-use-auto.RemoveStars: 'true', modernize-use-auto.MinTypeNameLength: '0'}}" +// RUN: %check_clang_tidy %s modernize-use-auto %t -- \ +// RUN: -config="{CheckOptions: {modernize-use-auto.RemoveStars: 'false', modernize-use-auto.MinTypeNameLength: '0'}}" + +void pointerToFunction() { + void (*(*(f1)))() = static_cast(nullptr); + // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use auto when initializing + // CHECK-FIXES-REMOVE: auto f1 = + // CHECK-FIXES: auto *f1 = +} + +void pointerToArray() { + int(*a1)[2] = new int[10][2]; + // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use auto when initializing + // CHECK-FIXES-REMOVE: auto a1 = + // CHECK-FIXES: auto *a1 = +} + +void memberFunctionPointer() { + class A { + void f(); + }; + void(A::* a1)() = static_cast(nullptr); + // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use auto when initializing + // CHECK-FIXES-REMOVE: auto a1 = + // CHECK-FIXES: auto *a1 = +} + diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-emplace.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-emplace.cpp index fead2b6151d0..f7b1ad55f5df 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-emplace.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-emplace.cpp @@ -1183,6 +1183,11 @@ struct NonTrivialWithVector { std::vector it; }; +struct NonTrivialWithIntAndVector { + int x; + std::vector it; +}; + struct NonTrivialWithCtor { NonTrivialWithCtor(); NonTrivialWithCtor(std::vector const&); @@ -1332,6 +1337,14 @@ void testBracedInitTemporaries() { v3.push_back(NonTrivialWithCtor{{}}); v3.push_back({{0}}); v3.push_back({{}}); + + std::vector v4; + + // These should not be noticed or fixed; after the correction, the code won't + // compile. + v4.push_back(NonTrivialWithIntAndVector{1, {}}); + v4.push_back(NonTrivialWithIntAndVector{}); + v4.push_back({}); } void testWithPointerTypes() { diff --git a/clang-tools-extra/test/clang-tidy/checkers/performance/for-range-copy.cpp b/clang-tools-extra/test/clang-tidy/checkers/performance/for-range-copy.cpp index 1a2eedc9e65c..f9d06898ca03 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/performance/for-range-copy.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/performance/for-range-copy.cpp @@ -47,6 +47,11 @@ struct S { S &operator=(const S &); }; +struct Point { + ~Point() {} + int x, y; +}; + struct Convertible { operator S() const { return S(); @@ -87,6 +92,10 @@ void instantiated() { // CHECK-MESSAGES: [[@LINE-1]]:16: warning: the loop variable's type is {{.*}} // CHECK-FIXES: {{^}} for (const S& S2 : View>()) {} + for (const auto [X, Y] : View>()) {} + // CHECK-MESSAGES: [[@LINE-1]]:19: warning: the loop variable's type is + // CHECK-FIXES: {{^}} for (const auto& [X, Y] : View>()) {} + for (const T T2 : View>()) {} // CHECK-MESSAGES: [[@LINE-1]]:16: warning: the loop variable's type is {{.*}} // CHECK-FIXES: {{^}} for (const T& T2 : View>()) {} @@ -123,11 +132,6 @@ struct Mutable { ~Mutable() {} }; -struct Point { - ~Point() {} - int x, y; -}; - Mutable& operator<<(Mutable &Out, bool B) { Out.setBool(B); return Out; @@ -144,6 +148,7 @@ void useByValue(Mutable M); void useByConstValue(const Mutable M); void mutate(Mutable *M); void mutate(Mutable &M); +void mutate(int &); void onceConstOnceMutated(const Mutable &M1, Mutable &M2); void negativeVariableIsMutated() { @@ -234,6 +239,22 @@ void positiveOnlyAccessedFieldAsConst() { } } +void positiveOnlyUsedAsConstBinding() { + for (auto [X, Y] : View>()) { + // CHECK-MESSAGES: [[@LINE-1]]:13: warning: loop variable is copied but + // CHECK-FIXES: for (const auto& [X, Y] : View>()) { + use(X); + use(Y); + } +} + +void negativeMutatedBinding() { + for (auto [X, Y] : View>()) { + use(X); + mutate(Y); + } +} + void positiveOnlyUsedInCopyConstructor() { for (auto A : View>()) { // CHECK-MESSAGES: [[@LINE-1]]:13: warning: loop variable is copied but only used as const reference; consider making it a const reference [performance-for-range-copy] diff --git a/clang-tools-extra/test/clang-tidy/checkers/performance/noexcept-swap.cpp b/clang-tools-extra/test/clang-tidy/checkers/performance/noexcept-swap.cpp index dfc71a2bb9ab..6b266707d77f 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/performance/noexcept-swap.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/performance/noexcept-swap.cpp @@ -32,6 +32,10 @@ void swap(A &, A &); // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: swap functions should be marked noexcept [performance-noexcept-swap] // CHECK-FIXES: void swap(A &, A &) noexcept ; +void iter_swap(A &, A &); +// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: swap functions should be marked noexcept [performance-noexcept-swap] +// CHECK-FIXES: void iter_swap(A &, A &) noexcept ; + struct B { static constexpr bool kFalse = false; void swap(B &) noexcept(kFalse); diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/avoid-nested-conditional-operator.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/avoid-nested-conditional-operator.cpp new file mode 100644 index 000000000000..847df08199e3 --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/avoid-nested-conditional-operator.cpp @@ -0,0 +1,24 @@ +// RUN: %check_clang_tidy %s readability-avoid-nested-conditional-operator %t + +int NestInConditional = (true ? true : false) ? 1 : 2; +// CHECK-MESSAGES: :[[@LINE-1]]:26: warning: conditional operator is used as sub-expression of parent conditional operator, refrain from using nested conditional operators +// CHECK-MESSAGES: :[[@LINE-2]]:25: note: parent conditional operator here + +int NestInTrue = true ? (true ? 1 : 2) : 2; +// CHECK-MESSAGES: :[[@LINE-1]]:26: warning: conditional operator is used as sub-expression of parent conditional operator, refrain from using nested conditional operators +// CHECK-MESSAGES: :[[@LINE-2]]:18: note: parent conditional operator here + +int NestInFalse = true ? 1 : true ? 1 : 2; +// CHECK-MESSAGES: :[[@LINE-1]]:30: warning: conditional operator is used as sub-expression of parent conditional operator, refrain from using nested conditional operators +// CHECK-MESSAGES: :[[@LINE-2]]:19: note: parent conditional operator here +int NestInFalse2 = true ? 1 : (true ? 1 : 2); +// CHECK-MESSAGES: :[[@LINE-1]]:32: warning: conditional operator is used as sub-expression of parent conditional operator, refrain from using nested conditional operators +// CHECK-MESSAGES: :[[@LINE-2]]:20: note: parent conditional operator here + +int NestWithParensis = true ? 1 : ((((true ? 1 : 2)))); +// CHECK-MESSAGES: :[[@LINE-1]]:39: warning: conditional operator is used as sub-expression of parent conditional operator, refrain from using nested conditional operators +// CHECK-MESSAGES: :[[@LINE-2]]:24: note: parent conditional operator here + +#define CONDITIONAL_EXPR (true ? 1 : 2) +// not diag for macro since it will not reduce readability +int NestWithMacro = true ? CONDITIONAL_EXPR : 2; diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/avoid-return-with-void-value.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/avoid-return-with-void-value.cpp new file mode 100644 index 000000000000..f00407c99ce5 --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/avoid-return-with-void-value.cpp @@ -0,0 +1,69 @@ +// RUN: %check_clang_tidy %s readability-avoid-return-with-void-value %t +// RUN: %check_clang_tidy -check-suffixes=,INCLUDE-MACROS %s readability-avoid-return-with-void-value %t \ +// RUN: -- -config="{CheckOptions: [{key: readability-avoid-return-with-void-value.IgnoreMacros, value: false}]}" \ +// RUN: -- +// RUN: %check_clang_tidy -check-suffixes=LENIENT %s readability-avoid-return-with-void-value %t \ +// RUN: -- -config="{CheckOptions: [{key: readability-avoid-return-with-void-value.StrictMode, value: false}]}" \ +// RUN: -- + +void f1(); + +void f2() { + return f1(); + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] + // CHECK-MESSAGES-LENIENT: :[[@LINE-2]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] +} + +void f3(bool b) { + if (b) return f1(); + // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] + return f2(); + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] + // CHECK-MESSAGES-LENIENT: :[[@LINE-2]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] +} + +template +T f4() {} + +void f5() { + return f4(); + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] + // CHECK-MESSAGES-LENIENT: :[[@LINE-2]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] +} + +void f6() { return; } + +int f7() { return 1; } + +int f8() { return f7(); } + +void f9() { + return (void)f7(); + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] + // CHECK-MESSAGES-LENIENT: :[[@LINE-2]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] +} + +#define RETURN_VOID return (void)1 + +void f10() { + RETURN_VOID; + // CHECK-MESSAGES-INCLUDE-MACROS: :[[@LINE-1]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] +} + +template +struct C { + C(A) {} +}; + +template +C f11() { return {}; } + +using VOID = void; + +VOID f12(); + +VOID f13() { + return f12(); + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] + // CHECK-MESSAGES-LENIENT: :[[@LINE-2]]:5: warning: return statement within a void function should not have a specified return value [readability-avoid-return-with-void-value] +} diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp index 3b9e06084183..84bdbd58b85e 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp @@ -33,7 +33,7 @@ class TemplatedContainer { public: bool operator==(const TemplatedContainer& other) const; bool operator!=(const TemplatedContainer& other) const; - int size() const; + unsigned long size() const; bool empty() const; }; @@ -42,7 +42,7 @@ class PrivateEmpty { public: bool operator==(const PrivateEmpty& other) const; bool operator!=(const PrivateEmpty& other) const; - int size() const; + unsigned long size() const; private: bool empty() const; }; @@ -61,7 +61,7 @@ struct EnumSize { class Container { public: bool operator==(const Container& other) const; - int size() const; + unsigned long size() const; bool empty() const; }; @@ -70,13 +70,13 @@ class Derived : public Container { class Container2 { public: - int size() const; + unsigned long size() const; bool empty() const { return size() == 0; } }; class Container3 { public: - int size() const; + unsigned long size() const; bool empty() const; }; @@ -85,7 +85,7 @@ bool Container3::empty() const { return this->size() == 0; } class Container4 { public: bool operator==(const Container4& rhs) const; - int size() const; + unsigned long size() const; bool empty() const { return *this == Container4(); } }; @@ -815,3 +815,49 @@ bool testNotEmptyStringLiterals(const std::string& s) using namespace std::string_literals; return s == "foo"s; } + +namespace PR72619 { + struct SS { + bool empty() const; + int size() const; + }; + + struct SU { + bool empty() const; + unsigned size() const; + }; + + void f(const SU& s) { + if (s.size() < 0) {} + if (0 > s.size()) {} + if (s.size() >= 0) {} + if (0 <= s.size()) {} + if (s.size() < 1) + ; + // CHECK-MESSAGES: :[[@LINE-2]]:9: warning: the 'empty' method should be used to check for emptiness instead of 'size' [readability-container-size-empty] + // CHECK-FIXES: {{^ }}if (s.empty()){{$}} + if (1 > s.size()) + ; + // CHECK-MESSAGES: :[[@LINE-2]]:13: warning: the 'empty' method should be used to check for emptiness instead of 'size' [readability-container-size-empty] + // CHECK-FIXES: {{^ }}if (s.empty()){{$}} + if (s.size() <= 0) + ; + // CHECK-MESSAGES: :[[@LINE-2]]:9: warning: the 'empty' method should be used to check for emptiness instead of 'size' [readability-container-size-empty] + // CHECK-FIXES: {{^ }}if (s.empty()){{$}} + if (0 >= s.size()) + ; + // CHECK-MESSAGES: :[[@LINE-2]]:14: warning: the 'empty' method should be used to check for emptiness instead of 'size' [readability-container-size-empty] + // CHECK-FIXES: {{^ }}if (s.empty()){{$}} + } + + void f(const SS& s) { + if (s.size() < 0) {} + if (0 > s.size()) {} + if (s.size() >= 0) {} + if (0 <= s.size()) {} + if (s.size() < 1) {} + if (1 > s.size()) {} + if (s.size() <= 0) {} + if (0 >= s.size()) {} + } +} diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion-allow-in-conditions.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion-allow-in-conditions.cpp index e393e297140c..ef591940917c 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion-allow-in-conditions.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion-allow-in-conditions.cpp @@ -12,13 +12,14 @@ int* functionReturningPointer(); struct Struct { int member; unsigned bitfield : 1; + bool boolfield : 1; }; void regularImplicitConversionIntegerToBoolIsNotIgnored() { int integer = 0; functionTaking(integer); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> bool [readability-implicit-bool-conversion] + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> 'bool' [readability-implicit-bool-conversion] // CHECK-FIXES: functionTaking(integer != 0); } @@ -28,6 +29,8 @@ void implicitConversionIntegerToBoolInConditionalsIsAllowed() { if (!s.member) {} if (s.bitfield) {} if (!s.bitfield) {} + if (s.boolfield == true) {} + if (s.boolfield != true) {} if (functionReturningInt()) {} if (!functionReturningInt()) {} if (functionReturningInt() && functionReturningPointer()) {} @@ -51,12 +54,12 @@ void implicitConversionIntegerToBoolInConditionalsIsAllowed() { void regularImplicitConversionPointerToBoolIsNotIgnored() { int* pointer = nullptr; functionTaking(pointer); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int *' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int *' -> 'bool' // CHECK-FIXES: functionTaking(pointer != nullptr); int Struct::* memberPointer = &Struct::member; functionTaking(memberPointer); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int Struct::*' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int Struct::*' -> 'bool' // CHECK-FIXES: functionTaking(memberPointer != nullptr); } diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion-cxx98.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion-cxx98.cpp index 861677ac6c30..e4c47b06d8f8 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion-cxx98.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion-cxx98.cpp @@ -15,31 +15,31 @@ struct Struct { void useOldNullMacroInReplacements() { int* pointer = NULL; functionTaking(pointer); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int *' -> bool [readability-implicit-bool-conversion] + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int *' -> 'bool' [readability-implicit-bool-conversion] // CHECK-FIXES: functionTaking(pointer != 0); int Struct::* memberPointer = NULL; functionTaking(!memberPointer); - // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: implicit conversion 'int Struct::*' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: implicit conversion 'int Struct::*' -> 'bool' // CHECK-FIXES: functionTaking(memberPointer == 0); } void fixFalseLiteralConvertingToNullPointer() { functionTaking(false); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion bool -> 'int *' + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'bool' -> 'int *' // CHECK-FIXES: functionTaking(0); int* pointer = NULL; if (pointer == false) {} - // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: implicit conversion bool -> 'int *' + // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: implicit conversion 'bool' -> 'int *' // CHECK-FIXES: if (pointer == 0) {} functionTaking(false); - // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: implicit conversion bool -> 'int Struct::*' + // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: implicit conversion 'bool' -> 'int Struct::*' // CHECK-FIXES: functionTaking(0); int Struct::* memberPointer = NULL; if (memberPointer != false) {} - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion bool -> 'int Struct::*' + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'bool' -> 'int Struct::*' // CHECK-FIXES: if (memberPointer != 0) {} } diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion.cpp index f7f5d506a9ce..6f1f29729718 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion.cpp @@ -21,30 +21,30 @@ void implicitConversionFromBoolSimpleCases() { functionTaking(boolean); functionTaking(boolean); - // CHECK-MESSAGES: :[[@LINE-1]]:23: warning: implicit conversion bool -> 'int' [readability-implicit-bool-conversion] + // CHECK-MESSAGES: :[[@LINE-1]]:23: warning: implicit conversion 'bool' -> 'int' [readability-implicit-bool-conversion] // CHECK-FIXES: functionTaking(static_cast(boolean)); functionTaking(boolean); - // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: implicit conversion bool -> 'unsigned long' + // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: implicit conversion 'bool' -> 'unsigned long' // CHECK-FIXES: functionTaking(static_cast(boolean)); functionTaking(boolean); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion bool -> 'char' + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'bool' -> 'char' // CHECK-FIXES: functionTaking(static_cast(boolean)); functionTaking(boolean); - // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: implicit conversion bool -> 'float' + // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: implicit conversion 'bool' -> 'float' // CHECK-FIXES: functionTaking(static_cast(boolean)); functionTaking(boolean); - // CHECK-MESSAGES: :[[@LINE-1]]:26: warning: implicit conversion bool -> 'double' + // CHECK-MESSAGES: :[[@LINE-1]]:26: warning: implicit conversion 'bool' -> 'double' // CHECK-FIXES: functionTaking(static_cast(boolean)); } float implicitConversionFromBoolInReturnValue() { bool boolean = false; return boolean; - // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: implicit conversion bool -> 'float' + // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: implicit conversion 'bool' -> 'float' // CHECK-FIXES: return static_cast(boolean); } @@ -58,15 +58,15 @@ void implicitConversionFromBoolInSingleBoolExpressions(bool b1, bool b2) { boolean = b2 != false; int integer = boolean - 3; - // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: implicit conversion bool -> 'int' + // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: implicit conversion 'bool' -> 'int' // CHECK-FIXES: int integer = static_cast(boolean) - 3; float floating = boolean / 0.3f; - // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: implicit conversion bool -> 'float' + // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: implicit conversion 'bool' -> 'float' // CHECK-FIXES: float floating = static_cast(boolean) / 0.3f; char character = boolean; - // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: implicit conversion bool -> 'char' + // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: implicit conversion 'bool' -> 'char' // CHECK-FIXES: char character = static_cast(boolean); } @@ -75,41 +75,41 @@ void implicitConversionFromBoollInComplexBoolExpressions() { bool anotherBoolean = false; int integer = boolean && anotherBoolean; - // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: implicit conversion bool -> 'int' + // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: implicit conversion 'bool' -> 'int' // CHECK-FIXES: int integer = static_cast(boolean && anotherBoolean); unsigned long unsignedLong = (! boolean) + 4ul; - // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: implicit conversion bool -> 'unsigned long' + // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: implicit conversion 'bool' -> 'unsigned long' // CHECK-FIXES: unsigned long unsignedLong = static_cast(! boolean) + 4ul; float floating = (boolean || anotherBoolean) * 0.3f; - // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: implicit conversion bool -> 'float' + // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: implicit conversion 'bool' -> 'float' // CHECK-FIXES: float floating = static_cast(boolean || anotherBoolean) * 0.3f; double doubleFloating = (boolean && (anotherBoolean || boolean)) * 0.3; - // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: implicit conversion bool -> 'double' + // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: implicit conversion 'bool' -> 'double' // CHECK-FIXES: double doubleFloating = static_cast(boolean && (anotherBoolean || boolean)) * 0.3; } void implicitConversionFromBoolLiterals() { functionTaking(true); - // CHECK-MESSAGES: :[[@LINE-1]]:23: warning: implicit conversion bool -> 'int' + // CHECK-MESSAGES: :[[@LINE-1]]:23: warning: implicit conversion 'bool' -> 'int' // CHECK-FIXES: functionTaking(1); functionTaking(false); - // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: implicit conversion bool -> 'unsigned long' + // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: implicit conversion 'bool' -> 'unsigned long' // CHECK-FIXES: functionTaking(0u); functionTaking(true); - // CHECK-MESSAGES: :[[@LINE-1]]:31: warning: implicit conversion bool -> 'signed char' + // CHECK-MESSAGES: :[[@LINE-1]]:31: warning: implicit conversion 'bool' -> 'signed char' // CHECK-FIXES: functionTaking(1); functionTaking(false); - // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: implicit conversion bool -> 'float' + // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: implicit conversion 'bool' -> 'float' // CHECK-FIXES: functionTaking(0.0f); functionTaking(true); - // CHECK-MESSAGES: :[[@LINE-1]]:26: warning: implicit conversion bool -> 'double' + // CHECK-MESSAGES: :[[@LINE-1]]:26: warning: implicit conversion 'bool' -> 'double' // CHECK-FIXES: functionTaking(1.0); } @@ -118,11 +118,11 @@ void implicitConversionFromBoolInComparisons() { int integer = 0; functionTaking(boolean == integer); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion bool -> 'int' + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'bool' -> 'int' // CHECK-FIXES: functionTaking(static_cast(boolean) == integer); functionTaking(integer != boolean); - // CHECK-MESSAGES: :[[@LINE-1]]:35: warning: implicit conversion bool -> 'int' + // CHECK-MESSAGES: :[[@LINE-1]]:35: warning: implicit conversion 'bool' -> 'int' // CHECK-FIXES: functionTaking(integer != static_cast(boolean)); } @@ -171,59 +171,59 @@ void useOfTemplateFunction() { void implicitConversionToBoolSimpleCases() { int integer = 10; functionTaking(integer); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: functionTaking(integer != 0); unsigned long unsignedLong = 10; functionTaking(unsignedLong); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'unsigned long' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'unsigned long' -> 'bool' // CHECK-FIXES: functionTaking(unsignedLong != 0u); float floating = 0.0f; functionTaking(floating); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'float' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'float' -> 'bool' // CHECK-FIXES: functionTaking(floating != 0.0f); double doubleFloating = 1.0f; functionTaking(doubleFloating); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'double' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'double' -> 'bool' // CHECK-FIXES: functionTaking(doubleFloating != 0.0); signed char character = 'a'; functionTaking(character); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'signed char' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'signed char' -> 'bool' // CHECK-FIXES: functionTaking(character != 0); int* pointer = nullptr; functionTaking(pointer); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int *' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int *' -> 'bool' // CHECK-FIXES: functionTaking(pointer != nullptr); auto pointerToMember = &Struct::member; functionTaking(pointerToMember); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int Struct::*' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int Struct::*' -> 'bool' // CHECK-FIXES: functionTaking(pointerToMember != nullptr); } void implicitConversionToBoolInSingleExpressions() { int integer = 10; bool boolComingFromInt = integer; - // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: bool boolComingFromInt = integer != 0; float floating = 10.0f; bool boolComingFromFloat = floating; - // CHECK-MESSAGES: :[[@LINE-1]]:30: warning: implicit conversion 'float' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:30: warning: implicit conversion 'float' -> 'bool' // CHECK-FIXES: bool boolComingFromFloat = floating != 0.0f; signed char character = 'a'; bool boolComingFromChar = character; - // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: implicit conversion 'signed char' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: implicit conversion 'signed char' -> 'bool' // CHECK-FIXES: bool boolComingFromChar = character != 0; int* pointer = nullptr; bool boolComingFromPointer = pointer; - // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: implicit conversion 'int *' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: implicit conversion 'int *' -> 'bool' // CHECK-FIXES: bool boolComingFromPointer = pointer != nullptr; } @@ -233,131 +233,131 @@ void implicitConversionToBoolInComplexExpressions() { int integer = 10; int anotherInteger = 20; bool boolComingFromInteger = integer + anotherInteger; - // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: bool boolComingFromInteger = (integer + anotherInteger) != 0; float floating = 0.2f; bool boolComingFromFloating = floating - 0.3f || boolean; - // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: implicit conversion 'float' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: implicit conversion 'float' -> 'bool' // CHECK-FIXES: bool boolComingFromFloating = ((floating - 0.3f) != 0.0f) || boolean; double doubleFloating = 0.3; bool boolComingFromDoubleFloating = (doubleFloating - 0.4) && boolean; - // CHECK-MESSAGES: :[[@LINE-1]]:39: warning: implicit conversion 'double' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:39: warning: implicit conversion 'double' -> 'bool' // CHECK-FIXES: bool boolComingFromDoubleFloating = ((doubleFloating - 0.4) != 0.0) && boolean; } void implicitConversionInNegationExpressions() { int integer = 10; bool boolComingFromNegatedInt = !integer; - // CHECK-MESSAGES: :[[@LINE-1]]:36: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:36: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: bool boolComingFromNegatedInt = integer == 0; float floating = 10.0f; bool boolComingFromNegatedFloat = ! floating; - // CHECK-MESSAGES: :[[@LINE-1]]:39: warning: implicit conversion 'float' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:39: warning: implicit conversion 'float' -> 'bool' // CHECK-FIXES: bool boolComingFromNegatedFloat = floating == 0.0f; signed char character = 'a'; bool boolComingFromNegatedChar = (! character); - // CHECK-MESSAGES: :[[@LINE-1]]:39: warning: implicit conversion 'signed char' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:39: warning: implicit conversion 'signed char' -> 'bool' // CHECK-FIXES: bool boolComingFromNegatedChar = (character == 0); int* pointer = nullptr; bool boolComingFromNegatedPointer = not pointer; - // CHECK-MESSAGES: :[[@LINE-1]]:43: warning: implicit conversion 'int *' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:43: warning: implicit conversion 'int *' -> 'bool' // CHECK-FIXES: bool boolComingFromNegatedPointer = pointer == nullptr; } void implicitConversionToBoolInControlStatements() { int integer = 10; if (integer) {} - // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: if (integer != 0) {} long int longInteger = 0.2f; for (;longInteger;) {} - // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: implicit conversion 'long' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: implicit conversion 'long' -> 'bool' // CHECK-FIXES: for (;longInteger != 0;) {} float floating = 0.3f; while (floating) {} - // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: implicit conversion 'float' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: implicit conversion 'float' -> 'bool' // CHECK-FIXES: while (floating != 0.0f) {} double doubleFloating = 0.4; do {} while (doubleFloating); - // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: implicit conversion 'double' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: implicit conversion 'double' -> 'bool' // CHECK-FIXES: do {} while (doubleFloating != 0.0); } bool implicitConversionToBoolInReturnValue() { float floating = 1.0f; return floating; - // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: implicit conversion 'float' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: implicit conversion 'float' -> 'bool' // CHECK-FIXES: return floating != 0.0f; } void implicitConversionToBoolFromLiterals() { functionTaking(0); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: functionTaking(false); functionTaking(1); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: functionTaking(true); functionTaking(2ul); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'unsigned long' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'unsigned long' -> 'bool' // CHECK-FIXES: functionTaking(true); functionTaking(0.0f); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'float' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'float' -> 'bool' // CHECK-FIXES: functionTaking(false); functionTaking(1.0f); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'float' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'float' -> 'bool' // CHECK-FIXES: functionTaking(true); functionTaking(2.0); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'double' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'double' -> 'bool' // CHECK-FIXES: functionTaking(true); functionTaking('\0'); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'char' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'char' -> 'bool' // CHECK-FIXES: functionTaking(false); functionTaking('a'); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'char' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'char' -> 'bool' // CHECK-FIXES: functionTaking(true); functionTaking(""); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'const char *' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'const char *' -> 'bool' // CHECK-FIXES: functionTaking(true); functionTaking("abc"); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'const char *' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'const char *' -> 'bool' // CHECK-FIXES: functionTaking(true); functionTaking(NULL); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'long' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'long' -> 'bool' // CHECK-FIXES: functionTaking(false); } void implicitConversionToBoolFromUnaryMinusAndZeroLiterals() { functionTaking(-0); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: functionTaking((-0) != 0); functionTaking(-0.0f); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'float' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'float' -> 'bool' // CHECK-FIXES: functionTaking((-0.0f) != 0.0f); functionTaking(-0.0); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'double' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'double' -> 'bool' // CHECK-FIXES: functionTaking((-0.0) != 0.0); } @@ -370,11 +370,11 @@ void implicitConversionToBoolInWithOverloadedOperators() { UserStruct s; functionTaking(s(0)); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: functionTaking(s(0) != 0); functionTaking(s + 2); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: functionTaking((s + 2) != 0); } @@ -446,27 +446,27 @@ struct S { int c : 2; S(bool a, bool b, bool c) : a(a), b(b), c(c) {} - // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: implicit conversion bool -> 'int' - // CHECK-MESSAGES: :[[@LINE-2]]:45: warning: implicit conversion bool -> 'int' + // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: implicit conversion 'bool' -> 'int' + // CHECK-MESSAGES: :[[@LINE-2]]:45: warning: implicit conversion 'bool' -> 'int' // CHECK-FIXES: S(bool a, bool b, bool c) : a(static_cast(a)), b(b), c(static_cast(c)) {} }; bool f(S& s) { functionTaking(s.a); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: functionTaking(s.a != 0); functionTaking(s.b); // CHECK-FIXES: functionTaking(s.b); s.a = true; - // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: implicit conversion bool -> 'int' + // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: implicit conversion 'bool' -> 'int' // CHECK-FIXES: s.a = 1; s.b = true; // CHECK-FIXES: s.b = true; s.c = true; - // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: implicit conversion bool -> 'int' + // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: implicit conversion 'bool' -> 'int' // CHECK-FIXES: s.c = 1; functionTaking(s.c); - // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:24: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: functionTaking(s.c != 0); } @@ -475,14 +475,14 @@ bool f(S& s) { int implicitConversionReturnInt() { return true; - // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: implicit conversion bool -> 'int' + // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: implicit conversion 'bool' -> 'int' // CHECK-FIXES: return 1 } int implicitConversionReturnIntWithParens() { return (true); - // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: implicit conversion bool -> 'int' + // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: implicit conversion 'bool' -> 'int' // CHECK-FIXES: return 1 } @@ -490,14 +490,14 @@ int implicitConversionReturnIntWithParens() bool implicitConversionReturnBool() { return 1; - // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: return true } bool implicitConversionReturnBoolWithParens() { return (1); - // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: implicit conversion 'int' -> bool + // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: implicit conversion 'int' -> 'bool' // CHECK-FIXES: return true } @@ -508,3 +508,19 @@ namespace PR47000 { using IntType = int; int to_int2(bool x) { return IntType{x}; } } + +namespace PR71867 { + bool foo(bool x) { + return x ? 1 : false; + // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: implicit conversion 'int' -> 'bool' + // CHECK-MESSAGES: :[[@LINE-2]]:20: warning: implicit conversion 'bool' -> 'int' + // CHECK-FIXES: return (x ? 1 : 0) != 0; + } + + bool boo(bool x) { + return x ? true : 0; + // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: implicit conversion 'int' -> 'bool' + // CHECK-MESSAGES: :[[@LINE-2]]:16: warning: implicit conversion 'bool' -> 'int' + // CHECK-FIXES: return (x ? 1 : 0) != 0; + } +} diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-casting.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-casting.cpp new file mode 100644 index 000000000000..30cac6bd5cca --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-casting.cpp @@ -0,0 +1,223 @@ +// RUN: %check_clang_tidy -std=c++11-or-later %s readability-redundant-casting %t -- -- -fno-delayed-template-parsing +// RUN: %check_clang_tidy -std=c++11-or-later -check-suffix=,MACROS %s readability-redundant-casting %t -- \ +// RUN: -config='{CheckOptions: { readability-redundant-casting.IgnoreMacros: false }}' \ +// RUN: -- -fno-delayed-template-parsing +// RUN: %check_clang_tidy -std=c++11-or-later -check-suffix=,ALIASES %s readability-redundant-casting %t -- \ +// RUN: -config='{CheckOptions: { readability-redundant-casting.IgnoreTypeAliases: true }}' \ +// RUN: -- -fno-delayed-template-parsing + +struct A {}; +struct B : A {}; +A getA(); + +void testRedundantStaticCasting(A& value) { + A& a1 = static_cast(value); + // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: redundant explicit casting to the same type 'A' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-MESSAGES: :[[@LINE-3]]:36: note: source type originates from referencing this parameter + // CHECK-FIXES: {{^}} A& a1 = value; +} + +void testRedundantConstCasting1(A& value) { + A& a2 = const_cast(value); + // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: redundant explicit casting to the same type 'A' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-MESSAGES: :[[@LINE-3]]:36: note: source type originates from referencing this parameter + // CHECK-FIXES: {{^}} A& a2 = value; +} + +void testRedundantConstCasting2(const A& value) { + const A& a3 = const_cast(value); + // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: redundant explicit casting to the same type 'const A' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-MESSAGES: :[[@LINE-3]]:42: note: source type originates from referencing this parameter + // CHECK-FIXES: {{^}} const A& a3 = value; +} + +void testRedundantReinterpretCasting(A& value) { + A& a4 = reinterpret_cast(value); + // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: redundant explicit casting to the same type 'A' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-MESSAGES: :[[@LINE-3]]:41: note: source type originates from referencing this parameter + // CHECK-FIXES: {{^}} A& a4 = value; +} + +void testRedundantCCasting(A& value) { + A& a5 = (A&)(value); + // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: redundant explicit casting to the same type 'A' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-MESSAGES: :[[@LINE-3]]:31: note: source type originates from referencing this parameter + // CHECK-FIXES: {{^}} A& a5 = value; +} + +void testDoubleCasting(A& value) { + A& a6 = static_cast(reinterpret_cast(value)); + // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: redundant explicit casting to the same type 'A' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-MESSAGES: :[[@LINE-2]]:27: warning: redundant explicit casting to the same type 'A' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-MESSAGES: :[[@LINE-4]]:27: note: source type originates from referencing this parameter + // CHECK-FIXES: {{^}} A& a6 = value; +} + +void testDiffrentTypesCast(B& value) { + A& a7 = static_cast(value); +} + +void testCastingWithAuto() { + auto a = getA(); + A& a8 = static_cast(a); + // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: redundant explicit casting to the same type 'A' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-MESSAGES: :[[@LINE-3]]:8: note: source type originates from referencing this variable + // CHECK-FIXES: {{^}} A& a8 = a; +} + +void testCastingWithConstAuto() { + const auto a = getA(); + const A& a9 = static_cast(a); + // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: redundant explicit casting to the same type 'const A' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-MESSAGES: :[[@LINE-3]]:14: note: source type originates from referencing this variable + // CHECK-FIXES: {{^}} const A& a9 = a; +} + +void testCastingWithAutoPtr(A& ptr) { + auto* a = &ptr; + A* a10 = static_cast(a); + // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: redundant explicit casting to the same type 'A *' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-MESSAGES: :[[@LINE-3]]:9: note: source type originates from referencing this variable + // CHECK-FIXES: {{^}} A* a10 = a; +} + +template +void testRedundantTemplateCasting(T& value) { + A& a = static_cast(value); + T& t = static_cast(value); + // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: redundant explicit casting to the same type 'T' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-MESSAGES: :[[@LINE-4]]:38: note: source type originates from referencing this parameter + // CHECK-FIXES: {{^}} T& t = value; +} + +void testTemplate() { + A value; + testRedundantTemplateCasting(value); +} + +void testValidRefConstCast() { + const auto a = getA(); + A& a11 = const_cast(a); +} + +void testValidPtrConstCast(const A* ptr) { + A* a12 = const_cast(ptr); +} + +#define CAST(X) static_cast(X) + +void testMacroCasting(int value) { + int a = CAST(value); + // CHECK-MESSAGES-MACROS: :[[@LINE-1]]:11: warning: redundant explicit casting to the same type 'int' as the sub-expression, remove this casting [readability-redundant-casting] +} + +#define PTR_NAME name + +void testMacroCasting(A* PTR_NAME) { + A* a13 = static_cast(PTR_NAME); + // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: redundant explicit casting to the same type 'A *' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-FIXES: {{^}} A* a13 = PTR_NAME; +} + +struct CastBool { + operator bool() const { + return true; + } +}; + +void testUserOperatorCast(const CastBool& value) { + bool b = static_cast(value); +} + +using TypeA = A; + +void testTypedefCast(A& value) { + TypeA& a = static_cast(value); + // CHECK-MESSAGES-ALIASES: :[[@LINE-1]]:14: warning: redundant explicit casting to the same type 'TypeA' (aka 'A') as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-FIXES-ALIASES: {{^}} TypeA& a = value; +} + +void testTypedefCast2(TypeA& value) { + A& a = static_cast(value); + // CHECK-MESSAGES-ALIASES: :[[@LINE-1]]:10: warning: redundant explicit casting to the same type 'A' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-FIXES-ALIASES: {{^}} A& a = value; +} + +void testFunctionalCastWithPrimitive(int a) { + int b = int(a); + // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: redundant explicit casting to the same type 'int' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-FIXES: {{^}} int b = a; +} + +void testFunctionalCastWithInitExpr(unsigned a) { + unsigned b = ~unsigned{!a}; + unsigned c = unsigned{0}; +} + +void testBinaryOperator(char c) { + int a = int(c - 'C'); +} + +struct BIT { + bool b:1; +}; + +template +void make(Args&& ...); + +void testBinaryOperator(BIT b) { + make((bool)b.b); +} + +struct Class { + using Iterator = const char*; + + Iterator begin() { + return static_cast(first()); +// CHECK-MESSAGES-ALIASES: :[[@LINE-1]]:12: warning: redundant explicit casting to the same type 'Iterator' (aka 'const char *') as the sub-expression, remove this casting [readability-redundant-casting] +// CHECK-MESSAGES-ALIASES: :[[@LINE+4]]:15: note: source type originates from the invocation of this method +// CHECK-FIXES-ALIASES: {{^}} return first(); + } + + const char* first(); +}; + +void testAddOperation(int aa, int bb) { + int c = static_cast(aa + bb) * aa; + // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: redundant explicit casting to the same type 'int' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-FIXES: {{^}} int c = (aa + bb) * aa; +} + +void testAddOperationWithParen(int a, int b) { + int c = static_cast((a+b))*a; + // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: redundant explicit casting to the same type 'int' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-FIXES: {{^}} int c = (a+b)*a; +} + +void testRValueCast(int&& a) { + int&& b = static_cast(a); + int&& c = static_cast(10); + // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: redundant explicit casting to the same type 'int' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-FIXES: {{^}} int&& c = 10; +} + +template +void testRedundantNTTPCasting() { + int a = static_cast(V); + // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: redundant explicit casting to the same type 'int' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-MESSAGES: :[[@LINE-4]]:15: note: source type originates from referencing this non-type template parameter + // CHECK-FIXES: {{^}} int a = V; +} + +template +void testValidNTTPCasting() { + int a = static_cast(V); +} + +template +void testRedundantDependentNTTPCasting() { + T a = static_cast(V); + // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: redundant explicit casting to the same type 'T' as the sub-expression, remove this casting [readability-redundant-casting] + // CHECK-MESSAGES: :[[@LINE-4]]:25: note: source type originates from referencing this non-type template parameter + // CHECK-FIXES: {{^}} T a = V; +} diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-inline-specifier.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-inline-specifier.cpp new file mode 100644 index 000000000000..cdd98d8fdc20 --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-inline-specifier.cpp @@ -0,0 +1,137 @@ +// RUN: %check_clang_tidy -std=c++17 %s readability-redundant-inline-specifier %t +// RUN: %check_clang_tidy -std=c++17 -check-suffixes=,STRICT %s readability-redundant-inline-specifier %t -- -config="{CheckOptions: {readability-redundant-inline-specifier.StrictMode: 'true'}}" + +template inline T f() +// CHECK-MESSAGES-STRICT: :[[@LINE-1]]:23: warning: function 'f' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] +// CHECK-FIXES-STRICT: template T f() +{ + return T{}; +} + +template <> inline double f() = delete; +// CHECK-MESSAGES: :[[@LINE-1]]:13: warning: function 'f' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] +// CHECK-FIXES: template <> double f() = delete; + +inline int g(float a) +{ + return static_cast(a - 5.F); +} + +inline int g(double) = delete; +// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: function 'g' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] +// CHECK-FIXES: int g(double) = delete; + +class C +{ + public: + inline C& operator=(const C&) = delete; + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: function 'operator=' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] + // CHECK-FIXES: C& operator=(const C&) = delete; + + inline C(const C&) = default; + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: function 'C' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] + // CHECK-FIXES: C(const C&) = default; + + constexpr inline C& operator=(int a); + // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: function 'operator=' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] + // CHECK-FIXES: constexpr C& operator=(int a); + + inline C() {} + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: function 'C' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] + // CHECK-FIXES: C() {} + + constexpr inline C(int); + // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: function 'C' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] + // CHECK-FIXES: constexpr C(int); + + inline int Get42() const { return 42; } + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: function 'Get42' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] + // CHECK-FIXES: int Get42() const { return 42; } + + static inline constexpr int C_STATIC = 42; + // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: variable 'C_STATIC' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] + // CHECK-FIXES: static constexpr int C_STATIC = 42; + + static constexpr int C_STATIC_2 = 42; +}; + +constexpr inline int Get42() { return 42; } +// CHECK-MESSAGES: :[[@LINE-1]]:11: warning: function 'Get42' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] +// CHECK-FIXES: constexpr int Get42() { return 42; } + + +static constexpr inline int NAMESPACE_STATIC = 42; + +inline static int fn0(int i) +// CHECK-MESSAGES-STRICT: :[[@LINE-1]]:1: warning: function 'fn0' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] +// CHECK-FIXES-STRICT: static int fn0(int i) +{ + return i - 1; +} + +static constexpr inline int fn1(int i) +// CHECK-MESSAGES: :[[@LINE-1]]:18: warning: function 'fn1' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] +// CHECK-FIXES: static constexpr int fn1(int i) +{ + return i - 1; +} + +namespace +{ + inline int fn2(int i) + // CHECK-MESSAGES-STRICT: :[[@LINE-1]]:5: warning: function 'fn2' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] + // CHECK-FIXES-STRICT: int fn2(int i) + { + return i - 1; + } + + inline constexpr int fn3(int i) + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: function 'fn3' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] + // CHECK-FIXES: constexpr int fn3(int i) + { + return i - 1; + } + + inline constexpr int MY_CONSTEXPR_VAR = 42; + // CHECK-MESSAGES-STRICT: :[[@LINE-1]]:5: warning: variable 'MY_CONSTEXPR_VAR' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] + // CHECK-FIXES-STRICT: constexpr int MY_CONSTEXPR_VAR = 42; +} + +namespace ns +{ + inline int fn4(int i) + { + return i - 1; + } + + inline constexpr int fn5(int i) + // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: function 'fn5' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] + // CHECK-FIXES: constexpr int fn5(int i) + { + return i - 1; + } +} + +auto fn6 = [](){}; + +template inline T fn7(); + +template T fn7() +{ + return T{}; +} + +template T fn8(); + +template inline T fn8() +// CHECK-MESSAGES-STRICT: :[[@LINE-1]]:23: warning: function 'fn8' has inline specifier but is implicitly inlined [readability-redundant-inline-specifier] +// CHECK-FIXES-STRICT: template T fn8() +{ + return T{}; +} + +#define INLINE_MACRO() inline void fn9() { } +INLINE_MACRO() + +#define INLINE_KW inline +INLINE_KW void fn10() { } diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-member-init.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-member-init.cpp index d8ca406581a3..17b2714abca0 100644 --- a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-member-init.cpp +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-member-init.cpp @@ -250,3 +250,55 @@ struct NF15 { S s2; }; }; + +// Direct in-class initialization with default constructor +struct D1 { + S f1 {}; + // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: initializer for member 'f1' is redundant + // CHECK-FIXES: S f1; +}; + +// Direct in-class initialization with constructor with default argument +struct D2 { + T f2 {}; + // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: initializer for member 'f2' is redundant + // CHECK-FIXES: T f2; +}; + +// Direct in-class initialization with default constructor (assign) +struct D3 { + S f3 = {}; + // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: initializer for member 'f3' is redundant + // CHECK-FIXES: S f3; +}; + +// Direct in-class initialization with constructor with default argument (assign) +struct D4 { + T f4 = {}; + // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: initializer for member 'f4' is redundant + // CHECK-FIXES: T f4; +}; + +// Templated class independent type +template +struct D5 { + S f5 /*comment*/ = S(); + // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: initializer for member 'f5' is redundant + // CHECK-FIXES: S f5 /*comment*/; +}; +D5 d5i; +D5 d5s; + +struct D6 { + UsesCleanup uc2{}; + // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: initializer for member 'uc2' is redundant + // CHECK-FIXES: UsesCleanup uc2; +}; + +template +struct D7 { + V f7; +}; + +D7 d7i; +D7 d7s; diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/simplify-boolean-expr-macros.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/simplify-boolean-expr-macros.cpp new file mode 100644 index 000000000000..7d0cfe7e27dc --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/readability/simplify-boolean-expr-macros.cpp @@ -0,0 +1,20 @@ +// RUN: %check_clang_tidy -check-suffixes=,MACROS %s readability-simplify-boolean-expr %t + +// Ignore expressions in macros. +// RUN: %check_clang_tidy %s readability-simplify-boolean-expr %t \ +// RUN: -- -config="{CheckOptions: {readability-simplify-boolean-expr.IgnoreMacros: true}}" \ +// RUN: -- + +#define NEGATE(expr) !(expr) + +bool without_macro(bool a, bool b) { + return !(!a && b); + // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: boolean expression can be simplified by DeMorgan's theorem + // CHECK-FIXES: return a || !b; +} + +bool macro(bool a, bool b) { + return NEGATE(!a && b); + // CHECK-MESSAGES-MACROS: :[[@LINE-1]]:12: warning: boolean expression can be simplified by DeMorgan's theorem + // CHECK-FIXES: return NEGATE(!a && b); +} diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt index 9f814478c455..5f2b7f064da4 100644 --- a/clang/CMakeLists.txt +++ b/clang/CMakeLists.txt @@ -193,6 +193,12 @@ set(C_INCLUDE_DIRS "" CACHE STRING set(GCC_INSTALL_PREFIX "" CACHE PATH "Directory where gcc is installed." ) set(DEFAULT_SYSROOT "" CACHE STRING "Default to all compiler invocations for --sysroot=." ) +if(GCC_INSTALL_PREFIX) + message(WARNING "GCC_INSTALL_PREFIX is deprecated and will be removed. Use " + "configuration files (https://clang.llvm.org/docs/UsersManual.html#configuration-files)" + "to specify the default --gcc-install-dir= or --gcc-triple=. --gcc-toolchain= is discouraged. " + "See https://github.com/llvm/llvm-project/pull/77537 for detail.") +endif() set(ENABLE_LINKER_BUILD_ID OFF CACHE BOOL "pass --build-id to ld") diff --git a/clang/bindings/python/README.txt b/clang/bindings/python/README.txt index 44c715e5de56..3e509662144f 100644 --- a/clang/bindings/python/README.txt +++ b/clang/bindings/python/README.txt @@ -10,7 +10,7 @@ runner. For example: -- $ env PYTHONPATH=$(echo ~/llvm/clang/bindings/python/) \ CLANG_LIBRARY_PATH=$(llvm-config --libdir) \ - python -m unittest discover -v + python3 -m unittest discover -v tests.cindex.test_index.test_create ... ok ... diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py index d780ee353a13..754f03d718e8 100644 --- a/clang/bindings/python/clang/cindex.py +++ b/clang/bindings/python/clang/cindex.py @@ -66,81 +66,50 @@ import clang.enumerations +import collections.abc import os -import sys - -if sys.version_info[0] == 3: - # Python 3 strings are unicode, translate them to/from utf8 for C-interop. - class c_interop_string(c_char_p): - def __init__(self, p=None): - if p is None: - p = "" - if isinstance(p, str): - p = p.encode("utf8") - super(c_char_p, self).__init__(p) - def __str__(self): - return self.value - - @property - def value(self): - if super(c_char_p, self).value is None: - return None - return super(c_char_p, self).value.decode("utf8") - - @classmethod - def from_param(cls, param): - if isinstance(param, str): - return cls(param) - if isinstance(param, bytes): - return cls(param) - if param is None: - # Support passing null to C functions expecting char arrays - return None - raise TypeError( - "Cannot convert '{}' to '{}'".format(type(param).__name__, cls.__name__) - ) - - @staticmethod - def to_python_string(x, *args): - return x.value - def b(x): - if isinstance(x, bytes): - return x - return x.encode("utf8") +# Python 3 strings are unicode, translate them to/from utf8 for C-interop. +class c_interop_string(c_char_p): + def __init__(self, p=None): + if p is None: + p = "" + if isinstance(p, str): + p = p.encode("utf8") + super(c_char_p, self).__init__(p) -elif sys.version_info[0] == 2: - # Python 2 strings are utf8 byte strings, no translation is needed for - # C-interop. - c_interop_string = c_char_p - - def _to_python_string(x, *args): - return x - - c_interop_string.to_python_string = staticmethod(_to_python_string) + def __str__(self): + return self.value - def b(x): - return x + @property + def value(self): + if super(c_char_p, self).value is None: + return None + return super(c_char_p, self).value.decode("utf8") + @classmethod + def from_param(cls, param): + if isinstance(param, str): + return cls(param) + if isinstance(param, bytes): + return cls(param) + if param is None: + # Support passing null to C functions expecting char arrays + return None + raise TypeError( + "Cannot convert '{}' to '{}'".format(type(param).__name__, cls.__name__) + ) -# Importing ABC-s directly from collections is deprecated since Python 3.7, -# will stop working in Python 3.8. -# See: https://docs.python.org/dev/whatsnew/3.7.html#id3 -if sys.version_info[:2] >= (3, 7): - from collections import abc as collections_abc -else: - import collections as collections_abc + @staticmethod + def to_python_string(x, *args): + return x.value -# We only support PathLike objects on Python version with os.fspath present -# to be consistent with the Python standard library. On older Python versions -# we only support strings and we have dummy fspath to just pass them through. -try: - fspath = os.fspath -except AttributeError: - def fspath(x): +def b(x): + if isinstance(x, bytes): return x + return x.encode("utf8") # ctypes doesn't implicitly convert c_void_p to the appropriate wrapper @@ -202,7 +171,7 @@ def __init__(self, enumeration, message): ### Structures and Utility Classes ### -class CachedProperty(object): +class CachedProperty: """Decorator that lazy-loads the value of a property. The first time the property is accessed, the original property function is @@ -392,7 +361,7 @@ def __repr__(self): return "" % (self.start, self.end) -class Diagnostic(object): +class Diagnostic: """ A Diagnostic is a single instance of a Clang diagnostic. It includes the diagnostic severity, the message, the location the diagnostic occurred, as @@ -433,7 +402,7 @@ def spelling(self): @property def ranges(self): - class RangeIterator(object): + class RangeIterator: def __init__(self, diag): self.diag = diag @@ -449,7 +418,7 @@ def __getitem__(self, key): @property def fixits(self): - class FixItIterator(object): + class FixItIterator: def __init__(self, diag): self.diag = diag @@ -468,7 +437,7 @@ def __getitem__(self, key): @property def children(self): - class ChildDiagnosticsIterator(object): + class ChildDiagnosticsIterator: def __init__(self, diag): self.diag_set = conf.lib.clang_getChildDiagnostics(diag) @@ -532,7 +501,7 @@ def from_param(self): return self.ptr -class FixIt(object): +class FixIt: """ A FixIt represents a transformation to be applied to the source to "fix-it". The fix-it should be applied by replacing the given source range @@ -547,7 +516,7 @@ def __repr__(self): return "" % (self.range, self.value) -class TokenGroup(object): +class TokenGroup: """Helper class to facilitate token management. Tokens are allocated from libclang in chunks. They must be disposed of as a @@ -603,7 +572,7 @@ def get_tokens(tu, extent): yield token -class TokenKind(object): +class TokenKind: """Describes a specific type of a Token.""" _value_map = {} # int -> TokenKind @@ -642,7 +611,7 @@ def register(value, name): ### Cursor Kinds ### -class BaseEnumeration(object): +class BaseEnumeration: """ Common base class for named enumerations held in sync with Index.h values. @@ -2059,7 +2028,7 @@ def from_cursor_result(res, fn, args): return res -class StorageClass(object): +class StorageClass: """ Describes the storage class of a declaration """ @@ -2353,7 +2322,7 @@ def argument_types(self): container is a Type instance. """ - class ArgumentsIterator(collections_abc.Sequence): + class ArgumentsIterator(collections.abc.Sequence): def __init__(self, parent): self.parent = parent self.length = None @@ -2608,7 +2577,7 @@ def __ne__(self, other): # a void*. -class ClangObject(object): +class ClangObject: """ A helper for Clang objects. This class helps act as an intermediary for the ctypes library and the Clang CIndex library. @@ -2656,8 +2625,8 @@ class _CXUnsavedFile(Structure): } -class CompletionChunk(object): - class Kind(object): +class CompletionChunk: + class Kind: def __init__(self, name): self.name = name @@ -2747,7 +2716,7 @@ def isKindResultType(self): class CompletionString(ClangObject): - class Availability(object): + class Availability: def __init__(self, name): self.name = name @@ -2849,7 +2818,7 @@ def results(self): @property def diagnostics(self): - class DiagnosticsItr(object): + class DiagnosticsItr: def __init__(self, ccr): self.ccr = ccr @@ -3003,13 +2972,13 @@ def from_source( if hasattr(contents, "read"): contents = contents.read() contents = b(contents) - unsaved_array[i].name = b(fspath(name)) + unsaved_array[i].name = b(os.fspath(name)) unsaved_array[i].contents = contents unsaved_array[i].length = len(contents) ptr = conf.lib.clang_parseTranslationUnit( index, - fspath(filename) if filename is not None else None, + os.fspath(filename) if filename is not None else None, args_array, len(args), unsaved_array, @@ -3040,7 +3009,7 @@ def from_ast_file(cls, filename, index=None): if index is None: index = Index.create() - ptr = conf.lib.clang_createTranslationUnit(index, fspath(filename)) + ptr = conf.lib.clang_createTranslationUnit(index, os.fspath(filename)) if not ptr: raise TranslationUnitLoadError(filename) @@ -3159,7 +3128,7 @@ def diagnostics(self): Return an iterable (and indexable) object containing the diagnostics. """ - class DiagIterator(object): + class DiagIterator: def __init__(self, tu): self.tu = tu @@ -3193,7 +3162,7 @@ def reparse(self, unsaved_files=None, options=0): if hasattr(contents, "read"): contents = contents.read() contents = b(contents) - unsaved_files_array[i].name = b(fspath(name)) + unsaved_files_array[i].name = b(os.fspath(name)) unsaved_files_array[i].contents = contents unsaved_files_array[i].length = len(contents) ptr = conf.lib.clang_reparseTranslationUnit( @@ -3217,7 +3186,11 @@ def save(self, filename): """ options = conf.lib.clang_defaultSaveOptions(self) result = int( - conf.lib.clang_saveTranslationUnit(self, fspath(filename), options) + conf.lib.clang_saveTranslationUnit( + self, + os.fspath(filename), + options, + ) ) if result != 0: raise TranslationUnitSaveError(result, "Error saving TranslationUnit.") @@ -3261,12 +3234,12 @@ def codeComplete( if hasattr(contents, "read"): contents = contents.read() contents = b(contents) - unsaved_files_array[i].name = b(fspath(name)) + unsaved_files_array[i].name = b(os.fspath(name)) unsaved_files_array[i].contents = contents unsaved_files_array[i].length = len(contents) ptr = conf.lib.clang_codeCompleteAt( self, - fspath(path), + os.fspath(path), line, column, unsaved_files_array, @@ -3300,7 +3273,9 @@ class File(ClangObject): @staticmethod def from_name(translation_unit, file_name): """Retrieve a file handle within the given translation unit.""" - return File(conf.lib.clang_getFile(translation_unit, fspath(file_name))) + return File( + conf.lib.clang_getFile(translation_unit, os.fspath(file_name)), + ) @property def name(self): @@ -3328,7 +3303,7 @@ def from_result(res, fn, args): return res -class FileInclusion(object): +class FileInclusion: """ The FileInclusion class represents the inclusion of one source file by another via a '#include' directive or as the input file for the translation @@ -3377,7 +3352,7 @@ def __init__(self, enumeration, message): Exception.__init__(self, "Error %d: %s" % (enumeration, message)) -class CompileCommand(object): +class CompileCommand: """Represents the compile command used to build a file""" def __init__(self, cmd, ccmds): @@ -3409,7 +3384,7 @@ def arguments(self): yield conf.lib.clang_CompileCommand_getArg(self.cmd, i) -class CompileCommands(object): +class CompileCommands: """ CompileCommands is an iterable object containing all CompileCommand that can be used for building a specific file. @@ -3460,7 +3435,7 @@ def fromDirectory(buildDir): errorCode = c_uint() try: cdb = conf.lib.clang_CompilationDatabase_fromDirectory( - fspath(buildDir), byref(errorCode) + os.fspath(buildDir), byref(errorCode) ) except CompilationDatabaseError as e: raise CompilationDatabaseError( @@ -3474,7 +3449,7 @@ def getCompileCommands(self, filename): build filename. Returns None if filename is not found in the database. """ return conf.lib.clang_CompilationDatabase_getCompileCommands( - self, fspath(filename) + self, os.fspath(filename) ) def getAllCompileCommands(self): @@ -3865,7 +3840,7 @@ def register(item): register(f) -class Config(object): +class Config: library_path = None library_file = None compatibility_check = True @@ -3880,7 +3855,7 @@ def set_library_path(path): "any other functionalities in libclang." ) - Config.library_path = fspath(path) + Config.library_path = os.fspath(path) @staticmethod def set_library_file(filename): @@ -3891,7 +3866,7 @@ def set_library_file(filename): "any other functionalities in libclang." ) - Config.library_file = fspath(filename) + Config.library_file = os.fspath(filename) @staticmethod def set_compatibility_check(check_status): diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake index c4673c8a54c5..eee37c5e7901 100644 --- a/clang/cmake/caches/Fuchsia-stage2.cmake +++ b/clang/cmake/caches/Fuchsia-stage2.cmake @@ -6,7 +6,7 @@ set(LLVM_TARGETS_TO_BUILD X86;ARM;AArch64;RISCV CACHE STRING "") set(PACKAGE_VENDOR Fuchsia CACHE STRING "") -set(_FUCHSIA_ENABLE_PROJECTS "bolt;clang;clang-tools-extra;lld;llvm;polly") +set(_FUCHSIA_ENABLE_PROJECTS "bolt;clang;clang-tools-extra;libc;lld;llvm;polly") set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "") set(LLVM_ENABLE_BACKTRACES OFF CACHE BOOL "") @@ -22,8 +22,11 @@ set(LLVM_ENABLE_TERMINFO OFF CACHE BOOL "") set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "") set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "") set(LLVM_ENABLE_ZLIB ON CACHE BOOL "") +set(LLVM_FORCE_BUILD_RUNTIME ON CACHE BOOL "") set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "") set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "") +set(LLVM_LIBC_FULL_BUILD ON CACHE BOOL "") +set(LIBC_HDRGEN_ONLY ON CACHE BOOL "") set(LLVM_STATIC_LINK_CXX_STDLIB ON CACHE BOOL "") set(LLVM_USE_RELATIVE_PATHS_IN_FILES ON CACHE BOOL "") set(LLDB_ENABLE_CURSES OFF CACHE BOOL "") @@ -297,6 +300,39 @@ if(FUCHSIA_SDK) set(LLVM_RUNTIME_MULTILIB_hwasan+noexcept_TARGETS "aarch64-unknown-fuchsia;riscv64-unknown-fuchsia" CACHE STRING "") endif() +foreach(target riscv32-unknown-elf) + list(APPEND BUILTIN_TARGETS "${target}") + set(BUILTINS_${target}_CMAKE_SYSTEM_NAME Generic CACHE STRING "") + set(BUILTINS_${target}_CMAKE_SYSTEM_PROCESSOR RISCV CACHE STRING "") + set(BUILTINS_${target}_CMAKE_SYSROOT "" CACHE STRING "") + set(BUILTINS_${target}_CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "") + foreach(lang C;CXX;ASM) + set(BUILTINS_${target}_CMAKE_${lang}_FLAGS "--target=${target} -march=rv32imafc -mabi=ilp32f" CACHE STRING "") + endforeach() + foreach(type SHARED;MODULE;EXE) + set(BUILTINS_${target}_CMAKE_${type}_LINKER_FLAGS "-fuse-ld=lld" CACHE STRING "") + endforeach() + set(BUILTINS_${target}_COMPILER_RT_BAREMETAL_BUILD ON CACHE BOOL "") + + list(APPEND RUNTIME_TARGETS "${target}") + set(RUNTIMES_${target}_CMAKE_SYSTEM_NAME Generic CACHE STRING "") + set(RUNTIMES_${target}_CMAKE_SYSTEM_PROCESSOR RISCV CACHE STRING "") + set(RUNTIMES_${target}_CMAKE_SYSROOT "" CACHE STRING "") + set(RUNTIMES_${target}_CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "") + set(RUNTIMES_${target}_CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY CACHE STRING "") + foreach(lang C;CXX;ASM) + set(RUNTIMES_${target}_CMAKE_${lang}_FLAGS "--target=${target} -march=rv32imafc -mabi=ilp32f" CACHE STRING "") + endforeach() + foreach(type SHARED;MODULE;EXE) + set(RUNTIMES_${target}_CMAKE_${type}_LINKER_FLAGS "-fuse-ld=lld" CACHE STRING "") + endforeach() + set(RUNTIMES_${target}_LLVM_LIBC_FULL_BUILD ON CACHE BOOL "") + set(RUNTIMES_${target}_LIBC_ENABLE_USE_BY_CLANG ON CACHE BOOL "") + set(RUNTIMES_${target}_LLVM_INCLUDE_TESTS OFF CACHE BOOL "") + set(RUNTIMES_${target}_LLVM_ENABLE_ASSERTIONS OFF CACHE BOOL "") + set(RUNTIMES_${target}_LLVM_ENABLE_RUNTIMES "libc" CACHE STRING "") +endforeach() + set(LLVM_BUILTIN_TARGETS "${BUILTIN_TARGETS}" CACHE STRING "") set(LLVM_RUNTIME_TARGETS "${RUNTIME_TARGETS}" CACHE STRING "") diff --git a/clang/cmake/caches/Fuchsia.cmake b/clang/cmake/caches/Fuchsia.cmake index dad434be720d..fe925901eb3d 100644 --- a/clang/cmake/caches/Fuchsia.cmake +++ b/clang/cmake/caches/Fuchsia.cmake @@ -6,7 +6,7 @@ set(LLVM_TARGETS_TO_BUILD X86;ARM;AArch64;RISCV CACHE STRING "") set(PACKAGE_VENDOR Fuchsia CACHE STRING "") -set(_FUCHSIA_ENABLE_PROJECTS "bolt;clang;clang-tools-extra;lld;llvm;polly") +set(_FUCHSIA_ENABLE_PROJECTS "bolt;clang;clang-tools-extra;libc;lld;llvm;polly") set(LLVM_ENABLE_DIA_SDK OFF CACHE BOOL "") set(LLVM_ENABLE_LIBEDIT OFF CACHE BOOL "") @@ -18,6 +18,7 @@ set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "") set(LLVM_ENABLE_ZLIB OFF CACHE BOOL "") set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "") set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "") +set(LIBC_HDRGEN_ONLY ON CACHE BOOL "") set(LLVM_USE_RELATIVE_PATHS_IN_FILES ON CACHE BOOL "") set(LLDB_ENABLE_CURSES OFF CACHE BOOL "") set(LLDB_ENABLE_LIBEDIT OFF CACHE BOOL "") @@ -38,9 +39,25 @@ set(_FUCHSIA_BOOTSTRAP_PASSTHROUGH CURL_ROOT OpenSSL_ROOT httplib_ROOT + + # Deprecated CursesAndPanel_ROOT + + CURSES_INCLUDE_DIRS + CURSES_LIBRARIES + PANEL_LIBRARIES + + # Deprecated Terminfo_ROOT + + Terminfo_LIBRARIES + + # Deprecated LibEdit_ROOT + + LibEdit_INCLUDE_DIRS + LibEdit_LIBRARIES + FUCHSIA_ENABLE_LLDB LLDB_ENABLE_CURSES LLDB_ENABLE_LIBEDIT diff --git a/clang/cmake/caches/PGO.cmake b/clang/cmake/caches/PGO.cmake index e1d0585e453f..15bc755d110d 100644 --- a/clang/cmake/caches/PGO.cmake +++ b/clang/cmake/caches/PGO.cmake @@ -2,7 +2,7 @@ set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") set(LLVM_ENABLE_PROJECTS "clang;lld" CACHE STRING "") -set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi" CACHE STRING "") +set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "") set(LLVM_TARGETS_TO_BUILD Native CACHE STRING "") set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "") diff --git a/clang/cmake/caches/Release.cmake b/clang/cmake/caches/Release.cmake new file mode 100644 index 000000000000..1ca9138b9807 --- /dev/null +++ b/clang/cmake/caches/Release.cmake @@ -0,0 +1,93 @@ +# Plain options configure the first build. +# BOOTSTRAP_* options configure the second build. +# BOOTSTRAP_BOOTSTRAP_* options configure the third build. + +# General Options +set(LLVM_RELEASE_ENABLE_LTO THIN CACHE STRING "") +set(LLVM_RELEASE_ENABLE_PGO ON CACHE BOOL "") + +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") + +# Stage 1 Bootstrap Setup +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") +if (LLVM_RELEASE_ENABLE_PGO) + set(CLANG_BOOTSTRAP_TARGETS + generate-profdata + stage2 + stage2-clang + stage2-distribution + stage2-install + stage2-install-distribution + stage2-install-distribution-toolchain + stage2-check-all + stage2-check-llvm + stage2-check-clang + stage2-test-suite CACHE STRING "") +else() + set(CLANG_BOOTSTRAP_TARGETS + clang + check-all + check-llvm + check-clang + test-suite + stage3 + stage3-clang + stage3-check-all + stage3-check-llvm + stage3-check-clang + stage3-install + stage3-test-suite CACHE STRING "") +endif() + +# Stage 1 Options +set(STAGE1_PROJECTS "clang") +set(STAGE1_RUNTIMES "") + +if (LLVM_RELEASE_ENABLE_PGO) + list(APPEND STAGE1_PROJECTS "lld") + list(APPEND STAGE1_RUNTIMES "compiler-rt") +endif() + +set(LLVM_ENABLE_RUNTIMES ${STAGE1_RUNTIMES} CACHE STRING "") +set(LLVM_ENABLE_PROJECTS ${STAGE1_PROJECTS} CACHE STRING "") + +set(LLVM_TARGETS_TO_BUILD Native CACHE STRING "") + +# Stage 2 Bootstrap Setup +set(BOOTSTRAP_CLANG_ENABLE_BOOTSTRAP ON CACHE STRING "") +set(BOOTSTRAP_CLANG_BOOTSTRAP_TARGETS + clang + check-all + check-llvm + check-clang CACHE STRING "") + +# Stage 2 Options +set(STAGE2_PROJECTS "clang") +set(STAGE2_RUNTIMES "") + +if (LLVM_RELEASE_ENABLE_LTO OR LLVM_RELEASE_ENABLE_PGO) + list(APPEND STAGE2_PROJECTS "lld") +endif() + +if (LLVM_RELEASE_ENABLE_PGO) + set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED IR CACHE STRING "") + list(APPEND STAGE2_RUNTIMES "compiler-rt") + set(BOOTSTRAP_LLVM_ENABLE_LTO ${LLVM_RELEASE_ENABLE_LTO}) + if (LLVM_RELEASE_ENABLE_LTO) + set(BOOTSTRAP_LLVM_ENABLE_LLD ON CACHE BOOL "") + endif() +endif() + +set(BOOTSTRAP_LLVM_ENABLE_PROJECTS ${STAGE2_PROJECTS} CACHE STRING "") +set(BOOTSTRAP_LLVM_ENABLE_RUNTIMES ${STAGE2_RUNTIMES} CACHE STRING "") +if (NOT LLVM_RELEASE_ENABLE_PGO) + set(BOOTSTRAP_LLVM_TARGETS_TO_BUILD Native CACHE STRING "") +endif() + +# Stage 3 Options +set(BOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "") +set(BOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_PROJECTS "clang;lld;lldb;clang-tools-extra;bolt;polly;mlir;flang" CACHE STRING "") +set(BOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_LTO ${LLVM_RELEASE_ENABLE_LTO} CACHE STRING "") +if (LLVM_RELEASE_ENABLE_LTO) + set(BOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_LLD ON CACHE BOOL "") +endif() diff --git a/clang/docs/BoundsSafety.rst b/clang/docs/BoundsSafety.rst new file mode 100644 index 000000000000..f1837675ec9b --- /dev/null +++ b/clang/docs/BoundsSafety.rst @@ -0,0 +1,998 @@ +================================================== +``-fbounds-safety``: Enforcing bounds safety for C +================================================== + +.. contents:: + :local: + +Overview +======== + +``-fbounds-safety`` is a C extension to enforce bounds safety to prevent +out-of-bounds (OOB) memory accesses, which remain a major source of security +vulnerabilities in C. ``-fbounds-safety`` aims to eliminate this class of bugs +by turning OOB accesses into deterministic traps. + +The ``-fbounds-safety`` extension offers bounds annotations that programmers can +use to attach bounds to pointers. For example, programmers can add the +``__counted_by(N)`` annotation to parameter ``ptr``, indicating that the pointer +has ``N`` valid elements: + +.. code-block:: c + + void foo(int *__counted_by(N) ptr, size_t N); + +Using this bounds information, the compiler inserts bounds checks on every +pointer dereference, ensuring that the program does not access memory outside +the specified bounds. The compiler requires programmers to provide enough bounds +information so that the accesses can be checked at either run time or compile +time — and it rejects code if it cannot. + +The most important contribution of ``-fbounds-safety`` is how it reduces the +programmer's annotation burden by reconciling bounds annotations at ABI +boundaries with the use of implicit wide pointers (a.k.a. "fat" pointers) that +carry bounds information on local variables without the need for annotations. We +designed this model so that it preserves ABI compatibility with C while +minimizing adoption effort. + +The ``-fbounds-safety`` extension has been adopted on millions of lines of +production C code and proven to work in a consumer operating system setting. The +extension was designed to enable incremental adoption — a key requirement in +real-world settings where modifying an entire project and its dependencies all +at once is often not possible. It also addresses multiple of other practical +challenges that have made existing approaches to safer C dialects difficult to +adopt, offering these properties that make it widely adoptable in practice: + +* It is designed to preserve the Application Binary Interface (ABI). +* It interoperates well with plain C code. +* It can be adopted partially and incrementally while still providing safety + benefits. +* It is a conforming extension to C. +* Consequently, source code that adopts the extension can continue to be + compiled by toolchains that do not support the extension (CAVEAT: this still + requires inclusion of a header file macro-defining bounds annotations to + empty). +* It has a relatively low adoption cost. + +This document discusses the key designs of ``-fbounds-safety``. The document is +subject to be actively updated with a more detailed specification. The +implementation plan can be found in :doc:`BoundsSafetyImplPlans`. + + +Programming Model +================= + +Overview +-------- + +``-fbounds-safety`` ensures that pointers are not used to access memory beyond +their bounds by performing bounds checking. If a bounds check fails, the program +will deterministically trap before out-of-bounds memory is accessed. + +In our model, every pointer has an explicit or implicit bounds attribute that +determines its bounds and ensures guaranteed bounds checking. Consider the +example below where the ``__counted_by(count)`` annotation indicates that +parameter ``p`` points to a buffer of integers containing ``count`` elements. An +off-by-one error is present in the loop condition, leading to ``p[i]`` being +out-of-bounds access during the loop's final iteration. The compiler inserts a +bounds check before ``p`` is dereferenced to ensure that the access remains +within the specified bounds. + +.. code-block:: c + + void fill_array_with_indices(int *__counted_by(count) p, unsigned count) { + // off-by-one error (i < count) + for (unsigned i = 0; i <= count; ++i) { + // bounds check inserted: + // if (i >= count) trap(); + p[i] = i; + } + } + +A bounds annotation defines an invariant for the pointer type, and the model +ensures that this invariant remains true. In the example below, pointer ``p`` +annotated with ``__counted_by(count)`` must always point to a memory buffer +containing at least ``count`` elements of the pointee type. Changing the value +of ``count``, like in the example below, may violate this invariant and permit +out-of-bounds access to the pointer. To avoid this, the compiler employs +compile-time restrictions and emits run-time checks as necessary to ensure the +new count value doesn't exceed the actual length of the buffer. Section +`Maintaining correctness of bounds annotations`_ provides more details about +this programming model. + +.. code-block:: c + + int g; + + void foo(int *__counted_by(count) p, size_t count) { + count++; // may violate the invariant of __counted_by + count--; // may violate the invariant of __counted_by if count was 0. + count = g; // may violate the invariant of __counted_by + // depending on the value of `g`. + } + +The requirement to annotate all pointers with explicit bounds information could +present a significant adoption burden. To tackle this issue, the model +incorporates the concept of a "wide pointer" (a.k.a. fat pointer) – a larger +pointer that carries bounds information alongside the pointer value. Utilizing +wide pointers can potentially reduce the adoption burden, as it contains bounds +information internally and eliminates the need for explicit bounds annotations. +However, wide pointers differ from standard C pointers in their data layout, +which may result in incompatibilities with the application binary interface +(ABI). Breaking the ABI complicates interoperability with external code that has +not adopted the same programming model. + +``-fbounds-safety`` harmonizes the wide pointer and the bounds annotation +approaches to reduce the adoption burden while maintaining the ABI. In this +model, local variables of pointer type are implicitly treated as wide pointers, +allowing them to carry bounds information without requiring explicit bounds +annotations. Please note that this approach doesn't apply to function parameters +which are considered ABI-visible. As local variables are typically hidden from +the ABI, this approach has a marginal impact on it. In addition, +``-fbounds-safety`` employs compile-time restrictions to prevent implicit wide +pointers from silently breaking the ABI (see `ABI implications of default bounds +annotations`_). Pointers associated with any other variables, including function +parameters, are treated as single object pointers (i.e., ``__single``), ensuring +that they always have the tightest bounds by default and offering a strong +bounds safety guarantee. + +By implementing default bounds annotations based on ABI visibility, a +considerable portion of C code can operate without modifications within this +programming model, reducing the adoption burden. + +The rest of the section will discuss individual bounds annotations and the +programming model in more detail. + +Bounds annotations +------------------ + +Annotation for pointers to a single object +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The C language allows pointer arithmetic on arbitrary pointers and this has been +a source of many bounds safety issues. In practice, many pointers are merely +pointing to a single object and incrementing or decrementing such a pointer +immediately makes the pointer go out-of-bounds. To prevent this unsafety, +``-fbounds-safety`` provides the annotation ``__single`` that causes pointer +arithmetic on annotated pointers to be a compile time error. + +* ``__single`` : indicates that the pointer is either pointing to a single + object or null. Hence, pointers with ``__single`` do not permit pointer + arithmetic nor being subscripted with a non-zero index. Dereferencing a + ``__single`` pointer is allowed but it requires a null check. Upper and lower + bounds checks are not required because the ``__single`` pointer should point + to a valid object unless it's null. + +``__single`` is the default annotation for ABI-visible pointers. This +gives strong security guarantees in that these pointers cannot be incremented or +decremented unless they have an explicit, overriding bounds annotation that can +be used to verify the safety of the operation. The compiler issues an error when +a ``__single`` pointer is utilized for pointer arithmetic or array access, as +these operations would immediately cause the pointer to exceed its bounds. +Consequently, this prompts programmers to provide sufficient bounds information +to pointers. In the following example, the pointer on parameter p is +single-by-default, and is employed for array access. As a result, the compiler +generates an error suggesting to add ``__counted_by`` to the pointer. + +.. code-block:: c + + void fill_array_with_indices(int *p, unsigned count) { + for (unsigned i = 0; i < count; ++i) { + p[i] = i; // error + } + } + + +External bounds annotations +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +"External" bounds annotations provide a way to express a relationship between a +pointer variable and another variable (or expression) containing the bounds +information of the pointer. In the following example, ``__counted_by(count)`` +annotation expresses the bounds of parameter p using another parameter count. +This model works naturally with many C interfaces and structs because the bounds +of a pointer is often available adjacent to the pointer itself, e.g., at another +parameter of the same function prototype, or at another field of the same struct +declaration. + +.. code-block:: c + + void fill_array_with_indices(int *__counted_by(count) p, size_t count) { + // off-by-one error + for (size_t i = 0; i <= count; ++i) + p[i] = i; + } + +External bounds annotations include ``__counted_by``, ``__sized_by``, and +``__ended_by``. These annotations do not change the pointer representation, +meaning they do not have ABI implications. + +* ``__counted_by(N)`` : The pointer points to memory that contains ``N`` + elements of pointee type. ``N`` is an expression of integer type which can be + a simple reference to declaration, a constant including calls to constant + functions, or an arithmetic expression that does not have side effect. The + ``__counted_by`` annotation cannot apply to pointers to incomplete types or + types without size such as ``void *``. Instead, ``__sized_by`` can be used to + describe the byte count. +* ``__sized_by(N)`` : The pointer points to memory that contains ``N`` bytes. + Just like the argument of ``__counted_by``, ``N`` is an expression of integer + type which can be a constant, a simple reference to a declaration, or an + arithmetic expression that does not have side effects. This is mainly used for + pointers to incomplete types or types without size such as ``void *``. +* ``__ended_by(P)`` : The pointer has the upper bound of value ``P``, which is + one past the last element of the pointer. In other words, this annotation + describes a range that starts with the pointer that has this annotation and + ends with ``P`` which is the argument of the annotation. ``P`` itself may be + annotated with ``__ended_by(Q)``. In this case, the end of the range extends + to the pointer ``Q``. This is used for "iterator" support in C where you're + iterating from one pointer value to another until a final pointer value is + reached (and the final pointer value is not dereferencable). + +Accessing a pointer outside the specified bounds causes a run-time trap or a +compile-time error. Also, the model maintains correctness of bounds annotations +when the pointer and/or the related value containing the bounds information are +updated or passed as arguments. This is done by compile-time restrictions or +run-time checks (see `Maintaining correctness of bounds annotations`_ +for more detail). For instance, initializing ``buf`` with ``null`` while +assigning non-zero value to ``count``, as shown in the following example, would +violate the ``__counted_by`` annotation because a null pointer does not point to +any valid memory location. To avoid this, the compiler produces either a +compile-time error or run-time trap. + +.. code-block:: c + + void null_with_count_10(int *__counted_by(count) buf, unsigned count) { + buf = 0; + // This is not allowed as it creates a null pointer with non-zero length + count = 10; + } + +However, there are use cases where a pointer is either a null pointer or is +pointing to memory of the specified size. To support this idiom, +``-fbounds-safety`` provides ``*_or_null`` variants, +``__counted_by_or_null(N)``, ``__sized_by_or_null(N)``, and +``__ended_by_or_null(P)``. Accessing a pointer with any of these bounds +annotations will require an extra null check to avoid a null pointer +dereference. + +Internal bounds annotations +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A wide pointer (sometimes known as a "fat" pointer) is a pointer that carries +additional bounds information internally (as part of its data). The bounds +require additional storage space making wide pointers larger than normal +pointers, hence the name "wide pointer". The memory layout of a wide pointer is +equivalent to a struct with the pointer, upper bound, and (optionally) lower +bound as its fields as shown below. + +.. code-block:: c + + struct wide_pointer_datalayout { + void* pointer; // Address used for dereferences and pointer arithmetic + void* upper_bound; // Points one past the highest address that can be + // accessed + void* lower_bound; // (Optional) Points to lowest address that can be + // accessed + }; + +Even with this representational change, wide pointers act syntactically as +normal pointers to allow standard pointer operations, such as pointer +dereference (``*p``), array subscript (``p[i]``), member access (``p->``), and +pointer arithmetic, with some restrictions on bounds-unsafe uses. + +``-fbounds-safety`` has a set of "internal" bounds annotations to turn pointers +into wide pointers. These are ``__bidi_indexable`` and ``__indexable``. When a +pointer has either of these annotations, the compiler changes the pointer to the +corresponding wide pointer. This means these annotations will break the ABI and +will not be compatible with plain C, and thus they should generally not be used +in ABI surfaces. + +* ``__bidi_indexable`` : A pointer with this annotation becomes a wide pointer + to carry the upper bound and the lower bound, the layout of which is + equivalent to ``struct { T *ptr; T *upper_bound; T *lower_bound; };``. As the + name indicates, pointers with this annotation are "bidirectionally indexable", + meaning that they can be indexed with either a negative or a positive offset + and the pointers can be incremented or decremented using pointer arithmetic. A + ``__bidi_indexable`` pointer is allowed to hold an out-of-bounds pointer + value. While creating an OOB pointer is undefined behavior in C, + ``-fbounds-safety`` makes it well-defined behavior. That is, pointer + arithmetic overflow with ``__bidi_indexable`` is defined as equivalent of + two's complement integer computation, and at the LLVM IR level this means + ``getelementptr`` won't get ``inbounds`` keyword. Accessing memory using the + OOB pointer is prevented via a run-time bounds check. + +* ``__indexable`` : A pointer with this annotation becomes a wide pointer + carrying the upper bound (but no explicit lower bound), the layout of which is + equivalent to ``struct { T *ptr; T *upper_bound; };``. Since ``__indexable`` + pointers do not have a separate lower bound, the pointer value itself acts as + the lower bound. An ``__indexable`` pointer can only be incremented or indexed + in the positive direction. Indexing it in the negative direction will trigger + a compile-time error. Otherwise, the compiler inserts a run-time + check to ensure pointer arithmetic doesn't make the pointer smaller than the + original ``__indexable`` pointer (Note that ``__indexable`` doesn't have a + lower bound so the pointer value is effectively the lower bound). As pointer + arithmetic overflow will make the pointer smaller than the original pointer, + it will cause a trap at runtime. Similar to ``__bidi_indexable``, an + ``__indexable`` pointer is allowed to have a pointer value above the upper + bound and creating such a pointer is well-defined behavior. Dereferencing such + a pointer, however, will cause a run-time trap. + +* ``__bidi_indexable`` offers the best flexibility out of all the pointer + annotations in this model, as ``__bidi_indexable`` pointers can be used for + any pointer operation. However, this comes with the largest code size and + memory cost out of the available pointer annotations in this model. In some + cases, use of the ``__bidi_indexable`` annotation may be duplicating bounds + information that exists elsewhere in the program. In such cases, using + external bounds annotations may be a better choice. + +``__bidi_indexable`` is the default annotation for non-ABI visible pointers, +such as local pointer variables — that is, if the programmer does not specify +another bounds annotation, a local pointer variable is implicitly +``__bidi_indexable``. Since ``__bidi_indexable`` pointers automatically carry +bounds information and have no restrictions on kinds of pointer operations that +can be used with these pointers, most code inside a function works as is without +modification. In the example below, ``int *buf`` doesn't require manual +annotation as it's implicitly ``int *__bidi_indexable buf``, carrying the bounds +information passed from the return value of malloc, which is necessary to insert +bounds checking for ``buf[i]``. + +.. code-block:: c + + void *__sized_by(size) malloc(size_t size); + + int *__counted_by(n) get_array_with_0_to_n_1(size_t n) { + int *buf = malloc(sizeof(int) * n); + for (size_t i = 0; i < n; ++i) + buf[i] = i; + return buf; + } + +Annotations for sentinel-delimited arrays +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A C string is an array of characters. The null terminator — the first null +character ('\0') element in the array — marks the end of the string. +``-fbounds-safety`` provides ``__null_terminated`` to annotate C strings and the +generalized form ``__terminated_by(T)`` to annotate pointers and arrays with an +end marked by a sentinel value. The model prevents dereferencing a +``__terminated_by`` pointer beyond its end. Calculating the location of the end +(i.e., the address of the sentinel value), requires reading the entire array in +memory and would have some performance costs. To avoid an unintended performance +hit, the model puts some restrictions on how these pointers can be used. +``__terminated_by`` pointers cannot be indexed and can only be incremented one +element at a time. To allow these operations, the pointers must be explicitly +converted to ``__indexable`` pointers using the intrinsic function +``__unsafe_terminated_by_to_indexable(P, T)`` (or +``__unsafe_null_terminated_to_indexable(P)``) which converts the +``__terminated_by`` pointer ``P`` to an ``__indexable`` pointer. + +* ``__null_terminated`` : The pointer or array is terminated by ``NULL`` or + ``0``. Modifying the terminator or incrementing the pointer beyond it is + prevented at run time. + +* ``__terminated_by(T)`` : The pointer or array is terminated by ``T`` which is + a constant expression. Accessing or incrementing the pointer beyond the + terminator is not allowed. This is a generalization of ``__null_terminated`` + which is defined as ``__terminated_by(0)``. + +Annotation for interoperating with bounds-unsafe code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A pointer with the ``__unsafe_indexable`` annotation behaves the same as a plain +C pointer. That is, the pointer does not have any bounds information and pointer +operations are not checked. + +``__unsafe_indexable`` can be used to mark pointers from system headers or +pointers from code that has not adopted -fbounds safety. This enables +interoperation between code using ``-fbounds-safety`` and code that does not. + +Default pointer types +--------------------- + +ABI visibility and default annotations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Requiring ``-fbounds-safety`` adopters to add bounds annotations to all pointers +in the codebase would be a significant adoption burden. To avoid this and to +secure all pointers by default, ``-fbounds-safety`` applies default bounds +annotations to pointer types. +Default annotations apply to pointer types of declarations + +``-fbounds-safety`` applies default bounds annotations to pointer types used in +declarations. The default annotations are determined by the ABI visibility of +the pointer. A pointer type is ABI-visible if changing its size or +representation affects the ABI. For instance, changing the size of a type used +in a function parameter will affect the ABI and thus pointers used in function +parameters are ABI-visible pointers. On the other hand, changing the types of +local variables won't have such ABI implications. Hence, ``-fbounds-safety`` +considers the outermost pointer types of local variables as non-ABI visible. The +rest of the pointers such as nested pointer types, pointer types of global +variables, struct fields, and function prototypes are considered ABI-visible. + +All ABI-visible pointers are treated as ``__single`` by default unless annotated +otherwise. This default both preserves ABI and makes these pointers safe by +default. This behavior can be controlled with macros, i.e., +``__ptrcheck_abi_assume_*ATTR*()``, to set the default annotation for +ABI-visible pointers to be either ``__single``, ``__bidi_indexable``, +``__indexable``, or ``__unsafe_indexable``. For instance, +``__ptrcheck_abi_assume_unsafe_indexable()`` will make all ABI-visible pointers +be ``__unsafe_indexable``. Non-ABI visible pointers — the outermost pointer +types of local variables — are ``__bidi_indexable`` by default, so that these +pointers have the bounds information necessary to perform bounds checks without +the need for a manual annotation. All ``const char`` pointers or any typedefs +equivalent to ``const char`` pointers are ``__null_terminated`` by default. This +means that ``char8_t`` is ``unsigned char`` so ``const char8_t *`` won't be +``__null_terminated`` by default. Similarly, ``const wchar_t *`` won't be +``__null_terminated`` by default unless the platform defines it as ``typedef +char wchar_t``. Please note, however, that the programmers can still explicitly +use ``__null_terminated`` in any other pointers, e.g., ``char8_t +*__null_terminated``, ``wchar_t *__null_terminated``, ``int +*__null_terminated``, etc. if they should be treated as ``__null_terminated``. +The same applies to other annotations. +In system headers, the default pointer attribute for ABI-visible pointers is set +to ``__unsafe_indexable`` by default. + +The ``__ptrcheck_abi_assume_*ATTR*()`` macros are defined as pragmas in the +toolchain header (See `Portability with toolchains that do not support the +extension`_ for more details about the toolchain header): + +.. code-block:: C + +#define __ptrcheck_abi_assume_single() \ + _Pragma("clang abi_ptr_attr set(single)") + +#define __ptrcheck_abi_assume_indexable() \ + _Pragma("clang abi_ptr_attr set(indexable)") + +#define __ptrcheck_abi_assume_bidi_indexable() \ + _Pragma("clang abi_ptr_attr set(bidi_indexable)") + +#define __ptrcheck_abi_assume_unsafe_indexable() \ + _Pragma("clang abi_ptr_attr set(unsafe_indexable)") + + +ABI implications of default bounds annotations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Although simply modifying types of a local variable doesn't normally impact the +ABI, taking the address of such a modified type could create a pointer type that +has an ABI mismatch. Looking at the following example, ``int *local`` is +implicitly ``int *__bidi_indexable`` and thus the type of ``&local`` is a +pointer to ``int *__bidi_indexable``. On the other hand, in ``void foo(int +**)``, the parameter type is a pointer to ``int *__single`` (i.e., ``void +foo(int *__single *__single)``) (or a pointer to ``int *__unsafe_indexable`` if +it's from a system header). The compiler reports an error for casts between +pointers whose elements have incompatible pointer attributes. This way, +``-fbounds-safety`` prevents pointers that are implicitly ``__bidi_indexable`` +from silently escaping thereby breaking the ABI. + +.. code-block:: c + + void foo(int **); + + void bar(void) { + int *local = 0; + // error: passing 'int *__bidi_indexable*__bidi_indexable' to parameter of + // incompatible nested pointer type 'int *__single*__single' + foo(&local); + } + +A local variable may still be exposed to the ABI if ``typeof()`` takes the type +of local variable to define an interface as shown in the following example. + +.. code-block:: C + + // bar.c + void bar(int *) { ... } + + // foo.c + void foo(void) { + int *p; // implicitly `int *__bidi_indexable p` + extern void bar(typeof(p)); // creates an interface of type + // `void bar(int *__bidi_indexable)` + } + +Doing this may break the ABI if the parameter is not ``__bidi_indexable`` at the +definition of function ``bar()`` which is likely the case because parameters are +``__single`` by default without an explicit annotation. + +In order to avoid an implicitly wide pointer from silently breaking the ABI, the +compiler reports a warning when ``typeof()`` is used on an implicit wide pointer +at any ABI visible context (e.g., function prototype, struct definition, etc.). + +.. _Default pointer types in typeof: + +Default pointer types in ``typeof()`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When ``typeof()`` takes an expression, it respects the bounds annotation on +the expression type, including the bounds annotation is implcit. For example, +the global variable ``g`` in the following code is implicitly ``__single`` so +``typeof(g)`` gets ``char *__single``. The similar is true for the parameter +``p``, so ``typeof(p)`` returns ``void *__single``. The local variable ``l`` is +implicitly ``__bidi_indexable``, so ``typeof(l)`` becomes +``int *__bidi_indexable``. + +.. code-block:: C + + char *g; // typeof(g) == char *__single + + void foo(void *p) { + // typeof(p) == void *__single + + int *l; // typeof(l) == int *__bidi_indexable + } + +When the type of expression has an "external" bounds annotation, e.g., +``__sized_by``, ``__counted_by``, etc., the compiler may report an error on +``typeof`` if the annotation creates a dependency with another declaration or +variable. For example, the compiler reports an error on ``typeof(p1)`` shown in +the following code because allowing it can potentially create another type +dependent on the parameter ``size`` in a different context (Please note that an +external bounds annotation on a parameter may only refer to another parameter of +the same function). On the other hand, ``typeof(p2)`` works resulting in ``int +*__counted_by(10)``, since it doesn't depend on any other declaration. + +.. TODO: add a section describing constraints on external bounds annotations + +.. code-block:: C + + void foo(int *__counted_by(size) p1, size_t size) { + // typeof(p1) == int *__counted_by(size) + // -> a compiler error as it tries to create another type + // dependent on `size`. + + int *__counted_by(10) p2; // typeof(p2) == int *__counted_by(10) + // -> no error + + } + +When ``typeof()`` takes a type name, the compiler doesn't apply an implicit +bounds annotation on the named pointer types. For example, ``typeof(int*)`` +returns ``int *`` without any bounds annotation. A bounds annotation may be +added after the fact depending on the context. In the following example, +``typeof(int *)`` returns ``int *`` so it's equivalent as the local variable is +declared as ``int *l``, so it eventually becomes implicitly +``__bidi_indexable``. + +.. code-block:: c + + void foo(void) { + typeof(int *) l; // `int *__bidi_indexable` (same as `int *l`) + } + +The programmers can still explicitly add a bounds annotation on the types named +inside ``typeof``, e.g., ``typeof(int *__bidi_indexable)``, which evaluates to +``int *__bidi_indexable``. + + +Default pointer types in ``sizeof()`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When ``sizeof()`` takes a type name, the compiler doesn't apply an implicit +bounds annotation on the named pointer types. This means if a bounds annotation +is not specified, the evaluated pointer type is treated identically to a plain C +pointer type. Therefore, ``sizeof(int*)`` remains the same with or without +``-fbounds-safety``. That said, programmers can explicitly add attribute to the +types, e.g., ``sizeof(int *__bidi_indexable)``, in which case the sizeof +evaluates to the size of type ``int *__bidi_indexable`` (the value equivalent to +``3 * sizeof(int*)``). + +When ``sizeof()`` takes an expression, i.e., ``sizeof(expr``, it behaves as +``sizeof(typeof(expr))``, except that ``sizeof(expr)`` does not report an error +with ``expr`` that has a type with an external bounds annotation dependent on +another declaration, whereas ``typeof()`` on the same expression would be an +error as described in :ref:`Default pointer types in typeof`. +The following example describes this behavior. + +.. code-block:: c + + void foo(int *__counted_by(size) p, size_t size) { + // sizeof(p) == sizeof(int *__counted_by(size)) == sizeof(int *) + // typeof(p): error + }; + +Default pointer types in ``alignof()`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``alignof()`` only takes a type name as the argument and it doesn't take an +expression. Similar to ``sizeof()`` and ``typeof``, the compiler doesn't apply +an implicit bounds annotation on the pointer types named inside ``alignof()``. +Therefore, ``alignof(T *)`` remains the same with or without +``-fbounds-safety``, evaluating into the alignment of the raw pointer ``T *``. +The programmers can explicitly add a bounds annotation to the types, e.g., +``alignof(int *__bidi_indexable)``, which returns the alignment of ``int +*__bidi_indexable``. A bounds annotation including an internal bounds annotation +(i.e., ``__indexable`` and ``__bidi_indexable``) doesn't affect the alignment of +the original pointer. Therefore, ``alignof(int *__bidi_indexable)`` is equal to +``alignof(int *)``. + + +Default pointer types used in C-style casts +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A pointer type used in a C-style cast (e.g., ``(int *)src``) inherits the same +pointer attribute in the type of src. For instance, if the type of src is ``T +*__single`` (with ``T`` being an arbitrary C type), ``(int *)src`` will be ``int +*__single``. The reasoning behind this behavior is so that a C-style cast +doesn't introduce any unexpected side effects caused by an implicit cast of +bounds attribute. + +Pointer casts can have explicit bounds annotations. For instance, ``(int +*__bidi_indexable)src`` casts to ``int *__bidi_indexable`` as long as src has a +bounds annotation that can implicitly convert to ``__bidi_indexable``. If +``src`` has type ``int *__single``, it can implicitly convert to ``int +*__bidi_indexable`` which then will have the upper bound pointing to one past +the first element. However, if src has type ``int *__unsafe_indexable``, the +explicit cast ``(int *__bidi_indexable)src`` will cause an error because +``__unsafe_indexable`` cannot cast to ``__bidi_indexable`` as +``__unsafe_indexable`` doesn't have bounds information. `Cast rules`_ describes +in more detail what kinds of casts are allowed between pointers with different +bounds annotations. + +Default pointer types in typedef +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Pointer types in ``typedef``\s do not have implicit default bounds annotations. +Instead, the bounds annotation is determined when the ``typedef`` is used. The +following example shows that no pointer annotation is specified in the ``typedef +pint_t`` while each instance of ``typedef``'ed pointer gets its bounds +annotation based on the context in which the type is used. + +.. code-block:: c + + typedef int * pint_t; // int * + + pint_t glob; // int *__single glob; + + void foo(void) { + pint_t local; // int *__bidi_indexable local; + } + +Pointer types in a ``typedef`` can still have explicit annotations, e.g., +``typedef int *__single``, in which case the bounds annotation ``__single`` will +apply to every use of the ``typedef``. + +Array to pointer promotion to secure arrays (including VLAs) +------------------------------------------------------------ + +Arrays on function prototypes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In C, arrays on function prototypes are promoted (or "decayed") to a pointer to +its first element (e.g., ``&arr[0]``). In ``-fbounds-safety``, arrays are also +decayed to pointers, but with the addition of an implicit bounds annotation, +which includes variable-length arrays (VLAs). As shown in the following example, +arrays on function prototypes are decalyed to corresponding ``__counted_by`` +pointers. + +.. code-block:: c + + // Function prototype: void foo(int n, int *__counted_by(n) arr); + void foo(int n, int arr[n]); + + // Function prototype: void bar(int *__counted_by(10) arr); + void bar(int arr[10]); + +This means the array parameters are treated as `__counted_by` pointers within +the function and callers of the function also see them as the corresponding +`__counted_by` pointers. + +Incomplete arrays on function prototypes will cause a compiler error unless it +has ``__counted_by`` annotation in its bracket. + +.. code-block:: c + + void f1(int n, int arr[]); // error + + void f3(int n, int arr[__counted_by(n)]); // ok + + void f2(int n, int arr[n]); // ok, decays to int *__counted_by(n) + + void f4(int n, int *__counted_by(n) arr); // ok + + void f5(int n, int *arr); // ok, but decays to int *__single, + // and cannot be used for pointer arithmetic + +Array references +^^^^^^^^^^^^^^^^ + +In C, similar to arrays on the function prototypes, a reference to array is +automatically promoted (or "decayed") to a pointer to its first element (e.g., +``&arr[0]``). + +In `-fbounds-safety`, array references are promoted to ``__bidi_indexable`` +pointers which contain the upper and lower bounds of the array, with the +equivalent of ``&arr[0]`` serving as the lower bound and ``&arr[array_size]`` +(or one past the last element) serving as the upper bound. This applies to all +types of arrays including constant-length arrays, variable-length arrays (VLAs), +and flexible array members annotated with `__counted_by`. + +In the following example, reference to ``vla`` promotes to ``int +*__bidi_indexable``, with ``&vla[n]`` as the upper bound and ``&vla[0]`` as the +lower bound. Then, it's copied to ``int *p``, which is implicitly ``int +*__bidi_indexable p``. Please note that value of ``n`` used to create the upper +bound is ``10``, not ``100``, in this case because ``10`` is the actual length +of ``vla``, the value of ``n`` at the time when the array is being allocated. + +.. code-block:: c + + void foo(void) { + int n = 10; + int vla[n]; + n = 100; + int *p = vla; // { .ptr: &vla[0], .upper: &vla[10], .lower: &vla[0] } + // it's `&vla[10]` because the value of `n` was 10 at the + // time when the array is actually allocated. + // ... + } + +By promoting array references to ``__bidi_indexable``, all array accesses are +bounds checked in ``-fbounds-safety``, just as ``__bidi_indexable`` pointers +are. + +Maintaining correctness of bounds annotations +--------------------------------------------- + +``-fbounds-safety`` maintains correctness of bounds annotations by performing +additional checks when a pointer object and/or its related value containing the +bounds information is updated. + +For example, ``__single`` expresses an invariant that the pointer must either +point to a single valid object or be a null pointer. To maintain this invariant, +the compiler inserts checks when initializing a ``__single`` pointer, as shown +in the following example: + +.. code-block:: c + + void foo(void *__sized_by(size) vp, size_t size) { + // Inserted check: + // if ((int*)upper_bound(vp) - (int*)vp < sizeof(int) && !!vp) trap(); + int *__single ip = (int *)vp; + } + +Additionally, an explicit bounds annotation such as ``int *__counted_by(count) +buf`` defines a relationship between two variables, ``buf`` and ``count``: +namely, that ``buf`` has ``count`` number of elements available. This +relationship must hold even after any of these related variables are updated. To +this end, the model requires that assignments to ``buf`` and ``count`` must be +side by side, with no side effects between them. This prevents ``buf`` and +``count`` from temporarily falling out of sync due to updates happening at a +distance. + +The example below shows a function ``alloc_buf`` that initializes a struct that +members that use the ``__counted_by`` annotation. The compiler allows these +assignments because ``sbuf->buf`` and ``sbuf->count`` are updated side by side +without any side effects in between the assignments. + +Furthermore, the compiler inserts additional run-time checks to ensure the new +``buf`` has at least as many elements as the new ``count`` indicates as shown in +the transformed pseudo code of function ``alloc_buf()`` in the example below. + +.. code-block:: c + + typedef struct { + int *__counted_by(count) buf; + size_t count; + } sized_buf_t; + + void alloc_buf(sized_buf_t *sbuf, sized_t nelems) { + sbuf->buf = (int *)malloc(sizeof(int) * nelems); + sbuf->count = nelems; + } + + // Transformed pseudo code: + void alloc_buf(sized_buf_t *sbuf, sized_t nelems) { + // Materialize RHS values: + int *tmp_ptr = (int *)malloc(sizeof(int) * nelems); + int tmp_count = nelems; + // Inserted check: + // - checks to ensure that `lower <= tmp_ptr <= upper` + // - if (upper(tmp_ptr) - tmp_ptr < tmp_count) trap(); + sbuf->buf = tmp_ptr; + sbuf->count = tmp_count; + } + +Whether the compiler can optimize such run-time checks depends on how the upper +bound of the pointer is derived. If the source pointer has ``__sized_by``, +``__counted_by``, or a variant of such, the compiler assumes that the upper +bound calculation doesn't overflow, e.g., ``ptr + size`` (where the type of +``ptr`` is ``void *__sized_by(size)``), because when the ``__sized_by`` pointer +is initialized, ``-fbounds-safety`` inserts run-time checks to ensure that ``ptr ++ size`` doesn't overflow and that ``size >= 0``. + +Assuming the upper bound calculation doesn't overflow, the compiler can simplify +the trap condition ``upper(tmp_ptr) - tmp_ptr < tmp_count`` to ``size < +tmp_count`` so if both ``size`` and ``tmp_count`` values are known at compile +time such that ``0 <= tmp_count <= size``, the optimizer can remove the check. + +``ptr + size`` may still overflow if the ``__sized_by`` pointer is created from +code that doesn't enable ``-fbounds-safety``, which is undefined behavior. + +In the previous code example with the transformed ``alloc_buf()``, the upper +bound of ``tmp_ptr`` is derived from ``void *__sized_by_or_null(size)``, which +is the return type of ``malloc()``. Hence, the pointer arithmetic doesn't +overflow or ``tmp_ptr`` is null. Therefore, if ``nelems`` was given as a +compile-time constant, the compiler could remove the checks. + +Cast rules +---------- + +``-fbounds-safety`` does not enforce overall type safety and bounds invariants +can still be violated by incorrect casts in some cases. That said, +``-fbounds-safety`` prevents type conversions that change bounds attributes in a +way to violate the bounds invariant of the destination's pointer annotation. +Type conversions that change bounds attributes may be allowed if it does not +violate the invariant of the destination or that can be verified at run time. +Here are some of the important cast rules. + +Two pointers that have different bounds annotations on their nested pointer +types are incompatible and cannot implicitly cast to each other. For example, +``T *__single *__single`` cannot be converted to ``T *__bidi_indexable +*__single``. Such a conversion between incompatible nested bounds annotations +can be allowed using an explicit cast (e.g., C-style cast). Hereafter, the rules +only apply to the top pointer types. ``__unsafe_indexable`` cannot be converted +to any other safe pointer types (``__single``, ``__bidi_indexable``, +``__counted_by``, etc) using a cast. The extension provides builtins to force +this conversion, ``__unsafe_forge_bidi_indexable(type, pointer, char_count)`` to +convert pointer to a ``__bidi_indexable`` pointer of type with ``char_count`` +bytes available and ``__unsafe_forge_single(type, pointer)`` to convert pointer +to a single pointer of type type. The following examples show the usage of these +functions. Function ``example_forge_bidi()`` gets an external buffer from an +unsafe library by calling ``get_buf()`` which returns ``void +*__unsafe_indexable.`` Under the type rules, this cannot be directly assigned to +``void *buf`` (implicitly ``void *__bidi_indexable``). Thus, +``__unsafe_forge_bidi_indexable`` is used to manually create a +``__bidi_indexable`` from the unsafe buffer. + +.. code-block:: c + + // unsafe_library.h + void *__unsafe_indexable get_buf(void); + size_t get_buf_size(void); + + // my_source1.c (enables -fbounds-safety) + #include "unsafe_library.h" + void example_forge_bidi(void) { + void *buf = + __unsafe_forge_bidi_indexable(void *, get_buf(), get_buf_size()); + // ... + } + + // my_source2.c (enables -fbounds-safety) + #include + void example_forge_single(void) { + FILE *fp = __unsafe_forge_single(FILE *, fopen("mypath", "rb")); + // ... + } + +* Function ``example_forge_single`` takes a file handle by calling fopen defined + in system header ``stdio.h``. Assuming ``stdio.h`` did not adopt + ``-fbounds-safety``, the return type of ``fopen`` would implicitly be ``FILE + *__unsafe_indexable`` and thus it cannot be directly assigned to ``FILE *fp`` + in the bounds-safe source. To allow this operation, ``__unsafe_forge_single`` + is used to create a ``__single`` from the return value of ``fopen``. + +* Similar to ``__unsafe_indexable``, any non-pointer type (including ``int``, + ``intptr_t``, ``uintptr_t``, etc.) cannot be converted to any safe pointer + type because these don't have bounds information. ``__unsafe_forge_single`` or + ``__unsafe_forge_bidi_indexable`` must be used to force the conversion. + +* Any safe pointer types can cast to ``__unsafe_indexable`` because it doesn't + have any invariant to maintain. + +* ``__single`` casts to ``__bidi_indexable`` if the pointee type has a known + size. After the conversion, the resulting ``__bidi_indexable`` has the size of + a single object of the pointee type of ``__single``. ``__single`` cannot cast + to ``__bidi_indexable`` if the pointee type is incomplete or sizeless. For + example, ``void *__single`` cannot convert to ``void *__bidi_indexable`` + because void is an incomplete type and thus the compiler cannot correctly + determine the upper bound of a single void pointer. + +* Similarly, ``__single`` can cast to ``__indexable`` if the pointee type has a + known size. The resulting ``__indexable`` has the size of a single object of + the pointee type. + +* ``__single`` casts to ``__counted_by(E)`` only if ``E`` is 0 or 1. + +* ``__single`` can cast to ``__single`` including when they have different + pointee types as long as it is allowed in the underlying C standard. + ``-fbounds-safety`` doesn't guarantee type safety. + +* ``__bidi_indexable`` and ``__indexable`` can cast to ``__single``. The + compiler may insert run-time checks to ensure the pointer has at least a + single element or is a null pointer. + +* ``__bidi_indexable`` casts to ``__indexable`` if the pointer does not have an + underflow. The compiler may insert run-time checks to ensure the pointer is + not below the lower bound. + +* ``__indexable`` casts to ``__bidi_indexable``. The resulting + ``__bidi_indexable`` gets the lower bound same as the pointer value. + +* A type conversion may involve both a bitcast and a bounds annotation cast. For + example, casting from ``int *__bidi_indexable`` to ``char *__single`` involve + a bitcast (``int *`` to ``char *``) and a bounds annotation cast + (``__bidi_indexable`` to ``__single``). In this case, the compiler performs + the bitcast and then converts the bounds annotation. This means, ``int + *__bidi_indexable`` will be converted to ``char *__bidi_indexable`` and then + to ``char *__single``. + +* ``__terminated_by(T)`` cannot cast to any safe pointer type without the same + ``__terminated_by(T)`` attribute. To perform the cast, programmers can use an + intrinsic function such as ``__unsafe_terminated_by_to_indexable(P)`` to force + the conversion. + +* ``__terminated_by(T)`` can cast to ``__unsafe_indexable``. + +* Any type without ``__terminated_by(T)`` cannot cast to ``__terminated_by(T)`` + without explicitly using an intrinsic function to allow it. + + + ``__unsafe_terminated_by_from_indexable(T, PTR [, PTR_TO_TERM])`` casts any + safe pointer PTR to a ``__terminated_by(T)`` pointer. ``PTR_TO_TERM`` is an + optional argument where the programmer can provide the exact location of the + terminator. With this argument, the function can skip reading the entire + array in order to locate the end of the pointer (or the upper bound). + Providing an incorrect ``PTR_TO_TERM`` causes a run-time trap. + + + ``__unsafe_forge_terminated_by(T, P, E)`` creates ``T __terminated_by(E)`` + pointer given any pointer ``P``. Tmust be a pointer type. + +Portability with toolchains that do not support the extension +------------------------------------------------------------- + +The language model is designed so that it doesn't alter the semantics of the +original C program, other than introducing deterministic traps where otherwise +the behavior is undefined and/or unsafe. Clang provides a toolchain header +(``ptrcheck.h``) that macro-defines the annotations as type attributes when +``-fbounds-safety`` is enabled and defines them to empty when the extension is +disabled. Thus, the code adopting ``-fbounds-safety`` can compile with +toolchains that do not support this extension, by including the header or adding +macros to define the annotations to empty. For example, the toolchain not +supporting this extension may not have a header defining ``__counted_by``, so +the code using ``__counted_by`` must define it as nothing or include a header +that has the define. + +.. code-block:: c + + #if defined(__has_feature) && __has_feature(bounds_safety) + #define __counted_by(T) __attribute__((__counted_by__(T))) + // ... other bounds annotations + #else #define __counted_by(T) // defined as nothing + // ... other bounds annotations + #endif + + // expands to `void foo(int * ptr, size_t count);` + // when extension is not enabled or not available + void foo(int *__counted_by(count) ptr, size_t count); + +Other potential applications of bounds annotations +================================================== + +The bounds annotations provided by the ``-fbounds-safety`` programming model +have potential use cases beyond the language extension itself. For example, +static and dynamic analysis tools could use the bounds information to improve +diagnostics for out-of-bounds accesses, even if ``-fbounds-safety`` is not used. +The bounds annotations could be used to improve C interoperability with +bounds-safe languages, providing a better mapping to bounds-safe types in the +safe language interface. The bounds annotations can also serve as documentation +specifying the relationship between declarations. + +Limitations +=========== + +``-fbounds-safety`` aims to bring the bounds safety guarantee to the C language, +and it does not guarantee other types of memory safety properties. Consequently, +it may not prevent some of the secondary bounds safety violations caused by +other types of safety violations such as type confusion. For instance, +``-fbounds-safety`` does not perform type-safety checks on conversions between +`__single`` pointers of different pointee types (e.g., ``char *__single`` → +``void *__single`` → ``int *__single``) beyond what the foundation languages +(C/C++) already offer. + +``-fbounds-safety`` heavily relies on run-time checks to keep the bounds safety +and the soundness of the type system. This may incur significant code size +overhead in unoptimized builds and leaving some of the adoption mistakes to be +caught only at run time. This is not a fundamental limitation, however, because +incrementally adding necessary static analysis will allow us to catch issues +early on and remove unnecessary bounds checks in unoptimized builds. \ No newline at end of file diff --git a/clang/docs/BoundsSafetyImplPlans.rst b/clang/docs/BoundsSafetyImplPlans.rst new file mode 100644 index 000000000000..4fbf87f96635 --- /dev/null +++ b/clang/docs/BoundsSafetyImplPlans.rst @@ -0,0 +1,255 @@ +============================================ +Implementation plans for ``-fbounds-safety`` +============================================ + +.. contents:: + :local: + +External bounds annotations +=========================== + +The bounds annotations are C type attributes appertaining to pointer types. If +an attribute is added to the position of a declaration attribute, e.g., ``int +*ptr __counted_by(size)``, the attribute appertains to the outermost pointer +type of the declaration (``int *``). + +New sugar types +=============== + +An external bounds annotation creates a type sugar of the underlying pointer +types. We will introduce a new sugar type, ``DynamicBoundsPointerType`` to +represent ``__counted_by`` or ``__sized_by``. Using ``AttributedType`` would not +be sufficient because the type needs to hold the count or size expression as +well as some metadata necessary for analysis, while this type may be implemented +through inheritance from ``AttributedType``. Treating the annotations as type +sugars means two types with incompatible external bounds annotations may be +considered canonically the same types. This is sometimes necessary, for example, +to make the ``__counted_by`` and friends not participate in function +overloading. However, this design requires a separate logic to walk through the +entire type hierarchy to check type compatibility of bounds annotations. + +Late parsing for C +================== + +A bounds annotation such as ``__counted_by(count)`` can be added to type of a +struct field declaration where count is another field of the same struct +declared later. Similarly, the annotation may apply to type of a function +parameter declaration which precedes the parameter count in the same function. +This means parsing the argument of bounds annotations must be done after the +parser has the whole context of a struct or a function declaration. Clang has +late parsing logic for C++ declaration attributes that require late parsing, +while the C declaration attributes and C/C++ type attributes do not have the +same logic. This requires introducing late parsing logic for C/C++ type +attributes. + +Internal bounds annotations +=========================== + +``__indexable`` and ``__bidi_indexable`` alter pointer representations to be +equivalent to a struct with the pointer and the corresponding bounds fields. +Despite this difference in their representations, they are still pointers in +terms of types of operations that are allowed and their semantics. For instance, +a pointer dereference on a ``__bidi_indexable`` pointer will return the +dereferenced value same as plain C pointers, modulo the extra bounds checks +being performed before dereferencing the wide pointer. This means mapping the +wide pointers to struct types with equivalent layout won’t be sufficient. To +represent the wide pointers in Clang AST, we add an extra field in the +PointerType class to indicate the internal bounds of the pointer. This ensures +pointers of different representations are mapped to different canonical types +while they are still treated as pointers. + +In LLVM IR, wide pointers will be emitted as structs of equivalent +representations. Clang CodeGen will handle them as Aggregate in +``TypeEvaluationKind (TEK)``. ``AggExprEmitter`` was extended to handle pointer +operations returning wide pointers. Alternatively, a new ``TEK`` and an +expression emitter dedicated to wide pointers could be introduced. + +Default bounds annotations +========================== + +The model may implicitly add ``__bidi_indexable`` or ``__single`` depending on +the context of the declaration that has the pointer type. ``__bidi_indexable`` +implicitly adds to local variables, while ``__single`` implicitly adds to +pointer types specifying struct fields, function parameters, or global +variables. This means the parser may first create the pointer type without any +default pointer attribute and then recreate the type once the parser has the +declaration context and determined the default attribute accordingly. + +This also requires the parser to reset the type of the declaration with the +newly created type with the right default attribute. + +Promotion expression +==================== + +A new expression will be introduced to represent the conversion from a pointer +with an external bounds annotation, such as ``__counted_by``, to +``__bidi_indexable``. This type of conversion cannot be handled by normal +CastExprs because it requires an extra subexpression(s) to provide the bounds +information necessary to create a wide pointer. + +Bounds check expression +======================= + +Bounds checks are part of semantics defined in the ``-fbounds-safety`` language +model. Hence, exposing the bounds checks and other semantic actions in the AST +is desirable. A new expression for bounds checks has been added to the AST. The +bounds check expression has a ``BoundsCheckKind`` to indicate the kind of checks +and has the additional sub-expressions that are necessary to perform the check +according to the kind. + +Paired assignment check +======================= + +``-fbounds-safety`` enforces that variables or fields related with the same +external bounds annotation (e.g., ``buf`` and ``count`` related with +``__counted_by`` in the example below) must be updated side by side within the +same basic block and without side effect in between. + +.. code-block:: c + + typedef struct { + int *__counted_by(count) buf; size_t count; + } sized_buf_t; + + void alloc_buf(sized_buf_t *sbuf, sized_t nelems) { + sbuf->buf = (int *)malloc(sizeof(int) * nelems); + sbuf->count = nelems; + } + +To implement this rule, the compiler requires a linear representation of +statements to understand the ordering and the adjacency between the two or more +assignments. The Clang CFG is used to implement this analysis as Clang CFG +provides a linear view of statements within each ``CFGBlock`` (Clang +``CFGBlock`` represents a single basic block in a source-level CFG). + +Bounds check optimizations +========================== + +In ``-fbounds-safety``, the Clang frontend emits run-time checks for every +memory dereference if the type system or analyses in the frontend couldn’t +verify its bounds safety. The implementation relies on LLVM optimizations to +remove redundant run-time checks. Using this optimization strategy, if the +original source code already has bounds checks, the fewer additional checks +``-fbounds-safety`` will introduce. The LLVM ``ConstraintElimination`` pass is +design to remove provable redundant checks (please check Florian Hahn’s +presentation in 2021 LLVM Dev Meeting and the implementation to learn more). In +the following example, ``-fbounds-safety`` implicitly adds the redundant bounds +checks that the optimizer can remove: + +.. code-block:: c + + void fill_array_with_indices(int *__counted_by(count) p, size_t count) { + for (size_t i = 0; i < count; ++i) { + // implicit bounds checks: + // if (p + i < p || p + i + 1 > p + count) trap(); + p[i] = i; + } + } + +``ConstraintElimination`` collects the following facts and determines if the +bounds checks can be safely removed: + +* Inside the for-loop, ``0 <= i < count``, hence ``1 <= i + 1 <= count``. +* Pointer arithmetic ``p + count`` in the if-condition doesn’t wrap. +* ``-fbounds-safety`` treats pointer arithmetic overflow as deterministically + two’s complement computation, not an undefined behavior. Therefore, + getelementptr does not typically have inbounds keyword. However, the compiler + does emit inbounds for ``p + count`` in this case because + ``__counted_by(count)`` has the invariant that p has at least as many as + elements as count. Using this information, ``ConstraintElimination`` is able + to determine ``p + count`` doesn’t wrap. +* Accordingly, ``p + i`` and ``p + i + 1`` also don’t wrap. +* Therefore, ``p <= p + i`` and ``p + i + 1 <= p + count``. +* The if-condition simplifies to false and becomes dead code that the subsequent + optimization passes can remove. + +``OptRemarks`` can be utilized to provide insights into performance tuning. It +has the capability to report on checks that it cannot eliminate, possibly with +reasons, allowing programmers to adjust their code to unlock further +optimizations. + +Debugging +========= + +Internal bounds annotations +--------------------------- + +Internal bounds annotations change a pointer into a wide pointer. The debugger +needs to understand that wide pointers are essentially pointers with a struct +layout. To handle this, a wide pointer is described as a record type in the +debug info. The type name has a special name prefix (e.g., +``__bounds_safety$bidi_indexable``) which can be recognized by a debug info +consumer to provide support that goes beyond showing the internal structure of +the wide pointer. There are no DWARF extensions needed to support wide pointers. +In our implementation, LLDB recognizes wide pointer types by name and +reconstructs them as wide pointer Clang AST types for use in the expression +evaluator. + +External bounds annotations +--------------------------- + +Similar to internal bounds annotations, external bound annotations are described +as a typedef to their underlying pointer type in the debug info, and the bounds +are encoded as strings in the typedef’s name (e.g., +``__bounds_safety$counted_by:N``). + +Recognizing ``-fbounds-safety`` traps +------------------------------------- + +Clang emits debug info for ``-fbounds-safety`` traps as inlined functions, where +the function name encodes the error message. LLDB implements a frame recognizer +to surface a human-readable error cause to the end user. A debug info consumer +that is unaware of this sees an inlined function whose name encodes an error +message (e.g., : ``__bounds_safety$Bounds check failed``). + +Expression Parsing +------------------ + +In our implementation, LLDB’s expression evaluator does not enable the +``-fbounds-safety`` language option because it’s currently unable to fully +reconstruct the pointers with external bounds annotations, and also because the +evaluator operates in C++ mode, utilizing C++ reference types, while +``-fbounds-safety`` does not currently support C++. This means LLDB’s expression +evaluator can only evaluate a subset of the ``-fbounds-safety`` language model. +Specifically, it’s capable of evaluating the wide pointers that already exist in +the source code. All other expressions are evaluated according to C/C++ +semantics. + +C++ support +=========== + +C++ has multiple options to write code in a bounds-safe manner, such as +following the bounds-safety core guidelines and/or using hardened libc++ along +with the `C++ Safe Buffer model +`_. However, these +techniques may require ABI changes and may not be applicable to code +interoperating with C. When the ABI of an existing program needs to be preserved +and for headers shared between C and C++, ``-fbounds-safety`` offers a potential +solution. + +``-fbounds-safety`` is not currently supported in C++, but we believe the +general approach would be applicable for future efforts. + +Upstreaming plan +================ + +Gradual updates with experimental flag +-------------------------------------- + +The upstreaming will take place as a series of smaller PRs and we will guard our +implementation with an experimental flag ``-fexperimental-bounds-safety`` until +the usable model is fully upstreamed. Once the model is ready for use, we will +expose the flag ``-fbounds-safety``. + +Possible patch sets +------------------- + +* External bounds annotations and the (late) parsing logic. +* Internal bounds annotations (wide pointers) and their parsing logic. +* Clang code generation for wide pointers with debug information. +* Pointer cast semantics involving bounds annotations (this could be divided + into multiple sub-PRs). +* CFG analysis for pairs of related pointer and count assignments and the likes. +* Bounds check expressions in AST and the Clang code generation (this could also + be divided into multiple sub-PRs). + diff --git a/clang/docs/ClangFormat.rst b/clang/docs/ClangFormat.rst index f52f35550d03..819d9ee9f9cd 100644 --- a/clang/docs/ClangFormat.rst +++ b/clang/docs/ClangFormat.rst @@ -131,6 +131,31 @@ An easy way to create the ``.clang-format`` file is: Available style options are described in :doc:`ClangFormatStyleOptions`. +.clang-format-ignore +==================== + +You can create ``.clang-format-ignore`` files to make ``clang-format`` ignore +certain files. A ``.clang-format-ignore`` file consists of patterns of file path +names. It has the following format: + +* A blank line is skipped. +* Leading and trailing spaces of a line are trimmed. +* A line starting with a hash (``#``) is a comment. +* A non-comment line is a single pattern. +* The slash (``/``) is used as the directory separator. +* A pattern is relative to the directory of the ``.clang-format-ignore`` file + (or the root directory if the pattern starts with a slash). Patterns + containing drive names (e.g. ``C:``) are not supported. +* Patterns follow the rules specified in `POSIX 2.13.1, 2.13.2, and Rule 1 of + 2.13.3 `_. +* A pattern is negated if it starts with a bang (``!``). + +To match all files in a directory, use e.g. ``foo/bar/*``. To match all files in +the directory of the ``.clang-format-ignore`` file, use ``*``. +Multiple ``.clang-format-ignore`` files are supported similar to the +``.clang-format`` files, with a lower directory level file voiding the higher +level ones. Vim Integration =============== diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst index 3d42571e82d8..4dc0de3a90f2 100644 --- a/clang/docs/ClangFormatStyleOptions.rst +++ b/clang/docs/ClangFormatStyleOptions.rst @@ -392,6 +392,23 @@ the configuration (without a prefix: ``Auto``). a &= 2; bbb = 2; + * ``bool AlignFunctionPointers`` Only for ``AlignConsecutiveDeclarations``. Whether function pointers are + aligned. + + .. code-block:: c++ + + true: + unsigned i; + int &r; + int *p; + int (*f)(); + + false: + unsigned i; + int &r; + int *p; + int (*f)(); + * ``bool PadOperators`` Only for ``AlignConsecutiveAssignments``. Whether short assignment operators are left-padded to the same length as long ones in order to put all assignment operators to the right of the left hand side. @@ -517,6 +534,23 @@ the configuration (without a prefix: ``Auto``). a &= 2; bbb = 2; + * ``bool AlignFunctionPointers`` Only for ``AlignConsecutiveDeclarations``. Whether function pointers are + aligned. + + .. code-block:: c++ + + true: + unsigned i; + int &r; + int *p; + int (*f)(); + + false: + unsigned i; + int &r; + int *p; + int (*f)(); + * ``bool PadOperators`` Only for ``AlignConsecutiveAssignments``. Whether short assignment operators are left-padded to the same length as long ones in order to put all assignment operators to the right of the left hand side. @@ -642,6 +676,23 @@ the configuration (without a prefix: ``Auto``). a &= 2; bbb = 2; + * ``bool AlignFunctionPointers`` Only for ``AlignConsecutiveDeclarations``. Whether function pointers are + aligned. + + .. code-block:: c++ + + true: + unsigned i; + int &r; + int *p; + int (*f)(); + + false: + unsigned i; + int &r; + int *p; + int (*f)(); + * ``bool PadOperators`` Only for ``AlignConsecutiveAssignments``. Whether short assignment operators are left-padded to the same length as long ones in order to put all assignment operators to the right of the left hand side. @@ -768,6 +819,23 @@ the configuration (without a prefix: ``Auto``). a &= 2; bbb = 2; + * ``bool AlignFunctionPointers`` Only for ``AlignConsecutiveDeclarations``. Whether function pointers are + aligned. + + .. code-block:: c++ + + true: + unsigned i; + int &r; + int *p; + int (*f)(); + + false: + unsigned i; + int &r; + int *p; + int (*f)(); + * ``bool PadOperators`` Only for ``AlignConsecutiveAssignments``. Whether short assignment operators are left-padded to the same length as long ones in order to put all assignment operators to the right of the left hand side. @@ -4364,6 +4432,11 @@ the configuration (without a prefix: ``Auto``). **PenaltyBreakOpenParenthesis** (``Unsigned``) :versionbadge:`clang-format 14` :ref:`¶ ` The penalty for breaking after ``(``. +.. _PenaltyBreakScopeResolution: + +**PenaltyBreakScopeResolution** (``Unsigned``) :versionbadge:`clang-format 18` :ref:`¶ ` + The penalty for breaking after ``::``. + .. _PenaltyBreakString: **PenaltyBreakString** (``Unsigned``) :versionbadge:`clang-format 3.7` :ref:`¶ ` @@ -4926,6 +4999,11 @@ the configuration (without a prefix: ``Auto``). int bar; int bar; } // namespace b } // namespace b +.. _SkipMacroDefinitionBody: + +**SkipMacroDefinitionBody** (``Boolean``) :versionbadge:`clang-format 18` :ref:`¶ ` + Do not format macro definition body. + .. _SortIncludes: **SortIncludes** (``SortIncludesOptions``) :versionbadge:`clang-format 3.8` :ref:`¶ ` diff --git a/clang/docs/CommandGuide/clang.rst b/clang/docs/CommandGuide/clang.rst index e1c872cdc553..643365215f87 100644 --- a/clang/docs/CommandGuide/clang.rst +++ b/clang/docs/CommandGuide/clang.rst @@ -302,7 +302,7 @@ Language Selection and Mode Options .. option:: -fmsc-version= Set ``_MSC_VER``. When on Windows, this defaults to either the same value as - the currently installed version of cl.exe, or ``1920``. Not set otherwise. + the currently installed version of cl.exe, or ``1933``. Not set otherwise. .. option:: -fborland-extensions diff --git a/clang/docs/HLSL/FunctionCalls.rst b/clang/docs/HLSL/FunctionCalls.rst new file mode 100644 index 000000000000..7317de2163f8 --- /dev/null +++ b/clang/docs/HLSL/FunctionCalls.rst @@ -0,0 +1,321 @@ +=================== +HLSL Function Calls +=================== + +.. contents:: + :local: + +Introduction +============ + +This document describes the design and implementation of HLSL's function call +semantics in Clang. This includes details related to argument conversion and +parameter lifetimes. + +This document does not seek to serve as official documentation for HLSL's +call semantics, but does provide an overview to assist a reader. The +authoritative documentation for HLSL's language semantics is the `draft language +specification `_. + +Argument Semantics +================== + +In HLSL, all function arguments are passed by value in and out of functions. +HLSL has 3 keywords which denote the parameter semantics (``in``, ``out`` and +``inout``). In a function declaration a parameter may be annotated any of the +following ways: + +#. - denotes input +#. ``in`` - denotes input +#. ``out`` - denotes output +#. ``in out`` - denotes input and output +#. ``out in`` - denotes input and output +#. ``inout`` - denotes input and output + +Parameters that are exclusively input behave like C/C++ parameters that are +passed by value. + +For parameters that are output (or input and output), a temporary value is +created in the caller. The temporary value is then passed by-address. For +output-only parameters, the temporary is uninitialized when passed (if the +parameter is not explicitly initialized inside the function an undefined value +is stored back to the argument expression). For parameters that are both input +and output, the temporary is initialized from the lvalue argument expression +through implicit or explicit casting from the lvalue argument type to the +parameter type. + +On return of the function, the values of any parameter temporaries are written +back to the argument expression through an inverted conversion sequence (if an +``out`` parameter was not initialized in the function, the uninitialized value +may be written back). + +Parameters of constant-sized array type are also passed with value semantics. +This requires input parameters of arrays to construct temporaries and the +temporaries go through array-to-pointer decay when initializing parameters. + +Implementations are allowed to avoid unnecessary temporaries, and HLSL's strict +no-alias rules can enable some trivial optimizations. + +Array Temporaries +----------------- + +Given the following example: + +.. code-block:: c++ + + void fn(float a[4]) { + a[0] = a[1] + a[2] + a[3]; + } + + float4 main() : SV_Target { + float arr[4] = {1, 1, 1, 1}; + fn(arr); + return float4(arr[0], arr[1], arr[2], arr[3]); + } + +In C or C++, the array parameter decays to a pointer, so after the call to +``fn``, the value of ``arr[0]`` is ``3``. In HLSL, the array is passed by value, +so modifications inside ``fn`` do not propagate out. + +.. note:: + + DXC may pass unsized arrays directly as decayed pointers, which is an + unfortunate behavior divergence. + +Out Parameter Temporaries +------------------------- + +.. code-block:: c++ + + void Init(inout int X, inout int Y) { + Y = 2; + X = 1; + } + + void main() { + int V; + Init(V, V); // MSVC (or clang-cl) V == 2, Clang V == 1 + } + +In the above example the ``Init`` function's behavior depends on the C++ +implementation. C++ does not define the order in which parameters are +initialized or destroyed. In MSVC and Clang's MSVC compatibility mode, arguments +are emitted right-to-left and destroyed left-to-right. This means that the +parameter initialization and destruction occurs in the order: {``Y``, ``X``, +``~X``, ``~Y``}. This causes the write-back of the value of ``Y`` to occur last, +so the resulting value of ``V`` is ``2``. In the Itanium C++ ABI, the parameter +ordering is reversed, so the initialization and destruction occurs in the order: +{``X``, ``Y``, ``~Y``, ``X``}. This causes the write-back of the value ``X`` to +occur last, resulting in the value of ``V`` being set to ``1``. + +.. code-block:: c++ + + void Trunc(inout int3 V) { } + + + void main() { + float3 F = {1.5, 2.6, 3.3}; + Trunc(F); // F == {1.0, 2.0, 3.0} + } + +In the above example, the argument expression ``F`` undergoes element-wise +conversion from a float vector to an integer vector to create a temporary +``int3``. On expiration the temporary undergoes elementwise conversion back to +the floating point vector type ``float3``. This results in an implicit +element-wise conversion of the vector even if the value is unused in the +function (effectively truncating the floating point values). + + +.. code-block:: c++ + + void UB(out int X) {} + + void main() { + int X = 7; + UB(X); // X is undefined! + } + +In this example an initialized value is passed to an ``out`` parameter. +Parameters marked ``out`` are not initialized by the argument expression or +implicitly by the function. They must be explicitly initialized. In this case +the argument is not initialized in the function so the temporary is still +uninitialized when it is copied back to the argument expression. This is +undefined behavior in HLSL, and any use of the argument after the call is a use +of an undefined value which may be illegal in the target (DXIL programs with +used or potentially used ``undef`` or ``poison`` values fail validation). + +Clang Implementation +==================== + +.. note:: + + The implementation described here is a proposal. It has not yet been fully + implemented, so the current state of Clang's sources may not reflect this + design. A prototype implementation was built on DXC which is Clang-3.7 based. + The prototype can be found + `here `_. A lot + of the changes in the prototype implementation are restoring Clang-3.7 code + that was previously modified to its original state. + +The implementation in clang depends on two new AST nodes and minor extensions to +Clang's existing support for Objective-C write-back arguments. The goal of this +design is to capture the semantic details of HLSL function calls in the AST, and +minimize the amount of magic that needs to occur during IR generation. + +The two new AST nodes are ``HLSLArrayTemporaryExpr`` and ``HLSLOutParamExpr``, +which respectively represent the temporaries used for passing arrays by value +and the temporaries created for function outputs. + +Array Temporaries +----------------- + +The ``HLSLArrayTemporaryExpr`` represents temporary values for input +constant-sized array arguments. This applies for all constant-sized array +arguments regardless of whether or not the parameter is constant-sized or +unsized. + +.. code-block:: c++ + + void SizedArray(float a[4]); + void UnsizedArray(float a[]); + + void main() { + float arr[4] = {1, 1, 1, 1}; + SizedArray(arr); + UnsizedArray(arr); + } + +In the example above, the following AST is generated for the call to +``SizedArray``: + +.. code-block:: text + + CallExpr 'void' + |-ImplicitCastExpr 'void (*)(float [4])' + | `-DeclRefExpr 'void (float [4])' lvalue Function 'SizedArray' 'void (float [4])' + `-HLSLArrayTemporaryExpr 'float [4]' + `-DeclRefExpr 'float [4]' lvalue Var 'arr' 'float [4]' + +In the example above, the following AST is generated for the call to +``UnsizedArray``: + +.. code-block:: text + + CallExpr 'void' + |-ImplicitCastExpr 'void (*)(float [])' + | `-DeclRefExpr 'void (float [])' lvalue Function 'UnsizedArray' 'void (float [])' + `-HLSLArrayTemporaryExpr 'float [4]' + `-DeclRefExpr 'float [4]' lvalue Var 'arr' 'float [4]' + +In both of these cases the argument expression is of known array size so we can +initialize an appropriately sized temporary. + +It is illegal in HLSL to convert an unsized array to a sized array: + +.. code-block:: c++ + + void SizedArray(float a[4]); + void UnsizedArray(float a[]) { + SizedArray(a); // Cannot convert float[] to float[4] + } + +When converting a sized array to an unsized array, an array temporary can also +be inserted. Given the following code: + +.. code-block:: c++ + + void UnsizedArray(float a[]); + void SizedArray(float a[4]) { + UnsizedArray(a); + } + +An expected AST should be something like: + +.. code-block:: text + + CallExpr 'void' + |-ImplicitCastExpr 'void (*)(float [])' + | `-DeclRefExpr 'void (float [])' lvalue Function 'UnsizedArray' 'void (float [])' + `-HLSLArrayTemporaryExpr 'float [4]' + `-DeclRefExpr 'float [4]' lvalue Var 'arr' 'float [4]' + +Out Parameter Temporaries +------------------------- + +Output parameters are defined in HLSL as *casting expiring values* (cx-values), +which is a term made up for HLSL. A cx-value is a temporary value which may be +the result of a cast, and stores its value back to an lvalue when the value +expires. + +To represent this concept in Clang we introduce a new ``HLSLOutParamExpr``. An +``HLSLOutParamExpr`` has two forms, one with a single sub-expression and one +with two sub-expressions. + +The single sub-expression form is used when the argument expression and the +function parameter are the same type, so no cast is required. As in this +example: + +.. code-block:: c++ + + void Init(inout int X) { + X = 1; + } + + void main() { + int V; + Init(V); + } + +The expected AST formulation for this code would be something like: + +.. code-block:: text + + CallExpr 'void' + |-ImplicitCastExpr 'void (*)(int &)' + | `-DeclRefExpr 'void (int &)' lvalue Function 'Init' 'void (int &)' + |-HLSLOutParamExpr 'int' lvalue inout + `-DeclRefExpr 'int' lvalue Var 'V' 'int' + +The ``HLSLOutParamExpr`` captures that the value is ``inout`` vs ``out`` to +denote whether or not the temporary is initialized from the sub-expression. If +no casting is required the sub-expression denotes the lvalue expression that the +cx-value will be copied to when the value expires. + +The two sub-expression form of the AST node is required when the argument type +is not the same as the parameter type. Given this example: + +.. code-block:: c++ + + void Trunc(inout int3 V) { } + + + void main() { + float3 F = {1.5, 2.6, 3.3}; + Trunc(F); + } + +For this case the ``HLSLOutParamExpr`` will have sub-expressions to record both +casting expression sequences for the initialization and write back: + +.. code-block:: text + + -CallExpr 'void' + |-ImplicitCastExpr 'void (*)(int3 &)' + | `-DeclRefExpr 'void (int3 &)' lvalue Function 'inc_i32' 'void (int3 &)' + `-HLSLOutParamExpr 'int3' lvalue inout + |-ImplicitCastExpr 'float3' + | `-ImplicitCastExpr 'int3' + | `-OpaqueValueExpr 'int3' lvalue + `-ImplicitCastExpr 'int3' + `-ImplicitCastExpr 'float3' + `-DeclRefExpr 'float3' lvalue 'F' 'float3' + +In this formation the write-back casts are captured as the first sub-expression +and they cast from an ``OpaqueValueExpr``. In IR generation we can use the +``OpaqueValueExpr`` as a placeholder for the ``HLSLOutParamExpr``'s temporary +value on function return. + +In code generation this can be implemented with some targeted extensions to the +Objective-C write-back support. Specifically extending CGCall.cpp's +``EmitWriteback`` function to support casting expressions and emission of +aggregate lvalues. diff --git a/clang/docs/HLSL/HLSLDocs.rst b/clang/docs/HLSL/HLSLDocs.rst index a02dd2e8a962..1f232129548d 100644 --- a/clang/docs/HLSL/HLSLDocs.rst +++ b/clang/docs/HLSL/HLSLDocs.rst @@ -14,3 +14,4 @@ HLSL Design and Implementation HLSLIRReference ResourceTypes EntryFunctions + FunctionCalls diff --git a/clang/docs/InternalsManual.rst b/clang/docs/InternalsManual.rst index 05fadf5a0344..b3e2b870ae5f 100644 --- a/clang/docs/InternalsManual.rst +++ b/clang/docs/InternalsManual.rst @@ -931,7 +931,7 @@ the option appears on the command line, the argument value is simply copied. .. code-block:: text def isysroot : JoinedOrSeparate<["-"], "isysroot">, - Visibility<[ClangOption, CC1Option]>, + Visibility<[ClangOption, CC1Option, FlangOption]>, MarshallingInfoString, [{"/"}]>; **List of Strings** @@ -3364,7 +3364,7 @@ Multiple occurrences accumulate prefixes. For example, Specifying Diagnostics ^^^^^^^^^^^^^^^^^^^^^^ -Indicating that a line expects an error or a warning is simple. Put a comment +Indicating that a line expects an error or a warning is easy. Put a comment on the line that has the diagnostic, use ``expected-{error,warning,remark,note}`` to tag if it's an expected error, warning, remark, or note (respectively), and place the expected text between @@ -3373,6 +3373,9 @@ enough to ensure that the correct diagnostic was emitted. (Note: full text should be included in test cases unless there is a compelling reason to use truncated text instead.) +For a full description of the matching behavior, including more complex +matching scenarios, see :ref:`matching ` below. + Here's an example of the most commonly used way to specify expected diagnostics: @@ -3458,8 +3461,33 @@ A range can also be specified by ``-``. For example: In this example, the diagnostic may appear only once, if at all. +.. _DiagnosticMatching: + +Matching Modes +~~~~~~~~~~~~~~ + +The default matching mode is simple string, which looks for the expected text +that appears between the first `{{` and `}}` pair of the comment. The string is +interpreted just as-is, with one exception: the sequence `\n` is converted to a +single newline character. This mode matches the emitted diagnostic when the +text appears as a substring at any position of the emitted message. + +To enable matching against desired strings that contain `}}` or `{{`, the +string-mode parser accepts opening delimiters of more than two curly braces, +like `{{{`. It then looks for a closing delimiter of equal "width" (i.e `}}}`). +For example: + +.. code-block:: c++ + + // expected-note {{{evaluates to '{{2, 3, 4}} == {0, 3, 4}'}}} + +The intent is to allow the delimeter to be wider than the longest `{` or `}` +brace sequence in the content, so that if your expected text contains `{{{` +(three braces) it may be delimited with `{{{{` (four braces), and so on. + Regex matching mode may be selected by appending ``-re`` to the diagnostic type -and including regexes wrapped in double curly braces in the directive, such as: +and including regexes wrapped in double curly braces (`{{` and `}}`) in the +directive, such as: .. code-block:: text @@ -3471,6 +3499,8 @@ Examples matching error: "variable has incomplete type 'struct s'" // expected-error {{variable has incomplete type 'struct s'}} // expected-error {{variable has incomplete type}} + // expected-error {{{variable has incomplete type}}} + // expected-error {{{{variable has incomplete type}}}} // expected-error-re {{variable has type 'struct {{.}}'}} // expected-error-re {{variable has type 'struct {{.*}}'}} diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst index 23a7f4f5d5b9..c1420079f751 100644 --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -829,6 +829,7 @@ to ``float``; see below for more information on this emulation. see below. * ``_Float16`` is supported on the following targets: + * 32-bit ARM (natively on some architecture versions) * 64-bit ARM (AArch64) (natively on ARMv8.2a and above) * AMDGPU (natively) @@ -837,6 +838,7 @@ to ``float``; see below for more information on this emulation. * RISC-V (natively if Zfh or Zhinx is available) * ``__bf16`` is supported on the following targets (currently never natively): + * 32-bit ARM * 64-bit ARM (AArch64) * RISC-V diff --git a/clang/docs/LibASTMatchersReference.html b/clang/docs/LibASTMatchersReference.html index fcd3114bb523..c40d679e383b 100644 --- a/clang/docs/LibASTMatchersReference.html +++ b/clang/docs/LibASTMatchersReference.html @@ -1629,6 +1629,17 @@

Node Matchers

+Matcher<Stmt>cxxFoldExprMatcher<CXXFoldExpr>... +
Matches C++17 fold expressions.
+
+Example matches `(0 + ... + args)`:
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
+
+ + Matcher<Stmt>cxxForRangeStmtMatcher<CXXForRangeStmt>... 
Matches range-based for statements.
 
@@ -2965,11 +2976,18 @@ 
Narrowing Matchers

 
 
 Matcher<BinaryOperator>hasOperatorNamestd::string Name
-
Matches the operator Name of operator expressions (binary or
-unary).
+
Matches the operator Name of operator expressions and fold expressions
+(binary or unary).
 
 Example matches a || b (matcher = binaryOperator(hasOperatorName("||")))
   !(a || b)
+
+Example matches `(0 + ... + args)`
+    (matcher = cxxFoldExpr(hasOperatorName("+")))
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
 

 
 
@@ -3430,6 +3448,91 @@ 
Narrowing Matchers

 

 
 
+Matcher<CXXFoldExpr>hasOperatorNamestd::string Name
+
Matches the operator Name of operator expressions and fold expressions
+(binary or unary).
+
+Example matches a || b (matcher = binaryOperator(hasOperatorName("||")))
+  !(a || b)
+
+Example matches `(0 + ... + args)`
+    (matcher = cxxFoldExpr(hasOperatorName("+")))
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
+

+
+
+Matcher<CXXFoldExpr>isBinaryFold
+
Matches binary fold expressions, i.e. fold expressions with an initializer.
+
+Example matches `(0 + ... + args)`
+    (matcher = cxxFoldExpr(isBinaryFold()))
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
+
+  template <typename... Args>
+  auto multiply(Args... args) {
+      return (args * ...);
+  }
+

+
+
+Matcher<CXXFoldExpr>isLeftFold
+
Matches left-folding fold expressions.
+
+Example matches `(0 + ... + args)`
+    (matcher = cxxFoldExpr(isLeftFold()))
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
+
+  template <typename... Args>
+  auto multiply(Args... args) {
+      return (args * ... * 1);
+  }
+

+
+
+Matcher<CXXFoldExpr>isRightFold
+
Matches right-folding fold expressions.
+
+Example matches `(args * ... * 1)`
+    (matcher = cxxFoldExpr(isRightFold()))
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
+
+  template <typename... Args>
+  auto multiply(Args... args) {
+      return (args * ... * 1);
+  }
+

+
+
+Matcher<CXXFoldExpr>isUnaryFold
+
Matches unary fold expressions, i.e. fold expressions without an
+initializer.
+
+Example matches `(args * ...)`
+    (matcher = cxxFoldExpr(isUnaryFold()))
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
+
+  template <typename... Args>
+  auto multiply(Args... args) {
+      return (args * ...);
+  }
+

+
+
 Matcher<CXXMethodDecl>isConst
 
Matches if the given method declaration is const.
 
@@ -3599,11 +3702,18 @@ 
Narrowing Matchers

 
 
 Matcher<CXXOperatorCallExpr>hasOperatorNamestd::string Name
-
Matches the operator Name of operator expressions (binary or
-unary).
+
Matches the operator Name of operator expressions and fold expressions
+(binary or unary).
 
 Example matches a || b (matcher = binaryOperator(hasOperatorName("||")))
   !(a || b)
+
+Example matches `(0 + ... + args)`
+    (matcher = cxxFoldExpr(hasOperatorName("+")))
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
 

 
 
@@ -3757,11 +3867,18 @@ 
Narrowing Matchers

 
 
 Matcher<CXXRewrittenBinaryOperator>hasOperatorNamestd::string Name
-
Matches the operator Name of operator expressions (binary or
-unary).
+
Matches the operator Name of operator expressions and fold expressions
+(binary or unary).
 
 Example matches a || b (matcher = binaryOperator(hasOperatorName("||")))
   !(a || b)
+
+Example matches `(0 + ... + args)`
+    (matcher = cxxFoldExpr(hasOperatorName("+")))
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
 

 
 
@@ -5711,12 +5828,19 @@ 
Narrowing Matchers

 

 
 
-Matcher<UnaryOperator>hasOperatorNamestd::string Name
-
Matches the operator Name of operator expressions (binary or
-unary).
+Matcher<UnaryOperator>hasOperatorNamestd::string Name
+
Matches the operator Name of operator expressions and fold expressions
+(binary or unary).
 
 Example matches a || b (matcher = binaryOperator(hasOperatorName("||")))
   !(a || b)
+
+Example matches `(0 + ... + args)`
+    (matcher = cxxFoldExpr(hasOperatorName("+")))
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
 

 
 
@@ -6453,7 +6577,7 @@ 
AST Traversal Matchers

 
 Matcher<BinaryOperator>hasEitherOperandMatcher<Expr> InnerMatcher
 
Matches if either the left hand side or the right hand side of a
-binary operator matches.
+binary operator or fold expression matches.
 

 
 
@@ -6466,7 +6590,8 @@ 
AST Traversal Matchers

 
 
 Matcher<BinaryOperator>hasOperandsMatcher<Expr> Matcher1, Matcher<Expr> Matcher2
-
Matches if both matchers match with opposite sides of the binary operator.
+
Matches if both matchers match with opposite sides of the binary operator
+or fold expression.
 
 Example matcher = binaryOperator(hasOperands(integerLiteral(equals(1),
                                              integerLiteral(equals(2)))
@@ -6885,6 +7010,112 @@ 
AST Traversal Matchers

 

 
 
+Matcher<CXXFoldExpr>calleeMatcher<Stmt> InnerMatcher
+
Matches if the call or fold expression's callee expression matches.
+
+Given
+  class Y { void x() { this->x(); x(); Y y; y.x(); } };
+  void f() { f(); }
+callExpr(callee(expr()))
+  matches this->x(), x(), y.x(), f()
+with callee(...)
+  matching this->x, x, y.x, f respectively
+
+Given
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
+
+  template <typename... Args>
+  auto multiply(Args... args) {
+      return (args * ... * 1);
+  }
+cxxFoldExpr(callee(expr()))
+  matches (args * ... * 1)
+with callee(...)
+  matching *
+
+Note: Callee cannot take the more general internal::Matcher<Expr>
+because this introduces ambiguous overloads with calls to Callee taking a
+internal::Matcher<Decl>, as the matcher hierarchy is purely
+implemented in terms of implicit casts.
+

+
+
+Matcher<CXXFoldExpr>hasEitherOperandMatcher<Expr> InnerMatcher
+
Matches if either the left hand side or the right hand side of a
+binary operator or fold expression matches.
+

+
+
+Matcher<CXXFoldExpr>hasFoldInitast_matchers::Matcher<Expr> InnerMacher
+
Matches the operand that does not contain the parameter pack.
+
+Example matches `(0 + ... + args)` and `(args * ... * 1)`
+    (matcher = cxxFoldExpr(hasFoldInit(expr())))
+  with hasFoldInit(...)
+    matching `0` and `1` respectively
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
+
+  template <typename... Args>
+  auto multiply(Args... args) {
+      return (args * ... * 1);
+  }
+

+
+
+Matcher<CXXFoldExpr>hasLHSMatcher<Expr> InnerMatcher
+
Matches the left hand side of binary operator expressions.
+
+Example matches a (matcher = binaryOperator(hasLHS()))
+  a || b
+

+
+
+Matcher<CXXFoldExpr>hasOperandsMatcher<Expr> Matcher1, Matcher<Expr> Matcher2
+
Matches if both matchers match with opposite sides of the binary operator
+or fold expression.
+
+Example matcher = binaryOperator(hasOperands(integerLiteral(equals(1),
+                                             integerLiteral(equals(2)))
+  1 + 2 // Match
+  2 + 1 // Match
+  1 + 1 // No match
+  2 + 2 // No match
+

+
+
+Matcher<CXXFoldExpr>hasPatternast_matchers::Matcher<Expr> InnerMacher
+
Matches the operand that contains the parameter pack.
+
+Example matches `(0 + ... + args)`
+    (matcher = cxxFoldExpr(hasPattern(expr())))
+  with hasPattern(...)
+    matching `args`
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
+
+  template <typename... Args>
+  auto multiply(Args... args) {
+      return (args * ... * 1);
+  }
+

+
+
+Matcher<CXXFoldExpr>hasRHSMatcher<Expr> InnerMatcher
+
Matches the right hand side of binary operator expressions.
+
+Example matches b (matcher = binaryOperator(hasRHS()))
+  a || b
+

+
+
 Matcher<CXXForRangeStmt>hasBodyMatcher<Stmt> InnerMatcher
 
Matches a 'for', 'while', 'while' statement or a function or coroutine
 definition that has a given body. Note that in case of functions or
@@ -7179,7 +7410,7 @@ 
AST Traversal Matchers

 
 Matcher<CXXOperatorCallExpr>hasEitherOperandMatcher<Expr> InnerMatcher
 
Matches if either the left hand side or the right hand side of a
-binary operator matches.
+binary operator or fold expression matches.
 

 
 
@@ -7192,7 +7423,8 @@ 
AST Traversal Matchers

 
 
 Matcher<CXXOperatorCallExpr>hasOperandsMatcher<Expr> Matcher1, Matcher<Expr> Matcher2
-
Matches if both matchers match with opposite sides of the binary operator.
+
Matches if both matchers match with opposite sides of the binary operator
+or fold expression.
 
 Example matcher = binaryOperator(hasOperands(integerLiteral(equals(1),
                                              integerLiteral(equals(2)))
@@ -7317,9 +7549,9 @@ 
AST Traversal Matchers

 

 
 
-Matcher<CXXRewrittenBinaryOperator>hasEitherOperandMatcher<Expr> InnerMatcher
-
Matches if either the left hand side or the right hand side of a
-binary operator matches.
+Matcher<CXXRewrittenBinaryOperator>hasEitherOperandMatcher<Expr> InnerMatcher
+
Matches if either the left hand side or the right hand side of a
+binary operator or fold expression matches.
 

 
 
@@ -7331,8 +7563,9 @@ 
AST Traversal Matchers

 

 
 
-Matcher<CXXRewrittenBinaryOperator>hasOperandsMatcher<Expr> Matcher1, Matcher<Expr> Matcher2
-
Matches if both matchers match with opposite sides of the binary operator.
+Matcher<CXXRewrittenBinaryOperator>hasOperandsMatcher<Expr> Matcher1, Matcher<Expr> Matcher2
+
Matches if both matchers match with opposite sides of the binary operator
+or fold expression.
 
 Example matcher = binaryOperator(hasOperands(integerLiteral(equals(1),
                                              integerLiteral(equals(2)))
@@ -7436,8 +7669,8 @@ 
AST Traversal Matchers

 

 
 
-Matcher<CallExpr>calleeMatcher<Decl> InnerMatcher
-
Matches 1) if the call expression's callee's declaration matches the
+Matcher<CallExpr>calleeMatcher<Decl> InnerMatcher
+
Matches 1) if the call expression's callee's declaration matches the
 given matcher; or 2) if the Obj-C message expression's callee's method
 declaration matches the given matcher.
 
@@ -7458,7 +7691,7 @@ 
AST Traversal Matchers

 
 
 Matcher<CallExpr>calleeMatcher<Stmt> InnerMatcher
-
Matches if the call expression's callee expression matches.
+
Matches if the call or fold expression's callee expression matches.
 
 Given
   class Y { void x() { this->x(); x(); Y y; y.x(); } };
@@ -7468,6 +7701,21 @@ 
AST Traversal Matchers

 with callee(...)
   matching this->x, x, y.x, f respectively
 
+Given
+  template <typename... Args>
+  auto sum(Args... args) {
+      return (0 + ... + args);
+  }
+
+  template <typename... Args>
+  auto multiply(Args... args) {
+      return (args * ... * 1);
+  }
+cxxFoldExpr(callee(expr()))
+  matches (args * ... * 1)
+with callee(...)
+  matching *
+
 Note: Callee cannot take the more general internal::Matcher<Expr>
 because this introduces ambiguous overloads with calls to Callee taking a
 internal::Matcher<Decl>, as the matcher hierarchy is purely
@@ -9087,8 +9335,8 @@ 
AST Traversal Matchers

 

 
 
-Matcher<ObjCMessageExpr>calleeMatcher<Decl> InnerMatcher
-
Matches 1) if the call expression's callee's declaration matches the
+Matcher<ObjCMessageExpr>calleeMatcher<Decl> InnerMatcher
+
Matches 1) if the call expression's callee's declaration matches the
 given matcher; or 2) if the Obj-C message expression's callee's method
 declaration matches the given matcher.
 
diff --git a/clang/docs/LibASTMatchersTutorial.rst b/clang/docs/LibASTMatchersTutorial.rst
index 37c9f178fa8d..d2883688ebfa 100644
--- a/clang/docs/LibASTMatchersTutorial.rst
+++ b/clang/docs/LibASTMatchersTutorial.rst
@@ -22,7 +22,7 @@ started guide `_.
 
 .. code-block:: console
 
-      cd ~/clang-llvm
+      mkdir ~/clang-llvm && cd ~/clang-llvm
       git clone https://github.com/llvm/llvm-project.git
 
 Next you need to obtain the CMake build system and Ninja build tool.
@@ -33,11 +33,11 @@ Next you need to obtain the CMake build system and Ninja build tool.
       git clone https://github.com/martine/ninja.git
       cd ninja
       git checkout release
-      ./bootstrap.py
+      ./configure.py --bootstrap
       sudo cp ninja /usr/bin/
 
       cd ~/clang-llvm
-      git clone git://cmake.org/stage/cmake.git
+      git clone https://gitlab.kitware.com/cmake/cmake.git
       cd cmake
       git checkout next
       ./bootstrap
@@ -50,7 +50,7 @@ Okay. Now we'll build Clang!
 
       cd ~/clang-llvm
       mkdir build && cd build
-      cmake -G Ninja ../llvm -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DLLVM_BUILD_TESTS=ON  # Enable tests; default is off.
+      cmake -G Ninja ../llvm-project/llvm -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" -DCMAKE_BUILD_TYPE=Release -DLLVM_BUILD_TESTS=ON
       ninja
       ninja check       # Test LLVM only.
       ninja clang-test  # Test Clang only.
@@ -65,7 +65,7 @@ Finally, we want to set Clang as its own compiler.
 .. code-block:: console
 
       cd ~/clang-llvm/build
-      ccmake ../llvm
+      ccmake ../llvm-project/llvm
 
 The second command will bring up a GUI for configuring Clang. You need
 to set the entry for ``CMAKE_CXX_COMPILER``. Press ``'t'`` to turn on
@@ -89,7 +89,7 @@ live in the ``clang-tools-extra`` repository.
 
 .. code-block:: console
 
-      cd ~/clang-llvm
+      cd ~/clang-llvm/llvm-project
       mkdir clang-tools-extra/loop-convert
       echo 'add_subdirectory(loop-convert)' >> clang-tools-extra/CMakeLists.txt
       vim clang-tools-extra/loop-convert/CMakeLists.txt
@@ -314,7 +314,7 @@ handiwork:
 
 .. code-block:: console
 
-      cd ~/clang-llvm/llvm/llvm_build/
+      cd ~/clang-llvm/build/
       ninja loop-convert
       vim ~/test-files/simple-loops.cc
       bin/loop-convert ~/test-files/simple-loops.cc
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 3c08d1808b0e..060bc7669b72 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -37,6 +37,35 @@ These changes are ones which we think may surprise users when upgrading to
 Clang |release| because of the opportunity they pose for disruption to existing
 code bases.
 
+- Fix a bug in reversed argument for templated operators.
+  This breaks code in C++20 which was previously accepted in C++17.
+  Clang did not properly diagnose such casese in C++20 before this change. Eg:
+
+  .. code-block:: cpp
+
+    struct P {};
+    template bool operator==(const P&, const S&);
+
+    struct A : public P {};
+    struct B : public P {};
+
+    // This equality is now ambiguous in C++20.
+    bool ambiguous(A a, B b) { return a == b; }
+
+    template bool operator!=(const P&, const S&);
+    // Ok. Found a matching operator!=.
+    bool fine(A a, B b) { return a == b; }
+
+  To reduce such widespread breakages, as an extension, Clang accepts this code
+  with an existing warning ``-Wambiguous-reversed-operator`` warning.
+  Fixes `GH `_.
+
+- The CMake variable ``GCC_INSTALL_PREFIX`` (which sets the default
+  ``--gcc-toolchain=``) is deprecated and will be removed. Specify
+  ``--gcc-install-dir=`` or ``--gcc-triple=`` in a `configuration file
+  ` as a
+  replacement.
+  (`#77537 `_)
 
 C/C++ Language Potentially Breaking Changes
 -------------------------------------------
@@ -108,6 +137,13 @@ C++ Specific Potentially Breaking Changes
   when targetting MSVC to match the behavior of MSVC.
   (`MSVC Docs `_)
 
+- Remove the hardcoded path to the imported modules for C++20 named modules. Now we
+  require all the dependent modules to specified from the command line.
+  See (`#62707: `_).
+
+- Forbid `import XXX;` in C++ to find module `XXX` comes from explicit clang modules.
+  See (`#64755: `_).
+
 ABI Changes in This Version
 ---------------------------
 - Following the SystemV ABI for x86-64, ``__int128`` arguments will no longer
@@ -147,6 +183,10 @@ C++ Language Changes
 
 C++20 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
+- Implemented `P1907R1 ` which extends allowed non-type template argument
+  kinds with e.g. floating point values and pointers and references to subobjects.
+  This feature is still experimental. Accordingly, ``__cpp_nontype_template_args`` was not updated.
+  However, its support can be tested with ``__has_extension(cxx_generalized_nttp)``.
 
 C++23 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
@@ -155,6 +195,7 @@ C++23 Feature Support
   `CWG2653 `_, `CWG2687 `_). Because the
   support for this feature is still experimental, the feature test macro ``__cpp_explicit_this_parameter``
   was not set in this version.
+  However, its support can be tested with ``__has_extension(cxx_explicit_this_parameter)``.
 
 - Added a separate warning to warn the use of attributes on lambdas as a C++23 extension
   in previous language versions: ``-Wc++23-lambda-attributes``.
@@ -193,6 +234,16 @@ C++2c Feature Support
 
 Resolutions to C++ Defect Reports
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+- Implemented `CWG2137 `_ which allows
+  list-initialization from objects of the same type.
+- Implemented `CWG2311 `_: given a prvalue ``e`` of object type
+  ``T``, ``T{e}`` will try to resolve an initializer list constructor and will use it if successful (CWG2137).
+  Otherwise, if there is no initializer list constructor, the copy will be elided as if it was ``T(e)``.
+
+- Implemented `CWG2598 `_ and `CWG2096 `_,
+  making unions (that have either no members or at least one literal member) literal types.
+  (`#77924: `_).
+
 
 C Language Changes
 ------------------
@@ -202,6 +253,11 @@ C Language Changes
 - Enums will now be represented in TBAA metadata using their actual underlying
   integer type. Previously they were treated as chars, which meant they could
   alias with all other types.
+- Clang now supports the C-only attribute ``counted_by``. When applied to a
+  struct's flexible array member, it points to the struct field that holds the
+  number of elements in the flexible array member. This information can improve
+  the results of the array bound sanitizer and the
+  ``__builtin_dynamic_object_size`` builtin.
 
 C23 Feature Support
 ^^^^^^^^^^^^^^^^^^^
@@ -220,6 +276,13 @@ C23 Feature Support
   previously implemented allowing a label at the end of a compound statement,
   and now we've implemented allowing a label to be followed by a declaration
   instead of a statement.
+- Implemented
+  `N2940 `_ which
+  removes support for trigraphs in C23 and later. In earlier language modes,
+  trigraphs remain enabled by default in conforming modes (e.g. ``-std=c17``)
+  and disabled by default in GNU and Microsoft modes (e.g., ``-std=gnu17`` or
+  ``-fms-compatibility``). If needed, you can enable trigraphs by passing
+  ``-ftrigraphs``.
 
 Non-comprehensive list of changes in this release
 -------------------------------------------------
@@ -238,6 +301,8 @@ Non-comprehensive list of changes in this release
 
 * Added ``#pragma clang fp reciprocal``.
 
+* The version of Unicode used by Clang (primarily to parse identifiers) has been updated to 15.1.
+
 New Compiler Flags
 ------------------
 
@@ -257,7 +322,7 @@ New Compiler Flags
   handlers will be smaller. A throw expression of a type with a
   potentially-throwing destructor will lead to an error.
 
-* ``-fopenacc`` was added as a part of the effort to support OpenACC in clang.
+* ``-fopenacc`` was added as a part of the effort to support OpenACC in Clang.
 
 * ``-fcx-limited-range`` enables the naive mathematical formulas for complex
   division and multiplication with no NaN checking of results. The default is
@@ -268,6 +333,16 @@ New Compiler Flags
   division. See SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8
   (1962). The default is ``-fno-cx-fortran-rules``.
 
+* ``-fvisibility-global-new-delete=`` gives more freedom to users to
+  control how and if Clang forces a visibility for the replaceable new and
+  delete declarations. The option takes 4 values: ``force-hidden``,
+  ``force-protected``, ``force-default`` and ``source``; ``force-default`` is
+  the default. Option values with prefix ``force-`` assign such declarations
+  an implicit visibility attribute with the corresponding visibility. An option
+  value of ``source`` implies that no implicit attribute is added. Without the
+  attribute the replaceable global new and delete operators behave normally
+  (like other functions) with respect to visibility attributes, pragmas and
+  options (e.g ``--fvisibility=``).
 
 Deprecated Compiler Flags
 -------------------------
@@ -284,6 +359,9 @@ Modified Compiler Flags
   ``rtdcall``. This new default CC only works for M68k and will use the new
   ``m68k_rtdcc`` CC on every functions that are not variadic. The ``-mrtd``
   driver/frontend flag has the same effect when targeting M68k.
+* ``-fvisibility-global-new-delete-hidden`` is now a deprecated spelling of
+  ``-fvisibility-global-new-delete=force-hidden`` (``-fvisibility-global-new-delete=``
+  is new in this release).
 
 Removed Compiler Flags
 -------------------------
@@ -357,6 +435,7 @@ Attribute Changes in Clang
 - Clang now introduced ``[[clang::coro_lifetimebound]]`` attribute.
   All parameters of a function are considered to be lifetime bound if the function
   returns a type annotated with ``[[clang::coro_lifetimebound]]`` and ``[[clang::coro_return_type]]``.
+  This analysis can be disabled for a function by annotating the function with ``[[clang::coro_disable_lifetimebound]]``.
 
 Improvements to Clang's diagnostics
 -----------------------------------
@@ -518,7 +597,55 @@ Improvements to Clang's diagnostics
 - Clang now diagnoses definitions of friend function specializations, e.g. ``friend void f<>(int) {}``.
 - Clang now diagnoses narrowing conversions involving const references.
   (`#63151: `_).
+- Clang now diagnoses unexpanded packs within the template argument lists of function template specializations.
+- The warning `-Wnan-infinity-disabled` is now emitted when ``INFINITY``
+  or ``NAN`` are used in arithmetic operations or function arguments in
+  floating-point mode where ``INFINITY`` or ``NAN`` don't have the expected
+  values.
+
+- Clang now diagnoses attempts to bind a bitfield to an NTTP of a reference type as erroneous
+  converted constant expression and not as a reference to subobject.
+- Clang now diagnoses ``auto`` and ``decltype(auto)`` in declarations of conversion function template
+  (`CWG1878: `_)
+- Clang now diagnoses the requirement that non-template friend declarations with requires clauses
+  and template friend declarations with a constraint that depends on a template parameter from an
+  enclosing template must be a definition.
+- Clang now diagnoses incorrect usage of ``const`` and ``pure`` attributes, so ``-Wignored-attributes`` diagnoses more cases.
+- Clang now emits more descriptive diagnostics for 'unusual' expressions (e.g. incomplete index
+  expressions on matrix types or builtin functions without an argument list) as placement-args
+  to new-expressions.
+
+  Before:
+
+  .. code-block:: text
+
+    error: no matching function for call to 'operator new'
+       13 |     new (__builtin_memset) S {};
+          |     ^   ~~~~~~~~~~~~~~~~~~
+
+    note: candidate function not viable: no known conversion from '' to 'int' for 2nd argument
+        5 |     void* operator new(__SIZE_TYPE__, int);
+          |           ^
+
+  After:
+
+  .. code-block:: text
 
+    error: builtin functions must be directly called
+       13 |     new (__builtin_memset) S {};
+          |          ^
+
+- Clang now diagnoses import before module declarations but not in global
+  module fragment.
+  (`#67627: `_).
+
+- Clang now diagnoses include headers with angle in module purviews, which is
+  not usually intended.
+  (`#68615: `_)
+
+- Clang now won't mention invisible namespace when diagnose invisible declarations
+  inside namespace. The original diagnostic message is confusing.
+  (`#73893: `_)
 
 Improvements to Clang's time-trace
 ----------------------------------
@@ -607,7 +734,9 @@ Bug Fixes in This Version
 - Clang will correctly evaluate ``noexcept`` expression for template functions
   of template classes. Fixes
   (`#68543 `_,
-  `#42496 `_)
+  `#42496 `_,
+  `#77071 `_,
+  `#77411 `_)
 - Fixed an issue when a shift count larger than ``__INT64_MAX__``, in a right
   shift operation, could result in missing warnings about
   ``shift count >= width of type`` or internal compiler error.
@@ -636,6 +765,8 @@ Bug Fixes in This Version
   Fixes (`#67317 `_)
 - Clang now properly diagnoses use of stand-alone OpenMP directives after a
   label (including ``case`` or ``default`` labels).
+- Fix compiler memory leak for enums with underlying type larger than 64 bits.
+  Fixes (`#78311 `_)
 
   Before:
 
@@ -685,10 +816,49 @@ Bug Fixes in This Version
   (`#65568 `_)
 - Fix an issue where clang doesn't respect detault template arguments that
   are added in a later redeclaration for CTAD.
-  Fixes (#69987 `_)
+  Fixes (`#69987 `_)
 - Fix an issue where CTAD fails for explicit type conversion.
-  Fixes (#64347 `_)
-
+  Fixes (`#64347 `_)
+- Fix crash when using C++ only tokens like ``::`` in C compiler clang.
+  Fixes (`#73559 `_)
+- Clang now accepts recursive non-dependent calls to functions with deduced
+  return type.
+  Fixes (`#71015 `_)
+- Fix assertion failure when initializing union containing struct with
+  flexible array member using empty initializer list.
+  Fixes (`#77085 `_)
+- Fix assertion crash due to failed scope restoring caused by too-early VarDecl
+  invalidation by invalid initializer Expr.
+  Fixes (`#30908 `_)
+- Clang now emits correct source location for code-coverage regions in `if constexpr`
+  and `if consteval` branches. Untaken branches are now skipped.
+  Fixes (`#54419 `_)
+- Fix assertion failure when declaring a template friend function with
+  a constrained parameter in a template class that declares a class method
+  or lambda at different depth.
+  Fixes (`#75426 `_)
+- Fix an issue where clang cannot find conversion function with template
+  parameter when instantiation of template class.
+  Fixes (`#77583 `_)
+- Fix an issue where CTAD fails for function-type/array-type arguments.
+  Fixes (`#51710 `_)
+- Fix crashes when using the binding decl from an invalid structured binding.
+  Fixes (`#67495 `_) and
+  (`#72198 `_)
+- Fix assertion failure when call noreturn-attribute function with musttail
+  attribute.
+  Fixes (`#76631 `_)
+  - The MS ``__noop`` builtin without an argument list is now accepted
+  in the placement-args of new-expressions, matching MSVC's behaviour.
+- Fix an issue that caused MS ``__decspec(property)`` accesses as well as
+  Objective-C++ property accesses to not be converted to a function call
+  to the getter in the placement-args of new-expressions.
+  Fixes (`#65053 `_)
+- Fix an issue with missing symbol definitions when the first coroutine
+  statement appears in a discarded ``if constexpr`` branch.
+  Fixes (`#78290 `_)
+- Fixed assertion failure with deleted overloaded unary operators.
+  Fixes (`#78314 `_)
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -716,7 +886,8 @@ Bug Fixes to C++ Support
 
 - Clang emits an error on substitution failure within lambda body inside a
   requires-expression. This fixes:
-  (`#64138 `_).
+  (`#64138 `_) and
+  (`#71684 `_).
 
 - Update ``FunctionDeclBitfields.NumFunctionDeclBits``. This fixes:
   (`#64171 `_).
@@ -843,6 +1014,46 @@ Bug Fixes to C++ Support
 - Fix crash when parsing nested requirement. Fixes:
   (`#73112 `_)
 
+- Fixed a crash caused by using return type requirement in a lambda. Fixes:
+  (`#63808 `_)
+  (`#64607 `_)
+  (`#64086 `_)
+
+- Fixed a crash where we lost uninstantiated constraints on placeholder NTTP packs. Fixes:
+  (`#63837 `_)
+
+- Fixed a regression where clang forgets how to substitute into constraints on template-template
+  parameters. Fixes:
+  (`#57410 `_) and
+  (`#76604 `_)
+
+- Fix a bug where clang would produce inconsistent values when
+  ``std::source_location::current()`` was used in a function template.
+  Fixes (`#78128 `_)
+
+- Clang now allows parenthesized initialization of arrays in `operator new[]`.
+  Fixes: (`#68198 `_)
+
+- Fixes CTAD for aggregates on nested template classes. Fixes:
+  (`#77599 `_)
+
+- Fix crash when importing the same module with an dynamic initializer twice
+  in different visibility.
+  Fixes (`#67893 `_)
+
+- Fix a false-positive ODR violation for different definitions for `std::align_val_t`.
+  Fixes (`#76638 `_)
+
+- Remove recorded `#pragma once` state for headers included in named modules.
+  Fixes (`#77995 `_)
+
+- Set the ``__cpp_auto_cast`` feature test macro in C++23 mode.
+
+- Fix crash for inconsistent deducing state of function return types
+  in importing modules.
+  Fixes (`#78830 `_)
+  Fixes (`#60085 `_)
+
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 - Fixed an import failure of recursive friend class template.
@@ -856,6 +1067,12 @@ Bug Fixes to AST Handling
 - Fixed a bug where RecursiveASTVisitor fails to visit the
   initializer of a bitfield.
   `Issue 64916 `_
+- Fixed a bug where range-loop-analysis checks for trivial copyability,
+  rather than trivial copy-constructibility
+  `Issue 47355 `_
+- Fixed a bug where Template Instantiation failed to handle Lambda Expressions
+  with certain types of Attributes.
+  (`#76521 `_)
 
 Miscellaneous Bug Fixes
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -877,6 +1094,8 @@ Miscellaneous Clang Crashes Fixed
 - Fixed a crash when a lambda marked as ``static`` referenced a captured
   variable in an expression.
   `Issue 74608 `_
+- Fixed a crash with modules and a ``constexpr`` destructor.
+  `Issue 68702 `_
 
 
 OpenACC Specific Changes
@@ -899,6 +1118,9 @@ AMDGPU Support
   arguments in C ABI. Callee is responsible for allocating stack memory and
   copying the value of the struct if modified. Note that AMDGPU backend still
   supports byval for struct arguments.
+- The default value for ``-mcode-object-version`` is now 5.
+  See `AMDHSA Code Object V5 Metadata `_
+  for more details.
 
 X86 Support
 ^^^^^^^^^^^
@@ -910,6 +1132,15 @@ X86 Support
   * Support intrinsic of ``_uwrmsr``.
 - Support ISA of ``AVX10.1``.
 - ``-march=pantherlake`` and ``-march=clearwaterforest`` are now supported.
+- Added ABI handling for ``__float128`` to match with GCC.
+- Emit warnings for options to enable knl/knm specific ISAs: AVX512PF, AVX512ER
+  and PREFETCHWT1. From next version (LLVM 19), these ISAs' intrinsic supports
+  will be deprecated:
+  * intrinsic series of *_exp2a23_*
+  * intrinsic series of *_rsqrt28_*
+  * intrinsic series of *_rcp28_*
+  * intrinsic series of *_prefetch_i[3|6][2|4]gather_*
+  * intrinsic series of *_prefetch_i[3|6][2|4]scatter_*
 
 Arm and AArch64 Support
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -966,11 +1197,32 @@ Windows Support
 
 LoongArch Support
 ^^^^^^^^^^^^^^^^^
+- Added builtins support for all LSX (128-bits SIMD) and LASX (256-bits SIMD)
+  instructions.
+- Added builtins support for approximate calculation instructions that were
+  introduced in LoongArch Reference Manual V1.10.
+- Made ``-mcmodel=`` compatible with LoongArch gcc that accepted ``normal``,
+  ``medium`` and ``extreme``.
+- The ``model`` attribute was now supported for overriding the default code
+  model used to access global variables. The following values were supported:
+  ``normal``, ``medium`` and ``extreme``.
+
+  *Example Code*:
+
+  .. code-block:: c
+
+     int var __attribute((model("extreme")));
+
+- Default to ``-fno-direct-access-external-data`` for non-PIC.
+- An ABI mismatch with gcc/g++ about empty structs/unions passing was fixed.
+- ``_mcount`` was generated instead of ``mcount``.
 
 RISC-V Support
 ^^^^^^^^^^^^^^
 - Unaligned memory accesses can be toggled by ``-m[no-]unaligned-access`` or the
   aliases ``-m[no-]strict-align``.
+- CodeGen of RV32E/RV64E was supported experimentally.
+- CodeGen of ilp32e/lp64e was supported experimentally.
 
 - Default ABI with F but without D was changed to ilp32f for RV32 and to lp64f
   for RV64.
@@ -1032,6 +1284,9 @@ AST Matchers
 - Add ``convertVectorExpr``.
 - Add ``dependentSizedExtVectorType``.
 - Add ``macroQualifiedType``.
+- Add ``CXXFoldExpr`` related matchers: ``cxxFoldExpr``, ``callee``,
+  ``hasInit``, ``hasPattern``, ``isRightFold``, ``isLeftFold``,
+  ``isUnaryFold``, ``isBinaryFold``, ``hasOperator``, ``hasLHS``, ``hasRHS``, ``hasEitherOperand``.
 
 clang-format
 ------------
@@ -1041,6 +1296,10 @@ clang-format
 - Add ``BreakAdjacentStringLiterals`` option.
 - Add ``ObjCPropertyAttributeOrder`` which can be used to sort ObjC property
   attributes (like ``nonatomic, strong, nullable``).
+- Add ``PenaltyBreakScopeResolution`` option.
+- Add ``.clang-format-ignore`` files.
+- Add ``AlignFunctionPointers`` sub-option for ``AlignConsecutiveDeclarations``.
+- Add ``SkipMacroDefinitionBody`` option.
 
 libclang
 --------
@@ -1126,9 +1385,13 @@ Improvements
 ^^^^^^^^^^^^
 
 - Improved the ``unix.StdCLibraryFunctions`` checker by modeling more
-  functions like ``send``, ``recv``, ``readlink`` and ``errno`` behavior.
+  functions like ``send``, ``recv``, ``readlink``, ``fflush``, ``mkdtemp``,
+  ``getcwd`` and ``errno`` behavior.
   (`52ac71f92d38 `_,
+  `#77040 `_,
+  `#76671 `_,
   `#71373 `_,
+  `#76557 `_,
   `#71392 `_)
 
 - Fixed a false negative for when accessing a nonnull property (ObjC).
@@ -1154,15 +1417,20 @@ Improvements
   (`c3a87ddad62a `_,
   `0954dc3fb921 `_)
 
-- Improved the ``alpha.unix.Stream`` checker by modeling more functions like,
-  ``fflush``, ``fputs``, ``fgetc``, ``fputc``, ``fopen``, ``fopen``, ``fgets``.
-  (`#74296 `_,
+- Improved the ``alpha.unix.Stream`` checker by modeling more functions
+  ``fputs``, ``fputc``, ``fgets``, ``fgetc``, ``fdopen``, ``ungetc``, ``fflush``,
+  ``getdelim``, ``getline`` and no not recognize alternative
+  ``fopen`` and ``tmpfile`` implementations.
+  (`#78693 `_,
+  `#76776 `_,
+  `#74296 `_,
   `#73335 `_,
   `#72627 `_,
   `#71518 `_,
   `#72016 `_,
   `#70540 `_,
-  `#73638 `_)
+  `#73638 `_,
+  `#77331 `_)
 
 - The ``alpha.security.taint.TaintPropagation`` checker no longer propagates
   taint on ``strlen`` and ``strnlen`` calls, unless these are marked
@@ -1179,6 +1447,14 @@ Improvements
 - Checkers can query constraint bounds to improve diagnostic messages.
   (`#74141 `_)
 
+- Improved the generated initializers for modules. Now the calls to initialize
+  functions from imported module units can be omitted if the initialize
+  function is known to be empty.
+  (`#56794 `_)
+
+- Clang now allow to export declarations within linkage-specification.
+  (`#71347 `_)
+
 Moved checkers
 ^^^^^^^^^^^^^^
 
diff --git a/clang/docs/SourceBasedCodeCoverage.rst b/clang/docs/SourceBasedCodeCoverage.rst
index 2be6fa3d717a..cee706289284 100644
--- a/clang/docs/SourceBasedCodeCoverage.rst
+++ b/clang/docs/SourceBasedCodeCoverage.rst
@@ -64,6 +64,11 @@ To compile code with coverage enabled, pass ``-fprofile-instr-generate
 Note that linking together code with and without coverage instrumentation is
 supported. Uninstrumented code simply won't be accounted for in reports.
 
+To compile code with Modified Condition/Decision Coverage (MC/DC) enabled,
+pass ``-fcoverage-mcdc`` in addition to the clang options specified above.
+MC/DC is an advanced form of code coverage most applicable in the embedded
+space.
+
 Running the instrumented program
 ================================
 
@@ -211,6 +216,10 @@ region counts:
     |      4|    1|}
     ------------------
 
+If the application was instrumented for Modified Condition/Decision Coverage
+(MC/DC) using the clang option ``-fcoverage-mcdc``, an MC/DC subview can be
+enabled using ``--show-mcdc`` that will show detailed MC/DC information for
+each complex condition boolean expression containing at most six conditions.
 
 To generate a file-level summary of coverage statistics instead of a
 line-oriented report, try:
@@ -259,7 +268,7 @@ the exported data at a high level in the llvm-cov source code.
 Interpreting reports
 ====================
 
-There are five statistics tracked in a coverage summary:
+There are six statistics tracked in a coverage summary:
 
 * Function coverage is the percentage of functions which have been executed at
   least once. A function is considered to be executed if any of its
@@ -288,10 +297,28 @@ There are five statistics tracked in a coverage summary:
   that is comprised of two individual conditions, each of which evaluates to
   either true or false, producing four total branch outcomes.
 
-Of these five statistics, function coverage is usually the least granular while
-branch coverage is the most granular. 100% branch coverage for a function
-implies 100% region coverage for a function. The project-wide totals for each
-statistic are listed in the summary.
+* Modified Condition/Decision Coverage (MC/DC) is the percentage of individual
+  branch conditions that have been shown to independently affect the decision
+  outcome of the boolean expression they comprise. This is accomplished using
+  the analysis of executed control flow through the expression (i.e. test
+  vectors) to show that as a condition's outcome is varied between "true" and
+  false", the decision's outcome also varies between "true" and false", while
+  the outcome of all other conditions is held fixed (or they are masked out as
+  unevaluatable, as happens in languages whose logical operators have
+  short-circuit semantics).  MC/DC builds on top of branch coverage and
+  requires that all code blocks and all execution paths have been tested.  This
+  statistic is hidden by default in reports, but it can be enabled via the
+  ``-show-mcdc-summary`` option as long as code was also compiled using the
+  clang option ``-fcoverage-mcdc``.
+
+  * Boolean expressions that are only comprised of one condition (and therefore
+    have no logical operators) are not included in MC/DC analysis and are
+    trivially deducible using branch coverage.
+
+Of these six statistics, function coverage is usually the least granular while
+branch coverage (with MC/DC) is the most granular. 100% branch coverage for a
+function implies 100% region coverage for a function. The project-wide totals
+for each statistic are listed in the summary.
 
 Format compatibility guarantees
 ===============================
@@ -453,6 +480,21 @@ Branch coverage is tied directly to branch-generating conditions in the source
 code.  Users should not see hidden branches that aren't actually tied to the
 source code.
 
+MC/DC Instrumentation
+---------------------
+
+When instrumenting for Modified Condition/Decision Coverage (MC/DC) using the
+clang option ``-fcoverage-mcdc``, users are limited to at most **six** leaf-level
+conditions in a boolean expression.  A warning will be generated for boolean
+expressions that contain more than six, and they will not be instrumented for
+MC/DC.
+
+Also, if a boolean expression is embedded in the nest of another boolean
+expression but separated by a non-logical operator, this is also not supported.
+For example, in ``x = (a && b && c && func(d && f))``, the ``d && f`` case
+starts a new boolean expression that is separated from the other conditions by
+the operator ``func()``.  When this is encountered, a warning will be generated
+and the boolean expression will not be instrumented.
 
 Switch statements
 -----------------
diff --git a/clang/docs/StandardCPlusPlusModules.rst b/clang/docs/StandardCPlusPlusModules.rst
index 497045757403..81043ff25be0 100644
--- a/clang/docs/StandardCPlusPlusModules.rst
+++ b/clang/docs/StandardCPlusPlusModules.rst
@@ -143,7 +143,7 @@ Then we type:
 .. code-block:: console
 
   $ clang++ -std=c++20 Hello.cppm --precompile -o Hello.pcm
-  $ clang++ -std=c++20 use.cpp -fprebuilt-module-path=. Hello.pcm -o Hello.out
+  $ clang++ -std=c++20 use.cpp -fmodule-file=Hello=Hello.pcm Hello.pcm -o Hello.out
   $ ./Hello.out
   Hello World!
 
@@ -200,15 +200,15 @@ Then we are able to compile the example by the following command:
   $ clang++ -std=c++20 interface_part.cppm --precompile -o M-interface_part.pcm
   $ clang++ -std=c++20 impl_part.cppm --precompile -fprebuilt-module-path=. -o M-impl_part.pcm
   $ clang++ -std=c++20 M.cppm --precompile -fprebuilt-module-path=. -o M.pcm
-  $ clang++ -std=c++20 Impl.cpp -fmodule-file=M=M.pcm -c -o Impl.o
+  $ clang++ -std=c++20 Impl.cpp -fprebuilt-module-path=. -c -o Impl.o
 
   # Compiling the user
   $ clang++ -std=c++20 User.cpp -fprebuilt-module-path=. -c -o User.o
 
   # Compiling the module and linking it together
-  $ clang++ -std=c++20 M-interface_part.pcm -c -o M-interface_part.o
-  $ clang++ -std=c++20 M-impl_part.pcm -c -o M-impl_part.o
-  $ clang++ -std=c++20 M.pcm -c -o M.o
+  $ clang++ -std=c++20 M-interface_part.pcm -fprebuilt-module-path=. -c -o M-interface_part.o
+  $ clang++ -std=c++20 M-impl_part.pcm -fprebuilt-module-path=. -c -o M-impl_part.o
+  $ clang++ -std=c++20 M.pcm -fprebuilt-module-path=. -c -o M.o
   $ clang++ User.o M-interface_part.o  M-impl_part.o M.o Impl.o -o a.out
 
 We explain the options in the following sections.
@@ -218,7 +218,6 @@ How to enable standard C++ modules
 
 Currently, standard C++ modules are enabled automatically
 if the language standard is ``-std=c++20`` or newer.
-The ``-fmodules-ts`` option is deprecated and is planned to be removed.
 
 How to produce a BMI
 ~~~~~~~~~~~~~~~~~~~~
@@ -346,6 +345,9 @@ In case all ``-fprebuilt-module-path=``, ``-fmodule-file==`` will take the second
 highest precedence.
 
+We need to specify all the dependent (directly and indirectly) BMIs.
+See https://github.com/llvm/llvm-project/issues/62707 for detail.
+
 When we compile a ``module implementation unit``, we must specify the BMI of the corresponding
 ``primary module interface unit``.
 Since the language specification says a module implementation unit implicitly imports
@@ -690,14 +692,68 @@ the BMI within ``clang-cl.exe``.
 
 This is tracked in: https://github.com/llvm/llvm-project/issues/64118
 
-delayed template parsing is not supported/broken with C++ modules
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+false positive ODR violation diagnostic due to using inconsistent qualified but the same type
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ODR violation is a pretty common issue when using modules.
+Sometimes the program violated the One Definition Rule actually.
+But sometimes it shows the compiler gives false positive diagnostics.
+
+One often reported example is:
+
+.. code-block:: c++
+
+  // part.cc
+  module;
+  typedef long T;
+  namespace ns {
+  inline void fun() {
+      (void)(T)0;
+  }
+  }
+  export module repro:part;
+
+  // repro.cc
+  module;
+  typedef long T;
+  namespace ns {
+      using ::T;
+  }
+  namespace ns {
+  inline void fun() {
+      (void)(T)0;
+  }
+  }
+  export module repro;
+  export import :part;
+
+Currently the compiler complains about the inconsistent definition of `fun()` in
+2 module units. This is incorrect. Since both definitions of `fun()` has the same
+spelling and `T` refers to the same type entity finally. So the program should be
+fine.
+
+This is tracked in https://github.com/llvm/llvm-project/issues/78850.
+
+Using TU-local entity in other units
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Module units are translation units. So the entities which should only be local to the
+module unit itself shouldn't be used by other units in any means.
+
+In the language side, to address the idea formally, the language specification defines
+the concept of ``TU-local`` and ``exposure`` in
+`basic.link/p14 `_,
+`basic.link/p15 `_,
+`basic.link/p16 `_,
+`basic.link/p17 `_ and
+`basic.link/p18 `_.
 
-The feature `-fdelayed-template-parsing` can't work well with C++ modules now.
-Note that this is significant on Windows since the option will be enabled by default
-on Windows.
+However, the compiler doesn't support these 2 ideas formally.
+This results in unclear and confusing diagnostic messages.
+And it is worse that the compiler may import TU-local entities to other units without any
+diagnostics.
 
-This is tracked in: https://github.com/llvm/llvm-project/issues/61068
+This is tracked in https://github.com/llvm/llvm-project/issues/78173.
 
 Header Units
 ============
diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index 7c30570437e8..ff2d4a68b8e5 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -2809,6 +2809,110 @@ indexed format, regardeless whether it is produced by frontend or the IR pass.
   overhead. ``prefer-atomic`` will be transformed to ``atomic`` when supported
   by the target, or ``single`` otherwise.
 
+Fine Tuning Profile Collection
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The PGO infrastructure provides user program knobs to fine tune profile
+collection. Specifically, the PGO runtime provides the following functions
+that can be used to control the regions in the program where profiles should
+be collected.
+
+ * ``void __llvm_profile_set_filename(const char *Name)``: changes the name of
+   the profile file to ``Name``.
+ * ``void __llvm_profile_reset_counters(void)``: resets all counters to zero.
+ * ``int __llvm_profile_dump(void)``: write the profile data to disk.
+ * ``int __llvm_orderfile_dump(void)``: write the order file to disk.
+
+For example, the following pattern can be used to skip profiling program
+initialization, profile two specific hot regions, and skip profiling program
+cleanup:
+
+.. code-block:: c
+
+    int main() {
+      initialize();
+
+      // Reset all profile counters to 0 to omit profile collected during
+      // initialize()'s execution.
+      __llvm_profile_reset_counters();
+      ... hot region 1
+      // Dump the profile for hot region 1.
+      __llvm_profile_set_filename("region1.profraw");
+      __llvm_profile_dump();
+
+      // Reset counters before proceeding to hot region 2.
+      __llvm_profile_reset_counters();
+      ... hot region 2
+      // Dump the profile for hot region 2.
+      __llvm_profile_set_filename("region2.profraw");
+      __llvm_profile_dump();
+
+      // Since the profile has been dumped, no further profile data
+      // will be collected beyond the above __llvm_profile_dump().
+      cleanup();
+      return 0;
+    }
+
+These APIs' names can be introduced to user programs in two ways.
+They can be declared as weak symbols on platforms which support
+treating weak symbols as ``null`` during linking. For example, the user can
+have
+
+.. code-block:: c
+
+    __attribute__((weak)) int __llvm_profile_dump(void);
+
+    // Then later in the same source file
+    if (__llvm_profile_dump)
+      if (__llvm_profile_dump() != 0) { ... }
+    // The first if condition tests if the symbol is actually defined.
+    // Profile dumping only happens if the symbol is defined. Hence,
+    // the user program works correctly during normal (not profile-generate)
+    // executions.
+
+Alternatively, the user program can include the header
+``profile/instr_prof_interface.h``, which contains the API names. For example,
+
+.. code-block:: c
+
+    #include "profile/instr_prof_interface.h"
+
+    // Then later in the same source file
+    if (__llvm_profile_dump() != 0) { ... }
+
+The user code does not need to check if the API names are defined, because
+these names are automatically replaced by ``(0)`` or the equivalence of noop
+if the ``clang`` is not compiling for profile generation.
+
+Such replacement can happen because ``clang`` adds one of two macros depending
+on the ``-fprofile-generate`` and the ``-fprofile-use`` flags.
+
+ * ``__LLVM_INSTR_PROFILE_GENERATE``: defined when one of
+   ``-fprofile[-instr]-generate``/``-fcs-profile-generate`` is in effect.
+ * ``__LLVM_INSTR_PROFILE_USE``: defined when one of
+   ``-fprofile-use``/``-fprofile-instr-use`` is in effect.
+
+The two macros can be used to provide more flexibiilty so a user program
+can execute code specifically intended for profile generate or profile use.
+For example, a user program can have special logging during profile generate:
+
+.. code-block:: c
+
+    #if __LLVM_INSTR_PROFILE_GENERATE
+    expensive_logging_of_full_program_state();
+    #endif
+
+The logging is automatically excluded during a normal build of the program,
+hence it does not impact performance during a normal execution.
+
+It is advised to use such fine tuning only in a program's cold regions. The weak
+symbols can introduce extra control flow (the ``if`` checks), while the macros
+(hence declarations they guard in ``profile/instr_prof_interface.h``)
+can change the control flow of the functions that use them between profile
+generation and profile use (which can lead to discarded counters in such
+functions). Using these APIs in the program's cold regions introduces less
+overhead and leads to more optimized code.
+
 Disabling Instrumentation
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -3963,6 +4067,60 @@ implicitly included in later levels.
 - ``-march=x86-64-v3``: (close to Haswell) AVX, AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE, XSAVE
 - ``-march=x86-64-v4``: AVX512F, AVX512BW, AVX512CD, AVX512DQ, AVX512VL
 
+`Intel AVX10 ISA `_ is
+a major new vector ISA incorporating the modern vectorization aspects of
+Intel AVX-512. This ISA will be supported on all future Intel processors.
+Users are supposed to use the new options ``-mavx10.N`` and ``-mavx10.N-512``
+on these processors and should not use traditional AVX512 options anymore.
+
+The ``N`` in ``-mavx10.N`` represents a continuous integer number starting
+from ``1``. ``-mavx10.N`` is an alias of ``-mavx10.N-256``, which means to
+enable all instructions within AVX10 version N at a maximum vector length of
+256 bits. ``-mavx10.N-512`` enables all instructions at a maximum vector
+length of 512 bits, which is a superset of instructions ``-mavx10.N`` enabled.
+
+Current binaries built with AVX512 features can run on Intel AVX10/512 capable
+processors without re-compile, but cannot run on AVX10/256 capable processors.
+Users need to re-compile their code with ``-mavx10.N``, and maybe update some
+code that calling to 512-bit X86 specific intrinsics and passing or returning
+512-bit vector types in function call, if they want to run on AVX10/256 capable
+processors. Binaries built with ``-mavx10.N`` can run on both AVX10/256 and
+AVX10/512 capable processors.
+
+Users can add a ``-mno-evex512`` in the command line with AVX512 options if
+they want to run the binary on both legacy AVX512 and new AVX10/256 capable
+processors. The option has the same constraints as ``-mavx10.N``, i.e.,
+cannot call to 512-bit X86 specific intrinsics and pass or return 512-bit vector
+types in function call.
+
+Users should avoid using AVX512 features in function target attributes when
+developing code for AVX10. If they have to do so, they need to add an explicit
+``evex512`` or ``no-evex512`` together with AVX512 features for 512-bit or
+non-512-bit functions respectively to avoid unexpected code generation. Both
+command line option and target attribute of EVEX512 feature can only be used
+with AVX512. They don't affect vector size of AVX10.
+
+User should not mix the use AVX10 and AVX512 options together at any time,
+because the option combinations are conflicting sometimes. For example, a
+combination of ``-mavx512f -mavx10.1-256`` doesn't show a clear intention to
+compiler, since instructions in AVX512F and AVX10.1/256 intersect but do not
+overlap. In this case, compiler will emit warning for it, but the behavior
+is determined. It will generate the same code as option ``-mavx10.1-512``.
+A similar case is ``-mavx512f -mavx10.2-256``, which equals to
+``-mavx10.1-512 -mavx10.2-256``, because ``avx10.2-256`` implies ``avx10.1-256``
+and ``-mavx512f -mavx10.1-256`` equals to ``-mavx10.1-512``.
+
+There are some new macros introduced with AVX10 support. ``-mavx10.1-256`` will
+enable ``__AVX10_1__`` and ``__EVEX256__``, while ``-mavx10.1-512`` enables
+``__AVX10_1__``, ``__EVEX256__``, ``__EVEX512__``  and ``__AVX10_1_512__``.
+Besides, both ``-mavx10.1-256`` and ``-mavx10.1-512`` will enable all AVX512
+feature specific macros. A AVX512 feature will enable both ``__EVEX256__``,
+``__EVEX512__`` and its own macro. So ``__EVEX512__`` can be used to guard code
+that can run on both legacy AVX512 and AVX10/512 capable processors but cannot
+run on AVX10/256, while a AVX512 macro like ``__AVX512F__`` cannot tell the
+difference among the three options. Users need to check additional macros
+``__AVX10_1__`` and ``__EVEX512__`` if they want to make distinction.
+
 ARM
 ^^^
 
@@ -4382,11 +4540,11 @@ Execute ``clang-cl /?`` to see a list of supported options:
       -fmerge-all-constants   Allow merging of constants
       -fms-compatibility-version=
                               Dot-separated value representing the Microsoft compiler version
-                              number to report in _MSC_VER (0 = don't define it (default))
+                              number to report in _MSC_VER (0 = don't define it; default is same value as installed cl.exe, or 1933)
       -fms-compatibility      Enable full Microsoft Visual C++ compatibility
       -fms-extensions         Accept some non-standard constructs supported by the Microsoft compiler
       -fmsc-version=   Microsoft compiler version number to report in _MSC_VER
-                              (0 = don't define it (default))
+                              (0 = don't define it; default is same value as installed cl.exe, or 1933)
       -fno-addrsig            Don't emit an address-significance table
       -fno-builtin-    Disable implicit builtin knowledge of a specific function
       -fno-builtin            Disable implicit builtin knowledge of functions
diff --git a/clang/docs/index.rst b/clang/docs/index.rst
index 5453a19564b8..a35a867b96bd 100644
--- a/clang/docs/index.rst
+++ b/clang/docs/index.rst
@@ -35,6 +35,8 @@ Using Clang as a Compiler
    SanitizerCoverage
    SanitizerStats
    SanitizerSpecialCaseList
+   BoundsSafety
+   BoundsSafetyImplPlans
    ControlFlowIntegrity
    LTOVisibility
    SafeStack
diff --git a/clang/include/clang/AST/APNumericStorage.h b/clang/include/clang/AST/APNumericStorage.h
new file mode 100644
index 000000000000..95eddbcd86e8
--- /dev/null
+++ b/clang/include/clang/AST/APNumericStorage.h
@@ -0,0 +1,71 @@
+//===--- APNumericStorage.h - Store APInt/APFloat in ASTContext -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_AST_APNUMERICSTORAGE_H
+#define LLVM_CLANG_AST_APNUMERICSTORAGE_H
+
+#include "llvm/ADT/APFloat.h"
+#include "llvm/ADT/APInt.h"
+
+namespace clang {
+class ASTContext;
+
+/// Used by IntegerLiteral/FloatingLiteral/EnumConstantDecl to store the
+/// numeric without leaking memory.
+///
+/// For large floats/integers, APFloat/APInt will allocate memory from the heap
+/// to represent these numbers.  Unfortunately, when we use a BumpPtrAllocator
+/// to allocate IntegerLiteral/FloatingLiteral nodes the memory associated with
+/// the APFloat/APInt values will never get freed. APNumericStorage uses
+/// ASTContext's allocator for memory allocation.
+class APNumericStorage {
+  union {
+    uint64_t VAL;   ///< Used to store the <= 64 bits integer value.
+    uint64_t *pVal; ///< Used to store the >64 bits integer value.
+  };
+  unsigned BitWidth;
+
+  bool hasAllocation() const { return llvm::APInt::getNumWords(BitWidth) > 1; }
+
+  APNumericStorage(const APNumericStorage &) = delete;
+  void operator=(const APNumericStorage &) = delete;
+
+protected:
+  APNumericStorage() : VAL(0), BitWidth(0) {}
+
+  llvm::APInt getIntValue() const {
+    unsigned NumWords = llvm::APInt::getNumWords(BitWidth);
+    if (NumWords > 1)
+      return llvm::APInt(BitWidth, NumWords, pVal);
+    else
+      return llvm::APInt(BitWidth, VAL);
+  }
+  void setIntValue(const ASTContext &C, const llvm::APInt &Val);
+};
+
+class APIntStorage : private APNumericStorage {
+public:
+  llvm::APInt getValue() const { return getIntValue(); }
+  void setValue(const ASTContext &C, const llvm::APInt &Val) {
+    setIntValue(C, Val);
+  }
+};
+
+class APFloatStorage : private APNumericStorage {
+public:
+  llvm::APFloat getValue(const llvm::fltSemantics &Semantics) const {
+    return llvm::APFloat(Semantics, getIntValue());
+  }
+  void setValue(const ASTContext &C, const llvm::APFloat &Val) {
+    setIntValue(C, Val.bitcastToAPInt());
+  }
+};
+
+} // end namespace clang
+
+#endif // LLVM_CLANG_AST_APNUMERICSTORAGE_H
diff --git a/clang/include/clang/AST/Attr.h b/clang/include/clang/AST/Attr.h
index 1b831c9511e2..8e9b7ad8b468 100644
--- a/clang/include/clang/AST/Attr.h
+++ b/clang/include/clang/AST/Attr.h
@@ -25,6 +25,7 @@
 #include "clang/Basic/Sanitizers.h"
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/Frontend/HLSL/HLSLResource.h"
+#include "llvm/Support/CodeGen.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index a807bcdd76b3..f26fb5ad5f13 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_CLANG_AST_DECL_H
 #define LLVM_CLANG_AST_DECL_H
 
+#include "clang/AST/APNumericStorage.h"
 #include "clang/AST/APValue.h"
 #include "clang/AST/ASTContextAllocate.h"
 #include "clang/AST/DeclAccessPair.h"
@@ -2293,8 +2294,8 @@ class FunctionDecl : public DeclaratorDecl,
 
   /// Whether this virtual function is pure, i.e. makes the containing class
   /// abstract.
-  bool isPure() const { return FunctionDeclBits.IsPure; }
-  void setPure(bool P = true);
+  bool isPureVirtual() const { return FunctionDeclBits.IsPureVirtual; }
+  void setIsPureVirtual(bool P = true);
 
   /// Whether this templated function will be late parsed.
   bool isLateTemplateParsed() const {
@@ -3251,15 +3252,16 @@ class FieldDecl : public DeclaratorDecl, public Mergeable {
 /// that is defined.  For example, in "enum X {a,b}", each of a/b are
 /// EnumConstantDecl's, X is an instance of EnumDecl, and the type of a/b is a
 /// TagType for the X EnumDecl.
-class EnumConstantDecl : public ValueDecl, public Mergeable {
+class EnumConstantDecl : public ValueDecl,
+                         public Mergeable,
+                         public APIntStorage {
   Stmt *Init; // an integer constant expression
-  llvm::APSInt Val; // The value.
+  bool IsUnsigned;
 
 protected:
-  EnumConstantDecl(DeclContext *DC, SourceLocation L,
+  EnumConstantDecl(const ASTContext &C, DeclContext *DC, SourceLocation L,
                    IdentifierInfo *Id, QualType T, Expr *E,
-                   const llvm::APSInt &V)
-    : ValueDecl(EnumConstant, DC, L, Id, T), Init((Stmt*)E), Val(V) {}
+                   const llvm::APSInt &V);
 
 public:
   friend class StmtIteratorBase;
@@ -3272,10 +3274,15 @@ class EnumConstantDecl : public ValueDecl, public Mergeable {
 
   const Expr *getInitExpr() const { return (const Expr*) Init; }
   Expr *getInitExpr() { return (Expr*) Init; }
-  const llvm::APSInt &getInitVal() const { return Val; }
+  llvm::APSInt getInitVal() const {
+    return llvm::APSInt(getValue(), IsUnsigned);
+  }
 
   void setInitExpr(Expr *E) { Init = (Stmt*) E; }
-  void setInitVal(const llvm::APSInt &V) { Val = V; }
+  void setInitVal(const ASTContext &C, const llvm::APSInt &V) {
+    setValue(C, V);
+    IsUnsigned = V.isUnsigned();
+  }
 
   SourceRange getSourceRange() const override LLVM_READONLY;
 
diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h
index 10dcbdb262d8..9a4736019d1b 100644
--- a/clang/include/clang/AST/DeclBase.h
+++ b/clang/include/clang/AST/DeclBase.h
@@ -19,6 +19,7 @@
 #include "clang/AST/SelectorLocationsKind.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
+#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/Specifiers.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -488,6 +489,15 @@ class alignas(8) Decl {
   // Return true if this is a FileContext Decl.
   bool isFileContextDecl() const;
 
+  /// Whether it resembles a flexible array member. This is a static member
+  /// because we want to be able to call it with a nullptr. That allows us to
+  /// perform non-Decl specific checks based on the object's type and strict
+  /// flex array level.
+  static bool isFlexibleArrayMemberLike(
+      ASTContext &Context, const Decl *D, QualType Ty,
+      LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel,
+      bool IgnoreTemplateOrMacroSubstitution);
+
   ASTContext &getASTContext() const LLVM_READONLY;
 
   /// Helper to get the language options from the ASTContext.
@@ -538,17 +548,18 @@ class alignas(8) Decl {
     return hasAttrs() ? getAttrs().end() : nullptr;
   }
 
-  template 
-  void dropAttr() {
+  template  void dropAttrs() {
     if (!HasAttrs) return;
 
     AttrVec &Vec = getAttrs();
-    llvm::erase_if(Vec, [](Attr *A) { return isa(A); });
+    llvm::erase_if(Vec, [](Attr *A) { return isa(A); });
 
     if (Vec.empty())
       HasAttrs = false;
   }
 
+  template  void dropAttr() { dropAttrs(); }
+
   template 
   llvm::iterator_range> specific_attrs() const {
     return llvm::make_range(specific_attr_begin(), specific_attr_end());
@@ -1697,7 +1708,7 @@ class DeclContext {
     LLVM_PREFERRED_TYPE(bool)
     uint64_t IsVirtualAsWritten : 1;
     LLVM_PREFERRED_TYPE(bool)
-    uint64_t IsPure : 1;
+    uint64_t IsPureVirtual : 1;
     LLVM_PREFERRED_TYPE(bool)
     uint64_t HasInheritedPrototype : 1;
     LLVM_PREFERRED_TYPE(bool)
diff --git a/clang/include/clang/AST/DeclCXX.h b/clang/include/clang/AST/DeclCXX.h
index 432293583576..9cebaff63bb0 100644
--- a/clang/include/clang/AST/DeclCXX.h
+++ b/clang/include/clang/AST/DeclCXX.h
@@ -266,7 +266,7 @@ class CXXRecordDecl : public RecordDecl {
   friend class LambdaExpr;
   friend class ODRDiagsEmitter;
 
-  friend void FunctionDecl::setPure(bool);
+  friend void FunctionDecl::setIsPureVirtual(bool);
   friend void TagDecl::startDefinition();
 
   /// Values used in DefinitionData fields to represent special members.
@@ -1425,6 +1425,9 @@ class CXXRecordDecl : public RecordDecl {
   /// (C++11 [class]p6).
   bool isTriviallyCopyable() const;
 
+  /// Determine whether this class is considered trivially copyable per
+  bool isTriviallyCopyConstructible() const;
+
   /// Determine whether this class is considered trivial.
   ///
   /// C++11 [class]p6:
@@ -1436,31 +1439,20 @@ class CXXRecordDecl : public RecordDecl {
 
   /// Determine whether this class is a literal type.
   ///
-  /// C++11 [basic.types]p10:
+  /// C++20 [basic.types]p10:
   ///   A class type that has all the following properties:
-  ///     - it has a trivial destructor
-  ///     - every constructor call and full-expression in the
-  ///       brace-or-equal-intializers for non-static data members (if any) is
-  ///       a constant expression.
-  ///     - it is an aggregate type or has at least one constexpr constructor
-  ///       or constructor template that is not a copy or move constructor, and
-  ///     - all of its non-static data members and base classes are of literal
-  ///       types
-  ///
-  /// We resolve DR1361 by ignoring the second bullet. We resolve DR1452 by
-  /// treating types with trivial default constructors as literal types.
-  ///
-  /// Only in C++17 and beyond, are lambdas literal types.
-  bool isLiteral() const {
-    const LangOptions &LangOpts = getLangOpts();
-    return (LangOpts.CPlusPlus20 ? hasConstexprDestructor()
-                                          : hasTrivialDestructor()) &&
-           (!isLambda() || LangOpts.CPlusPlus17) &&
-           !hasNonLiteralTypeFieldsOrBases() &&
-           (isAggregate() || isLambda() ||
-            hasConstexprNonCopyMoveConstructor() ||
-            hasTrivialDefaultConstructor());
-  }
+  ///     - it has a constexpr destructor
+  ///     - all of its non-static non-variant data members and base classes
+  ///       are of non-volatile literal types, and it:
+  ///        - is a closure type
+  ///        - is an aggregate union type that has either no variant members
+  ///          or at least one variant member of non-volatile literal type
+  ///        - is a non-union aggregate type for which each of its anonymous
+  ///          union members satisfies the above requirements for an aggregate
+  ///          union type, or
+  ///        - has at least one constexpr constructor or constructor template
+  ///          that is not a copy or move constructor.
+  bool isLiteral() const;
 
   /// Determine whether this is a structural type.
   bool isStructural() const {
@@ -2044,6 +2036,14 @@ class RequiresExprBodyDecl : public Decl, public DeclContext {
   // Implement isa/cast/dyncast/etc.
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classofKind(Kind K) { return K == RequiresExprBody; }
+
+  static DeclContext *castToDeclContext(const RequiresExprBodyDecl *D) {
+    return static_cast(const_cast(D));
+  }
+
+  static RequiresExprBodyDecl *castFromDeclContext(const DeclContext *DC) {
+    return static_cast(const_cast(DC));
+  }
 };
 
 /// Represents a static or instance method of a struct/union/class.
@@ -2110,7 +2110,7 @@ class CXXMethodDecl : public FunctionDecl {
 
     // Member function is virtual if it is marked explicitly so, or if it is
     // declared in __interface -- then it is automatically pure virtual.
-    if (CD->isVirtualAsWritten() || CD->isPure())
+    if (CD->isVirtualAsWritten() || CD->isPureVirtual())
       return true;
 
     return CD->size_overridden_methods() != 0;
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index a41f2d66b37b..9820bd11da86 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_CLANG_AST_EXPR_H
 #define LLVM_CLANG_AST_EXPR_H
 
+#include "clang/AST/APNumericStorage.h"
 #include "clang/AST/APValue.h"
 #include "clang/AST/ASTVector.h"
 #include "clang/AST/ComputeDependence.h"
@@ -1135,7 +1136,6 @@ class ConstantExpr final
     return ConstantExprBits.APValueKind != APValue::None;
   }
   APValue getAPValueResult() const;
-  APValue &getResultAsAPValue() const { return APValueResult(); }
   llvm::APSInt getResultAsAPSInt() const;
   // Iterators
   child_range children() { return child_range(&SubExpr, &SubExpr+1); }
@@ -1479,57 +1479,6 @@ class DeclRefExpr final
   }
 };
 
-/// Used by IntegerLiteral/FloatingLiteral to store the numeric without
-/// leaking memory.
-///
-/// For large floats/integers, APFloat/APInt will allocate memory from the heap
-/// to represent these numbers.  Unfortunately, when we use a BumpPtrAllocator
-/// to allocate IntegerLiteral/FloatingLiteral nodes the memory associated with
-/// the APFloat/APInt values will never get freed. APNumericStorage uses
-/// ASTContext's allocator for memory allocation.
-class APNumericStorage {
-  union {
-    uint64_t VAL;    ///< Used to store the <= 64 bits integer value.
-    uint64_t *pVal;  ///< Used to store the >64 bits integer value.
-  };
-  unsigned BitWidth;
-
-  bool hasAllocation() const { return llvm::APInt::getNumWords(BitWidth) > 1; }
-
-  APNumericStorage(const APNumericStorage &) = delete;
-  void operator=(const APNumericStorage &) = delete;
-
-protected:
-  APNumericStorage() : VAL(0), BitWidth(0) { }
-
-  llvm::APInt getIntValue() const {
-    unsigned NumWords = llvm::APInt::getNumWords(BitWidth);
-    if (NumWords > 1)
-      return llvm::APInt(BitWidth, NumWords, pVal);
-    else
-      return llvm::APInt(BitWidth, VAL);
-  }
-  void setIntValue(const ASTContext &C, const llvm::APInt &Val);
-};
-
-class APIntStorage : private APNumericStorage {
-public:
-  llvm::APInt getValue() const { return getIntValue(); }
-  void setValue(const ASTContext &C, const llvm::APInt &Val) {
-    setIntValue(C, Val);
-  }
-};
-
-class APFloatStorage : private APNumericStorage {
-public:
-  llvm::APFloat getValue(const llvm::fltSemantics &Semantics) const {
-    return llvm::APFloat(Semantics, getIntValue());
-  }
-  void setValue(const ASTContext &C, const llvm::APFloat &Val) {
-    setIntValue(C, Val.bitcastToAPInt());
-  }
-};
-
 class IntegerLiteral : public Expr, public APIntStorage {
   SourceLocation Loc;
 
@@ -4806,6 +4755,17 @@ class SourceLocExpr final : public Expr {
     return T->getStmtClass() == SourceLocExprClass;
   }
 
+  static bool MayBeDependent(SourceLocIdentKind Kind) {
+    switch (Kind) {
+    case SourceLocIdentKind::Function:
+    case SourceLocIdentKind::FuncSig:
+    case SourceLocIdentKind::SourceLocStruct:
+      return true;
+    default:
+      return false;
+    }
+  }
+
 private:
   friend class ASTStmtReader;
 };
diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h
index 242780164318..9a7c632c36c5 100644
--- a/clang/include/clang/AST/ExprCXX.h
+++ b/clang/include/clang/AST/ExprCXX.h
@@ -2210,14 +2210,11 @@ enum class CXXNewInitializationStyle {
   /// New-expression has no initializer as written.
   None,
 
-  /// New-expression has no written initializer, but has an implicit one.
-  Implicit,
-
   /// New-expression has a C++98 paren-delimited initializer.
-  Call,
+  Parens,
 
   /// New-expression has a C++11 list-initializer.
-  List
+  Braces
 };
 
 /// Represents a new-expression for memory allocation and constructor
@@ -2388,17 +2385,7 @@ class CXXNewExpr final
   bool isGlobalNew() const { return CXXNewExprBits.IsGlobalNew; }
 
   /// Whether this new-expression has any initializer at all.
-  bool hasInitializer() const {
-    switch (getInitializationStyle()) {
-    case CXXNewInitializationStyle::None:
-      return false;
-    case CXXNewInitializationStyle::Implicit:
-    case CXXNewInitializationStyle::Call:
-    case CXXNewInitializationStyle::List:
-      return true;
-    }
-    llvm_unreachable("Unknown initializer");
-  }
+  bool hasInitializer() const { return CXXNewExprBits.HasInitializer; }
 
   /// The kind of initializer this new-expression has.
   CXXNewInitializationStyle getInitializationStyle() const {
diff --git a/clang/include/clang/AST/ODRHash.h b/clang/include/clang/AST/ODRHash.h
index cedf644520fc..a1caa6d39a87 100644
--- a/clang/include/clang/AST/ODRHash.h
+++ b/clang/include/clang/AST/ODRHash.h
@@ -25,6 +25,7 @@
 
 namespace clang {
 
+class APValue;
 class Decl;
 class IdentifierInfo;
 class NestedNameSpecifier;
@@ -101,6 +102,8 @@ class ODRHash {
   // Save booleans until the end to lower the size of data to process.
   void AddBoolean(bool value);
 
+  void AddStructuralValue(const APValue &);
+
   static bool isSubDeclToBeProcessed(const Decl *D, const DeclContext *Parent);
 
 private:
diff --git a/clang/include/clang/AST/PropertiesBase.td b/clang/include/clang/AST/PropertiesBase.td
index d86c4eba6a22..0270c086d06b 100644
--- a/clang/include/clang/AST/PropertiesBase.td
+++ b/clang/include/clang/AST/PropertiesBase.td
@@ -808,6 +808,20 @@ let Class = PropertyTypeCase in {
     return TemplateArgument(ctx, value, type, isDefaulted);
   }]>;
 }
+let Class = PropertyTypeCase in {
+  def : Property<"value", APValue> {
+    let Read = [{ node.getAsStructuralValue() }];
+  }
+  def : Property<"type", QualType> {
+    let Read = [{ node.getStructuralValueType() }];
+  }
+  def : Property<"isDefaulted", Bool> {
+    let Read = [{ node.getIsDefaulted() }];
+  }
+  def : Creator<[{
+    return TemplateArgument(ctx, type, value, isDefaulted);
+  }]>;
+}
 let Class = PropertyTypeCase in {
   def : Property<"name", TemplateName> {
     let Read = [{ node.getAsTemplateOrTemplatePattern() }];
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 8f2714e142bb..2aee6a947141 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -850,6 +850,7 @@ bool RecursiveASTVisitor::TraverseTemplateArgument(
   case TemplateArgument::Declaration:
   case TemplateArgument::Integral:
   case TemplateArgument::NullPtr:
+  case TemplateArgument::StructuralValue:
     return true;
 
   case TemplateArgument::Type:
@@ -882,6 +883,7 @@ bool RecursiveASTVisitor::TraverseTemplateArgumentLoc(
   case TemplateArgument::Declaration:
   case TemplateArgument::Integral:
   case TemplateArgument::NullPtr:
+  case TemplateArgument::StructuralValue:
     return true;
 
   case TemplateArgument::Type: {
diff --git a/clang/include/clang/AST/Stmt.h b/clang/include/clang/AST/Stmt.h
index da7b37ce0e12..55eca4007d17 100644
--- a/clang/include/clang/AST/Stmt.h
+++ b/clang/include/clang/AST/Stmt.h
@@ -868,9 +868,11 @@ class alignas(void *) Stmt {
     LLVM_PREFERRED_TYPE(bool)
     unsigned UsualArrayDeleteWantsSize : 1;
 
-    /// What kind of initializer do we have? Could be none, parens, or braces.
-    /// In storage, we distinguish between "none, and no initializer expr", and
-    /// "none, but an implicit initializer expr".
+    // Is initializer expr present?
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned HasInitializer : 1;
+
+    /// What kind of initializer syntax used? Could be none, parens, or braces.
     LLVM_PREFERRED_TYPE(CXXNewInitializationStyle)
     unsigned StoredInitializationStyle : 2;
 
@@ -1631,8 +1633,10 @@ class CompoundStmt final
                               SourceLocation RB);
 
   // Build an empty compound statement with a location.
-  explicit CompoundStmt(SourceLocation Loc)
-      : Stmt(CompoundStmtClass), LBraceLoc(Loc), RBraceLoc(Loc) {
+  explicit CompoundStmt(SourceLocation Loc) : CompoundStmt(Loc, Loc) {}
+
+  CompoundStmt(SourceLocation Loc, SourceLocation EndLoc)
+      : Stmt(CompoundStmtClass), LBraceLoc(Loc), RBraceLoc(EndLoc) {
     CompoundStmtBits.NumStmts = 0;
     CompoundStmtBits.HasFPFeatures = 0;
   }
diff --git a/clang/include/clang/AST/TemplateArgumentVisitor.h b/clang/include/clang/AST/TemplateArgumentVisitor.h
index 190aa97adf45..cf0d32201580 100644
--- a/clang/include/clang/AST/TemplateArgumentVisitor.h
+++ b/clang/include/clang/AST/TemplateArgumentVisitor.h
@@ -37,6 +37,7 @@ class Base {
       DISPATCH(Declaration);
       DISPATCH(NullPtr);
       DISPATCH(Integral);
+      DISPATCH(StructuralValue);
       DISPATCH(Template);
       DISPATCH(TemplateExpansion);
       DISPATCH(Expression);
@@ -59,6 +60,7 @@ class Base {
   VISIT_METHOD(Declaration);
   VISIT_METHOD(NullPtr);
   VISIT_METHOD(Integral);
+  VISIT_METHOD(StructuralValue);
   VISIT_METHOD(Template);
   VISIT_METHOD(TemplateExpansion);
   VISIT_METHOD(Expression);
diff --git a/clang/include/clang/AST/TemplateBase.h b/clang/include/clang/AST/TemplateBase.h
index b7cd71f17c94..fea2c8ccfee6 100644
--- a/clang/include/clang/AST/TemplateBase.h
+++ b/clang/include/clang/AST/TemplateBase.h
@@ -50,6 +50,7 @@ template <> struct PointerLikeTypeTraits {
 
 namespace clang {
 
+class APValue;
 class ASTContext;
 class Expr;
 struct PrintingPolicy;
@@ -80,6 +81,13 @@ class TemplateArgument {
     /// that was provided for an integral non-type template parameter.
     Integral,
 
+    /// The template argument is a non-type template argument that can't be
+    /// represented by the special-case Declaration, NullPtr, or Integral
+    /// forms. These values are only ever produced by constant evaluation,
+    /// so cannot be dependent.
+    /// TODO: merge Declaration, NullPtr and Integral into this?
+    StructuralValue,
+
     /// The template argument is a template name that was provided for a
     /// template template parameter.
     Template,
@@ -130,6 +138,14 @@ class TemplateArgument {
     };
     void *Type;
   };
+  struct V {
+    LLVM_PREFERRED_TYPE(ArgKind)
+    unsigned Kind : 31;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned IsDefaulted : 1;
+    APValue *Value;
+    void *Type;
+  };
   struct A {
     LLVM_PREFERRED_TYPE(ArgKind)
     unsigned Kind : 31;
@@ -156,11 +172,19 @@ class TemplateArgument {
   union {
     struct DA DeclArg;
     struct I Integer;
+    struct V Value;
     struct A Args;
     struct TA TemplateArg;
     struct TV TypeOrValue;
   };
 
+  void initFromType(QualType T, bool IsNullPtr, bool IsDefaulted);
+  void initFromDeclaration(ValueDecl *D, QualType QT, bool IsDefaulted);
+  void initFromIntegral(const ASTContext &Ctx, const llvm::APSInt &Value,
+                        QualType Type, bool IsDefaulted);
+  void initFromStructural(const ASTContext &Ctx, QualType Type,
+                          const APValue &V, bool IsDefaulted);
+
 public:
   /// Construct an empty, invalid template argument.
   constexpr TemplateArgument() : TypeOrValue({Null, 0, /* IsDefaulted */ 0}) {}
@@ -168,25 +192,22 @@ class TemplateArgument {
   /// Construct a template type argument.
   TemplateArgument(QualType T, bool isNullPtr = false,
                    bool IsDefaulted = false) {
-    TypeOrValue.Kind = isNullPtr ? NullPtr : Type;
-    TypeOrValue.IsDefaulted = IsDefaulted;
-    TypeOrValue.V = reinterpret_cast(T.getAsOpaquePtr());
+    initFromType(T, isNullPtr, IsDefaulted);
   }
 
-  /// Construct a template argument that refers to a
-  /// declaration, which is either an external declaration or a
-  /// template declaration.
+  /// Construct a template argument that refers to a (non-dependent)
+  /// declaration.
   TemplateArgument(ValueDecl *D, QualType QT, bool IsDefaulted = false) {
-    assert(D && "Expected decl");
-    DeclArg.Kind = Declaration;
-    DeclArg.IsDefaulted = IsDefaulted;
-    DeclArg.QT = QT.getAsOpaquePtr();
-    DeclArg.D = D;
+    initFromDeclaration(D, QT, IsDefaulted);
   }
 
   /// Construct an integral constant template argument. The memory to
   /// store the value is allocated with Ctx.
-  TemplateArgument(ASTContext &Ctx, const llvm::APSInt &Value, QualType Type,
+  TemplateArgument(const ASTContext &Ctx, const llvm::APSInt &Value,
+                   QualType Type, bool IsDefaulted = false);
+
+  /// Construct a template argument from an arbitrary constant value.
+  TemplateArgument(const ASTContext &Ctx, QualType Type, const APValue &Value,
                    bool IsDefaulted = false);
 
   /// Construct an integral constant template argument with the same
@@ -297,7 +318,7 @@ class TemplateArgument {
   /// Retrieve the type for a type template argument.
   QualType getAsType() const {
     assert(getKind() == Type && "Unexpected kind");
-    return QualType::getFromOpaquePtr(reinterpret_cast(TypeOrValue.V));
+    return QualType::getFromOpaquePtr(reinterpret_cast(TypeOrValue.V));
   }
 
   /// Retrieve the declaration for a declaration non-type
@@ -315,7 +336,7 @@ class TemplateArgument {
   /// Retrieve the type for null non-type template argument.
   QualType getNullPtrType() const {
     assert(getKind() == NullPtr && "Unexpected kind");
-    return QualType::getFromOpaquePtr(reinterpret_cast(TypeOrValue.V));
+    return QualType::getFromOpaquePtr(reinterpret_cast(TypeOrValue.V));
   }
 
   /// Retrieve the template name for a template name argument.
@@ -371,6 +392,14 @@ class TemplateArgument {
   /// default template parameter.
   bool getIsDefaulted() const { return (bool)TypeOrValue.IsDefaulted; }
 
+  /// Get the value of a StructuralValue.
+  const APValue &getAsStructuralValue() const { return *Value.Value; }
+
+  /// Get the type of a StructuralValue.
+  QualType getStructuralValueType() const {
+    return QualType::getFromOpaquePtr(Value.Type);
+  }
+
   /// If this is a non-type template argument, get its type. Otherwise,
   /// returns a null QualType.
   QualType getNonTypeTemplateArgumentType() const;
@@ -516,6 +545,7 @@ class TemplateArgumentLoc {
     assert(Argument.getKind() == TemplateArgument::NullPtr ||
            Argument.getKind() == TemplateArgument::Integral ||
            Argument.getKind() == TemplateArgument::Declaration ||
+           Argument.getKind() == TemplateArgument::StructuralValue ||
            Argument.getKind() == TemplateArgument::Expression);
   }
 
@@ -541,13 +571,9 @@ class TemplateArgumentLoc {
   /// - Fetches the full source range of the argument.
   SourceRange getSourceRange() const LLVM_READONLY;
 
-  const TemplateArgument &getArgument() const {
-    return Argument;
-  }
+  const TemplateArgument &getArgument() const { return Argument; }
 
-  TemplateArgumentLocInfo getLocInfo() const {
-    return LocInfo;
-  }
+  TemplateArgumentLocInfo getLocInfo() const { return LocInfo; }
 
   TypeSourceInfo *getTypeSourceInfo() const {
     if (Argument.getKind() != TemplateArgument::Type)
@@ -575,6 +601,11 @@ class TemplateArgumentLoc {
     return LocInfo.getAsExpr();
   }
 
+  Expr *getSourceStructuralValueExpression() const {
+    assert(Argument.getKind() == TemplateArgument::StructuralValue);
+    return LocInfo.getAsExpr();
+  }
+
   NestedNameSpecifierLoc getTemplateQualifierLoc() const {
     if (Argument.getKind() != TemplateArgument::Template &&
         Argument.getKind() != TemplateArgument::TemplateExpansion)
@@ -606,8 +637,7 @@ class TemplateArgumentListInfo {
 public:
   TemplateArgumentListInfo() = default;
 
-  TemplateArgumentListInfo(SourceLocation LAngleLoc,
-                           SourceLocation RAngleLoc)
+  TemplateArgumentListInfo(SourceLocation LAngleLoc, SourceLocation RAngleLoc)
       : LAngleLoc(LAngleLoc), RAngleLoc(RAngleLoc) {}
 
   // This can leak if used in an AST node, use ASTTemplateArgumentListInfo
@@ -626,21 +656,15 @@ class TemplateArgumentListInfo {
     return Arguments.data();
   }
 
-  llvm::ArrayRef arguments() const {
-    return Arguments;
-  }
+  llvm::ArrayRef arguments() const { return Arguments; }
 
   const TemplateArgumentLoc &operator[](unsigned I) const {
     return Arguments[I];
   }
 
-  TemplateArgumentLoc &operator[](unsigned I) {
-    return Arguments[I];
-  }
+  TemplateArgumentLoc &operator[](unsigned I) { return Arguments[I]; }
 
-  void addArgument(const TemplateArgumentLoc &Loc) {
-    Arguments.push_back(Loc);
-  }
+  void addArgument(const TemplateArgumentLoc &Loc) { Arguments.push_back(Loc); }
 };
 
 /// Represents an explicit template argument list in C++, e.g.,
diff --git a/clang/include/clang/AST/TextNodeDumper.h b/clang/include/clang/AST/TextNodeDumper.h
index 2f4ed082a0c7..732749ad305e 100644
--- a/clang/include/clang/AST/TextNodeDumper.h
+++ b/clang/include/clang/AST/TextNodeDumper.h
@@ -252,6 +252,8 @@ class TextNodeDumper
   void VisitGotoStmt(const GotoStmt *Node);
   void VisitCaseStmt(const CaseStmt *Node);
   void VisitReturnStmt(const ReturnStmt *Node);
+  void VisitCoawaitExpr(const CoawaitExpr *Node);
+  void VisitCoreturnStmt(const CoreturnStmt *Node);
   void VisitCompoundStmt(const CompoundStmt *Node);
   void VisitConstantExpr(const ConstantExpr *Node);
   void VisitCallExpr(const CallExpr *Node);
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 0a6ec8ecfef7..f9790e05b926 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -920,6 +920,9 @@ class QualType {
   /// Return true if this is a trivially copyable type (C++0x [basic.types]p9)
   bool isTriviallyCopyableType(const ASTContext &Context) const;
 
+  /// Return true if this is a trivially copyable type
+  bool isTriviallyCopyConstructibleType(const ASTContext &Context) const;
+
   /// Return true if this is a trivially relocatable type.
   bool isTriviallyRelocatableType(const ASTContext &Context) const;
 
@@ -4035,6 +4038,22 @@ class FunctionType : public Type {
   /// because TrailingObjects cannot handle repeated types.
   struct ExceptionType { QualType Type; };
 
+  /// A simple holder for various uncommon bits which do not fit in
+  /// FunctionTypeBitfields. Aligned to alignof(void *) to maintain the
+  /// alignment of subsequent objects in TrailingObjects.
+  struct alignas(void *) FunctionTypeExtraBitfields {
+    /// The number of types in the exception specification.
+    /// A whole unsigned is not needed here and according to
+    /// [implimits] 8 bits would be enough here.
+    unsigned NumExceptionType : 10;
+
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned HasArmTypeAttributes : 1;
+
+    FunctionTypeExtraBitfields()
+        : NumExceptionType(0), HasArmTypeAttributes(false) {}
+  };
+
   /// The AArch64 SME ACLE (Arm C/C++ Language Extensions) define a number
   /// of function type attributes that can be set on function types, including
   /// function pointers.
@@ -4042,26 +4061,43 @@ class FunctionType : public Type {
     SME_NormalFunction = 0,
     SME_PStateSMEnabledMask = 1 << 0,
     SME_PStateSMCompatibleMask = 1 << 1,
-    SME_PStateZASharedMask = 1 << 2,
-    SME_PStateZAPreservedMask = 1 << 3,
-    SME_AttributeMask = 0b111'111 // We only support maximum 6 bits because of the
-                                  // bitmask in FunctionTypeExtraBitfields.
+
+    // Describes the value of the state using ArmStateValue.
+    SME_ZAShift = 2,
+    SME_ZAMask = 0b111 << SME_ZAShift,
+    SME_ZT0Shift = 5,
+    SME_ZT0Mask = 0b111 << SME_ZT0Shift,
+
+    SME_AttributeMask =
+        0b111'111'11 // We can't support more than 8 bits because of
+                     // the bitmask in FunctionTypeExtraBitfields.
   };
 
-  /// A simple holder for various uncommon bits which do not fit in
-  /// FunctionTypeBitfields. Aligned to alignof(void *) to maintain the
-  /// alignment of subsequent objects in TrailingObjects.
-  struct alignas(void *) FunctionTypeExtraBitfields {
-    /// The number of types in the exception specification.
-    /// A whole unsigned is not needed here and according to
-    /// [implimits] 8 bits would be enough here.
-    unsigned NumExceptionType : 10;
+  enum ArmStateValue : unsigned {
+    ARM_None = 0,
+    ARM_Preserves = 1,
+    ARM_In = 2,
+    ARM_Out = 3,
+    ARM_InOut = 4,
+  };
+
+  static ArmStateValue getArmZAState(unsigned AttrBits) {
+    return (ArmStateValue)((AttrBits & SME_ZAMask) >> SME_ZAShift);
+  }
+
+  static ArmStateValue getArmZT0State(unsigned AttrBits) {
+    return (ArmStateValue)((AttrBits & SME_ZT0Mask) >> SME_ZT0Shift);
+  }
 
+  /// A holder for Arm type attributes as described in the Arm C/C++
+  /// Language extensions which are not particularly common to all
+  /// types and therefore accounted separately from FunctionTypeBitfields.
+  struct alignas(void *) FunctionTypeArmAttributes {
     /// Any AArch64 SME ACLE type attributes that need to be propagated
     /// on declarations and function pointers.
-    unsigned AArch64SMEAttributes : 6;
-    FunctionTypeExtraBitfields()
-        : NumExceptionType(0), AArch64SMEAttributes(SME_NormalFunction) {}
+    unsigned AArch64SMEAttributes : 8;
+
+    FunctionTypeArmAttributes() : AArch64SMEAttributes(SME_NormalFunction) {}
   };
 
 protected:
@@ -4160,7 +4196,8 @@ class FunctionProtoType final
       public llvm::FoldingSetNode,
       private llvm::TrailingObjects<
           FunctionProtoType, QualType, SourceLocation,
-          FunctionType::FunctionTypeExtraBitfields, FunctionType::ExceptionType,
+          FunctionType::FunctionTypeExtraBitfields,
+          FunctionType::FunctionTypeArmAttributes, FunctionType::ExceptionType,
           Expr *, FunctionDecl *, FunctionType::ExtParameterInfo, Qualifiers> {
   friend class ASTContext; // ASTContext creates these.
   friend TrailingObjects;
@@ -4233,6 +4270,8 @@ class FunctionProtoType final
     ExceptionSpecInfo() = default;
 
     ExceptionSpecInfo(ExceptionSpecificationType EST) : Type(EST) {}
+
+    void instantiate();
   };
 
   /// Extra information about a function prototype. ExtProtoInfo is not
@@ -4242,7 +4281,7 @@ class FunctionProtoType final
     FunctionType::ExtInfo ExtInfo;
     unsigned Variadic : 1;
     unsigned HasTrailingReturn : 1;
-    unsigned AArch64SMEAttributes : 6;
+    unsigned AArch64SMEAttributes : 8;
     Qualifiers TypeQuals;
     RefQualifierKind RefQualifier = RQ_None;
     ExceptionSpecInfo ExceptionSpec;
@@ -4265,7 +4304,11 @@ class FunctionProtoType final
 
     bool requiresFunctionProtoTypeExtraBitfields() const {
       return ExceptionSpec.Type == EST_Dynamic ||
-             AArch64SMEAttributes != SME_NormalFunction;
+             requiresFunctionProtoTypeArmAttributes();
+    }
+
+    bool requiresFunctionProtoTypeArmAttributes() const {
+      return AArch64SMEAttributes != SME_NormalFunction;
     }
 
     void setArmSMEAttribute(AArch64SMETypeAttributes Kind, bool Enable = true) {
@@ -4285,6 +4328,10 @@ class FunctionProtoType final
     return isVariadic();
   }
 
+  unsigned numTrailingObjects(OverloadToken) const {
+    return hasArmTypeAttributes();
+  }
+
   unsigned numTrailingObjects(OverloadToken) const {
     return hasExtraBitfields();
   }
@@ -4373,6 +4420,12 @@ class FunctionProtoType final
 
   }
 
+  bool hasArmTypeAttributes() const {
+    return FunctionTypeBits.HasExtraBitfields &&
+           getTrailingObjects()
+               ->HasArmTypeAttributes;
+  }
+
   bool hasExtQualifiers() const {
     return FunctionTypeBits.HasExtQuals;
   }
@@ -4584,9 +4637,9 @@ class FunctionProtoType final
   /// Return a bitmask describing the SME attributes on the function type, see
   /// AArch64SMETypeAttributes for their values.
   unsigned getAArch64SMEAttributes() const {
-    if (!hasExtraBitfields())
+    if (!hasArmTypeAttributes())
       return SME_NormalFunction;
-    return getTrailingObjects()
+    return getTrailingObjects()
         ->AArch64SMEAttributes;
   }
 
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index 82a26356c58f..dc1f49525a00 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -2062,6 +2062,18 @@ extern const internal::VariadicDynCastAllOfMatcher
 extern const internal::VariadicDynCastAllOfMatcher
     cxxOperatorCallExpr;
 
+/// Matches C++17 fold expressions.
+///
+/// Example matches `(0 + ... + args)`:
+/// \code
+///   template 
+///   auto sum(Args... args) {
+///       return (0 + ... + args);
+///   }
+/// \endcode
+extern const internal::VariadicDynCastAllOfMatcher
+    cxxFoldExpr;
+
 /// Matches rewritten binary operators
 ///
 /// Example matches use of "<":
@@ -3881,7 +3893,7 @@ AST_MATCHER_P(ObjCMessageExpr, numSelectorArgs, unsigned, N) {
   return Node.getSelector().getNumArgs() == N;
 }
 
-/// Matches if the call expression's callee expression matches.
+/// Matches if the call or fold expression's callee expression matches.
 ///
 /// Given
 /// \code
@@ -3893,13 +3905,32 @@ AST_MATCHER_P(ObjCMessageExpr, numSelectorArgs, unsigned, N) {
 /// with callee(...)
 ///   matching this->x, x, y.x, f respectively
 ///
+/// Given
+/// \code
+///   template 
+///   auto sum(Args... args) {
+///       return (0 + ... + args);
+///   }
+///
+///   template 
+///   auto multiply(Args... args) {
+///       return (args * ... * 1);
+///   }
+/// \endcode
+/// cxxFoldExpr(callee(expr()))
+///   matches (args * ... * 1)
+/// with callee(...)
+///   matching *
+///
 /// Note: Callee cannot take the more general internal::Matcher
 /// because this introduces ambiguous overloads with calls to Callee taking a
 /// internal::Matcher, as the matcher hierarchy is purely
 /// implemented in terms of implicit casts.
-AST_MATCHER_P(CallExpr, callee, internal::Matcher,
-              InnerMatcher) {
-  const Expr *ExprNode = Node.getCallee();
+AST_POLYMORPHIC_MATCHER_P_OVERLOAD(callee,
+                                   AST_POLYMORPHIC_SUPPORTED_TYPES(CallExpr,
+                                                                   CXXFoldExpr),
+                                   internal::Matcher, InnerMatcher, 0) {
+  const auto *ExprNode = Node.getCallee();
   return (ExprNode != nullptr &&
           InnerMatcher.matches(*ExprNode, Finder, Builder));
 }
@@ -4532,6 +4563,121 @@ AST_POLYMORPHIC_MATCHER_P2(hasArgument,
   return InnerMatcher.matches(*Arg->IgnoreParenImpCasts(), Finder, Builder);
 }
 
+/// Matches the operand that does not contain the parameter pack.
+///
+/// Example matches `(0 + ... + args)` and `(args * ... * 1)`
+///     (matcher = cxxFoldExpr(hasFoldInit(expr())))
+///   with hasFoldInit(...)
+///     matching `0` and `1` respectively
+/// \code
+///   template 
+///   auto sum(Args... args) {
+///       return (0 + ... + args);
+///   }
+///
+///   template 
+///   auto multiply(Args... args) {
+///       return (args * ... * 1);
+///   }
+/// \endcode
+AST_MATCHER_P(CXXFoldExpr, hasFoldInit, ast_matchers::internal::Matcher,
+              InnerMacher) {
+  const auto *const Init = Node.getInit();
+  return Init && InnerMacher.matches(*Init, Finder, Builder);
+}
+
+/// Matches the operand that contains the parameter pack.
+///
+/// Example matches `(0 + ... + args)`
+///     (matcher = cxxFoldExpr(hasPattern(expr())))
+///   with hasPattern(...)
+///     matching `args`
+/// \code
+///   template 
+///   auto sum(Args... args) {
+///       return (0 + ... + args);
+///   }
+///
+///   template 
+///   auto multiply(Args... args) {
+///       return (args * ... * 1);
+///   }
+/// \endcode
+AST_MATCHER_P(CXXFoldExpr, hasPattern, ast_matchers::internal::Matcher,
+              InnerMacher) {
+  const Expr *const Pattern = Node.getPattern();
+  return Pattern && InnerMacher.matches(*Pattern, Finder, Builder);
+}
+
+/// Matches right-folding fold expressions.
+///
+/// Example matches `(args * ... * 1)`
+///     (matcher = cxxFoldExpr(isRightFold()))
+/// \code
+///   template 
+///   auto sum(Args... args) {
+///       return (0 + ... + args);
+///   }
+///
+///   template 
+///   auto multiply(Args... args) {
+///       return (args * ... * 1);
+///   }
+/// \endcode
+AST_MATCHER(CXXFoldExpr, isRightFold) { return Node.isRightFold(); }
+
+/// Matches left-folding fold expressions.
+///
+/// Example matches `(0 + ... + args)`
+///     (matcher = cxxFoldExpr(isLeftFold()))
+/// \code
+///   template 
+///   auto sum(Args... args) {
+///       return (0 + ... + args);
+///   }
+///
+///   template 
+///   auto multiply(Args... args) {
+///       return (args * ... * 1);
+///   }
+/// \endcode
+AST_MATCHER(CXXFoldExpr, isLeftFold) { return Node.isLeftFold(); }
+
+/// Matches unary fold expressions, i.e. fold expressions without an
+/// initializer.
+///
+/// Example matches `(args * ...)`
+///     (matcher = cxxFoldExpr(isUnaryFold()))
+/// \code
+///   template 
+///   auto sum(Args... args) {
+///       return (0 + ... + args);
+///   }
+///
+///   template 
+///   auto multiply(Args... args) {
+///       return (args * ...);
+///   }
+/// \endcode
+AST_MATCHER(CXXFoldExpr, isUnaryFold) { return Node.getInit() == nullptr; }
+
+/// Matches binary fold expressions, i.e. fold expressions with an initializer.
+///
+/// Example matches `(0 + ... + args)`
+///     (matcher = cxxFoldExpr(isBinaryFold()))
+/// \code
+///   template 
+///   auto sum(Args... args) {
+///       return (0 + ... + args);
+///   }
+///
+///   template 
+///   auto multiply(Args... args) {
+///       return (args * ...);
+///   }
+/// \endcode
+AST_MATCHER(CXXFoldExpr, isBinaryFold) { return Node.getInit() != nullptr; }
+
 /// Matches the n'th item of an initializer list expression.
 ///
 /// Example matches y.
@@ -5709,17 +5855,27 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(equals,
     .matchesNode(Node);
 }
 
-/// Matches the operator Name of operator expressions (binary or
-/// unary).
+/// Matches the operator Name of operator expressions and fold expressions
+/// (binary or unary).
 ///
 /// Example matches a || b (matcher = binaryOperator(hasOperatorName("||")))
 /// \code
 ///   !(a || b)
 /// \endcode
+///
+/// Example matches `(0 + ... + args)`
+///     (matcher = cxxFoldExpr(hasOperatorName("+")))
+/// \code
+///   template 
+///   auto sum(Args... args) {
+///       return (0 + ... + args);
+///   }
+/// \endcode
 AST_POLYMORPHIC_MATCHER_P(
     hasOperatorName,
     AST_POLYMORPHIC_SUPPORTED_TYPES(BinaryOperator, CXXOperatorCallExpr,
-                                    CXXRewrittenBinaryOperator, UnaryOperator),
+                                    CXXRewrittenBinaryOperator, CXXFoldExpr,
+                                    UnaryOperator),
     std::string, Name) {
   if (std::optional OpName = internal::getOpName(Node))
     return *OpName == Name;
@@ -5789,11 +5945,12 @@ AST_POLYMORPHIC_MATCHER(
 /// \code
 ///   a || b
 /// \endcode
-AST_POLYMORPHIC_MATCHER_P(hasLHS,
-                          AST_POLYMORPHIC_SUPPORTED_TYPES(
-                              BinaryOperator, CXXOperatorCallExpr,
-                              CXXRewrittenBinaryOperator, ArraySubscriptExpr),
-                          internal::Matcher, InnerMatcher) {
+AST_POLYMORPHIC_MATCHER_P(
+    hasLHS,
+    AST_POLYMORPHIC_SUPPORTED_TYPES(BinaryOperator, CXXOperatorCallExpr,
+                                    CXXRewrittenBinaryOperator,
+                                    ArraySubscriptExpr, CXXFoldExpr),
+    internal::Matcher, InnerMatcher) {
   const Expr *LeftHandSide = internal::getLHS(Node);
   return (LeftHandSide != nullptr &&
           InnerMatcher.matches(*LeftHandSide, Finder, Builder));
@@ -5805,29 +5962,31 @@ AST_POLYMORPHIC_MATCHER_P(hasLHS,
 /// \code
 ///   a || b
 /// \endcode
-AST_POLYMORPHIC_MATCHER_P(hasRHS,
-                          AST_POLYMORPHIC_SUPPORTED_TYPES(
-                              BinaryOperator, CXXOperatorCallExpr,
-                              CXXRewrittenBinaryOperator, ArraySubscriptExpr),
-                          internal::Matcher, InnerMatcher) {
+AST_POLYMORPHIC_MATCHER_P(
+    hasRHS,
+    AST_POLYMORPHIC_SUPPORTED_TYPES(BinaryOperator, CXXOperatorCallExpr,
+                                    CXXRewrittenBinaryOperator,
+                                    ArraySubscriptExpr, CXXFoldExpr),
+    internal::Matcher, InnerMatcher) {
   const Expr *RightHandSide = internal::getRHS(Node);
   return (RightHandSide != nullptr &&
           InnerMatcher.matches(*RightHandSide, Finder, Builder));
 }
 
 /// Matches if either the left hand side or the right hand side of a
-/// binary operator matches.
+/// binary operator or fold expression matches.
 AST_POLYMORPHIC_MATCHER_P(
     hasEitherOperand,
     AST_POLYMORPHIC_SUPPORTED_TYPES(BinaryOperator, CXXOperatorCallExpr,
-                                    CXXRewrittenBinaryOperator),
+                                    CXXFoldExpr, CXXRewrittenBinaryOperator),
     internal::Matcher, InnerMatcher) {
   return internal::VariadicDynCastAllOfMatcher()(
              anyOf(hasLHS(InnerMatcher), hasRHS(InnerMatcher)))
       .matches(Node, Finder, Builder);
 }
 
-/// Matches if both matchers match with opposite sides of the binary operator.
+/// Matches if both matchers match with opposite sides of the binary operator
+/// or fold expression.
 ///
 /// Example matcher = binaryOperator(hasOperands(integerLiteral(equals(1),
 ///                                              integerLiteral(equals(2)))
@@ -5840,7 +5999,7 @@ AST_POLYMORPHIC_MATCHER_P(
 AST_POLYMORPHIC_MATCHER_P2(
     hasOperands,
     AST_POLYMORPHIC_SUPPORTED_TYPES(BinaryOperator, CXXOperatorCallExpr,
-                                    CXXRewrittenBinaryOperator),
+                                    CXXFoldExpr, CXXRewrittenBinaryOperator),
     internal::Matcher, Matcher1, internal::Matcher, Matcher2) {
   return internal::VariadicDynCastAllOfMatcher()(
              anyOf(allOf(hasLHS(Matcher1), hasRHS(Matcher2)),
@@ -6192,9 +6351,7 @@ AST_POLYMORPHIC_MATCHER(isFinal,
 ///   };
 /// \endcode
 ///   matches A::x
-AST_MATCHER(CXXMethodDecl, isPure) {
-  return Node.isPure();
-}
+AST_MATCHER(CXXMethodDecl, isPure) { return Node.isPureVirtual(); }
 
 /// Matches if the given method declaration is const.
 ///
diff --git a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
index 7136d0d2fe08..47d912c73dd7 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
@@ -2195,6 +2195,9 @@ inline std::optional getOpName(const CXXOperatorCallExpr &Node) {
   }
   return BinaryOperator::getOpcodeStr(*optBinaryOpcode);
 }
+inline StringRef getOpName(const CXXFoldExpr &Node) {
+  return BinaryOperator::getOpcodeStr(Node.getOperator());
+}
 
 /// Matches overloaded operators with a specific name.
 ///
diff --git a/clang/include/clang/Analysis/Analyses/UnsafeBufferUsage.h b/clang/include/clang/Analysis/Analyses/UnsafeBufferUsage.h
index 8a2d56668e32..b28f2c6b99c5 100644
--- a/clang/include/clang/Analysis/Analyses/UnsafeBufferUsage.h
+++ b/clang/include/clang/Analysis/Analyses/UnsafeBufferUsage.h
@@ -66,7 +66,7 @@ class UnsafeBufferUsageHandler {
 
   /// Invoked when an unsafe operation over raw pointers is found.
   virtual void handleUnsafeOperation(const Stmt *Operation,
-                                     bool IsRelatedToDecl) = 0;
+                                     bool IsRelatedToDecl, ASTContext &Ctx) = 0;
 
   /// Invoked when a fix is suggested against a variable. This function groups
   /// all variables that must be fixed together (i.e their types must be changed
diff --git a/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def b/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def
index 757ee452ced7..c97661688365 100644
--- a/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def
+++ b/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def
@@ -30,6 +30,7 @@ WARNING_GADGET(Decrement)
 WARNING_GADGET(ArraySubscript)
 WARNING_GADGET(PointerArithmetic)
 WARNING_GADGET(UnsafeBufferUsageAttr)
+WARNING_GADGET(DataInvocation)
 FIXABLE_GADGET(ULCArraySubscript)          // `DRE[any]` in an Unspecified Lvalue Context
 FIXABLE_GADGET(DerefSimplePtrArithFixable)
 FIXABLE_GADGET(PointerDereference)
diff --git a/clang/include/clang/Analysis/CFG.h b/clang/include/clang/Analysis/CFG.h
index 67383bb316d3..9f776ca6cc26 100644
--- a/clang/include/clang/Analysis/CFG.h
+++ b/clang/include/clang/Analysis/CFG.h
@@ -1215,7 +1215,9 @@ class CFG {
   //===--------------------------------------------------------------------===//
 
   class BuildOptions {
-    std::bitset alwaysAddMask;
+    // Stmt::lastStmtConstant has the same value as the last Stmt kind,
+    // so make sure we add one to account for this!
+    std::bitset alwaysAddMask;
 
   public:
     using ForcedBlkExprs = llvm::DenseMap;
diff --git a/clang/include/clang/Analysis/FlowSensitive/ControlFlowContext.h b/clang/include/clang/Analysis/FlowSensitive/ControlFlowContext.h
index 768387a121b9..405e93287a05 100644
--- a/clang/include/clang/Analysis/FlowSensitive/ControlFlowContext.h
+++ b/clang/include/clang/Analysis/FlowSensitive/ControlFlowContext.h
@@ -32,7 +32,8 @@ namespace dataflow {
 class ControlFlowContext {
 public:
   /// Builds a ControlFlowContext from a `FunctionDecl`.
-  /// `Func.hasBody()` must be true, and `Func.isTemplated()` must be false.
+  /// `Func.doesThisDeclarationHaveABody()` must be true, and
+  /// `Func.isTemplated()` must be false.
   static llvm::Expected build(const FunctionDecl &Func);
 
   /// Builds a ControlFlowContext from an AST node. `D` is the function in which
diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h
index 1dffbe8a09c3..b95095d2184c 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h
@@ -186,6 +186,12 @@ template  struct DataflowAnalysisState {
 /// the dataflow analysis cannot be performed successfully. Otherwise, calls
 /// `PostVisitCFG` on each CFG element with the final analysis results at that
 /// program point.
+///
+/// `MaxBlockVisits` caps the number of block visits during analysis. See
+/// `runTypeErasedDataflowAnalysis` for a full description. The default value is
+/// essentially arbitrary -- large enough to accommodate what seems like any
+/// reasonable CFG, but still small enough to limit the cost of hitting the
+/// limit.
 template 
 llvm::Expected>>>
@@ -194,7 +200,8 @@ runDataflowAnalysis(
     const Environment &InitEnv,
     std::function &)>
-        PostVisitCFG = nullptr) {
+        PostVisitCFG = nullptr,
+    std::int32_t MaxBlockVisits = 20'000) {
   std::function
       PostVisitCFGClosure = nullptr;
@@ -212,7 +219,7 @@ runDataflowAnalysis(
   }
 
   auto TypeErasedBlockStates = runTypeErasedDataflowAnalysis(
-      CFCtx, Analysis, InitEnv, PostVisitCFGClosure);
+      CFCtx, Analysis, InitEnv, PostVisitCFGClosure, MaxBlockVisits);
   if (!TypeErasedBlockStates)
     return TypeErasedBlockStates.takeError();
 
@@ -261,6 +268,10 @@ auto createAnalysis(ASTContext &ASTCtx, Environment &Env)
 ///   iterations.
 /// - This limit is still low enough to keep runtimes acceptable (on typical
 ///   machines) in cases where we hit the limit.
+///
+/// `MaxBlockVisits` caps the number of block visits during analysis. See
+/// `runDataflowAnalysis` for a full description and explanation of the default
+/// value.
 template 
 llvm::Expected> diagnoseFunction(
     const FunctionDecl &FuncDecl, ASTContext &ASTCtx,
@@ -268,7 +279,8 @@ llvm::Expected> diagnoseFunction(
         const CFGElement &, ASTContext &,
         const TransferStateForDiagnostics &)>
         Diagnoser,
-    std::int64_t MaxSATIterations = 1'000'000'000) {
+    std::int64_t MaxSATIterations = 1'000'000'000,
+    std::int32_t MaxBlockVisits = 20'000) {
   llvm::Expected Context =
       ControlFlowContext::build(FuncDecl);
   if (!Context)
@@ -293,7 +305,8 @@ llvm::Expected> diagnoseFunction(
                             State.Lattice.Value),
                         State.Env));
                 llvm::move(EltDiagnostics, std::back_inserter(Diagnostics));
-              })
+              },
+              MaxBlockVisits)
               .takeError())
     return std::move(Err);
 
diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
index e8c27d6c1203..1543f900e401 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
@@ -172,7 +172,8 @@ class Environment {
   ///
   /// Requirements:
   ///
-  ///  The function must have a body.
+  ///  The function must have a body, i.e.
+  ///  `FunctionDecl::doesThisDecalarationHaveABody()` must be true.
   void initialize();
 
   /// Returns a new environment that is a copy of this one.
diff --git a/clang/include/clang/Analysis/FlowSensitive/Formula.h b/clang/include/clang/Analysis/FlowSensitive/Formula.h
index 982e400c1def..0e6352403a83 100644
--- a/clang/include/clang/Analysis/FlowSensitive/Formula.h
+++ b/clang/include/clang/Analysis/FlowSensitive/Formula.h
@@ -75,6 +75,10 @@ class alignas(const Formula *) Formula {
     return static_cast(Value);
   }
 
+  bool isLiteral(bool b) const {
+    return kind() == Literal && static_cast(Value) == b;
+  }
+
   ArrayRef operands() const {
     return ArrayRef(reinterpret_cast(this + 1),
                     numOperands(kind()));
diff --git a/clang/include/clang/Analysis/FlowSensitive/Transfer.h b/clang/include/clang/Analysis/FlowSensitive/Transfer.h
index 58bb77c4905c..7713df747cb7 100644
--- a/clang/include/clang/Analysis/FlowSensitive/Transfer.h
+++ b/clang/include/clang/Analysis/FlowSensitive/Transfer.h
@@ -25,10 +25,17 @@ namespace dataflow {
 /// Maps statements to the environments of basic blocks that contain them.
 class StmtToEnvMap {
 public:
+  // `CurBlockID` is the ID of the block currently being processed, and
+  // `CurState` is the pending state currently associated with this block. These
+  // are supplied separately as the pending state for the current block may not
+  // yet be represented in `BlockToState`.
   StmtToEnvMap(const ControlFlowContext &CFCtx,
                llvm::ArrayRef>
-                   BlockToState)
-      : CFCtx(CFCtx), BlockToState(BlockToState) {}
+                   BlockToState,
+               unsigned CurBlockID,
+               const TypeErasedDataflowAnalysisState &CurState)
+      : CFCtx(CFCtx), BlockToState(BlockToState), CurBlockID(CurBlockID),
+        CurState(CurState) {}
 
   /// Returns the environment of the basic block that contains `S`.
   /// The result is guaranteed never to be null.
@@ -37,6 +44,8 @@ class StmtToEnvMap {
 private:
   const ControlFlowContext &CFCtx;
   llvm::ArrayRef> BlockToState;
+  unsigned CurBlockID;
+  const TypeErasedDataflowAnalysisState &CurState;
 };
 
 /// Evaluates `S` and updates `Env` accordingly.
diff --git a/clang/include/clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h b/clang/include/clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h
index 67c323dbf45e..a0ca7440230b 100644
--- a/clang/include/clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h
+++ b/clang/include/clang/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.h
@@ -138,13 +138,20 @@ struct TypeErasedDataflowAnalysisState {
 /// dataflow analysis cannot be performed successfully. Otherwise, calls
 /// `PostVisitCFG` on each CFG element with the final analysis results at that
 /// program point.
+///
+/// `MaxBlockVisits` caps the number of block visits during analysis. It doesn't
+/// distinguish between repeat visits to the same block and visits to distinct
+/// blocks. This parameter is a backstop to prevent infinite loops, in the case
+/// of bugs in the lattice and/or transfer functions that prevent the analysis
+/// from converging.
 llvm::Expected>>
 runTypeErasedDataflowAnalysis(
     const ControlFlowContext &CFCtx, TypeErasedDataflowAnalysis &Analysis,
     const Environment &InitEnv,
     std::function
-        PostVisitCFG = nullptr);
+        PostVisitCFG,
+    std::int32_t MaxBlockVisits);
 
 } // namespace dataflow
 } // namespace clang
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 6d022411f6e3..2bed6f25ba46 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -146,6 +146,11 @@ def ExternalGlobalVar : SubsetSubjectisLocalExternDecl()}],
                              "external global variables">;
 
+def NonTLSGlobalVar : SubsetSubjecthasGlobalStorage() &&
+                               S->getTLSKind() == 0}],
+                             "non-TLS global variables">;
+
 def InlineFunction : SubsetSubjectisInlineSpecified()}], "inline functions">;
 
@@ -304,6 +309,9 @@ class VariadicEnumArgument values,
   bit IsExternalType = isExternalType;
 }
 
+// Represents an attribute wrapped by another attribute.
+class WrappedAttr : Argument;
+
 // This handles one spelling of an attribute.
 class Spelling {
   string Name = name;
@@ -437,6 +445,7 @@ def TargetAArch64 : TargetArch<["aarch64", "aarch64_be", "aarch64_32"]>;
 def TargetAnyArm : TargetArch;
 def TargetAVR : TargetArch<["avr"]>;
 def TargetBPF : TargetArch<["bpfel", "bpfeb"]>;
+def TargetLoongArch : TargetArch<["loongarch32", "loongarch64"]>;
 def TargetMips32 : TargetArch<["mips", "mipsel"]>;
 def TargetAnyMips : TargetArch<["mips", "mipsel", "mips64", "mips64el"]>;
 def TargetMSP430 : TargetArch<["msp430"]>;
@@ -1127,6 +1136,14 @@ def CoroLifetimeBound : InheritableAttr {
   let SimpleHandler = 1;
 }
 
+def CoroDisableLifetimeBound : InheritableAttr {
+  let Spellings = [Clang<"coro_disable_lifetimebound">];
+  let Subjects = SubjectList<[Function]>;
+  let LangOpts = [CPlusPlus];
+  let Documentation = [CoroLifetimeBoundDoc];
+  let SimpleHandler = 1;
+}
+
 // OSObject-based attributes.
 def OSConsumed : InheritableParamAttr {
   let Spellings = [Clang<"os_consumed">];
@@ -2283,7 +2300,7 @@ def ObjCBridgeRelated : InheritableAttr {
 def NSErrorDomain : InheritableAttr {
   let Spellings = [GNU<"ns_error_domain">];
   let Subjects = SubjectList<[Enum], ErrorDiag>;
-  let Args = [DeclArgument];
+  let Args = [IdentifierArgument<"ErrorDomain">];
   let Documentation = [NSErrorDomainDocs];
 }
 
@@ -2531,16 +2548,48 @@ def ArmStreamingCompatible : TypeAttr, TargetSpecificAttr {
   let Documentation = [ArmSmeStreamingCompatibleDocs];
 }
 
-def ArmSharedZA : TypeAttr, TargetSpecificAttr {
-  let Spellings = [RegularKeyword<"__arm_shared_za">];
+def ArmNew : InheritableAttr, TargetSpecificAttr {
+  let Spellings = [RegularKeyword<"__arm_new">];
+  let Args = [VariadicStringArgument<"NewArgs">];
+  let Subjects = SubjectList<[Function], ErrorDiag>;
+  let Documentation = [ArmNewDocs];
+
+  let AdditionalMembers = [{
+    bool isNewZA() const {
+      return llvm::is_contained(newArgs(), "za");
+    }
+    bool isNewZT0() const {
+      return llvm::is_contained(newArgs(), "zt0");
+    }
+  }];
+}
+
+def ArmIn : TypeAttr, TargetSpecificAttr {
+  let Spellings = [RegularKeyword<"__arm_in">];
+  let Args = [VariadicStringArgument<"InArgs">];
+  let Subjects = SubjectList<[HasFunctionProto], ErrorDiag>;
+  let Documentation = [ArmInDocs];
+}
+
+def ArmOut : TypeAttr, TargetSpecificAttr {
+  let Spellings = [RegularKeyword<"__arm_out">];
+  let Args = [VariadicStringArgument<"OutArgs">];
+  let Subjects = SubjectList<[HasFunctionProto], ErrorDiag>;
+  let Documentation = [ArmOutDocs];
+}
+
+def ArmInOut : TypeAttr, TargetSpecificAttr {
+  let Spellings = [RegularKeyword<"__arm_inout">];
+  let Args = [VariadicStringArgument<"InOutArgs">];
   let Subjects = SubjectList<[HasFunctionProto], ErrorDiag>;
-  let Documentation = [ArmSmeSharedZADocs];
+  let Documentation = [ArmInOutDocs];
 }
 
-def ArmPreservesZA : TypeAttr, TargetSpecificAttr {
-  let Spellings = [RegularKeyword<"__arm_preserves_za">];
+def ArmPreserves : TypeAttr, TargetSpecificAttr {
+  let Spellings = [RegularKeyword<"__arm_preserves">];
+  let Args = [VariadicStringArgument<"PreserveArgs">];
   let Subjects = SubjectList<[HasFunctionProto], ErrorDiag>;
-  let Documentation = [ArmSmePreservesZADocs];
+  let Documentation = [ArmPreservesDocs];
 }
 
 def ArmLocallyStreaming : InheritableAttr, TargetSpecificAttr {
@@ -2549,14 +2598,6 @@ def ArmLocallyStreaming : InheritableAttr, TargetSpecificAttr {
   let Documentation = [ArmSmeLocallyStreamingDocs];
 }
 
-def ArmNewZA : InheritableAttr, TargetSpecificAttr {
-  let Spellings = [RegularKeyword<"__arm_new_za">];
-  let Subjects = SubjectList<[Function], ErrorDiag>;
-  let Documentation = [ArmSmeNewZADocs];
-}
-def : MutualExclusions<[ArmNewZA, ArmSharedZA]>;
-def : MutualExclusions<[ArmNewZA, ArmPreservesZA]>;
-
 
 def Pure : InheritableAttr {
   let Spellings = [GCC<"pure">];
@@ -2619,6 +2660,22 @@ def SwiftError : InheritableAttr {
   let Documentation = [SwiftErrorDocs];
 }
 
+def SwiftImportAsNonGeneric : InheritableAttr {
+  // This attribute has no spellings as it is only ever created implicitly
+  // from API notes.
+  let Spellings = [];
+  let SemaHandler = 0;
+  let Documentation = [InternalOnly];
+}
+
+def SwiftImportPropertyAsAccessors : InheritableAttr {
+  // This attribute has no spellings as it is only ever created implicitly
+  // from API notes.
+  let Spellings = [];
+  let SemaHandler = 0;
+  let Documentation = [InternalOnly];
+}
+
 def SwiftName : InheritableAttr {
   let Spellings = [GNU<"swift_name">];
   let Args = [StringArgument<"Name">];
@@ -2640,6 +2697,31 @@ def SwiftPrivate : InheritableAttr {
   let SimpleHandler = 1;
 }
 
+def SwiftVersionedAddition : Attr {
+  // This attribute has no spellings as it is only ever created implicitly
+  // from API notes.
+  let Spellings = [];
+  let Args = [VersionArgument<"Version">, WrappedAttr<"AdditionalAttr">,
+              BoolArgument<"IsReplacedByActive">];
+  let SemaHandler = 0;
+  let Documentation = [InternalOnly];
+}
+
+def SwiftVersionedRemoval : Attr {
+  // This attribute has no spellings as it is only ever created implicitly
+  // from API notes.
+  let Spellings = [];
+  let Args = [VersionArgument<"Version">, UnsignedArgument<"RawKind">,
+              BoolArgument<"IsReplacedByActive">];
+  let SemaHandler = 0;
+  let Documentation = [InternalOnly];
+  let AdditionalMembers = [{
+    attr::Kind getAttrKindToRemove() const {
+      return static_cast(getRawKind());
+    }
+  }];
+}
+
 def NoDeref : TypeAttr {
   let Spellings = [Clang<"noderef">];
   let Documentation = [NoDerefDocs];
@@ -2736,6 +2818,15 @@ def PragmaClangTextSection : InheritableAttr {
   let Documentation = [InternalOnly];
 }
 
+def CodeModel : InheritableAttr, TargetSpecificAttr {
+  let Spellings = [GCC<"model">];
+  let Args = [EnumArgument<"Model", "llvm::CodeModel::Model",
+              ["normal", "medium", "extreme"], ["Small", "Medium", "Large"],
+              /*opt=*/0, /*fake=*/0, /*isExternalType=*/1>];
+  let Subjects = SubjectList<[NonTLSGlobalVar], ErrorDiag>;
+  let Documentation = [CodeModelDocs];
+}
+
 def Sentinel : InheritableAttr {
   let Spellings = [GCC<"sentinel">];
   let Args = [DefaultIntArgument<"Sentinel", 0>,
@@ -4368,3 +4459,21 @@ def CodeAlign: StmtAttr {
     static constexpr int MaximumAlignment = 4096;
   }];
 }
+
+def CountedBy : InheritableAttr {
+  let Spellings = [Clang<"counted_by">];
+  let Subjects = SubjectList<[Field]>;
+  let Args = [IdentifierArgument<"CountedByField">];
+  let Documentation = [CountedByDocs];
+  let LangOpts = [COnly];
+  // FIXME: This is ugly. Let using a DeclArgument would be nice, but a Decl
+  // isn't yet available due to the fact that we're still parsing the
+  // structure. Maybe that code could be changed sometime in the future.
+  code AdditionalMembers = [{
+    private:
+      SourceRange CountedByFieldLoc;
+    public:
+      SourceRange getCountedByFieldLoc() const { return CountedByFieldLoc; }
+      void setCountedByFieldLoc(SourceRange Loc) { CountedByFieldLoc = Loc; }
+  }];
+}
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 98a7ecc7fd7d..7e633f8e2635 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -57,6 +57,15 @@ global variable or function should be in after translation.
   let Heading = "section, __declspec(allocate)";
 }
 
+def CodeModelDocs : Documentation {
+  let Category = DocCatVariable;
+  let Content = [{
+The ``model`` attribute allows overriding the translation unit's
+code model (specified by ``-mcmodel``) for a specific global variable.
+  }];
+  let Heading = "model";
+}
+
 def UsedDocs : Documentation {
   let Category = DocCatFunction;
   let Content = [{
@@ -3612,7 +3621,7 @@ Attribute ``trivial_abi`` has no effect in the following cases:
 
 def MSInheritanceDocs : Documentation {
   let Category = DocCatDecl;
-  let Heading = "__single_inhertiance, __multiple_inheritance, __virtual_inheritance";
+  let Heading = "__single_inheritance, __multiple_inheritance, __virtual_inheritance";
   let Content = [{
 This collection of keywords is enabled under ``-fms-extensions`` and controls
 the pointer-to-member representation used on ``*-*-win32`` targets.
@@ -6852,30 +6861,73 @@ without changing modes.
   }];
 }
 
-def ArmSmeSharedZADocs : Documentation {
+def ArmInDocs : Documentation {
   let Category = DocCatArmSmeAttributes;
   let Content = [{
-The ``__arm_shared_za`` keyword applies to prototyped function types and specifies
-that the function shares SME's matrix storage (ZA) with its caller.  This
-means that:
+The ``__arm_in`` keyword applies to prototyped function types and specifies
+that the function shares a given state S with its caller.  For ``__arm_in``, the
+function takes the state S as input and returns with the state S unchanged.
 
-* the function requires that the processor implements the Scalable Matrix
-  Extension (SME).
+The attribute takes string arguments to instruct the compiler which state
+is shared.  The supported states for S are:
 
-* the function enters with ZA in an active state.
+* ``"za"`` for Matrix Storage (requires SME)
 
-* the function returns with ZA in an active state.
+The attributes ``__arm_in(S)``, ``__arm_out(S)``, ``__arm_inout(S)`` and
+``__arm_preserves(S)`` are all mutually exclusive for the same state S.
   }];
 }
 
-def ArmSmePreservesZADocs : Documentation {
+def ArmOutDocs : Documentation {
   let Category = DocCatArmSmeAttributes;
   let Content = [{
-The ``__arm_preserves_za`` keyword applies to prototyped function types and
-specifies that the function does not modify ZA state.
+The ``__arm_out`` keyword applies to prototyped function types and specifies
+that the function shares a given state S with its caller.  For ``__arm_out``,
+the function ignores the incoming state for S and returns new state for S.
+
+The attribute takes string arguments to instruct the compiler which state
+is shared.  The supported states for S are:
+
+* ``"za"`` for Matrix Storage (requires SME)
+
+The attributes ``__arm_in(S)``, ``__arm_out(S)``, ``__arm_inout(S)`` and
+``__arm_preserves(S)`` are all mutually exclusive for the same state S.
   }];
 }
 
+def ArmInOutDocs : Documentation {
+  let Category = DocCatArmSmeAttributes;
+  let Content = [{
+The ``__arm_inout`` keyword applies to prototyped function types and specifies
+that the function shares a given state S with its caller.  For ``__arm_inout``,
+the function takes the state S as input and returns new state for S.
+
+The attribute takes string arguments to instruct the compiler which state
+is shared.  The supported states for S are:
+
+* ``"za"`` for Matrix Storage (requires SME)
+
+The attributes ``__arm_in(S)``, ``__arm_out(S)``, ``__arm_inout(S)`` and
+``__arm_preserves(S)`` are all mutually exclusive for the same state S.
+  }];
+}
+
+def ArmPreservesDocs : Documentation {
+  let Category = DocCatArmSmeAttributes;
+  let Content = [{
+The ``__arm_preserves`` keyword applies to prototyped function types and
+specifies that the function does not read a given state S and returns
+with state S unchanged.
+
+The attribute takes string arguments to instruct the compiler which state
+is shared.  The supported states for S are:
+
+* ``"za"`` for Matrix Storage (requires SME)
+
+The attributes ``__arm_in(S)``, ``__arm_out(S)``, ``__arm_inout(S)`` and
+``__arm_preserves(S)`` are all mutually exclusive for the same state S.
+  }];
+}
 
 def ArmSmeLocallyStreamingDocs : Documentation {
   let Category = DocCatArmSmeAttributes;
@@ -6898,13 +6950,18 @@ at the end of the function.
   }];
 }
 
-def ArmSmeNewZADocs : Documentation {
+def ArmNewDocs : Documentation {
   let Category = DocCatArmSmeAttributes;
   let Content = [{
-The ``__arm_new_za`` keyword applies to function declarations and specifies
-that the function will be set up with a fresh ZA context.
+The ``__arm_new`` keyword applies to function declarations and specifies
+that the function will create a new scope for state S.
+
+The attribute takes string arguments to instruct the compiler for which state
+to create new scope.  The supported states for S are:
 
-This means that:
+* ``"za"`` for Matrix Storage (requires SME)
+
+For state ``"za"``, this means that:
 
 * the function requires that the target processor implements the Scalable Matrix
   Extension (SME).
@@ -6915,8 +6972,8 @@ This means that:
 
 * the function will disable PSTATE.ZA (by setting it to 0) before returning.
 
-For ``__arm_new_za`` functions Clang will set up the ZA context automatically
-on entry to the function, and disable it before returning. For example, if ZA is
+For ``__arm_new("za")`` functions Clang will set up the ZA context automatically
+on entry to the function and disable it before returning. For example, if ZA is
 in a dormant state Clang will generate the code to commit a lazy-save and set up
 a new ZA state before executing user code.
   }];
@@ -7671,9 +7728,12 @@ The ``[[clang::coro_lifetimebound]]`` is a class attribute which can be applied
 to a coroutine return type (`CRT`_) (i.e.
 it should also be annotated with ``[[clang::coro_return_type]]``).
 
-All parameters of a function are considered to be lifetime bound. See `documentation`_
-of ``[[clang::lifetimebound]]`` for more details.
-if the function returns a coroutine return type (CRT) annotated with ``[[clang::coro_lifetimebound]]``.
+All parameters of a function are considered to be lifetime bound if the function returns a
+coroutine return type (CRT) annotated with ``[[clang::coro_lifetimebound]]``.
+This lifetime bound analysis can be disabled for a coroutine wrapper or a coroutine by annotating the function
+with ``[[clang::coro_disable_lifetimebound]]`` function attribute .
+See `documentation`_ of ``[[clang::lifetimebound]]`` for details about lifetime bound analysis.
+
 
 Reference parameters of a coroutine are susceptible to capturing references to temporaries or local variables.
 
@@ -7703,7 +7763,7 @@ Both coroutines and coroutine wrappers are part of this analysis.
   };
 
   Task coro(const int& a) { co_return a + 1; }
-  Task [[clang::coro_wrapper]] coro_wrapper(const int& a, const int& b) {
+  [[clang::coro_wrapper]] Task coro_wrapper(const int& a, const int& b) {
     return a > b ? coro(a) : coro(b);
   }
   Task temporary_reference() {
@@ -7718,7 +7778,100 @@ Both coroutines and coroutine wrappers are part of this analysis.
     return coro(a); // warning: returning address of stack variable `a`.
   }
 
+This analysis can be disabled for all calls to a particular function by annotating the function
+with function attribute ``[[clang::coro_disable_lifetimebound]]``.
+For example, this could be useful for coroutine wrappers which accept reference parameters
+but do not pass them to the underlying coroutine or pass them by value.
+
+.. code-block:: c++
+
+  Task coro(int a) { co_return a + 1; }
+  [[clang::coro_wrapper, clang::coro_disable_lifetimebound]] Task coro_wrapper(const int& a) {
+    return coro(a + 1);
+  }
+  void use() {
+    auto task = coro_wrapper(1); // use of temporary is fine as the argument is not lifetime bound.
+  }
+
 .. _`documentation`: https://clang.llvm.org/docs/AttributeReference.html#lifetimebound
 .. _`CRT`: https://clang.llvm.org/docs/AttributeReference.html#coro-return-type
 }];
 }
+
+def CountedByDocs : Documentation {
+  let Category = DocCatField;
+  let Content = [{
+Clang supports the ``counted_by`` attribute on the flexible array member of a
+structure in C. The argument for the attribute is the name of a field member
+holding the count of elements in the flexible array. This information can be
+used to improve the results of the array bound sanitizer and the
+``__builtin_dynamic_object_size`` builtin. The ``count`` field member must be
+within the same non-anonymous, enclosing struct as the flexible array member.
+
+This example specifies that the flexible array member ``array`` has the number
+of elements allocated for it in ``count``:
+
+.. code-block:: c
+
+  struct bar;
+
+  struct foo {
+    size_t count;
+    char other;
+    struct bar *array[] __attribute__((counted_by(count)));
+  };
+
+This establishes a relationship between ``array`` and ``count``. Specifically,
+``array`` must have at least ``count`` number of elements available. It's the
+user's responsibility to ensure that this relationship is maintained through
+changes to the structure.
+
+In the following example, the allocated array erroneously has fewer elements
+than what's specified by ``p->count``. This would result in an out-of-bounds
+access not being detected.
+
+.. code-block:: c
+
+  #define SIZE_INCR 42
+
+  struct foo *p;
+
+  void foo_alloc(size_t count) {
+    p = malloc(MAX(sizeof(struct foo),
+                   offsetof(struct foo, array[0]) + count * sizeof(struct bar *)));
+    p->count = count + SIZE_INCR;
+  }
+
+The next example updates ``p->count``, but breaks the relationship requirement
+that ``p->array`` must have at least ``p->count`` number of elements available:
+
+.. code-block:: c
+
+  #define SIZE_INCR 42
+
+  struct foo *p;
+
+  void foo_alloc(size_t count) {
+    p = malloc(MAX(sizeof(struct foo),
+                   offsetof(struct foo, array[0]) + count * sizeof(struct bar *)));
+    p->count = count;
+  }
+
+  void use_foo(int index, int val) {
+    p->count += SIZE_INCR + 1; /* 'count' is now larger than the number of elements of 'array'. */
+    p->array[index] = val;     /* The sanitizer can't properly check this access. */
+  }
+
+In this example, an update to ``p->count`` maintains the relationship
+requirement:
+
+.. code-block:: c
+
+  void use_foo(int index, int val) {
+    if (p->count == 0)
+      return;
+    --p->count;
+    p->array[index] = val;
+  }
+  }];
+}
diff --git a/clang/include/clang/Basic/AttributeCommonInfo.h b/clang/include/clang/Basic/AttributeCommonInfo.h
index 018b92fdc11f..ef2ddf525c98 100644
--- a/clang/include/clang/Basic/AttributeCommonInfo.h
+++ b/clang/include/clang/Basic/AttributeCommonInfo.h
@@ -255,6 +255,19 @@ class AttributeCommonInfo {
     return SpellingIndex != SpellingNotCalculated;
   }
 };
+
+inline bool doesKeywordAttributeTakeArgs(tok::TokenKind Kind) {
+  switch (Kind) {
+  default:
+    return false;
+#define KEYWORD_ATTRIBUTE(NAME, HASARG, ...)                                   \
+  case tok::kw_##NAME:                                                         \
+    return HASARG;
+#include "clang/Basic/RegularKeywordAttrInfo.inc"
+#undef KEYWORD_ATTRIBUTE
+  }
+}
+
 } // namespace clang
 
 #endif // LLVM_CLANG_BASIC_ATTRIBUTECOMMONINFO_H
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.def b/clang/include/clang/Basic/BuiltinsAArch64.def
index 82a1ba3c82ad..31ec84143f65 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.def
+++ b/clang/include/clang/Basic/BuiltinsAArch64.def
@@ -68,6 +68,9 @@ TARGET_BUILTIN(__builtin_arm_ldg, "v*v*", "t", "mte")
 TARGET_BUILTIN(__builtin_arm_stg, "vv*", "t", "mte")
 TARGET_BUILTIN(__builtin_arm_subp, "Uiv*v*", "t", "mte")
 
+// SME state function
+BUILTIN(__builtin_arm_get_sme_state, "vULi*ULi*", "n")
+
 // Memory Operations
 TARGET_BUILTIN(__builtin_arm_mops_memset_tag, "v*v*iz", "", "mte,mops")
 
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index e562ef04a301..d208342d9c51 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -255,6 +255,10 @@ TARGET_BUILTIN(__builtin_amdgcn_sudot4, "iIbiIbiiIb", "nc", "dot8-insts")
 TARGET_BUILTIN(__builtin_amdgcn_sdot8, "SiSiSiSiIb", "nc", "dot1-insts")
 TARGET_BUILTIN(__builtin_amdgcn_udot8, "UiUiUiUiIb", "nc", "dot7-insts")
 TARGET_BUILTIN(__builtin_amdgcn_sudot8, "iIbiIbiiIb", "nc", "dot8-insts")
+TARGET_BUILTIN(__builtin_amdgcn_dot4_f32_fp8_bf8, "fUiUif", "nc", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_dot4_f32_bf8_fp8, "fUiUif", "nc", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_dot4_f32_fp8_fp8, "fUiUif", "nc", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_dot4_f32_bf8_bf8, "fUiUif", "nc", "gfx12-insts")
 
 //===----------------------------------------------------------------------===//
 // GFX10+ only builtins.
@@ -397,19 +401,20 @@ TARGET_BUILTIN(__builtin_amdgcn_smfmac_f32_32x32x32_bf8_fp8, "V16fV2iV4iV16fiIiI
 TARGET_BUILTIN(__builtin_amdgcn_smfmac_f32_32x32x32_fp8_bf8, "V16fV2iV4iV16fiIiIi", "nc", "fp8-insts")
 TARGET_BUILTIN(__builtin_amdgcn_smfmac_f32_32x32x32_fp8_fp8, "V16fV2iV4iV16fiIiIi", "nc", "fp8-insts")
 
-TARGET_BUILTIN(__builtin_amdgcn_cvt_f32_bf8, "fiIi", "nc", "fp8-insts")
-TARGET_BUILTIN(__builtin_amdgcn_cvt_f32_fp8, "fiIi", "nc", "fp8-insts")
-TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_f32_bf8, "V2fiIb", "nc", "fp8-insts")
-TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_f32_fp8, "V2fiIb", "nc", "fp8-insts")
-TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_bf8_f32, "iffiIb", "nc", "fp8-insts")
-TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_fp8_f32, "iffiIb", "nc", "fp8-insts")
-TARGET_BUILTIN(__builtin_amdgcn_cvt_sr_bf8_f32, "ifiiIi", "nc", "fp8-insts")
-TARGET_BUILTIN(__builtin_amdgcn_cvt_sr_fp8_f32, "ifiiIi", "nc", "fp8-insts")
+TARGET_BUILTIN(__builtin_amdgcn_cvt_f32_bf8, "fiIi", "nc", "fp8-conversion-insts")
+TARGET_BUILTIN(__builtin_amdgcn_cvt_f32_fp8, "fiIi", "nc", "fp8-conversion-insts")
+TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_f32_bf8, "V2fiIb", "nc", "fp8-conversion-insts")
+TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_f32_fp8, "V2fiIb", "nc", "fp8-conversion-insts")
+TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_bf8_f32, "iffiIb", "nc", "fp8-conversion-insts")
+TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_fp8_f32, "iffiIb", "nc", "fp8-conversion-insts")
+TARGET_BUILTIN(__builtin_amdgcn_cvt_sr_bf8_f32, "ifiiIi", "nc", "fp8-conversion-insts")
+TARGET_BUILTIN(__builtin_amdgcn_cvt_sr_fp8_f32, "ifiiIi", "nc", "fp8-conversion-insts")
 
 //===----------------------------------------------------------------------===//
 // GFX12+ only builtins.
 //===----------------------------------------------------------------------===//
 
+TARGET_BUILTIN(__builtin_amdgcn_s_sleep_var, "vUi", "n", "gfx12-insts")
 TARGET_BUILTIN(__builtin_amdgcn_permlane16_var,  "UiUiUiUiIbIb", "nc", "gfx12-insts")
 TARGET_BUILTIN(__builtin_amdgcn_permlanex16_var, "UiUiUiUiIbIb", "nc", "gfx12-insts")
 TARGET_BUILTIN(__builtin_amdgcn_s_barrier_signal, "vIi", "n", "gfx12-insts")
@@ -423,6 +428,13 @@ TARGET_BUILTIN(__builtin_amdgcn_s_wakeup_barrier, "vi", "n", "gfx12-insts")
 TARGET_BUILTIN(__builtin_amdgcn_s_barrier_leave, "b", "n", "gfx12-insts")
 TARGET_BUILTIN(__builtin_amdgcn_s_get_barrier_state, "Uii", "n", "gfx12-insts")
 
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v2i32, "V2iV2i*1", "nc", "gfx12-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v8i16, "V8sV8s*1", "nc", "gfx12-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v8f16, "V8hV8h*1", "nc", "gfx12-insts,wavefrontsize32")
+
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_i32, "ii*1", "nc", "gfx12-insts,wavefrontsize64")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v4i16, "V4sV4s*1", "nc", "gfx12-insts,wavefrontsize64")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v4f16, "V4hV4h*1", "nc", "gfx12-insts,wavefrontsize64")
 
 #undef BUILTIN
 #undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsLoongArchBase.def b/clang/include/clang/Basic/BuiltinsLoongArchBase.def
index cbb239223aae..a5a07c167908 100644
--- a/clang/include/clang/Basic/BuiltinsLoongArchBase.def
+++ b/clang/include/clang/Basic/BuiltinsLoongArchBase.def
@@ -51,3 +51,8 @@ TARGET_BUILTIN(__builtin_loongarch_iocsrwr_d, "vUWiUi", "nc", "64bit")
 
 TARGET_BUILTIN(__builtin_loongarch_lddir_d, "WiWiIUWi", "nc", "64bit")
 TARGET_BUILTIN(__builtin_loongarch_ldpte_d, "vWiIUWi", "nc", "64bit")
+
+TARGET_BUILTIN(__builtin_loongarch_frecipe_s, "ff", "nc", "f,frecipe")
+TARGET_BUILTIN(__builtin_loongarch_frecipe_d, "dd", "nc", "d,frecipe")
+TARGET_BUILTIN(__builtin_loongarch_frsqrte_s, "ff", "nc", "f,frecipe")
+TARGET_BUILTIN(__builtin_loongarch_frsqrte_d, "dd", "nc", "d,frecipe")
diff --git a/clang/include/clang/Basic/BuiltinsLoongArchLASX.def b/clang/include/clang/Basic/BuiltinsLoongArchLASX.def
index 3de200f665b6..4cf51cc000f6 100644
--- a/clang/include/clang/Basic/BuiltinsLoongArchLASX.def
+++ b/clang/include/clang/Basic/BuiltinsLoongArchLASX.def
@@ -657,9 +657,15 @@ TARGET_BUILTIN(__builtin_lasx_xvfsqrt_d, "V4dV4d", "nc", "lasx")
 TARGET_BUILTIN(__builtin_lasx_xvfrecip_s, "V8fV8f", "nc", "lasx")
 TARGET_BUILTIN(__builtin_lasx_xvfrecip_d, "V4dV4d", "nc", "lasx")
 
+TARGET_BUILTIN(__builtin_lasx_xvfrecipe_s, "V8fV8f", "nc", "lasx,frecipe")
+TARGET_BUILTIN(__builtin_lasx_xvfrecipe_d, "V4dV4d", "nc", "lasx,frecipe")
+
 TARGET_BUILTIN(__builtin_lasx_xvfrsqrt_s, "V8fV8f", "nc", "lasx")
 TARGET_BUILTIN(__builtin_lasx_xvfrsqrt_d, "V4dV4d", "nc", "lasx")
 
+TARGET_BUILTIN(__builtin_lasx_xvfrsqrte_s, "V8fV8f", "nc", "lasx,frecipe")
+TARGET_BUILTIN(__builtin_lasx_xvfrsqrte_d, "V4dV4d", "nc", "lasx,frecipe")
+
 TARGET_BUILTIN(__builtin_lasx_xvfcvtl_s_h, "V8fV16s", "nc", "lasx")
 TARGET_BUILTIN(__builtin_lasx_xvfcvth_s_h, "V8fV16s", "nc", "lasx")
 TARGET_BUILTIN(__builtin_lasx_xvfcvtl_d_s, "V4dV8f", "nc", "lasx")
diff --git a/clang/include/clang/Basic/BuiltinsLoongArchLSX.def b/clang/include/clang/Basic/BuiltinsLoongArchLSX.def
index 8e6aec886c50..c90f4dc5458f 100644
--- a/clang/include/clang/Basic/BuiltinsLoongArchLSX.def
+++ b/clang/include/clang/Basic/BuiltinsLoongArchLSX.def
@@ -641,9 +641,15 @@ TARGET_BUILTIN(__builtin_lsx_vfsqrt_d, "V2dV2d", "nc", "lsx")
 TARGET_BUILTIN(__builtin_lsx_vfrecip_s, "V4fV4f", "nc", "lsx")
 TARGET_BUILTIN(__builtin_lsx_vfrecip_d, "V2dV2d", "nc", "lsx")
 
+TARGET_BUILTIN(__builtin_lsx_vfrecipe_s, "V4fV4f", "nc", "lsx,frecipe")
+TARGET_BUILTIN(__builtin_lsx_vfrecipe_d, "V2dV2d", "nc", "lsx,frecipe")
+
 TARGET_BUILTIN(__builtin_lsx_vfrsqrt_s, "V4fV4f", "nc", "lsx")
 TARGET_BUILTIN(__builtin_lsx_vfrsqrt_d, "V2dV2d", "nc", "lsx")
 
+TARGET_BUILTIN(__builtin_lsx_vfrsqrte_s, "V4fV4f", "nc", "lsx,frecipe")
+TARGET_BUILTIN(__builtin_lsx_vfrsqrte_d, "V2dV2d", "nc", "lsx,frecipe")
+
 TARGET_BUILTIN(__builtin_lsx_vfcvtl_s_h, "V4fV8s", "nc", "lsx")
 TARGET_BUILTIN(__builtin_lsx_vfcvtl_d_s, "V2dV4f", "nc", "lsx")
 
diff --git a/clang/include/clang/Basic/BuiltinsPPC.def b/clang/include/clang/Basic/BuiltinsPPC.def
index a35488ed3dfa..88ae0ce94085 100644
--- a/clang/include/clang/Basic/BuiltinsPPC.def
+++ b/clang/include/clang/Basic/BuiltinsPPC.def
@@ -110,6 +110,10 @@ BUILTIN(__builtin_ppc_fctiw, "dd", "")
 BUILTIN(__builtin_ppc_fctiwz, "dd", "")
 BUILTIN(__builtin_ppc_fctudz, "dd", "")
 BUILTIN(__builtin_ppc_fctuwz, "dd", "")
+
+// fence builtin prevents all instructions moved across it
+BUILTIN(__builtin_ppc_fence, "v", "")
+
 BUILTIN(__builtin_ppc_swdiv_nochk, "ddd", "")
 BUILTIN(__builtin_ppc_swdivs_nochk, "fff", "")
 BUILTIN(__builtin_ppc_alignx, "vIivC*", "nc")
diff --git a/clang/include/clang/Basic/CMakeLists.txt b/clang/include/clang/Basic/CMakeLists.txt
index 28baa2e45e42..be216150c71b 100644
--- a/clang/include/clang/Basic/CMakeLists.txt
+++ b/clang/include/clang/Basic/CMakeLists.txt
@@ -45,10 +45,10 @@ clang_tablegen(AttrSubMatchRulesList.inc -gen-clang-attr-subject-match-rule-list
   SOURCE Attr.td
   TARGET ClangAttrSubjectMatchRuleList)
 
-clang_tablegen(AttrTokenKinds.inc -gen-clang-attr-token-kinds
+clang_tablegen(RegularKeywordAttrInfo.inc -gen-clang-regular-keyword-attr-info
   -I ${CMAKE_CURRENT_SOURCE_DIR}/../../
   SOURCE Attr.td
-  TARGET ClangAttrTokenKinds
+  TARGET ClangRegularKeywordAttrInfo
   )
 
 clang_tablegen(AttrHasAttributeImpl.inc -gen-clang-attr-has-attribute-impl
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index 0acb5ae134ea..2f2e45d5cf63 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -12,6 +12,9 @@
 // that have enumeration type and VALUE_CODEGENOPT is a code
 // generation option that describes a value rather than a flag.
 //
+// AFFECTING_VALUE_CODEGENOPT is used for code generation options that can
+// affect the AST.
+//
 //===----------------------------------------------------------------------===//
 #ifndef CODEGENOPT
 #  error Define the CODEGENOPT macro to handle language options
@@ -27,6 +30,11 @@ CODEGENOPT(Name, Bits, Default)
 CODEGENOPT(Name, Bits, Default)
 #endif
 
+#ifndef AFFECTING_VALUE_CODEGENOPT
+#  define AFFECTING_VALUE_CODEGENOPT(Name, Bits, Default) \
+VALUE_CODEGENOPT(Name, Bits, Default)
+#endif
+
 CODEGENOPT(DisableIntegratedAS, 1, 0) ///< -no-integrated-as
 CODEGENOPT(RelaxELFRelocations, 1, 1) ///< -Wa,-mrelax-relocations={yes,no}
 CODEGENOPT(AsmVerbose        , 1, 0) ///< -dA, -fverbose-asm.
@@ -193,8 +201,10 @@ ENUM_CODEGENOPT(ObjCDispatchMethod, ObjCDispatchMethodKind, 2, Legacy)
 CODEGENOPT(ObjCConvertMessagesToRuntimeCalls , 1, 1)
 CODEGENOPT(ObjCAvoidHeapifyLocalBlocks, 1, 0)
 
-VALUE_CODEGENOPT(OptimizationLevel, 2, 0) ///< The -O[0-3] option specified.
-VALUE_CODEGENOPT(OptimizeSize, 2, 0) ///< If -Os (==1) or -Oz (==2) is specified.
+
+// The optimization options affect frontend options, whicn in turn do affect the AST.
+AFFECTING_VALUE_CODEGENOPT(OptimizationLevel, 2, 0) ///< The -O[0-3] option specified.
+AFFECTING_VALUE_CODEGENOPT(OptimizeSize, 2, 0) ///< If -Os (==1) or -Oz (==2) is specified.
 
 CODEGENOPT(AtomicProfileUpdate , 1, 0) ///< Set -fprofile-update=atomic
 /// Choose profile instrumenation kind or no instrumentation.
@@ -209,6 +219,7 @@ CODEGENOPT(CoverageMapping , 1, 0) ///< Generate coverage mapping regions to
                                    ///< enable code coverage analysis.
 CODEGENOPT(DumpCoverageMapping , 1, 0) ///< Dump the generated coverage mapping
                                        ///< regions.
+CODEGENOPT(MCDCCoverage , 1, 0) ///< Enable MC/DC code coverage criteria.
 
   /// If -fpcc-struct-return or -freg-struct-return is specified.
 ENUM_CODEGENOPT(StructReturnConvention, StructReturnConventionKind, 2, SRCK_Default)
@@ -436,3 +447,4 @@ CODEGENOPT(CtorDtorReturnThis, 1, 0)
 #undef CODEGENOPT
 #undef ENUM_CODEGENOPT
 #undef VALUE_CODEGENOPT
+#undef AFFECTING_VALUE_CODEGENOPT
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index 6952b48e898a..3f8fe385fef3 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -494,6 +494,9 @@ class CodeGenOptions : public CodeGenOptionsBase {
     return getProfileInstr() == ProfileCSIRInstr;
   }
 
+  /// Check if any form of instrumentation is on.
+  bool hasProfileInstr() const { return getProfileInstr() != ProfileNone; }
+
   /// Check if Clang profile use is on.
   bool hasProfileClangUse() const {
     return getProfileUse() == ProfileClangInstr;
diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td
index 65a33f61a694..b1bada65cb6b 100644
--- a/clang/include/clang/Basic/DiagnosticCommonKinds.td
+++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td
@@ -70,6 +70,10 @@ def warn_pragma_debug_missing_argument : Warning<
 def warn_pragma_debug_unexpected_argument : Warning<
   "unexpected argument to debug command">, InGroup;
 
+def warn_fp_nan_inf_when_disabled : Warning<
+  "use of %select{infinity|NaN}0%select{| via a macro}1 is undefined behavior "
+  "due to the currently enabled floating-point options">,
+  InGroup>;
 }
 
 // Parse && Sema
@@ -349,15 +353,18 @@ def warn_invalid_feature_combination : Warning<
 def warn_target_unrecognized_env : Warning<
   "mismatch between architecture and environment in target triple '%0'; did you mean '%1'?">,
   InGroup;
+def warn_knl_knm_isa_support_removed : Warning<
+  "KNL, KNM related Intel Xeon Phi CPU's specific ISA's supports will be removed in LLVM 19.">,
+  InGroup>;
 
 // Source manager
 def err_cannot_open_file : Error<"cannot open file '%0': %1">, DefaultFatal;
 def err_file_modified : Error<
   "file '%0' modified since it was first processed">, DefaultFatal;
 def err_file_too_large : Error<
-  "sorry, unsupported: file '%0' is too large for Clang to process">;
+  "file '%0' is too large for Clang to process">;
 def err_sloc_space_too_large : Error<
-  "sorry, the translation unit is too large for Clang to process: ran out of source locations">, DefaultFatal;
+  "translation unit is too large for Clang to process: ran out of source locations">, DefaultFatal;
 def err_unsupported_bom : Error<"%0 byte order mark detected in '%1', but "
   "encoding is not supported">, DefaultFatal;
 def err_unable_to_rename_temp : Error<
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 676f1a62b49d..094fe1950941 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -427,6 +427,8 @@ def warn_drv_clang_unsupported : Warning<
   "the clang compiler does not support '%0'">;
 def warn_drv_deprecated_arg : Warning<
   "argument '%0' is deprecated, use '%1' instead">, InGroup;
+def warn_drv_deprecated_custom : Warning<
+  "argument '%0' is deprecated, %1">, InGroup;
 def warn_drv_assuming_mfloat_abi_is : Warning<
   "unknown platform, assuming -mfloat-abi=%0">;
 def warn_drv_unsupported_float_abi_by_lib : Warning<
@@ -472,7 +474,7 @@ def warn_unsupported_branch_protection: Warning <
 def err_sls_hardening_arm_not_supported : Error<
   "-mharden-sls is only supported on armv7-a or later">;
 def warn_drv_large_data_threshold_invalid_code_model: Warning<
-  "'%0' only applies to medium code model">,
+  "'%0' only applies to medium and large code models">,
   InGroup;
 
 def note_drv_command_failed_diag_msg : Note<
@@ -656,6 +658,13 @@ def err_drv_trivial_auto_var_init_stop_after_missing_dependency : Error<
 def err_drv_trivial_auto_var_init_stop_after_invalid_value : Error<
   "'-ftrivial-auto-var-init-stop-after=*' only accepts positive integers">;
 
+def err_drv_trivial_auto_var_init_max_size_missing_dependency : Error<
+  "'-ftrivial-auto-var-init-max-size=*' is used without "
+  "'-ftrivial-auto-var-init=zero' or '-ftrivial-auto-var-init=pattern'">;
+
+def err_drv_trivial_auto_var_init_max_size_invalid_value : Error<
+  "'-ftrivial-auto-var-init-max-size=*' only accepts positive integers (in bytes)">;
+
 def warn_drv_msp430_hwmult_unsupported : Warning<
   "the given MCU does not support hardware multiply, but '-mhwmult' is set to "
   "%0">, InGroup;
@@ -735,10 +744,10 @@ def err_drv_dxc_missing_target_profile : Error<
 def err_drv_hlsl_unsupported_target : Error<
   "HLSL code generation is unsupported for target '%0'">;
 def err_drv_hlsl_bad_shader_required_in_target : Error<
-  "shader %select{model|stage}0 is required in target '%1' for HLSL code generation">;
+  "%select{shader model|Vulkan environment|shader stage}0 is required as %select{OS|environment}1 in target '%2' for HLSL code generation">;
 
 def err_drv_hlsl_bad_shader_unsupported : Error<
-  "shader %select{model|stage}0 '%1' in target '%2' is invalid for HLSL code generation">;
+  "%select{shader model|Vulkan environment|shader stage}0 '%1' in target '%2' is invalid for HLSL code generation">;
 def warn_drv_dxc_missing_dxv : Warning<"dxv not found. "
     "Resulting DXIL will not be validated or signed for use in release environments.">,
     InGroup;
@@ -786,4 +795,7 @@ def warn_android_unversioned_fallback : Warning<
   " directories will not be used in Clang 19. Provide a versioned directory"
   " for the target version or lower instead.">,
   InGroup>;
+
+def err_drv_triple_version_invalid : Error<
+  "version '%0' in target triple '%1' is invalid">;
 }
diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
index 568000106a84..85ecfdf9de62 100644
--- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td
+++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
@@ -167,7 +167,7 @@ def err_verify_no_such_marker : Error<
 def err_verify_missing_start : Error<
     "cannot find start ('{{') of expected %0">;
 def err_verify_missing_end : Error<
-    "cannot find end ('}}') of expected %0">;
+    "cannot find end ('%1') of expected %0">;
 def err_verify_invalid_content : Error<
     "invalid expected %0: %1">;
 def err_verify_missing_regex : Error<
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index e4b1069cde18..a30ab27566ec 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1364,6 +1364,17 @@ def err_acc_invalid_clause : Error<"invalid OpenACC clause %0">;
 def err_acc_missing_directive : Error<"expected OpenACC directive">;
 def err_acc_invalid_open_paren
     : Error<"expected clause-list or newline in OpenACC directive">;
+def err_acc_invalid_default_clause_kind
+    : Error<"invalid value for 'default' clause; expected 'present' or 'none'">;
+def err_acc_invalid_tag_kind
+    : Error<"invalid tag %0 on '%1' %select{directive|clause}2">;
+def err_acc_expected_reduction_operator
+    : Error<"missing reduction operator, expected '+', '*', 'max', 'min', '&', "
+            "'|', '^', '&&', or '||', follwed by a ':'">;
+def err_acc_invalid_reduction_operator
+    : Error<"invalid reduction operator,  expected '+', '*', 'max', 'min', "
+            "'&', '|', '^', '&&', or '||'">;
+def err_acc_incorrect_bind_arg : Error<"expected identifier or string literal">;
 
 // OpenMP support.
 def warn_pragma_omp_ignored : Warning<
@@ -1607,6 +1618,9 @@ def err_pragma_invalid_keyword : Error<
 def err_pragma_pipeline_invalid_keyword : Error<
     "invalid argument; expected 'disable'">;
 
+// API notes.
+def err_type_unparsed : Error<"unparsed tokens following type">;
+
 // Pragma unroll support.
 def warn_pragma_unroll_cuda_value_in_parens : Warning<
   "argument to '#pragma unroll' should not be in parentheses in CUDA C/C++">,
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 248d987c9942..176abb0bd784 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -695,6 +695,13 @@ def warn_maybe_falloff_nonvoid_function : Warning<
 def warn_falloff_nonvoid_function : Warning<
   "non-void function does not return a value">,
   InGroup;
+def warn_const_attr_with_pure_attr : Warning<
+  "'const' attribute imposes more restrictions; 'pure' attribute ignored">,
+  InGroup;
+def warn_pure_function_returns_void : Warning<
+  "'%select{pure|const}0' attribute on function returning 'void'; attribute ignored">,
+  InGroup;
+
 def err_maybe_falloff_nonvoid_block : Error<
   "non-void block does not return a value in all control paths">;
 def err_falloff_nonvoid_block : Error<
@@ -2256,6 +2263,8 @@ def warn_cxx17_compat_aggregate_init_paren_list : Warning<
 def err_reference_bind_to_bitfield : Error<
   "%select{non-const|volatile}0 reference cannot bind to "
   "bit-field%select{| %1}2">;
+def err_reference_bind_to_bitfield_in_cce : Error<
+  "reference cannot bind to bit-field in converted constant expression">;
 def err_reference_bind_to_vector_element : Error<
   "%select{non-const|volatile}0 reference cannot bind to vector element">;
 def err_reference_bind_to_matrix_element : Error<
@@ -2412,7 +2421,8 @@ def err_auto_not_allowed : Error<
   "|in type allocated by 'new'|in K&R-style function parameter"
   "|in template parameter|in friend declaration|in function prototype that is "
   "not a function declaration|in requires expression parameter"
-  "|in array declaration}1">;
+  "|in array declaration"
+  "|in declaration of conversion function template}1">;
 def err_dependent_deduced_tst : Error<
   "typename specifier refers to "
   "%select{class template|function template|variable template|alias template|"
@@ -3114,6 +3124,8 @@ def err_musttail_scope : Error<
   "cannot perform a tail call from this return statement">;
 def err_musttail_no_variadic : Error<
   "%0 attribute may not be used with variadic functions">;
+def err_musttail_no_return : Error<
+  "%0 attribute may not be used with no-return-attribute functions">;
 
 def err_nsobject_attribute : Error<
   "'NSObject' attribute is for pointer types only">;
@@ -3161,6 +3173,9 @@ def warn_attribute_arm_sm_incompat_builtin : Warning<
 def warn_attribute_arm_za_builtin_no_za_state : Warning<
   "builtin call is not valid when calling from a function without active ZA state">,
   InGroup>;
+def warn_attribute_arm_zt0_builtin_no_zt0_state : Warning<
+  "builtin call is not valid when calling from a function without active ZT0 state">,
+  InGroup>;
 def err_sve_vector_in_non_sve_target : Error<
   "SVE vector type %0 cannot be used in a target without sve">;
 def err_attribute_riscv_rvv_bits_unsupported : Error<
@@ -3418,6 +3433,8 @@ def warn_objc_redundant_literal_use : Warning<
 def err_attr_tlsmodel_arg : Error<"tls_model must be \"global-dynamic\", "
   "\"local-dynamic\", \"initial-exec\" or \"local-exec\"">;
 
+def err_attr_codemodel_arg : Error<"code model '%0' is not supported on this target">;
+
 def err_aix_attr_unsupported_tls_model : Error<"TLS model '%0' is not yet supported on AIX">;
 
 def err_tls_var_aligned_over_maximum : Error<
@@ -3697,10 +3714,20 @@ def err_sme_call_in_non_sme_target : Error<
   "call to a streaming function requires 'sme'">;
 def err_sme_za_call_no_za_state : Error<
   "call to a shared ZA function requires the caller to have ZA state">;
+def err_sme_zt0_call_no_zt0_state : Error<
+  "call to a shared ZT0 function requires the caller to have ZT0 state">;
 def err_sme_definition_using_sm_in_non_sme_target : Error<
   "function executed in streaming-SVE mode requires 'sme'">;
 def err_sme_definition_using_za_in_non_sme_target : Error<
   "function using ZA state requires 'sme'">;
+def err_sme_definition_using_zt0_in_non_sme2_target : Error<
+  "function using ZT0 state requires 'sme2'">;
+def err_conflicting_attributes_arm_state : Error<
+  "conflicting attributes for state '%0'">;
+def err_unknown_arm_state : Error<
+  "unknown state '%0'">;
+def err_missing_arm_state : Error<
+  "missing state for %0">;
 def err_cconv_change : Error<
   "function declared '%0' here was previously declared "
   "%select{'%2'|without calling convention}1">;
@@ -5139,8 +5166,6 @@ def err_non_type_template_arg_subobject : Error<
   "non-type template argument refers to subobject '%0'">;
 def err_non_type_template_arg_addr_label_diff : Error<
   "template argument / label address difference / what did you expect?">;
-def err_non_type_template_arg_unsupported : Error<
-  "sorry, non-type template argument of type %0 is not yet supported">;
 def err_template_arg_not_convertible : Error<
   "non-type template argument of type %0 cannot be converted to a value "
   "of type %1">;
@@ -5192,9 +5217,6 @@ def err_template_arg_not_object_or_func : Error<
   "non-type template argument does not refer to an object or function">;
 def err_template_arg_not_pointer_to_member_form : Error<
   "non-type template argument is not a pointer to member constant">;
-def err_template_arg_member_ptr_base_derived_not_supported : Error<
-  "sorry, non-type template argument of pointer-to-member type %1 that refers "
-  "to member %q0 of a different class is not supported yet">;
 def err_template_arg_invalid : Error<
   "non-type template argument '%0' is invalid">;
 def ext_template_arg_extra_parens : ExtWarn<
@@ -6163,23 +6185,19 @@ def err_illegal_initializer_type : Error<"illegal initializer type %0">;
 def ext_init_list_type_narrowing : ExtWarn<
   "type %0 cannot be narrowed to %1 in initializer list">,
   InGroup, DefaultError, SFINAEFailure;
-// *_narrowing_const_reference diagnostics have the same messages, but are
-// controlled by -Wc++11-narrowing-const-reference for narrowing involving a
-// const reference.
 def ext_init_list_type_narrowing_const_reference : ExtWarn<
-  "type %0 cannot be narrowed to %1 in initializer list">,
+  ext_init_list_type_narrowing.Summary>,
   InGroup, DefaultError, SFINAEFailure;
 def ext_init_list_variable_narrowing : ExtWarn<
   "non-constant-expression cannot be narrowed from type %0 to %1 in "
   "initializer list">, InGroup, DefaultError, SFINAEFailure;
 def ext_init_list_variable_narrowing_const_reference : ExtWarn<
-  "non-constant-expression cannot be narrowed from type %0 to %1 in "
-  "initializer list">, InGroup, DefaultError, SFINAEFailure;
+  ext_init_list_variable_narrowing.Summary>, InGroup, DefaultError, SFINAEFailure;
 def ext_init_list_constant_narrowing : ExtWarn<
   "constant expression evaluates to %0 which cannot be narrowed to type %1">,
   InGroup, DefaultError, SFINAEFailure;
 def ext_init_list_constant_narrowing_const_reference : ExtWarn<
-  "constant expression evaluates to %0 which cannot be narrowed to type %1">,
+  ext_init_list_constant_narrowing.Summary>,
   InGroup, DefaultError, SFINAEFailure;
 def warn_init_list_type_narrowing : Warning<
   "type %0 cannot be narrowed to %1 in initializer list in C++11">,
@@ -6446,6 +6464,19 @@ def warn_superclass_variable_sized_type_not_at_end : Warning<
   "field %0 can overwrite instance variable %1 with variable sized type %2"
   " in superclass %3">, InGroup;
 
+def err_flexible_array_count_not_in_same_struct : Error<
+  "'counted_by' field %0 isn't within the same struct as the flexible array">;
+def err_counted_by_attr_not_on_flexible_array_member : Error<
+  "'counted_by' only applies to C99 flexible array members">;
+def err_counted_by_attr_refers_to_flexible_array : Error<
+  "'counted_by' cannot refer to the flexible array %0">;
+def err_counted_by_must_be_in_structure : Error<
+  "field %0 in 'counted_by' not inside structure">;
+def err_flexible_array_counted_by_attr_field_not_integer : Error<
+  "field %0 in 'counted_by' must be a non-boolean integer type">;
+def note_flexible_array_counted_by_attr_field : Note<
+  "field %0 declared here">;
+
 let CategoryName = "ARC Semantic Issue" in {
 
 // ARC-mode diagnostics.
@@ -6992,6 +7023,11 @@ def err_member_decl_does_not_match : Error<
   "does not match any declaration in %1">;
 def err_friend_decl_with_def_arg_must_be_def : Error<
   "friend declaration specifying a default argument must be a definition">;
+def err_friend_decl_with_enclosing_temp_constraint_must_be_def : Error<
+  "friend declaration with a constraint that depends on an enclosing "
+  "template parameter must be a definition">;
+def err_non_temp_friend_decl_with_requires_clause_must_be_def : Error<
+  "non-template friend declaration with a requires clause must be a definition">;
 def err_friend_decl_with_def_arg_redeclared : Error<
   "friend declaration specifying a default argument must be the only declaration">;
 def err_friend_decl_does_not_match : Error<
@@ -9918,10 +9954,10 @@ def warn_new_dangling_initializer_list : Warning<
   "will be destroyed at the end of the full-expression">,
   InGroup;
 def warn_unsupported_lifetime_extension : Warning<
-  "sorry, lifetime extension of "
+  "lifetime extension of "
   "%select{temporary|backing array of initializer list}0 created "
-  "by aggregate initialization using default member initializer "
-  "is not supported; lifetime of %select{temporary|backing array}0 "
+  "by aggregate initialization using a default member initializer "
+  "is not yet supported; lifetime of %select{temporary|backing array}0 "
   "will end at the end of the full-expression">, InGroup;
 
 // For non-floating point, expressions of the form x == x or x != x
@@ -12069,7 +12105,7 @@ def warn_unsafe_buffer_variable : Warning<
   InGroup, DefaultIgnore;
 def warn_unsafe_buffer_operation : Warning<
   "%select{unsafe pointer operation|unsafe pointer arithmetic|"
-  "unsafe buffer access|function introduces unsafe buffer manipulation}0">,
+  "unsafe buffer access|function introduces unsafe buffer manipulation|unsafe invocation of span::data}0">,
   InGroup, DefaultIgnore;
 def note_unsafe_buffer_operation : Note<
   "used%select{| in pointer arithmetic| in buffer access}0 here">;
diff --git a/clang/include/clang/Basic/DiagnosticSerializationKinds.td b/clang/include/clang/Basic/DiagnosticSerializationKinds.td
index 3cb2cd32cf6d..11c706ebf84b 100644
--- a/clang/include/clang/Basic/DiagnosticSerializationKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSerializationKinds.td
@@ -129,10 +129,8 @@ def warn_module_system_bit_conflict : Warning<
   "as a non-system module; any difference in diagnostic options will be ignored">,
   InGroup;
 
-def warn_reading_std_cxx_module_by_implicit_paths : Warning<
-  "it is deprecated to read module '%0' implicitly; it is going to be removed in clang 18; "
-  "consider to specify the dependencies explicitly">,
-  InGroup>;
+def err_failed_to_find_module_file : Error<
+  "failed to find module file for module '%0'">;
 } // let CategoryName
 
 let CategoryName = "AST Serialization Issue" in {
diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def
index 06efac0cf1ab..5fad5fc3623c 100644
--- a/clang/include/clang/Basic/Features.def
+++ b/clang/include/clang/Basic/Features.def
@@ -269,6 +269,10 @@ EXTENSION(cxx_fixed_enum, true)
 EXTENSION(cxx_binary_literals, true)
 EXTENSION(cxx_init_captures, LangOpts.CPlusPlus11)
 EXTENSION(cxx_variable_templates, LangOpts.CPlusPlus)
+//C++20
+EXTENSION(cxx_generalized_nttp, LangOpts.CPlusPlus20)
+//C++23
+EXTENSION(cxx_explicit_this_parameter, LangOpts.CPlusPlus23)
 // Miscellaneous language extensions
 EXTENSION(overloadable_unmarked, true)
 EXTENSION(pragma_clang_attribute_namespaces, true)
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 21abc346cf17..8fc75e1cca03 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -220,7 +220,7 @@ BENIGN_LANGOPT(NoSignedZero      , 1, 0, "Permit Floating Point optimization wit
 BENIGN_LANGOPT(AllowRecip        , 1, 0, "Permit Floating Point reciprocal")
 BENIGN_LANGOPT(ApproxFunc        , 1, 0, "Permit Floating Point approximation")
 
-ENUM_LANGOPT(ComplexRange, ComplexRangeKind, 2, CX_Full, "Enable use of range reduction for complex arithmetics.")
+ENUM_LANGOPT(ComplexRange, ComplexRangeKind, 2, CX_None, "Enable use of range reduction for complex arithmetics.")
 
 BENIGN_LANGOPT(ObjCGCBitmapPrint , 1, 0, "printing of GC's bitmap layout for __weak/__strong ivars")
 
@@ -260,6 +260,7 @@ LANGOPT(OpenMPTeamSubscription  , 1, 0, "Assume distributed loops do not have mo
 LANGOPT(OpenMPNoThreadState  , 1, 0, "Assume that no thread in a parallel region will modify an ICV.")
 LANGOPT(OpenMPNoNestedParallelism  , 1, 0, "Assume that no thread in a parallel region will encounter a parallel region")
 LANGOPT(OpenMPOffloadMandatory  , 1, 0, "Assert that offloading is mandatory and do not create a host fallback.")
+LANGOPT(OpenMPForceUSM     , 1, 0, "Enable OpenMP unified shared memory mode via compiler.")
 LANGOPT(NoGPULib  , 1, 0, "Indicate a build without the standard GPU libraries.")
 LANGOPT(RenderScript      , 1, 0, "RenderScript")
 
@@ -308,7 +309,8 @@ BENIGN_LANGOPT(IgnoreXCOFFVisibility, 1, 0, "All the visibility attributes that
 BENIGN_LANGOPT(VisibilityInlinesHiddenStaticLocalVar, 1, 0,
                "hidden visibility for static local variables in inline C++ "
                "methods when -fvisibility-inlines hidden is enabled")
-LANGOPT(GlobalAllocationFunctionVisibilityHidden , 1, 0, "hidden visibility for global operator new and delete declaration")
+ENUM_LANGOPT(GlobalAllocationFunctionVisibility, VisibilityForcedKinds, 3, VisibilityForcedKinds::ForceDefault,
+             "How to apply visibility to global operator new and delete declarations")
 LANGOPT(NewInfallible , 1, 0, "Treats throwing global C++ operator new as always returning valid memory (annotates with __attribute__((returns_nonnull)) and throw()). This is detectable in source.")
 BENIGN_LANGOPT(ParseUnknownAnytype, 1, 0, "__unknown_anytype")
 BENIGN_LANGOPT(DebuggerSupport , 1, 0, "debugger support")
@@ -359,16 +361,16 @@ BENIGN_ENUM_LANGOPT(TypeVisibilityMode, Visibility, 3, DefaultVisibility,
              "default visibility for types [-ftype-visibility]")
 LANGOPT(SetVisibilityForExternDecls, 1, 0,
         "apply global symbol visibility to external declarations without an explicit visibility")
-LANGOPT(VisibilityFromDLLStorageClass, 1, 0,
-        "set the visiblity of globals from their DLL storage class [-fvisibility-from-dllstorageclass]")
-ENUM_LANGOPT(DLLExportVisibility, Visibility, 3, DefaultVisibility,
-             "visibility for functions and variables with dllexport annotations [-fvisibility-from-dllstorageclass]")
-ENUM_LANGOPT(NoDLLStorageClassVisibility, Visibility, 3, HiddenVisibility,
-             "visibility for functions and variables without an explicit DLL storage class [-fvisibility-from-dllstorageclass]")
-ENUM_LANGOPT(ExternDeclDLLImportVisibility, Visibility, 3, DefaultVisibility,
-             "visibility for external declarations with dllimport annotations [-fvisibility-from-dllstorageclass]")
-ENUM_LANGOPT(ExternDeclNoDLLStorageClassVisibility, Visibility, 3, HiddenVisibility,
-             "visibility for external declarations without an explicit DLL storage class [-fvisibility-from-dllstorageclass]")
+BENIGN_LANGOPT(VisibilityFromDLLStorageClass, 1, 0,
+               "override the visibility of globals based on their final DLL storage class [-fvisibility-from-dllstorageclass]")
+BENIGN_ENUM_LANGOPT(DLLExportVisibility, VisibilityFromDLLStorageClassKinds, 3, VisibilityFromDLLStorageClassKinds::Default,
+             "how to adjust the visibility for functions and variables with dllexport annotations [-fvisibility-dllexport]")
+BENIGN_ENUM_LANGOPT(NoDLLStorageClassVisibility, VisibilityFromDLLStorageClassKinds, 3, VisibilityFromDLLStorageClassKinds::Hidden,
+             "how to adjust the visibility for functions and variables without an explicit DLL storage class [-fvisibility-nodllstorageclass]")
+BENIGN_ENUM_LANGOPT(ExternDeclDLLImportVisibility, VisibilityFromDLLStorageClassKinds, 3, VisibilityFromDLLStorageClassKinds::Default,
+             "how to adjust the visibility for external declarations with dllimport annotations [-fvisibility-externs-dllimport]")
+BENIGN_ENUM_LANGOPT(ExternDeclNoDLLStorageClassVisibility, VisibilityFromDLLStorageClassKinds, 3,  VisibilityFromDLLStorageClassKinds::Hidden,
+             "how to adjust the visibility for external declarations without an explicit DLL storage class [-fvisibility-externs-nodllstorageclass]")
 BENIGN_LANGOPT(SemanticInterposition        , 1, 0, "semantic interposition")
 BENIGN_LANGOPT(HalfNoSemanticInterposition, 1, 0,
                "Like -fno-semantic-interposition but don't use local aliases")
@@ -378,6 +380,8 @@ ENUM_LANGOPT(TrivialAutoVarInit, TrivialAutoVarInitKind, 2, TrivialAutoVarInitKi
              "trivial automatic variable initialization")
 VALUE_LANGOPT(TrivialAutoVarInitStopAfter, 32, 0,
              "stop trivial automatic variable initialization after the specified number of instances. Must be greater than 0.")
+VALUE_LANGOPT(TrivialAutoVarInitMaxSize, 32, 0,
+             "stop trivial automatic variable initialization if var size exceeds the specified size (in bytes). Must be greater than 0.")
 ENUM_LANGOPT(SignedOverflowBehavior, SignedOverflowBehaviorTy, 2, SOB_Undefined,
              "signed integer overflow handling")
 ENUM_LANGOPT(ThreadModel  , ThreadModelKind, 2, ThreadModelKind::POSIX, "Thread Model")
@@ -457,6 +461,7 @@ ENUM_LANGOPT(SignReturnAddressKey, SignReturnAddressKeyKind, 1, SignReturnAddres
              "Key used for return address signing")
 LANGOPT(BranchTargetEnforcement, 1, 0, "Branch-target enforcement enabled")
 LANGOPT(BranchProtectionPAuthLR, 1, 0, "Use PC as a diversifier using PAuthLR NOP instructions.")
+LANGOPT(GuardedControlStack, 1, 0, "Guarded control stack enabled")
 
 LANGOPT(SpeculativeLoadHardening, 1, 0, "Speculative load hardening enabled")
 
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index 9f986fce2d44..c1cc5548ef10 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -381,6 +381,28 @@ class LangOptions : public LangOptionsBase {
     All,
   };
 
+  enum class VisibilityForcedKinds {
+    /// Force hidden visibility
+    ForceHidden,
+    /// Force protected visibility
+    ForceProtected,
+    /// Force default visibility
+    ForceDefault,
+    /// Don't alter the visibility
+    Source,
+  };
+
+  enum class VisibilityFromDLLStorageClassKinds {
+    /// Keep the IR-gen assigned visibility.
+    Keep,
+    /// Override the IR-gen assigned visibility with default visibility.
+    Default,
+    /// Override the IR-gen assigned visibility with hidden visibility.
+    Hidden,
+    /// Override the IR-gen assigned visibility with protected visibility.
+    Protected,
+  };
+
   enum class StrictFlexArraysLevelKind {
     /// Any trailing array member is a FAM.
     Default = 0,
@@ -392,7 +414,7 @@ class LangOptions : public LangOptionsBase {
     IncompleteOnly = 3,
   };
 
-  enum ComplexRangeKind { CX_Full, CX_Limited, CX_Fortran };
+  enum ComplexRangeKind { CX_Full, CX_Limited, CX_Fortran, CX_None };
 
 public:
   /// The used language standard.
@@ -662,6 +684,26 @@ class LangOptions : public LangOptionsBase {
            DefaultVisiblityExportMapping::All;
   }
 
+  bool hasGlobalAllocationFunctionVisibility() const {
+    return getGlobalAllocationFunctionVisibility() !=
+           VisibilityForcedKinds::Source;
+  }
+
+  bool hasDefaultGlobalAllocationFunctionVisibility() const {
+    return getGlobalAllocationFunctionVisibility() ==
+           VisibilityForcedKinds::ForceDefault;
+  }
+
+  bool hasProtectedGlobalAllocationFunctionVisibility() const {
+    return getGlobalAllocationFunctionVisibility() ==
+           VisibilityForcedKinds::ForceProtected;
+  }
+
+  bool hasHiddenGlobalAllocationFunctionVisibility() const {
+    return getGlobalAllocationFunctionVisibility() ==
+           VisibilityForcedKinds::ForceHidden;
+  }
+
   /// Remap path prefix according to -fmacro-prefix-path option.
   void remapPathPrefix(SmallVectorImpl &Path) const;
 
diff --git a/clang/include/clang/Basic/ObjCRuntime.h b/clang/include/clang/Basic/ObjCRuntime.h
index 500b2462f007..1ccf60f0b7be 100644
--- a/clang/include/clang/Basic/ObjCRuntime.h
+++ b/clang/include/clang/Basic/ObjCRuntime.h
@@ -100,16 +100,24 @@ class ObjCRuntime {
   bool isLegacyDispatchDefaultForArch(llvm::Triple::ArchType Arch) {
     // The GNUstep runtime uses a newer dispatch method by default from
     // version 1.6 onwards
-    if (getKind() == GNUstep && getVersion() >= VersionTuple(1, 6)) {
-      if (Arch == llvm::Triple::arm ||
-          Arch == llvm::Triple::x86 ||
-          Arch == llvm::Triple::x86_64)
-        return false;
-    }
-    else if ((getKind() ==  MacOSX) && isNonFragile() &&
-             (getVersion() >= VersionTuple(10, 0)) &&
-             (getVersion() < VersionTuple(10, 6)))
-        return Arch != llvm::Triple::x86_64;
+    if (getKind() == GNUstep) {
+      switch (Arch) {
+      case llvm::Triple::arm:
+      case llvm::Triple::x86:
+      case llvm::Triple::x86_64:
+        return !(getVersion() >= VersionTuple(1, 6));
+      case llvm::Triple::aarch64:
+      case llvm::Triple::mips64:
+        return !(getVersion() >= VersionTuple(1, 9));
+      case llvm::Triple::riscv64:
+        return !(getVersion() >= VersionTuple(2, 2));
+      default:
+        return true;
+      }
+    } else if ((getKind() == MacOSX) && isNonFragile() &&
+               (getVersion() >= VersionTuple(10, 0)) &&
+               (getVersion() < VersionTuple(10, 6)))
+      return Arch != llvm::Triple::x86_64;
     // Except for deployment target of 10.5 or less,
     // Mac runtimes use legacy dispatch everywhere now.
     return true;
@@ -203,7 +211,13 @@ class ObjCRuntime {
     case GCC:
       return false;
     case GNUstep:
-      return false;
+      // This could be enabled for all versions, except for the fact that the
+      // implementation of `objc_retain` and friends prior to 2.2 call [object
+      // retain] in their fall-back paths, which leads to infinite recursion if
+      // the runtime is built with this enabled.  Since distributions typically
+      // build all Objective-C things with the same compiler version and flags,
+      // it's better to be conservative here.
+      return (getVersion() >= VersionTuple(2, 2));
     case ObjFW:
       return false;
     }
@@ -240,7 +254,7 @@ class ObjCRuntime {
     case GCC:
       return false;
     case GNUstep:
-      return false;
+      return getVersion() >= VersionTuple(2, 2);
     case ObjFW:
       return false;
     }
@@ -258,6 +272,8 @@ class ObjCRuntime {
       return getVersion() >= VersionTuple(12, 2);
     case WatchOS:
       return getVersion() >= VersionTuple(5, 2);
+    case GNUstep:
+      return getVersion() >= VersionTuple(2, 2);
     default:
       return false;
     }
@@ -455,7 +471,8 @@ class ObjCRuntime {
     case iOS: return true;
     case WatchOS: return true;
     case GCC: return false;
-    case GNUstep: return false;
+    case GNUstep:
+      return (getVersion() >= VersionTuple(2, 2));
     case ObjFW: return false;
     }
     llvm_unreachable("bad kind");
diff --git a/clang/include/clang/Basic/OpenACCKinds.h b/clang/include/clang/Basic/OpenACCKinds.h
index 3117d584d347..6487a95910ed 100644
--- a/clang/include/clang/Basic/OpenACCKinds.h
+++ b/clang/include/clang/Basic/OpenACCKinds.h
@@ -14,6 +14,9 @@
 #ifndef LLVM_CLANG_BASIC_OPENACCKINDS_H
 #define LLVM_CLANG_BASIC_OPENACCKINDS_H
 
+#include "clang/Basic/Diagnostic.h"
+#include "llvm/Support/ErrorHandling.h"
+
 namespace clang {
 // Represents the Construct/Directive kind of a pragma directive. Note the
 // OpenACC standard is inconsistent between calling these Construct vs
@@ -62,6 +65,75 @@ enum class OpenACCDirectiveKind {
   Invalid,
 };
 
+inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &Out,
+                                             OpenACCDirectiveKind K) {
+  switch (K) {
+  case OpenACCDirectiveKind::Parallel:
+    return Out << "parallel";
+
+  case OpenACCDirectiveKind::Serial:
+    return Out << "serial";
+
+  case OpenACCDirectiveKind::Kernels:
+    return Out << "kernels";
+
+  case OpenACCDirectiveKind::Data:
+    return Out << "data";
+
+  case OpenACCDirectiveKind::EnterData:
+    return Out << "enter data";
+
+  case OpenACCDirectiveKind::ExitData:
+    return Out << "exit data";
+
+  case OpenACCDirectiveKind::HostData:
+    return Out << "host_data";
+
+  case OpenACCDirectiveKind::Loop:
+    return Out << "loop";
+
+  case OpenACCDirectiveKind::Cache:
+    return Out << "cache";
+
+  case OpenACCDirectiveKind::ParallelLoop:
+    return Out << "parallel loop";
+
+  case OpenACCDirectiveKind::SerialLoop:
+    return Out << "serial loop";
+
+  case OpenACCDirectiveKind::KernelsLoop:
+    return Out << "kernels loop";
+
+  case OpenACCDirectiveKind::Atomic:
+    return Out << "atomic";
+
+  case OpenACCDirectiveKind::Declare:
+    return Out << "declare";
+
+  case OpenACCDirectiveKind::Init:
+    return Out << "init";
+
+  case OpenACCDirectiveKind::Shutdown:
+    return Out << "shutdown";
+
+  case OpenACCDirectiveKind::Set:
+    return Out << "set";
+
+  case OpenACCDirectiveKind::Update:
+    return Out << "update";
+
+  case OpenACCDirectiveKind::Wait:
+    return Out << "wait";
+
+  case OpenACCDirectiveKind::Routine:
+    return Out << "routine";
+
+  case OpenACCDirectiveKind::Invalid:
+    return Out << "";
+  }
+  llvm_unreachable("Uncovered directive kind");
+}
+
 enum class OpenACCAtomicKind {
   Read,
   Write,
@@ -72,25 +144,256 @@ enum class OpenACCAtomicKind {
 
 /// Represents the kind of an OpenACC clause.
 enum class OpenACCClauseKind {
-  // 'finalize' clause, allowed on 'exit data' directive.
+  /// 'finalize' clause, allowed on 'exit data' directive.
   Finalize,
-  // 'if_present' clause, allowed on 'host_data' and 'update' directives.
+  /// 'if_present' clause, allowed on 'host_data' and 'update' directives.
   IfPresent,
-  // 'seq' clause, allowed on 'loop' and 'routine' directives.
+  /// 'seq' clause, allowed on 'loop' and 'routine' directives.
   Seq,
-  // 'independent' clause, allowed on 'loop' directives.
+  /// 'independent' clause, allowed on 'loop' directives.
   Independent,
-  // 'auto' clause, allowed on 'loop' directives.
+  /// 'auto' clause, allowed on 'loop' directives.
   Auto,
-  // 'worker' clause, allowed on 'loop' and 'routine' directives.
+  /// 'worker' clause, allowed on 'loop', Combined, and 'routine' directives.
   Worker,
-  // 'vector' clause, allowed on 'loop' and 'routine' directives. Takes no
-  // arguments for 'routine', so the 'loop' version is not yet implemented
-  // completely.
+  /// 'vector' clause, allowed on 'loop', Combined, and 'routine' directives.
   Vector,
-  // 'nohost' clause, allowed on 'routine' directives.
+  /// 'nohost' clause, allowed on 'routine' directives.
   NoHost,
-  // Represents an invalid clause, for the purposes of parsing.
+  /// 'default' clause, allowed on parallel, serial, kernel (and compound)
+  /// constructs.
+  Default,
+  /// 'if' clause, allowed on all the Compute Constructs, Data Constructs,
+  /// Executable Constructs, and Combined Constructs.
+  If,
+  /// 'self' clause, allowed on Compute and Combined Constructs, plus 'update'.
+  Self,
+  /// 'copy' clause, allowed on Compute and Combined Constructs, plus 'data' and
+  /// 'declare'.
+  Copy,
+  /// 'use_device' clause, allowed on 'host_data' construct.
+  UseDevice,
+  /// 'attach' clause, allowed on Compute and Combined constructs, plus 'data'
+  /// and 'enter data'.
+  Attach,
+  /// 'delete' clause, allowed on the 'exit data' construct.
+  Delete,
+  /// 'detach' clause, allowed on the 'exit data' construct.
+  Detach,
+  /// 'device' clause, allowed on the 'update' construct.
+  Device,
+  /// 'deviceptr' clause, allowed on Compute and Combined Constructs, plus
+  /// 'data' and 'declare'.
+  DevicePtr,
+  /// 'device_resident' clause, allowed on the 'declare' construct.
+  DeviceResident,
+  /// 'firstprivate' clause, allowed on 'parallel', 'serial', 'parallel loop',
+  /// and 'serial loop' constructs.
+  FirstPrivate,
+  /// 'host' clause, allowed on 'update' construct.
+  Host,
+  /// 'link' clause, allowed on 'declare' construct.
+  Link,
+  /// 'no_create' clause, allowed on allowed on Compute and Combined constructs,
+  /// plus 'data'.
+  NoCreate,
+  /// 'present' clause, allowed on Compute and Combined constructs, plus 'data'
+  /// and 'declare'.
+  Present,
+  /// 'private' clause, allowed on 'parallel', 'serial', 'loop', 'parallel
+  /// loop', and 'serial loop' constructs.
+  Private,
+  /// 'copyout' clause, allowed on Compute and Combined constructs, plus 'data',
+  /// 'exit data', and 'declare'.
+  CopyOut,
+  /// 'copyin' clause, allowed on Compute and Combined constructs, plus 'data',
+  /// 'enter data', and 'declare'.
+  CopyIn,
+  /// 'copyin' clause, allowed on Compute and Combined constructs, plus 'data',
+  /// 'enter data', and 'declare'.
+  Create,
+  /// 'reduction' clause, allowed on Parallel, Serial, Loop, and the combined
+  /// constructs.
+  Reduction,
+  /// 'collapse' clause, allowed on 'loop' and Combined constructs.
+  Collapse,
+  /// 'bind' clause, allowed on routine constructs.
+  Bind,
+  /// 'vector_length' clause, allowed on 'parallel', 'kernels', 'parallel loop',
+  /// and 'kernels loop' constructs.
+  VectorLength,
+  /// 'num_gangs' clause, allowed on 'parallel', 'kernels', parallel loop', and
+  /// 'kernels loop' constructs.
+  NumGangs,
+  /// 'num_workers' clause, allowed on 'parallel', 'kernels', parallel loop',
+  /// and 'kernels loop' constructs.
+  NumWorkers,
+  /// 'device_num' clause, allowed on 'init', 'shutdown', and 'set' constructs.
+  DeviceNum,
+  /// 'default_async' clause, allowed on 'set' construct.
+  DefaultAsync,
+  /// 'device_type' clause, allowed on Constructs, 'data', 'init', 'shutdown',
+  /// 'set', update', 'loop', 'routine', and Combined constructs.
+  DeviceType,
+  /// 'dtype' clause, an alias for 'device_type', stored separately for
+  /// diagnostic purposes.
+  DType,
+
+  /// Represents an invalid clause, for the purposes of parsing.
+  Invalid,
+};
+
+inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &Out,
+                                             OpenACCClauseKind K) {
+  switch (K) {
+  case OpenACCClauseKind::Finalize:
+    return Out << "finalize";
+
+  case OpenACCClauseKind::IfPresent:
+    return Out << "if_present";
+
+  case OpenACCClauseKind::Seq:
+    return Out << "seq";
+
+  case OpenACCClauseKind::Independent:
+    return Out << "independent";
+
+  case OpenACCClauseKind::Auto:
+    return Out << "auto";
+
+  case OpenACCClauseKind::Worker:
+    return Out << "worker";
+
+  case OpenACCClauseKind::Vector:
+    return Out << "vector";
+
+  case OpenACCClauseKind::NoHost:
+    return Out << "nohost";
+
+  case OpenACCClauseKind::Default:
+    return Out << "default";
+
+  case OpenACCClauseKind::If:
+    return Out << "if";
+
+  case OpenACCClauseKind::Self:
+    return Out << "self";
+
+  case OpenACCClauseKind::Copy:
+    return Out << "copy";
+
+  case OpenACCClauseKind::UseDevice:
+    return Out << "use_device";
+
+  case OpenACCClauseKind::Attach:
+    return Out << "attach";
+
+  case OpenACCClauseKind::Delete:
+    return Out << "delete";
+
+  case OpenACCClauseKind::Detach:
+    return Out << "detach";
+
+  case OpenACCClauseKind::Device:
+    return Out << "device";
+
+  case OpenACCClauseKind::DevicePtr:
+    return Out << "deviceptr";
+
+  case OpenACCClauseKind::DeviceResident:
+    return Out << "device_resident";
+
+  case OpenACCClauseKind::FirstPrivate:
+    return Out << "firstprivate";
+
+  case OpenACCClauseKind::Host:
+    return Out << "host";
+
+  case OpenACCClauseKind::Link:
+    return Out << "link";
+
+  case OpenACCClauseKind::NoCreate:
+    return Out << "no_create";
+
+  case OpenACCClauseKind::Present:
+    return Out << "present";
+
+  case OpenACCClauseKind::Private:
+    return Out << "private";
+
+  case OpenACCClauseKind::CopyOut:
+    return Out << "copyout";
+
+  case OpenACCClauseKind::CopyIn:
+    return Out << "copyin";
+
+  case OpenACCClauseKind::Create:
+    return Out << "create";
+
+  case OpenACCClauseKind::Reduction:
+    return Out << "reduction";
+
+  case OpenACCClauseKind::Collapse:
+    return Out << "collapse";
+
+  case OpenACCClauseKind::Bind:
+    return Out << "bind";
+
+  case OpenACCClauseKind::VectorLength:
+    return Out << "vector_length";
+
+  case OpenACCClauseKind::NumGangs:
+    return Out << "num_gangs";
+
+  case OpenACCClauseKind::NumWorkers:
+    return Out << "num_workers";
+
+  case OpenACCClauseKind::DeviceNum:
+    return Out << "device_num";
+
+  case OpenACCClauseKind::DefaultAsync:
+    return Out << "default_async";
+
+  case OpenACCClauseKind::DeviceType:
+    return Out << "device_type";
+
+  case OpenACCClauseKind::DType:
+    return Out << "dtype";
+
+  case OpenACCClauseKind::Invalid:
+    return Out << "";
+  }
+  llvm_unreachable("Uncovered clause kind");
+}
+enum class OpenACCDefaultClauseKind {
+  /// 'none' option.
+  None,
+  /// 'present' option.
+  Present,
+  /// Not a valid option.
+  Invalid,
+};
+
+enum class OpenACCReductionOperator {
+  /// '+'.
+  Addition,
+  /// '*'.
+  Multiplication,
+  /// 'max'.
+  Max,
+  /// 'min'.
+  Min,
+  /// '&'.
+  BitwiseAnd,
+  /// '|'.
+  BitwiseOr,
+  /// '^'.
+  BitwiseXOr,
+  /// '&&'.
+  And,
+  /// '||'.
+  Or,
+  /// Invalid Reduction Clause Kind.
   Invalid,
 };
 } // namespace clang
diff --git a/clang/include/clang/Basic/TargetBuiltins.h b/clang/include/clang/Basic/TargetBuiltins.h
index 32a43bcd365b..1a8e4123ed7e 100644
--- a/clang/include/clang/Basic/TargetBuiltins.h
+++ b/clang/include/clang/Basic/TargetBuiltins.h
@@ -304,7 +304,9 @@ namespace clang {
     bool isZExtReturn() const { return Flags & IsZExtReturn; }
     bool isByteIndexed() const { return Flags & IsByteIndexed; }
     bool isOverloadNone() const { return Flags & IsOverloadNone; }
-    bool isOverloadWhile() const { return Flags & IsOverloadWhile; }
+    bool isOverloadWhileOrMultiVecCvt() const {
+      return Flags & IsOverloadWhileOrMultiVecCvt;
+    }
     bool isOverloadDefault() const { return !(Flags & OverloadKindMask); }
     bool isOverloadWhileRW() const { return Flags & IsOverloadWhileRW; }
     bool isOverloadCvt() const { return Flags & IsOverloadCvt; }
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index ac3c324c6c29..3eb23ebdacf0 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -1373,6 +1373,7 @@ class TargetInfo : public TransferrableTargetInfo,
         LangOptions::SignReturnAddressKeyKind::AKey;
     bool BranchTargetEnforcement = false;
     bool BranchProtectionPAuthLR = false;
+    bool GuardedControlStack = false;
   };
 
   /// Determine if the Architecture in this TargetInfo supports branch
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index 3f0e1e1a7d45..c10e2adfbe6e 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -760,10 +760,11 @@ KEYWORD(__builtin_available              , KEYALL)
 KEYWORD(__builtin_sycl_unique_stable_name, KEYSYCL)
 
 // Keywords defined by Attr.td.
+// The "EMPTY ## X" is used to prevent early macro-expansion of the keyword.
 #ifndef KEYWORD_ATTRIBUTE
-#define KEYWORD_ATTRIBUTE(X) KEYWORD(X, KEYALL)
+#define KEYWORD_ATTRIBUTE(X, HASARG, EMPTY) KEYWORD(EMPTY ## X, KEYALL)
 #endif
-#include "clang/Basic/AttrTokenKinds.inc"
+#include "clang/Basic/RegularKeywordAttrInfo.inc"
 
 // Clang-specific keywords enabled only in testing.
 TESTING_KEYWORD(__unknown_anytype , KEYALL)
diff --git a/clang/include/clang/Basic/TokenKinds.h b/clang/include/clang/Basic/TokenKinds.h
index e4857405bc7f..7529b922619a 100644
--- a/clang/include/clang/Basic/TokenKinds.h
+++ b/clang/include/clang/Basic/TokenKinds.h
@@ -109,8 +109,8 @@ bool isPragmaAnnotation(TokenKind K);
 
 inline constexpr bool isRegularKeywordAttribute(TokenKind K) {
   return (false
-#define KEYWORD_ATTRIBUTE(X) || (K == tok::kw_##X)
-#include "clang/Basic/AttrTokenKinds.inc"
+#define KEYWORD_ATTRIBUTE(X, ...) || (K == tok::kw_##X)
+#include "clang/Basic/RegularKeywordAttrInfo.inc"
   );
 }
 
diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index aac3bd486de9..695e1bddf9ff 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -21,19 +21,19 @@ include "arm_sve_sme_incl.td"
 multiclass ZALoad ch> {
   let TargetGuard = "sme" in {
     def NAME # _H : MInst<"svld1_hor_" # n_suffix, "vimPQ", t,
-                          [IsLoad, IsOverloadNone, IsStreaming, IsSharedZA],
+                          [IsLoad, IsOverloadNone, IsStreaming, IsInOutZA],
                           MemEltTyDefault, i_prefix # "_horiz", ch>;
 
     def NAME # _H_VNUM : MInst<"svld1_hor_vnum_" # n_suffix, "vimPQl", t,
-                               [IsLoad, IsOverloadNone, IsStreaming, IsSharedZA],
+                               [IsLoad, IsOverloadNone, IsStreaming, IsInOutZA],
                                MemEltTyDefault, i_prefix # "_horiz", ch>;
 
     def NAME # _V : MInst<"svld1_ver_" # n_suffix, "vimPQ", t,
-                          [IsLoad, IsOverloadNone, IsStreaming, IsSharedZA],
+                          [IsLoad, IsOverloadNone, IsStreaming, IsInOutZA],
                           MemEltTyDefault, i_prefix # "_vert", ch>;
 
     def NAME # _V_VNUM : MInst<"svld1_ver_vnum_" # n_suffix, "vimPQl", t,
-                               [IsLoad, IsOverloadNone, IsStreaming, IsSharedZA],
+                               [IsLoad, IsOverloadNone, IsStreaming, IsInOutZA],
                                MemEltTyDefault, i_prefix # "_vert", ch>;
   }
 }
@@ -45,11 +45,11 @@ defm SVLD1_ZA64 : ZALoad<"za64", "l", "aarch64_sme_ld1d", [ImmCheck<0, ImmCheck0
 defm SVLD1_ZA128 : ZALoad<"za128", "q", "aarch64_sme_ld1q", [ImmCheck<0, ImmCheck0_15>]>;
 
 def SVLDR_VNUM_ZA : MInst<"svldr_vnum_za", "vmQl", "",
-                          [IsOverloadNone, IsStreamingCompatible, IsSharedZA],
+                          [IsOverloadNone, IsStreamingCompatible, IsInOutZA],
                           MemEltTyDefault, "aarch64_sme_ldr">;
 
 def SVLDR_ZA : MInst<"svldr_za", "vmQ", "",
-                          [IsOverloadNone, IsStreamingCompatible, IsSharedZA],
+                          [IsOverloadNone, IsStreamingCompatible, IsInOutZA],
                           MemEltTyDefault, "aarch64_sme_ldr", []>;
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -58,19 +58,19 @@ def SVLDR_ZA : MInst<"svldr_za", "vmQ", "",
 multiclass ZAStore ch> {
   let TargetGuard = "sme" in {
     def NAME # _H : MInst<"svst1_hor_" # n_suffix, "vimP%", t,
-                          [IsStore, IsOverloadNone, IsStreaming, IsSharedZA, IsPreservesZA],
+                          [IsStore, IsOverloadNone, IsStreaming, IsInZA],
                           MemEltTyDefault, i_prefix # "_horiz", ch>;
 
     def NAME # _H_VNUM : MInst<"svst1_hor_vnum_" # n_suffix, "vimP%l", t,
-                               [IsStore, IsOverloadNone, IsStreaming, IsSharedZA, IsPreservesZA],
+                               [IsStore, IsOverloadNone, IsStreaming, IsInZA],
                                MemEltTyDefault, i_prefix # "_horiz", ch>;
 
     def NAME # _V : MInst<"svst1_ver_" # n_suffix, "vimP%", t,
-                          [IsStore, IsOverloadNone, IsStreaming, IsSharedZA, IsPreservesZA],
+                          [IsStore, IsOverloadNone, IsStreaming, IsInZA],
                           MemEltTyDefault, i_prefix # "_vert", ch>;
 
     def NAME # _V_VNUM : MInst<"svst1_ver_vnum_" # n_suffix, "vimP%l", t,
-                               [IsStore, IsOverloadNone, IsStreaming, IsSharedZA, IsPreservesZA],
+                               [IsStore, IsOverloadNone, IsStreaming, IsInZA],
                                MemEltTyDefault, i_prefix # "_vert", ch>;
   }
 }
@@ -82,11 +82,11 @@ defm SVST1_ZA64 : ZAStore<"za64", "l", "aarch64_sme_st1d", [ImmCheck<0, ImmCheck
 defm SVST1_ZA128 : ZAStore<"za128", "q", "aarch64_sme_st1q", [ImmCheck<0, ImmCheck0_15>]>;
 
 def SVSTR_VNUM_ZA : MInst<"svstr_vnum_za", "vm%l", "",
-                          [IsOverloadNone, IsStreamingCompatible, IsSharedZA, IsPreservesZA],
+                          [IsOverloadNone, IsStreamingCompatible, IsInZA],
                           MemEltTyDefault, "aarch64_sme_str">;
 
 def SVSTR_ZA : MInst<"svstr_za", "vm%", "",
-                      [IsOverloadNone, IsStreamingCompatible, IsSharedZA, IsPreservesZA],
+                      [IsOverloadNone, IsStreamingCompatible, IsInZA],
                       MemEltTyDefault, "aarch64_sme_str", []>;
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -96,11 +96,11 @@ multiclass ZARead ch>
   let TargetGuard = "sme" in {
     def NAME # _H : SInst<"svread_hor_" # n_suffix # "[_{d}]", "ddPim", t,
                           MergeOp1, i_prefix # "_horiz",
-                          [IsReadZA, IsStreaming, IsSharedZA, IsPreservesZA], ch>;
+                          [IsReadZA, IsStreaming, IsInZA], ch>;
 
     def NAME # _V : SInst<"svread_ver_" # n_suffix # "[_{d}]", "ddPim", t,
                           MergeOp1, i_prefix # "_vert",
-                          [IsReadZA, IsStreaming, IsSharedZA, IsPreservesZA], ch>;
+                          [IsReadZA, IsStreaming, IsInZA], ch>;
   }
 }
 
@@ -117,11 +117,11 @@ multiclass ZAWrite ch
   let TargetGuard = "sme" in {
     def NAME # _H : SInst<"svwrite_hor_" # n_suffix # "[_{d}]", "vimPd", t,
                           MergeOp1, i_prefix # "_horiz",
-                          [IsWriteZA, IsStreaming, IsSharedZA], ch>;
+                          [IsWriteZA, IsStreaming, IsInOutZA], ch>;
 
     def NAME # _V : SInst<"svwrite_ver_" # n_suffix # "[_{d}]", "vimPd", t,
                           MergeOp1, i_prefix # "_vert",
-                          [IsWriteZA, IsStreaming, IsSharedZA], ch>;
+                          [IsWriteZA, IsStreaming, IsInOutZA], ch>;
   }
 }
 
@@ -136,10 +136,10 @@ defm SVWRITE_ZA128 : ZAWrite<"za128", "csilUcUsUiUlhbfd", "aarch64_sme_writeq",
 
 let TargetGuard = "sme" in {
   def SVZERO_MASK_ZA : SInst<"svzero_mask_za", "vi", "", MergeNone, "aarch64_sme_zero",
-                             [IsOverloadNone, IsStreamingCompatible, IsSharedZA],
+                             [IsOverloadNone, IsStreamingCompatible, IsInOutZA],
                              [ImmCheck<0, ImmCheck0_255>]>;
   def SVZERO_ZA      : SInst<"svzero_za", "v", "", MergeNone, "aarch64_sme_zero",
-                             [IsOverloadNone, IsStreamingCompatible, IsSharedZA]>;
+                             [IsOverloadNone, IsStreamingCompatible, IsOutZA]>;
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -149,7 +149,7 @@ multiclass ZACount {
   let TargetGuard = "sme" in {
     def NAME : SInst<"sv" # n_suffix, "nv", "", MergeNone,
                       "aarch64_sme_" # n_suffix,
-                      [IsOverloadNone, IsStreamingCompatible, IsPreservesZA]>;
+                      [IsOverloadNone, IsStreamingCompatible]>;
   }
 }
 
@@ -164,13 +164,13 @@ defm SVCNTSD : ZACount<"cntsd">;
 multiclass ZAAdd {
   let TargetGuard = "sme" in {
     def NAME # _ZA32: SInst<"sv" # n_suffix # "_za32[_{d}]", "viPPd", "iUi", MergeOp1,
-                      "aarch64_sme_" # n_suffix, [IsStreaming, IsSharedZA],
+                      "aarch64_sme_" # n_suffix, [IsStreaming, IsInOutZA],
                       [ImmCheck<0, ImmCheck0_3>]>;
   }
 
   let TargetGuard = "sme-i16i64" in {
     def NAME # _ZA64: SInst<"sv" # n_suffix # "_za64[_{d}]", "viPPd", "lUl", MergeOp1,
-                     "aarch64_sme_" # n_suffix, [IsStreaming, IsSharedZA],
+                     "aarch64_sme_" # n_suffix, [IsStreaming, IsInOutZA],
                      [ImmCheck<0, ImmCheck0_7>]>;
   }
 }
@@ -186,7 +186,7 @@ multiclass ZAIntOuterProd {
     def NAME # _ZA32_B: SInst<"sv" # n_suffix2 # "_za32[_{d}]",
                               "viPPdd", !cond(!eq(n_suffix1, "s") : "", true: "U") # "c",
                               MergeOp1, "aarch64_sme_" # n_suffix1 # n_suffix2 # "_wide",
-                              [IsStreaming, IsSharedZA],
+                              [IsStreaming, IsInOutZA],
                               [ImmCheck<0, ImmCheck0_3>]>;
   }
 
@@ -194,7 +194,7 @@ multiclass ZAIntOuterProd {
     def NAME # _ZA64_H: SInst<"sv" # n_suffix2 # "_za64[_{d}]",
                               "viPPdd", !cond(!eq(n_suffix1, "s") : "", true: "U") # "s",
                               MergeOp1, "aarch64_sme_" # n_suffix1 # n_suffix2 # "_wide",
-                              [IsStreaming, IsSharedZA],
+                              [IsStreaming, IsInOutZA],
                               [ImmCheck<0, ImmCheck0_7>]>;
   }
 }
@@ -213,7 +213,7 @@ multiclass ZAIntOuterProdMixedSigns {
                               "viPPd" # !cond(!eq(n_suffix1, "su") : "u", true: "x"),
                               !cond(!eq(n_suffix1, "su") : "", true: "U") # "c",
                               MergeOp1, "aarch64_sme_" # n_suffix1 # n_suffix2 # "_wide",
-                              [IsStreaming, IsSharedZA],
+                              [IsStreaming, IsInOutZA],
                               [ImmCheck<0, ImmCheck0_3>]>;
   }
 
@@ -222,7 +222,7 @@ multiclass ZAIntOuterProdMixedSigns {
                               "viPPd" # !cond(!eq(n_suffix1, "su") : "u", true: "x"),
                               !cond(!eq(n_suffix1, "su") : "", true: "U") # "s",
                               MergeOp1, "aarch64_sme_" # n_suffix1 # n_suffix2 # "_wide",
-                              [IsStreaming, IsSharedZA],
+                              [IsStreaming, IsInOutZA],
                               [ImmCheck<0, ImmCheck0_7>]>;
   }
 }
@@ -239,24 +239,24 @@ multiclass ZAFPOuterProd {
   let TargetGuard = "sme" in {
     def NAME # _ZA32_B: SInst<"sv" # n_suffix # "_za32[_{d}]", "viPPdd", "h",
                               MergeOp1, "aarch64_sme_" # n_suffix # "_wide",
-                              [IsStreaming, IsSharedZA],
+                              [IsStreaming, IsInOutZA],
                               [ImmCheck<0, ImmCheck0_3>]>;
 
     def NAME # _ZA32_H: SInst<"sv" # n_suffix # "_za32[_{d}]", "viPPdd", "b",
                               MergeOp1, "aarch64_sme_" # n_suffix # "_wide",
-                              [IsStreaming, IsSharedZA],
+                              [IsStreaming, IsInOutZA],
                               [ImmCheck<0, ImmCheck0_3>]>;
 
     def NAME # _ZA32_S: SInst<"sv" # n_suffix # "_za32[_{d}]", "viPPdd", "f",
                               MergeOp1, "aarch64_sme_" # n_suffix,
-                              [IsStreaming, IsSharedZA],
+                              [IsStreaming, IsInOutZA],
                               [ImmCheck<0, ImmCheck0_3>]>;
   }
 
   let TargetGuard = "sme-f64f64" in {
     def NAME # _ZA64_D: SInst<"sv" # n_suffix # "_za64[_{d}]", "viPPdd", "d",
                               MergeOp1, "aarch64_sme_" # n_suffix,
-                              [IsStreaming, IsSharedZA],
+                              [IsStreaming, IsInOutZA],
                               [ImmCheck<0, ImmCheck0_7>]>;
   }
 }
@@ -269,29 +269,29 @@ defm SVMOPS : ZAFPOuterProd<"mops">;
 
 multiclass ZAAddSub {
   let TargetGuard = "sme2" in {
-    def NAME # _WRITE_SINGLE_ZA32_VG1X2_I32 : Inst<"sv" # n_suffix # "_write[_single]_za32[_{d}]_vg1x2", "vm2d", "iUi", MergeNone, "aarch64_sme_" # n_suffix # "_write_single_za_vg1x2", [IsStreaming, IsSharedZA], []>;
-    def NAME # _WRITE_SINGLE_ZA32_VG1X4_I32 : Inst<"sv" # n_suffix # "_write[_single]_za32[_{d}]_vg1x4", "vm4d", "iUi", MergeNone, "aarch64_sme_" # n_suffix # "_write_single_za_vg1x4", [IsStreaming, IsSharedZA], []>;
+    def NAME # _WRITE_SINGLE_ZA32_VG1X2_I32 : Inst<"sv" # n_suffix # "_write[_single]_za32[_{d}]_vg1x2", "vm2d", "iUi", MergeNone, "aarch64_sme_" # n_suffix # "_write_single_za_vg1x2", [IsStreaming, IsInOutZA], []>;
+    def NAME # _WRITE_SINGLE_ZA32_VG1X4_I32 : Inst<"sv" # n_suffix # "_write[_single]_za32[_{d}]_vg1x4", "vm4d", "iUi", MergeNone, "aarch64_sme_" # n_suffix # "_write_single_za_vg1x4", [IsStreaming, IsInOutZA], []>;
 
-    def NAME # _WRITE_ZA32_VG1X2_I32 : Inst<"sv" # n_suffix # "_write_za32[_{d}]_vg1x2", "vm22", "iUi", MergeNone, "aarch64_sme_" # n_suffix # "_write_za_vg1x2", [IsStreaming, IsSharedZA], []>;
-    def NAME # _WRITE_ZA32_VG1X4_I32 : Inst<"sv" # n_suffix # "_write_za32[_{d}]_vg1x4", "vm44", "iUi", MergeNone, "aarch64_sme_" # n_suffix # "_write_za_vg1x4", [IsStreaming, IsSharedZA], []>;
+    def NAME # _WRITE_ZA32_VG1X2_I32 : Inst<"sv" # n_suffix # "_write_za32[_{d}]_vg1x2", "vm22", "iUi", MergeNone, "aarch64_sme_" # n_suffix # "_write_za_vg1x2", [IsStreaming, IsInOutZA], []>;
+    def NAME # _WRITE_ZA32_VG1X4_I32 : Inst<"sv" # n_suffix # "_write_za32[_{d}]_vg1x4", "vm44", "iUi", MergeNone, "aarch64_sme_" # n_suffix # "_write_za_vg1x4", [IsStreaming, IsInOutZA], []>;
 
-    def NAME # _ZA32_VG1x2_I32 : Inst<"sv" # n_suffix # "_za32[_{d}]_vg1x2", "vm2", "iUif", MergeNone, "aarch64_sme_" # n_suffix # "_za32_vg1x2", [IsStreaming, IsSharedZA], []>;
-    def NAME # _ZA32_VG1X4_I32 : Inst<"sv" # n_suffix # "_za32[_{d}]_vg1x4", "vm4", "iUif", MergeNone, "aarch64_sme_" # n_suffix # "_za32_vg1x4", [IsStreaming, IsSharedZA], []>;
+    def NAME # _ZA32_VG1x2_I32 : Inst<"sv" # n_suffix # "_za32[_{d}]_vg1x2", "vm2", "iUif", MergeNone, "aarch64_sme_" # n_suffix # "_za32_vg1x2", [IsStreaming, IsInOutZA], []>;
+    def NAME # _ZA32_VG1X4_I32 : Inst<"sv" # n_suffix # "_za32[_{d}]_vg1x4", "vm4", "iUif", MergeNone, "aarch64_sme_" # n_suffix # "_za32_vg1x4", [IsStreaming, IsInOutZA], []>;
 
     let TargetGuard = "sme-i16i64" in {
-      def NAME # _WRITE_SINGLE_ZA64_VG1X2_I64 : Inst<"sv" # n_suffix # "_write[_single]_za64[_{d}]_vg1x2", "vm2d", "lUl", MergeNone, "aarch64_sme_" # n_suffix # "_write_single_za_vg1x2", [IsStreaming, IsSharedZA], []>;
-      def NAME # _WRITE_SINGLE_ZA64_VG1X4_I64 : Inst<"sv" # n_suffix # "_write[_single]_za64[_{d}]_vg1x4", "vm4d", "lUl", MergeNone, "aarch64_sme_" # n_suffix # "_write_single_za_vg1x4", [IsStreaming, IsSharedZA], []>;
+      def NAME # _WRITE_SINGLE_ZA64_VG1X2_I64 : Inst<"sv" # n_suffix # "_write[_single]_za64[_{d}]_vg1x2", "vm2d", "lUl", MergeNone, "aarch64_sme_" # n_suffix # "_write_single_za_vg1x2", [IsStreaming, IsInOutZA], []>;
+      def NAME # _WRITE_SINGLE_ZA64_VG1X4_I64 : Inst<"sv" # n_suffix # "_write[_single]_za64[_{d}]_vg1x4", "vm4d", "lUl", MergeNone, "aarch64_sme_" # n_suffix # "_write_single_za_vg1x4", [IsStreaming, IsInOutZA], []>;
 
-      def NAME # _WRITE_ZA64_VG1x2_I64 : Inst<"sv" # n_suffix # "_write_za64[_{d}]_vg1x2", "vm22", "lUl", MergeNone, "aarch64_sme_" # n_suffix # "_write_za_vg1x2", [IsStreaming, IsSharedZA], []>;
-      def NAME # _WRITE_ZA64_VG1x4_I64 : Inst<"sv" # n_suffix # "_write_za64[_{d}]_vg1x4", "vm44", "lUl", MergeNone, "aarch64_sme_" # n_suffix # "_write_za_vg1x4", [IsStreaming, IsSharedZA], []>;
+      def NAME # _WRITE_ZA64_VG1x2_I64 : Inst<"sv" # n_suffix # "_write_za64[_{d}]_vg1x2", "vm22", "lUl", MergeNone, "aarch64_sme_" # n_suffix # "_write_za_vg1x2", [IsStreaming, IsInOutZA], []>;
+      def NAME # _WRITE_ZA64_VG1x4_I64 : Inst<"sv" # n_suffix # "_write_za64[_{d}]_vg1x4", "vm44", "lUl", MergeNone, "aarch64_sme_" # n_suffix # "_write_za_vg1x4", [IsStreaming, IsInOutZA], []>;
 
-      def NAME # _ZA64_VG1X2_I64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x2", "vm2", "lUl", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x2", [IsStreaming, IsSharedZA], []>;
-      def NAME # _ZA64_VG1X4_I64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x4", "vm4", "lUl", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x4", [IsStreaming, IsSharedZA], []>;
+      def NAME # _ZA64_VG1X2_I64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x2", "vm2", "lUl", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x2", [IsStreaming, IsInOutZA], []>;
+      def NAME # _ZA64_VG1X4_I64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x4", "vm4", "lUl", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x4", [IsStreaming, IsInOutZA], []>;
     }
 
     let TargetGuard = "sme-f64f64" in {
-      def NAME # _ZA64_VG1X2_F64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x2", "vm2", "d", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x2", [IsStreaming, IsSharedZA], []>;
-      def NAME # _ZA64_VG1X4_F64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x4", "vm4", "d", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x4", [IsStreaming, IsSharedZA], []>;
+      def NAME # _ZA64_VG1X2_F64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x2", "vm2", "d", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x2", [IsStreaming, IsInOutZA], []>;
+      def NAME # _ZA64_VG1X4_F64 : Inst<"sv" # n_suffix # "_za64[_{d}]_vg1x4", "vm4", "d", MergeNone, "aarch64_sme_" # n_suffix # "_za64_vg1x4", [IsStreaming, IsInOutZA], []>;
     }
   }
 }
@@ -306,12 +306,12 @@ defm SVSUB : ZAAddSub<"sub">;
 //
 
 multiclass ZAWrite_VG checks> {
-  def NAME # _VG2_H : Inst<"svwrite_hor_" # n # "[_{d}]_vg2",   "vim2", t, MergeNone, i # "_hor_vg2", [IsSharedZA, IsStreaming], checks>;
-  def NAME # _VG2_V : Inst<"svwrite_ver_" # n # "[_{d}]_vg2",   "vim2", t, MergeNone, i # "_ver_vg2", [IsSharedZA, IsStreaming], checks>;
-  def NAME # _VG4_H : Inst<"svwrite_hor_" # n # "[_{d}]_vg4",   "vim4", t, MergeNone, i # "_hor_vg4", [IsSharedZA, IsStreaming], checks>;
-  def NAME # _VG4_V : Inst<"svwrite_ver_" # n # "[_{d}]_vg4",   "vim4", t, MergeNone, i # "_ver_vg4", [IsSharedZA, IsStreaming], checks>;
-  def NAME # _VG1x2 : Inst<"svwrite_"     # n # "[_{d}]_vg1x2", "vm2",  t, MergeNone, i # "_vg1x2",   [IsSharedZA, IsStreaming], []>;
-  def NAME # _VG1x4 : Inst<"svwrite_"     # n # "[_{d}]_vg1x4", "vm4",  t, MergeNone, i # "_vg1x4",   [IsSharedZA, IsStreaming], []>;
+  def NAME # _VG2_H : Inst<"svwrite_hor_" # n # "[_{d}]_vg2",   "vim2", t, MergeNone, i # "_hor_vg2", [IsInOutZA, IsStreaming], checks>;
+  def NAME # _VG2_V : Inst<"svwrite_ver_" # n # "[_{d}]_vg2",   "vim2", t, MergeNone, i # "_ver_vg2", [IsInOutZA, IsStreaming], checks>;
+  def NAME # _VG4_H : Inst<"svwrite_hor_" # n # "[_{d}]_vg4",   "vim4", t, MergeNone, i # "_hor_vg4", [IsInOutZA, IsStreaming], checks>;
+  def NAME # _VG4_V : Inst<"svwrite_ver_" # n # "[_{d}]_vg4",   "vim4", t, MergeNone, i # "_ver_vg4", [IsInOutZA, IsStreaming], checks>;
+  def NAME # _VG1x2 : Inst<"svwrite_"     # n # "[_{d}]_vg1x2", "vm2",  t, MergeNone, i # "_vg1x2",   [IsInOutZA, IsStreaming], []>;
+  def NAME # _VG1x4 : Inst<"svwrite_"     # n # "[_{d}]_vg1x4", "vm4",  t, MergeNone, i # "_vg1x4",   [IsInOutZA, IsStreaming], []>;
 }
 
 let TargetGuard = "sme2" in {
@@ -322,12 +322,12 @@ let TargetGuard = "sme2" in {
 }
 
 multiclass ZARead_VG checks> {
-  def NAME # _VG2_H : Inst<"svread_hor_" # n # "_{d}_vg2",   "2im", t, MergeNone, i # "_hor_vg2", [IsSharedZA, IsPreservesZA, IsStreaming], checks>;
-  def NAME # _VG2_V : Inst<"svread_ver_" # n # "_{d}_vg2",   "2im", t, MergeNone, i # "_ver_vg2", [IsSharedZA, IsPreservesZA, IsStreaming], checks>;
-  def NAME # _VG4_H : Inst<"svread_hor_" # n # "_{d}_vg4",   "4im", t, MergeNone, i # "_hor_vg4", [IsSharedZA, IsPreservesZA, IsStreaming], checks>;
-  def NAME # _VG4_V : Inst<"svread_ver_" # n # "_{d}_vg4",   "4im", t, MergeNone, i # "_ver_vg4", [IsSharedZA, IsPreservesZA, IsStreaming], checks>;
-  def NAME # _VG1x2 : Inst<"svread_"     # n # "_{d}_vg1x2", "2m",  t, MergeNone, i # "_vg1x2",   [IsSharedZA, IsPreservesZA, IsStreaming], []>;
-  def NAME # _VG1x4 : Inst<"svread_"     # n # "_{d}_vg1x4", "4m",  t, MergeNone, i # "_vg1x4",   [IsSharedZA, IsPreservesZA, IsStreaming], []>;
+  def NAME # _VG2_H : Inst<"svread_hor_" # n # "_{d}_vg2",   "2im", t, MergeNone, i # "_hor_vg2", [IsInZA, IsStreaming], checks>;
+  def NAME # _VG2_V : Inst<"svread_ver_" # n # "_{d}_vg2",   "2im", t, MergeNone, i # "_ver_vg2", [IsInZA, IsStreaming], checks>;
+  def NAME # _VG4_H : Inst<"svread_hor_" # n # "_{d}_vg4",   "4im", t, MergeNone, i # "_hor_vg4", [IsInZA, IsStreaming], checks>;
+  def NAME # _VG4_V : Inst<"svread_ver_" # n # "_{d}_vg4",   "4im", t, MergeNone, i # "_ver_vg4", [IsInZA, IsStreaming], checks>;
+  def NAME # _VG1x2 : Inst<"svread_"     # n # "_{d}_vg1x2", "2m",  t, MergeNone, i # "_vg1x2",   [IsInZA, IsStreaming], []>;
+  def NAME # _VG1x4 : Inst<"svread_"     # n # "_{d}_vg1x4", "4m",  t, MergeNone, i # "_vg1x4",   [IsInZA, IsStreaming], []>;
 }
 
 let TargetGuard = "sme2" in {
@@ -342,331 +342,331 @@ let TargetGuard = "sme2" in {
 //
 
 let TargetGuard = "sme2" in {
-  def SVSMOPA  : Inst<"svmopa_za32[_{d}]_m", "viPPdd", "s", MergeNone, "aarch64_sme_smopa_za32", [IsSharedZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>;
-  def SVUSMOPA : Inst<"svmopa_za32[_{d}]_m", "viPPdd", "Us", MergeNone, "aarch64_sme_umopa_za32", [IsSharedZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>;
+  def SVSMOPA  : Inst<"svmopa_za32[_{d}]_m", "viPPdd", "s", MergeNone, "aarch64_sme_smopa_za32", [IsInOutZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>;
+  def SVUSMOPA : Inst<"svmopa_za32[_{d}]_m", "viPPdd", "Us", MergeNone, "aarch64_sme_umopa_za32", [IsInOutZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>;
 
-  def SVSMOPS  : Inst<"svmops_za32[_{d}]_m", "viPPdd", "s", MergeNone, "aarch64_sme_smops_za32", [IsSharedZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>;
-  def SVUSMOPS : Inst<"svmops_za32[_{d}]_m", "viPPdd", "Us", MergeNone, "aarch64_sme_umops_za32", [IsSharedZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>;
+  def SVSMOPS  : Inst<"svmops_za32[_{d}]_m", "viPPdd", "s", MergeNone, "aarch64_sme_smops_za32", [IsInOutZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>;
+  def SVUSMOPS : Inst<"svmops_za32[_{d}]_m", "viPPdd", "Us", MergeNone, "aarch64_sme_umops_za32", [IsInOutZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>;
 
-  def SVBMOPA : Inst<"svbmopa_za32[_{d}]_m", "viPPdd", "iUi", MergeNone, "aarch64_sme_bmopa_za32", [IsSharedZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>;
+  def SVBMOPA : Inst<"svbmopa_za32[_{d}]_m", "viPPdd", "iUi", MergeNone, "aarch64_sme_bmopa_za32", [IsInOutZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>;
 
-  def SVBMOPS : Inst<"svbmops_za32[_{d}]_m", "viPPdd", "iUi", MergeNone, "aarch64_sme_bmops_za32", [IsSharedZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>;
+  def SVBMOPS : Inst<"svbmops_za32[_{d}]_m", "viPPdd", "iUi", MergeNone, "aarch64_sme_bmops_za32", [IsInOutZA, IsStreaming], [ImmCheck<0, ImmCheck0_3>]>;
 
   // VERTICAL DOT-PRODUCT
-  def SVVDOT_LANE_ZA32_VG1x2_S : Inst<"svvdot_lane_za32[_{d}]_vg1x2", "vm2di", "s", MergeNone, "aarch64_sme_svdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVVDOT_LANE_ZA32_VG1x4_S : Inst<"svvdot_lane_za32[_{d}]_vg1x4", "vm4di", "c", MergeNone, "aarch64_sme_svdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVVDOT_LANE_ZA32_VG1x2_U : Inst<"svvdot_lane_za32[_{d}]_vg1x2", "vm2di", "Us", MergeNone, "aarch64_sme_uvdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVVDOT_LANE_ZA32_VG1x4_U : Inst<"svvdot_lane_za32[_{d}]_vg1x4", "vm4di", "Uc", MergeNone, "aarch64_sme_uvdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVVDOT_LANE_ZA32_VG1x2_F : Inst<"svvdot_lane_za32[_{d}]_vg1x2", "vm2di", "hb", MergeNone, "aarch64_sme_fvdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVSUVDOT_LANE_ZA32_VG1x4 : Inst<"svsuvdot_lane_za32[_{d}]_vg1x4", "vm4di", "c", MergeNone, "aarch64_sme_suvdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVUSVDOT_LANE_ZA32_VG1x4 : Inst<"svusvdot_lane_za32[_{d}]_vg1x4", "vm4di", "Uc", MergeNone, "aarch64_sme_usvdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVVDOT_LANE_ZA32_VG1x2_S : Inst<"svvdot_lane_za32[_{d}]_vg1x2", "vm2di", "s", MergeNone, "aarch64_sme_svdot_lane_za32_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVVDOT_LANE_ZA32_VG1x4_S : Inst<"svvdot_lane_za32[_{d}]_vg1x4", "vm4di", "c", MergeNone, "aarch64_sme_svdot_lane_za32_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVVDOT_LANE_ZA32_VG1x2_U : Inst<"svvdot_lane_za32[_{d}]_vg1x2", "vm2di", "Us", MergeNone, "aarch64_sme_uvdot_lane_za32_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVVDOT_LANE_ZA32_VG1x4_U : Inst<"svvdot_lane_za32[_{d}]_vg1x4", "vm4di", "Uc", MergeNone, "aarch64_sme_uvdot_lane_za32_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVVDOT_LANE_ZA32_VG1x2_F : Inst<"svvdot_lane_za32[_{d}]_vg1x2", "vm2di", "hb", MergeNone, "aarch64_sme_fvdot_lane_za32_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVSUVDOT_LANE_ZA32_VG1x4 : Inst<"svsuvdot_lane_za32[_{d}]_vg1x4", "vm4di", "c", MergeNone, "aarch64_sme_suvdot_lane_za32_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVUSVDOT_LANE_ZA32_VG1x4 : Inst<"svusvdot_lane_za32[_{d}]_vg1x4", "vm4di", "Uc", MergeNone, "aarch64_sme_usvdot_lane_za32_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
 
   // Multi-vector signed & unsigned integer dot-product
-  def SVDOT_MULTI_ZA32_VG1x2_S  : Inst<"svdot_za32[_{d}]_vg1x2", "vm22", "cs", MergeNone, "aarch64_sme_sdot_za32_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_MULTI_ZA32_VG1x4_S  : Inst<"svdot_za32[_{d}]_vg1x4", "vm44", "cs", MergeNone, "aarch64_sme_sdot_za32_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_MULTI_ZA32_VG1x2_U  : Inst<"svdot_za32[_{d}]_vg1x2", "vm22", "UcUs", MergeNone, "aarch64_sme_udot_za32_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_MULTI_ZA32_VG1x4_U  : Inst<"svdot_za32[_{d}]_vg1x4", "vm44", "UcUs", MergeNone, "aarch64_sme_udot_za32_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_SINGLE_ZA32_VG1x2_S : Inst<"svdot[_single]_za32[_{d}]_vg1x2", "vm2d", "cs", MergeNone, "aarch64_sme_sdot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_SINGLE_ZA32_VG1x4_S : Inst<"svdot[_single]_za32[_{d}]_vg1x4", "vm4d", "cs", MergeNone, "aarch64_sme_sdot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_SINGLE_ZA32_VG1x2_U : Inst<"svdot[_single]_za32[_{d}]_vg1x2", "vm2d", "UcUs", MergeNone, "aarch64_sme_udot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_SINGLE_ZA32_VG1x4_U : Inst<"svdot[_single]_za32[_{d}]_vg1x4", "vm4d", "UcUs", MergeNone, "aarch64_sme_udot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_LANE_ZA32_VG1x2_S   : Inst<"svdot_lane_za32[_{d}]_vg1x2", "vm2di", "cs", MergeNone, "aarch64_sme_sdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVDOT_LANE_ZA32_VG1x4_S   : Inst<"svdot_lane_za32[_{d}]_vg1x4", "vm4di", "cs", MergeNone, "aarch64_sme_sdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVDOT_LANE_ZA32_VG1x2_U   : Inst<"svdot_lane_za32[_{d}]_vg1x2", "vm2di", "UcUs", MergeNone, "aarch64_sme_udot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVDOT_LANE_ZA32_VG1x4_U   : Inst<"svdot_lane_za32[_{d}]_vg1x4", "vm4di", "UcUs", MergeNone, "aarch64_sme_udot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-
-  def SVUSDOT_SINGLE_ZA32_VG1x2 : Inst<"svusdot[_single]_za32[_{d}]_vg1x2", "vm2.dx", "Uc", MergeNone, "aarch64_sme_usdot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVUSDOT_SINGLE_ZA32_VG1x4 : Inst<"svusdot[_single]_za32[_{d}]_vg1x4", "vm4.dx", "Uc", MergeNone, "aarch64_sme_usdot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVUSDOT_MULTI_ZA32_VG1x2  : Inst<"svusdot_za32[_{d}]_vg1x2", "vm2.d2.x", "Uc", MergeNone, "aarch64_sme_usdot_za32_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVUSDOT_MULTI_ZA32_VG1x4  : Inst<"svusdot_za32[_{d}]_vg1x4", "vm4.d4.x", "Uc", MergeNone, "aarch64_sme_usdot_za32_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVUSDOT_LANE_ZA32_VG1x2   : Inst<"svusdot_lane_za32[_{d}]_vg1x2", "vm2.dxi", "Uc", MergeNone, "aarch64_sme_usdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVUSDOT_LANE_ZA32_VG1x4   : Inst<"svusdot_lane_za32[_{d}]_vg1x4", "vm4.dxi", "Uc", MergeNone, "aarch64_sme_usdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-
-  def SVSUDOT_SINGLE_ZA32_VG1x2 : Inst<"svsudot[_single]_za32[_{d}]_vg1x2", "vm2.du", "c", MergeNone, "aarch64_sme_sudot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVSUDOT_SINGLE_ZA32_VG1x4 : Inst<"svsudot[_single]_za32[_{d}]_vg1x4", "vm4.du", "c", MergeNone, "aarch64_sme_sudot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>;
+  def SVDOT_MULTI_ZA32_VG1x2_S  : Inst<"svdot_za32[_{d}]_vg1x2", "vm22", "cs", MergeNone, "aarch64_sme_sdot_za32_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_MULTI_ZA32_VG1x4_S  : Inst<"svdot_za32[_{d}]_vg1x4", "vm44", "cs", MergeNone, "aarch64_sme_sdot_za32_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_MULTI_ZA32_VG1x2_U  : Inst<"svdot_za32[_{d}]_vg1x2", "vm22", "UcUs", MergeNone, "aarch64_sme_udot_za32_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_MULTI_ZA32_VG1x4_U  : Inst<"svdot_za32[_{d}]_vg1x4", "vm44", "UcUs", MergeNone, "aarch64_sme_udot_za32_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_SINGLE_ZA32_VG1x2_S : Inst<"svdot[_single]_za32[_{d}]_vg1x2", "vm2d", "cs", MergeNone, "aarch64_sme_sdot_single_za32_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_SINGLE_ZA32_VG1x4_S : Inst<"svdot[_single]_za32[_{d}]_vg1x4", "vm4d", "cs", MergeNone, "aarch64_sme_sdot_single_za32_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_SINGLE_ZA32_VG1x2_U : Inst<"svdot[_single]_za32[_{d}]_vg1x2", "vm2d", "UcUs", MergeNone, "aarch64_sme_udot_single_za32_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_SINGLE_ZA32_VG1x4_U : Inst<"svdot[_single]_za32[_{d}]_vg1x4", "vm4d", "UcUs", MergeNone, "aarch64_sme_udot_single_za32_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_LANE_ZA32_VG1x2_S   : Inst<"svdot_lane_za32[_{d}]_vg1x2", "vm2di", "cs", MergeNone, "aarch64_sme_sdot_lane_za32_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVDOT_LANE_ZA32_VG1x4_S   : Inst<"svdot_lane_za32[_{d}]_vg1x4", "vm4di", "cs", MergeNone, "aarch64_sme_sdot_lane_za32_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVDOT_LANE_ZA32_VG1x2_U   : Inst<"svdot_lane_za32[_{d}]_vg1x2", "vm2di", "UcUs", MergeNone, "aarch64_sme_udot_lane_za32_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVDOT_LANE_ZA32_VG1x4_U   : Inst<"svdot_lane_za32[_{d}]_vg1x4", "vm4di", "UcUs", MergeNone, "aarch64_sme_udot_lane_za32_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+
+  def SVUSDOT_SINGLE_ZA32_VG1x2 : Inst<"svusdot[_single]_za32[_{d}]_vg1x2", "vm2.dx", "Uc", MergeNone, "aarch64_sme_usdot_single_za32_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVUSDOT_SINGLE_ZA32_VG1x4 : Inst<"svusdot[_single]_za32[_{d}]_vg1x4", "vm4.dx", "Uc", MergeNone, "aarch64_sme_usdot_single_za32_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVUSDOT_MULTI_ZA32_VG1x2  : Inst<"svusdot_za32[_{d}]_vg1x2", "vm2.d2.x", "Uc", MergeNone, "aarch64_sme_usdot_za32_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVUSDOT_MULTI_ZA32_VG1x4  : Inst<"svusdot_za32[_{d}]_vg1x4", "vm4.d4.x", "Uc", MergeNone, "aarch64_sme_usdot_za32_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVUSDOT_LANE_ZA32_VG1x2   : Inst<"svusdot_lane_za32[_{d}]_vg1x2", "vm2.dxi", "Uc", MergeNone, "aarch64_sme_usdot_lane_za32_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVUSDOT_LANE_ZA32_VG1x4   : Inst<"svusdot_lane_za32[_{d}]_vg1x4", "vm4.dxi", "Uc", MergeNone, "aarch64_sme_usdot_lane_za32_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+
+  def SVSUDOT_SINGLE_ZA32_VG1x2 : Inst<"svsudot[_single]_za32[_{d}]_vg1x2", "vm2.du", "c", MergeNone, "aarch64_sme_sudot_single_za32_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVSUDOT_SINGLE_ZA32_VG1x4 : Inst<"svsudot[_single]_za32[_{d}]_vg1x4", "vm4.du", "c", MergeNone, "aarch64_sme_sudot_single_za32_vg1x4", [IsStreaming, IsInOutZA], []>;
 
   // Multi-multi sudot builtins are mapped to usdot, with zn & zm operands swapped
-  def SVSUDOT_MULTI_ZA32_VG1x2  : Inst<"svsudot_za32[_{d}]_vg1x2", "vm2.d2.u", "c", MergeNone, "aarch64_sme_usdot_za32_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVSUDOT_MULTI_ZA32_VG1x4  : Inst<"svsudot_za32[_{d}]_vg1x4", "vm4.d4.u", "c", MergeNone, "aarch64_sme_usdot_za32_vg1x4", [IsStreaming, IsSharedZA], []>;
+  def SVSUDOT_MULTI_ZA32_VG1x2  : Inst<"svsudot_za32[_{d}]_vg1x2", "vm2.d2.u", "c", MergeNone, "aarch64_sme_usdot_za32_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVSUDOT_MULTI_ZA32_VG1x4  : Inst<"svsudot_za32[_{d}]_vg1x4", "vm4.d4.u", "c", MergeNone, "aarch64_sme_usdot_za32_vg1x4", [IsStreaming, IsInOutZA], []>;
 
-  def SVSUDOT_LANE_ZA32_VG1x2   : Inst<"svsudot_lane_za32[_{d}]_vg1x2", "vm2.dui", "c", MergeNone, "aarch64_sme_sudot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVSUDOT_LANE_ZA32_VG1x4   : Inst<"svsudot_lane_za32[_{d}]_vg1x4", "vm4.dui", "c", MergeNone, "aarch64_sme_sudot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVSUDOT_LANE_ZA32_VG1x2   : Inst<"svsudot_lane_za32[_{d}]_vg1x2", "vm2.dui", "c", MergeNone, "aarch64_sme_sudot_lane_za32_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVSUDOT_LANE_ZA32_VG1x4   : Inst<"svsudot_lane_za32[_{d}]_vg1x4", "vm4.dui", "c", MergeNone, "aarch64_sme_sudot_lane_za32_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
 
   // Multi-vector half-precision/BFloat16 floating-point dot-product
-  def SVDOT_MULTI_ZA32_VG1x2_F16  : Inst<"svdot_za32[_{d}]_vg1x2", "vm22", "bh", MergeNone, "aarch64_sme_fdot_za32_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_MULTI_ZA32_VG1x4_F16  : Inst<"svdot_za32[_{d}]_vg1x4", "vm44", "bh", MergeNone, "aarch64_sme_fdot_za32_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_SINGLE_ZA32_VG1x2_F16 : Inst<"svdot[_single]_za32[_{d}]_vg1x2", "vm2d", "bh", MergeNone, "aarch64_sme_fdot_single_za32_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_SINGLE_ZA32_VG1x4_F16 : Inst<"svdot[_single]_za32[_{d}]_vg1x4", "vm4d", "bh", MergeNone, "aarch64_sme_fdot_single_za32_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_LANE_ZA32_VG1x2_F16   : Inst<"svdot_lane_za32[_{d}]_vg1x2", "vm2di", "bh", MergeNone, "aarch64_sme_fdot_lane_za32_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVDOT_LANE_ZA32_VG1x4_F16   : Inst<"svdot_lane_za32[_{d}]_vg1x4", "vm4di", "bh", MergeNone, "aarch64_sme_fdot_lane_za32_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVDOT_MULTI_ZA32_VG1x2_F16  : Inst<"svdot_za32[_{d}]_vg1x2", "vm22", "bh", MergeNone, "aarch64_sme_fdot_za32_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_MULTI_ZA32_VG1x4_F16  : Inst<"svdot_za32[_{d}]_vg1x4", "vm44", "bh", MergeNone, "aarch64_sme_fdot_za32_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_SINGLE_ZA32_VG1x2_F16 : Inst<"svdot[_single]_za32[_{d}]_vg1x2", "vm2d", "bh", MergeNone, "aarch64_sme_fdot_single_za32_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_SINGLE_ZA32_VG1x4_F16 : Inst<"svdot[_single]_za32[_{d}]_vg1x4", "vm4d", "bh", MergeNone, "aarch64_sme_fdot_single_za32_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_LANE_ZA32_VG1x2_F16   : Inst<"svdot_lane_za32[_{d}]_vg1x2", "vm2di", "bh", MergeNone, "aarch64_sme_fdot_lane_za32_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVDOT_LANE_ZA32_VG1x4_F16   : Inst<"svdot_lane_za32[_{d}]_vg1x4", "vm4di", "bh", MergeNone, "aarch64_sme_fdot_lane_za32_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
 }
 
 let TargetGuard = "sme2,sme-i16i64" in {
-  def SVVDOT_LANE_ZA64_VG1x4_S : Inst<"svvdot_lane_za64[_{d}]_vg1x4", "vm4di", "s", MergeNone, "aarch64_sme_svdot_lane_za64_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>;
-  def SVVDOT_LANE_ZA64_VG1x4_U : Inst<"svvdot_lane_za64[_{d}]_vg1x4", "vm4di", "Us", MergeNone, "aarch64_sme_uvdot_lane_za64_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>;
-
-  def SVDOT_MULTI_ZA64_VG1x2_S16  : Inst<"svdot_za64[_{d}]_vg1x2", "vm22", "s", MergeNone, "aarch64_sme_sdot_za64_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_MULTI_ZA64_VG1x4_S16  : Inst<"svdot_za64[_{d}]_vg1x4", "vm44", "s", MergeNone, "aarch64_sme_sdot_za64_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_MULTI_ZA64_VG1x2_U16  : Inst<"svdot_za64[_{d}]_vg1x2", "vm22", "Us", MergeNone, "aarch64_sme_udot_za64_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_MULTI_ZA64_VG1x4_U16  : Inst<"svdot_za64[_{d}]_vg1x4", "vm44", "Us", MergeNone, "aarch64_sme_udot_za64_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_SINGLE_ZA64_VG1x2_S16 : Inst<"svdot[_single]_za64[_{d}]_vg1x2", "vm2d", "s", MergeNone, "aarch64_sme_sdot_single_za64_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_SINGLE_ZA64_VG1x4_S16 : Inst<"svdot[_single]_za64[_{d}]_vg1x4", "vm4d", "s", MergeNone, "aarch64_sme_sdot_single_za64_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_SINGLE_ZA64_VG1x2_U16 : Inst<"svdot[_single]_za64[_{d}]_vg1x2", "vm2d", "Us", MergeNone, "aarch64_sme_udot_single_za64_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_SINGLE_ZA64_VG1x4_U16 : Inst<"svdot[_single]_za64[_{d}]_vg1x4", "vm4d", "Us", MergeNone, "aarch64_sme_udot_single_za64_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVDOT_LANE_ZA64_VG1x2_S16   : Inst<"svdot_lane_za64[_{d}]_vg1x2", "vm2di", "s", MergeNone, "aarch64_sme_sdot_lane_za64_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>;
-  def SVDOT_LANE_ZA64_VG1x4_S16   : Inst<"svdot_lane_za64[_{d}]_vg1x4", "vm4di", "s", MergeNone, "aarch64_sme_sdot_lane_za64_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>;
-  def SVDOT_LANE_ZA64_VG1x2_U16   : Inst<"svdot_lane_za64[_{d}]_vg1x2", "vm2di", "Us", MergeNone, "aarch64_sme_udot_lane_za64_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>;
-  def SVDOT_LANE_ZA64_VG1x4_U16   : Inst<"svdot_lane_za64[_{d}]_vg1x4", "vm4di", "Us", MergeNone, "aarch64_sme_udot_lane_za64_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>;
+  def SVVDOT_LANE_ZA64_VG1x4_S : Inst<"svvdot_lane_za64[_{d}]_vg1x4", "vm4di", "s", MergeNone, "aarch64_sme_svdot_lane_za64_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_1>]>;
+  def SVVDOT_LANE_ZA64_VG1x4_U : Inst<"svvdot_lane_za64[_{d}]_vg1x4", "vm4di", "Us", MergeNone, "aarch64_sme_uvdot_lane_za64_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_1>]>;
+
+  def SVDOT_MULTI_ZA64_VG1x2_S16  : Inst<"svdot_za64[_{d}]_vg1x2", "vm22", "s", MergeNone, "aarch64_sme_sdot_za64_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_MULTI_ZA64_VG1x4_S16  : Inst<"svdot_za64[_{d}]_vg1x4", "vm44", "s", MergeNone, "aarch64_sme_sdot_za64_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_MULTI_ZA64_VG1x2_U16  : Inst<"svdot_za64[_{d}]_vg1x2", "vm22", "Us", MergeNone, "aarch64_sme_udot_za64_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_MULTI_ZA64_VG1x4_U16  : Inst<"svdot_za64[_{d}]_vg1x4", "vm44", "Us", MergeNone, "aarch64_sme_udot_za64_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_SINGLE_ZA64_VG1x2_S16 : Inst<"svdot[_single]_za64[_{d}]_vg1x2", "vm2d", "s", MergeNone, "aarch64_sme_sdot_single_za64_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_SINGLE_ZA64_VG1x4_S16 : Inst<"svdot[_single]_za64[_{d}]_vg1x4", "vm4d", "s", MergeNone, "aarch64_sme_sdot_single_za64_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_SINGLE_ZA64_VG1x2_U16 : Inst<"svdot[_single]_za64[_{d}]_vg1x2", "vm2d", "Us", MergeNone, "aarch64_sme_udot_single_za64_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_SINGLE_ZA64_VG1x4_U16 : Inst<"svdot[_single]_za64[_{d}]_vg1x4", "vm4d", "Us", MergeNone, "aarch64_sme_udot_single_za64_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVDOT_LANE_ZA64_VG1x2_S16   : Inst<"svdot_lane_za64[_{d}]_vg1x2", "vm2di", "s", MergeNone, "aarch64_sme_sdot_lane_za64_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_1>]>;
+  def SVDOT_LANE_ZA64_VG1x4_S16   : Inst<"svdot_lane_za64[_{d}]_vg1x4", "vm4di", "s", MergeNone, "aarch64_sme_sdot_lane_za64_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_1>]>;
+  def SVDOT_LANE_ZA64_VG1x2_U16   : Inst<"svdot_lane_za64[_{d}]_vg1x2", "vm2di", "Us", MergeNone, "aarch64_sme_udot_lane_za64_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_1>]>;
+  def SVDOT_LANE_ZA64_VG1x4_U16   : Inst<"svdot_lane_za64[_{d}]_vg1x4", "vm4di", "Us", MergeNone, "aarch64_sme_udot_lane_za64_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_1>]>;
 }
 
 // FMLA/FMLS
 let TargetGuard = "sme2" in {
-  def SVMLA_MULTI_VG1x2_F32 : Inst<"svmla_za32[_{d}]_vg1x2", "vm22", "f", MergeNone, "aarch64_sme_fmla_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLA_MULTI_VG1x4_F32 : Inst<"svmla_za32[_{d}]_vg1x4", "vm44", "f", MergeNone, "aarch64_sme_fmla_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLS_MULTI_VG1x2_F32 : Inst<"svmls_za32[_{d}]_vg1x2", "vm22", "f", MergeNone, "aarch64_sme_fmls_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLS_MULTI_VG1x4_F32 : Inst<"svmls_za32[_{d}]_vg1x4", "vm44", "f", MergeNone, "aarch64_sme_fmls_vg1x4", [IsStreaming, IsSharedZA], []>;
-
-  def SVMLA_SINGLE_VG1x2_F32 : Inst<"svmla[_single]_za32[_{d}]_vg1x2", "vm2d", "f", MergeNone, "aarch64_sme_fmla_single_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLA_SINGLE_VG1x4_F32 : Inst<"svmla[_single]_za32[_{d}]_vg1x4", "vm4d", "f", MergeNone, "aarch64_sme_fmla_single_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLS_SINGLE_VG1x2_F32 : Inst<"svmls[_single]_za32[_{d}]_vg1x2", "vm2d", "f", MergeNone, "aarch64_sme_fmls_single_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLS_SINGLE_VG1x4_F32 : Inst<"svmls[_single]_za32[_{d}]_vg1x4", "vm4d", "f", MergeNone, "aarch64_sme_fmls_single_vg1x4", [IsStreaming, IsSharedZA], []>;
-
-  def SVMLA_LANE_VG1x2_F32 : Inst<"svmla_lane_za32[_{d}]_vg1x2", "vm2di", "f", MergeNone, "aarch64_sme_fmla_lane_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVMLA_LANE_VG1x4_F32 : Inst<"svmla_lane_za32[_{d}]_vg1x4", "vm4di", "f", MergeNone, "aarch64_sme_fmla_lane_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVMLS_LANE_VG1x2_F32 : Inst<"svmls_lane_za32[_{d}]_vg1x2", "vm2di", "f", MergeNone, "aarch64_sme_fmls_lane_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
-  def SVMLS_LANE_VG1x4_F32 : Inst<"svmls_lane_za32[_{d}]_vg1x4", "vm4di", "f", MergeNone, "aarch64_sme_fmls_lane_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVMLA_MULTI_VG1x2_F32 : Inst<"svmla_za32[_{d}]_vg1x2", "vm22", "f", MergeNone, "aarch64_sme_fmla_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLA_MULTI_VG1x4_F32 : Inst<"svmla_za32[_{d}]_vg1x4", "vm44", "f", MergeNone, "aarch64_sme_fmla_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_MULTI_VG1x2_F32 : Inst<"svmls_za32[_{d}]_vg1x2", "vm22", "f", MergeNone, "aarch64_sme_fmls_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_MULTI_VG1x4_F32 : Inst<"svmls_za32[_{d}]_vg1x4", "vm44", "f", MergeNone, "aarch64_sme_fmls_vg1x4", [IsStreaming, IsInOutZA], []>;
+
+  def SVMLA_SINGLE_VG1x2_F32 : Inst<"svmla[_single]_za32[_{d}]_vg1x2", "vm2d", "f", MergeNone, "aarch64_sme_fmla_single_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLA_SINGLE_VG1x4_F32 : Inst<"svmla[_single]_za32[_{d}]_vg1x4", "vm4d", "f", MergeNone, "aarch64_sme_fmla_single_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_SINGLE_VG1x2_F32 : Inst<"svmls[_single]_za32[_{d}]_vg1x2", "vm2d", "f", MergeNone, "aarch64_sme_fmls_single_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_SINGLE_VG1x4_F32 : Inst<"svmls[_single]_za32[_{d}]_vg1x4", "vm4d", "f", MergeNone, "aarch64_sme_fmls_single_vg1x4", [IsStreaming, IsInOutZA], []>;
+
+  def SVMLA_LANE_VG1x2_F32 : Inst<"svmla_lane_za32[_{d}]_vg1x2", "vm2di", "f", MergeNone, "aarch64_sme_fmla_lane_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVMLA_LANE_VG1x4_F32 : Inst<"svmla_lane_za32[_{d}]_vg1x4", "vm4di", "f", MergeNone, "aarch64_sme_fmla_lane_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVMLS_LANE_VG1x2_F32 : Inst<"svmls_lane_za32[_{d}]_vg1x2", "vm2di", "f", MergeNone, "aarch64_sme_fmls_lane_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
+  def SVMLS_LANE_VG1x4_F32 : Inst<"svmls_lane_za32[_{d}]_vg1x4", "vm4di", "f", MergeNone, "aarch64_sme_fmls_lane_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_3>]>;
 }
 
 let TargetGuard = "sme2,sme-f64f64" in {
-  def SVMLA_MULTI_VG1x2_F64 : Inst<"svmla_za64[_{d}]_vg1x2", "vm22", "d", MergeNone, "aarch64_sme_fmla_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLA_MULTI_VG1x4_F64 : Inst<"svmla_za64[_{d}]_vg1x4", "vm44", "d", MergeNone, "aarch64_sme_fmla_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLS_MULTI_VG1x2_F64 : Inst<"svmls_za64[_{d}]_vg1x2", "vm22", "d", MergeNone, "aarch64_sme_fmls_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLS_MULTI_VG1x4_F64 : Inst<"svmls_za64[_{d}]_vg1x4", "vm44", "d", MergeNone, "aarch64_sme_fmls_vg1x4", [IsStreaming, IsSharedZA], []>;
-
-  def SVMLA_SINGLE_VG1x2_F64 : Inst<"svmla[_single]_za64[_{d}]_vg1x2", "vm2d", "d", MergeNone, "aarch64_sme_fmla_single_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLA_SINGLE_VG1x4_F64 : Inst<"svmla[_single]_za64[_{d}]_vg1x4", "vm4d", "d", MergeNone, "aarch64_sme_fmla_single_vg1x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLS_SINGLE_VG1x2_F64 : Inst<"svmls[_single]_za64[_{d}]_vg1x2", "vm2d", "d", MergeNone, "aarch64_sme_fmls_single_vg1x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLS_SINGLE_VG1x4_F64 : Inst<"svmls[_single]_za64[_{d}]_vg1x4", "vm4d", "d", MergeNone, "aarch64_sme_fmls_single_vg1x4", [IsStreaming, IsSharedZA], []>;
-
-  def SVMLA_LANE_VG1x2_F64 : Inst<"svmla_lane_za64[_{d}]_vg1x2", "vm2di", "d", MergeNone, "aarch64_sme_fmla_lane_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>;
-  def SVMLA_LANE_VG1x4_F64 : Inst<"svmla_lane_za64[_{d}]_vg1x4", "vm4di", "d", MergeNone, "aarch64_sme_fmla_lane_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>;
-  def SVMLS_LANE_VG1x2_F64 : Inst<"svmls_lane_za64[_{d}]_vg1x2", "vm2di", "d", MergeNone, "aarch64_sme_fmls_lane_vg1x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>;
-  def SVMLS_LANE_VG1x4_F64 : Inst<"svmls_lane_za64[_{d}]_vg1x4", "vm4di", "d", MergeNone, "aarch64_sme_fmls_lane_vg1x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_1>]>;
+  def SVMLA_MULTI_VG1x2_F64 : Inst<"svmla_za64[_{d}]_vg1x2", "vm22", "d", MergeNone, "aarch64_sme_fmla_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLA_MULTI_VG1x4_F64 : Inst<"svmla_za64[_{d}]_vg1x4", "vm44", "d", MergeNone, "aarch64_sme_fmla_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_MULTI_VG1x2_F64 : Inst<"svmls_za64[_{d}]_vg1x2", "vm22", "d", MergeNone, "aarch64_sme_fmls_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_MULTI_VG1x4_F64 : Inst<"svmls_za64[_{d}]_vg1x4", "vm44", "d", MergeNone, "aarch64_sme_fmls_vg1x4", [IsStreaming, IsInOutZA], []>;
+
+  def SVMLA_SINGLE_VG1x2_F64 : Inst<"svmla[_single]_za64[_{d}]_vg1x2", "vm2d", "d", MergeNone, "aarch64_sme_fmla_single_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLA_SINGLE_VG1x4_F64 : Inst<"svmla[_single]_za64[_{d}]_vg1x4", "vm4d", "d", MergeNone, "aarch64_sme_fmla_single_vg1x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_SINGLE_VG1x2_F64 : Inst<"svmls[_single]_za64[_{d}]_vg1x2", "vm2d", "d", MergeNone, "aarch64_sme_fmls_single_vg1x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLS_SINGLE_VG1x4_F64 : Inst<"svmls[_single]_za64[_{d}]_vg1x4", "vm4d", "d", MergeNone, "aarch64_sme_fmls_single_vg1x4", [IsStreaming, IsInOutZA], []>;
+
+  def SVMLA_LANE_VG1x2_F64 : Inst<"svmla_lane_za64[_{d}]_vg1x2", "vm2di", "d", MergeNone, "aarch64_sme_fmla_lane_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_1>]>;
+  def SVMLA_LANE_VG1x4_F64 : Inst<"svmla_lane_za64[_{d}]_vg1x4", "vm4di", "d", MergeNone, "aarch64_sme_fmla_lane_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_1>]>;
+  def SVMLS_LANE_VG1x2_F64 : Inst<"svmls_lane_za64[_{d}]_vg1x2", "vm2di", "d", MergeNone, "aarch64_sme_fmls_lane_vg1x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_1>]>;
+  def SVMLS_LANE_VG1x4_F64 : Inst<"svmls_lane_za64[_{d}]_vg1x4", "vm4di", "d", MergeNone, "aarch64_sme_fmls_lane_vg1x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_1>]>;
 }
 
 // FMLAL/FMLSL/UMLAL/SMLAL
 // SMLALL/UMLALL/USMLALL/SUMLALL
 let TargetGuard = "sme2" in {
   // MULTI MLAL
-  def SVMLAL_MULTI_VG2x2_F16 : Inst<"svmla_za32[_{d}]_vg2x2", "vm22", "bh", MergeNone, "aarch64_sme_fmlal_vg2x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_MULTI_VG2x4_F16 : Inst<"svmla_za32[_{d}]_vg2x4", "vm44", "bh", MergeNone, "aarch64_sme_fmlal_vg2x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_MULTI_VG2x2_S16 : Inst<"svmla_za32[_{d}]_vg2x2", "vm22", "s", MergeNone, "aarch64_sme_smlal_vg2x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_MULTI_VG2x4_S16 : Inst<"svmla_za32[_{d}]_vg2x4", "vm44", "s", MergeNone, "aarch64_sme_smlal_vg2x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_MULTI_VG2x2_U16 : Inst<"svmla_za32[_{d}]_vg2x2", "vm22", "Us", MergeNone, "aarch64_sme_umlal_vg2x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_MULTI_VG2x4_U16 : Inst<"svmla_za32[_{d}]_vg2x4", "vm44", "Us", MergeNone, "aarch64_sme_umlal_vg2x4", [IsStreaming, IsSharedZA], []>;
-
-  def SVMLAL_MULTI_VG4x2_S8 : Inst<"svmla_za32[_{d}]_vg4x2", "vm22", "c", MergeNone, "aarch64_sme_smla_za32_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_MULTI_VG4x2_U8 : Inst<"svmla_za32[_{d}]_vg4x2", "vm22", "Uc", MergeNone, "aarch64_sme_umla_za32_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_MULTI_VG4x4_S8 : Inst<"svmla_za32[_{d}]_vg4x4", "vm44", "c", MergeNone, "aarch64_sme_smla_za32_vg4x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_MULTI_VG4x4_U8 : Inst<"svmla_za32[_{d}]_vg4x4", "vm44", "Uc", MergeNone, "aarch64_sme_umla_za32_vg4x4", [IsStreaming, IsSharedZA], []>;
+  def SVMLAL_MULTI_VG2x2_F16 : Inst<"svmla_za32[_{d}]_vg2x2", "vm22", "bh", MergeNone, "aarch64_sme_fmlal_vg2x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_MULTI_VG2x4_F16 : Inst<"svmla_za32[_{d}]_vg2x4", "vm44", "bh", MergeNone, "aarch64_sme_fmlal_vg2x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_MULTI_VG2x2_S16 : Inst<"svmla_za32[_{d}]_vg2x2", "vm22", "s", MergeNone, "aarch64_sme_smlal_vg2x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_MULTI_VG2x4_S16 : Inst<"svmla_za32[_{d}]_vg2x4", "vm44", "s", MergeNone, "aarch64_sme_smlal_vg2x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_MULTI_VG2x2_U16 : Inst<"svmla_za32[_{d}]_vg2x2", "vm22", "Us", MergeNone, "aarch64_sme_umlal_vg2x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_MULTI_VG2x4_U16 : Inst<"svmla_za32[_{d}]_vg2x4", "vm44", "Us", MergeNone, "aarch64_sme_umlal_vg2x4", [IsStreaming, IsInOutZA], []>;
+
+  def SVMLAL_MULTI_VG4x2_S8 : Inst<"svmla_za32[_{d}]_vg4x2", "vm22", "c", MergeNone, "aarch64_sme_smla_za32_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_MULTI_VG4x2_U8 : Inst<"svmla_za32[_{d}]_vg4x2", "vm22", "Uc", MergeNone, "aarch64_sme_umla_za32_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_MULTI_VG4x4_S8 : Inst<"svmla_za32[_{d}]_vg4x4", "vm44", "c", MergeNone, "aarch64_sme_smla_za32_vg4x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_MULTI_VG4x4_U8 : Inst<"svmla_za32[_{d}]_vg4x4", "vm44", "Uc", MergeNone, "aarch64_sme_umla_za32_vg4x4", [IsStreaming, IsInOutZA], []>;
 
   // MULTI MLSL
-  def SVMLSL_MULTI_VG2x2_F16 : Inst<"svmls_za32[_{d}]_vg2x2", "vm22", "bh", MergeNone, "aarch64_sme_fmlsl_vg2x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_MULTI_VG2x4_F16 : Inst<"svmls_za32[_{d}]_vg2x4", "vm44", "bh", MergeNone, "aarch64_sme_fmlsl_vg2x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_MULTI_VG2x2_S16 : Inst<"svmls_za32[_{d}]_vg2x2", "vm22", "s", MergeNone, "aarch64_sme_smlsl_vg2x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_MULTI_VG2x4_S16 : Inst<"svmls_za32[_{d}]_vg2x4", "vm44", "s", MergeNone, "aarch64_sme_smlsl_vg2x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_MULTI_VG2x2_U16 : Inst<"svmls_za32[_{d}]_vg2x2", "vm22", "Us", MergeNone, "aarch64_sme_umlsl_vg2x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_MULTI_VG2x4_U16 : Inst<"svmls_za32[_{d}]_vg2x4", "vm44", "Us", MergeNone, "aarch64_sme_umlsl_vg2x4", [IsStreaming, IsSharedZA], []>;
-
-  def SVMLSL_MULTI_VG4x2_S8 : Inst<"svmls_za32[_{d}]_vg4x2", "vm22", "c", MergeNone, "aarch64_sme_smls_za32_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_MULTI_VG4x2_U8 : Inst<"svmls_za32[_{d}]_vg4x2", "vm22", "Uc", MergeNone, "aarch64_sme_umls_za32_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_MULTI_VG4x4_S8 : Inst<"svmls_za32[_{d}]_vg4x4", "vm44", "c", MergeNone, "aarch64_sme_smls_za32_vg4x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_MULTI_VG4x4_U8 : Inst<"svmls_za32[_{d}]_vg4x4", "vm44", "Uc", MergeNone, "aarch64_sme_umls_za32_vg4x4", [IsStreaming, IsSharedZA], []>;
+  def SVMLSL_MULTI_VG2x2_F16 : Inst<"svmls_za32[_{d}]_vg2x2", "vm22", "bh", MergeNone, "aarch64_sme_fmlsl_vg2x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_MULTI_VG2x4_F16 : Inst<"svmls_za32[_{d}]_vg2x4", "vm44", "bh", MergeNone, "aarch64_sme_fmlsl_vg2x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_MULTI_VG2x2_S16 : Inst<"svmls_za32[_{d}]_vg2x2", "vm22", "s", MergeNone, "aarch64_sme_smlsl_vg2x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_MULTI_VG2x4_S16 : Inst<"svmls_za32[_{d}]_vg2x4", "vm44", "s", MergeNone, "aarch64_sme_smlsl_vg2x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_MULTI_VG2x2_U16 : Inst<"svmls_za32[_{d}]_vg2x2", "vm22", "Us", MergeNone, "aarch64_sme_umlsl_vg2x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_MULTI_VG2x4_U16 : Inst<"svmls_za32[_{d}]_vg2x4", "vm44", "Us", MergeNone, "aarch64_sme_umlsl_vg2x4", [IsStreaming, IsInOutZA], []>;
+
+  def SVMLSL_MULTI_VG4x2_S8 : Inst<"svmls_za32[_{d}]_vg4x2", "vm22", "c", MergeNone, "aarch64_sme_smls_za32_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_MULTI_VG4x2_U8 : Inst<"svmls_za32[_{d}]_vg4x2", "vm22", "Uc", MergeNone, "aarch64_sme_umls_za32_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_MULTI_VG4x4_S8 : Inst<"svmls_za32[_{d}]_vg4x4", "vm44", "c", MergeNone, "aarch64_sme_smls_za32_vg4x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_MULTI_VG4x4_U8 : Inst<"svmls_za32[_{d}]_vg4x4", "vm44", "Uc", MergeNone, "aarch64_sme_umls_za32_vg4x4", [IsStreaming, IsInOutZA], []>;
 
   // SINGLE MLAL
-  def SVMLAL_SINGLE_VG2x1_F16 : Inst<"svmla_za32[_{d}]_vg2x1",          "vmdd", "bh", MergeNone, "aarch64_sme_fmlal_single_vg2x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG2x2_F16 : Inst<"svmla[_single]_za32[_{d}]_vg2x2", "vm2d", "bh", MergeNone, "aarch64_sme_fmlal_single_vg2x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG2x4_F16 : Inst<"svmla[_single]_za32[_{d}]_vg2x4", "vm4d", "bh", MergeNone, "aarch64_sme_fmlal_single_vg2x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG2x1_S16 : Inst<"svmla_za32[_{d}]_vg2x1",          "vmdd", "s", MergeNone, "aarch64_sme_smlal_single_vg2x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG2x2_S16 : Inst<"svmla[_single]_za32[_{d}]_vg2x2", "vm2d", "s", MergeNone, "aarch64_sme_smlal_single_vg2x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG2x4_S16 : Inst<"svmla[_single]_za32[_{d}]_vg2x4", "vm4d", "s", MergeNone, "aarch64_sme_smlal_single_vg2x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG2x1_U16 : Inst<"svmla_za32[_{d}]_vg2x1",          "vmdd", "Us", MergeNone, "aarch64_sme_umlal_single_vg2x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG2x2_U16 : Inst<"svmla[_single]_za32[_{d}]_vg2x2", "vm2d", "Us", MergeNone, "aarch64_sme_umlal_single_vg2x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG2x4_U16 : Inst<"svmla[_single]_za32[_{d}]_vg2x4", "vm4d", "Us", MergeNone, "aarch64_sme_umlal_single_vg2x4", [IsStreaming, IsSharedZA], []>;
-
-  def SVMLAL_SINGLE_VG4x1_S8  : Inst<"svmla_za32[_{d}]_vg4x1",          "vmdd", "c", MergeNone, "aarch64_sme_smla_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG4x1_U8  : Inst<"svmla_za32[_{d}]_vg4x1",          "vmdd", "Uc", MergeNone, "aarch64_sme_umla_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG4x2_S8  : Inst<"svmla[_single]_za32[_{d}]_vg4x2", "vm2d", "c", MergeNone, "aarch64_sme_smla_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG4x2_U8  : Inst<"svmla[_single]_za32[_{d}]_vg4x2", "vm2d", "Uc", MergeNone, "aarch64_sme_umla_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG4x4_S8  : Inst<"svmla[_single]_za32[_{d}]_vg4x4", "vm4d", "c", MergeNone, "aarch64_sme_smla_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG4x4_U8  : Inst<"svmla[_single]_za32[_{d}]_vg4x4", "vm4d", "Uc", MergeNone, "aarch64_sme_umla_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>;
+  def SVMLAL_SINGLE_VG2x1_F16 : Inst<"svmla_za32[_{d}]_vg2x1",          "vmdd", "bh", MergeNone, "aarch64_sme_fmlal_single_vg2x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG2x2_F16 : Inst<"svmla[_single]_za32[_{d}]_vg2x2", "vm2d", "bh", MergeNone, "aarch64_sme_fmlal_single_vg2x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG2x4_F16 : Inst<"svmla[_single]_za32[_{d}]_vg2x4", "vm4d", "bh", MergeNone, "aarch64_sme_fmlal_single_vg2x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG2x1_S16 : Inst<"svmla_za32[_{d}]_vg2x1",          "vmdd", "s", MergeNone, "aarch64_sme_smlal_single_vg2x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG2x2_S16 : Inst<"svmla[_single]_za32[_{d}]_vg2x2", "vm2d", "s", MergeNone, "aarch64_sme_smlal_single_vg2x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG2x4_S16 : Inst<"svmla[_single]_za32[_{d}]_vg2x4", "vm4d", "s", MergeNone, "aarch64_sme_smlal_single_vg2x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG2x1_U16 : Inst<"svmla_za32[_{d}]_vg2x1",          "vmdd", "Us", MergeNone, "aarch64_sme_umlal_single_vg2x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG2x2_U16 : Inst<"svmla[_single]_za32[_{d}]_vg2x2", "vm2d", "Us", MergeNone, "aarch64_sme_umlal_single_vg2x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG2x4_U16 : Inst<"svmla[_single]_za32[_{d}]_vg2x4", "vm4d", "Us", MergeNone, "aarch64_sme_umlal_single_vg2x4", [IsStreaming, IsInOutZA], []>;
+
+  def SVMLAL_SINGLE_VG4x1_S8  : Inst<"svmla_za32[_{d}]_vg4x1",          "vmdd", "c", MergeNone, "aarch64_sme_smla_za32_single_vg4x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG4x1_U8  : Inst<"svmla_za32[_{d}]_vg4x1",          "vmdd", "Uc", MergeNone, "aarch64_sme_umla_za32_single_vg4x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG4x2_S8  : Inst<"svmla[_single]_za32[_{d}]_vg4x2", "vm2d", "c", MergeNone, "aarch64_sme_smla_za32_single_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG4x2_U8  : Inst<"svmla[_single]_za32[_{d}]_vg4x2", "vm2d", "Uc", MergeNone, "aarch64_sme_umla_za32_single_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG4x4_S8  : Inst<"svmla[_single]_za32[_{d}]_vg4x4", "vm4d", "c", MergeNone, "aarch64_sme_smla_za32_single_vg4x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG4x4_U8  : Inst<"svmla[_single]_za32[_{d}]_vg4x4", "vm4d", "Uc", MergeNone, "aarch64_sme_umla_za32_single_vg4x4", [IsStreaming, IsInOutZA], []>;
 
   // SINGLE MLSL
-  def SVMLSL_SINGLE_VG2x1_F16 : Inst<"svmls_za32[_{d}]_vg2x1",          "vmdd", "bh", MergeNone, "aarch64_sme_fmlsl_single_vg2x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG2x2_F16 : Inst<"svmls[_single]_za32[_{d}]_vg2x2", "vm2d", "bh", MergeNone, "aarch64_sme_fmlsl_single_vg2x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG2x4_F16 : Inst<"svmls[_single]_za32[_{d}]_vg2x4", "vm4d", "bh", MergeNone, "aarch64_sme_fmlsl_single_vg2x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG2x1_S16 : Inst<"svmls_za32[_{d}]_vg2x1",          "vmdd", "s", MergeNone, "aarch64_sme_smlsl_single_vg2x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG2x2_S16 : Inst<"svmls[_single]_za32[_{d}]_vg2x2", "vm2d", "s", MergeNone, "aarch64_sme_smlsl_single_vg2x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG2x4_S16 : Inst<"svmls[_single]_za32[_{d}]_vg2x4", "vm4d", "s", MergeNone, "aarch64_sme_smlsl_single_vg2x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG2x1_U16 : Inst<"svmls_za32[_{d}]_vg2x1",          "vmdd", "Us", MergeNone, "aarch64_sme_umlsl_single_vg2x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG2x2_U16 : Inst<"svmls[_single]_za32[_{d}]_vg2x2", "vm2d", "Us", MergeNone, "aarch64_sme_umlsl_single_vg2x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG2x4_U16 : Inst<"svmls[_single]_za32[_{d}]_vg2x4", "vm4d", "Us", MergeNone, "aarch64_sme_umlsl_single_vg2x4", [IsStreaming, IsSharedZA], []>;
-
-  def SVMLSL_SINGLE_VG4x1_S8  : Inst<"svmls_za32[_{d}]_vg4x1",          "vmdd", "c", MergeNone, "aarch64_sme_smls_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG4x1_U8  : Inst<"svmls_za32[_{d}]_vg4x1",          "vmdd", "Uc", MergeNone, "aarch64_sme_umls_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG4x2_S8  : Inst<"svmls[_single]_za32[_{d}]_vg4x2", "vm2d", "c", MergeNone, "aarch64_sme_smls_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG4x2_U8  : Inst<"svmls[_single]_za32[_{d}]_vg4x2", "vm2d", "Uc", MergeNone, "aarch64_sme_umls_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG4x4_S8  : Inst<"svmls[_single]_za32[_{d}]_vg4x4", "vm4d", "c", MergeNone, "aarch64_sme_smls_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG4x4_U8  : Inst<"svmls[_single]_za32[_{d}]_vg4x4", "vm4d", "Uc", MergeNone, "aarch64_sme_umls_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>;
+  def SVMLSL_SINGLE_VG2x1_F16 : Inst<"svmls_za32[_{d}]_vg2x1",          "vmdd", "bh", MergeNone, "aarch64_sme_fmlsl_single_vg2x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG2x2_F16 : Inst<"svmls[_single]_za32[_{d}]_vg2x2", "vm2d", "bh", MergeNone, "aarch64_sme_fmlsl_single_vg2x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG2x4_F16 : Inst<"svmls[_single]_za32[_{d}]_vg2x4", "vm4d", "bh", MergeNone, "aarch64_sme_fmlsl_single_vg2x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG2x1_S16 : Inst<"svmls_za32[_{d}]_vg2x1",          "vmdd", "s", MergeNone, "aarch64_sme_smlsl_single_vg2x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG2x2_S16 : Inst<"svmls[_single]_za32[_{d}]_vg2x2", "vm2d", "s", MergeNone, "aarch64_sme_smlsl_single_vg2x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG2x4_S16 : Inst<"svmls[_single]_za32[_{d}]_vg2x4", "vm4d", "s", MergeNone, "aarch64_sme_smlsl_single_vg2x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG2x1_U16 : Inst<"svmls_za32[_{d}]_vg2x1",          "vmdd", "Us", MergeNone, "aarch64_sme_umlsl_single_vg2x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG2x2_U16 : Inst<"svmls[_single]_za32[_{d}]_vg2x2", "vm2d", "Us", MergeNone, "aarch64_sme_umlsl_single_vg2x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG2x4_U16 : Inst<"svmls[_single]_za32[_{d}]_vg2x4", "vm4d", "Us", MergeNone, "aarch64_sme_umlsl_single_vg2x4", [IsStreaming, IsInOutZA], []>;
+
+  def SVMLSL_SINGLE_VG4x1_S8  : Inst<"svmls_za32[_{d}]_vg4x1",          "vmdd", "c", MergeNone, "aarch64_sme_smls_za32_single_vg4x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG4x1_U8  : Inst<"svmls_za32[_{d}]_vg4x1",          "vmdd", "Uc", MergeNone, "aarch64_sme_umls_za32_single_vg4x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG4x2_S8  : Inst<"svmls[_single]_za32[_{d}]_vg4x2", "vm2d", "c", MergeNone, "aarch64_sme_smls_za32_single_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG4x2_U8  : Inst<"svmls[_single]_za32[_{d}]_vg4x2", "vm2d", "Uc", MergeNone, "aarch64_sme_umls_za32_single_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG4x4_S8  : Inst<"svmls[_single]_za32[_{d}]_vg4x4", "vm4d", "c", MergeNone, "aarch64_sme_smls_za32_single_vg4x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG4x4_U8  : Inst<"svmls[_single]_za32[_{d}]_vg4x4", "vm4d", "Uc", MergeNone, "aarch64_sme_umls_za32_single_vg4x4", [IsStreaming, IsInOutZA], []>;
 
   // INDEXED MLAL
-  def SVMLAL_LANE_VG2x1_F16 : Inst<"svmla_lane_za32[_{d}]_vg2x1", "vmddi", "bh", MergeNone, "aarch64_sme_fmlal_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLAL_LANE_VG2x2_F16 : Inst<"svmla_lane_za32[_{d}]_vg2x2", "vm2di", "bh", MergeNone, "aarch64_sme_fmlal_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLAL_LANE_VG2x4_F16 : Inst<"svmla_lane_za32[_{d}]_vg2x4", "vm4di", "bh", MergeNone, "aarch64_sme_fmlal_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLAL_LANE_VG2x1_S16 : Inst<"svmla_lane_za32[_{d}]_vg2x1", "vmddi", "s", MergeNone, "aarch64_sme_smlal_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLAL_LANE_VG2x2_S16 : Inst<"svmla_lane_za32[_{d}]_vg2x2", "vm2di", "s", MergeNone, "aarch64_sme_smlal_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLAL_LANE_VG2x4_S16 : Inst<"svmla_lane_za32[_{d}]_vg2x4", "vm4di", "s", MergeNone, "aarch64_sme_smlal_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLAL_LANE_VG2x1_U16 : Inst<"svmla_lane_za32[_{d}]_vg2x1", "vmddi", "Us", MergeNone, "aarch64_sme_umlal_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLAL_LANE_VG2x2_U16 : Inst<"svmla_lane_za32[_{d}]_vg2x2", "vm2di", "Us", MergeNone, "aarch64_sme_umlal_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLAL_LANE_VG2x4_U16 : Inst<"svmla_lane_za32[_{d}]_vg2x4", "vm4di", "Us", MergeNone, "aarch64_sme_umlal_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-
-  def SVMLAL_LANE_VG4x1_S8 : Inst<"svmla_lane_za32[_{d}]_vg4x1", "vmddi", "c", MergeNone, "aarch64_sme_smla_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLAL_LANE_VG4x1_U8 : Inst<"svmla_lane_za32[_{d}]_vg4x1", "vmddi", "Uc", MergeNone, "aarch64_sme_umla_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLAL_LANE_VG4x2_S8 : Inst<"svmla_lane_za32[_{d}]_vg4x2", "vm2di", "c", MergeNone, "aarch64_sme_smla_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLAL_LANE_VG4x2_U8 : Inst<"svmla_lane_za32[_{d}]_vg4x2", "vm2di", "Uc", MergeNone, "aarch64_sme_umla_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLAL_LANE_VG4x4_S8 : Inst<"svmla_lane_za32[_{d}]_vg4x4", "vm4di", "c", MergeNone, "aarch64_sme_smla_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLAL_LANE_VG4x4_U8 : Inst<"svmla_lane_za32[_{d}]_vg4x4", "vm4di", "Uc", MergeNone, "aarch64_sme_umla_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLAL_LANE_VG2x1_F16 : Inst<"svmla_lane_za32[_{d}]_vg2x1", "vmddi", "bh", MergeNone, "aarch64_sme_fmlal_lane_vg2x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG2x2_F16 : Inst<"svmla_lane_za32[_{d}]_vg2x2", "vm2di", "bh", MergeNone, "aarch64_sme_fmlal_lane_vg2x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG2x4_F16 : Inst<"svmla_lane_za32[_{d}]_vg2x4", "vm4di", "bh", MergeNone, "aarch64_sme_fmlal_lane_vg2x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG2x1_S16 : Inst<"svmla_lane_za32[_{d}]_vg2x1", "vmddi", "s", MergeNone, "aarch64_sme_smlal_lane_vg2x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG2x2_S16 : Inst<"svmla_lane_za32[_{d}]_vg2x2", "vm2di", "s", MergeNone, "aarch64_sme_smlal_lane_vg2x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG2x4_S16 : Inst<"svmla_lane_za32[_{d}]_vg2x4", "vm4di", "s", MergeNone, "aarch64_sme_smlal_lane_vg2x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG2x1_U16 : Inst<"svmla_lane_za32[_{d}]_vg2x1", "vmddi", "Us", MergeNone, "aarch64_sme_umlal_lane_vg2x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG2x2_U16 : Inst<"svmla_lane_za32[_{d}]_vg2x2", "vm2di", "Us", MergeNone, "aarch64_sme_umlal_lane_vg2x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG2x4_U16 : Inst<"svmla_lane_za32[_{d}]_vg2x4", "vm4di", "Us", MergeNone, "aarch64_sme_umlal_lane_vg2x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+
+  def SVMLAL_LANE_VG4x1_S8 : Inst<"svmla_lane_za32[_{d}]_vg4x1", "vmddi", "c", MergeNone, "aarch64_sme_smla_za32_lane_vg4x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLAL_LANE_VG4x1_U8 : Inst<"svmla_lane_za32[_{d}]_vg4x1", "vmddi", "Uc", MergeNone, "aarch64_sme_umla_za32_lane_vg4x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLAL_LANE_VG4x2_S8 : Inst<"svmla_lane_za32[_{d}]_vg4x2", "vm2di", "c", MergeNone, "aarch64_sme_smla_za32_lane_vg4x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLAL_LANE_VG4x2_U8 : Inst<"svmla_lane_za32[_{d}]_vg4x2", "vm2di", "Uc", MergeNone, "aarch64_sme_umla_za32_lane_vg4x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLAL_LANE_VG4x4_S8 : Inst<"svmla_lane_za32[_{d}]_vg4x4", "vm4di", "c", MergeNone, "aarch64_sme_smla_za32_lane_vg4x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLAL_LANE_VG4x4_U8 : Inst<"svmla_lane_za32[_{d}]_vg4x4", "vm4di", "Uc", MergeNone, "aarch64_sme_umla_za32_lane_vg4x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
 
   // INDEXED MLSL
-  def SVMLSL_LANE_VG2x1_F16 : Inst<"svmls_lane_za32[_{d}]_vg2x1", "vmddi", "bh", MergeNone, "aarch64_sme_fmlsl_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLSL_LANE_VG2x2_F16 : Inst<"svmls_lane_za32[_{d}]_vg2x2", "vm2di", "bh", MergeNone, "aarch64_sme_fmlsl_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLSL_LANE_VG2x4_F16 : Inst<"svmls_lane_za32[_{d}]_vg2x4", "vm4di", "bh", MergeNone, "aarch64_sme_fmlsl_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLSL_LANE_VG2x1_S16 : Inst<"svmls_lane_za32[_{d}]_vg2x1", "vmddi", "s", MergeNone, "aarch64_sme_smlsl_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLSL_LANE_VG2x2_S16 : Inst<"svmls_lane_za32[_{d}]_vg2x2", "vm2di", "s", MergeNone, "aarch64_sme_smlsl_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLSL_LANE_VG2x4_S16 : Inst<"svmls_lane_za32[_{d}]_vg2x4", "vm4di", "s", MergeNone, "aarch64_sme_smlsl_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLSL_LANE_VG2x1_U16 : Inst<"svmls_lane_za32[_{d}]_vg2x1", "vmddi", "Us", MergeNone, "aarch64_sme_umlsl_lane_vg2x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLSL_LANE_VG2x2_U16 : Inst<"svmls_lane_za32[_{d}]_vg2x2", "vm2di", "Us", MergeNone, "aarch64_sme_umlsl_lane_vg2x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLSL_LANE_VG2x4_U16 : Inst<"svmls_lane_za32[_{d}]_vg2x4", "vm4di", "Us", MergeNone, "aarch64_sme_umlsl_lane_vg2x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-
-  def SVMLSL_LANE_VG4x1_S8 : Inst<"svmls_lane_za32[_{d}]_vg4x1", "vmddi", "c", MergeNone, "aarch64_sme_smls_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLSL_LANE_VG4x1_U8 : Inst<"svmls_lane_za32[_{d}]_vg4x1", "vmddi", "Uc", MergeNone, "aarch64_sme_umls_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLSL_LANE_VG4x2_S8 : Inst<"svmls_lane_za32[_{d}]_vg4x2", "vm2di", "c", MergeNone, "aarch64_sme_smls_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLSL_LANE_VG4x2_U8 : Inst<"svmls_lane_za32[_{d}]_vg4x2", "vm2di", "Uc", MergeNone, "aarch64_sme_umls_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLSL_LANE_VG4x4_S8 : Inst<"svmls_lane_za32[_{d}]_vg4x4", "vm4di", "c", MergeNone, "aarch64_sme_smls_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVMLSL_LANE_VG4x4_U8 : Inst<"svmls_lane_za32[_{d}]_vg4x4", "vm4di", "Uc", MergeNone, "aarch64_sme_umls_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLSL_LANE_VG2x1_F16 : Inst<"svmls_lane_za32[_{d}]_vg2x1", "vmddi", "bh", MergeNone, "aarch64_sme_fmlsl_lane_vg2x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG2x2_F16 : Inst<"svmls_lane_za32[_{d}]_vg2x2", "vm2di", "bh", MergeNone, "aarch64_sme_fmlsl_lane_vg2x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG2x4_F16 : Inst<"svmls_lane_za32[_{d}]_vg2x4", "vm4di", "bh", MergeNone, "aarch64_sme_fmlsl_lane_vg2x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG2x1_S16 : Inst<"svmls_lane_za32[_{d}]_vg2x1", "vmddi", "s", MergeNone, "aarch64_sme_smlsl_lane_vg2x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG2x2_S16 : Inst<"svmls_lane_za32[_{d}]_vg2x2", "vm2di", "s", MergeNone, "aarch64_sme_smlsl_lane_vg2x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG2x4_S16 : Inst<"svmls_lane_za32[_{d}]_vg2x4", "vm4di", "s", MergeNone, "aarch64_sme_smlsl_lane_vg2x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG2x1_U16 : Inst<"svmls_lane_za32[_{d}]_vg2x1", "vmddi", "Us", MergeNone, "aarch64_sme_umlsl_lane_vg2x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG2x2_U16 : Inst<"svmls_lane_za32[_{d}]_vg2x2", "vm2di", "Us", MergeNone, "aarch64_sme_umlsl_lane_vg2x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG2x4_U16 : Inst<"svmls_lane_za32[_{d}]_vg2x4", "vm4di", "Us", MergeNone, "aarch64_sme_umlsl_lane_vg2x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+
+  def SVMLSL_LANE_VG4x1_S8 : Inst<"svmls_lane_za32[_{d}]_vg4x1", "vmddi", "c", MergeNone, "aarch64_sme_smls_za32_lane_vg4x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLSL_LANE_VG4x1_U8 : Inst<"svmls_lane_za32[_{d}]_vg4x1", "vmddi", "Uc", MergeNone, "aarch64_sme_umls_za32_lane_vg4x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLSL_LANE_VG4x2_S8 : Inst<"svmls_lane_za32[_{d}]_vg4x2", "vm2di", "c", MergeNone, "aarch64_sme_smls_za32_lane_vg4x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLSL_LANE_VG4x2_U8 : Inst<"svmls_lane_za32[_{d}]_vg4x2", "vm2di", "Uc", MergeNone, "aarch64_sme_umls_za32_lane_vg4x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLSL_LANE_VG4x4_S8 : Inst<"svmls_lane_za32[_{d}]_vg4x4", "vm4di", "c", MergeNone, "aarch64_sme_smls_za32_lane_vg4x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVMLSL_LANE_VG4x4_U8 : Inst<"svmls_lane_za32[_{d}]_vg4x4", "vm4di", "Uc", MergeNone, "aarch64_sme_umls_za32_lane_vg4x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
 
   // SINGLE SUMLALL
   // Single sumla maps to usmla, with zn & zm operands swapped
-  def SVSUMLALL_SINGLE_VG4x1 : Inst<"svsumla_za32[_{d}]_vg4x1",          "vmdu",   "c", MergeNone, "aarch64_sme_usmla_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>;
+  def SVSUMLALL_SINGLE_VG4x1 : Inst<"svsumla_za32[_{d}]_vg4x1",          "vmdu",   "c", MergeNone, "aarch64_sme_usmla_za32_single_vg4x1", [IsStreaming, IsInOutZA], []>;
 
-  def SVSUMLALL_SINGLE_VG4x2 : Inst<"svsumla[_single]_za32[_{d}]_vg4x2", "vm2.du", "c", MergeNone, "aarch64_sme_sumla_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVSUMLALL_SINGLE_VG4x4 : Inst<"svsumla[_single]_za32[_{d}]_vg4x4", "vm4.du", "c", MergeNone, "aarch64_sme_sumla_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>;
+  def SVSUMLALL_SINGLE_VG4x2 : Inst<"svsumla[_single]_za32[_{d}]_vg4x2", "vm2.du", "c", MergeNone, "aarch64_sme_sumla_za32_single_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVSUMLALL_SINGLE_VG4x4 : Inst<"svsumla[_single]_za32[_{d}]_vg4x4", "vm4.du", "c", MergeNone, "aarch64_sme_sumla_za32_single_vg4x4", [IsStreaming, IsInOutZA], []>;
 
   // Multi-multi sumla builtins are mapped to usmla, with zn & zm operands swapped
-  def SVSUMLALL_MULTI_VG4x2 : Inst<"svsumla_za32[_{d}]_vg4x2", "vm2.d2.u", "c", MergeNone, "aarch64_sme_usmla_za32_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVSUMLALL_MULTI_VG4x4 : Inst<"svsumla_za32[_{d}]_vg4x4", "vm4.d4.u", "c", MergeNone, "aarch64_sme_usmla_za32_vg4x4", [IsStreaming, IsSharedZA], []>;
+  def SVSUMLALL_MULTI_VG4x2 : Inst<"svsumla_za32[_{d}]_vg4x2", "vm2.d2.u", "c", MergeNone, "aarch64_sme_usmla_za32_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVSUMLALL_MULTI_VG4x4 : Inst<"svsumla_za32[_{d}]_vg4x4", "vm4.d4.u", "c", MergeNone, "aarch64_sme_usmla_za32_vg4x4", [IsStreaming, IsInOutZA], []>;
 
   // INDEXED SUMLALL
-  def SVSUMLALL_LANE_VG4x1 : Inst<"svsumla_lane_za32[_{d}]_vg4x1", "vmdui", "c", MergeNone, "aarch64_sme_sumla_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVSUMLALL_LANE_VG4x2 : Inst<"svsumla_lane_za32[_{d}]_vg4x2", "vm2ui", "c", MergeNone, "aarch64_sme_sumla_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVSUMLALL_LANE_VG4x4 : Inst<"svsumla_lane_za32[_{d}]_vg4x4", "vm4ui", "c", MergeNone, "aarch64_sme_sumla_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVSUMLALL_LANE_VG4x1 : Inst<"svsumla_lane_za32[_{d}]_vg4x1", "vmdui", "c", MergeNone, "aarch64_sme_sumla_za32_lane_vg4x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVSUMLALL_LANE_VG4x2 : Inst<"svsumla_lane_za32[_{d}]_vg4x2", "vm2ui", "c", MergeNone, "aarch64_sme_sumla_za32_lane_vg4x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVSUMLALL_LANE_VG4x4 : Inst<"svsumla_lane_za32[_{d}]_vg4x4", "vm4ui", "c", MergeNone, "aarch64_sme_sumla_za32_lane_vg4x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
 
   // SINGLE USMLALL
-  def SVUSMLALL_SINGLE_VG4x1 : Inst<"svusmla_za32[_{d}]_vg4x1",          "vmdx",   "Uc", MergeNone, "aarch64_sme_usmla_za32_single_vg4x1", [IsStreaming, IsSharedZA], []>;
-  def SVUSMLALL_SINGLE_VG4x2 : Inst<"svusmla[_single]_za32[_{d}]_vg4x2", "vm2.dx", "Uc", MergeNone, "aarch64_sme_usmla_za32_single_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVUSMLALL_SINGLE_VG4x4 : Inst<"svusmla[_single]_za32[_{d}]_vg4x4", "vm4.dx", "Uc", MergeNone, "aarch64_sme_usmla_za32_single_vg4x4", [IsStreaming, IsSharedZA], []>;
+  def SVUSMLALL_SINGLE_VG4x1 : Inst<"svusmla_za32[_{d}]_vg4x1",          "vmdx",   "Uc", MergeNone, "aarch64_sme_usmla_za32_single_vg4x1", [IsStreaming, IsInOutZA], []>;
+  def SVUSMLALL_SINGLE_VG4x2 : Inst<"svusmla[_single]_za32[_{d}]_vg4x2", "vm2.dx", "Uc", MergeNone, "aarch64_sme_usmla_za32_single_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVUSMLALL_SINGLE_VG4x4 : Inst<"svusmla[_single]_za32[_{d}]_vg4x4", "vm4.dx", "Uc", MergeNone, "aarch64_sme_usmla_za32_single_vg4x4", [IsStreaming, IsInOutZA], []>;
 
   // MULTI USMLALL
-  def SVUSMLALL_MULTI_VG4x2 : Inst<"svusmla_za32[_{d}]_vg4x2", "vm2.d2.x", "Uc", MergeNone, "aarch64_sme_usmla_za32_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVUSMLALL_MULTI_VG4x4 : Inst<"svusmla_za32[_{d}]_vg4x4", "vm4.d4.x", "Uc", MergeNone, "aarch64_sme_usmla_za32_vg4x4", [IsStreaming, IsSharedZA], []>;
+  def SVUSMLALL_MULTI_VG4x2 : Inst<"svusmla_za32[_{d}]_vg4x2", "vm2.d2.x", "Uc", MergeNone, "aarch64_sme_usmla_za32_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVUSMLALL_MULTI_VG4x4 : Inst<"svusmla_za32[_{d}]_vg4x4", "vm4.d4.x", "Uc", MergeNone, "aarch64_sme_usmla_za32_vg4x4", [IsStreaming, IsInOutZA], []>;
 
   // INDEXED USMLALL
-  def SVUSMLALL_LANE_VG4x1 : Inst<"svusmla_lane_za32[_{d}]_vg4x1", "vmdxi", "Uc", MergeNone, "aarch64_sme_usmla_za32_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVUSMLALL_LANE_VG4x2 : Inst<"svusmla_lane_za32[_{d}]_vg4x2", "vm2xi", "Uc", MergeNone, "aarch64_sme_usmla_za32_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
-  def SVUSMLALL_LANE_VG4x4 : Inst<"svusmla_lane_za32[_{d}]_vg4x4", "vm4xi", "Uc", MergeNone, "aarch64_sme_usmla_za32_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVUSMLALL_LANE_VG4x1 : Inst<"svusmla_lane_za32[_{d}]_vg4x1", "vmdxi", "Uc", MergeNone, "aarch64_sme_usmla_za32_lane_vg4x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVUSMLALL_LANE_VG4x2 : Inst<"svusmla_lane_za32[_{d}]_vg4x2", "vm2xi", "Uc", MergeNone, "aarch64_sme_usmla_za32_lane_vg4x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
+  def SVUSMLALL_LANE_VG4x4 : Inst<"svusmla_lane_za32[_{d}]_vg4x4", "vm4xi", "Uc", MergeNone, "aarch64_sme_usmla_za32_lane_vg4x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_15>]>;
 }
 
 let TargetGuard = "sme2,sme-i16i64" in {
   // MULTI MLAL
-  def SVMLAL_MULTI_VG4x2_S16 : Inst<"svmla_za64[_{d}]_vg4x2", "vm22", "s", MergeNone, "aarch64_sme_smla_za64_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_MULTI_VG4x2_U16 : Inst<"svmla_za64[_{d}]_vg4x2", "vm22", "Us", MergeNone, "aarch64_sme_umla_za64_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_MULTI_VG4x4_S16 : Inst<"svmla_za64[_{d}]_vg4x4", "vm44", "s", MergeNone, "aarch64_sme_smla_za64_vg4x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_MULTI_VG4x4_U16 : Inst<"svmla_za64[_{d}]_vg4x4", "vm44", "Us", MergeNone, "aarch64_sme_umla_za64_vg4x4", [IsStreaming, IsSharedZA], []>;
+  def SVMLAL_MULTI_VG4x2_S16 : Inst<"svmla_za64[_{d}]_vg4x2", "vm22", "s", MergeNone, "aarch64_sme_smla_za64_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_MULTI_VG4x2_U16 : Inst<"svmla_za64[_{d}]_vg4x2", "vm22", "Us", MergeNone, "aarch64_sme_umla_za64_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_MULTI_VG4x4_S16 : Inst<"svmla_za64[_{d}]_vg4x4", "vm44", "s", MergeNone, "aarch64_sme_smla_za64_vg4x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_MULTI_VG4x4_U16 : Inst<"svmla_za64[_{d}]_vg4x4", "vm44", "Us", MergeNone, "aarch64_sme_umla_za64_vg4x4", [IsStreaming, IsInOutZA], []>;
 
   // MULTI MLSL
-  def SVMLSL_MULTI_VG4x2_S16 : Inst<"svmls_za64[_{d}]_vg4x2", "vm22", "s", MergeNone, "aarch64_sme_smls_za64_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_MULTI_VG4x2_U16 : Inst<"svmls_za64[_{d}]_vg4x2", "vm22", "Us", MergeNone, "aarch64_sme_umls_za64_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_MULTI_VG4x4_S16 : Inst<"svmls_za64[_{d}]_vg4x4", "vm44", "s", MergeNone, "aarch64_sme_smls_za64_vg4x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_MULTI_VG4x4_U16 : Inst<"svmls_za64[_{d}]_vg4x4", "vm44", "Us", MergeNone, "aarch64_sme_umls_za64_vg4x4", [IsStreaming, IsSharedZA], []>;
+  def SVMLSL_MULTI_VG4x2_S16 : Inst<"svmls_za64[_{d}]_vg4x2", "vm22", "s", MergeNone, "aarch64_sme_smls_za64_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_MULTI_VG4x2_U16 : Inst<"svmls_za64[_{d}]_vg4x2", "vm22", "Us", MergeNone, "aarch64_sme_umls_za64_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_MULTI_VG4x4_S16 : Inst<"svmls_za64[_{d}]_vg4x4", "vm44", "s", MergeNone, "aarch64_sme_smls_za64_vg4x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_MULTI_VG4x4_U16 : Inst<"svmls_za64[_{d}]_vg4x4", "vm44", "Us", MergeNone, "aarch64_sme_umls_za64_vg4x4", [IsStreaming, IsInOutZA], []>;
 
   // SINGLE MLAL
-  def SVMLAL_SINGLE_VG4x1_S16 : Inst<"svmla_za64[_{d}]_vg4x1",          "vmdd", "s", MergeNone, "aarch64_sme_smla_za64_single_vg4x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG4x1_U16 : Inst<"svmla_za64[_{d}]_vg4x1",          "vmdd", "Us", MergeNone, "aarch64_sme_umla_za64_single_vg4x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG4x2_S16 : Inst<"svmla[_single]_za64[_{d}]_vg4x2", "vm2d", "s", MergeNone, "aarch64_sme_smla_za64_single_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG4x2_U16 : Inst<"svmla[_single]_za64[_{d}]_vg4x2", "vm2d", "Us", MergeNone, "aarch64_sme_umla_za64_single_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG4x4_S16 : Inst<"svmla[_single]_za64[_{d}]_vg4x4", "vm4d", "s", MergeNone, "aarch64_sme_smla_za64_single_vg4x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLAL_SINGLE_VG4x4_U16 : Inst<"svmla[_single]_za64[_{d}]_vg4x4", "vm4d", "Us", MergeNone, "aarch64_sme_umla_za64_single_vg4x4", [IsStreaming, IsSharedZA], []>;
+  def SVMLAL_SINGLE_VG4x1_S16 : Inst<"svmla_za64[_{d}]_vg4x1",          "vmdd", "s", MergeNone, "aarch64_sme_smla_za64_single_vg4x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG4x1_U16 : Inst<"svmla_za64[_{d}]_vg4x1",          "vmdd", "Us", MergeNone, "aarch64_sme_umla_za64_single_vg4x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG4x2_S16 : Inst<"svmla[_single]_za64[_{d}]_vg4x2", "vm2d", "s", MergeNone, "aarch64_sme_smla_za64_single_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG4x2_U16 : Inst<"svmla[_single]_za64[_{d}]_vg4x2", "vm2d", "Us", MergeNone, "aarch64_sme_umla_za64_single_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG4x4_S16 : Inst<"svmla[_single]_za64[_{d}]_vg4x4", "vm4d", "s", MergeNone, "aarch64_sme_smla_za64_single_vg4x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLAL_SINGLE_VG4x4_U16 : Inst<"svmla[_single]_za64[_{d}]_vg4x4", "vm4d", "Us", MergeNone, "aarch64_sme_umla_za64_single_vg4x4", [IsStreaming, IsInOutZA], []>;
 
   // SINGLE MLSL
-  def SVMLSL_SINGLE_VG4x1_S16 : Inst<"svmls_za64[_{d}]_vg4x1",          "vmdd", "s", MergeNone, "aarch64_sme_smls_za64_single_vg4x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG4x1_U16 : Inst<"svmls_za64[_{d}]_vg4x1",          "vmdd", "Us", MergeNone, "aarch64_sme_umls_za64_single_vg4x1", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG4x2_S16 : Inst<"svmls[_single]_za64[_{d}]_vg4x2", "vm2d", "s", MergeNone, "aarch64_sme_smls_za64_single_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG4x2_U16 : Inst<"svmls[_single]_za64[_{d}]_vg4x2", "vm2d", "Us", MergeNone, "aarch64_sme_umls_za64_single_vg4x2", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG4x4_S16 : Inst<"svmls[_single]_za64[_{d}]_vg4x4", "vm4d", "s", MergeNone, "aarch64_sme_smls_za64_single_vg4x4", [IsStreaming, IsSharedZA], []>;
-  def SVMLSL_SINGLE_VG4x4_U16 : Inst<"svmls[_single]_za64[_{d}]_vg4x4", "vm4d", "Us", MergeNone, "aarch64_sme_umls_za64_single_vg4x4", [IsStreaming, IsSharedZA], []>;
+  def SVMLSL_SINGLE_VG4x1_S16 : Inst<"svmls_za64[_{d}]_vg4x1",          "vmdd", "s", MergeNone, "aarch64_sme_smls_za64_single_vg4x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG4x1_U16 : Inst<"svmls_za64[_{d}]_vg4x1",          "vmdd", "Us", MergeNone, "aarch64_sme_umls_za64_single_vg4x1", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG4x2_S16 : Inst<"svmls[_single]_za64[_{d}]_vg4x2", "vm2d", "s", MergeNone, "aarch64_sme_smls_za64_single_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG4x2_U16 : Inst<"svmls[_single]_za64[_{d}]_vg4x2", "vm2d", "Us", MergeNone, "aarch64_sme_umls_za64_single_vg4x2", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG4x4_S16 : Inst<"svmls[_single]_za64[_{d}]_vg4x4", "vm4d", "s", MergeNone, "aarch64_sme_smls_za64_single_vg4x4", [IsStreaming, IsInOutZA], []>;
+  def SVMLSL_SINGLE_VG4x4_U16 : Inst<"svmls[_single]_za64[_{d}]_vg4x4", "vm4d", "Us", MergeNone, "aarch64_sme_umls_za64_single_vg4x4", [IsStreaming, IsInOutZA], []>;
 
   // INDEXED MLAL
-  def SVMLAL_LANE_VG4x1_S16 : Inst<"svmla_lane_za64[_{d}]_vg4x1", "vmddi", "s", MergeNone, "aarch64_sme_smla_za64_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLAL_LANE_VG4x1_U16 : Inst<"svmla_lane_za64[_{d}]_vg4x1", "vmddi", "Us", MergeNone, "aarch64_sme_umla_za64_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLAL_LANE_VG4x2_S16 : Inst<"svmla_lane_za64[_{d}]_vg4x2", "vm2di", "s", MergeNone, "aarch64_sme_smla_za64_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLAL_LANE_VG4x2_U16 : Inst<"svmla_lane_za64[_{d}]_vg4x2", "vm2di", "Us", MergeNone, "aarch64_sme_umla_za64_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLAL_LANE_VG4x4_S16 : Inst<"svmla_lane_za64[_{d}]_vg4x4", "vm4di", "s", MergeNone, "aarch64_sme_smla_za64_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLAL_LANE_VG4x4_U16 : Inst<"svmla_lane_za64[_{d}]_vg4x4", "vm4di", "Us", MergeNone, "aarch64_sme_umla_za64_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG4x1_S16 : Inst<"svmla_lane_za64[_{d}]_vg4x1", "vmddi", "s", MergeNone, "aarch64_sme_smla_za64_lane_vg4x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG4x1_U16 : Inst<"svmla_lane_za64[_{d}]_vg4x1", "vmddi", "Us", MergeNone, "aarch64_sme_umla_za64_lane_vg4x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG4x2_S16 : Inst<"svmla_lane_za64[_{d}]_vg4x2", "vm2di", "s", MergeNone, "aarch64_sme_smla_za64_lane_vg4x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG4x2_U16 : Inst<"svmla_lane_za64[_{d}]_vg4x2", "vm2di", "Us", MergeNone, "aarch64_sme_umla_za64_lane_vg4x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG4x4_S16 : Inst<"svmla_lane_za64[_{d}]_vg4x4", "vm4di", "s", MergeNone, "aarch64_sme_smla_za64_lane_vg4x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLAL_LANE_VG4x4_U16 : Inst<"svmla_lane_za64[_{d}]_vg4x4", "vm4di", "Us", MergeNone, "aarch64_sme_umla_za64_lane_vg4x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
 
   // INDEXED MLSL
-  def SVMLSL_LANE_VG4x1_S16 : Inst<"svmls_lane_za64[_{d}]_vg4x1", "vmddi", "s", MergeNone, "aarch64_sme_smls_za64_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLSL_LANE_VG4x1_U16 : Inst<"svmls_lane_za64[_{d}]_vg4x1", "vmddi", "Us", MergeNone, "aarch64_sme_umls_za64_lane_vg4x1", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLSL_LANE_VG4x2_S16 : Inst<"svmls_lane_za64[_{d}]_vg4x2", "vm2di", "s", MergeNone, "aarch64_sme_smls_za64_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLSL_LANE_VG4x2_U16 : Inst<"svmls_lane_za64[_{d}]_vg4x2", "vm2di", "Us", MergeNone, "aarch64_sme_umls_za64_lane_vg4x2", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLSL_LANE_VG4x4_S16 : Inst<"svmls_lane_za64[_{d}]_vg4x4", "vm4di", "s", MergeNone, "aarch64_sme_smls_za64_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVMLSL_LANE_VG4x4_U16 : Inst<"svmls_lane_za64[_{d}]_vg4x4", "vm4di", "Us", MergeNone, "aarch64_sme_umls_za64_lane_vg4x4", [IsStreaming, IsSharedZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG4x1_S16 : Inst<"svmls_lane_za64[_{d}]_vg4x1", "vmddi", "s", MergeNone, "aarch64_sme_smls_za64_lane_vg4x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG4x1_U16 : Inst<"svmls_lane_za64[_{d}]_vg4x1", "vmddi", "Us", MergeNone, "aarch64_sme_umls_za64_lane_vg4x1", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG4x2_S16 : Inst<"svmls_lane_za64[_{d}]_vg4x2", "vm2di", "s", MergeNone, "aarch64_sme_smls_za64_lane_vg4x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG4x2_U16 : Inst<"svmls_lane_za64[_{d}]_vg4x2", "vm2di", "Us", MergeNone, "aarch64_sme_umls_za64_lane_vg4x2", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG4x4_S16 : Inst<"svmls_lane_za64[_{d}]_vg4x4", "vm4di", "s", MergeNone, "aarch64_sme_smls_za64_lane_vg4x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVMLSL_LANE_VG4x4_U16 : Inst<"svmls_lane_za64[_{d}]_vg4x4", "vm4di", "Us", MergeNone, "aarch64_sme_umls_za64_lane_vg4x4", [IsStreaming, IsInOutZA], [ImmCheck<3, ImmCheck0_7>]>;
 }
 
 //
 // Spill and fill of ZT0
 //
 let TargetGuard = "sme2" in {
-  def SVLDR_ZT : Inst<"svldr_zt", "viQ", "", MergeNone, "aarch64_sme_ldr_zt", [IsOverloadNone, IsStreamingCompatible, IsSharedZA], [ImmCheck<0, ImmCheck0_0>]>;
-  def SVSTR_ZT : Inst<"svstr_zt", "vi%", "", MergeNone, "aarch64_sme_str_zt", [IsOverloadNone, IsStreamingCompatible, IsSharedZA, IsPreservesZA], [ImmCheck<0, ImmCheck0_0>]>;
+  def SVLDR_ZT : Inst<"svldr_zt", "viQ", "", MergeNone, "aarch64_sme_ldr_zt", [IsOverloadNone, IsStreamingCompatible, IsInOutZT0], [ImmCheck<0, ImmCheck0_0>]>;
+  def SVSTR_ZT : Inst<"svstr_zt", "vi%", "", MergeNone, "aarch64_sme_str_zt", [IsOverloadNone, IsStreamingCompatible, IsInZT0], [ImmCheck<0, ImmCheck0_0>]>;
 }
 
 //
 // Zero ZT0
 //
 let TargetGuard = "sme2" in {
-  def SVZERO_ZT : Inst<"svzero_zt", "vi", "", MergeNone, "aarch64_sme_zero_zt", [IsOverloadNone, IsStreamingCompatible, IsSharedZA], [ImmCheck<0, ImmCheck0_0>]>;
+  def SVZERO_ZT : Inst<"svzero_zt", "vi", "", MergeNone, "aarch64_sme_zero_zt", [IsOverloadNone, IsStreamingCompatible, IsOutZT0], [ImmCheck<0, ImmCheck0_0>]>;
 }
 
 //
 // lookup table expand four contiguous registers
 //
 let TargetGuard = "sme2" in {
-  def SVLUTI2_LANE_ZT_X4 : Inst<"svluti2_lane_zt_{d}_x4", "4.di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti2_lane_zt_x4", [IsStreaming, IsSharedZA, IsPreservesZA], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_3>]>;
-  def SVLUTI4_LANE_ZT_X4 : Inst<"svluti4_lane_zt_{d}_x4", "4.di[i", "sUsiUibhf", MergeNone, "aarch64_sme_luti4_lane_zt_x4", [IsStreaming, IsSharedZA, IsPreservesZA], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_1>]>;
+  def SVLUTI2_LANE_ZT_X4 : Inst<"svluti2_lane_zt_{d}_x4", "4.di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti2_lane_zt_x4", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_3>]>;
+  def SVLUTI4_LANE_ZT_X4 : Inst<"svluti4_lane_zt_{d}_x4", "4.di[i", "sUsiUibhf", MergeNone, "aarch64_sme_luti4_lane_zt_x4", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_1>]>;
 }
 
 //
 // lookup table expand one register
 //
 let TargetGuard = "sme2" in {
-  def SVLUTI2_LANE_ZT : Inst<"svluti2_lane_zt_{d}", "di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti2_lane_zt", [IsStreaming, IsSharedZA, IsPreservesZA], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_15>]>;
-  def SVLUTI4_LANE_ZT : Inst<"svluti4_lane_zt_{d}", "di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti4_lane_zt", [IsStreaming, IsSharedZA, IsPreservesZA], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_7>]>;
+  def SVLUTI2_LANE_ZT : Inst<"svluti2_lane_zt_{d}", "di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti2_lane_zt", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_15>]>;
+  def SVLUTI4_LANE_ZT : Inst<"svluti4_lane_zt_{d}", "di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti4_lane_zt", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_7>]>;
 }
 
 //
 // lookup table expand two contiguous registers
 //
 let TargetGuard = "sme2" in {
-  def SVLUTI2_LANE_ZT_X2 : Inst<"svluti2_lane_zt_{d}_x2", "2.di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti2_lane_zt_x2", [IsStreaming, IsSharedZA, IsPreservesZA], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_7>]>;
-  def SVLUTI4_LANE_ZT_X2 : Inst<"svluti4_lane_zt_{d}_x2", "2.di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti4_lane_zt_x2", [IsStreaming, IsSharedZA, IsPreservesZA], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_3>]>;
+  def SVLUTI2_LANE_ZT_X2 : Inst<"svluti2_lane_zt_{d}_x2", "2.di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti2_lane_zt_x2", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_7>]>;
+  def SVLUTI4_LANE_ZT_X2 : Inst<"svluti4_lane_zt_{d}_x2", "2.di[i", "cUcsUsiUibhf", MergeNone, "aarch64_sme_luti4_lane_zt_x2", [IsStreaming, IsInZT0], [ImmCheck<0, ImmCheck0_0>, ImmCheck<2, ImmCheck0_3>]>;
 }
diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index 91f62c4c7633..6da30e08e752 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -454,11 +454,11 @@ let TargetGuard = "sve,bf16" in {
 
 let TargetGuard = "sve2p1" in {
   // Contiguous truncating store from quadword (single vector).
-  def SVST1UWQ      : MInst<"svst1uwq[_{d}]", "vPcd", "iUif",  [IsStore], MemEltTyInt32, "aarch64_sve_st1uwq">;
-  def SVST1UWQ_VNUM : MInst<"svst1uwq_vnum[_{d}]", "vPcld", "iUif", [IsStore], MemEltTyInt32, "aarch64_sve_st1uwq">;
+  def SVST1UWQ      : MInst<"svst1wq[_{d}]", "vPcd", "iUif",  [IsStore], MemEltTyInt32, "aarch64_sve_st1wq">;
+  def SVST1UWQ_VNUM : MInst<"svst1wq_vnum[_{d}]", "vPcld", "iUif", [IsStore], MemEltTyInt32, "aarch64_sve_st1wq">;
 
-  def SVST1UDQ      : MInst<"svst1udq[_{d}]", "vPcd", "lUld",  [IsStore], MemEltTyInt64, "aarch64_sve_st1udq">;
-  def SVST1UDQ_VNUM : MInst<"svst1udq_vnum[_{d}]", "vPcld", "lUld", [IsStore], MemEltTyInt64, "aarch64_sve_st1udq">;
+  def SVST1UDQ      : MInst<"svst1dq[_{d}]", "vPcd", "lUld",  [IsStore], MemEltTyInt64, "aarch64_sve_st1dq">;
+  def SVST1UDQ_VNUM : MInst<"svst1dq_vnum[_{d}]", "vPcld", "lUld", [IsStore], MemEltTyInt64, "aarch64_sve_st1dq">;
 
   // Store one vector (vector base + scalar offset)
   def SVST1Q_SCATTER_U64BASE_OFFSET : MInst<"svst1q_scatter[_{2}base]_offset[_{d}]",  "vPgld", "cUcsUsiUilUlfhdb", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1q_scatter_scalar_offset">;
@@ -543,13 +543,13 @@ def SVADRD : SInst<"svadrd[_{0}base]_[{2}]index",  "uud", "ilUiUl", MergeNone, "
 ////////////////////////////////////////////////////////////////////////////////
 // Scalar to vector
 
-def SVDUPQ_8  : SInst<"svdupq[_n]_{d}", "dssssssssssssssss",  "cUc", MergeNone>;
-def SVDUPQ_16 : SInst<"svdupq[_n]_{d}", "dssssssss",  "sUsh", MergeNone>;
+def SVDUPQ_8  : SInst<"svdupq[_n]_{d}", "dssssssssssssssss",  "cUc", MergeNone, "", [IsStreamingCompatible]>;
+def SVDUPQ_16 : SInst<"svdupq[_n]_{d}", "dssssssss",  "sUsh", MergeNone, "", [IsStreamingCompatible]>;
 let TargetGuard = "sve,bf16" in {
-  def SVDUPQ_BF16 : SInst<"svdupq[_n]_{d}", "dssssssss",  "b", MergeNone>;
+  def SVDUPQ_BF16 : SInst<"svdupq[_n]_{d}", "dssssssss",  "b", MergeNone, "", [IsStreamingCompatible]>;
 }
-def SVDUPQ_32 : SInst<"svdupq[_n]_{d}", "dssss",  "iUif", MergeNone>;
-def SVDUPQ_64 : SInst<"svdupq[_n]_{d}", "dss",  "lUld", MergeNone>;
+def SVDUPQ_32 : SInst<"svdupq[_n]_{d}", "dssss",  "iUif", MergeNone, "", [IsStreamingCompatible]>;
+def SVDUPQ_64 : SInst<"svdupq[_n]_{d}", "dss",  "lUld", MergeNone, "", [IsStreamingCompatible]>;
 
 multiclass svdup_base {
   def NAME : SInst;
@@ -638,8 +638,8 @@ def SVQADD_N_U : SInst<"svqadd[_n_{d}]", "dda",  "UcUsUiUl", MergeNone, "aarch64
 def SVQSUB_N_S : SInst<"svqsub[_n_{d}]", "dda",  "csil",     MergeNone, "aarch64_sve_sqsub_x", [IsStreamingCompatible]>;
 def SVQSUB_N_U : SInst<"svqsub[_n_{d}]", "dda",  "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x", [IsStreamingCompatible]>;
 
-def SVDOT_LANE_S : SInst<"svdot_lane[_{d}]",  "ddqqi",  "il",   MergeNone, "aarch64_sve_sdot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
-def SVDOT_LANE_U : SInst<"svdot_lane[_{d}]",  "ddqqi",  "UiUl", MergeNone, "aarch64_sve_udot_lane", [], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
+def SVDOT_LANE_S : SInst<"svdot_lane[_{d}]",  "ddqqi",  "il",   MergeNone, "aarch64_sve_sdot_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
+def SVDOT_LANE_U : SInst<"svdot_lane[_{d}]",  "ddqqi",  "UiUl", MergeNone, "aarch64_sve_udot_lane", [IsStreamingCompatible], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
 
 ////////////////////////////////////////////////////////////////////////////////
 // Logical operations
@@ -749,14 +749,14 @@ def SVCMPLS_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone,
 ////////////////////////////////////////////////////////////////////////////////
 // While comparisons
 
-def SVWHILELE_S32 : SInst<"svwhilele_{d}[_{1}]", "Pkk", "PcPsPiPl",     MergeNone, "aarch64_sve_whilele", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILELE_S64 : SInst<"svwhilele_{d}[_{1}]", "Pll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilele", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILELO_U32 : SInst<"svwhilelt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILELO_U64 : SInst<"svwhilelt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILELS_U32 : SInst<"svwhilele_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILELS_U64 : SInst<"svwhilele_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILELT_S32 : SInst<"svwhilelt_{d}[_{1}]", "Pkk", "PcPsPiPl",     MergeNone, "aarch64_sve_whilelt", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILELT_S64 : SInst<"svwhilelt_{d}[_{1}]", "Pll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilelt", [IsOverloadWhile, IsStreamingCompatible]>;
+def SVWHILELE_S32 : SInst<"svwhilele_{d}[_{1}]", "Pkk", "PcPsPiPl",     MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILELE_S64 : SInst<"svwhilele_{d}[_{1}]", "Pll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILELO_U32 : SInst<"svwhilelt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILELO_U64 : SInst<"svwhilelt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILELS_U32 : SInst<"svwhilele_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILELS_U64 : SInst<"svwhilele_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILELT_S32 : SInst<"svwhilelt_{d}[_{1}]", "Pkk", "PcPsPiPl",     MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILELT_S64 : SInst<"svwhilelt_{d}[_{1}]", "Pll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
 
 ////////////////////////////////////////////////////////////////////////////////
 // Counting bit
@@ -835,14 +835,14 @@ def SVSCALE_N_M : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeOp1,  "aarch64_sv
 def SVSCALE_N_X : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeAny,  "aarch64_sve_fscale", [IsStreamingCompatible]>;
 def SVSCALE_N_Z : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeZero, "aarch64_sve_fscale", [IsStreamingCompatible]>;
 
-defm SVMAD_F  : SInstZPZZZ<"svmad",  "hfd", "aarch64_sve_fmad",  "aarch64_sve_fmla_u",  [ReverseMergeAnyAccOp]>;
-defm SVMLA_F  : SInstZPZZZ<"svmla",  "hfd", "aarch64_sve_fmla",  "aarch64_sve_fmla_u">;
-defm SVMLS_F  : SInstZPZZZ<"svmls",  "hfd", "aarch64_sve_fmls",  "aarch64_sve_fmls_u">;
-defm SVMSB_F  : SInstZPZZZ<"svmsb",  "hfd", "aarch64_sve_fmsb",  "aarch64_sve_fmls_u",  [ReverseMergeAnyAccOp]>;
-defm SVNMAD_F : SInstZPZZZ<"svnmad", "hfd", "aarch64_sve_fnmad", "aarch64_sve_fnmla_u", [ReverseMergeAnyAccOp]>;
-defm SVNMLA_F : SInstZPZZZ<"svnmla", "hfd", "aarch64_sve_fnmla", "aarch64_sve_fnmla_u">;
-defm SVNMLS_F : SInstZPZZZ<"svnmls", "hfd", "aarch64_sve_fnmls", "aarch64_sve_fnmls_u">;
-defm SVNMSB_F : SInstZPZZZ<"svnmsb", "hfd", "aarch64_sve_fnmsb", "aarch64_sve_fnmls_u", [ReverseMergeAnyAccOp]>;
+defm SVMAD_F  : SInstZPZZZ<"svmad",  "hfd", "aarch64_sve_fmad",  "aarch64_sve_fmla_u",  [IsStreamingCompatible, ReverseMergeAnyAccOp]>;
+defm SVMLA_F  : SInstZPZZZ<"svmla",  "hfd", "aarch64_sve_fmla",  "aarch64_sve_fmla_u", [IsStreamingCompatible]>;
+defm SVMLS_F  : SInstZPZZZ<"svmls",  "hfd", "aarch64_sve_fmls",  "aarch64_sve_fmls_u", [IsStreamingCompatible]>;
+defm SVMSB_F  : SInstZPZZZ<"svmsb",  "hfd", "aarch64_sve_fmsb",  "aarch64_sve_fmls_u",  [IsStreamingCompatible, ReverseMergeAnyAccOp]>;
+defm SVNMAD_F : SInstZPZZZ<"svnmad", "hfd", "aarch64_sve_fnmad", "aarch64_sve_fnmla_u", [IsStreamingCompatible, ReverseMergeAnyAccOp]>;
+defm SVNMLA_F : SInstZPZZZ<"svnmla", "hfd", "aarch64_sve_fnmla", "aarch64_sve_fnmla_u", [IsStreamingCompatible]>;
+defm SVNMLS_F : SInstZPZZZ<"svnmls", "hfd", "aarch64_sve_fnmls", "aarch64_sve_fnmls_u", [IsStreamingCompatible]>;
+defm SVNMSB_F : SInstZPZZZ<"svnmsb", "hfd", "aarch64_sve_fnmsb", "aarch64_sve_fnmls_u", [IsStreamingCompatible, ReverseMergeAnyAccOp]>;
 
 def SVCADD_M : SInst<"svcadd[_{d}]", "dPddi",  "hfd", MergeOp1,  "aarch64_sve_fcadd", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRot90_270>]>;
 def SVCADD_X : SInst<"svcadd[_{d}]", "dPddi",  "hfd", MergeAny,  "aarch64_sve_fcadd", [IsStreamingCompatible], [ImmCheck<3, ImmCheckComplexRot90_270>]>;
@@ -881,11 +881,11 @@ def SVACLE  : SInst<"svacle[_{d}]",  "PPdd", "hfd", MergeNone, "aarch64_sve_facg
 def SVACLT  : SInst<"svaclt[_{d}]",  "PPdd", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare, IsStreamingCompatible]>;
 def SVCMPUO : SInst<"svcmpuo[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpuo", [IsStreamingCompatible]>;
 
-def SVACGE_N  : SInst<"svacge[_n_{d}]",  "PPda", "hfd", MergeNone, "aarch64_sve_facge">;
-def SVACGT_N  : SInst<"svacgt[_n_{d}]",  "PPda", "hfd", MergeNone, "aarch64_sve_facgt">;
-def SVACLE_N  : SInst<"svacle[_n_{d}]",  "PPda", "hfd", MergeNone, "aarch64_sve_facge", [ReverseCompare]>;
-def SVACLT_N  : SInst<"svaclt[_n_{d}]",  "PPda", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare]>;
-def SVCMPUO_N : SInst<"svcmpuo[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpuo">;
+def SVACGE_N  : SInst<"svacge[_n_{d}]",  "PPda", "hfd", MergeNone, "aarch64_sve_facge", [IsStreamingCompatible]>;
+def SVACGT_N  : SInst<"svacgt[_n_{d}]",  "PPda", "hfd", MergeNone, "aarch64_sve_facgt", [IsStreamingCompatible]>;
+def SVACLE_N  : SInst<"svacle[_n_{d}]",  "PPda", "hfd", MergeNone, "aarch64_sve_facge", [ReverseCompare, IsStreamingCompatible]>;
+def SVACLT_N  : SInst<"svaclt[_n_{d}]",  "PPda", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare, IsStreamingCompatible]>;
+def SVCMPUO_N : SInst<"svcmpuo[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpuo", [IsStreamingCompatible]>;
 
 def SVCMPEQ_F : SInst<"svcmpeq[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpeq", [IsStreamingCompatible]>;
 def SVCMPNE_F : SInst<"svcmpne[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpne", [IsStreamingCompatible]>;
@@ -1023,15 +1023,15 @@ def SVCOMPACT    : SInst<"svcompact[_{d}]",   "dPd",  "ilUiUlfd",        MergeNo
 // splat of any possible lane. It is upto LLVM to pick a more efficient
 // instruction such as DUP (indexed) if the lane index fits the range of the
 // instruction's immediate.
-def SVDUP_LANE   : SInst<"svdup_lane[_{d}]",  "ddL",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl">;
+def SVDUP_LANE   : SInst<"svdup_lane[_{d}]",  "ddL",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl", [IsStreamingCompatible]>;
 let TargetGuard = "sve,bf16" in {
 def SVDUP_LANE_BF16 :
-                   SInst<"svdup_lane[_{d}]",  "ddL",  "b",               MergeNone, "aarch64_sve_tbl">;
+                   SInst<"svdup_lane[_{d}]",  "ddL",  "b",               MergeNone, "aarch64_sve_tbl", [IsStreamingCompatible]>;
 }
 
-def SVDUPQ_LANE  : SInst<"svdupq_lane[_{d}]", "ddn",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_dupq_lane">;
+def SVDUPQ_LANE  : SInst<"svdupq_lane[_{d}]", "ddn",  "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_dupq_lane", [IsStreamingCompatible]>;
 let TargetGuard = "sve,bf16" in {
-  def SVDUPQ_LANE_BF16  : SInst<"svdupq_lane[_{d}]", "ddn",  "b", MergeNone, "aarch64_sve_dupq_lane">;
+  def SVDUPQ_LANE_BF16  : SInst<"svdupq_lane[_{d}]", "ddn",  "b", MergeNone, "aarch64_sve_dupq_lane", [IsStreamingCompatible]>;
 }
 def SVEXT        : SInst<"svext[_{d}]",       "dddi", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ext", [IsStreamingCompatible], [ImmCheck<2, ImmCheckExtract, 1>]>;
 defm SVLASTA     : SVEPerm<"svlasta[_{d}]",   "sPd",  "aarch64_sve_lasta">;
@@ -1109,11 +1109,11 @@ def SVPFALSE : SInst<"svpfalse[_b]", "Pv", "", MergeNone, "", [IsOverloadNone, I
 def SVPTRUE_PAT : SInst<"svptrue_pat_{d}", "PI", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [IsStreamingCompatible]>;
 def SVPTRUE     : SInst<"svptrue_{d}",     "Pv",  "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [IsAppendSVALL, IsStreamingCompatible]>;
 
-def SVDUPQ_B8      : SInst<"svdupq[_n]_{d}",  "Pssssssssssssssss",  "Pc", MergeNone>;
-def SVDUPQ_B16     : SInst<"svdupq[_n]_{d}", "Pssssssss",  "Ps", MergeNone>;
-def SVDUPQ_B32     : SInst<"svdupq[_n]_{d}", "Pssss",  "Pi", MergeNone>;
-def SVDUPQ_B64     : SInst<"svdupq[_n]_{d}", "Pss",  "Pl", MergeNone>;
-def SVDUP_N_B      : SInst<"svdup[_n]_{d}",  "Ps", "PcPsPiPl", MergeNone>;
+def SVDUPQ_B8      : SInst<"svdupq[_n]_{d}",  "Pssssssssssssssss",  "Pc", MergeNone, "", [IsStreamingCompatible]>;
+def SVDUPQ_B16     : SInst<"svdupq[_n]_{d}", "Pssssssss",  "Ps", MergeNone, "", [IsStreamingCompatible]>;
+def SVDUPQ_B32     : SInst<"svdupq[_n]_{d}", "Pssss",  "Pi", MergeNone, "", [IsStreamingCompatible]>;
+def SVDUPQ_B64     : SInst<"svdupq[_n]_{d}", "Pss",  "Pl", MergeNone, "", [IsStreamingCompatible]>;
+def SVDUP_N_B      : SInst<"svdup[_n]_{d}",  "Ps", "PcPsPiPl", MergeNone, "", [IsStreamingCompatible]>;
 
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -1268,10 +1268,10 @@ def SVZIP2Q      : SInst<"svzip2q[_{d}]",     "ddd",  "csilUcUsUiUlhfd", MergeNo
 let TargetGuard = "sve,bf16,f64mm" in {
 def SVTRN1Q_BF16      : SInst<"svtrn1q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_trn1q">;
 def SVTRN2Q_BF16      : SInst<"svtrn2q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_trn2q">;
-def SVUZP1Q_BF16      : SInst<"svuzp1q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_uzp1q">;
-def SVUZP2Q_BF16      : SInst<"svuzp2q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_uzp2q">;
-def SVZIP1Q_BF16      : SInst<"svzip1q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_zip1q">;
-def SVZIP2Q_BF16      : SInst<"svzip2q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_zip2q">;
+def SVUZP1Q_BF16      : SInst<"svuzp1q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_uzp1q", [IsStreamingCompatible]>;
+def SVUZP2Q_BF16      : SInst<"svuzp2q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_uzp2q", [IsStreamingCompatible]>;
+def SVZIP1Q_BF16      : SInst<"svzip1q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_zip1q", [IsStreamingCompatible]>;
+def SVZIP2Q_BF16      : SInst<"svzip2q[_{d}]",     "ddd",  "b", MergeNone, "aarch64_sve_zip2q", [IsStreamingCompatible]>;
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -1281,9 +1281,9 @@ def SVUNDEF_2 : SInst<"svundef2_{d}", "2v", "csilUcUsUiUlhfd", MergeNone, "", [I
 def SVUNDEF_3 : SInst<"svundef3_{d}", "3v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, IsStreamingCompatible]>;
 def SVUNDEF_4 : SInst<"svundef4_{d}", "4v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, IsStreamingCompatible]>;
 
-def SVCREATE_2 : SInst<"svcreate2[_{d}]", "2dd",   "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate]>;
-def SVCREATE_3 : SInst<"svcreate3[_{d}]", "3ddd",  "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate]>;
-def SVCREATE_4 : SInst<"svcreate4[_{d}]", "4dddd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate]>;
+def SVCREATE_2 : SInst<"svcreate2[_{d}]", "2dd",   "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate, IsStreamingCompatible]>;
+def SVCREATE_3 : SInst<"svcreate3[_{d}]", "3ddd",  "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate, IsStreamingCompatible]>;
+def SVCREATE_4 : SInst<"svcreate4[_{d}]", "4dddd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate, IsStreamingCompatible]>;
 
 let TargetGuard = "sve,bf16" in {
 def SVUNDEF_1_BF16 : SInst<"svundef_{d}",  "dv", "b", MergeNone, "", [IsUndef, IsStreamingCompatible]>;
@@ -1291,65 +1291,70 @@ def SVUNDEF_2_BF16 : SInst<"svundef2_{d}", "2v", "b", MergeNone, "", [IsUndef, I
 def SVUNDEF_3_BF16 : SInst<"svundef3_{d}", "3v", "b", MergeNone, "", [IsUndef, IsStreamingCompatible]>;
 def SVUNDEF_4_BF16 : SInst<"svundef4_{d}", "4v", "b", MergeNone, "", [IsUndef, IsStreamingCompatible]>;
 
-def SVCREATE_2_BF16 : SInst<"svcreate2[_{d}]", "2dd",   "b", MergeNone, "", [IsTupleCreate]>;
-def SVCREATE_3_BF16 : SInst<"svcreate3[_{d}]", "3ddd",  "b", MergeNone, "", [IsTupleCreate]>;
-def SVCREATE_4_BF16 : SInst<"svcreate4[_{d}]", "4dddd", "b", MergeNone, "", [IsTupleCreate]>;
+def SVCREATE_2_BF16 : SInst<"svcreate2[_{d}]", "2dd",   "b", MergeNone, "", [IsTupleCreate, IsStreamingCompatible]>;
+def SVCREATE_3_BF16 : SInst<"svcreate3[_{d}]", "3ddd",  "b", MergeNone, "", [IsTupleCreate, IsStreamingCompatible]>;
+def SVCREATE_4_BF16 : SInst<"svcreate4[_{d}]", "4dddd", "b", MergeNone, "", [IsTupleCreate, IsStreamingCompatible]>;
 }
 
-let TargetGuard = "sve2p1" in {
-  def SVCREATE_2_B : SInst<"svcreate2[_{d}]", "2dd",   "Pc", MergeNone, "", [IsTupleCreate]>;
-  def SVCREATE_4_B : SInst<"svcreate4[_{d}]", "4dddd", "Pc", MergeNone, "", [IsTupleCreate]>;
+let TargetGuard = "sve2p1|sme2" in {
+  def SVCREATE_2_B : SInst<"svcreate2[_b]", "2dd",   "Pc", MergeNone, "", [IsTupleCreate, IsStreamingCompatible]>;
+  def SVCREATE_4_B : SInst<"svcreate4[_b]", "4dddd", "Pc", MergeNone, "", [IsTupleCreate, IsStreamingCompatible]>;
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 // Vector insertion and extraction
-def SVGET_2 : SInst<"svget2[_{d}]", "d2i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_1>]>;
-def SVGET_3 : SInst<"svget3[_{d}]", "d3i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_2>]>;
-def SVGET_4 : SInst<"svget4[_{d}]", "d4i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_3>]>;
+def SVGET_2 : SInst<"svget2[_{d}]", "d2i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>;
+def SVGET_3 : SInst<"svget3[_{d}]", "d3i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_2>]>;
+def SVGET_4 : SInst<"svget4[_{d}]", "d4i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>;
 
-def SVSET_2 : SInst<"svset2[_{d}]", "22id", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_1>]>;
-def SVSET_3 : SInst<"svset3[_{d}]", "33id", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_2>]>;
-def SVSET_4 : SInst<"svset4[_{d}]", "44id", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_3>]>;
+def SVSET_2 : SInst<"svset2[_{d}]", "22id", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleSet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>;
+def SVSET_3 : SInst<"svset3[_{d}]", "33id", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleSet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_2>]>;
+def SVSET_4 : SInst<"svset4[_{d}]", "44id", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleSet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>;
 
 let TargetGuard = "sve,bf16" in {
-def SVGET_2_BF16 : SInst<"svget2[_{d}]", "d2i", "b", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_1>]>;
-def SVGET_3_BF16 : SInst<"svget3[_{d}]", "d3i", "b", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_2>]>;
-def SVGET_4_BF16 : SInst<"svget4[_{d}]", "d4i", "b", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_3>]>;
+def SVGET_2_BF16 : SInst<"svget2[_{d}]", "d2i", "b", MergeNone, "", [IsTupleGet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>;
+def SVGET_3_BF16 : SInst<"svget3[_{d}]", "d3i", "b", MergeNone, "", [IsTupleGet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_2>]>;
+def SVGET_4_BF16 : SInst<"svget4[_{d}]", "d4i", "b", MergeNone, "", [IsTupleGet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>;
 
-def SVSET_2_BF16 : SInst<"svset2[_{d}]", "22id", "b", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_1>]>;
-def SVSET_3_BF16 : SInst<"svset3[_{d}]", "33id", "b", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_2>]>;
-def SVSET_4_BF16 : SInst<"svset4[_{d}]", "44id", "b", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_3>]>;
+def SVSET_2_BF16 : SInst<"svset2[_{d}]", "22id", "b", MergeNone, "", [IsTupleSet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>;
+def SVSET_3_BF16 : SInst<"svset3[_{d}]", "33id", "b", MergeNone, "", [IsTupleSet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_2>]>;
+def SVSET_4_BF16 : SInst<"svset4[_{d}]", "44id", "b", MergeNone, "", [IsTupleSet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>;
 }
 
-let TargetGuard = "sve2p1" in {
-  def SVGET_2_B : SInst<"svget2[_{d}]", "d2i", "Pc", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_1>]>;
-  def SVGET_4_B : SInst<"svget4[_{d}]", "d4i", "Pc", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_3>]>;
+let TargetGuard = "sve2p1|sme2" in {
+  def SVGET_2_B : SInst<"svget2[_b]", "d2i", "Pc", MergeNone, "", [IsTupleGet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>;
+  def SVGET_4_B : SInst<"svget4[_b]", "d4i", "Pc", MergeNone, "", [IsTupleGet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>;
+
+  def SVSET_2_B : SInst<"svset2[_b]", "22id", "Pc", MergeNone, "", [IsTupleSet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>;
+  def SVSET_4_B : SInst<"svset4[_b]", "44id", "Pc", MergeNone, "", [IsTupleSet, IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>;
+}
 
-  def SVSET_2_B : SInst<"svset2[_{d}]", "22id", "Pc", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_1>]>;
-  def SVSET_4_B : SInst<"svset4[_{d}]", "44id", "Pc", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_3>]>;
+let TargetGuard = "sve2p1|sme2" in {
+  def SVUNDEF_2_B: Inst<"svundef2_b", "2", "Pc", MergeNone, "", [IsUndef, IsStreamingCompatible], []>;
+  def SVUNDEF_4_B: Inst<"svundef4_b", "4", "Pc", MergeNone, "", [IsUndef, IsStreamingCompatible], []>;
 }
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 WhileGE/GT
 let TargetGuard = "sve2" in {
-def SVWHILEGE_S32 : SInst<"svwhilege_{d}[_{1}]", "Pkk", "PcPsPiPl",     MergeNone, "aarch64_sve_whilege", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILEGE_S64 : SInst<"svwhilege_{d}[_{1}]", "Pll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilege", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILEGT_S32 : SInst<"svwhilegt_{d}[_{1}]", "Pkk", "PcPsPiPl",     MergeNone, "aarch64_sve_whilegt", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILEGT_S64 : SInst<"svwhilegt_{d}[_{1}]", "Pll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilegt", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILEHI_U32 : SInst<"svwhilegt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILEHI_U64 : SInst<"svwhilegt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhile, IsStreamingCompatible]>;
-def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhile, IsStreamingCompatible]>;
+def SVWHILEGE_S32 : SInst<"svwhilege_{d}[_{1}]", "Pkk", "PcPsPiPl",     MergeNone, "aarch64_sve_whilege", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILEGE_S64 : SInst<"svwhilege_{d}[_{1}]", "Pll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilege", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILEGT_S32 : SInst<"svwhilegt_{d}[_{1}]", "Pkk", "PcPsPiPl",     MergeNone, "aarch64_sve_whilegt", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILEGT_S64 : SInst<"svwhilegt_{d}[_{1}]", "Pll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilegt", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILEHI_U32 : SInst<"svwhilegt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILEHI_U64 : SInst<"svwhilegt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
+def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, IsStreamingCompatible]>;
 }
 
 let TargetGuard = "sve2p1|sme2"  in {
-  def SVWHILEGE_S64_X2 : SInst<"svwhilege_{d}[_{1}]_x2", "2ll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilege_x2">;
-  def SVWHILEGT_S64_X2 : SInst<"svwhilegt_{d}[_{1}]_x2", "2ll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilegt_x2">;
-  def SVWHILEHI_U64_X2 : SInst<"svwhilegt_{d}[_{1}]_x2", "2nn", "PcPsPiPl",     MergeNone, "aarch64_sve_whilehi_x2">;
-  def SVWHILEHS_U64_X2 : SInst<"svwhilege_{d}[_{1}]_x2", "2nn", "PcPsPiPl",     MergeNone, "aarch64_sve_whilehs_x2">;
-  def SVWHILELE_S64_X2 : SInst<"svwhilele_{d}[_{1}]_x2", "2ll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilele_x2">;
-  def SVWHILELT_S64_X2 : SInst<"svwhilelt_{d}[_{1}]_x2", "2ll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilelt_x2">;
-  def SVWHILELO_U64_X2 : SInst<"svwhilelt_{d}[_{1}]_x2", "2nn", "PcPsPiPl",     MergeNone, "aarch64_sve_whilelo_x2">;
-  def SVWHILELS_U64_X2 : SInst<"svwhilele_{d}[_{1}]_x2", "2nn", "PcPsPiPl",     MergeNone, "aarch64_sve_whilels_x2">;
+  def SVWHILEGE_S64_X2 : SInst<"svwhilege_{d}[_{1}]_x2", "2ll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilege_x2", [IsStreamingOrSVE2p1]>;
+  def SVWHILEGT_S64_X2 : SInst<"svwhilegt_{d}[_{1}]_x2", "2ll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilegt_x2", [IsStreamingOrSVE2p1]>;
+  def SVWHILEHI_U64_X2 : SInst<"svwhilegt_{d}[_{1}]_x2", "2nn", "PcPsPiPl",     MergeNone, "aarch64_sve_whilehi_x2", [IsStreamingOrSVE2p1]>;
+  def SVWHILEHS_U64_X2 : SInst<"svwhilege_{d}[_{1}]_x2", "2nn", "PcPsPiPl",     MergeNone, "aarch64_sve_whilehs_x2", [IsStreamingOrSVE2p1]>;
+  def SVWHILELE_S64_X2 : SInst<"svwhilele_{d}[_{1}]_x2", "2ll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilele_x2", [IsStreamingOrSVE2p1]>;
+  def SVWHILELT_S64_X2 : SInst<"svwhilelt_{d}[_{1}]_x2", "2ll", "PcPsPiPl",     MergeNone, "aarch64_sve_whilelt_x2", [IsStreamingOrSVE2p1]>;
+  def SVWHILELO_U64_X2 : SInst<"svwhilelt_{d}[_{1}]_x2", "2nn", "PcPsPiPl",     MergeNone, "aarch64_sve_whilelo_x2", [IsStreamingOrSVE2p1]>;
+  def SVWHILELS_U64_X2 : SInst<"svwhilele_{d}[_{1}]_x2", "2nn", "PcPsPiPl",     MergeNone, "aarch64_sve_whilels_x2", [IsStreamingOrSVE2p1]>;
 
 }
 
@@ -1826,8 +1831,8 @@ def SVPMULLB_PAIR   : SInst<"svpmullb_pair[_{d}]",   "ddd",  "UcUi",         Mer
 def SVPMULLB_PAIR_N : SInst<"svpmullb_pair[_n_{d}]", "dda",  "UcUi",         MergeNone, "aarch64_sve_pmullb_pair", [IsStreamingCompatible]>;
 def SVPMULLT        : SInst<"svpmullt[_{d}]",        "dhh",  "UsUl",         MergeNone, "", [IsStreamingCompatible]>;
 def SVPMULLT_N      : SInst<"svpmullt[_n_{d}]",      "dhR",  "UsUl",         MergeNone, "", [IsStreamingCompatible]>;
-def SVPMULLT_PAIR   : SInst<"svpmullt_pair[_{d}]",   "ddd",  "UcUi",         MergeNone, "aarch64_sve_pmullt_pair">;
-def SVPMULLT_PAIR_N : SInst<"svpmullt_pair[_n_{d}]", "dda",  "UcUi",         MergeNone, "aarch64_sve_pmullt_pair">;
+def SVPMULLT_PAIR   : SInst<"svpmullt_pair[_{d}]",   "ddd",  "UcUi",         MergeNone, "aarch64_sve_pmullt_pair", [IsStreamingCompatible]>;
+def SVPMULLT_PAIR_N : SInst<"svpmullt_pair[_n_{d}]", "dda",  "UcUi",         MergeNone, "aarch64_sve_pmullt_pair", [IsStreamingCompatible]>;
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -1948,14 +1953,10 @@ def SVBGRP_N : SInst<"svbgrp[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sv
 }
 
 let TargetGuard = "sve2p1|sme" in {
-def SVPSEL_B : SInst<"svpsel_lane_b8",  "PPPm", "Pc", MergeNone, "", [IsStreamingCompatible], []>;
-def SVPSEL_H : SInst<"svpsel_lane_b16", "PPPm", "Ps", MergeNone, "", [IsStreamingCompatible], []>;
-def SVPSEL_S : SInst<"svpsel_lane_b32", "PPPm", "Pi", MergeNone, "", [IsStreamingCompatible], []>;
-def SVPSEL_D : SInst<"svpsel_lane_b64", "PPPm", "Pl", MergeNone, "", [IsStreamingCompatible], []>;
-def SVPSEL_COUNT_ALIAS_B : SInst<"svpsel_lane_c8",  "}}Pm", "Pc", MergeNone, "", [IsStreamingCompatible], []>;
-def SVPSEL_COUNT_ALIAS_H : SInst<"svpsel_lane_c16", "}}Pm", "Ps", MergeNone, "", [IsStreamingCompatible], []>;
-def SVPSEL_COUNT_ALIAS_S : SInst<"svpsel_lane_c32", "}}Pm", "Pi", MergeNone, "", [IsStreamingCompatible], []>;
-def SVPSEL_COUNT_ALIAS_D : SInst<"svpsel_lane_c64", "}}Pm", "Pl", MergeNone, "", [IsStreamingCompatible], []>;
+def SVPSEL_B : SInst<"svpsel_lane_b8",  "PPPm", "Pc", MergeNone, "", [IsStreamingOrSVE2p1], []>;
+def SVPSEL_H : SInst<"svpsel_lane_b16", "PPPm", "Ps", MergeNone, "", [IsStreamingOrSVE2p1], []>;
+def SVPSEL_S : SInst<"svpsel_lane_b32", "PPPm", "Pi", MergeNone, "", [IsStreamingOrSVE2p1], []>;
+def SVPSEL_D : SInst<"svpsel_lane_b64", "PPPm", "Pl", MergeNone, "", [IsStreamingOrSVE2p1], []>;
 }
 
 // Standalone sve2.1 builtins
@@ -1976,39 +1977,42 @@ def SVFMINQV: SInst<"svminqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fminq
 }
 
 let TargetGuard = "sve2p1|sme2" in {
-//FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available
-def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>;
-def SVPEXT_X2     : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>;
+def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [IsStreamingOrSVE2p1], [ImmCheck<1, ImmCheck0_3>]>;
+def SVPEXT_X2     : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [IsStreamingOrSVE2p1], [ImmCheck<1, ImmCheck0_1>]>;
+
+def SVPSEL_COUNT_ALIAS_B : SInst<"svpsel_lane_c8",  "}}Pm", "Pc", MergeNone, "", [IsStreamingOrSVE2p1], []>;
+def SVPSEL_COUNT_ALIAS_H : SInst<"svpsel_lane_c16", "}}Pm", "Ps", MergeNone, "", [IsStreamingOrSVE2p1], []>;
+def SVPSEL_COUNT_ALIAS_S : SInst<"svpsel_lane_c32", "}}Pm", "Pi", MergeNone, "", [IsStreamingOrSVE2p1], []>;
+def SVPSEL_COUNT_ALIAS_D : SInst<"svpsel_lane_c64", "}}Pm", "Pl", MergeNone, "", [IsStreamingOrSVE2p1], []>;
 
-def SVWHILEGE_COUNT  : SInst<"svwhilege_{d}[_{1}]",  "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
-def SVWHILEGT_COUNT  : SInst<"svwhilegt_{d}[_{1}]",  "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
-def SVWHILELE_COUNT  : SInst<"svwhilele_{d}[_{1}]",  "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
-def SVWHILELT_COUNT  : SInst<"svwhilelt_{d}[_{1}]",  "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
-def SVWHILELO_COUNT  : SInst<"svwhilelt_{d}[_{1}]",  "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
-def SVWHILELS_COUNT  : SInst<"svwhilele_{d}[_{1}]",  "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
-def SVWHILEHI_COUNT  : SInst<"svwhilegt_{d}[_{1}]",  "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
-def SVWHILEHS_COUNT  : SInst<"svwhilege_{d}[_{1}]",  "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILEGE_COUNT  : SInst<"svwhilege_{d}[_{1}]",  "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILEGT_COUNT  : SInst<"svwhilegt_{d}[_{1}]",  "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILELE_COUNT  : SInst<"svwhilele_{d}[_{1}]",  "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILELT_COUNT  : SInst<"svwhilelt_{d}[_{1}]",  "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILELO_COUNT  : SInst<"svwhilelt_{d}[_{1}]",  "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILELS_COUNT  : SInst<"svwhilele_{d}[_{1}]",  "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILEHI_COUNT  : SInst<"svwhilegt_{d}[_{1}]",  "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
+def SVWHILEHS_COUNT  : SInst<"svwhilege_{d}[_{1}]",  "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<2, ImmCheck2_4_Mul2>]>;
 }
 
 multiclass MultiVecLoad {
-  // FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available (SME2 requires __arm_streaming)
-  def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "cUc",   [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "iUif",  [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "lUld",  [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "cUc",   [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-  def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-  def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "iUif",  [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-  def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "lUld",  [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-
-  def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "cUc",   [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "iUif",  [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "lUld",  [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "cUc",   [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-  def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-  def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "iUif",  [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-  def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "lUld",  [IsStructLoad, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "cUc",   [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "iUif",  [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "lUld",  [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "cUc",   [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "iUif",  [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "lUld",  [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+
+  def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "cUc",   [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "iUif",  [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "lUld",  [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "cUc",   [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "iUif",  [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "lUld",  [IsStructLoad, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
 }
 
 let TargetGuard = "sve2p1|sme2" in {
@@ -2017,24 +2021,23 @@ let TargetGuard = "sve2p1|sme2" in {
 }
 
 multiclass MultiVecStore {
-  // FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available (SME2 requires __arm_streaming)
-  def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "cUc",   [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "iUif",  [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "lUld",  [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "cUc",   [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-  def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-  def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "iUif",  [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-  def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "lUld",  [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-
-  def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "cUc",   [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "iUif",  [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "lUld",  [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
-  def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "cUc",   [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-  def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-  def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "iUif",  [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
-  def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "lUld",  [IsStructStore, IsStreamingCompatible], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "cUc",   [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "iUif",  [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "lUld",  [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "cUc",   [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "iUif",  [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "lUld",  [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+
+  def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "cUc",   [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "iUif",  [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "lUld",  [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">;
+  def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "cUc",   [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "iUif",  [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
+  def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "lUld",  [IsStructStore, IsStreamingOrSVE2p1], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">;
 }
 
 let TargetGuard = "sve2p1|sme2" in {
@@ -2042,30 +2045,29 @@ let TargetGuard = "sve2p1|sme2" in {
   defm STNT1 : MultiVecStore<"stnt1">;
 }
 
-let TargetGuard = "sve2p1" in {
-def SVDOT_X2_S : SInst<"svdot[_{d}_{2}_{3}]", "ddhh", "i",  MergeNone, "aarch64_sve_sdot_x2", [], []>;
-def SVDOT_X2_U : SInst<"svdot[_{d}_{2}_{3}]", "ddhh", "Ui", MergeNone, "aarch64_sve_udot_x2", [], []>;
-def SVDOT_X2_F : SInst<"svdot[_{d}_{2}_{3}]", "ddhh", "f",  MergeNone, "aarch64_sve_fdot_x2", [], []>;
-def SVDOT_LANE_X2_S : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "i",  MergeNone, "aarch64_sve_sdot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>;
-def SVDOT_LANE_X2_U : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "Ui", MergeNone, "aarch64_sve_udot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>;
-def SVDOT_LANE_X2_F : SInst<"svdot_lane[_{d}_{2}_{3}]", "ddhhi", "f",  MergeNone, "aarch64_sve_fdot_lane_x2", [], [ImmCheck<3, ImmCheck0_3>]>;
+let TargetGuard = "sve2p1|sme2" in {
+def SVDOT_X2_S : SInst<"svdot[_{d}_{2}]", "ddhh", "i",  MergeNone, "aarch64_sve_sdot_x2", [IsStreamingOrSVE2p1], []>;
+def SVDOT_X2_U : SInst<"svdot[_{d}_{2}]", "ddhh", "Ui", MergeNone, "aarch64_sve_udot_x2", [IsStreamingOrSVE2p1], []>;
+def SVDOT_X2_F : SInst<"svdot[_{d}_{2}]", "ddhh", "f",  MergeNone, "aarch64_sve_fdot_x2", [IsStreamingOrSVE2p1], []>;
+def SVDOT_LANE_X2_S : SInst<"svdot_lane[_{d}_{2}]", "ddhhi", "i",  MergeNone, "aarch64_sve_sdot_lane_x2", [IsStreamingOrSVE2p1], [ImmCheck<3, ImmCheck0_3>]>;
+def SVDOT_LANE_X2_U : SInst<"svdot_lane[_{d}_{2}]", "ddhhi", "Ui", MergeNone, "aarch64_sve_udot_lane_x2", [IsStreamingOrSVE2p1], [ImmCheck<3, ImmCheck0_3>]>;
+def SVDOT_LANE_X2_F : SInst<"svdot_lane[_{d}_{2}]", "ddhhi", "f",  MergeNone, "aarch64_sve_fdot_lane_x2", [IsStreamingOrSVE2p1], [ImmCheck<3, ImmCheck0_3>]>;
 }
 
-let TargetGuard = "sve2p1|sme" in {
-def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil",     MergeNone, "aarch64_sve_sclamp", [], []>;
-def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [], []>;
+let TargetGuard = "sve2p1|sme2" in {
+def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil",     MergeNone, "aarch64_sve_sclamp", [IsStreamingOrSVE2p1], []>;
+def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [IsStreamingOrSVE2p1], []>;
 
 defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUlbhfd", "aarch64_sve_revd">;
 }
 
 let TargetGuard = "sve2p1|sme2" in {
-  //FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available
-  def SVPTRUE_COUNT  : SInst<"svptrue_{d}", "}v", "QcQsQiQl", MergeNone, "aarch64_sve_ptrue_{d}", [IsOverloadNone, IsStreamingCompatible], []>;
+  def SVPTRUE_COUNT  : SInst<"svptrue_{d}", "}v", "QcQsQiQl", MergeNone, "aarch64_sve_ptrue_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], []>;
 
-  def SVPFALSE_COUNT_ALIAS : SInst<"svpfalse_c", "}v", "", MergeNone, "", [IsOverloadNone, IsStreamingCompatible]>;
+  def SVPFALSE_COUNT_ALIAS : SInst<"svpfalse_c", "}v", "", MergeNone, "", [IsOverloadNone, IsStreamingOrSVE2p1]>;
 
-  def SVFCLAMP   : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [IsStreamingCompatible], []>;
-  def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<1, ImmCheck2_4_Mul2>]>;
+  def SVFCLAMP   : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [IsStreamingOrSVE2p1], []>;
+  def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<1, ImmCheck2_4_Mul2>]>;
 }
 
 let TargetGuard = "(sve2|sme2),b16b16" in {
@@ -2212,7 +2214,7 @@ let TargetGuard = "sve2p1" in {
   def SVTBLQ : SInst<"svtblq[_{d}]", "ddu", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_tblq">;
   def SVTBXQ : SInst<"svtbxq[_{d}]", "dddu", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_tbxq">;
   // EXTQ
-  def EXTQ : SInst<"svextq_lane[_{d}]", "dddk", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_extq_lane", [], [ImmCheck<2, ImmCheck0_15>]>;
+  def EXTQ : SInst<"svextq[_{d}]", "dddk", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_extq", [], [ImmCheck<2, ImmCheck0_15>]>;
   // PMOV
   // Move to Pred
   multiclass PMOV_TO_PRED flags=[], ImmCheckType immCh > {
@@ -2242,15 +2244,15 @@ let TargetGuard = "sme2" in {
   def SVCVT_F16_X2  : SInst<"svcvt_f16[_f32_x2]", "e2", "f", MergeNone, "aarch64_sve_fcvt_x2", [IsStreaming],[]>;
   def SVCVT_BF16_X2 : SInst<"svcvt_bf16[_f32_x2]", "$2", "f", MergeNone, "aarch64_sve_bfcvt_x2", [IsOverloadNone, IsStreaming],[]>;
 
-  def SVCVT_F32_U32_X2 : SInst<"svcvt_{d}[_u32_x2]", "2.d2.u", "f", MergeNone, "aarch64_sve_fcvtu_x2", [IsStreaming], []>;
-  def SVCVT_U32_F32_X2 : SInst<"svcvt_u32[_{d}_x2]", "2.u2.d", "f", MergeNone, "aarch64_sve_ucvtf_x2", [IsStreaming], []>;
-  def SVCVT_F32_S32_X2 : SInst<"svcvt_{d}[_s32_x2]", "2.d2.x", "f", MergeNone, "aarch64_sve_fcvts_x2", [IsStreaming], []>;
-  def SVCVT_S32_F32_X2 : SInst<"svcvt_s32[_{d}_x2]", "2.x2.d", "f", MergeNone, "aarch64_sve_scvtf_x2", [IsStreaming], []>;
+  def SVCVT_F32_U32_X2 : SInst<"svcvt_{d}[_u32_x2]", "2.d2.u", "f",  MergeNone, "aarch64_sve_ucvtf_x2",  [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>;
+  def SVCVT_U32_F32_X2 : SInst<"svcvt_{d}[_f32_x2]", "2.d2.M", "Ui", MergeNone, "aarch64_sve_fcvtzu_x2", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>;
+  def SVCVT_F32_S32_X2 : SInst<"svcvt_{d}[_s32_x2]", "2.d2.x", "f",  MergeNone, "aarch64_sve_scvtf_x2",  [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>;
+  def SVCVT_S32_F32_X2 : SInst<"svcvt_{d}[_f32_x2]", "2.d2.M", "i",  MergeNone, "aarch64_sve_fcvtzs_x2", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>;
 
-  def SVCVT_F32_U32_X4 : SInst<"svcvt_{d}[_u32_x4]", "4.d4.u", "f", MergeNone, "aarch64_sve_fcvtu_x4", [IsStreaming], []>;
-  def SVCVT_U32_F32_X4 : SInst<"svcvt_u32[_{d}_x4]", "4.u4.d", "f", MergeNone, "aarch64_sve_ucvtf_x4", [IsStreaming], []>;
-  def SVCVT_F32_S32_X4 : SInst<"svcvt_{d}[_s32_x4]", "4.d4.x", "f", MergeNone, "aarch64_sve_fcvts_x4", [IsStreaming], []>;
-  def SVCVT_S32_F32_X4 : SInst<"svcvt_s32[_{d}_x4]", "4.x4.d", "f", MergeNone, "aarch64_sve_scvtf_x4", [IsStreaming], []>;
+  def SVCVT_F32_U32_X4 : SInst<"svcvt_{d}[_u32_x4]", "4.d4.u", "f",  MergeNone, "aarch64_sve_ucvtf_x4",  [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>;
+  def SVCVT_U32_F32_X4 : SInst<"svcvt_{d}[_f32_x4]", "4.d4.M", "Ui", MergeNone, "aarch64_sve_fcvtzu_x4", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>;
+  def SVCVT_F32_S32_X4 : SInst<"svcvt_{d}[_s32_x4]", "4.d4.x", "f",  MergeNone, "aarch64_sve_scvtf_x4",  [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>;
+  def SVCVT_S32_F32_X4 : SInst<"svcvt_{d}[_f32_x4]", "4.d4.M", "i",  MergeNone, "aarch64_sve_fcvtzs_x4", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>;
 }
 
 //
@@ -2326,10 +2328,9 @@ let TargetGuard = "sme2" in {
 
 let TargetGuard = "sve2p1|sme2" in {
 // == BFloat16 multiply-subtract ==
-// FIXME: Make all of these IsStreamingOrSVE2p1 once that is added
-  def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone, IsStreamingCompatible], []>;
-  def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone, IsStreamingCompatible], []>;
+  def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone, IsStreamingOrSVE2p1], []>;
+  def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone, IsStreamingOrSVE2p1], []>;
 
-  def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>;
-  def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone, IsStreamingCompatible], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<3, ImmCheck0_7>]>;
+  def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone, IsStreamingOrSVE2p1], [ImmCheck<3, ImmCheck0_7>]>;
 }
diff --git a/clang/include/clang/Basic/arm_sve_sme_incl.td b/clang/include/clang/Basic/arm_sve_sme_incl.td
index 0dba8493bad2..9a6ea9898ef7 100644
--- a/clang/include/clang/Basic/arm_sve_sme_incl.td
+++ b/clang/include/clang/Basic/arm_sve_sme_incl.td
@@ -194,39 +194,44 @@ def FirstSplatOperand         : FlagType<0x00000400>;
 // These flags are used to specify which scalar operand
 // needs to be duplicated/splatted into a vector.
 //      :                                     :
-def SplatOperandMask          : FlagType<0x00001C00>;
-def IsLoad                    : FlagType<0x00002000>;
-def IsStore                   : FlagType<0x00004000>;
-def IsGatherLoad              : FlagType<0x00008000>;
-def IsScatterStore            : FlagType<0x00010000>;
-def IsStructLoad              : FlagType<0x00020000>;
-def IsStructStore             : FlagType<0x00040000>;
-def IsZExtReturn              : FlagType<0x00080000>; // Return value is sign-extend by default
-def IsOverloadNone            : FlagType<0x00100000>; // Intrinsic does not take any overloaded types.
-def IsOverloadWhile           : FlagType<0x00200000>; // Use {default type, typeof(operand1)} as overloaded types.
-def IsOverloadWhileRW         : FlagType<0x00400000>; // Use {pred(default type), typeof(operand0)} as overloaded types.
-def IsOverloadCvt             : FlagType<0x00800000>; // Use {typeof(operand0), typeof(last operand)} as overloaded types.
-def OverloadKindMask          : FlagType<0x00E00000>; // When the masked values are all '0', the default type is used as overload type.
-def IsByteIndexed             : FlagType<0x01000000>;
-def IsAppendSVALL             : FlagType<0x02000000>; // Appends SV_ALL as the last operand.
-def IsInsertOp1SVALL          : FlagType<0x04000000>; // Inserts SV_ALL as the second operand.
-def IsPrefetch                : FlagType<0x08000000>; // Contiguous prefetches.
-def IsGatherPrefetch          : FlagType<0x10000000>;
-def ReverseCompare            : FlagType<0x20000000>; // Compare operands must be swapped.
-def ReverseUSDOT              : FlagType<0x40000000>; // Unsigned/signed operands must be swapped.
-def IsUndef                   : FlagType<0x80000000>; // Codegen `undef` of given type.
-def IsTupleCreate             : FlagType<0x100000000>;
-def IsTupleGet                : FlagType<0x200000000>;
-def IsTupleSet                : FlagType<0x400000000>;
-def ReverseMergeAnyBinOp      : FlagType<0x800000000>; // e.g. Implement SUBR_X using SUB_X.
-def ReverseMergeAnyAccOp      : FlagType<0x1000000000>; // e.g. Implement MSB_X using MLS_X.
-def IsStreaming               : FlagType<0x2000000000>;
-def IsStreamingCompatible     : FlagType<0x4000000000>;
-def IsSharedZA                : FlagType<0x8000000000>;
-def IsPreservesZA             : FlagType<0x10000000000>;
-def IsReadZA                  : FlagType<0x20000000000>;
-def IsWriteZA                 : FlagType<0x40000000000>;
-def IsReductionQV             : FlagType<0x80000000000>;
+def SplatOperandMask                : FlagType<0x00001C00>;
+def IsLoad                          : FlagType<0x00002000>;
+def IsStore                         : FlagType<0x00004000>;
+def IsGatherLoad                    : FlagType<0x00008000>;
+def IsScatterStore                  : FlagType<0x00010000>;
+def IsStructLoad                    : FlagType<0x00020000>;
+def IsStructStore                   : FlagType<0x00040000>;
+def IsZExtReturn                    : FlagType<0x00080000>; // Return value is sign-extend by default
+def IsOverloadNone                  : FlagType<0x00100000>; // Intrinsic does not take any overloaded types.
+def IsOverloadWhileOrMultiVecCvt    : FlagType<0x00200000>; // Use {default type, typeof(operand1)} as overloaded types.
+def IsOverloadWhileRW               : FlagType<0x00400000>; // Use {pred(default type), typeof(operand0)} as overloaded types.
+def IsOverloadCvt                   : FlagType<0x00800000>; // Use {typeof(operand0), typeof(last operand)} as overloaded types.
+def OverloadKindMask                : FlagType<0x00E00000>; // When the masked values are all '0', the default type is used as overload type.
+def IsByteIndexed                   : FlagType<0x01000000>;
+def IsAppendSVALL                   : FlagType<0x02000000>; // Appends SV_ALL as the last operand.
+def IsInsertOp1SVALL                : FlagType<0x04000000>; // Inserts SV_ALL as the second operand.
+def IsPrefetch                      : FlagType<0x08000000>; // Contiguous prefetches.
+def IsGatherPrefetch                : FlagType<0x10000000>;
+def ReverseCompare                  : FlagType<0x20000000>; // Compare operands must be swapped.
+def ReverseUSDOT                    : FlagType<0x40000000>; // Unsigned/signed operands must be swapped.
+def IsUndef                         : FlagType<0x80000000>; // Codegen `undef` of given type.
+def IsTupleCreate                   : FlagType<0x100000000>;
+def IsTupleGet                      : FlagType<0x200000000>;
+def IsTupleSet                      : FlagType<0x400000000>;
+def ReverseMergeAnyBinOp            : FlagType<0x800000000>; // e.g. Implement SUBR_X using SUB_X.
+def ReverseMergeAnyAccOp            : FlagType<0x1000000000>; // e.g. Implement MSB_X using MLS_X.
+def IsStreaming                     : FlagType<0x2000000000>;
+def IsStreamingCompatible           : FlagType<0x4000000000>;
+def IsReadZA                        : FlagType<0x8000000000>;
+def IsWriteZA                       : FlagType<0x10000000000>;
+def IsReductionQV                   : FlagType<0x20000000000>;
+def IsStreamingOrSVE2p1             : FlagType<0x40000000000>; // Use for intrinsics that are common between sme/sme2 and sve2p1.
+def IsInZA                          : FlagType<0x80000000000>;
+def IsOutZA                         : FlagType<0x100000000000>;
+def IsInOutZA                       : FlagType<0x200000000000>;
+def IsInZT0                         : FlagType<0x400000000000>;
+def IsOutZT0                        : FlagType<0x800000000000>;
+def IsInOutZT0                      : FlagType<0x1000000000000>;
 
 // These must be kept in sync with the flags in include/clang/Basic/TargetBuiltins.h
 class ImmCheckType {
diff --git a/clang/include/clang/Basic/riscv_sifive_vector.td b/clang/include/clang/Basic/riscv_sifive_vector.td
index e19a34f7632f..ef5114d6105e 100644
--- a/clang/include/clang/Basic/riscv_sifive_vector.td
+++ b/clang/include/clang/Basic/riscv_sifive_vector.td
@@ -109,7 +109,7 @@ multiclass RVVVFWMACCBuiltinSet> suffixes_prototypes> {
       Name = NAME,
       HasMasked = false,
       Log2LMUL = [-2, -1, 0, 1, 2] in
-    defm NAME : RVVOutOp1Op2BuiltinSet;
+    defm NAME : RVVOutOp1Op2BuiltinSet;
 }
 
 multiclass RVVVQMACCDODBuiltinSet> suffixes_prototypes> {
@@ -121,11 +121,11 @@ multiclass RVVVQMACCDODBuiltinSet> suffixes_prototypes> {
 }
 
 multiclass RVVVQMACCQOQBuiltinSet> suffixes_prototypes> {
-   let OverloadedName = NAME,
-       Name = NAME,
-       HasMasked = false,
-       Log2LMUL = [-1, 0, 1, 2] in
-     defm NAME : RVVOutOp1Op2BuiltinSet;
+  let OverloadedName = NAME,
+      Name = NAME,
+      HasMasked = false,
+      Log2LMUL = [-1, 0, 1, 2] in
+    defm NAME : RVVOutOp1Op2BuiltinSet;
 }
 
 multiclass RVVVFNRCLIPBuiltinSet {
diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td
index e7d78b03511f..8bde08105250 100644
--- a/clang/include/clang/Basic/riscv_vector.td
+++ b/clang/include/clang/Basic/riscv_vector.td
@@ -117,7 +117,7 @@ multiclass RVVIndexedLoad {
         defvar eew = eew_list[0];
         defvar eew_type = eew_list[1];
         let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask",
-            RequiredFeatures = !if(!eq(type, "x"), ["ZvfhminOrZvfh"],
+            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
                                                    []) in {
           def: RVVOutOp1Builtin<"v", "vPCe" # eew_type # "Uv", type>;
             if !not(IsFloat.val) then {
@@ -128,7 +128,7 @@ multiclass RVVIndexedLoad {
       defvar eew64 = "64";
       defvar eew64_type = "(Log2EEW:6)";
       let Name = op # eew64 # "_v", IRName = op, MaskedIRName = op # "_mask",
-          RequiredFeatures = !if(!eq(type, "x"), ["ZvfhminOrZvfh", "RV64"],
+          RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin", "RV64"],
                                                  ["RV64"]) in {
           def: RVVOutOp1Builtin<"v", "vPCe" # eew64_type # "Uv", type>;
             if !not(IsFloat.val) then {
@@ -222,7 +222,7 @@ multiclass RVVIndexedStore {
           defvar eew = eew_list[0];
           defvar eew_type = eew_list[1];
           let Name = op # eew  # "_v", IRName = op, MaskedIRName = op # "_mask",
-          RequiredFeatures = !if(!eq(type, "x"), ["ZvfhminOrZvfh"],
+          RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
                                                  []) in  {
             def : RVVBuiltin<"v", "0Pe" # eew_type # "Uvv", type>;
             if !not(IsFloat.val) then {
@@ -233,7 +233,7 @@ multiclass RVVIndexedStore {
         defvar eew64 = "64";
         defvar eew64_type = "(Log2EEW:6)";
         let Name = op # eew64  # "_v", IRName = op, MaskedIRName = op # "_mask",
-            RequiredFeatures = !if(!eq(type, "x"), ["ZvfhminOrZvfh", "RV64"],
+            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin", "RV64"],
                                                    ["RV64"]) in  {
           def : RVVBuiltin<"v", "0Pe" # eew64_type # "Uvv", type>;
           if !not(IsFloat.val) then {
@@ -681,7 +681,7 @@ let HasBuiltinAlias = false,
 def vlm: RVVVLEMaskBuiltin;
 defm vle8: RVVVLEBuiltin<["c"]>;
 defm vle16: RVVVLEBuiltin<["s"]>;
-let Name = "vle16_v", RequiredFeatures = ["ZvfhminOrZvfh"] in
+let Name = "vle16_v", RequiredFeatures = ["Zvfhmin"] in
   defm vle16_h: RVVVLEBuiltin<["x"]>;
 defm vle32: RVVVLEBuiltin<["i","f"]>;
 defm vle64: RVVVLEBuiltin<["l","d"]>;
@@ -689,7 +689,7 @@ defm vle64: RVVVLEBuiltin<["l","d"]>;
 def vsm : RVVVSEMaskBuiltin;
 defm vse8 : RVVVSEBuiltin<["c"]>;
 defm vse16: RVVVSEBuiltin<["s"]>;
-let Name = "vse16_v", RequiredFeatures = ["ZvfhminOrZvfh"] in
+let Name = "vse16_v", RequiredFeatures = ["Zvfhmin"] in
   defm vse16_h: RVVVSEBuiltin<["x"]>;
 defm vse32: RVVVSEBuiltin<["i","f"]>;
 defm vse64: RVVVSEBuiltin<["l","d"]>;
@@ -697,14 +697,14 @@ defm vse64: RVVVSEBuiltin<["l","d"]>;
 // 7.5. Vector Strided Instructions
 defm vlse8: RVVVLSEBuiltin<["c"]>;
 defm vlse16: RVVVLSEBuiltin<["s"]>;
-let Name = "vlse16_v", RequiredFeatures = ["ZvfhminOrZvfh"] in
+let Name = "vlse16_v", RequiredFeatures = ["Zvfhmin"] in
   defm vlse16_h: RVVVLSEBuiltin<["x"]>;
 defm vlse32: RVVVLSEBuiltin<["i","f"]>;
 defm vlse64: RVVVLSEBuiltin<["l","d"]>;
 
 defm vsse8 : RVVVSSEBuiltin<["c"]>;
 defm vsse16: RVVVSSEBuiltin<["s"]>;
-let Name = "vsse16_v", RequiredFeatures = ["ZvfhminOrZvfh"] in
+let Name = "vsse16_v", RequiredFeatures = ["Zvfhmin"] in
   defm vsse16_h: RVVVSSEBuiltin<["x"]>;
 defm vsse32: RVVVSSEBuiltin<["i","f"]>;
 defm vsse64: RVVVSSEBuiltin<["l","d"]>;
@@ -719,7 +719,7 @@ defm : RVVIndexedStore<"vsoxei">;
 // 7.7. Unit-stride Fault-Only-First Loads
 defm vle8ff: RVVVLEFFBuiltin<["c"]>;
 defm vle16ff: RVVVLEFFBuiltin<["s"]>;
-let Name = "vle16ff_v", RequiredFeatures = ["ZvfhminOrZvfh"] in
+let Name = "vle16ff_v", RequiredFeatures = ["Zvfhmin"] in
   defm vle16ff: RVVVLEFFBuiltin<["x"]>;
 defm vle32ff: RVVVLEFFBuiltin<["i", "f"]>;
 defm vle64ff: RVVVLEFFBuiltin<["l", "d"]>;
@@ -738,7 +738,7 @@ multiclass RVVUnitStridedSegLoadTuple {
             IRName = op # nf,
             MaskedIRName = op # nf # "_mask",
             NF = nf,
-            RequiredFeatures = !if(!eq(type, "x"), ["ZvfhminOrZvfh"],
+            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
                                                    []),
             ManualCodegen = [{
     {
@@ -800,7 +800,7 @@ multiclass RVVUnitStridedSegStoreTuple {
           MaskedIRName = op # nf # "_mask",
           NF = nf,
           HasMaskedOffOperand = false,
-          RequiredFeatures = !if(!eq(type, "x"), ["ZvfhminOrZvfh"],
+          RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
                                                  []),
           ManualCodegen = [{
     {
@@ -852,7 +852,7 @@ multiclass RVVUnitStridedSegLoadFFTuple {
             IRName = op # nf # "ff",
             MaskedIRName = op # nf # "ff_mask",
             NF = nf,
-            RequiredFeatures = !if(!eq(type, "x"), ["ZvfhminOrZvfh"],
+            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
                                                    []),
             ManualCodegen = [{
     {
@@ -927,7 +927,7 @@ multiclass RVVStridedSegLoadTuple {
             IRName = op # nf,
             MaskedIRName = op # nf # "_mask",
             NF = nf,
-            RequiredFeatures = !if(!eq(type, "x"), ["ZvfhminOrZvfh"],
+            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
                                                    []),
             ManualCodegen = [{
     {
@@ -991,7 +991,7 @@ multiclass RVVStridedSegStoreTuple {
             NF = nf,
             HasMaskedOffOperand = false,
             MaskedPolicyScheme = NonePolicy,
-            RequiredFeatures = !if(!eq(type, "x"), ["ZvfhminOrZvfh"],
+            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
                                                    []),
             ManualCodegen = [{
     {
@@ -1040,7 +1040,7 @@ multiclass RVVIndexedSegLoadTuple {
             IRName = op # nf,
             MaskedIRName = op # nf # "_mask",
             NF = nf,
-            RequiredFeatures = !if(!eq(type, "x"), ["ZvfhminOrZvfh"],
+            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
                                                    []),
             ManualCodegen = [{
     {
@@ -1103,7 +1103,7 @@ multiclass RVVIndexedSegStoreTuple {
             NF = nf,
             HasMaskedOffOperand = false,
             MaskedPolicyScheme = NonePolicy,
-            RequiredFeatures = !if(!eq(type, "x"), ["ZvfhminOrZvfh"],
+            RequiredFeatures = !if(!eq(type, "x"), ["Zvfhmin"],
                                                    []),
             ManualCodegen = [{
     {
@@ -1345,7 +1345,7 @@ let HasMasked = false,
                                    [["v", "Uv", "UvUv"]]>;
     defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csilfd",
                                    [["v", "v", "vv"]]>;
-    let RequiredFeatures = ["ZvfhminOrZvfh"] in
+    let RequiredFeatures = ["Zvfhmin"] in
       defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "x",
                                     [["v", "v", "vv"]]>;
   let SupportOverloading = false in
@@ -1841,7 +1841,7 @@ let HasMasked = false,
     }] in {
   defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "fd",
                                     [["vvm", "v", "vvvm"]]>;
-  let RequiredFeatures = ["ZvfhminOrZvfh"] in
+  let RequiredFeatures = ["Zvfhmin"] in
     defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "x",
                                       [["vvm", "v", "vvvm"]]>;
   defm vfmerge : RVVOutOp1BuiltinSet<"vfmerge", "xfd",
@@ -1869,7 +1869,7 @@ let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
   def vfwcvt_f_xu_v : RVVConvBuiltin<"Fw", "FwUv", "csi", "vfwcvt_f">;
   def vfwcvt_f_x_v : RVVConvBuiltin<"Fw", "Fwv", "csi", "vfwcvt_f">;
   def vfwcvt_f_f_v : RVVConvBuiltin<"w", "wv", "f", "vfwcvt_f">;
-  let RequiredFeatures = ["ZvfhminOrZvfh"] in
+  let RequiredFeatures = ["Zvfhmin"] in
     def vfwcvt_f_f_v_fp16 : RVVConvBuiltin<"w", "wv", "x", "vfwcvt_f"> {
       let Name = "vfwcvt_f_f_v";
       let IRName = "vfwcvt_f_f_v";
@@ -1966,7 +1966,7 @@ let ManualCodegen = [{
       }
       let OverloadedName = "vfncvt_f" in {
         defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "f", [["v", "vwu"]]>;
-        let RequiredFeatures = ["ZvfhminOrZvfh"] in
+        let RequiredFeatures = ["Zvfhmin"] in
         defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "x", [["v", "vwu"]]>;
       }
     }
@@ -2011,7 +2011,7 @@ let ManualCodegen = [{
     }
     let OverloadedName = "vfncvt_f" in {
       defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "f", [["v", "vw"]]>;
-      let RequiredFeatures = ["ZvfhminOrZvfh"] in
+      let RequiredFeatures = ["Zvfhmin"] in
       defm : RVVConvBuiltinSet<"vfncvt_f_f_w", "x", [["v", "vw"]]>;
     }
   }
@@ -2271,7 +2271,7 @@ let HasMasked = false, HasVL = false, IRName = "" in {
     def vreinterpret_u_f : RVVBuiltin<"FvUv", "UvFv", "il", "Uv">;
     def vreinterpret_f_i : RVVBuiltin<"vFv", "Fvv", "il", "Fv">;
     def vreinterpret_f_u : RVVBuiltin<"UvFv", "FvUv", "il", "Fv">;
-    let RequiredFeatures = ["ZvfhminOrZvfh"] in {
+    let RequiredFeatures = ["Zvfhmin"] in {
       def vreinterpret_i_h : RVVBuiltin<"Fvv", "vFv", "s", "v">;
       def vreinterpret_u_h : RVVBuiltin<"FvUv", "UvFv", "s", "Uv">;
       def vreinterpret_h_i : RVVBuiltin<"vFv", "Fvv", "s", "Fv">;
diff --git a/clang/include/clang/Basic/riscv_vector_common.td b/clang/include/clang/Basic/riscv_vector_common.td
index 4036ce8e6903..040db6f0cdbf 100644
--- a/clang/include/clang/Basic/riscv_vector_common.td
+++ b/clang/include/clang/Basic/riscv_vector_common.td
@@ -41,7 +41,7 @@
 //   x: float16_t (half)
 //   f: float32_t (float)
 //   d: float64_t (double)
-//   b: bfloat16_t (bfloat16)
+//   y: bfloat16_t (bfloat16)
 //
 // This way, given an LMUL, a record with a TypeRange "sil" will cause the
 // definition of 3 builtins. Each type "t" in the TypeRange (in this example
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 7acc25a2fc15..76747009065d 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -777,6 +777,8 @@ def gcc_install_dir_EQ : Joined<["--"], "gcc-install-dir=">,
 def gcc_toolchain : Joined<["--"], "gcc-toolchain=">, Flags<[NoXarchOption]>,
   HelpText<"Specify a directory where Clang can find 'include' and 'lib{,32,64}/gcc{,-cross}/$triple/$version'. "
   "Clang will use the GCC installation with the largest version">;
+def gcc_triple_EQ : Joined<["--"], "gcc-triple=">,
+  HelpText<"Search for the GCC installation with the specified triple.">;
 def CC : Flag<["-"], "CC">, Visibility<[ClangOption, CC1Option]>,
   Group,
     HelpText<"Include comments from within macros in preprocessed output">,
@@ -1699,6 +1701,12 @@ defm coverage_mapping : BoolFOption<"coverage-mapping",
           "Generate coverage mapping to enable code coverage analysis">,
   NegFlag, BothFlags<
           [], [ClangOption, CLOption]>>;
+defm mcdc_coverage : BoolFOption<"coverage-mcdc",
+  CodeGenOpts<"MCDCCoverage">, DefaultFalse,
+  PosFlag,
+  NegFlag,
+          BothFlags<[], [ClangOption, CLOption]>>;
 def fprofile_generate : Flag<["-"], "fprofile-generate">,
     Group, Visibility<[ClangOption, CLOption]>,
     HelpText<"Generate instrumented code to collect execution counts into default.profraw (overridden by LLVM_PROFILE_FILE env var)">;
@@ -3455,6 +3463,10 @@ def fopenmp_offload_mandatory : Flag<["-"], "fopenmp-offload-mandatory">, Group<
   Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>,
   HelpText<"Do not create a host fallback if offloading to the device fails.">,
   MarshallingInfoFlag>;
+def fopenmp_force_usm : Flag<["-"], "fopenmp-force-usm">, Group,
+  Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>,
+  HelpText<"Force behvaior as if the user specified pragma omp requires unified_shared_memory.">,
+  MarshallingInfoFlag>;
 def fopenmp_target_jit : Flag<["-"], "fopenmp-target-jit">, Group,
   Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CLOption]>,
   HelpText<"Emit code that can be JIT compiled for OpenMP offloading. Implies -foffload-lto=full">;
@@ -3660,6 +3672,10 @@ def ftrivial_auto_var_init_stop_after : Joined<["-"], "ftrivial-auto-var-init-st
   Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
   HelpText<"Stop initializing trivial automatic stack variables after the specified number of instances">,
   MarshallingInfoInt>;
+def ftrivial_auto_var_init_max_size : Joined<["-"], "ftrivial-auto-var-init-max-size=">, Group,
+  Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
+  HelpText<"Stop initializing trivial automatic stack variables if var size exceeds the specified number of instances (in bytes)">,
+  MarshallingInfoInt>;
 def fstandalone_debug : Flag<["-"], "fstandalone-debug">, Group,
   Visibility<[ClangOption, CLOption, DXCOption]>,
   HelpText<"Emit full debug info for all types used by the program">;
@@ -3854,27 +3870,32 @@ def dA : Flag<["-"], "dA">, Alias;
 defm visibility_from_dllstorageclass : BoolFOption<"visibility-from-dllstorageclass",
   LangOpts<"VisibilityFromDLLStorageClass">, DefaultFalse,
   PosFlag,
+          "Override the visibility of globals based on their final DLL storage class.">,
   NegFlag>;
+class MarshallingInfoVisibilityFromDLLStorage
+  : MarshallingInfoEnum,
+    Values<"keep,hidden,protected,default">,
+    NormalizedValuesScope<"LangOptions::VisibilityFromDLLStorageClassKinds">,
+    NormalizedValues<["Keep", "Hidden", "Protected", "Default"]> {}
 def fvisibility_dllexport_EQ : Joined<["-"], "fvisibility-dllexport=">, Group,
   Visibility<[ClangOption, CC1Option]>,
-  HelpText<"The visibility for dllexport definitions [-fvisibility-from-dllstorageclass]">,
-  MarshallingInfoVisibility, "DefaultVisibility">,
+  HelpText<"The visibility for dllexport definitions. If Keep is specified the visibility is not adjusted [-fvisibility-from-dllstorageclass]">,
+  MarshallingInfoVisibilityFromDLLStorage, "Default">,
   ShouldParseIf;
 def fvisibility_nodllstorageclass_EQ : Joined<["-"], "fvisibility-nodllstorageclass=">, Group,
   Visibility<[ClangOption, CC1Option]>,
-  HelpText<"The visibility for definitions without an explicit DLL export class [-fvisibility-from-dllstorageclass]">,
-  MarshallingInfoVisibility, "HiddenVisibility">,
+  HelpText<"The visibility for definitions without an explicit DLL storage class. If Keep is specified the visibility is not adjusted [-fvisibility-from-dllstorageclass]">,
+  MarshallingInfoVisibilityFromDLLStorage, "Hidden">,
   ShouldParseIf;
 def fvisibility_externs_dllimport_EQ : Joined<["-"], "fvisibility-externs-dllimport=">, Group,
   Visibility<[ClangOption, CC1Option]>,
-  HelpText<"The visibility for dllimport external declarations [-fvisibility-from-dllstorageclass]">,
-  MarshallingInfoVisibility, "DefaultVisibility">,
+  HelpText<"The visibility for dllimport external declarations. If Keep is specified the visibility is not adjusted [-fvisibility-from-dllstorageclass]">,
+  MarshallingInfoVisibilityFromDLLStorage, "Default">,
   ShouldParseIf;
 def fvisibility_externs_nodllstorageclass_EQ : Joined<["-"], "fvisibility-externs-nodllstorageclass=">, Group,
   Visibility<[ClangOption, CC1Option]>,
-  HelpText<"The visibility for external declarations without an explicit DLL dllstorageclass [-fvisibility-from-dllstorageclass]">,
-  MarshallingInfoVisibility, "HiddenVisibility">,
+  HelpText<"The visibility for external declarations without an explicit DLL storage class. If Keep is specified the visibility is not adjusted [-fvisibility-from-dllstorageclass]">,
+  MarshallingInfoVisibilityFromDLLStorage, "Hidden">,
   ShouldParseIf;
 def fvisibility_EQ : Joined<["-"], "fvisibility=">, Group,
   Visibility<[ClangOption, CC1Option]>,
@@ -3897,9 +3918,16 @@ def fvisibility_ms_compat : Flag<["-"], "fvisibility-ms-compat">, Group
   HelpText<"Give global types 'default' visibility and global functions and "
            "variables 'hidden' visibility by default">;
 def fvisibility_global_new_delete_hidden : Flag<["-"], "fvisibility-global-new-delete-hidden">, Group,
-  HelpText<"Give global C++ operator new and delete declarations hidden visibility">,
+  HelpText<"Give global C++ operator new and delete declarations hidden visibility">;
+class MarshallingInfoVisibilityGlobalNewDelete
+  : MarshallingInfoEnum,
+    Values<"force-default,force-protected,force-hidden,source">,
+    NormalizedValuesScope<"LangOptions::VisibilityForcedKinds">,
+    NormalizedValues<["ForceDefault", "ForceProtected", "ForceHidden", "Source"]> {}
+def fvisibility_global_new_delete_EQ : Joined<["-"], "fvisibility-global-new-delete=">, Group,
   Visibility<[ClangOption, CC1Option]>,
-  MarshallingInfoFlag>;
+  HelpText<"The visibility for global C++ operator new and delete declarations. If 'source' is specified the visibility is not adjusted">,
+  MarshallingInfoVisibilityGlobalNewDelete, "ForceDefault">;
 def mdefault_visibility_export_mapping_EQ : Joined<["-"], "mdefault-visibility-export-mapping=">,
   Values<"none,explicit,all">,
   NormalizedValuesScope<"LangOptions::DefaultVisiblityExportMapping">,
@@ -4263,7 +4291,7 @@ def iquote : JoinedOrSeparate<["-"], "iquote">, Group,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Add directory to QUOTE include search path">, MetaVarName<"">;
 def isysroot : JoinedOrSeparate<["-"], "isysroot">, Group,
-  Visibility<[ClangOption, CC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption]>,
   HelpText<"Set the system root directory (usually /)">, MetaVarName<"">,
   MarshallingInfoString, [{"/"}]>;
 def isystem : JoinedOrSeparate<["-"], "isystem">, Group,
@@ -4388,8 +4416,8 @@ def mcmodel_EQ : Joined<["-"], "mcmodel=">, Group,
   Visibility<[ClangOption, CC1Option]>,
   MarshallingInfoString, [{"default"}]>;
 def mlarge_data_threshold_EQ : Joined<["-"], "mlarge-data-threshold=">, Group,
-  Visibility<[ClangOption, CC1Option]>,
-  MarshallingInfoInt, "65535">;
+  Flags<[TargetSpecific]>, Visibility<[ClangOption, CC1Option]>,
+  MarshallingInfoInt, "0">;
 def mtls_size_EQ : Joined<["-"], "mtls-size=">, Group,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Specify bit size of immediate TLS offsets (AArch64 ELF only): "
@@ -4583,11 +4611,13 @@ let Flags = [TargetSpecific] in {
 def menable_experimental_extensions : Flag<["-"], "menable-experimental-extensions">, Group,
   HelpText<"Enable use of experimental RISC-V extensions.">;
 def mrvv_vector_bits_EQ : Joined<["-"], "mrvv-vector-bits=">, Group,
-  HelpText<"Specify the size in bits of an RVV vector register. Defaults to "
-           "the vector length agnostic value of \"scalable\". Accepts power of "
-           "2 values between 64 and 65536. Also accepts \"zvl\" "
-           "to use the value implied by -march/-mcpu. Value will be reflected "
-           "in __riscv_v_fixed_vlen preprocessor define (RISC-V only)">;
+  Visibility<[ClangOption, FlangOption]>,
+  HelpText<"Specify the size in bits of an RVV vector register">,
+  DocBrief<"Defaults to the vector length agnostic value of \"scalable\". "
+           "Accepts power of 2 values between 64 and 65536. Also accepts "
+           "\"zvl\" to use the value implied by -march/-mcpu. On Clang, value "
+           "will be reflected in __riscv_v_fixed_vlen preprocessor define "
+           "(RISC-V only)">;
 
 def munaligned_access : Flag<["-"], "munaligned-access">, Group,
   HelpText<"Allow memory accesses to be unaligned (AArch32/AArch64/LoongArch/RISC-V only)">;
@@ -4757,12 +4787,12 @@ defm amdgpu_ieee : BoolOption<"m", "amdgpu-ieee",
   NegFlag>, Group;
 
 def mcode_object_version_EQ : Joined<["-"], "mcode-object-version=">, Group,
-  HelpText<"Specify code object ABI version. Defaults to 4. (AMDGPU only)">,
+  HelpText<"Specify code object ABI version. Defaults to 5. (AMDGPU only)">,
   Visibility<[ClangOption, FlangOption, CC1Option, FC1Option]>,
   Values<"none,4,5">,
   NormalizedValuesScope<"llvm::CodeObjectVersionKind">,
   NormalizedValues<["COV_None", "COV_4", "COV_5"]>,
-  MarshallingInfoEnum, "COV_4">;
+  MarshallingInfoEnum, "COV_5">;
 
 defm cumode : SimpleMFlag<"cumode",
   "Specify CU wavefront", "Specify WGP wavefront",
@@ -5168,7 +5198,7 @@ def module_file_info : Flag<["-"], "module-file-info">, Flags<[]>,
   HelpText<"Provide information about a particular module file">;
 def mthumb : Flag<["-"], "mthumb">, Group;
 def mtune_EQ : Joined<["-"], "mtune=">, Group,
-  HelpText<"Only supported on AArch64, PowerPC, RISC-V, SystemZ, and X86">;
+  HelpText<"Only supported on AArch64, PowerPC, RISC-V, SPARC, SystemZ, and X86">;
 def multi__module : Flag<["-"], "multi_module">;
 def multiply__defined__unused : Separate<["-"], "multiply_defined_unused">;
 def multiply__defined : Separate<["-"], "multiply_defined">;
@@ -5195,12 +5225,12 @@ def nohipwrapperinc : Flag<["-"], "nohipwrapperinc">, Group,
   HelpText<"Do not include the default HIP wrapper headers and include paths">;
 def : Flag<["-"], "nocudainc">, Alias;
 def nogpulib : Flag<["-"], "nogpulib">, MarshallingInfoFlag>,
-  Visibility<[ClangOption, CC1Option]>,
+  Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Do not link device library for CUDA/HIP device compilation">;
 def : Flag<["-"], "nocudalib">, Alias;
-def gpulibc : Flag<["-"], "gpulibc">, Visibility<[ClangOption, CC1Option]>,
+def gpulibc : Flag<["-"], "gpulibc">, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Link the LLVM C Library for GPUs">;
-def nogpulibc : Flag<["-"], "nogpulibc">, Visibility<[ClangOption, CC1Option]>;
+def nogpulibc : Flag<["-"], "nogpulibc">, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>;
 def nodefaultlibs : Flag<["-"], "nodefaultlibs">;
 def nodriverkitlib : Flag<["-"], "nodriverkitlib">;
 def nofixprebinding : Flag<["-"], "nofixprebinding">;
@@ -5303,7 +5333,8 @@ def pthreads : Flag<["-"], "pthreads">;
 defm pthread : BoolOption<"", "pthread",
   LangOpts<"POSIXThreads">, DefaultFalse,
   PosFlag,
-  NegFlag, BothFlags<[], [ClangOption, CC1Option]>>;
+  NegFlag,
+  BothFlags<[], [ClangOption, CC1Option, FlangOption, FC1Option]>>;
 def pie : Flag<["-"], "pie">, Group;
 def static_pie : Flag<["-"], "static-pie">, Group;
 def read__only__relocs : Separate<["-"], "read_only_relocs">;
@@ -6039,11 +6070,11 @@ def mapx_features_EQ : CommaJoined<["-"], "mapx-features=">, Group, Values<"egpr,push2pop2,ppx,ndd,ccmp,cf">;
 def mno_apx_features_EQ : CommaJoined<["-"], "mno-apx-features=">, Group,
     HelpText<"Disable features of APX">, Values<"egpr,push2pop2,ppx,ndd,ccmp,cf">;
-// Features egpr, push2pop2 and ppx are validated with llvm-test-suite && cpu2017 on Intel SDE.
+// Features egpr, push2pop2, ppx and ndd are validated with llvm-test-suite && cpu2017 on Intel SDE.
 // For stability, we turn on these features only for -mapxf. After a feature pass the validation,
 // we will add it to -mapxf.
-def mapxf : Flag<["-"], "mapxf">, Alias, AliasArgs<["egpr","push2pop2","ppx"]>;
-def mno_apxf : Flag<["-"], "mno-apxf">, Alias, AliasArgs<["egpr","push2pop2","ppx"]>;
+def mapxf : Flag<["-"], "mapxf">, Alias, AliasArgs<["egpr","push2pop2","ppx", "ndd"]>;
+def mno_apxf : Flag<["-"], "mno-apxf">, Alias, AliasArgs<["egpr","push2pop2","ppx","ndd"]>;
 } // let Flags = [TargetSpecific]
 
 // VE feature flags
@@ -6293,6 +6324,11 @@ def fno_sycl : Flag<["-"], "fno-sycl">,
   Visibility<[ClangOption, CLOption]>,
   Group, HelpText<"Disables SYCL kernels compilation for device">;
 
+// OS-specific options
+let Flags = [TargetSpecific] in {
+defm android_pad_segment : BooleanFFlag<"android-pad-segment">, Group;
+} // let Flags = [TargetSpecific]
+
 //===----------------------------------------------------------------------===//
 // FLangOption + NoXarchOption
 //===----------------------------------------------------------------------===//
@@ -7008,6 +7044,8 @@ def mbranch_target_enforce : Flag<["-"], "mbranch-target-enforce">,
   MarshallingInfoFlag>;
 def mbranch_protection_pauth_lr : Flag<["-"], "mbranch-protection-pauth-lr">,
   MarshallingInfoFlag>;
+def mguarded_control_stack : Flag<["-"], "mguarded-control-stack">,
+  MarshallingInfoFlag>;
 def fno_dllexport_inlines : Flag<["-"], "fno-dllexport-inlines">,
   MarshallingInfoNegativeFlag>;
 def cfguard_no_checks : Flag<["-"], "cfguard-no-checks">,
@@ -8451,3 +8489,6 @@ def : Option<["/", "-"], "Qembed_debug", KIND_FLAG>, Group,
   HelpText<"Embed PDB in shader container (ignored)">;
 def spirv : DXCFlag<"spirv">,
   HelpText<"Generate SPIR-V code">;
+def fspv_target_env_EQ : Joined<["-"], "fspv-target-env=">, Group,
+  HelpText<"Specify the target environment">,
+  Values<"vulkan1.2, vulkan1.3">;
diff --git a/clang/include/clang/Driver/Types.def b/clang/include/clang/Driver/Types.def
index b889883125c4..f72c27e1ee70 100644
--- a/clang/include/clang/Driver/Types.def
+++ b/clang/include/clang/Driver/Types.def
@@ -37,8 +37,10 @@
 // C family source language (with and without preprocessing).
 TYPE("cpp-output",               PP_C,         INVALID,         "i",      phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("c",                        C,            PP_C,            "c",      phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
-TYPE("cl",                       CL,           PP_C,            "cl",     phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
-TYPE("clcpp",                    CLCXX,        PP_CXX,          "clcpp",  phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
+TYPE("cl",                       CL,           PP_CL,           "cl",     phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
+TYPE("cl-cpp-output",            PP_CL,        INVALID,         "cli",    phases::Compile, phases::Backend, phases::Assemble, phases::Link)
+TYPE("clcpp",                    CLCXX,        PP_CLCXX,        "clcpp",  phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
+TYPE("clcpp-cpp-output",         PP_CLCXX,     INVALID,         "clii",   phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("cuda-cpp-output",          PP_CUDA,      INVALID,         "cui",    phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("cuda",                     CUDA,         PP_CUDA,         "cu",     phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("cuda",                     CUDA_DEVICE,  PP_CUDA,         "cu",     phases::Preprocess, phases::Compile, phases::Backend, phases::Assemble, phases::Link)
diff --git a/clang/include/clang/ExtractAPI/API.h b/clang/include/clang/ExtractAPI/API.h
index b4c0e0ad39cd..0a0f1bd1e95f 100644
--- a/clang/include/clang/ExtractAPI/API.h
+++ b/clang/include/clang/ExtractAPI/API.h
@@ -169,6 +169,7 @@ struct APIRecord {
     RK_Enum,
     RK_StructField,
     RK_Struct,
+    RK_UnionField,
     RK_Union,
     RK_StaticField,
     RK_CXXField,
@@ -224,7 +225,7 @@ struct APIRecord {
   StringRef USR;
   StringRef Name;
   PresumedLoc Location;
-  AvailabilitySet Availabilities;
+  AvailabilityInfo Availability;
   LinkageInfo Linkage;
 
   /// Documentation comment lines attached to this symbol declaration.
@@ -256,12 +257,12 @@ struct APIRecord {
   APIRecord() = delete;
 
   APIRecord(RecordKind Kind, StringRef USR, StringRef Name,
-            PresumedLoc Location, AvailabilitySet Availabilities,
+            PresumedLoc Location, AvailabilityInfo Availability,
             LinkageInfo Linkage, const DocComment &Comment,
             DeclarationFragments Declaration, DeclarationFragments SubHeading,
             bool IsFromSystemHeader)
       : USR(USR), Name(Name), Location(Location),
-        Availabilities(std::move(Availabilities)), Linkage(Linkage),
+        Availability(std::move(Availability)), Linkage(Linkage),
         Comment(Comment), Declaration(Declaration), SubHeading(SubHeading),
         IsFromSystemHeader(IsFromSystemHeader), Kind(Kind) {}
 
@@ -274,10 +275,10 @@ struct APIRecord {
 
 struct NamespaceRecord : APIRecord {
   NamespaceRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                  AvailabilitySet Availabilities, LinkageInfo Linkage,
+                  AvailabilityInfo Availability, LinkageInfo Linkage,
                   const DocComment &Comment, DeclarationFragments Declaration,
                   DeclarationFragments SubHeading, bool IsFromSystemHeader)
-      : APIRecord(RK_Namespace, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(RK_Namespace, USR, Name, Loc, std::move(Availability),
                   Linkage, Comment, Declaration, SubHeading,
                   IsFromSystemHeader) {}
 
@@ -291,23 +292,23 @@ struct GlobalFunctionRecord : APIRecord {
   FunctionSignature Signature;
 
   GlobalFunctionRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                       AvailabilitySet Availabilities, LinkageInfo Linkage,
+                       AvailabilityInfo Availability, LinkageInfo Linkage,
                        const DocComment &Comment,
                        DeclarationFragments Declaration,
                        DeclarationFragments SubHeading,
                        FunctionSignature Signature, bool IsFromSystemHeader)
-      : APIRecord(RK_GlobalFunction, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(RK_GlobalFunction, USR, Name, Loc, std::move(Availability),
                   Linkage, Comment, Declaration, SubHeading,
                   IsFromSystemHeader),
         Signature(Signature) {}
 
   GlobalFunctionRecord(RecordKind Kind, StringRef USR, StringRef Name,
-                       PresumedLoc Loc, AvailabilitySet Availabilities,
+                       PresumedLoc Loc, AvailabilityInfo Availability,
                        LinkageInfo Linkage, const DocComment &Comment,
                        DeclarationFragments Declaration,
                        DeclarationFragments SubHeading,
                        FunctionSignature Signature, bool IsFromSystemHeader)
-      : APIRecord(Kind, USR, Name, Loc, std::move(Availabilities), Linkage,
+      : APIRecord(Kind, USR, Name, Loc, std::move(Availability), Linkage,
                   Comment, Declaration, SubHeading, IsFromSystemHeader),
         Signature(Signature) {}
 
@@ -323,14 +324,14 @@ struct GlobalFunctionTemplateRecord : GlobalFunctionRecord {
   Template Templ;
 
   GlobalFunctionTemplateRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                               AvailabilitySet Availabilities,
+                               AvailabilityInfo Availability,
                                LinkageInfo Linkage, const DocComment &Comment,
                                DeclarationFragments Declaration,
                                DeclarationFragments SubHeading,
                                FunctionSignature Signature, Template Template,
                                bool IsFromSystemHeader)
       : GlobalFunctionRecord(RK_GlobalFunctionTemplate, USR, Name, Loc,
-                             std::move(Availabilities), Linkage, Comment,
+                             std::move(Availability), Linkage, Comment,
                              Declaration, SubHeading, Signature,
                              IsFromSystemHeader),
         Templ(Template) {}
@@ -343,12 +344,12 @@ struct GlobalFunctionTemplateRecord : GlobalFunctionRecord {
 struct GlobalFunctionTemplateSpecializationRecord : GlobalFunctionRecord {
   GlobalFunctionTemplateSpecializationRecord(
       StringRef USR, StringRef Name, PresumedLoc Loc,
-      AvailabilitySet Availabilities, LinkageInfo Linkage,
+      AvailabilityInfo Availability, LinkageInfo Linkage,
       const DocComment &Comment, DeclarationFragments Declaration,
       DeclarationFragments SubHeading, FunctionSignature Signature,
       bool IsFromSystemHeader)
       : GlobalFunctionRecord(RK_GlobalFunctionTemplateSpecialization, USR, Name,
-                             Loc, std::move(Availabilities), Linkage, Comment,
+                             Loc, std::move(Availability), Linkage, Comment,
                              Declaration, SubHeading, Signature,
                              IsFromSystemHeader) {}
 
@@ -360,20 +361,20 @@ struct GlobalFunctionTemplateSpecializationRecord : GlobalFunctionRecord {
 /// This holds information associated with global functions.
 struct GlobalVariableRecord : APIRecord {
   GlobalVariableRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                       AvailabilitySet Availabilities, LinkageInfo Linkage,
+                       AvailabilityInfo Availability, LinkageInfo Linkage,
                        const DocComment &Comment,
                        DeclarationFragments Declaration,
                        DeclarationFragments SubHeading, bool IsFromSystemHeader)
-      : APIRecord(RK_GlobalVariable, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(RK_GlobalVariable, USR, Name, Loc, std::move(Availability),
                   Linkage, Comment, Declaration, SubHeading,
                   IsFromSystemHeader) {}
 
   GlobalVariableRecord(RecordKind Kind, StringRef USR, StringRef Name,
-                       PresumedLoc Loc, AvailabilitySet Availabilities,
+                       PresumedLoc Loc, AvailabilityInfo Availability,
                        LinkageInfo Linkage, const DocComment &Comment,
                        DeclarationFragments Declaration,
                        DeclarationFragments SubHeading, bool IsFromSystemHeader)
-      : APIRecord(Kind, USR, Name, Loc, std::move(Availabilities), Linkage,
+      : APIRecord(Kind, USR, Name, Loc, std::move(Availability), Linkage,
                   Comment, Declaration, SubHeading, IsFromSystemHeader) {}
 
   static bool classof(const APIRecord *Record) {
@@ -388,13 +389,13 @@ struct GlobalVariableTemplateRecord : GlobalVariableRecord {
   Template Templ;
 
   GlobalVariableTemplateRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                               AvailabilitySet Availabilities,
+                               AvailabilityInfo Availability,
                                LinkageInfo Linkage, const DocComment &Comment,
                                DeclarationFragments Declaration,
                                DeclarationFragments SubHeading,
                                class Template Template, bool IsFromSystemHeader)
       : GlobalVariableRecord(RK_GlobalVariableTemplate, USR, Name, Loc,
-                             std::move(Availabilities), Linkage, Comment,
+                             std::move(Availability), Linkage, Comment,
                              Declaration, SubHeading, IsFromSystemHeader),
         Templ(Template) {}
 
@@ -406,11 +407,11 @@ struct GlobalVariableTemplateRecord : GlobalVariableRecord {
 struct GlobalVariableTemplateSpecializationRecord : GlobalVariableRecord {
   GlobalVariableTemplateSpecializationRecord(
       StringRef USR, StringRef Name, PresumedLoc Loc,
-      AvailabilitySet Availabilities, LinkageInfo Linkage,
+      AvailabilityInfo Availability, LinkageInfo Linkage,
       const DocComment &Comment, DeclarationFragments Declaration,
       DeclarationFragments SubHeading, bool IsFromSystemHeader)
       : GlobalVariableRecord(RK_GlobalVariableTemplateSpecialization, USR, Name,
-                             Loc, std::move(Availabilities), Linkage, Comment,
+                             Loc, std::move(Availability), Linkage, Comment,
                              Declaration, SubHeading, IsFromSystemHeader) {}
 
   static bool classof(const APIRecord *Record) {
@@ -424,12 +425,12 @@ struct GlobalVariableTemplatePartialSpecializationRecord
 
   GlobalVariableTemplatePartialSpecializationRecord(
       StringRef USR, StringRef Name, PresumedLoc Loc,
-      AvailabilitySet Availabilities, LinkageInfo Linkage,
+      AvailabilityInfo Availability, LinkageInfo Linkage,
       const DocComment &Comment, DeclarationFragments Declaration,
       DeclarationFragments SubHeading, class Template Template,
       bool IsFromSystemHeader)
       : GlobalVariableRecord(RK_GlobalVariableTemplatePartialSpecialization,
-                             USR, Name, Loc, std::move(Availabilities), Linkage,
+                             USR, Name, Loc, std::move(Availability), Linkage,
                              Comment, Declaration, SubHeading,
                              IsFromSystemHeader),
         Templ(Template) {}
@@ -442,10 +443,10 @@ struct GlobalVariableTemplatePartialSpecializationRecord
 /// This holds information associated with enum constants.
 struct EnumConstantRecord : APIRecord {
   EnumConstantRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                     AvailabilitySet Availabilities, const DocComment &Comment,
+                     AvailabilityInfo Availability, const DocComment &Comment,
                      DeclarationFragments Declaration,
                      DeclarationFragments SubHeading, bool IsFromSystemHeader)
-      : APIRecord(RK_EnumConstant, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(RK_EnumConstant, USR, Name, Loc, std::move(Availability),
                   LinkageInfo::none(), Comment, Declaration, SubHeading,
                   IsFromSystemHeader) {}
 
@@ -462,10 +463,10 @@ struct EnumRecord : APIRecord {
   SmallVector> Constants;
 
   EnumRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-             AvailabilitySet Availabilities, const DocComment &Comment,
+             AvailabilityInfo Availability, const DocComment &Comment,
              DeclarationFragments Declaration, DeclarationFragments SubHeading,
              bool IsFromSystemHeader)
-      : APIRecord(RK_Enum, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(RK_Enum, USR, Name, Loc, std::move(Availability),
                   LinkageInfo::none(), Comment, Declaration, SubHeading,
                   IsFromSystemHeader) {}
 
@@ -478,17 +479,19 @@ struct EnumRecord : APIRecord {
 };
 
 /// This holds information associated with struct fields.
-struct StructFieldRecord : APIRecord {
-  StructFieldRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                    AvailabilitySet Availabilities, const DocComment &Comment,
+struct RecordFieldRecord : APIRecord {
+  RecordFieldRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
+                    AvailabilityInfo Availability, const DocComment &Comment,
                     DeclarationFragments Declaration,
-                    DeclarationFragments SubHeading, bool IsFromSystemHeader)
-      : APIRecord(RK_StructField, USR, Name, Loc, std::move(Availabilities),
+                    DeclarationFragments SubHeading, RecordKind Kind,
+                    bool IsFromSystemHeader)
+      : APIRecord(Kind, USR, Name, Loc, std::move(Availability),
                   LinkageInfo::none(), Comment, Declaration, SubHeading,
                   IsFromSystemHeader) {}
 
   static bool classof(const APIRecord *Record) {
-    return Record->getKind() == RK_StructField;
+    return Record->getKind() == RK_StructField ||
+           Record->getKind() == RK_UnionField;
   }
 
 private:
@@ -496,19 +499,20 @@ struct StructFieldRecord : APIRecord {
 };
 
 /// This holds information associated with structs.
-struct StructRecord : APIRecord {
-  SmallVector> Fields;
+struct RecordRecord : APIRecord {
+  SmallVector> Fields;
 
-  StructRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-               AvailabilitySet Availabilities, const DocComment &Comment,
+  RecordRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
+               AvailabilityInfo Availability, const DocComment &Comment,
                DeclarationFragments Declaration,
-               DeclarationFragments SubHeading, bool IsFromSystemHeader)
-      : APIRecord(RK_Struct, USR, Name, Loc, std::move(Availabilities),
+               DeclarationFragments SubHeading, RecordKind Kind,
+               bool IsFromSystemHeader)
+      : APIRecord(Kind, USR, Name, Loc, std::move(Availability),
                   LinkageInfo::none(), Comment, Declaration, SubHeading,
                   IsFromSystemHeader) {}
 
   static bool classof(const APIRecord *Record) {
-    return Record->getKind() == RK_Struct;
+    return Record->getKind() == RK_Struct || Record->getKind() == RK_Union;
   }
 
 private:
@@ -519,21 +523,21 @@ struct CXXFieldRecord : APIRecord {
   AccessControl Access;
 
   CXXFieldRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                 AvailabilitySet Availabilities, const DocComment &Comment,
+                 AvailabilityInfo Availability, const DocComment &Comment,
                  DeclarationFragments Declaration,
                  DeclarationFragments SubHeading, AccessControl Access,
                  bool IsFromSystemHeader)
-      : APIRecord(RK_CXXField, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(RK_CXXField, USR, Name, Loc, std::move(Availability),
                   LinkageInfo::none(), Comment, Declaration, SubHeading,
                   IsFromSystemHeader),
         Access(Access) {}
 
   CXXFieldRecord(RecordKind Kind, StringRef USR, StringRef Name,
-                 PresumedLoc Loc, AvailabilitySet Availabilities,
+                 PresumedLoc Loc, AvailabilityInfo Availability,
                  const DocComment &Comment, DeclarationFragments Declaration,
                  DeclarationFragments SubHeading, AccessControl Access,
                  bool IsFromSystemHeader)
-      : APIRecord(Kind, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(Kind, USR, Name, Loc, std::move(Availability),
                   LinkageInfo::none(), Comment, Declaration, SubHeading,
                   IsFromSystemHeader),
         Access(Access) {}
@@ -550,13 +554,13 @@ struct CXXFieldTemplateRecord : CXXFieldRecord {
   Template Templ;
 
   CXXFieldTemplateRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                         AvailabilitySet Availabilities,
+                         AvailabilityInfo Availability,
                          const DocComment &Comment,
                          DeclarationFragments Declaration,
                          DeclarationFragments SubHeading, AccessControl Access,
                          Template Template, bool IsFromSystemHeader)
       : CXXFieldRecord(RK_CXXFieldTemplate, USR, Name, Loc,
-                       std::move(Availabilities), Comment, Declaration,
+                       std::move(Availability), Comment, Declaration,
                        SubHeading, Access, IsFromSystemHeader),
         Templ(Template) {}
 
@@ -572,11 +576,11 @@ struct CXXMethodRecord : APIRecord {
   CXXMethodRecord() = delete;
 
   CXXMethodRecord(RecordKind Kind, StringRef USR, StringRef Name,
-                  PresumedLoc Loc, AvailabilitySet Availabilities,
+                  PresumedLoc Loc, AvailabilityInfo Availability,
                   const DocComment &Comment, DeclarationFragments Declaration,
                   DeclarationFragments SubHeading, FunctionSignature Signature,
                   AccessControl Access, bool IsFromSystemHeader)
-      : APIRecord(Kind, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(Kind, USR, Name, Loc, std::move(Availability),
                   LinkageInfo::none(), Comment, Declaration, SubHeading,
                   IsFromSystemHeader),
         Signature(Signature), Access(Access) {}
@@ -586,14 +590,13 @@ struct CXXMethodRecord : APIRecord {
 
 struct CXXConstructorRecord : CXXMethodRecord {
   CXXConstructorRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                       AvailabilitySet Availabilities,
-                       const DocComment &Comment,
+                       AvailabilityInfo Availability, const DocComment &Comment,
                        DeclarationFragments Declaration,
                        DeclarationFragments SubHeading,
                        FunctionSignature Signature, AccessControl Access,
                        bool IsFromSystemHeader)
       : CXXMethodRecord(RK_CXXConstructorMethod, USR, Name, Loc,
-                        std::move(Availabilities), Comment, Declaration,
+                        std::move(Availability), Comment, Declaration,
                         SubHeading, Signature, Access, IsFromSystemHeader) {}
   static bool classof(const APIRecord *Record) {
     return Record->getKind() == RK_CXXConstructorMethod;
@@ -605,13 +608,13 @@ struct CXXConstructorRecord : CXXMethodRecord {
 
 struct CXXDestructorRecord : CXXMethodRecord {
   CXXDestructorRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                      AvailabilitySet Availabilities, const DocComment &Comment,
+                      AvailabilityInfo Availability, const DocComment &Comment,
                       DeclarationFragments Declaration,
                       DeclarationFragments SubHeading,
                       FunctionSignature Signature, AccessControl Access,
                       bool IsFromSystemHeader)
       : CXXMethodRecord(RK_CXXDestructorMethod, USR, Name, Loc,
-                        std::move(Availabilities), Comment, Declaration,
+                        std::move(Availability), Comment, Declaration,
                         SubHeading, Signature, Access, IsFromSystemHeader) {}
   static bool classof(const APIRecord *Record) {
     return Record->getKind() == RK_CXXDestructorMethod;
@@ -623,14 +626,14 @@ struct CXXDestructorRecord : CXXMethodRecord {
 
 struct CXXStaticMethodRecord : CXXMethodRecord {
   CXXStaticMethodRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                        AvailabilitySet Availabilities,
+                        AvailabilityInfo Availability,
                         const DocComment &Comment,
                         DeclarationFragments Declaration,
                         DeclarationFragments SubHeading,
                         FunctionSignature Signature, AccessControl Access,
                         bool IsFromSystemHeader)
       : CXXMethodRecord(RK_CXXStaticMethod, USR, Name, Loc,
-                        std::move(Availabilities), Comment, Declaration,
+                        std::move(Availability), Comment, Declaration,
                         SubHeading, Signature, Access, IsFromSystemHeader) {}
   static bool classof(const APIRecord *Record) {
     return Record->getKind() == RK_CXXStaticMethod;
@@ -642,14 +645,14 @@ struct CXXStaticMethodRecord : CXXMethodRecord {
 
 struct CXXInstanceMethodRecord : CXXMethodRecord {
   CXXInstanceMethodRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                          AvailabilitySet Availabilities,
+                          AvailabilityInfo Availability,
                           const DocComment &Comment,
                           DeclarationFragments Declaration,
                           DeclarationFragments SubHeading,
                           FunctionSignature Signature, AccessControl Access,
                           bool IsFromSystemHeader)
       : CXXMethodRecord(RK_CXXInstanceMethod, USR, Name, Loc,
-                        std::move(Availabilities), Comment, Declaration,
+                        std::move(Availability), Comment, Declaration,
                         SubHeading, Signature, Access, IsFromSystemHeader) {}
 
   static bool classof(const APIRecord *Record) {
@@ -664,14 +667,14 @@ struct CXXMethodTemplateRecord : CXXMethodRecord {
   Template Templ;
 
   CXXMethodTemplateRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                          AvailabilitySet Availabilities,
+                          AvailabilityInfo Availability,
                           const DocComment &Comment,
                           DeclarationFragments Declaration,
                           DeclarationFragments SubHeading,
                           FunctionSignature Signature, AccessControl Access,
                           Template Template, bool IsFromSystemHeader)
       : CXXMethodRecord(RK_CXXMethodTemplate, USR, Name, Loc,
-                        std::move(Availabilities), Comment, Declaration,
+                        std::move(Availability), Comment, Declaration,
                         SubHeading, Signature, Access, IsFromSystemHeader),
         Templ(Template) {}
 
@@ -683,12 +686,12 @@ struct CXXMethodTemplateRecord : CXXMethodRecord {
 struct CXXMethodTemplateSpecializationRecord : CXXMethodRecord {
   CXXMethodTemplateSpecializationRecord(
       StringRef USR, StringRef Name, PresumedLoc Loc,
-      AvailabilitySet Availabilities, const DocComment &Comment,
+      AvailabilityInfo Availability, const DocComment &Comment,
       DeclarationFragments Declaration, DeclarationFragments SubHeading,
       FunctionSignature Signature, AccessControl Access,
       bool IsFromSystemHeader)
       : CXXMethodRecord(RK_CXXMethodTemplateSpecialization, USR, Name, Loc,
-                        std::move(Availabilities), Comment, Declaration,
+                        std::move(Availability), Comment, Declaration,
                         SubHeading, Signature, Access, IsFromSystemHeader) {}
 
   static bool classof(const APIRecord *Record) {
@@ -711,13 +714,13 @@ struct ObjCPropertyRecord : APIRecord {
   bool IsOptional;
 
   ObjCPropertyRecord(RecordKind Kind, StringRef USR, StringRef Name,
-                     PresumedLoc Loc, AvailabilitySet Availabilities,
+                     PresumedLoc Loc, AvailabilityInfo Availability,
                      const DocComment &Comment,
                      DeclarationFragments Declaration,
                      DeclarationFragments SubHeading, AttributeKind Attributes,
                      StringRef GetterName, StringRef SetterName,
                      bool IsOptional, bool IsFromSystemHeader)
-      : APIRecord(Kind, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(Kind, USR, Name, Loc, std::move(Availability),
                   LinkageInfo::none(), Comment, Declaration, SubHeading,
                   IsFromSystemHeader),
         Attributes(Attributes), GetterName(GetterName), SetterName(SetterName),
@@ -731,7 +734,7 @@ struct ObjCPropertyRecord : APIRecord {
 
 struct ObjCInstancePropertyRecord : ObjCPropertyRecord {
   ObjCInstancePropertyRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                             AvailabilitySet Availabilities,
+                             AvailabilityInfo Availability,
                              const DocComment &Comment,
                              DeclarationFragments Declaration,
                              DeclarationFragments SubHeading,
@@ -739,7 +742,7 @@ struct ObjCInstancePropertyRecord : ObjCPropertyRecord {
                              StringRef SetterName, bool IsOptional,
                              bool IsFromSystemHeader)
       : ObjCPropertyRecord(RK_ObjCInstanceProperty, USR, Name, Loc,
-                           std::move(Availabilities), Comment, Declaration,
+                           std::move(Availability), Comment, Declaration,
                            SubHeading, Attributes, GetterName, SetterName,
                            IsOptional, IsFromSystemHeader) {}
 
@@ -753,7 +756,7 @@ struct ObjCInstancePropertyRecord : ObjCPropertyRecord {
 
 struct ObjCClassPropertyRecord : ObjCPropertyRecord {
   ObjCClassPropertyRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                          AvailabilitySet Availabilities,
+                          AvailabilityInfo Availability,
                           const DocComment &Comment,
                           DeclarationFragments Declaration,
                           DeclarationFragments SubHeading,
@@ -761,7 +764,7 @@ struct ObjCClassPropertyRecord : ObjCPropertyRecord {
                           StringRef SetterName, bool IsOptional,
                           bool IsFromSystemHeader)
       : ObjCPropertyRecord(RK_ObjCClassProperty, USR, Name, Loc,
-                           std::move(Availabilities), Comment, Declaration,
+                           std::move(Availability), Comment, Declaration,
                            SubHeading, Attributes, GetterName, SetterName,
                            IsOptional, IsFromSystemHeader) {}
 
@@ -779,12 +782,12 @@ struct ObjCInstanceVariableRecord : APIRecord {
   AccessControl Access;
 
   ObjCInstanceVariableRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                             AvailabilitySet Availabilities,
+                             AvailabilityInfo Availability,
                              const DocComment &Comment,
                              DeclarationFragments Declaration,
                              DeclarationFragments SubHeading,
                              AccessControl Access, bool IsFromSystemHeader)
-      : APIRecord(RK_ObjCIvar, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(RK_ObjCIvar, USR, Name, Loc, std::move(Availability),
                   LinkageInfo::none(), Comment, Declaration, SubHeading,
                   IsFromSystemHeader),
         Access(Access) {}
@@ -804,11 +807,11 @@ struct ObjCMethodRecord : APIRecord {
   ObjCMethodRecord() = delete;
 
   ObjCMethodRecord(RecordKind Kind, StringRef USR, StringRef Name,
-                   PresumedLoc Loc, AvailabilitySet Availabilities,
+                   PresumedLoc Loc, AvailabilityInfo Availability,
                    const DocComment &Comment, DeclarationFragments Declaration,
                    DeclarationFragments SubHeading, FunctionSignature Signature,
                    bool IsFromSystemHeader)
-      : APIRecord(Kind, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(Kind, USR, Name, Loc, std::move(Availability),
                   LinkageInfo::none(), Comment, Declaration, SubHeading,
                   IsFromSystemHeader),
         Signature(Signature) {}
@@ -818,13 +821,13 @@ struct ObjCMethodRecord : APIRecord {
 
 struct ObjCInstanceMethodRecord : ObjCMethodRecord {
   ObjCInstanceMethodRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                           AvailabilitySet Availabilities,
+                           AvailabilityInfo Availability,
                            const DocComment &Comment,
                            DeclarationFragments Declaration,
                            DeclarationFragments SubHeading,
                            FunctionSignature Signature, bool IsFromSystemHeader)
       : ObjCMethodRecord(RK_ObjCInstanceMethod, USR, Name, Loc,
-                         std::move(Availabilities), Comment, Declaration,
+                         std::move(Availability), Comment, Declaration,
                          SubHeading, Signature, IsFromSystemHeader) {}
   static bool classof(const APIRecord *Record) {
     return Record->getKind() == RK_ObjCInstanceMethod;
@@ -836,13 +839,13 @@ struct ObjCInstanceMethodRecord : ObjCMethodRecord {
 
 struct ObjCClassMethodRecord : ObjCMethodRecord {
   ObjCClassMethodRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                        AvailabilitySet Availabilities,
+                        AvailabilityInfo Availability,
                         const DocComment &Comment,
                         DeclarationFragments Declaration,
                         DeclarationFragments SubHeading,
                         FunctionSignature Signature, bool IsFromSystemHeader)
       : ObjCMethodRecord(RK_ObjCClassMethod, USR, Name, Loc,
-                         std::move(Availabilities), Comment, Declaration,
+                         std::move(Availability), Comment, Declaration,
                          SubHeading, Signature, IsFromSystemHeader) {}
 
   static bool classof(const APIRecord *Record) {
@@ -880,13 +883,13 @@ struct StaticFieldRecord : CXXFieldRecord {
   SymbolReference Context;
 
   StaticFieldRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                    AvailabilitySet Availabilities, LinkageInfo Linkage,
+                    AvailabilityInfo Availability, LinkageInfo Linkage,
                     const DocComment &Comment, DeclarationFragments Declaration,
                     DeclarationFragments SubHeading, SymbolReference Context,
                     AccessControl Access, bool IsFromSystemHeader)
-      : CXXFieldRecord(RK_StaticField, USR, Name, Loc,
-                       std::move(Availabilities), Comment, Declaration,
-                       SubHeading, Access, IsFromSystemHeader),
+      : CXXFieldRecord(RK_StaticField, USR, Name, Loc, std::move(Availability),
+                       Comment, Declaration, SubHeading, Access,
+                       IsFromSystemHeader),
         Context(Context) {}
 
   static bool classof(const APIRecord *Record) {
@@ -905,11 +908,11 @@ struct ObjCContainerRecord : APIRecord {
   ObjCContainerRecord() = delete;
 
   ObjCContainerRecord(RecordKind Kind, StringRef USR, StringRef Name,
-                      PresumedLoc Loc, AvailabilitySet Availabilities,
+                      PresumedLoc Loc, AvailabilityInfo Availability,
                       LinkageInfo Linkage, const DocComment &Comment,
                       DeclarationFragments Declaration,
                       DeclarationFragments SubHeading, bool IsFromSystemHeader)
-      : APIRecord(Kind, USR, Name, Loc, std::move(Availabilities), Linkage,
+      : APIRecord(Kind, USR, Name, Loc, std::move(Availability), Linkage,
                   Comment, Declaration, SubHeading, IsFromSystemHeader) {}
 
   virtual ~ObjCContainerRecord() = 0;
@@ -922,11 +925,11 @@ struct CXXClassRecord : APIRecord {
   AccessControl Access;
 
   CXXClassRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                 AvailabilitySet Availabilities, const DocComment &Comment,
+                 AvailabilityInfo Availability, const DocComment &Comment,
                  DeclarationFragments Declaration,
                  DeclarationFragments SubHeading, RecordKind Kind,
                  AccessControl Access, bool IsFromSystemHeader)
-      : APIRecord(Kind, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(Kind, USR, Name, Loc, std::move(Availability),
                   LinkageInfo::none(), Comment, Declaration, SubHeading,
                   IsFromSystemHeader),
         Access(Access) {}
@@ -943,11 +946,11 @@ struct ClassTemplateRecord : CXXClassRecord {
   Template Templ;
 
   ClassTemplateRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                      AvailabilitySet Availabilities, const DocComment &Comment,
+                      AvailabilityInfo Availability, const DocComment &Comment,
                       DeclarationFragments Declaration,
                       DeclarationFragments SubHeading, Template Template,
                       AccessControl Access, bool IsFromSystemHeader)
-      : CXXClassRecord(USR, Name, Loc, std::move(Availabilities), Comment,
+      : CXXClassRecord(USR, Name, Loc, std::move(Availability), Comment,
                        Declaration, SubHeading, RK_ClassTemplate, Access,
                        IsFromSystemHeader),
         Templ(Template) {}
@@ -960,10 +963,10 @@ struct ClassTemplateRecord : CXXClassRecord {
 struct ClassTemplateSpecializationRecord : CXXClassRecord {
   ClassTemplateSpecializationRecord(
       StringRef USR, StringRef Name, PresumedLoc Loc,
-      AvailabilitySet Availabilities, const DocComment &Comment,
+      AvailabilityInfo Availability, const DocComment &Comment,
       DeclarationFragments Declaration, DeclarationFragments SubHeading,
       AccessControl Access, bool IsFromSystemHeader)
-      : CXXClassRecord(USR, Name, Loc, std::move(Availabilities), Comment,
+      : CXXClassRecord(USR, Name, Loc, std::move(Availability), Comment,
                        Declaration, SubHeading, RK_ClassTemplateSpecialization,
                        Access, IsFromSystemHeader) {}
 
@@ -976,10 +979,10 @@ struct ClassTemplatePartialSpecializationRecord : CXXClassRecord {
   Template Templ;
   ClassTemplatePartialSpecializationRecord(
       StringRef USR, StringRef Name, PresumedLoc Loc,
-      AvailabilitySet Availabilities, const DocComment &Comment,
+      AvailabilityInfo Availability, const DocComment &Comment,
       DeclarationFragments Declaration, DeclarationFragments SubHeading,
       Template Template, AccessControl Access, bool IsFromSystemHeader)
-      : CXXClassRecord(USR, Name, Loc, std::move(Availabilities), Comment,
+      : CXXClassRecord(USR, Name, Loc, std::move(Availability), Comment,
                        Declaration, SubHeading, RK_ClassTemplateSpecialization,
                        Access, IsFromSystemHeader),
         Templ(Template) {}
@@ -993,11 +996,11 @@ struct ConceptRecord : APIRecord {
   Template Templ;
 
   ConceptRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                AvailabilitySet Availabilities, const DocComment &Comment,
+                AvailabilityInfo Availability, const DocComment &Comment,
                 DeclarationFragments Declaration,
                 DeclarationFragments SubHeading, Template Template,
                 bool IsFromSystemHeader)
-      : APIRecord(RK_Concept, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(RK_Concept, USR, Name, Loc, std::move(Availability),
                   LinkageInfo::none(), Comment, Declaration, SubHeading,
                   IsFromSystemHeader),
         Templ(Template) {}
@@ -1010,12 +1013,12 @@ struct ObjCCategoryRecord : ObjCContainerRecord {
   bool IsFromExternalModule = false;
 
   ObjCCategoryRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                     AvailabilitySet Availabilities, const DocComment &Comment,
+                     AvailabilityInfo Availability, const DocComment &Comment,
                      DeclarationFragments Declaration,
                      DeclarationFragments SubHeading, SymbolReference Interface,
                      bool IsFromSystemHeader)
       : ObjCContainerRecord(RK_ObjCCategory, USR, Name, Loc,
-                            std::move(Availabilities), LinkageInfo::none(),
+                            std::move(Availability), LinkageInfo::none(),
                             Comment, Declaration, SubHeading,
                             IsFromSystemHeader),
         Interface(Interface) {}
@@ -1035,13 +1038,13 @@ struct ObjCInterfaceRecord : ObjCContainerRecord {
   SmallVector Categories;
 
   ObjCInterfaceRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                      AvailabilitySet Availabilities, LinkageInfo Linkage,
+                      AvailabilityInfo Availability, LinkageInfo Linkage,
                       const DocComment &Comment,
                       DeclarationFragments Declaration,
                       DeclarationFragments SubHeading,
                       SymbolReference SuperClass, bool IsFromSystemHeader)
       : ObjCContainerRecord(RK_ObjCInterface, USR, Name, Loc,
-                            std::move(Availabilities), Linkage, Comment,
+                            std::move(Availability), Linkage, Comment,
                             Declaration, SubHeading, IsFromSystemHeader),
         SuperClass(SuperClass) {}
 
@@ -1056,11 +1059,11 @@ struct ObjCInterfaceRecord : ObjCContainerRecord {
 /// This holds information associated with Objective-C protocols.
 struct ObjCProtocolRecord : ObjCContainerRecord {
   ObjCProtocolRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                     AvailabilitySet Availabilities, const DocComment &Comment,
+                     AvailabilityInfo Availability, const DocComment &Comment,
                      DeclarationFragments Declaration,
                      DeclarationFragments SubHeading, bool IsFromSystemHeader)
       : ObjCContainerRecord(RK_ObjCProtocol, USR, Name, Loc,
-                            std::move(Availabilities), LinkageInfo::none(),
+                            std::move(Availability), LinkageInfo::none(),
                             Comment, Declaration, SubHeading,
                             IsFromSystemHeader) {}
 
@@ -1078,7 +1081,7 @@ struct MacroDefinitionRecord : APIRecord {
                         DeclarationFragments Declaration,
                         DeclarationFragments SubHeading,
                         bool IsFromSystemHeader)
-      : APIRecord(RK_MacroDefinition, USR, Name, Loc, AvailabilitySet(),
+      : APIRecord(RK_MacroDefinition, USR, Name, Loc, AvailabilityInfo(),
                   LinkageInfo(), {}, Declaration, SubHeading,
                   IsFromSystemHeader) {}
 
@@ -1099,11 +1102,11 @@ struct TypedefRecord : APIRecord {
   SymbolReference UnderlyingType;
 
   TypedefRecord(StringRef USR, StringRef Name, PresumedLoc Loc,
-                AvailabilitySet Availabilities, const DocComment &Comment,
+                AvailabilityInfo Availability, const DocComment &Comment,
                 DeclarationFragments Declaration,
                 DeclarationFragments SubHeading, SymbolReference UnderlyingType,
                 bool IsFromSystemHeader)
-      : APIRecord(RK_Typedef, USR, Name, Loc, std::move(Availabilities),
+      : APIRecord(RK_Typedef, USR, Name, Loc, std::move(Availability),
                   LinkageInfo(), Comment, Declaration, SubHeading,
                   IsFromSystemHeader),
         UnderlyingType(UnderlyingType) {}
@@ -1191,7 +1194,7 @@ class APISet {
 public:
   NamespaceRecord *addNamespace(APIRecord *Parent, StringRef Name,
                                 StringRef USR, PresumedLoc Loc,
-                                AvailabilitySet Availability,
+                                AvailabilityInfo Availability,
                                 LinkageInfo Linkage, const DocComment &Comment,
                                 DeclarationFragments Declaration,
                                 DeclarationFragments SubHeading,
@@ -1204,13 +1207,13 @@ class APISet {
   /// to generate the USR for \c D and keep it alive in APISet.
   GlobalVariableRecord *
   addGlobalVar(StringRef Name, StringRef USR, PresumedLoc Loc,
-               AvailabilitySet Availability, LinkageInfo Linkage,
+               AvailabilityInfo Availability, LinkageInfo Linkage,
                const DocComment &Comment, DeclarationFragments Declaration,
                DeclarationFragments SubHeadin, bool IsFromSystemHeaderg);
 
   GlobalVariableTemplateRecord *
   addGlobalVariableTemplate(StringRef Name, StringRef USR, PresumedLoc Loc,
-                            AvailabilitySet Availability, LinkageInfo Linkage,
+                            AvailabilityInfo Availability, LinkageInfo Linkage,
                             const DocComment &Comment,
                             DeclarationFragments Declaration,
                             DeclarationFragments SubHeading, Template Template,
@@ -1224,14 +1227,14 @@ class APISet {
   /// to generate the USR for \c D and keep it alive in APISet.
   GlobalFunctionRecord *
   addGlobalFunction(StringRef Name, StringRef USR, PresumedLoc Loc,
-                    AvailabilitySet Availability, LinkageInfo Linkage,
+                    AvailabilityInfo Availability, LinkageInfo Linkage,
                     const DocComment &Comment, DeclarationFragments Declaration,
                     DeclarationFragments SubHeading,
                     FunctionSignature Signature, bool IsFromSystemHeader);
 
   GlobalFunctionTemplateRecord *addGlobalFunctionTemplate(
       StringRef Name, StringRef USR, PresumedLoc Loc,
-      AvailabilitySet Availability, LinkageInfo Linkage,
+      AvailabilityInfo Availability, LinkageInfo Linkage,
       const DocComment &Comment, DeclarationFragments Declaration,
       DeclarationFragments SubHeading, FunctionSignature Signature,
       Template Template, bool IsFromSystemHeader);
@@ -1239,7 +1242,7 @@ class APISet {
   GlobalFunctionTemplateSpecializationRecord *
   addGlobalFunctionTemplateSpecialization(
       StringRef Name, StringRef USR, PresumedLoc Loc,
-      AvailabilitySet Availability, LinkageInfo Linkage,
+      AvailabilityInfo Availability, LinkageInfo Linkage,
       const DocComment &Comment, DeclarationFragments Declaration,
       DeclarationFragments SubHeading, FunctionSignature Signature,
       bool IsFromSystemHeader);
@@ -1252,7 +1255,7 @@ class APISet {
   /// to generate the USR for \c D and keep it alive in APISet.
   EnumConstantRecord *
   addEnumConstant(EnumRecord *Enum, StringRef Name, StringRef USR,
-                  PresumedLoc Loc, AvailabilitySet Availability,
+                  PresumedLoc Loc, AvailabilityInfo Availability,
                   const DocComment &Comment, DeclarationFragments Declaration,
                   DeclarationFragments SubHeading, bool IsFromSystemHeader);
 
@@ -1263,45 +1266,46 @@ class APISet {
   /// APISet itself, and APISet::recordUSR(const Decl *D) is a helper method
   /// to generate the USR for \c D and keep it alive in APISet.
   EnumRecord *addEnum(StringRef Name, StringRef USR, PresumedLoc Loc,
-                      AvailabilitySet Availability, const DocComment &Comment,
+                      AvailabilityInfo Availability, const DocComment &Comment,
                       DeclarationFragments Declaration,
                       DeclarationFragments SubHeading, bool IsFromSystemHeader);
 
-  /// Create and add a struct field record into the API set.
+  /// Create and add a record field record into the API set.
   ///
   /// Note: the caller is responsible for keeping the StringRef \p Name and
   /// \p USR alive. APISet::copyString provides a way to copy strings into
   /// APISet itself, and APISet::recordUSR(const Decl *D) is a helper method
   /// to generate the USR for \c D and keep it alive in APISet.
-  StructFieldRecord *
-  addStructField(StructRecord *Struct, StringRef Name, StringRef USR,
-                 PresumedLoc Loc, AvailabilitySet Availability,
+  RecordFieldRecord *
+  addRecordField(RecordRecord *Record, StringRef Name, StringRef USR,
+                 PresumedLoc Loc, AvailabilityInfo Availability,
                  const DocComment &Comment, DeclarationFragments Declaration,
-                 DeclarationFragments SubHeading, bool IsFromSystemHeader);
+                 DeclarationFragments SubHeading, APIRecord::RecordKind Kind,
+                 bool IsFromSystemHeader);
 
-  /// Create and add a struct record into the API set.
+  /// Create and add a record record into the API set.
   ///
   /// Note: the caller is responsible for keeping the StringRef \p Name and
   /// \p USR alive. APISet::copyString provides a way to copy strings into
   /// APISet itself, and APISet::recordUSR(const Decl *D) is a helper method
   /// to generate the USR for \c D and keep it alive in APISet.
-  StructRecord *addStruct(StringRef Name, StringRef USR, PresumedLoc Loc,
-                          AvailabilitySet Availability,
+  RecordRecord *addRecord(StringRef Name, StringRef USR, PresumedLoc Loc,
+                          AvailabilityInfo Availability,
                           const DocComment &Comment,
                           DeclarationFragments Declaration,
                           DeclarationFragments SubHeading,
-                          bool IsFromSystemHeader);
+                          APIRecord::RecordKind Kind, bool IsFromSystemHeader);
 
   StaticFieldRecord *
   addStaticField(StringRef Name, StringRef USR, PresumedLoc Loc,
-                 AvailabilitySet Availabilities, LinkageInfo Linkage,
+                 AvailabilityInfo Availability, LinkageInfo Linkage,
                  const DocComment &Comment, DeclarationFragments Declaration,
                  DeclarationFragments SubHeading, SymbolReference Context,
                  AccessControl Access, bool IsFromSystemHeaderg);
 
   CXXFieldRecord *addCXXField(APIRecord *CXXClass, StringRef Name,
                               StringRef USR, PresumedLoc Loc,
-                              AvailabilitySet Availabilities,
+                              AvailabilityInfo Availability,
                               const DocComment &Comment,
                               DeclarationFragments Declaration,
                               DeclarationFragments SubHeading,
@@ -1309,12 +1313,12 @@ class APISet {
 
   CXXFieldTemplateRecord *addCXXFieldTemplate(
       APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-      AvailabilitySet Availability, const DocComment &Comment,
+      AvailabilityInfo Availability, const DocComment &Comment,
       DeclarationFragments Declaration, DeclarationFragments SubHeading,
       AccessControl Access, Template Template, bool IsFromSystemHeader);
 
   CXXClassRecord *addCXXClass(APIRecord *Parent, StringRef Name, StringRef USR,
-                              PresumedLoc Loc, AvailabilitySet Availability,
+                              PresumedLoc Loc, AvailabilityInfo Availability,
                               const DocComment &Comment,
                               DeclarationFragments Declaration,
                               DeclarationFragments SubHeading,
@@ -1323,76 +1327,76 @@ class APISet {
 
   ClassTemplateRecord *
   addClassTemplate(APIRecord *Parent, StringRef Name, StringRef USR,
-                   PresumedLoc Loc, AvailabilitySet Availability,
+                   PresumedLoc Loc, AvailabilityInfo Availability,
                    const DocComment &Comment, DeclarationFragments Declaration,
                    DeclarationFragments SubHeading, Template Template,
                    AccessControl Access, bool IsFromSystemHeader);
 
   ClassTemplateSpecializationRecord *addClassTemplateSpecialization(
       APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-      AvailabilitySet Availability, const DocComment &Comment,
+      AvailabilityInfo Availability, const DocComment &Comment,
       DeclarationFragments Declaration, DeclarationFragments SubHeading,
       AccessControl Access, bool IsFromSystemHeader);
 
   ClassTemplatePartialSpecializationRecord *
   addClassTemplatePartialSpecialization(
       APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-      AvailabilitySet Availability, const DocComment &Comment,
+      AvailabilityInfo Availability, const DocComment &Comment,
       DeclarationFragments Declaration, DeclarationFragments SubHeading,
       Template Template, AccessControl Access, bool IsFromSystemHeader);
 
   GlobalVariableTemplateSpecializationRecord *
   addGlobalVariableTemplateSpecialization(
       StringRef Name, StringRef USR, PresumedLoc Loc,
-      AvailabilitySet Availability, LinkageInfo Linkage,
+      AvailabilityInfo Availability, LinkageInfo Linkage,
       const DocComment &Comment, DeclarationFragments Declaration,
       DeclarationFragments SubHeading, bool IsFromSystemHeader);
 
   GlobalVariableTemplatePartialSpecializationRecord *
   addGlobalVariableTemplatePartialSpecialization(
       StringRef Name, StringRef USR, PresumedLoc Loc,
-      AvailabilitySet Availability, LinkageInfo Linkage,
+      AvailabilityInfo Availability, LinkageInfo Linkage,
       const DocComment &Comment, DeclarationFragments Declaration,
       DeclarationFragments SubHeading, Template Template,
       bool IsFromSystemHeader);
 
   CXXMethodRecord *addCXXInstanceMethod(
       APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-      AvailabilitySet Availability, const DocComment &Comment,
+      AvailabilityInfo Availability, const DocComment &Comment,
       DeclarationFragments Declaration, DeclarationFragments SubHeading,
       FunctionSignature Signature, AccessControl Access,
       bool IsFromSystemHeader);
 
   CXXMethodRecord *addCXXStaticMethod(
       APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-      AvailabilitySet Availability, const DocComment &Comment,
+      AvailabilityInfo Availability, const DocComment &Comment,
       DeclarationFragments Declaration, DeclarationFragments SubHeading,
       FunctionSignature Signature, AccessControl Access,
       bool IsFromSystemHeader);
 
   CXXMethodRecord *addCXXSpecialMethod(
       APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-      AvailabilitySet Availability, const DocComment &Comment,
+      AvailabilityInfo Availability, const DocComment &Comment,
       DeclarationFragments Declaration, DeclarationFragments SubHeading,
       FunctionSignature Signature, AccessControl Access,
       bool IsFromSystemHeader);
 
   CXXMethodTemplateRecord *addCXXMethodTemplate(
       APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-      AvailabilitySet Availability, const DocComment &Comment,
+      AvailabilityInfo Availability, const DocComment &Comment,
       DeclarationFragments Declaration, DeclarationFragments SubHeading,
       FunctionSignature Signature, AccessControl Access, Template Template,
       bool IsFromSystemHeader);
 
   CXXMethodTemplateSpecializationRecord *addCXXMethodTemplateSpec(
       APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-      AvailabilitySet Availability, const DocComment &Comment,
+      AvailabilityInfo Availability, const DocComment &Comment,
       DeclarationFragments Declaration, DeclarationFragments SubHeading,
       FunctionSignature Signature, AccessControl Access,
       bool IsFromSystemHeader);
 
   ConceptRecord *addConcept(StringRef Name, StringRef USR, PresumedLoc Loc,
-                            AvailabilitySet Availability,
+                            AvailabilityInfo Availability,
                             const DocComment &Comment,
                             DeclarationFragments Declaration,
                             DeclarationFragments SubHeading, Template Template,
@@ -1406,7 +1410,7 @@ class APISet {
   /// to generate the USR for \c D and keep it alive in APISet.
   ObjCCategoryRecord *
   addObjCCategory(StringRef Name, StringRef USR, PresumedLoc Loc,
-                  AvailabilitySet Availability, const DocComment &Comment,
+                  AvailabilityInfo Availability, const DocComment &Comment,
                   DeclarationFragments Declaration,
                   DeclarationFragments SubHeading, SymbolReference Interface,
                   bool IsFromSystemHeader, bool IsFromExternalModule);
@@ -1419,7 +1423,7 @@ class APISet {
   /// to generate the USR for \c D and keep it alive in APISet.
   ObjCInterfaceRecord *
   addObjCInterface(StringRef Name, StringRef USR, PresumedLoc Loc,
-                   AvailabilitySet Availability, LinkageInfo Linkage,
+                   AvailabilityInfo Availability, LinkageInfo Linkage,
                    const DocComment &Comment, DeclarationFragments Declaration,
                    DeclarationFragments SubHeading, SymbolReference SuperClass,
                    bool IsFromSystemHeader);
@@ -1432,7 +1436,7 @@ class APISet {
   /// to generate the USR for \c D and keep it alive in APISet.
   ObjCMethodRecord *
   addObjCMethod(ObjCContainerRecord *Container, StringRef Name, StringRef USR,
-                PresumedLoc Loc, AvailabilitySet Availability,
+                PresumedLoc Loc, AvailabilityInfo Availability,
                 const DocComment &Comment, DeclarationFragments Declaration,
                 DeclarationFragments SubHeading, FunctionSignature Signature,
                 bool IsInstanceMethod, bool IsFromSystemHeader);
@@ -1445,7 +1449,7 @@ class APISet {
   /// to generate the USR for \c D and keep it alive in APISet.
   ObjCPropertyRecord *
   addObjCProperty(ObjCContainerRecord *Container, StringRef Name, StringRef USR,
-                  PresumedLoc Loc, AvailabilitySet Availability,
+                  PresumedLoc Loc, AvailabilityInfo Availability,
                   const DocComment &Comment, DeclarationFragments Declaration,
                   DeclarationFragments SubHeading,
                   ObjCPropertyRecord::AttributeKind Attributes,
@@ -1460,7 +1464,7 @@ class APISet {
   /// to generate the USR for \c D and keep it alive in APISet.
   ObjCInstanceVariableRecord *addObjCInstanceVariable(
       ObjCContainerRecord *Container, StringRef Name, StringRef USR,
-      PresumedLoc Loc, AvailabilitySet Availability, const DocComment &Comment,
+      PresumedLoc Loc, AvailabilityInfo Availability, const DocComment &Comment,
       DeclarationFragments Declaration, DeclarationFragments SubHeading,
       ObjCInstanceVariableRecord::AccessControl Access,
       bool IsFromSystemHeader);
@@ -1473,7 +1477,7 @@ class APISet {
   /// to generate the USR for \c D and keep it alive in APISet.
   ObjCProtocolRecord *
   addObjCProtocol(StringRef Name, StringRef USR, PresumedLoc Loc,
-                  AvailabilitySet Availability, const DocComment &Comment,
+                  AvailabilityInfo Availability, const DocComment &Comment,
                   DeclarationFragments Declaration,
                   DeclarationFragments SubHeading, bool IsFromSystemHeader);
 
@@ -1498,7 +1502,7 @@ class APISet {
   /// to generate the USR for \c D and keep it alive in APISet.
   TypedefRecord *
   addTypedef(StringRef Name, StringRef USR, PresumedLoc Loc,
-             AvailabilitySet Availability, const DocComment &Comment,
+             AvailabilityInfo Availability, const DocComment &Comment,
              DeclarationFragments Declaration, DeclarationFragments SubHeading,
              SymbolReference UnderlyingType, bool IsFromSystemHeader);
 
@@ -1545,7 +1549,7 @@ class APISet {
     return GlobalVariableTemplatePartialSpecializations;
   }
   const RecordMap &getEnums() const { return Enums; }
-  const RecordMap &getStructs() const { return Structs; }
+  const RecordMap &getRecords() const { return Records; }
   const RecordMap &getCXXClasses() const { return CXXClasses; }
   const RecordMap &getCXXMethodTemplates() const {
     return CXXMethodTemplates;
@@ -1564,7 +1568,6 @@ class APISet {
   const RecordMap &getCXXFieldTemplates() const {
     return CXXFieldTemplates;
   }
-  const RecordMap &getConcepts() const { return Concepts; }
   const RecordMap &getClassTemplates() const {
     return ClassTemplates;
   }
@@ -1576,7 +1579,7 @@ class APISet {
   getClassTemplatePartialSpecializations() const {
     return ClassTemplatePartialSpecializations;
   }
-  const RecordMap &getRecords() const { return Concepts; }
+  const RecordMap &getConcepts() const { return Concepts; }
   const RecordMap &getObjCCategories() const {
     return ObjCCategories;
   }
@@ -1642,7 +1645,7 @@ class APISet {
   RecordMap Concepts;
   RecordMap StaticFields;
   RecordMap Enums;
-  RecordMap Structs;
+  RecordMap Records;
   RecordMap CXXClasses;
   RecordMap CXXFields;
   RecordMap CXXMethods;
diff --git a/clang/include/clang/ExtractAPI/AvailabilityInfo.h b/clang/include/clang/ExtractAPI/AvailabilityInfo.h
index 0af373135b66..3b8d6f46ed56 100644
--- a/clang/include/clang/ExtractAPI/AvailabilityInfo.h
+++ b/clang/include/clang/ExtractAPI/AvailabilityInfo.h
@@ -16,56 +16,31 @@
 #define LLVM_CLANG_EXTRACTAPI_AVAILABILITY_INFO_H
 
 #include "clang/AST/Decl.h"
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/Support/raw_ostream.h"
 
-using llvm::VersionTuple;
-
 namespace clang {
 namespace extractapi {
 
-/// Stores availability attributes of a symbol in a given domain.
+/// Stores availability attributes of a symbol.
 struct AvailabilityInfo {
   /// The domain for which this availability info item applies
   std::string Domain;
   VersionTuple Introduced;
   VersionTuple Deprecated;
   VersionTuple Obsoleted;
-  bool Unavailable;
-
-  AvailabilityInfo() = default;
-
-  AvailabilityInfo(StringRef Domain, VersionTuple I, VersionTuple D,
-                   VersionTuple O, bool U)
-      : Domain(Domain), Introduced(I), Deprecated(D), Obsoleted(O),
-        Unavailable(U) {}
-};
-
-class AvailabilitySet {
-private:
-  using AvailabilityList = llvm::SmallVector;
-  AvailabilityList Availabilities;
-
   bool UnconditionallyDeprecated = false;
   bool UnconditionallyUnavailable = false;
 
-public:
-  AvailabilitySet(const Decl *Decl);
-  AvailabilitySet() = default;
-
-  AvailabilityList::const_iterator begin() const {
-    return Availabilities.begin();
-  }
-
-  AvailabilityList::const_iterator end() const { return Availabilities.end(); }
+  AvailabilityInfo() = default;
 
+  /// Determine if this AvailabilityInfo represents the default availability.
+  bool isDefault() const { return *this == AvailabilityInfo(); }
   /// Check if the symbol is unconditionally deprecated.
   ///
   /// i.e. \code __attribute__((deprecated)) \endcode
   bool isUnconditionallyDeprecated() const { return UnconditionallyDeprecated; }
-
   /// Check if the symbol is unconditionally unavailable.
   ///
   /// i.e. \code __attribute__((unavailable)) \endcode
@@ -73,10 +48,28 @@ class AvailabilitySet {
     return UnconditionallyUnavailable;
   }
 
-  /// Determine if this AvailabilitySet represents default availability.
-  bool isDefault() const { return Availabilities.empty(); }
+  AvailabilityInfo(StringRef Domain, VersionTuple I, VersionTuple D,
+                   VersionTuple O, bool UD, bool UU)
+      : Domain(Domain), Introduced(I), Deprecated(D), Obsoleted(O),
+        UnconditionallyDeprecated(UD), UnconditionallyUnavailable(UU) {}
+
+  friend bool operator==(const AvailabilityInfo &Lhs,
+                         const AvailabilityInfo &Rhs);
+
+public:
+  static AvailabilityInfo createFromDecl(const Decl *Decl);
 };
 
+inline bool operator==(const AvailabilityInfo &Lhs,
+                       const AvailabilityInfo &Rhs) {
+  return std::tie(Lhs.Introduced, Lhs.Deprecated, Lhs.Obsoleted,
+                  Lhs.UnconditionallyDeprecated,
+                  Lhs.UnconditionallyUnavailable) ==
+         std::tie(Rhs.Introduced, Rhs.Deprecated, Rhs.Obsoleted,
+                  Rhs.UnconditionallyDeprecated,
+                  Rhs.UnconditionallyUnavailable);
+}
+
 } // namespace extractapi
 } // namespace clang
 
diff --git a/clang/include/clang/ExtractAPI/DeclarationFragments.h b/clang/include/clang/ExtractAPI/DeclarationFragments.h
index d719196b9a43..1b78c8b5931e 100644
--- a/clang/include/clang/ExtractAPI/DeclarationFragments.h
+++ b/clang/include/clang/ExtractAPI/DeclarationFragments.h
@@ -295,8 +295,9 @@ class DeclarationFragmentsBuilder {
   /// Build DeclarationFragments for a field declaration FieldDecl.
   static DeclarationFragments getFragmentsForField(const FieldDecl *);
 
-  /// Build DeclarationFragments for a struct record declaration RecordDecl.
-  static DeclarationFragments getFragmentsForStruct(const RecordDecl *);
+  /// Build DeclarationFragments for a struct/union record declaration
+  /// RecordDecl.
+  static DeclarationFragments getFragmentsForRecordDecl(const RecordDecl *);
 
   static DeclarationFragments getFragmentsForCXXClass(const CXXRecordDecl *);
 
diff --git a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
index a344fa7d5d8a..ac6f4e313540 100644
--- a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
+++ b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
@@ -14,10 +14,12 @@
 #ifndef LLVM_CLANG_EXTRACTAPI_EXTRACT_API_VISITOR_H
 #define LLVM_CLANG_EXTRACTAPI_EXTRACT_API_VISITOR_H
 
+#include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/Basic/OperatorKinds.h"
 #include "clang/Basic/Specifiers.h"
+#include "clang/ExtractAPI/AvailabilityInfo.h"
 #include "clang/ExtractAPI/DeclarationFragments.h"
 #include "llvm/ADT/FunctionExtras.h"
 
@@ -128,9 +130,10 @@ class ExtractAPIVisitorBase : public RecursiveASTVisitor {
   void recordEnumConstants(EnumRecord *EnumRecord,
                            const EnumDecl::enumerator_range Constants);
 
-  /// Collect API information for the struct fields and associate with the
+  /// Collect API information for the record fields and associate with the
   /// parent struct.
-  void recordStructFields(StructRecord *StructRecord,
+  void recordRecordFields(RecordRecord *RecordRecord,
+                          APIRecord::RecordKind FieldKind,
                           const RecordDecl::field_range Fields);
 
   /// Collect API information for the Objective-C methods and associate with the
@@ -263,7 +266,6 @@ bool ExtractAPIVisitorBase::VisitVarDecl(const VarDecl *Decl) {
       DeclarationFragmentsBuilder::getFragmentsForVar(Decl);
   DeclarationFragments SubHeading =
       DeclarationFragmentsBuilder::getSubHeading(Decl);
-
   if (Decl->isStaticDataMember()) {
     SymbolReference Context;
     // getDeclContext() should return a RecordDecl since we
@@ -272,13 +274,14 @@ bool ExtractAPIVisitorBase::VisitVarDecl(const VarDecl *Decl) {
     Context.Name = Record->getName();
     Context.USR = API.recordUSR(Record);
     auto Access = DeclarationFragmentsBuilder::getAccessControl(Decl);
-    API.addStaticField(Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment,
-                       Declaration, SubHeading, Context, Access,
-                       isInSystemHeader(Decl));
+    API.addStaticField(Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl),
+                       Linkage, Comment, Declaration, SubHeading, Context,
+                       Access, isInSystemHeader(Decl));
   } else
     // Add the global variable record to the API set.
-    API.addGlobalVar(Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment,
-                     Declaration, SubHeading, isInSystemHeader(Decl));
+    API.addGlobalVar(Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl),
+                     Linkage, Comment, Declaration, SubHeading,
+                     isInSystemHeader(Decl));
   return true;
 }
 
@@ -333,19 +336,19 @@ bool ExtractAPIVisitorBase::VisitFunctionDecl(
       DeclarationFragmentsBuilder::getSubHeading(Decl);
   FunctionSignature Signature =
       DeclarationFragmentsBuilder::getFunctionSignature(Decl);
-
   if (Decl->getTemplateSpecializationInfo())
     API.addGlobalFunctionTemplateSpecialization(
-        Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment,
+        Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl), Linkage,
+        Comment,
         DeclarationFragmentsBuilder::
             getFragmentsForFunctionTemplateSpecialization(Decl),
         SubHeading, Signature, isInSystemHeader(Decl));
   else
     // Add the function record to the API set.
     API.addGlobalFunction(
-        Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment,
-        DeclarationFragmentsBuilder::getFragmentsForFunction(Decl), SubHeading,
-        Signature, isInSystemHeader(Decl));
+        Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl), Linkage,
+        Comment, DeclarationFragmentsBuilder::getFragmentsForFunction(Decl),
+        SubHeading, Signature, isInSystemHeader(Decl));
   return true;
 }
 
@@ -379,10 +382,9 @@ bool ExtractAPIVisitorBase::VisitEnumDecl(const EnumDecl *Decl) {
       DeclarationFragmentsBuilder::getFragmentsForEnum(Decl);
   DeclarationFragments SubHeading =
       DeclarationFragmentsBuilder::getSubHeading(Decl);
-
-  EnumRecord *EnumRecord =
-      API.addEnum(API.copyString(Name), USR, Loc, AvailabilitySet(Decl),
-                  Comment, Declaration, SubHeading, isInSystemHeader(Decl));
+  EnumRecord *EnumRecord = API.addEnum(
+      API.copyString(Name), USR, Loc, AvailabilityInfo::createFromDecl(Decl),
+      Comment, Declaration, SubHeading, isInSystemHeader(Decl));
 
   // Now collect information about the enumerators in this enum.
   getDerivedExtractAPIVisitor().recordEnumConstants(EnumRecord,
@@ -496,8 +498,9 @@ bool ExtractAPIVisitorBase::VisitNamespaceDecl(
   DeclarationFragments SubHeading =
       DeclarationFragmentsBuilder::getSubHeading(Decl);
   APIRecord *Parent = determineParentRecord(Decl->getDeclContext());
-  API.addNamespace(Parent, Name, USR, Loc, AvailabilitySet(Decl), Linkage,
-                   Comment, Declaration, SubHeading, isInSystemHeader(Decl));
+  API.addNamespace(Parent, Name, USR, Loc,
+                   AvailabilityInfo::createFromDecl(Decl), Linkage, Comment,
+                   Declaration, SubHeading, isInSystemHeader(Decl));
 
   return true;
 }
@@ -524,17 +527,25 @@ bool ExtractAPIVisitorBase::VisitRecordDecl(const RecordDecl *Decl) {
 
   // Build declaration fragments and sub-heading for the struct.
   DeclarationFragments Declaration =
-      DeclarationFragmentsBuilder::getFragmentsForStruct(Decl);
+      DeclarationFragmentsBuilder::getFragmentsForRecordDecl(Decl);
   DeclarationFragments SubHeading =
       DeclarationFragmentsBuilder::getSubHeading(Decl);
 
-  StructRecord *StructRecord =
-      API.addStruct(Name, USR, Loc, AvailabilitySet(Decl), Comment, Declaration,
-                    SubHeading, isInSystemHeader(Decl));
+  auto RecordKind = APIRecord::RK_Struct;
+  auto FieldRecordKind = APIRecord::RK_StructField;
+
+  if (Decl->isUnion()) {
+    RecordKind = APIRecord::RK_Union;
+    FieldRecordKind = APIRecord::RK_UnionField;
+  }
+
+  RecordRecord *RecordRecord = API.addRecord(
+      Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl), Comment,
+      Declaration, SubHeading, RecordKind, isInSystemHeader(Decl));
 
   // Now collect information about the fields in this struct.
-  getDerivedExtractAPIVisitor().recordStructFields(StructRecord,
-                                                   Decl->fields());
+  getDerivedExtractAPIVisitor().recordRecordFields(
+      RecordRecord, FieldRecordKind, Decl->fields());
 
   return true;
 }
@@ -578,13 +589,13 @@ bool ExtractAPIVisitorBase::VisitCXXRecordDecl(
         DeclarationFragmentsBuilder::getFragmentsForRedeclarableTemplate(
             Decl->getDescribedClassTemplate()));
     CXXClassRecord = API.addClassTemplate(
-        Parent, Name, USR, Loc, AvailabilitySet(Decl), Comment, Declaration,
-        SubHeading, Template(Decl->getDescribedClassTemplate()), Access,
-        isInSystemHeader(Decl));
+        Parent, Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl), Comment,
+        Declaration, SubHeading, Template(Decl->getDescribedClassTemplate()),
+        Access, isInSystemHeader(Decl));
   } else
     CXXClassRecord = API.addCXXClass(
-        Parent, Name, USR, Loc, AvailabilitySet(Decl), Comment, Declaration,
-        SubHeading, Kind, Access, isInSystemHeader(Decl));
+        Parent, Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl), Comment,
+        Declaration, SubHeading, Kind, Access, isInSystemHeader(Decl));
 
   CXXClassRecord->Bases = getBases(Decl);
 
@@ -624,7 +635,7 @@ bool ExtractAPIVisitorBase::VisitCXXMethodDecl(
     FunctionTemplateDecl *TemplateDecl = Decl->getDescribedFunctionTemplate();
     API.addCXXMethodTemplate(
         API.findRecordForUSR(ParentUSR), Decl->getName(), USR, Loc,
-        AvailabilitySet(Decl), Comment,
+        AvailabilityInfo::createFromDecl(Decl), Comment,
         DeclarationFragmentsBuilder::getFragmentsForFunctionTemplate(
             TemplateDecl),
         SubHeading, DeclarationFragmentsBuilder::getFunctionSignature(Decl),
@@ -632,24 +643,27 @@ bool ExtractAPIVisitorBase::VisitCXXMethodDecl(
         Template(TemplateDecl), isInSystemHeader(Decl));
   } else if (Decl->getTemplateSpecializationInfo())
     API.addCXXMethodTemplateSpec(
-        Parent, Decl->getName(), USR, Loc, AvailabilitySet(Decl), Comment,
+        Parent, Decl->getName(), USR, Loc,
+        AvailabilityInfo::createFromDecl(Decl), Comment,
         DeclarationFragmentsBuilder::
             getFragmentsForFunctionTemplateSpecialization(Decl),
         SubHeading, Signature, Access, isInSystemHeader(Decl));
   else if (Decl->isOverloadedOperator())
     API.addCXXInstanceMethod(
         Parent, API.copyString(Decl->getNameAsString()), USR, Loc,
-        AvailabilitySet(Decl), Comment,
+        AvailabilityInfo::createFromDecl(Decl), Comment,
         DeclarationFragmentsBuilder::getFragmentsForOverloadedOperator(Decl),
         SubHeading, Signature, Access, isInSystemHeader(Decl));
   else if (Decl->isStatic())
     API.addCXXStaticMethod(
-        Parent, Decl->getName(), USR, Loc, AvailabilitySet(Decl), Comment,
+        Parent, Decl->getName(), USR, Loc,
+        AvailabilityInfo::createFromDecl(Decl), Comment,
         DeclarationFragmentsBuilder::getFragmentsForCXXMethod(Decl), SubHeading,
         Signature, Access, isInSystemHeader(Decl));
   else
     API.addCXXInstanceMethod(
-        Parent, Decl->getName(), USR, Loc, AvailabilitySet(Decl), Comment,
+        Parent, Decl->getName(), USR, Loc,
+        AvailabilityInfo::createFromDecl(Decl), Comment,
         DeclarationFragmentsBuilder::getFragmentsForCXXMethod(Decl), SubHeading,
         Signature, Access, isInSystemHeader(Decl));
 
@@ -678,13 +692,12 @@ bool ExtractAPIVisitorBase::VisitCXXConstructorDecl(
   FunctionSignature Signature =
       DeclarationFragmentsBuilder::getFunctionSignature(Decl);
   AccessControl Access = DeclarationFragmentsBuilder::getAccessControl(Decl);
-
   SmallString<128> ParentUSR;
   index::generateUSRForDecl(dyn_cast(Decl->getDeclContext()),
                             ParentUSR);
   API.addCXXInstanceMethod(API.findRecordForUSR(ParentUSR), Name, USR, Loc,
-                           AvailabilitySet(Decl), Comment, Declaration,
-                           SubHeading, Signature, Access,
+                           AvailabilityInfo::createFromDecl(Decl), Comment,
+                           Declaration, SubHeading, Signature, Access,
                            isInSystemHeader(Decl));
   return true;
 }
@@ -711,13 +724,12 @@ bool ExtractAPIVisitorBase::VisitCXXDestructorDecl(
   FunctionSignature Signature =
       DeclarationFragmentsBuilder::getFunctionSignature(Decl);
   AccessControl Access = DeclarationFragmentsBuilder::getAccessControl(Decl);
-
   SmallString<128> ParentUSR;
   index::generateUSRForDecl(dyn_cast(Decl->getDeclContext()),
                             ParentUSR);
   API.addCXXInstanceMethod(API.findRecordForUSR(ParentUSR), Name, USR, Loc,
-                           AvailabilitySet(Decl), Comment, Declaration,
-                           SubHeading, Signature, Access,
+                           AvailabilityInfo::createFromDecl(Decl), Comment,
+                           Declaration, SubHeading, Signature, Access,
                            isInSystemHeader(Decl));
   return true;
 }
@@ -740,8 +752,9 @@ bool ExtractAPIVisitorBase::VisitConceptDecl(const ConceptDecl *Decl) {
       DeclarationFragmentsBuilder::getFragmentsForConcept(Decl);
   DeclarationFragments SubHeading =
       DeclarationFragmentsBuilder::getSubHeading(Decl);
-  API.addConcept(Name, USR, Loc, AvailabilitySet(Decl), Comment, Declaration,
-                 SubHeading, Template(Decl), isInSystemHeader(Decl));
+  API.addConcept(Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl),
+                 Comment, Declaration, SubHeading, Template(Decl),
+                 isInSystemHeader(Decl));
   return true;
 }
 
@@ -768,8 +781,9 @@ bool ExtractAPIVisitorBase::VisitClassTemplateSpecializationDecl(
 
   APIRecord *Parent = determineParentRecord(Decl->getDeclContext());
   auto *ClassTemplateSpecializationRecord = API.addClassTemplateSpecialization(
-      Parent, Name, USR, Loc, AvailabilitySet(Decl), Comment, Declaration,
-      SubHeading, DeclarationFragmentsBuilder::getAccessControl(Decl),
+      Parent, Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl), Comment,
+      Declaration, SubHeading,
+      DeclarationFragmentsBuilder::getAccessControl(Decl),
       isInSystemHeader(Decl));
 
   ClassTemplateSpecializationRecord->Bases = getBases(Decl);
@@ -797,12 +811,11 @@ bool ExtractAPIVisitorBase::
       getFragmentsForClassTemplatePartialSpecialization(Decl);
   DeclarationFragments SubHeading =
       DeclarationFragmentsBuilder::getSubHeading(Decl);
-
   APIRecord *Parent = determineParentRecord(Decl->getDeclContext());
   auto *ClassTemplatePartialSpecRecord =
       API.addClassTemplatePartialSpecialization(
-          Parent, Name, USR, Loc, AvailabilitySet(Decl), Comment, Declaration,
-          SubHeading, Template(Decl),
+          Parent, Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl),
+          Comment, Declaration, SubHeading, Template(Decl),
           DeclarationFragmentsBuilder::getAccessControl(Decl),
           isInSystemHeader(Decl));
 
@@ -845,12 +858,13 @@ bool ExtractAPIVisitorBase::VisitVarTemplateDecl(
                             ParentUSR);
   if (Decl->getDeclContext()->getDeclKind() == Decl::CXXRecord)
     API.addCXXFieldTemplate(API.findRecordForUSR(ParentUSR), Name, USR, Loc,
-                            AvailabilitySet(Decl), Comment, Declaration,
-                            SubHeading,
+                            AvailabilityInfo::createFromDecl(Decl), Comment,
+                            Declaration, SubHeading,
                             DeclarationFragmentsBuilder::getAccessControl(Decl),
                             Template(Decl), isInSystemHeader(Decl));
   else
-    API.addGlobalVariableTemplate(Name, USR, Loc, AvailabilitySet(Decl),
+    API.addGlobalVariableTemplate(Name, USR, Loc,
+                                  AvailabilityInfo::createFromDecl(Decl),
                                   Linkage, Comment, Declaration, SubHeading,
                                   Template(Decl), isInSystemHeader(Decl));
   return true;
@@ -880,10 +894,9 @@ bool ExtractAPIVisitorBase::VisitVarTemplateSpecializationDecl(
           Decl);
   DeclarationFragments SubHeading =
       DeclarationFragmentsBuilder::getSubHeading(Decl);
-
   API.addGlobalVariableTemplateSpecialization(
-      Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment, Declaration,
-      SubHeading, isInSystemHeader(Decl));
+      Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl), Linkage, Comment,
+      Declaration, SubHeading, isInSystemHeader(Decl));
   return true;
 }
 
@@ -910,10 +923,9 @@ bool ExtractAPIVisitorBase::VisitVarTemplatePartialSpecializationDecl(
       getFragmentsForVarTemplatePartialSpecialization(Decl);
   DeclarationFragments SubHeading =
       DeclarationFragmentsBuilder::getSubHeading(Decl);
-
   API.addGlobalVariableTemplatePartialSpecialization(
-      Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment, Declaration,
-      SubHeading, Template(Decl), isInSystemHeader(Decl));
+      Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl), Linkage, Comment,
+      Declaration, SubHeading, Template(Decl), isInSystemHeader(Decl));
   return true;
 }
 
@@ -942,9 +954,8 @@ bool ExtractAPIVisitorBase::VisitFunctionTemplateDecl(
   FunctionSignature Signature =
       DeclarationFragmentsBuilder::getFunctionSignature(
           Decl->getTemplatedDecl());
-
   API.addGlobalFunctionTemplate(
-      Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment,
+      Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl), Linkage, Comment,
       DeclarationFragmentsBuilder::getFragmentsForFunctionTemplate(Decl),
       SubHeading, Signature, Template(Decl), isInSystemHeader(Decl));
 
@@ -983,8 +994,8 @@ bool ExtractAPIVisitorBase::VisitObjCInterfaceDecl(
   }
 
   ObjCInterfaceRecord *ObjCInterfaceRecord = API.addObjCInterface(
-      Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment, Declaration,
-      SubHeading, SuperClass, isInSystemHeader(Decl));
+      Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl), Linkage, Comment,
+      Declaration, SubHeading, SuperClass, isInSystemHeader(Decl));
 
   // Record all methods (selectors). This doesn't include automatically
   // synthesized property methods.
@@ -1023,9 +1034,9 @@ bool ExtractAPIVisitorBase::VisitObjCProtocolDecl(
   DeclarationFragments SubHeading =
       DeclarationFragmentsBuilder::getSubHeading(Decl);
 
-  ObjCProtocolRecord *ObjCProtocolRecord =
-      API.addObjCProtocol(Name, USR, Loc, AvailabilitySet(Decl), Comment,
-                          Declaration, SubHeading, isInSystemHeader(Decl));
+  ObjCProtocolRecord *ObjCProtocolRecord = API.addObjCProtocol(
+      Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl), Comment,
+      Declaration, SubHeading, isInSystemHeader(Decl));
 
   getDerivedExtractAPIVisitor().recordObjCMethods(ObjCProtocolRecord,
                                                   Decl->methods());
@@ -1055,8 +1066,8 @@ bool ExtractAPIVisitorBase::VisitTypedefNameDecl(
           dyn_cast(Decl->getUnderlyingType())) {
     if (const TagType *TagTy = dyn_cast(ET->desugar())) {
       if (Decl->getName() == TagTy->getDecl()->getName()) {
-        if (TagTy->getDecl()->isStruct()) {
-          modifyRecords(API.getStructs(), Decl->getName());
+        if (isa(TagTy->getDecl())) {
+          modifyRecords(API.getRecords(), Decl->getName());
         }
         if (TagTy->getDecl()->isEnum()) {
           modifyRecords(API.getEnums(), Decl->getName());
@@ -1080,7 +1091,8 @@ bool ExtractAPIVisitorBase::VisitTypedefNameDecl(
       TypedefUnderlyingTypeResolver(Context).getSymbolReferenceForType(Type,
                                                                        API);
 
-  API.addTypedef(Name, USR, Loc, AvailabilitySet(Decl), Comment,
+  API.addTypedef(Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl),
+                 Comment,
                  DeclarationFragmentsBuilder::getFragmentsForTypedef(Decl),
                  DeclarationFragmentsBuilder::getSubHeading(Decl), SymRef,
                  isInSystemHeader(Decl));
@@ -1122,8 +1134,9 @@ bool ExtractAPIVisitorBase::VisitObjCCategoryDecl(
   }
 
   ObjCCategoryRecord *ObjCCategoryRecord = API.addObjCCategory(
-      Name, USR, Loc, AvailabilitySet(Decl), Comment, Declaration, SubHeading,
-      Interface, isInSystemHeader(Decl), IsFromExternalModule);
+      Name, USR, Loc, AvailabilityInfo::createFromDecl(Decl), Comment,
+      Declaration, SubHeading, Interface, isInSystemHeader(Decl),
+      IsFromExternalModule);
 
   getDerivedExtractAPIVisitor().recordObjCMethods(ObjCCategoryRecord,
                                                   Decl->methods());
@@ -1160,17 +1173,18 @@ void ExtractAPIVisitorBase::recordEnumConstants(
     DeclarationFragments SubHeading =
         DeclarationFragmentsBuilder::getSubHeading(Constant);
 
-    API.addEnumConstant(EnumRecord, Name, USR, Loc, AvailabilitySet(Constant),
-                        Comment, Declaration, SubHeading,
-                        isInSystemHeader(Constant));
+    API.addEnumConstant(EnumRecord, Name, USR, Loc,
+                        AvailabilityInfo::createFromDecl(Constant), Comment,
+                        Declaration, SubHeading, isInSystemHeader(Constant));
   }
 }
 
 /// Collect API information for the struct fields and associate with the
 /// parent struct.
 template 
-void ExtractAPIVisitorBase::recordStructFields(
-    StructRecord *StructRecord, const RecordDecl::field_range Fields) {
+void ExtractAPIVisitorBase::recordRecordFields(
+    RecordRecord *RecordRecord, APIRecord::RecordKind FieldKind,
+    const RecordDecl::field_range Fields) {
   for (const auto *Field : Fields) {
     // Collect symbol information.
     StringRef Name = Field->getName();
@@ -1189,9 +1203,9 @@ void ExtractAPIVisitorBase::recordStructFields(
     DeclarationFragments SubHeading =
         DeclarationFragmentsBuilder::getSubHeading(Field);
 
-    API.addStructField(StructRecord, Name, USR, Loc, AvailabilitySet(Field),
-                       Comment, Declaration, SubHeading,
-                       isInSystemHeader(Field));
+    API.addRecordField(
+        RecordRecord, Name, USR, Loc, AvailabilityInfo::createFromDecl(Field),
+        Comment, Declaration, SubHeading, FieldKind, isInSystemHeader(Field));
   }
 }
 
@@ -1223,8 +1237,8 @@ bool ExtractAPIVisitorBase::VisitFieldDecl(const FieldDecl *Decl) {
   index::generateUSRForDecl(dyn_cast(Decl->getDeclContext()),
                             ParentUSR);
   API.addCXXField(API.findRecordForUSR(ParentUSR), Name, USR, Loc,
-                  AvailabilitySet(Decl), Comment, Declaration, SubHeading,
-                  Access, isInSystemHeader(Decl));
+                  AvailabilityInfo::createFromDecl(Decl), Comment, Declaration,
+                  SubHeading, Access, isInSystemHeader(Decl));
   return true;
 }
 
@@ -1255,13 +1269,13 @@ bool ExtractAPIVisitorBase::VisitCXXConversionDecl(
                             ParentUSR);
   if (Decl->isStatic())
     API.addCXXStaticMethod(API.findRecordForUSR(ParentUSR), Name, USR, Loc,
-                           AvailabilitySet(Decl), Comment, Declaration,
-                           SubHeading, Signature, Access,
+                           AvailabilityInfo::createFromDecl(Decl), Comment,
+                           Declaration, SubHeading, Signature, Access,
                            isInSystemHeader(Decl));
   else
     API.addCXXInstanceMethod(API.findRecordForUSR(ParentUSR), Name, USR, Loc,
-                             AvailabilitySet(Decl), Comment, Declaration,
-                             SubHeading, Signature, Access,
+                             AvailabilityInfo::createFromDecl(Decl), Comment,
+                             Declaration, SubHeading, Signature, Access,
                              isInSystemHeader(Decl));
   return true;
 }
@@ -1295,8 +1309,9 @@ void ExtractAPIVisitorBase::recordObjCMethods(
     FunctionSignature Signature =
         DeclarationFragmentsBuilder::getFunctionSignature(Method);
 
-    API.addObjCMethod(Container, Name, USR, Loc, AvailabilitySet(Method),
-                      Comment, Declaration, SubHeading, Signature,
+    API.addObjCMethod(Container, Name, USR, Loc,
+                      AvailabilityInfo::createFromDecl(Method), Comment,
+                      Declaration, SubHeading, Signature,
                       Method->isInstanceMethod(), isInSystemHeader(Method));
   }
 }
@@ -1334,8 +1349,8 @@ void ExtractAPIVisitorBase::recordObjCProperties(
       Attributes |= ObjCPropertyRecord::ReadOnly;
 
     API.addObjCProperty(
-        Container, Name, USR, Loc, AvailabilitySet(Property), Comment,
-        Declaration, SubHeading,
+        Container, Name, USR, Loc, AvailabilityInfo::createFromDecl(Property),
+        Comment, Declaration, SubHeading,
         static_cast(Attributes), GetterName,
         SetterName, Property->isOptional(),
         !(Property->getPropertyAttributes() &
@@ -1370,9 +1385,9 @@ void ExtractAPIVisitorBase::recordObjCInstanceVariables(
     ObjCInstanceVariableRecord::AccessControl Access =
         Ivar->getCanonicalAccessControl();
 
-    API.addObjCInstanceVariable(Container, Name, USR, Loc,
-                                AvailabilitySet(Ivar), Comment, Declaration,
-                                SubHeading, Access, isInSystemHeader(Ivar));
+    API.addObjCInstanceVariable(
+        Container, Name, USR, Loc, AvailabilityInfo::createFromDecl(Ivar),
+        Comment, Declaration, SubHeading, Access, isInSystemHeader(Ivar));
   }
 }
 
diff --git a/clang/include/clang/ExtractAPI/Serialization/SerializerBase.h b/clang/include/clang/ExtractAPI/Serialization/SerializerBase.h
index a188400a74d5..f0629a9ad56b 100644
--- a/clang/include/clang/ExtractAPI/Serialization/SerializerBase.h
+++ b/clang/include/clang/ExtractAPI/Serialization/SerializerBase.h
@@ -65,7 +65,7 @@ template  class APISetVisitor {
 
     getDerived()->traverseGlobalFunctionTemplateSpecializationRecords();
 
-    getDerived()->traverseStructRecords();
+    getDerived()->traverseRecordRecords();
 
     getDerived()->traverseObjCInterfaces();
 
@@ -98,9 +98,9 @@ template  class APISetVisitor {
       getDerived()->visitEnumRecord(*Enum.second);
   }
 
-  void traverseStructRecords() {
-    for (const auto &Struct : API.getStructs())
-      getDerived()->visitStructRecord(*Struct.second);
+  void traverseRecordRecords() {
+    for (const auto &Record : API.getRecords())
+      getDerived()->visitRecordRecord(*Record.second);
   }
 
   void traverseStaticFieldRecords() {
@@ -238,8 +238,8 @@ template  class APISetVisitor {
   /// Visit an enum record.
   void visitEnumRecord(const EnumRecord &Record){};
 
-  /// Visit a struct record.
-  void visitStructRecord(const StructRecord &Record){};
+  /// Visit a record record.
+  void visitRecordRecord(const RecordRecord &Record){};
 
   void visitStaticFieldRecord(const StaticFieldRecord &Record){};
 
diff --git a/clang/include/clang/ExtractAPI/Serialization/SymbolGraphSerializer.h b/clang/include/clang/ExtractAPI/Serialization/SymbolGraphSerializer.h
index a9b714dc4846..4249ac405fd2 100644
--- a/clang/include/clang/ExtractAPI/Serialization/SymbolGraphSerializer.h
+++ b/clang/include/clang/ExtractAPI/Serialization/SymbolGraphSerializer.h
@@ -170,8 +170,8 @@ class SymbolGraphSerializer : public APISetVisitor {
   /// Visit an enum record.
   void visitEnumRecord(const EnumRecord &Record);
 
-  /// Visit a struct record.
-  void visitStructRecord(const StructRecord &Record);
+  /// Visit a record record.
+  void visitRecordRecord(const RecordRecord &Record);
 
   void visitStaticFieldRecord(const StaticFieldRecord &Record);
 
diff --git a/clang/include/clang/Format/.clang-format b/clang/include/clang/Format/.clang-format
index d7331b3c8cf0..f95602cab0f7 100644
--- a/clang/include/clang/Format/.clang-format
+++ b/clang/include/clang/Format/.clang-format
@@ -1 +1,6 @@
-BasedOnStyle: clang-format
+BasedOnStyle: LLVM
+InsertBraces: true
+InsertNewlineAtEOF: true
+LineEnding: LF
+RemoveBracesLLVM: true
+RemoveParentheses: ReturnStatement
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index 8604dea689f9..bc9eecd42f9e 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -225,6 +225,22 @@ struct FormatStyle {
     ///   bbb = 2;
     /// \endcode
     bool AlignCompound;
+    /// Only for ``AlignConsecutiveDeclarations``. Whether function pointers are
+    /// aligned.
+    /// \code
+    ///   true:
+    ///   unsigned i;
+    ///   int     &r;
+    ///   int     *p;
+    ///   int      (*f)();
+    ///
+    ///   false:
+    ///   unsigned i;
+    ///   int     &r;
+    ///   int     *p;
+    ///   int (*f)();
+    /// \endcode
+    bool AlignFunctionPointers;
     /// Only for ``AlignConsecutiveAssignments``.  Whether short assignment
     /// operators are left-padded to the same length as long ones in order to
     /// put all assignment operators to the right of the left hand side.
@@ -247,7 +263,9 @@ struct FormatStyle {
     bool operator==(const AlignConsecutiveStyle &R) const {
       return Enabled == R.Enabled && AcrossEmptyLines == R.AcrossEmptyLines &&
              AcrossComments == R.AcrossComments &&
-             AlignCompound == R.AlignCompound && PadOperators == R.PadOperators;
+             AlignCompound == R.AlignCompound &&
+             AlignFunctionPointers == R.AlignFunctionPointers &&
+             PadOperators == R.PadOperators;
     }
     bool operator!=(const AlignConsecutiveStyle &R) const {
       return !(*this == R);
@@ -3037,6 +3055,7 @@ struct FormatStyle {
   bool isProto() const {
     return Language == LK_Proto || Language == LK_TextProto;
   }
+  bool isTableGen() const { return Language == LK_TableGen; }
 
   /// Language, this format style is targeted at.
   /// \version 3.5
@@ -3379,6 +3398,10 @@ struct FormatStyle {
   /// \version 14
   unsigned PenaltyBreakOpenParenthesis;
 
+  /// The penalty for breaking after ``::``.
+  /// \version 18
+  unsigned PenaltyBreakScopeResolution;
+
   /// The penalty for each line break introduced inside a string literal.
   /// \version 3.7
   unsigned PenaltyBreakString;
@@ -3909,6 +3932,10 @@ struct FormatStyle {
   /// \version 13
   unsigned ShortNamespaceLines;
 
+  /// Do not format macro definition body.
+  /// \version 18
+  bool SkipMacroDefinitionBody;
+
   /// Include sorting options.
   enum SortIncludesOptions : int8_t {
     /// Includes are never sorted.
@@ -4854,6 +4881,7 @@ struct FormatStyle {
            PenaltyBreakComment == R.PenaltyBreakComment &&
            PenaltyBreakFirstLessLess == R.PenaltyBreakFirstLessLess &&
            PenaltyBreakOpenParenthesis == R.PenaltyBreakOpenParenthesis &&
+           PenaltyBreakScopeResolution == R.PenaltyBreakScopeResolution &&
            PenaltyBreakString == R.PenaltyBreakString &&
            PenaltyBreakTemplateDeclaration ==
                R.PenaltyBreakTemplateDeclaration &&
@@ -4871,6 +4899,7 @@ struct FormatStyle {
            RequiresExpressionIndentation == R.RequiresExpressionIndentation &&
            SeparateDefinitionBlocks == R.SeparateDefinitionBlocks &&
            ShortNamespaceLines == R.ShortNamespaceLines &&
+           SkipMacroDefinitionBody == R.SkipMacroDefinitionBody &&
            SortIncludes == R.SortIncludes &&
            SortJavaStaticImport == R.SortJavaStaticImport &&
            SpaceAfterCStyleCast == R.SpaceAfterCStyleCast &&
diff --git a/clang/include/clang/Frontend/ASTUnit.h b/clang/include/clang/Frontend/ASTUnit.h
index fe99b3d5adbf..6af712afdcb6 100644
--- a/clang/include/clang/Frontend/ASTUnit.h
+++ b/clang/include/clang/Frontend/ASTUnit.h
@@ -902,7 +902,7 @@ class ASTUnit {
                     SourceManager &SourceMgr, FileManager &FileMgr,
                     SmallVectorImpl &StoredDiagnostics,
                     SmallVectorImpl &OwnedBuffers,
-                    std::unique_ptr Act = nullptr);
+                    std::unique_ptr Act);
 
   /// Save this translation unit to a file with the given name.
   ///
diff --git a/clang/include/clang/Frontend/Utils.h b/clang/include/clang/Frontend/Utils.h
index 143cf4359f00..604e42067a3f 100644
--- a/clang/include/clang/Frontend/Utils.h
+++ b/clang/include/clang/Frontend/Utils.h
@@ -43,12 +43,14 @@ class PCHContainerReader;
 class Preprocessor;
 class PreprocessorOptions;
 class PreprocessorOutputOptions;
+class CodeGenOptions;
 
 /// InitializePreprocessor - Initialize the preprocessor getting it and the
 /// environment ready to process a single file.
 void InitializePreprocessor(Preprocessor &PP, const PreprocessorOptions &PPOpts,
                             const PCHContainerReader &PCHContainerRdr,
-                            const FrontendOptions &FEOpts);
+                            const FrontendOptions &FEOpts,
+                            const CodeGenOptions &CodeGenOpts);
 
 /// DoPrintPreprocessedInput - Implement -E mode.
 void DoPrintPreprocessedInput(Preprocessor &PP, raw_ostream *OS,
diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h
index 01858dfcc90a..292fa566ae70 100644
--- a/clang/include/clang/Interpreter/Interpreter.h
+++ b/clang/include/clang/Interpreter/Interpreter.h
@@ -129,7 +129,7 @@ class Interpreter {
   llvm::Expected
   getSymbolAddressFromLinkerName(llvm::StringRef LinkerName) const;
 
-  enum InterfaceKind { NoAlloc, WithAlloc, CopyArray };
+  enum InterfaceKind { NoAlloc, WithAlloc, CopyArray, NewTag };
 
   const llvm::SmallVectorImpl &getValuePrintingInfo() const {
     return ValuePrintingInfo;
@@ -144,7 +144,7 @@ class Interpreter {
 
   llvm::DenseMap Dtors;
 
-  llvm::SmallVector ValuePrintingInfo;
+  llvm::SmallVector ValuePrintingInfo;
 };
 } // namespace clang
 
diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h
index 899e665e7454..b6ecc7e5ded9 100644
--- a/clang/include/clang/Lex/Lexer.h
+++ b/clang/include/clang/Lex/Lexer.h
@@ -198,11 +198,11 @@ class Lexer : public PreprocessorLexer {
   /// from.  Currently this is only used by _Pragma handling.
   SourceLocation getFileLoc() const { return FileLoc; }
 
-private:
   /// Lex - Return the next token in the file.  If this is the end of file, it
   /// return the tok::eof token.  This implicitly involves the preprocessor.
   bool Lex(Token &Result);
 
+private:
   /// Called when the preprocessor is in 'dependency scanning lexing mode'.
   bool LexDependencyDirectiveToken(Token &Result);
 
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 4ec21a8b6be2..2d9c53cdf5bd 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -2829,12 +2829,20 @@ class Preprocessor {
   }
 
   void emitMacroExpansionWarnings(const Token &Identifier) const {
-    if (Identifier.getIdentifierInfo()->isDeprecatedMacro())
+    IdentifierInfo *Info = Identifier.getIdentifierInfo();
+    if (Info->isDeprecatedMacro())
       emitMacroDeprecationWarning(Identifier);
 
-    if (Identifier.getIdentifierInfo()->isRestrictExpansion() &&
+    if (Info->isRestrictExpansion() &&
         !SourceMgr.isInMainFile(Identifier.getLocation()))
       emitRestrictExpansionWarning(Identifier);
+
+    if (Info->getName() == "INFINITY")
+      if (getLangOpts().NoHonorInfs)
+        emitRestrictInfNaNWarning(Identifier, 0);
+    if (Info->getName() == "NAN")
+      if (getLangOpts().NoHonorNaNs)
+        emitRestrictInfNaNWarning(Identifier, 1);
   }
 
   static void processPathForFileMacro(SmallVectorImpl &Path,
@@ -2850,6 +2858,8 @@ class Preprocessor {
   void emitMacroDeprecationWarning(const Token &Identifier) const;
   void emitRestrictExpansionWarning(const Token &Identifier) const;
   void emitFinalMacroWarning(const Token &Identifier, bool IsUndef) const;
+  void emitRestrictInfNaNWarning(const Token &Identifier,
+                                 unsigned DiagSelection) const;
 
   /// This boolean state keeps track if the current scanned token (by this PP)
   /// is in an "-Wunsafe-buffer-usage" opt-out region. Assuming PP scans a
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 2dbe090bd093..ffbde370e8f9 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_CLANG_PARSE_PARSER_H
 #define LLVM_CLANG_PARSE_PARSER_H
 
+#include "clang/Basic/OpenACCKinds.h"
 #include "clang/Basic/OperatorPrecedence.h"
 #include "clang/Lex/CodeCompletionHandler.h"
 #include "clang/Lex/Preprocessor.h"
@@ -234,6 +235,26 @@ class Parser : public CodeCompletionHandler {
   /// Parsing OpenACC directive mode.
   bool OpenACCDirectiveParsing = false;
 
+  /// Currently parsing a situation where an OpenACC array section could be
+  /// legal, such as a 'var-list'.
+  bool AllowOpenACCArraySections = false;
+
+  /// RAII object to set reset OpenACC parsing a context where Array Sections
+  /// are allowed.
+  class OpenACCArraySectionRAII {
+    Parser &P;
+
+  public:
+    OpenACCArraySectionRAII(Parser &P) : P(P) {
+      assert(!P.AllowOpenACCArraySections);
+      P.AllowOpenACCArraySections = true;
+    }
+    ~OpenACCArraySectionRAII() {
+      assert(P.AllowOpenACCArraySections);
+      P.AllowOpenACCArraySections = false;
+    }
+  };
+
   /// When true, we are directly inside an Objective-C message
   /// send expression.
   ///
@@ -3546,9 +3567,28 @@ class Parser : public CodeCompletionHandler {
   ExprResult ParseOpenACCIDExpression();
   /// Parses the variable list for the `cache` construct.
   void ParseOpenACCCacheVarList();
-  /// Parses a single variable in a variable list for the 'cache' construct.
-  bool ParseOpenACCCacheVar();
+  /// Parses a single variable in a variable list for OpenACC.
+  bool ParseOpenACCVar();
+  /// Parses the variable list for the variety of clauses that take a var-list,
+  /// including the optional Special Token listed for some,based on clause type.
+  bool ParseOpenACCClauseVarList(OpenACCClauseKind Kind);
+  /// Parses any parameters for an OpenACC Clause, including required/optional
+  /// parens.
+  bool ParseOpenACCClauseParams(OpenACCDirectiveKind DirKind,
+                                OpenACCClauseKind Kind);
+  /// Parses a single clause in a clause-list for OpenACC.
+  bool ParseOpenACCClause(OpenACCDirectiveKind DirKind);
+  /// Parses the clause-list for an OpenACC directive.
+  void ParseOpenACCClauseList(OpenACCDirectiveKind DirKind);
   bool ParseOpenACCWaitArgument();
+  /// Parses the clause of the 'bind' argument, which can be a string literal or
+  /// an ID expression.
+  ExprResult ParseOpenACCBindClauseArgument();
+  /// Parses the clause kind of 'int-expr', which can be any integral
+  /// expression.
+  ExprResult ParseOpenACCIntExpr();
+  /// Parses the 'device-type-list', which is a list of identifiers.
+  bool ParseOpenACCDeviceTypeList();
 
 private:
   //===--------------------------------------------------------------------===//
@@ -3624,6 +3664,18 @@ class Parser : public CodeCompletionHandler {
   ParseConceptDefinition(const ParsedTemplateInfo &TemplateInfo,
                          SourceLocation &DeclEnd);
 
+  /// Parse the given string as a type.
+  ///
+  /// This is a dangerous utility function currently employed only by API notes.
+  /// It is not a general entry-point for safely parsing types from strings.
+  ///
+  /// \param TypeStr The string to be parsed as a type.
+  /// \param Context The name of the context in which this string is being
+  /// parsed, which will be used in diagnostics.
+  /// \param IncludeLoc The location at which this parse was triggered.
+  TypeResult ParseTypeFromString(StringRef TypeStr, StringRef Context,
+                                 SourceLocation IncludeLoc);
+
   //===--------------------------------------------------------------------===//
   // Modules
   DeclGroupPtrTy ParseModuleDecl(Sema::ModuleImportState &ImportState);
diff --git a/clang/include/clang/Sema/ScopeInfo.h b/clang/include/clang/Sema/ScopeInfo.h
index b2f6e3289f41..6eaa74382685 100644
--- a/clang/include/clang/Sema/ScopeInfo.h
+++ b/clang/include/clang/Sema/ScopeInfo.h
@@ -66,7 +66,7 @@ namespace sema {
 /// parsed.
 class CompoundScopeInfo {
 public:
-  /// Whether this compound stamement contains `for' or `while' loops
+  /// Whether this compound statement contains `for' or `while' loops
   /// with empty bodies.
   bool HasEmptyLoopBodies = false;
 
@@ -168,7 +168,7 @@ class FunctionScopeInfo {
   /// to build, the initial and final coroutine suspend points
   bool NeedsCoroutineSuspends : 1;
 
-  /// An enumeration represeting the kind of the first coroutine statement
+  /// An enumeration representing the kind of the first coroutine statement
   /// in the function. One of co_return, co_await, or co_yield.
   unsigned char FirstCoroutineStmtKind : 2;
 
@@ -220,7 +220,7 @@ class FunctionScopeInfo {
   /// The initial and final coroutine suspend points.
   std::pair CoroutineSuspends;
 
-  /// The stack of currently active compound stamement scopes in the
+  /// The stack of currently active compound statement scopes in the
   /// function.
   SmallVector CompoundScopes;
 
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 5e3b57ea3322..1f1cbd11ff73 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -955,6 +955,10 @@ class Sema final {
     OpaqueParser = P;
   }
 
+  /// Callback to the parser to parse a type expressed as a string.
+  std::function
+      ParseTypeFromStringCallback;
+
   class DelayedDiagnostics;
 
   class DelayedDiagnosticsState {
@@ -2108,6 +2112,10 @@ class Sema final {
 
   bool CheckFunctionReturnType(QualType T, SourceLocation Loc);
 
+  /// Check an argument list for placeholders that we won't try to
+  /// handle later.
+  bool CheckArgsForPlaceholders(MultiExprArg args);
+
   /// Build a function type.
   ///
   /// This routine checks the function type according to C++ rules and
@@ -2153,7 +2161,7 @@ class Sema final {
                          SourceLocation Loc);
   QualType BuildBitIntType(bool IsUnsigned, Expr *BitWidth, SourceLocation Loc);
 
-  TypeSourceInfo *GetTypeForDeclarator(Declarator &D, Scope *S);
+  TypeSourceInfo *GetTypeForDeclarator(Declarator &D);
   TypeSourceInfo *GetTypeForDeclaratorCast(Declarator &D, QualType FromTy);
 
   /// Package the given type and TSI into a ParsedType.
@@ -2194,7 +2202,7 @@ class Sema final {
       SourceLocation TargetLoc, const FunctionProtoType *Source,
       bool SkipSourceFirstParameter, SourceLocation SourceLoc);
 
-  TypeResult ActOnTypeName(Scope *S, Declarator &D);
+  TypeResult ActOnTypeName(Declarator &D);
 
   /// The parser has parsed the context-sensitive type 'instancetype'
   /// in an Objective-C message declaration. Return the appropriate type.
@@ -3017,6 +3025,9 @@ class Sema final {
   ParmVarDecl *BuildParmVarDeclForTypedef(DeclContext *DC,
                                           SourceLocation Loc,
                                           QualType T);
+  QualType AdjustParameterTypeForObjCAutoRefCount(QualType T,
+                                                  SourceLocation NameLoc,
+                                                  TypeSourceInfo *TSInfo);
   ParmVarDecl *CheckParameter(DeclContext *DC, SourceLocation StartLoc,
                               SourceLocation NameLoc, IdentifierInfo *Name,
                               QualType T, TypeSourceInfo *TSInfo,
@@ -4799,6 +4810,8 @@ class Sema final {
   bool CheckAlwaysInlineAttr(const Stmt *OrigSt, const Stmt *CurSt,
                              const AttributeCommonInfo &A);
 
+  bool CheckCountedByAttr(Scope *Scope, const FieldDecl *FD);
+
   /// Adjust the calling convention of a method to be the ABI default if it
   /// wasn't specified explicitly.  This handles method types formed from
   /// function type typedefs and typename template arguments.
@@ -4814,6 +4827,29 @@ class Sema final {
   /// Valid types should not have multiple attributes with different CCs.
   const AttributedType *getCallingConvAttributedType(QualType T) const;
 
+  /// Check whether a nullability type specifier can be added to the given
+  /// type through some means not written in source (e.g. API notes).
+  ///
+  /// \param Type The type to which the nullability specifier will be
+  /// added. On success, this type will be updated appropriately.
+  ///
+  /// \param Nullability The nullability specifier to add.
+  ///
+  /// \param DiagLoc The location to use for diagnostics.
+  ///
+  /// \param AllowArrayTypes Whether to accept nullability specifiers on an
+  /// array type (e.g., because it will decay to a pointer).
+  ///
+  /// \param OverrideExisting Whether to override an existing, locally-specified
+  /// nullability specifier rather than complaining about the conflict.
+  ///
+  /// \returns true if nullability cannot be applied, false otherwise.
+  bool CheckImplicitNullabilityTypeSpecifier(QualType &Type,
+                                             NullabilityKind Nullability,
+                                             SourceLocation DiagLoc,
+                                             bool AllowArrayTypes,
+                                             bool OverrideExisting);
+
   /// Process the attributes before creating an attributed statement. Returns
   /// the semantic attributes that have been processed.
   void ProcessStmtAttributes(Stmt *Stmt, const ParsedAttributes &InAttrs,
@@ -5642,6 +5678,7 @@ class Sema final {
                       CorrectionCandidateCallback &CCC,
                       TemplateArgumentListInfo *ExplicitTemplateArgs = nullptr,
                       ArrayRef Args = std::nullopt,
+                      DeclContext *LookupCtx = nullptr,
                       TypoExpr **Out = nullptr);
 
   DeclResult LookupIvarInObjCMethod(LookupResult &Lookup, Scope *S,
@@ -7107,15 +7144,7 @@ class Sema final {
                                  NestedNameSpecInfo &IdInfo,
                                  bool EnteringContext);
 
-  /// The kind of conversion to check for. Either all attributes must match exactly,
-  /// or the converted type may add/drop '__arm_preserves_za'.
-  enum class AArch64SMECallConversionKind {
-    MatchExactly,
-    MayAddPreservesZA,
-    MayDropPreservesZA,
-  };
-  bool IsInvalidSMECallConversion(QualType FromType, QualType ToType,
-                                  AArch64SMECallConversionKind C);
+  bool IsInvalidSMECallConversion(QualType FromType, QualType ToType);
 
   /// The parser has parsed a nested-name-specifier
   /// 'template[opt] template-name < template-args >::'.
@@ -8566,8 +8595,8 @@ class Sema final {
                                           QualType ParamType,
                                           SourceLocation Loc);
   ExprResult
-  BuildExpressionFromIntegralTemplateArgument(const TemplateArgument &Arg,
-                                              SourceLocation Loc);
+  BuildExpressionFromNonTypeTemplateArgument(const TemplateArgument &Arg,
+                                             SourceLocation Loc);
 
   /// Enumeration describing how template parameter lists are compared
   /// for equality.
@@ -10263,11 +10292,13 @@ class Sema final {
     ~ConstraintEvalRAII() { TI.setEvaluateConstraints(OldValue); }
   };
 
-  // Unlike the above, this evaluates constraints, which should only happen at
-  // 'constraint checking' time.
+  // Must be used instead of SubstExpr at 'constraint checking' time.
   ExprResult
   SubstConstraintExpr(Expr *E,
                       const MultiLevelTemplateArgumentList &TemplateArgs);
+  // Unlike the above, this does not evaluates constraints.
+  ExprResult SubstConstraintExprWithoutSatisfaction(
+      Expr *E, const MultiLevelTemplateArgumentList &TemplateArgs);
 
   /// Substitute the given template arguments into a list of
   /// expressions, expanding pack expansions if required.
@@ -11218,6 +11249,11 @@ class Sema final {
   VarDecl *buildCoroutinePromise(SourceLocation Loc);
   void CheckCompletedCoroutineBody(FunctionDecl *FD, Stmt *&Body);
 
+  // Heuristically tells if the function is `get_return_object` member of a
+  // coroutine promise_type by matching the function name.
+  static bool CanBeGetReturnObject(const FunctionDecl *FD);
+  static bool CanBeGetReturnTypeOnAllocFailure(const FunctionDecl *FD);
+
   // As a clang extension, enforces that a non-coroutine function must be marked
   // with [[clang::coro_wrapper]] if it returns a type marked with
   // [[clang::coro_return_type]].
@@ -11344,9 +11380,12 @@ class Sema final {
   /// rigorous semantic checking in the new mapped directives.
   bool mapLoopConstruct(llvm::SmallVector &ClausesWithoutBind,
                         ArrayRef Clauses,
-                        OpenMPBindClauseKind BindKind,
+                        OpenMPBindClauseKind &BindKind,
                         OpenMPDirectiveKind &Kind,
-                        OpenMPDirectiveKind &PrevMappedDirective);
+                        OpenMPDirectiveKind &PrevMappedDirective,
+                        SourceLocation StartLoc, SourceLocation EndLoc,
+                        const DeclarationNameInfo &DirName,
+                        OpenMPDirectiveKind CancelRegion);
 
 public:
   /// The declarator \p D defines a function in the scope \p S which is nested
@@ -12967,7 +13006,7 @@ class Sema final {
   QualType FindCompositeObjCPointerType(ExprResult &LHS, ExprResult &RHS,
                                         SourceLocation QuestionLoc);
 
-  bool DiagnoseConditionalForNull(Expr *LHSExpr, Expr *RHSExpr,
+  bool DiagnoseConditionalForNull(const Expr *LHSExpr, const Expr *RHSExpr,
                                   SourceLocation QuestionLoc);
 
   void DiagnoseAlwaysNonNullPointer(Expr *E,
@@ -13893,8 +13932,9 @@ class Sema final {
 
   bool SemaBuiltinVAStart(unsigned BuiltinID, CallExpr *TheCall);
   bool SemaBuiltinVAStartARMMicrosoft(CallExpr *Call);
-  bool SemaBuiltinUnorderedCompare(CallExpr *TheCall);
-  bool SemaBuiltinFPClassification(CallExpr *TheCall, unsigned NumArgs);
+  bool SemaBuiltinUnorderedCompare(CallExpr *TheCall, unsigned BuiltinID);
+  bool SemaBuiltinFPClassification(CallExpr *TheCall, unsigned NumArgs,
+                                   unsigned BuiltinID);
   bool SemaBuiltinComplex(CallExpr *TheCall);
   bool SemaBuiltinVSX(CallExpr *TheCall);
   bool SemaBuiltinOSLogFormat(CallExpr *TheCall);
@@ -13998,6 +14038,8 @@ class Sema final {
                             SourceRange range,
                             llvm::SmallBitVector &CheckedVarArgs);
 
+  void CheckInfNaNFunction(const CallExpr *Call, const FunctionDecl *FDecl);
+
   void CheckAbsoluteValueFunction(const CallExpr *Call,
                                   const FunctionDecl *FDecl);
 
diff --git a/clang/include/clang/Sema/Template.h b/clang/include/clang/Sema/Template.h
index 2a553054a0ce..ce44aca797b0 100644
--- a/clang/include/clang/Sema/Template.h
+++ b/clang/include/clang/Sema/Template.h
@@ -564,6 +564,7 @@ enum class TemplateSubstitutionKind : char {
     const MultiLevelTemplateArgumentList &TemplateArgs;
     Sema::LateInstantiatedAttrVec* LateAttrs = nullptr;
     LocalInstantiationScope *StartingScope = nullptr;
+    // Whether to evaluate the C++20 constraints or simply substitute into them.
     bool EvaluateConstraints = true;
 
     /// A list of out-of-line class template partial
diff --git a/clang/include/clang/Sema/TypoCorrection.h b/clang/include/clang/Sema/TypoCorrection.h
index e0f8d152dbe5..09de164297e7 100644
--- a/clang/include/clang/Sema/TypoCorrection.h
+++ b/clang/include/clang/Sema/TypoCorrection.h
@@ -282,7 +282,7 @@ class CorrectionCandidateCallback {
 public:
   static const unsigned InvalidDistance = TypoCorrection::InvalidDistance;
 
-  explicit CorrectionCandidateCallback(IdentifierInfo *Typo = nullptr,
+  explicit CorrectionCandidateCallback(const IdentifierInfo *Typo = nullptr,
                                        NestedNameSpecifier *TypoNNS = nullptr)
       : Typo(Typo), TypoNNS(TypoNNS) {}
 
@@ -319,7 +319,7 @@ class CorrectionCandidateCallback {
   /// this method.
   virtual std::unique_ptr clone() = 0;
 
-  void setTypoName(IdentifierInfo *II) { Typo = II; }
+  void setTypoName(const IdentifierInfo *II) { Typo = II; }
   void setTypoNNS(NestedNameSpecifier *NNS) { TypoNNS = NNS; }
 
   // Flags for context-dependent keywords. WantFunctionLikeCasts is only
@@ -345,13 +345,13 @@ class CorrectionCandidateCallback {
            candidate.getCorrectionSpecifier() == TypoNNS;
   }
 
-  IdentifierInfo *Typo;
+  const IdentifierInfo *Typo;
   NestedNameSpecifier *TypoNNS;
 };
 
 class DefaultFilterCCC final : public CorrectionCandidateCallback {
 public:
-  explicit DefaultFilterCCC(IdentifierInfo *Typo = nullptr,
+  explicit DefaultFilterCCC(const IdentifierInfo *Typo = nullptr,
                             NestedNameSpecifier *TypoNNS = nullptr)
       : CorrectionCandidateCallback(Typo, TypoNNS) {}
 
@@ -365,6 +365,10 @@ class DefaultFilterCCC final : public CorrectionCandidateCallback {
 template 
 class DeclFilterCCC final : public CorrectionCandidateCallback {
 public:
+  explicit DeclFilterCCC(const IdentifierInfo *Typo = nullptr,
+                         NestedNameSpecifier *TypoNNS = nullptr)
+      : CorrectionCandidateCallback(Typo, TypoNNS) {}
+
   bool ValidateCandidate(const TypoCorrection &candidate) override {
     return candidate.getCorrectionDeclAs();
   }
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 21d791f5cd89..dd1451bbf2d2 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -550,6 +550,10 @@ class ASTReader
   /// declaration and the value is the deduced return type.
   llvm::SmallMapVector PendingDeducedTypeUpdates;
 
+  /// Functions has undededuced return type and we wish we can find the deduced
+  /// return type by iterating the redecls in other modules.
+  llvm::SmallVector PendingUndeducedFunctionDecls;
+
   /// Declarations that have been imported and have typedef names for
   /// linkage purposes.
   llvm::DenseMap, NamedDecl *>
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h
index 5d2c96e5bc9d..45187433c069 100644
--- a/clang/include/clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h
@@ -13,6 +13,7 @@
 namespace clang {
 namespace ento {
 namespace categories {
+extern const char *const AppleAPIMisuse;
 extern const char *const CoreFoundationObjectiveC;
 extern const char *const LogicError;
 extern const char *const MemoryRefCount;
diff --git a/clang/include/clang/StaticAnalyzer/Core/Checker.h b/clang/include/clang/StaticAnalyzer/Core/Checker.h
index 8a46282a595e..2ec54a837c42 100644
--- a/clang/include/clang/StaticAnalyzer/Core/Checker.h
+++ b/clang/include/clang/StaticAnalyzer/Core/Checker.h
@@ -193,9 +193,8 @@ class PostCall {
 
 class Location {
   template 
-  static void _checkLocation(void *checker,
-                             const SVal &location, bool isLoad, const Stmt *S,
-                             CheckerContext &C) {
+  static void _checkLocation(void *checker, SVal location, bool isLoad,
+                             const Stmt *S, CheckerContext &C) {
     ((const CHECKER *)checker)->checkLocation(location, isLoad, S, C);
   }
 
@@ -209,8 +208,7 @@ class Location {
 
 class Bind {
   template 
-  static void _checkBind(void *checker,
-                         const SVal &location, const SVal &val, const Stmt *S,
+  static void _checkBind(void *checker, SVal location, SVal val, const Stmt *S,
                          CheckerContext &C) {
     ((const CHECKER *)checker)->checkBind(location, val, S, C);
   }
@@ -456,10 +454,8 @@ namespace eval {
 
 class Assume {
   template 
-  static ProgramStateRef _evalAssume(void *checker,
-                                         ProgramStateRef state,
-                                         const SVal &cond,
-                                         bool assumption) {
+  static ProgramStateRef _evalAssume(void *checker, ProgramStateRef state,
+                                     SVal cond, bool assumption) {
     return ((const CHECKER *)checker)->evalAssume(state, cond, assumption);
   }
 
diff --git a/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h b/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
index 39583c443eda..a45ba1bc573e 100644
--- a/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
+++ b/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
@@ -488,13 +488,11 @@ class CheckerManager {
   using CheckCallFunc =
       CheckerFn;
 
-  using CheckLocationFunc =
-      CheckerFn;
+  using CheckLocationFunc = CheckerFn;
 
   using CheckBindFunc =
-      CheckerFn;
+      CheckerFn;
 
   using CheckEndAnalysisFunc =
       CheckerFn;
@@ -530,8 +528,7 @@ class CheckerManager {
                                  RegionAndSymbolInvalidationTraits *ITraits)>;
 
   using EvalAssumeFunc =
-      CheckerFn;
+      CheckerFn;
 
   using EvalCallFunc = CheckerFn;
 
diff --git a/clang/include/clang/Support/RISCVVIntrinsicUtils.h b/clang/include/clang/Support/RISCVVIntrinsicUtils.h
index c525d3443331..05a5e02e1390 100644
--- a/clang/include/clang/Support/RISCVVIntrinsicUtils.h
+++ b/clang/include/clang/Support/RISCVVIntrinsicUtils.h
@@ -488,7 +488,7 @@ class RVVIntrinsic {
 enum RVVRequire : uint32_t {
   RVV_REQ_None = 0,
   RVV_REQ_RV64 = 1 << 0,
-  RVV_REQ_ZvfhminOrZvfh = 1 << 1,
+  RVV_REQ_Zvfhmin = 1 << 1,
   RVV_REQ_Xsfvcp = 1 << 2,
   RVV_REQ_Xsfvfnrclipxfqf = 1 << 3,
   RVV_REQ_Xsfvfwmaccqqq = 1 << 4,
diff --git a/clang/lib/ARCMigrate/FileRemapper.cpp b/clang/lib/ARCMigrate/FileRemapper.cpp
index 7abc862ceecc..84024c3bafdc 100644
--- a/clang/lib/ARCMigrate/FileRemapper.cpp
+++ b/clang/lib/ARCMigrate/FileRemapper.cpp
@@ -43,7 +43,7 @@ std::string FileRemapper::getRemapInfoFile(StringRef outputDir) {
   assert(!outputDir.empty());
   SmallString<128> InfoFile = outputDir;
   llvm::sys::path::append(InfoFile, "remap");
-  return std::string(InfoFile.str());
+  return std::string(InfoFile);
 }
 
 bool FileRemapper::initFromDisk(StringRef outputDir, DiagnosticsEngine &Diag,
diff --git a/clang/lib/ARCMigrate/ObjCMT.cpp b/clang/lib/ARCMigrate/ObjCMT.cpp
index ed363a46a200..0786c81516b2 100644
--- a/clang/lib/ARCMigrate/ObjCMT.cpp
+++ b/clang/lib/ARCMigrate/ObjCMT.cpp
@@ -2201,7 +2201,7 @@ static std::string applyEditsToTemp(FileEntryRef FE,
   TmpOut.write(NewText.data(), NewText.size());
   TmpOut.close();
 
-  return std::string(TempPath.str());
+  return std::string(TempPath);
 }
 
 bool arcmt::getFileRemappingsFromFileList(
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 3c6ad5229380..8f3373521877 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -1321,6 +1321,13 @@ void ASTContext::InitBuiltinTypes(const TargetInfo &Target,
     InitBuiltinType(OMPArrayShapingTy, BuiltinType::OMPArrayShaping);
     InitBuiltinType(OMPIteratorTy, BuiltinType::OMPIterator);
   }
+  // Placeholder type for OpenACC array sections.
+  if (LangOpts.OpenACC) {
+    // FIXME: Once we implement OpenACC array sections in Sema, this will either
+    // be combined with the OpenMP type, or given its own type. In the meantime,
+    // just use the OpenMP type so that parsing can work.
+    InitBuiltinType(OMPArraySectionTy, BuiltinType::OMPArraySection);
+  }
   if (LangOpts.MatrixTypes)
     InitBuiltinType(IncompleteMatrixIdxTy, BuiltinType::IncompleteMatrixIdx);
 
@@ -2762,21 +2769,20 @@ bool ASTContext::hasUniqueObjectRepresentations(
     QualType Ty, bool CheckIfTriviallyCopyable) const {
   // C++17 [meta.unary.prop]:
   //   The predicate condition for a template specialization
-  //   has_unique_object_representations shall be
-  //   satisfied if and only if:
+  //   has_unique_object_representations shall be satisfied if and only if:
   //     (9.1) - T is trivially copyable, and
   //     (9.2) - any two objects of type T with the same value have the same
-  //     object representation, where two objects
-  //   of array or non-union class type are considered to have the same value
-  //   if their respective sequences of
-  //   direct subobjects have the same values, and two objects of union type
-  //   are considered to have the same
-  //   value if they have the same active member and the corresponding members
-  //   have the same value.
+  //     object representation, where:
+  //     - two objects of array or non-union class type are considered to have
+  //       the same value if their respective sequences of direct subobjects
+  //       have the same values, and
+  //     - two objects of union type are considered to have the same value if
+  //       they have the same active member and the corresponding members have
+  //       the same value.
   //   The set of scalar types for which this condition holds is
-  //   implementation-defined. [ Note: If a type has padding
-  //   bits, the condition does not hold; otherwise, the condition holds true
-  //   for unsigned integral types. -- end note ]
+  //   implementation-defined. [ Note: If a type has padding bits, the condition
+  //   does not hold; otherwise, the condition holds true for unsigned integral
+  //   types. -- end note ]
   assert(!Ty.isNull() && "Null QualType sent to unique object rep check");
 
   // Arrays are unique only if their element type is unique.
@@ -4496,10 +4502,11 @@ QualType ASTContext::getFunctionTypeInternal(
       EPI.ExceptionSpec.Type, EPI.ExceptionSpec.Exceptions.size());
   size_t Size = FunctionProtoType::totalSizeToAlloc<
       QualType, SourceLocation, FunctionType::FunctionTypeExtraBitfields,
-      FunctionType::ExceptionType, Expr *, FunctionDecl *,
-      FunctionProtoType::ExtParameterInfo, Qualifiers>(
+      FunctionType::FunctionTypeArmAttributes, FunctionType::ExceptionType,
+      Expr *, FunctionDecl *, FunctionProtoType::ExtParameterInfo, Qualifiers>(
       NumArgs, EPI.Variadic, EPI.requiresFunctionProtoTypeExtraBitfields(),
-      ESH.NumExceptionType, ESH.NumExprPtr, ESH.NumFunctionDeclPtr,
+      EPI.requiresFunctionProtoTypeArmAttributes(), ESH.NumExceptionType,
+      ESH.NumExprPtr, ESH.NumFunctionDeclPtr,
       EPI.ExtParameterInfos ? NumArgs : 0,
       EPI.TypeQuals.hasNonFastQualifiers() ? 1 : 0);
 
@@ -6765,6 +6772,11 @@ ASTContext::getCanonicalTemplateArgument(const TemplateArgument &Arg) const {
     case TemplateArgument::Integral:
       return TemplateArgument(Arg, getCanonicalType(Arg.getIntegralType()));
 
+    case TemplateArgument::StructuralValue:
+      return TemplateArgument(*this,
+                              getCanonicalType(Arg.getStructuralValueType()),
+                              Arg.getAsStructuralValue());
+
     case TemplateArgument::Type:
       return TemplateArgument(getCanonicalType(Arg.getAsType()),
                               /*isNullPtr*/ false, Arg.getIsDefaulted());
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 1b9f97539e25..552cb61c72ab 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -826,6 +826,17 @@ ASTNodeImporter::import(const TemplateArgument &From) {
                             From.getIsDefaulted());
   }
 
+  case TemplateArgument::StructuralValue: {
+    ExpectedType ToTypeOrErr = import(From.getStructuralValueType());
+    if (!ToTypeOrErr)
+      return ToTypeOrErr.takeError();
+    Expected ToValueOrErr = import(From.getAsStructuralValue());
+    if (!ToValueOrErr)
+      return ToValueOrErr.takeError();
+    return TemplateArgument(Importer.getToContext(), *ToTypeOrErr,
+                            *ToValueOrErr);
+  }
+
   case TemplateArgument::Template: {
     Expected ToTemplateOrErr = import(From.getAsTemplate());
     if (!ToTemplateOrErr)
@@ -2041,23 +2052,25 @@ ASTNodeImporter::ImportDeclContext(DeclContext *FromDC, bool ForceImport) {
     return ToDCOrErr.takeError();
   }
 
-  DeclContext *ToDC = *ToDCOrErr;
-  // Remove all declarations, which may be in wrong order in the
-  // lexical DeclContext and then add them in the proper order.
-  for (auto *D : FromDC->decls()) {
-    if (!MightNeedReordering(D))
-      continue;
+  if (const auto *FromRD = dyn_cast(FromDC)) {
+    DeclContext *ToDC = *ToDCOrErr;
+    // Remove all declarations, which may be in wrong order in the
+    // lexical DeclContext and then add them in the proper order.
+    for (auto *D : FromRD->decls()) {
+      if (!MightNeedReordering(D))
+        continue;
 
-    assert(D && "DC contains a null decl");
-    if (Decl *ToD = Importer.GetAlreadyImportedOrNull(D)) {
-      // Remove only the decls which we successfully imported.
-      assert(ToDC == ToD->getLexicalDeclContext() && ToDC->containsDecl(ToD));
-      // Remove the decl from its wrong place in the linked list.
-      ToDC->removeDecl(ToD);
-      // Add the decl to the end of the linked list.
-      // This time it will be at the proper place because the enclosing for
-      // loop iterates in the original (good) order of the decls.
-      ToDC->addDeclInternal(ToD);
+      assert(D && "DC contains a null decl");
+      if (Decl *ToD = Importer.GetAlreadyImportedOrNull(D)) {
+        // Remove only the decls which we successfully imported.
+        assert(ToDC == ToD->getLexicalDeclContext() && ToDC->containsDecl(ToD));
+        // Remove the decl from its wrong place in the linked list.
+        ToDC->removeDecl(ToD);
+        // Add the decl to the end of the linked list.
+        // This time it will be at the proper place because the enclosing for
+        // loop iterates in the original (good) order of the decls.
+        ToDC->addDeclInternal(ToD);
+      }
     }
   }
 
@@ -3577,6 +3590,8 @@ class IsTypeDeclaredInsideVisitor
     case TemplateArgument::NullPtr:
       // FIXME: The type is not allowed to be in the function?
       return CheckType(Arg.getNullPtrType());
+    case TemplateArgument::StructuralValue:
+      return CheckType(Arg.getStructuralValueType());
     case TemplateArgument::Pack:
       for (const auto &PackArg : Arg.getPackAsArray())
         if (checkTemplateArgument(PackArg))
@@ -3901,7 +3916,7 @@ ExpectedDecl ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
   ToFunction->setLexicalDeclContext(LexicalDC);
   ToFunction->setVirtualAsWritten(D->isVirtualAsWritten());
   ToFunction->setTrivial(D->isTrivial());
-  ToFunction->setPure(D->isPure());
+  ToFunction->setIsPureVirtual(D->isPureVirtual());
   ToFunction->setDefaulted(D->isDefaulted());
   ToFunction->setExplicitlyDefaulted(D->isExplicitlyDefaulted());
   ToFunction->setDeletedAsWritten(D->isDeletedAsWritten());
@@ -5934,15 +5949,22 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
   if (ToD)
     return ToD;
 
-  bool IsFriendTemplate = D->getFriendObjectKind() != Decl::FOK_None;
-  bool IsDependentContext = DC != LexicalDC ? LexicalDC->isDependentContext()
-                                            : DC->isDependentContext();
-  bool DependentFriend = IsFriendTemplate && IsDependentContext;
+  // Should check if a declaration is friend in a dependent context.
+  // Such templates are not linked together in a declaration chain.
+  // The ASTImporter strategy is to map existing forward declarations to
+  // imported ones only if strictly necessary, otherwise import these as new
+  // forward declarations. In case of the "dependent friend" declarations, new
+  // declarations are created, but not linked in a declaration chain.
+  auto IsDependentFriend = [](ClassTemplateDecl *TD) {
+    return TD->getFriendObjectKind() != Decl::FOK_None &&
+           TD->getLexicalDeclContext()->isDependentContext();
+  };
+  bool DependentFriend = IsDependentFriend(D);
 
   ClassTemplateDecl *FoundByLookup = nullptr;
 
   // We may already have a template of the same name; try to find and match it.
-  if (!DependentFriend && !DC->isFunctionOrMethod()) {
+  if (!DC->isFunctionOrMethod()) {
     SmallVector ConflictingDecls;
     auto FoundDecls = Importer.findDeclsInToCtx(DC, Name);
     for (auto *FoundDecl : FoundDecls) {
@@ -5958,10 +5980,13 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
 
         // FIXME: sufficient conditon for 'IgnoreTemplateParmDepth'?
         bool IgnoreTemplateParmDepth =
-            FoundTemplate->getFriendObjectKind() != Decl::FOK_None &&
-            !D->specializations().empty();
+            (FoundTemplate->getFriendObjectKind() != Decl::FOK_None) !=
+            (D->getFriendObjectKind() != Decl::FOK_None);
         if (IsStructuralMatch(D, FoundTemplate, /*Complain=*/true,
                               IgnoreTemplateParmDepth)) {
+          if (DependentFriend || IsDependentFriend(FoundTemplate))
+            continue;
+
           ClassTemplateDecl *TemplateWithDef =
               getTemplateDefinition(FoundTemplate);
           if (D->isThisDeclarationADefinition() && TemplateWithDef)
@@ -6148,6 +6173,11 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateSpecializationDecl(
                                                   InsertPos))
       // Add this partial specialization to the class template.
       ClassTemplate->AddPartialSpecialization(PartSpec2, InsertPos);
+    if (Expected ToInstOrErr =
+            import(PartialSpec->getInstantiatedFromMember()))
+      PartSpec2->setInstantiatedFromMember(*ToInstOrErr);
+    else
+      return ToInstOrErr.takeError();
 
     updateLookupTableForTemplateParameters(*ToTPList);
   } else { // Not a partial specialization.
@@ -6270,17 +6300,21 @@ ExpectedDecl ASTNodeImporter::VisitVarTemplateDecl(VarTemplateDecl *D) {
                                               D->getTemplatedDecl()))
         continue;
       if (IsStructuralMatch(D, FoundTemplate)) {
-        // The Decl in the "From" context has a definition, but in the
-        // "To" context we already have a definition.
+        // FIXME Check for ODR error if the two definitions have
+        // different initializers?
         VarTemplateDecl *FoundDef = getTemplateDefinition(FoundTemplate);
-        if (D->isThisDeclarationADefinition() && FoundDef)
-          // FIXME Check for ODR error if the two definitions have
-          // different initializers?
-          return Importer.MapImported(D, FoundDef);
-        if (FoundTemplate->getDeclContext()->isRecord() &&
-            D->getDeclContext()->isRecord())
-          return Importer.MapImported(D, FoundTemplate);
-
+        if (D->getDeclContext()->isRecord()) {
+          assert(FoundTemplate->getDeclContext()->isRecord() &&
+                 "Member variable template imported as non-member, "
+                 "inconsistent imported AST?");
+          if (FoundDef)
+            return Importer.MapImported(D, FoundDef);
+          if (!D->isThisDeclarationADefinition())
+            return Importer.MapImported(D, FoundTemplate);
+        } else {
+          if (FoundDef && D->isThisDeclarationADefinition())
+            return Importer.MapImported(D, FoundDef);
+        }
         FoundByLookup = FoundTemplate;
         break;
       }
@@ -9030,6 +9064,10 @@ class AttrImporter {
 public:
   AttrImporter(ASTImporter &I) : Importer(I), NImporter(I) {}
 
+  // Useful for accessing the imported attribute.
+  template  T *castAttrAs() { return cast(ToAttr); }
+  template  const T *castAttrAs() const { return cast(ToAttr); }
+
   // Create an "importer" for an attribute parameter.
   // Result of the 'value()' of that object is to be passed to the function
   // 'importAttr', in the order that is expected by the attribute class.
@@ -9243,6 +9281,15 @@ Expected ASTImporter::Import(const Attr *FromAttr) {
                   From->args_size());
     break;
   }
+  case attr::CountedBy: {
+    AI.cloneAttr(FromAttr);
+    const auto *CBA = cast(FromAttr);
+    Expected SR = Import(CBA->getCountedByFieldLoc()).get();
+    if (!SR)
+      return SR.takeError();
+    AI.castAttrAs()->setCountedByFieldLoc(SR.get());
+    break;
+  }
 
   default: {
     // The default branch works for attributes that have no arguments to import.
diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp
index 1f492b051e03..be7a850a2982 100644
--- a/clang/lib/AST/ASTStructuralEquivalence.cpp
+++ b/clang/lib/AST/ASTStructuralEquivalence.cpp
@@ -98,6 +98,8 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      QualType T1, QualType T2);
 static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      Decl *D1, Decl *D2);
+static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
+                                     const Stmt *S1, const Stmt *S2);
 static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      const TemplateArgument &Arg1,
                                      const TemplateArgument &Arg2);
@@ -437,12 +439,67 @@ class StmtComparer {
 };
 } // namespace
 
+static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
+                                     const UnaryOperator *E1,
+                                     const CXXOperatorCallExpr *E2) {
+  return UnaryOperator::getOverloadedOperator(E1->getOpcode()) ==
+             E2->getOperator() &&
+         IsStructurallyEquivalent(Context, E1->getSubExpr(), E2->getArg(0));
+}
+
+static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
+                                     const CXXOperatorCallExpr *E1,
+                                     const UnaryOperator *E2) {
+  return E1->getOperator() ==
+             UnaryOperator::getOverloadedOperator(E2->getOpcode()) &&
+         IsStructurallyEquivalent(Context, E1->getArg(0), E2->getSubExpr());
+}
+
+static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
+                                     const BinaryOperator *E1,
+                                     const CXXOperatorCallExpr *E2) {
+  return BinaryOperator::getOverloadedOperator(E1->getOpcode()) ==
+             E2->getOperator() &&
+         IsStructurallyEquivalent(Context, E1->getLHS(), E2->getArg(0)) &&
+         IsStructurallyEquivalent(Context, E1->getRHS(), E2->getArg(1));
+}
+
+static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
+                                     const CXXOperatorCallExpr *E1,
+                                     const BinaryOperator *E2) {
+  return E1->getOperator() ==
+             BinaryOperator::getOverloadedOperator(E2->getOpcode()) &&
+         IsStructurallyEquivalent(Context, E1->getArg(0), E2->getLHS()) &&
+         IsStructurallyEquivalent(Context, E1->getArg(1), E2->getRHS());
+}
+
 /// Determine structural equivalence of two statements.
 static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      const Stmt *S1, const Stmt *S2) {
   if (!S1 || !S2)
     return S1 == S2;
 
+  // Check for statements with similar syntax but different AST.
+  // A UnaryOperator node is more lightweight than a CXXOperatorCallExpr node.
+  // The more heavyweight node is only created if the definition-time name
+  // lookup had any results. The lookup results are stored CXXOperatorCallExpr
+  // only. The lookup results can be different in a "From" and "To" AST even if
+  // the compared structure is otherwise equivalent. For this reason we must
+  // treat a similar unary/binary operator node and CXXOperatorCall node as
+  // equivalent.
+  if (const auto *E2CXXOperatorCall = dyn_cast(S2)) {
+    if (const auto *E1Unary = dyn_cast(S1))
+      return IsStructurallyEquivalent(Context, E1Unary, E2CXXOperatorCall);
+    if (const auto *E1Binary = dyn_cast(S1))
+      return IsStructurallyEquivalent(Context, E1Binary, E2CXXOperatorCall);
+  }
+  if (const auto *E1CXXOperatorCall = dyn_cast(S1)) {
+    if (const auto *E2Unary = dyn_cast(S2))
+      return IsStructurallyEquivalent(Context, E1CXXOperatorCall, E2Unary);
+    if (const auto *E2Binary = dyn_cast(S2))
+      return IsStructurallyEquivalent(Context, E1CXXOperatorCall, E2Binary);
+  }
+
   // Compare the statements itself.
   StmtComparer Comparer(Context);
   if (!Comparer.IsEquivalent(S1, S2))
@@ -628,6 +685,9 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
     return IsStructurallyEquivalent(Context, Arg1.getAsExpr(),
                                     Arg2.getAsExpr());
 
+  case TemplateArgument::StructuralValue:
+    return Arg1.structurallyEquals(Arg2);
+
   case TemplateArgument::Pack:
     return IsStructurallyEquivalent(Context, Arg1.pack_elements(),
                                     Arg2.pack_elements());
@@ -1410,7 +1470,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
       Method1->isConst() == Method2->isConst() &&
       Method1->isVolatile() == Method2->isVolatile() &&
       Method1->isVirtual() == Method2->isVirtual() &&
-      Method1->isPure() == Method2->isPure() &&
+      Method1->isPureVirtual() == Method2->isPureVirtual() &&
       Method1->isDefaulted() == Method2->isDefaulted() &&
       Method1->isDeleted() == Method2->isDeleted();
   if (!PropertiesEqual)
@@ -1463,8 +1523,9 @@ IsStructurallyEquivalentLambdas(StructuralEquivalenceContext &Context,
 }
 
 /// Determine if context of a class is equivalent.
-static bool IsRecordContextStructurallyEquivalent(RecordDecl *D1,
-                                                  RecordDecl *D2) {
+static bool
+IsRecordContextStructurallyEquivalent(StructuralEquivalenceContext &Context,
+                                      RecordDecl *D1, RecordDecl *D2) {
   // The context should be completely equal, including anonymous and inline
   // namespaces.
   // We compare objects as part of full translation units, not subtrees of
@@ -1491,6 +1552,12 @@ static bool IsRecordContextStructurallyEquivalent(RecordDecl *D1,
         return false;
     }
 
+    if (auto *D1Spec = dyn_cast(DC1)) {
+      auto *D2Spec = dyn_cast(DC2);
+      if (!IsStructurallyEquivalent(Context, D1Spec, D2Spec))
+        return false;
+    }
+
     DC1 = DC1->getParent()->getNonTransparentContext();
     DC2 = DC2->getParent()->getNonTransparentContext();
   }
@@ -1544,7 +1611,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
   // If the records occur in different context (namespace), these should be
   // different. This is specially important if the definition of one or both
   // records is missing.
-  if (!IsRecordContextStructurallyEquivalent(D1, D2))
+  if (!IsRecordContextStructurallyEquivalent(Context, D1, D2))
     return false;
 
   // If both declarations are class template specializations, we know
diff --git a/clang/lib/AST/CMakeLists.txt b/clang/lib/AST/CMakeLists.txt
index fe3f8c485ec1..ebcb3952198a 100644
--- a/clang/lib/AST/CMakeLists.txt
+++ b/clang/lib/AST/CMakeLists.txt
@@ -75,6 +75,7 @@ add_clang_library(clangAST
   Interp/Function.cpp
   Interp/InterpBuiltin.cpp
   Interp/Floating.cpp
+  Interp/EvaluationResult.cpp
   Interp/Interp.cpp
   Interp/InterpBlock.cpp
   Interp/InterpFrame.cpp
diff --git a/clang/lib/AST/ComputeDependence.cpp b/clang/lib/AST/ComputeDependence.cpp
index 097753fd3267..584b58473294 100644
--- a/clang/lib/AST/ComputeDependence.cpp
+++ b/clang/lib/AST/ComputeDependence.cpp
@@ -603,6 +603,8 @@ ExprDependence clang::computeDependence(PredefinedExpr *E) {
 ExprDependence clang::computeDependence(CallExpr *E,
                                         llvm::ArrayRef PreArgs) {
   auto D = E->getCallee()->getDependence();
+  if (E->getType()->isDependentType())
+    D |= ExprDependence::Type;
   for (auto *A : llvm::ArrayRef(E->getArgs(), E->getNumArgs())) {
     if (A)
       D |= A->getDependence();
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 12e0a6faa4c3..26fdfa040796 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -343,6 +343,10 @@ LinkageComputer::getLVForTemplateArgumentList(ArrayRef Args,
       LV.merge(getTypeLinkageAndVisibility(Arg.getNullPtrType()));
       continue;
 
+    case TemplateArgument::StructuralValue:
+      LV.merge(getLVForValue(Arg.getAsStructuralValue(), computation));
+      continue;
+
     case TemplateArgument::Template:
     case TemplateArgument::TemplateExpansion:
       if (TemplateDecl *Template =
@@ -2835,7 +2839,7 @@ CharUnits VarDecl::getFlexibleArrayInitChars(const ASTContext &Ctx) const {
   if (!Ty || !Ty->getDecl()->hasFlexibleArrayMember())
     return CharUnits::Zero();
   auto *List = dyn_cast(getInit()->IgnoreParens());
-  if (!List)
+  if (!List || List->getNumInits() == 0)
     return CharUnits::Zero();
   const Expr *FlexibleInit = List->getInit(List->getNumInits() - 1);
   auto InitTy = Ctx.getAsConstantArrayType(FlexibleInit->getType());
@@ -3036,7 +3040,7 @@ FunctionDecl::FunctionDecl(Kind DK, ASTContext &C, DeclContext *DC,
   FunctionDeclBits.IsInline = isInlineSpecified;
   FunctionDeclBits.IsInlineSpecified = isInlineSpecified;
   FunctionDeclBits.IsVirtualAsWritten = false;
-  FunctionDeclBits.IsPure = false;
+  FunctionDeclBits.IsPureVirtual = false;
   FunctionDeclBits.HasInheritedPrototype = false;
   FunctionDeclBits.HasWrittenPrototype = true;
   FunctionDeclBits.IsDeleted = false;
@@ -3203,8 +3207,8 @@ void FunctionDecl::setBody(Stmt *B) {
     EndRangeLoc = B->getEndLoc();
 }
 
-void FunctionDecl::setPure(bool P) {
-  FunctionDeclBits.IsPure = P;
+void FunctionDecl::setIsPureVirtual(bool P) {
+  FunctionDeclBits.IsPureVirtual = P;
   if (P)
     if (auto *Parent = dyn_cast(getDeclContext()))
       Parent->markedVirtualFunctionPure();
@@ -5369,16 +5373,23 @@ void CapturedDecl::setBody(Stmt *B) { BodyAndNothrow.setPointer(B); }
 bool CapturedDecl::isNothrow() const { return BodyAndNothrow.getInt(); }
 void CapturedDecl::setNothrow(bool Nothrow) { BodyAndNothrow.setInt(Nothrow); }
 
+EnumConstantDecl::EnumConstantDecl(const ASTContext &C, DeclContext *DC,
+                                   SourceLocation L, IdentifierInfo *Id,
+                                   QualType T, Expr *E, const llvm::APSInt &V)
+    : ValueDecl(EnumConstant, DC, L, Id, T), Init((Stmt *)E) {
+  setInitVal(C, V);
+}
+
 EnumConstantDecl *EnumConstantDecl::Create(ASTContext &C, EnumDecl *CD,
                                            SourceLocation L,
                                            IdentifierInfo *Id, QualType T,
                                            Expr *E, const llvm::APSInt &V) {
-  return new (C, CD) EnumConstantDecl(CD, L, Id, T, E, V);
+  return new (C, CD) EnumConstantDecl(C, CD, L, Id, T, E, V);
 }
 
 EnumConstantDecl *
 EnumConstantDecl::CreateDeserialized(ASTContext &C, unsigned ID) {
-  return new (C, ID) EnumConstantDecl(nullptr, SourceLocation(), nullptr,
+  return new (C, ID) EnumConstantDecl(C, nullptr, SourceLocation(), nullptr,
                                       QualType(), nullptr, llvm::APSInt());
 }
 
diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp
index 5e03f0223d31..8163f9bdaf8d 100644
--- a/clang/lib/AST/DeclBase.cpp
+++ b/clang/lib/AST/DeclBase.cpp
@@ -29,7 +29,6 @@
 #include "clang/AST/Type.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
-#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/ObjCRuntime.h"
 #include "clang/Basic/PartialDiagnostic.h"
@@ -411,6 +410,79 @@ bool Decl::isFileContextDecl() const {
   return DC && DC->isFileContext();
 }
 
+bool Decl::isFlexibleArrayMemberLike(
+    ASTContext &Ctx, const Decl *D, QualType Ty,
+    LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel,
+    bool IgnoreTemplateOrMacroSubstitution) {
+  // For compatibility with existing code, we treat arrays of length 0 or
+  // 1 as flexible array members.
+  const auto *CAT = Ctx.getAsConstantArrayType(Ty);
+  if (CAT) {
+    using FAMKind = LangOptions::StrictFlexArraysLevelKind;
+
+    llvm::APInt Size = CAT->getSize();
+    if (StrictFlexArraysLevel == FAMKind::IncompleteOnly)
+      return false;
+
+    // GCC extension, only allowed to represent a FAM.
+    if (Size.isZero())
+      return true;
+
+    if (StrictFlexArraysLevel == FAMKind::ZeroOrIncomplete && Size.uge(1))
+      return false;
+
+    if (StrictFlexArraysLevel == FAMKind::OneZeroOrIncomplete && Size.uge(2))
+      return false;
+  } else if (!Ctx.getAsIncompleteArrayType(Ty)) {
+    return false;
+  }
+
+  if (const auto *OID = dyn_cast_if_present(D))
+    return OID->getNextIvar() == nullptr;
+
+  const auto *FD = dyn_cast_if_present(D);
+  if (!FD)
+    return false;
+
+  if (CAT) {
+    // GCC treats an array memeber of a union as an FAM if the size is one or
+    // zero.
+    llvm::APInt Size = CAT->getSize();
+    if (FD->getParent()->isUnion() && (Size.isZero() || Size.isOne()))
+      return true;
+  }
+
+  // Don't consider sizes resulting from macro expansions or template argument
+  // substitution to form C89 tail-padded arrays.
+  if (IgnoreTemplateOrMacroSubstitution) {
+    TypeSourceInfo *TInfo = FD->getTypeSourceInfo();
+    while (TInfo) {
+      TypeLoc TL = TInfo->getTypeLoc();
+
+      // Look through typedefs.
+      if (TypedefTypeLoc TTL = TL.getAsAdjusted()) {
+        const TypedefNameDecl *TDL = TTL.getTypedefNameDecl();
+        TInfo = TDL->getTypeSourceInfo();
+        continue;
+      }
+
+      if (auto CTL = TL.getAs()) {
+        if (const Expr *SizeExpr =
+                dyn_cast_if_present(CTL.getSizeExpr());
+            !SizeExpr || SizeExpr->getExprLoc().isMacroID())
+          return false;
+      }
+
+      break;
+    }
+  }
+
+  // Test that the field is the last in the structure.
+  RecordDecl::field_iterator FI(
+      DeclContext::decl_iterator(const_cast(FD)));
+  return ++FI == FD->getParent()->field_end();
+}
+
 TranslationUnitDecl *Decl::getTranslationUnitDecl() {
   if (auto *TUD = dyn_cast(this))
     return TUD;
@@ -930,20 +1002,14 @@ const AttrVec &Decl::getAttrs() const {
 
 Decl *Decl::castFromDeclContext (const DeclContext *D) {
   Decl::Kind DK = D->getDeclKind();
-  switch(DK) {
-#define DECL(NAME, BASE)
-#define DECL_CONTEXT(NAME) \
-    case Decl::NAME:       \
-      return static_cast(const_cast(D));
-#define DECL_CONTEXT_BASE(NAME)
-#include "clang/AST/DeclNodes.inc"
-    default:
+  switch (DK) {
 #define DECL(NAME, BASE)
-#define DECL_CONTEXT_BASE(NAME)                  \
-      if (DK >= first##NAME && DK <= last##NAME) \
-        return static_cast(const_cast(D));
+#define DECL_CONTEXT(NAME)                                                     \
+  case Decl::NAME:                                                             \
+    return static_cast(const_cast(D));
 #include "clang/AST/DeclNodes.inc"
-      llvm_unreachable("a decl that inherits DeclContext isn't handled");
+  default:
+    llvm_unreachable("a decl that inherits DeclContext isn't handled");
   }
 }
 
@@ -951,18 +1017,12 @@ DeclContext *Decl::castToDeclContext(const Decl *D) {
   Decl::Kind DK = D->getKind();
   switch(DK) {
 #define DECL(NAME, BASE)
-#define DECL_CONTEXT(NAME) \
-    case Decl::NAME:       \
-      return static_cast(const_cast(D));
-#define DECL_CONTEXT_BASE(NAME)
-#include "clang/AST/DeclNodes.inc"
-    default:
-#define DECL(NAME, BASE)
-#define DECL_CONTEXT_BASE(NAME)                                   \
-      if (DK >= first##NAME && DK <= last##NAME)                  \
-        return static_cast(const_cast(D));
+#define DECL_CONTEXT(NAME)                                                     \
+  case Decl::NAME:                                                             \
+    return static_cast(const_cast(D));
 #include "clang/AST/DeclNodes.inc"
-      llvm_unreachable("a decl that inherits DeclContext isn't handled");
+  default:
+    llvm_unreachable("a decl that inherits DeclContext isn't handled");
   }
 }
 
@@ -1129,20 +1189,14 @@ DeclContext::DeclContext(Decl::Kind K) {
 }
 
 bool DeclContext::classof(const Decl *D) {
-  switch (D->getKind()) {
+  Decl::Kind DK = D->getKind();
+  switch (DK) {
 #define DECL(NAME, BASE)
 #define DECL_CONTEXT(NAME) case Decl::NAME:
-#define DECL_CONTEXT_BASE(NAME)
 #include "clang/AST/DeclNodes.inc"
-      return true;
-    default:
-#define DECL(NAME, BASE)
-#define DECL_CONTEXT_BASE(NAME)                 \
-      if (D->getKind() >= Decl::first##NAME &&  \
-          D->getKind() <= Decl::last##NAME)     \
-        return true;
-#include "clang/AST/DeclNodes.inc"
-      return false;
+    return true;
+  default:
+    return false;
   }
 }
 
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index c944862fcefe..117e802dae2d 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -587,6 +587,19 @@ bool CXXRecordDecl::isTriviallyCopyable() const {
   return true;
 }
 
+bool CXXRecordDecl::isTriviallyCopyConstructible() const {
+
+  //   A trivially copy constructible class is a class that:
+  //   -- has no non-trivial copy constructors,
+  if (hasNonTrivialCopyConstructor())
+    return false;
+  //   -- has a trivial destructor.
+  if (!hasTrivialDestructor())
+    return false;
+
+  return true;
+}
+
 void CXXRecordDecl::markedVirtualFunctionPure() {
   // C++ [class.abstract]p2:
   //   A class is abstract if it has at least one pure virtual function.
@@ -1370,6 +1383,31 @@ void CXXRecordDecl::addedMember(Decl *D) {
   }
 }
 
+bool CXXRecordDecl::isLiteral() const {
+  const LangOptions &LangOpts = getLangOpts();
+  if (!(LangOpts.CPlusPlus20 ? hasConstexprDestructor()
+                             : hasTrivialDestructor()))
+    return false;
+
+  if (hasNonLiteralTypeFieldsOrBases()) {
+    // CWG2598
+    // is an aggregate union type that has either no variant
+    // members or at least one variant member of non-volatile literal type,
+    if (!isUnion())
+      return false;
+    bool HasAtLeastOneLiteralMember =
+        fields().empty() || any_of(fields(), [this](const FieldDecl *D) {
+          return !D->getType().isVolatileQualified() &&
+                 D->getType()->isLiteralType(getASTContext());
+        });
+    if (!HasAtLeastOneLiteralMember)
+      return false;
+  }
+
+  return isAggregate() || (isLambda() && LangOpts.CPlusPlus17) ||
+         hasConstexprNonCopyMoveConstructor() || hasTrivialDefaultConstructor();
+}
+
 void CXXRecordDecl::addedSelectedDestructor(CXXDestructorDecl *DD) {
   DD->setIneligibleOrNotSelected(false);
   addedEligibleSpecialMemberFunction(DD, SMF_Destructor);
@@ -2041,7 +2079,7 @@ void CXXRecordDecl::completeDefinition(CXXFinalOverriderMap *FinalOverriders) {
         //   A class is abstract if it contains or inherits at least one
         //   pure virtual function for which the final overrider is pure
         //   virtual.
-        if (SO->second.front().Method->isPure()) {
+        if (SO->second.front().Method->isPureVirtual()) {
           data().Abstract = true;
           Done = true;
           break;
@@ -2260,7 +2298,7 @@ CXXMethodDecl *CXXMethodDecl::getDevirtualizedMethod(const Expr *Base,
   // If the member function is marked 'final', we know that it can't be
   // overridden and can therefore devirtualize it unless it's pure virtual.
   if (hasAttr())
-    return isPure() ? nullptr : this;
+    return isPureVirtual() ? nullptr : this;
 
   // If Base is unknown, we cannot devirtualize.
   if (!Base)
@@ -2289,7 +2327,7 @@ CXXMethodDecl *CXXMethodDecl::getDevirtualizedMethod(const Expr *Base,
   // If that method is pure virtual, we can't devirtualize. If this code is
   // reached, the result would be UB, not a direct call to the derived class
   // function, and we can't assume the derived class function is defined.
-  if (DevirtualizedMethod->isPure())
+  if (DevirtualizedMethod->isPureVirtual())
     return nullptr;
 
   // If that method is marked final, we can devirtualize it.
diff --git a/clang/lib/AST/DeclPrinter.cpp b/clang/lib/AST/DeclPrinter.cpp
index 24da6f2ef32b..822ac12c4c7d 100644
--- a/clang/lib/AST/DeclPrinter.cpp
+++ b/clang/lib/AST/DeclPrinter.cpp
@@ -871,7 +871,7 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
 
   prettyPrintAttributes(D, Out, AttrPrintLoc::Right);
 
-  if (D->isPure())
+  if (D->isPureVirtual())
     Out << " = 0";
   else if (D->isDeletedAsWritten())
     Out << " = delete";
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index a90f92d07f86..f1efa98e175e 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -205,85 +205,22 @@ bool Expr::isKnownToHaveBooleanValue(bool Semantic) const {
 }
 
 bool Expr::isFlexibleArrayMemberLike(
-    ASTContext &Context,
+    ASTContext &Ctx,
     LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel,
     bool IgnoreTemplateOrMacroSubstitution) const {
-
-  // For compatibility with existing code, we treat arrays of length 0 or
-  // 1 as flexible array members.
-  const auto *CAT = Context.getAsConstantArrayType(getType());
-  if (CAT) {
-    llvm::APInt Size = CAT->getSize();
-
-    using FAMKind = LangOptions::StrictFlexArraysLevelKind;
-
-    if (StrictFlexArraysLevel == FAMKind::IncompleteOnly)
-      return false;
-
-    // GCC extension, only allowed to represent a FAM.
-    if (Size == 0)
-      return true;
-
-    if (StrictFlexArraysLevel == FAMKind::ZeroOrIncomplete && Size.uge(1))
-      return false;
-
-    if (StrictFlexArraysLevel == FAMKind::OneZeroOrIncomplete && Size.uge(2))
-      return false;
-  } else if (!Context.getAsIncompleteArrayType(getType()))
-    return false;
-
   const Expr *E = IgnoreParens();
+  const Decl *D = nullptr;
 
-  const NamedDecl *ND = nullptr;
-  if (const auto *DRE = dyn_cast(E))
-    ND = DRE->getDecl();
-  else if (const auto *ME = dyn_cast(E))
-    ND = ME->getMemberDecl();
+  if (const auto *ME = dyn_cast(E))
+    D = ME->getMemberDecl();
+  else if (const auto *DRE = dyn_cast(E))
+    D = DRE->getDecl();
   else if (const auto *IRE = dyn_cast(E))
-    return IRE->getDecl()->getNextIvar() == nullptr;
+    D = IRE->getDecl();
 
-  if (!ND)
-    return false;
-
-  // A flexible array member must be the last member in the class.
-  // FIXME: If the base type of the member expr is not FD->getParent(),
-  // this should not be treated as a flexible array member access.
-  if (const auto *FD = dyn_cast(ND)) {
-    // GCC treats an array memeber of a union as an FAM if the size is one or
-    // zero.
-    if (CAT) {
-      llvm::APInt Size = CAT->getSize();
-      if (FD->getParent()->isUnion() && (Size.isZero() || Size.isOne()))
-        return true;
-    }
-
-    // Don't consider sizes resulting from macro expansions or template argument
-    // substitution to form C89 tail-padded arrays.
-    if (IgnoreTemplateOrMacroSubstitution) {
-      TypeSourceInfo *TInfo = FD->getTypeSourceInfo();
-      while (TInfo) {
-        TypeLoc TL = TInfo->getTypeLoc();
-        // Look through typedefs.
-        if (TypedefTypeLoc TTL = TL.getAsAdjusted()) {
-          const TypedefNameDecl *TDL = TTL.getTypedefNameDecl();
-          TInfo = TDL->getTypeSourceInfo();
-          continue;
-        }
-        if (ConstantArrayTypeLoc CTL = TL.getAs()) {
-          const Expr *SizeExpr = dyn_cast(CTL.getSizeExpr());
-          if (!SizeExpr || SizeExpr->getExprLoc().isMacroID())
-            return false;
-        }
-        break;
-      }
-    }
-
-    RecordDecl::field_iterator FI(
-        DeclContext::decl_iterator(const_cast(FD)));
-    return ++FI == FD->getParent()->field_end();
-  }
-
-  return false;
+  return Decl::isFlexibleArrayMemberLike(Ctx, D, E->getType(),
+                                         StrictFlexArraysLevel,
+                                         IgnoreTemplateOrMacroSubstitution);
 }
 
 const ValueDecl *
@@ -753,7 +690,7 @@ std::string PredefinedExpr::ComputeName(PredefinedIdentKind IK,
 
         if (!Buffer.empty() && Buffer.front() == '\01')
           return std::string(Buffer.substr(1));
-        return std::string(Buffer.str());
+        return std::string(Buffer);
       }
       return std::string(ND->getIdentifier()->getName());
     }
@@ -1049,7 +986,7 @@ std::string FixedPointLiteral::getValueAsString(unsigned Radix) const {
   SmallString<64> S;
   FixedPointValueToString(
       S, llvm::APSInt::getUnsigned(getValue().getZExtValue()), Scale);
-  return std::string(S.str());
+  return std::string(S);
 }
 
 void CharacterLiteral::print(unsigned Val, CharacterLiteralKind Kind,
@@ -2282,7 +2219,10 @@ SourceLocExpr::SourceLocExpr(const ASTContext &Ctx, SourceLocIdentKind Kind,
     : Expr(SourceLocExprClass, ResultTy, VK_PRValue, OK_Ordinary),
       BuiltinLoc(BLoc), RParenLoc(RParenLoc), ParentContext(ParentContext) {
   SourceLocExprBits.Kind = llvm::to_underlying(Kind);
-  setDependence(ExprDependence::None);
+  // In dependent contexts, function names may change.
+  setDependence(MayBeDependent(Kind) && ParentContext->isDependentContext()
+                    ? ExprDependence::Value
+                    : ExprDependence::None);
 }
 
 StringRef SourceLocExpr::getBuiltinStr() const {
diff --git a/clang/lib/AST/ExprCXX.cpp b/clang/lib/AST/ExprCXX.cpp
index 83af7998f683..e61c11dffd88 100644
--- a/clang/lib/AST/ExprCXX.cpp
+++ b/clang/lib/AST/ExprCXX.cpp
@@ -194,14 +194,14 @@ CXXNewExpr::CXXNewExpr(bool IsGlobalNew, FunctionDecl *OperatorNew,
       DirectInitRange(DirectInitRange) {
 
   assert((Initializer != nullptr ||
-          InitializationStyle == CXXNewInitializationStyle::None ||
-          InitializationStyle == CXXNewInitializationStyle::Implicit) &&
-         "Only NoInit can have no initializer!");
+          InitializationStyle == CXXNewInitializationStyle::None) &&
+         "Only CXXNewInitializationStyle::None can have no initializer!");
 
   CXXNewExprBits.IsGlobalNew = IsGlobalNew;
   CXXNewExprBits.IsArray = ArraySize.has_value();
   CXXNewExprBits.ShouldPassAlignment = ShouldPassAlignment;
   CXXNewExprBits.UsualArrayDeleteWantsSize = UsualArrayDeleteWantsSize;
+  CXXNewExprBits.HasInitializer = Initializer != nullptr;
   CXXNewExprBits.StoredInitializationStyle =
       llvm::to_underlying(InitializationStyle);
   bool IsParenTypeId = TypeIdParens.isValid();
@@ -219,10 +219,10 @@ CXXNewExpr::CXXNewExpr(bool IsGlobalNew, FunctionDecl *OperatorNew,
     getTrailingObjects()[0] = TypeIdParens;
 
   switch (getInitializationStyle()) {
-  case CXXNewInitializationStyle::Call:
+  case CXXNewInitializationStyle::Parens:
     this->Range.setEnd(DirectInitRange.getEnd());
     break;
-  case CXXNewInitializationStyle::List:
+  case CXXNewInitializationStyle::Braces:
     this->Range.setEnd(getInitializer()->getSourceRange().getEnd());
     break;
   default:
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 3f595bbe74e7..0e6e0cc6db4c 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -5842,7 +5842,7 @@ static const CXXMethodDecl *HandleVirtualDispatch(
   // C++2a [class.abstract]p6:
   //   the effect of making a virtual call to a pure virtual function [...] is
   //   undefined
-  if (Callee->isPure()) {
+  if (Callee->isPureVirtual()) {
     Info.FFDiag(E, diag::note_constexpr_pure_virtual_call, 1) << Callee;
     Info.Note(Callee->getLocation(), diag::note_declared_at);
     return nullptr;
@@ -15444,11 +15444,13 @@ static bool EvaluateAsRValue(EvalInfo &Info, const Expr *E, APValue &Result) {
   if (Info.EnableNewConstInterp) {
     if (!Info.Ctx.getInterpContext().evaluateAsRValue(Info, E, Result))
       return false;
-  } else {
-    if (!::Evaluate(Result, Info, E))
-      return false;
+    return CheckConstantExpression(Info, E->getExprLoc(), E->getType(), Result,
+                                   ConstantExprKind::Normal);
   }
 
+  if (!::Evaluate(Result, Info, E))
+    return false;
+
   // Implicit lvalue-to-rvalue cast.
   if (E->isGLValue()) {
     LValue LV;
@@ -15676,6 +15678,13 @@ bool Expr::EvaluateAsConstantExpr(EvalResult &Result, const ASTContext &Ctx,
   EvalInfo Info(Ctx, Result, EM);
   Info.InConstantContext = true;
 
+  if (Info.EnableNewConstInterp) {
+    if (!Info.Ctx.getInterpContext().evaluate(Info, this, Result.Val))
+      return false;
+    return CheckConstantExpression(Info, getExprLoc(),
+                                   getStorageType(Ctx, this), Result.Val, Kind);
+  }
+
   // The type of the object we're initializing is 'const T' for a class NTTP.
   QualType T = getType();
   if (Kind == ConstantExprKind::ClassTemplateArgument)
@@ -15751,18 +15760,36 @@ bool Expr::EvaluateAsInitializer(APValue &Value, const ASTContext &Ctx,
   Info.setEvaluatingDecl(VD, Value);
   Info.InConstantContext = IsConstantInitialization;
 
+  SourceLocation DeclLoc = VD->getLocation();
+  QualType DeclTy = VD->getType();
+
   if (Info.EnableNewConstInterp) {
     auto &InterpCtx = const_cast(Ctx).getInterpContext();
     if (!InterpCtx.evaluateAsInitializer(Info, VD, Value))
       return false;
+
+    return CheckConstantExpression(Info, DeclLoc, DeclTy, Value,
+                                   ConstantExprKind::Normal);
   } else {
     LValue LVal;
     LVal.set(VD);
 
-    if (!EvaluateInPlace(Value, Info, LVal, this,
-                         /*AllowNonLiteralTypes=*/true) ||
-        EStatus.HasSideEffects)
-      return false;
+    {
+      // C++23 [intro.execution]/p5
+      // A full-expression is ... an init-declarator ([dcl.decl]) or a
+      // mem-initializer.
+      // So we need to make sure temporary objects are destroyed after having
+      // evaluated the expression (per C++23 [class.temporary]/p4).
+      //
+      // FIXME: Otherwise this may break test/Modules/pr68702.cpp because the
+      // serialization code calls ParmVarDecl::getDefaultArg() which strips the
+      // outermost FullExpr, such as ExprWithCleanups.
+      FullExpressionRAII Scope(Info);
+      if (!EvaluateInPlace(Value, Info, LVal, this,
+                           /*AllowNonLiteralTypes=*/true) ||
+          EStatus.HasSideEffects)
+        return false;
+    }
 
     // At this point, any lifetime-extended temporaries are completely
     // initialized.
@@ -15772,8 +15799,6 @@ bool Expr::EvaluateAsInitializer(APValue &Value, const ASTContext &Ctx,
       llvm_unreachable("Unhandled cleanup; missing full expression marker?");
   }
 
-  SourceLocation DeclLoc = VD->getLocation();
-  QualType DeclTy = VD->getType();
   return CheckConstantExpression(Info, DeclLoc, DeclTy, Value,
                                  ConstantExprKind::Normal) &&
          CheckMemoryLeaks(Info);
diff --git a/clang/lib/AST/FormatString.cpp b/clang/lib/AST/FormatString.cpp
index e0c9e18cfe3a..c5d14b4af7ff 100644
--- a/clang/lib/AST/FormatString.cpp
+++ b/clang/lib/AST/FormatString.cpp
@@ -488,7 +488,6 @@ ArgType::matchesType(ASTContext &C, QualType argTy) const {
                 return NoMatchPromotionTypeConfusion;
               break;
             case BuiltinType::Half:
-            case BuiltinType::Float16:
             case BuiltinType::Float:
               if (T == C.DoubleTy)
                 return MatchPromotion;
diff --git a/clang/lib/AST/Interp/ByteCodeEmitter.cpp b/clang/lib/AST/Interp/ByteCodeEmitter.cpp
index 045263447cbc..fd2a92d9d3f9 100644
--- a/clang/lib/AST/Interp/ByteCodeEmitter.cpp
+++ b/clang/lib/AST/Interp/ByteCodeEmitter.cpp
@@ -20,8 +20,7 @@
 using namespace clang;
 using namespace clang::interp;
 
-Expected
-ByteCodeEmitter::compileFunc(const FunctionDecl *FuncDecl) {
+Function *ByteCodeEmitter::compileFunc(const FunctionDecl *FuncDecl) {
   // Set up argument indices.
   unsigned ParamOffset = 0;
   SmallVector ParamTypes;
@@ -120,10 +119,6 @@ ByteCodeEmitter::compileFunc(const FunctionDecl *FuncDecl) {
 
   // Compile the function body.
   if (!IsEligibleForCompilation || !visitFunc(FuncDecl)) {
-    // Return a dummy function if compilation failed.
-    if (BailLocation)
-      return llvm::make_error(*BailLocation);
-
     Func->setIsFullyCompiled(true);
     return Func;
   }
@@ -183,12 +178,6 @@ int32_t ByteCodeEmitter::getOffset(LabelTy Label) {
   return 0ull;
 }
 
-bool ByteCodeEmitter::bail(const SourceLocation &Loc) {
-  if (!BailLocation)
-    BailLocation = Loc;
-  return false;
-}
-
 /// Helper to write bytecode and bail out if 32-bit offsets become invalid.
 /// Pointers will be automatically marshalled as 32-bit IDs.
 template 
diff --git a/clang/lib/AST/Interp/ByteCodeEmitter.h b/clang/lib/AST/Interp/ByteCodeEmitter.h
index 5520f8c30061..03de286582c9 100644
--- a/clang/lib/AST/Interp/ByteCodeEmitter.h
+++ b/clang/lib/AST/Interp/ByteCodeEmitter.h
@@ -17,7 +17,6 @@
 #include "PrimType.h"
 #include "Program.h"
 #include "Source.h"
-#include "llvm/Support/Error.h"
 
 namespace clang {
 namespace interp {
@@ -32,7 +31,7 @@ class ByteCodeEmitter {
 
 public:
   /// Compiles the function into the module.
-  llvm::Expected compileFunc(const FunctionDecl *FuncDecl);
+  Function *compileFunc(const FunctionDecl *FuncDecl);
 
 protected:
   ByteCodeEmitter(Context &Ctx, Program &P) : Ctx(Ctx), P(P) {}
@@ -49,11 +48,6 @@ class ByteCodeEmitter {
   virtual bool visitExpr(const Expr *E) = 0;
   virtual bool visitDecl(const VarDecl *E) = 0;
 
-  /// Bails out if a given node cannot be compiled.
-  bool bail(const Stmt *S) { return bail(S->getBeginLoc()); }
-  bool bail(const Decl *D) { return bail(D->getBeginLoc()); }
-  bool bail(const SourceLocation &Loc);
-
   /// Emits jumps.
   bool jumpTrue(const LabelTy &Label);
   bool jumpFalse(const LabelTy &Label);
@@ -81,8 +75,6 @@ class ByteCodeEmitter {
   LabelTy NextLabel = 0;
   /// Offset of the next local variable.
   unsigned NextLocalOffset = 0;
-  /// Location of a failure.
-  std::optional BailLocation;
   /// Label information for linker.
   llvm::DenseMap LabelOffsets;
   /// Location of label relocations.
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index e6b3097a80d8..cfcef067b92b 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -114,6 +114,8 @@ bool ByteCodeExprGen::VisitCastExpr(const CastExpr *CE) {
   }
 
   case CK_FloatingCast: {
+    if (DiscardResult)
+      return this->discard(SubExpr);
     if (!this->visit(SubExpr))
       return false;
     const auto *TargetSemantics = &Ctx.getFloatSemantics(CE->getType());
@@ -121,6 +123,8 @@ bool ByteCodeExprGen::VisitCastExpr(const CastExpr *CE) {
   }
 
   case CK_IntegralToFloating: {
+    if (DiscardResult)
+      return this->discard(SubExpr);
     std::optional FromT = classify(SubExpr->getType());
     if (!FromT)
       return false;
@@ -135,6 +139,9 @@ bool ByteCodeExprGen::VisitCastExpr(const CastExpr *CE) {
 
   case CK_FloatingToBoolean:
   case CK_FloatingToIntegral: {
+    if (DiscardResult)
+      return this->discard(SubExpr);
+
     std::optional ToT = classify(CE->getType());
 
     if (!ToT)
@@ -280,6 +287,33 @@ bool ByteCodeExprGen::VisitCastExpr(const CastExpr *CE) {
     return true;
   }
 
+  case CK_IntegralComplexToReal:
+  case CK_FloatingComplexToReal:
+    return this->emitComplexReal(SubExpr);
+
+  case CK_IntegralRealToComplex:
+  case CK_FloatingRealToComplex: {
+    // We're creating a complex value here, so we need to
+    // allocate storage for it.
+    if (!Initializing) {
+      std::optional LocalIndex =
+          allocateLocal(CE, /*IsExtended=*/true);
+      if (!LocalIndex)
+        return false;
+      if (!this->emitGetPtrLocal(*LocalIndex, CE))
+        return false;
+    }
+
+    // Init the complex value to {SubExpr, 0}.
+    if (!this->visitArrayElemInit(0, SubExpr))
+      return false;
+    // Zero-init the second element.
+    PrimType T = classifyPrim(SubExpr->getType());
+    if (!this->visitZeroInitializer(T, SubExpr->getType(), SubExpr))
+      return false;
+    return this->emitInitElem(T, 1, SubExpr);
+  }
+
   case CK_ToVoid:
     return discard(SubExpr);
 
@@ -364,7 +398,7 @@ bool ByteCodeExprGen::VisitBinaryOperator(const BinaryOperator *BO) {
   }
 
   if (!LT || !RT || !T)
-    return this->bail(BO);
+    return false;
 
   // Pointer arithmetic special case.
   if (BO->getOpcode() == BO_Add || BO->getOpcode() == BO_Sub) {
@@ -444,7 +478,7 @@ bool ByteCodeExprGen::VisitBinaryOperator(const BinaryOperator *BO) {
   case BO_LAnd:
     llvm_unreachable("Already handled earlier");
   default:
-    return this->bail(BO);
+    return false;
   }
 
   llvm_unreachable("Unhandled binary op");
@@ -497,7 +531,7 @@ bool ByteCodeExprGen::VisitPointerArithBinOp(const BinaryOperator *E) {
   else if (Op == BO_Sub)
     return this->emitSubOffset(OffsetType, E);
 
-  return this->bail(E);
+  return false;
 }
 
 template 
@@ -835,6 +869,10 @@ bool ByteCodeExprGen::VisitInitListExpr(const InitListExpr *E) {
 
   if (T->isAnyComplexType()) {
     unsigned NumInits = E->getNumInits();
+
+    if (NumInits == 1)
+      return this->delegate(E->inits()[0]);
+
     QualType ElemQT = E->getType()->getAs()->getElementType();
     PrimType ElemT = classifyPrim(ElemQT);
     if (NumInits == 0) {
@@ -2023,7 +2061,7 @@ bool ByteCodeExprGen::dereference(
   }
 
   if (LV->getType()->isAnyComplexType())
-    return visit(LV);
+    return this->delegate(LV);
 
   return false;
 }
@@ -2330,7 +2368,7 @@ bool ByteCodeExprGen::visitDecl(const VarDecl *VD) {
     auto GlobalIndex = P.getGlobal(VD);
     assert(GlobalIndex); // visitVarDecl() didn't return false.
     if (VarT) {
-      if (!this->emitGetGlobal(*VarT, *GlobalIndex, VD))
+      if (!this->emitGetGlobalUnchecked(*VarT, *GlobalIndex, VD))
         return false;
     } else {
       if (!this->emitGetPtrGlobal(*GlobalIndex, VD))
@@ -2351,7 +2389,9 @@ bool ByteCodeExprGen::visitDecl(const VarDecl *VD) {
   // Return the value
   if (VarT)
     return this->emitRet(*VarT, VD);
-  return this->emitRetValue(VD);
+
+  // Return non-primitive values as pointers here.
+  return this->emitRet(PT_Ptr, VD);
 }
 
 template 
@@ -2371,7 +2411,7 @@ bool ByteCodeExprGen::visitVarDecl(const VarDecl *VD) {
     std::optional GlobalIndex = P.createGlobal(VD, Init);
 
     if (!GlobalIndex)
-      return this->bail(VD);
+      return false;
 
     assert(Init);
     {
@@ -2760,23 +2800,16 @@ bool ByteCodeExprGen::VisitUnaryOperator(const UnaryOperator *E) {
     if (!this->visit(SubExpr))
       return false;
     return DiscardResult ? this->emitPop(*T, E) : this->emitComp(*T, E);
-  case UO_Real: { // __real x
-    assert(!T);
-    if (!this->visit(SubExpr))
-      return false;
-    if (!this->emitConstUint8(0, E))
-      return false;
-    if (!this->emitArrayElemPtrPopUint8(E))
-      return false;
-
-    // Since our _Complex implementation does not map to a primitive type,
-    // we sometimes have to do the lvalue-to-rvalue conversion here manually.
-    if (!SubExpr->isLValue())
-      return this->emitLoadPop(classifyPrim(E->getType()), E);
-    return true;
-  }
+  case UO_Real: // __real x
+    if (T)
+      return this->delegate(SubExpr);
+    return this->emitComplexReal(SubExpr);
   case UO_Imag: { // __imag x
-    assert(!T);
+    if (T) {
+      if (!this->discard(SubExpr))
+        return false;
+      return this->visitZeroInitializer(*T, SubExpr->getType(), SubExpr);
+    }
     if (!this->visit(SubExpr))
       return false;
     if (!this->emitConstUint8(1, E))
@@ -2941,10 +2974,32 @@ bool ByteCodeExprGen::emitPrimCast(PrimType FromT, PrimType ToT,
   return false;
 }
 
+/// Emits __real(SubExpr)
+template 
+bool ByteCodeExprGen::emitComplexReal(const Expr *SubExpr) {
+  assert(SubExpr->getType()->isAnyComplexType());
+
+  if (DiscardResult)
+    return this->discard(SubExpr);
+
+  if (!this->visit(SubExpr))
+    return false;
+  if (!this->emitConstUint8(0, SubExpr))
+    return false;
+  if (!this->emitArrayElemPtrPopUint8(SubExpr))
+    return false;
+
+  // Since our _Complex implementation does not map to a primitive type,
+  // we sometimes have to do the lvalue-to-rvalue conversion here manually.
+  if (!SubExpr->isLValue())
+    return this->emitLoadPop(*classifyComplexElementType(SubExpr->getType()),
+                             SubExpr);
+  return true;
+}
+
 /// When calling this, we have a pointer of the local-to-destroy
 /// on the stack.
 /// Emit destruction of record types (or arrays of record types).
-/// FIXME: Handle virtual destructors.
 template 
 bool ByteCodeExprGen::emitRecordDestruction(const Descriptor *Desc) {
   assert(Desc);
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.h b/clang/lib/AST/Interp/ByteCodeExprGen.h
index bbb13e97e725..df4cb736299c 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.h
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.h
@@ -191,10 +191,6 @@ class ByteCodeExprGen : public ConstStmtVisitor, bool>,
     if (!visitInitializer(Init))
       return false;
 
-    if ((Init->getType()->isArrayType() || Init->getType()->isRecordType()) &&
-        !this->emitCheckGlobalCtor(Init))
-      return false;
-
     return this->emitPopPtr(Init);
   }
 
@@ -294,6 +290,8 @@ class ByteCodeExprGen : public ConstStmtVisitor, bool>,
     return this->classify(ElemType);
   }
 
+  bool emitComplexReal(const Expr *SubExpr);
+
   bool emitRecordDestruction(const Descriptor *Desc);
   unsigned collectBaseOffset(const RecordType *BaseType,
                              const RecordType *DerivedType);
diff --git a/clang/lib/AST/Interp/ByteCodeStmtGen.cpp b/clang/lib/AST/Interp/ByteCodeStmtGen.cpp
index b1ab5fcf9cb6..a2d8c4e13010 100644
--- a/clang/lib/AST/Interp/ByteCodeStmtGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeStmtGen.cpp
@@ -142,6 +142,27 @@ bool ByteCodeStmtGen::visitFunc(const FunctionDecl *F) {
   // Classify the return type.
   ReturnType = this->classify(F->getReturnType());
 
+  auto emitFieldInitializer = [&](const Record::Field *F, unsigned FieldOffset,
+                                  const Expr *InitExpr) -> bool {
+    if (std::optional T = this->classify(InitExpr)) {
+      if (!this->visit(InitExpr))
+        return false;
+
+      if (F->isBitField())
+        return this->emitInitThisBitField(*T, F, FieldOffset, InitExpr);
+      return this->emitInitThisField(*T, FieldOffset, InitExpr);
+    }
+    // Non-primitive case. Get a pointer to the field-to-initialize
+    // on the stack and call visitInitialzer() for it.
+    if (!this->emitGetPtrThisField(FieldOffset, InitExpr))
+      return false;
+
+    if (!this->visitInitializer(InitExpr))
+      return false;
+
+    return this->emitPopPtr(InitExpr);
+  };
+
   // Emit custom code if this is a lambda static invoker.
   if (const auto *MD = dyn_cast(F);
       MD && MD->isLambdaStaticInvoker())
@@ -162,29 +183,8 @@ bool ByteCodeStmtGen::visitFunc(const FunctionDecl *F) {
       if (const FieldDecl *Member = Init->getMember()) {
         const Record::Field *F = R->getField(Member);
 
-        if (std::optional T = this->classify(InitExpr)) {
-          if (!this->visit(InitExpr))
-            return false;
-
-          if (F->isBitField()) {
-            if (!this->emitInitThisBitField(*T, F, InitExpr))
-              return false;
-          } else {
-            if (!this->emitInitThisField(*T, F->Offset, InitExpr))
-              return false;
-          }
-        } else {
-          // Non-primitive case. Get a pointer to the field-to-initialize
-          // on the stack and call visitInitialzer() for it.
-          if (!this->emitGetPtrThisField(F->Offset, InitExpr))
-            return false;
-
-          if (!this->visitInitializer(InitExpr))
-            return false;
-
-          if (!this->emitPopPtr(InitExpr))
-            return false;
-        }
+        if (!emitFieldInitializer(F, F->Offset, InitExpr))
+          return false;
       } else if (const Type *Base = Init->getBaseClass()) {
         // Base class initializer.
         // Get This Base and call initializer on it.
@@ -198,6 +198,26 @@ bool ByteCodeStmtGen::visitFunc(const FunctionDecl *F) {
           return false;
         if (!this->emitInitPtrPop(InitExpr))
           return false;
+      } else if (const IndirectFieldDecl *IFD = Init->getIndirectMember()) {
+        assert(IFD->getChainingSize() >= 2);
+
+        unsigned NestedFieldOffset = 0;
+        const Record::Field *NestedField = nullptr;
+        for (const NamedDecl *ND : IFD->chain()) {
+          const auto *FD = cast(ND);
+          const Record *FieldRecord =
+              this->P.getOrCreateRecord(FD->getParent());
+          assert(FieldRecord);
+
+          NestedField = FieldRecord->getField(FD);
+          assert(NestedField);
+
+          NestedFieldOffset += NestedField->Offset;
+        }
+        assert(NestedField);
+
+        if (!emitFieldInitializer(NestedField, NestedFieldOffset, InitExpr))
+          return false;
       } else {
         assert(Init->isDelegatingInitializer());
         if (!this->emitThis(InitExpr))
@@ -262,7 +282,7 @@ bool ByteCodeStmtGen::visitStmt(const Stmt *S) {
   default: {
     if (auto *Exp = dyn_cast(S))
       return this->discard(Exp);
-    return this->bail(S);
+    return false;
   }
   }
 }
diff --git a/clang/lib/AST/Interp/Context.cpp b/clang/lib/AST/Interp/Context.cpp
index 17abb7163583..75a300bcbace 100644
--- a/clang/lib/AST/Interp/Context.cpp
+++ b/clang/lib/AST/Interp/Context.cpp
@@ -30,18 +30,8 @@ Context::~Context() {}
 bool Context::isPotentialConstantExpr(State &Parent, const FunctionDecl *FD) {
   assert(Stk.empty());
   Function *Func = P->getFunction(FD);
-  if (!Func || !Func->hasBody()) {
-    if (auto R = ByteCodeStmtGen(*this, *P).compileFunc(FD)) {
-      Func = *R;
-    } else {
-      handleAllErrors(R.takeError(), [&Parent](ByteCodeGenError &Err) {
-        Parent.FFDiag(Err.getRange().getBegin(),
-                      diag::err_experimental_clang_interp_failed)
-            << Err.getRange();
-      });
-      return false;
-    }
-  }
+  if (!Func || !Func->hasBody())
+    Func = ByteCodeStmtGen(*this, *P).compileFunc(FD);
 
   APValue DummyResult;
   if (!Run(Parent, Func, DummyResult)) {
@@ -54,36 +44,90 @@ bool Context::isPotentialConstantExpr(State &Parent, const FunctionDecl *FD) {
 bool Context::evaluateAsRValue(State &Parent, const Expr *E, APValue &Result) {
   assert(Stk.empty());
   ByteCodeExprGen C(*this, *P, Parent, Stk, Result);
-  if (Check(Parent, C.interpretExpr(E))) {
-    assert(Stk.empty());
-#ifndef NDEBUG
-    // Make sure we don't rely on some value being still alive in
-    // InterpStack memory.
+
+  auto Res = C.interpretExpr(E);
+
+  if (Res.isInvalid()) {
     Stk.clear();
+    return false;
+  }
+
+  assert(Stk.empty());
+#ifndef NDEBUG
+  // Make sure we don't rely on some value being still alive in
+  // InterpStack memory.
+  Stk.clear();
 #endif
-    return true;
+
+  // Implicit lvalue-to-rvalue conversion.
+  if (E->isGLValue()) {
+    std::optional RValueResult = Res.toRValue();
+    if (!RValueResult) {
+      return false;
+    }
+    Result = *RValueResult;
+  } else {
+    Result = Res.toAPValue();
   }
 
+  return true;
+}
+
+bool Context::evaluate(State &Parent, const Expr *E, APValue &Result) {
+  assert(Stk.empty());
+  ByteCodeExprGen C(*this, *P, Parent, Stk, Result);
+
+  auto Res = C.interpretExpr(E);
+  if (Res.isInvalid()) {
+    Stk.clear();
+    return false;
+  }
+
+  assert(Stk.empty());
+#ifndef NDEBUG
+  // Make sure we don't rely on some value being still alive in
+  // InterpStack memory.
   Stk.clear();
-  return false;
+#endif
+  Result = Res.toAPValue();
+  return true;
 }
 
 bool Context::evaluateAsInitializer(State &Parent, const VarDecl *VD,
                                     APValue &Result) {
   assert(Stk.empty());
   ByteCodeExprGen C(*this, *P, Parent, Stk, Result);
-  if (Check(Parent, C.interpretDecl(VD))) {
-    assert(Stk.empty());
-#ifndef NDEBUG
-    // Make sure we don't rely on some value being still alive in
-    // InterpStack memory.
+
+  auto Res = C.interpretDecl(VD);
+  if (Res.isInvalid()) {
     Stk.clear();
-#endif
-    return true;
+    return false;
   }
 
+  assert(Stk.empty());
+#ifndef NDEBUG
+  // Make sure we don't rely on some value being still alive in
+  // InterpStack memory.
   Stk.clear();
-  return false;
+#endif
+
+  // Ensure global variables are fully initialized.
+  if (shouldBeGloballyIndexed(VD) && !Res.isInvalid() &&
+      (VD->getType()->isRecordType() || VD->getType()->isArrayType())) {
+    assert(Res.isLValue());
+
+    if (!Res.checkFullyInitialized(C.getState()))
+      return false;
+
+    // lvalue-to-rvalue conversion.
+    std::optional RValueResult = Res.toRValue();
+    if (!RValueResult)
+      return false;
+    Result = *RValueResult;
+
+  } else
+    Result = Res.toAPValue();
+  return true;
 }
 
 const LangOptions &Context::getLangOpts() const { return Ctx.getLangOpts(); }
@@ -234,12 +278,8 @@ const Function *Context::getOrCreateFunction(const FunctionDecl *FD) {
     return Func;
 
   if (!Func || WasNotDefined) {
-    if (auto R = ByteCodeStmtGen(*this, *P).compileFunc(FD))
-      Func = *R;
-    else {
-      llvm::consumeError(R.takeError());
-      return nullptr;
-    }
+    if (auto F = ByteCodeStmtGen(*this, *P).compileFunc(FD))
+      Func = F;
   }
 
   return Func;
diff --git a/clang/lib/AST/Interp/Context.h b/clang/lib/AST/Interp/Context.h
index 7649caef2242..ab83a8d13224 100644
--- a/clang/lib/AST/Interp/Context.h
+++ b/clang/lib/AST/Interp/Context.h
@@ -51,6 +51,9 @@ class Context final {
   /// Evaluates a toplevel expression as an rvalue.
   bool evaluateAsRValue(State &Parent, const Expr *E, APValue &Result);
 
+  /// Like evaluateAsRvalue(), but does no implicit lvalue-to-rvalue conversion.
+  bool evaluate(State &Parent, const Expr *E, APValue &Result);
+
   /// Evaluates a toplevel initializer.
   bool evaluateAsInitializer(State &Parent, const VarDecl *VD, APValue &Result);
 
diff --git a/clang/lib/AST/Interp/Descriptor.cpp b/clang/lib/AST/Interp/Descriptor.cpp
index 59a952135a2d..b330e54baf33 100644
--- a/clang/lib/AST/Interp/Descriptor.cpp
+++ b/clang/lib/AST/Interp/Descriptor.cpp
@@ -275,8 +275,8 @@ Descriptor::Descriptor(const DeclTy &D, const Descriptor *Elem, MetadataSize MD,
 }
 
 /// Unknown-size arrays of composite elements.
-Descriptor::Descriptor(const DeclTy &D, Descriptor *Elem, bool IsTemporary,
-                       UnknownSize)
+Descriptor::Descriptor(const DeclTy &D, const Descriptor *Elem,
+                       bool IsTemporary, UnknownSize)
     : Source(D), ElemSize(Elem->getAllocSize() + sizeof(InlineDescriptor)),
       Size(UnknownSizeMark), MDSize(0),
       AllocSize(alignof(void *) + sizeof(InitMapPtr)), ElemDesc(Elem),
@@ -286,7 +286,7 @@ Descriptor::Descriptor(const DeclTy &D, Descriptor *Elem, bool IsTemporary,
 }
 
 /// Composite records.
-Descriptor::Descriptor(const DeclTy &D, Record *R, MetadataSize MD,
+Descriptor::Descriptor(const DeclTy &D, const Record *R, MetadataSize MD,
                        bool IsConst, bool IsTemporary, bool IsMutable)
     : Source(D), ElemSize(std::max(alignof(void *), R->getFullSize())),
       Size(ElemSize), MDSize(MD.value_or(0)), AllocSize(Size + MDSize),
diff --git a/clang/lib/AST/Interp/Descriptor.h b/clang/lib/AST/Interp/Descriptor.h
index 8135f3d12f70..580c200f9095 100644
--- a/clang/lib/AST/Interp/Descriptor.h
+++ b/clang/lib/AST/Interp/Descriptor.h
@@ -100,7 +100,7 @@ struct Descriptor final {
   static constexpr MetadataSize InlineDescMD = sizeof(InlineDescriptor);
 
   /// Pointer to the record, if block contains records.
-  Record *const ElemRecord = nullptr;
+  const Record *const ElemRecord = nullptr;
   /// Descriptor of the array element.
   const Descriptor *const ElemDesc = nullptr;
   /// Flag indicating if the block is mutable.
@@ -135,10 +135,11 @@ struct Descriptor final {
              unsigned NumElems, bool IsConst, bool IsTemporary, bool IsMutable);
 
   /// Allocates a descriptor for an array of composites of unknown size.
-  Descriptor(const DeclTy &D, Descriptor *Elem, bool IsTemporary, UnknownSize);
+  Descriptor(const DeclTy &D, const Descriptor *Elem, bool IsTemporary,
+             UnknownSize);
 
   /// Allocates a descriptor for a record.
-  Descriptor(const DeclTy &D, Record *R, MetadataSize MD, bool IsConst,
+  Descriptor(const DeclTy &D, const Record *R, MetadataSize MD, bool IsConst,
              bool IsTemporary, bool IsMutable);
 
   Descriptor(const DeclTy &D, MetadataSize MD);
diff --git a/clang/lib/AST/Interp/EvalEmitter.cpp b/clang/lib/AST/Interp/EvalEmitter.cpp
index 0ff0bde8fd17..a60f893de8bd 100644
--- a/clang/lib/AST/Interp/EvalEmitter.cpp
+++ b/clang/lib/AST/Interp/EvalEmitter.cpp
@@ -19,7 +19,7 @@ using namespace clang::interp;
 
 EvalEmitter::EvalEmitter(Context &Ctx, Program &P, State &Parent,
                          InterpStack &Stk, APValue &Result)
-    : Ctx(Ctx), P(P), S(Parent, P, Stk, Ctx, this), Result(Result) {
+    : Ctx(Ctx), P(P), S(Parent, P, Stk, Ctx, this), EvalResult(&Ctx) {
   // Create a dummy frame for the interpreter which does not have locals.
   S.Current =
       new InterpFrame(S, /*Func=*/nullptr, /*Caller=*/nullptr, CodePtr());
@@ -33,20 +33,22 @@ EvalEmitter::~EvalEmitter() {
   }
 }
 
-llvm::Expected EvalEmitter::interpretExpr(const Expr *E) {
-  if (this->visitExpr(E))
-    return true;
-  if (BailLocation)
-    return llvm::make_error(*BailLocation);
-  return false;
+EvaluationResult EvalEmitter::interpretExpr(const Expr *E) {
+  EvalResult.setSource(E);
+
+  if (!this->visitExpr(E))
+    EvalResult.setInvalid();
+
+  return std::move(this->EvalResult);
 }
 
-llvm::Expected EvalEmitter::interpretDecl(const VarDecl *VD) {
-  if (this->visitDecl(VD))
-    return true;
-  if (BailLocation)
-    return llvm::make_error(*BailLocation);
-  return false;
+EvaluationResult EvalEmitter::interpretDecl(const VarDecl *VD) {
+  EvalResult.setSource(VD);
+
+  if (!this->visitDecl(VD))
+    EvalResult.setInvalid();
+
+  return std::move(this->EvalResult);
 }
 
 void EvalEmitter::emitLabel(LabelTy Label) {
@@ -77,12 +79,6 @@ Scope::Local EvalEmitter::createLocal(Descriptor *D) {
   return {Off, D};
 }
 
-bool EvalEmitter::bail(const SourceLocation &Loc) {
-  if (!BailLocation)
-    BailLocation = Loc;
-  return false;
-}
-
 bool EvalEmitter::jumpTrue(const LabelTy &Label) {
   if (isActive()) {
     if (S.Stk.pop())
@@ -116,125 +112,37 @@ template  bool EvalEmitter::emitRet(const SourceInfo &Info) {
   if (!isActive())
     return true;
   using T = typename PrimConv::T;
-  return ReturnValue(S.Stk.pop(), Result);
+  EvalResult.setValue(S.Stk.pop().toAPValue());
+  return true;
 }
 
-bool EvalEmitter::emitRetVoid(const SourceInfo &Info) { return true; }
+template <> bool EvalEmitter::emitRet(const SourceInfo &Info) {
+  if (!isActive())
+    return true;
+  EvalResult.setPointer(S.Stk.pop());
+  return true;
+}
+template <> bool EvalEmitter::emitRet(const SourceInfo &Info) {
+  if (!isActive())
+    return true;
+  EvalResult.setFunctionPointer(S.Stk.pop());
+  return true;
+}
+
+bool EvalEmitter::emitRetVoid(const SourceInfo &Info) {
+  EvalResult.setValid();
+  return true;
+}
 
 bool EvalEmitter::emitRetValue(const SourceInfo &Info) {
-  // Method to recursively traverse composites.
-  std::function Composite;
-  Composite = [this, &Composite](QualType Ty, const Pointer &Ptr, APValue &R) {
-    if (const auto *AT = Ty->getAs())
-      Ty = AT->getValueType();
-
-    if (const auto *RT = Ty->getAs()) {
-      const auto *Record = Ptr.getRecord();
-      assert(Record && "Missing record descriptor");
-
-      bool Ok = true;
-      if (RT->getDecl()->isUnion()) {
-        const FieldDecl *ActiveField = nullptr;
-        APValue Value;
-        for (const auto &F : Record->fields()) {
-          const Pointer &FP = Ptr.atField(F.Offset);
-          QualType FieldTy = F.Decl->getType();
-          if (FP.isActive()) {
-            if (std::optional T = Ctx.classify(FieldTy)) {
-              TYPE_SWITCH(*T, Ok &= ReturnValue(FP.deref(), Value));
-            } else {
-              Ok &= Composite(FieldTy, FP, Value);
-            }
-            break;
-          }
-        }
-        R = APValue(ActiveField, Value);
-      } else {
-        unsigned NF = Record->getNumFields();
-        unsigned NB = Record->getNumBases();
-        unsigned NV = Ptr.isBaseClass() ? 0 : Record->getNumVirtualBases();
-
-        R = APValue(APValue::UninitStruct(), NB, NF);
-
-        for (unsigned I = 0; I < NF; ++I) {
-          const Record::Field *FD = Record->getField(I);
-          QualType FieldTy = FD->Decl->getType();
-          const Pointer &FP = Ptr.atField(FD->Offset);
-          APValue &Value = R.getStructField(I);
-
-          if (std::optional T = Ctx.classify(FieldTy)) {
-            TYPE_SWITCH(*T, Ok &= ReturnValue(FP.deref(), Value));
-          } else {
-            Ok &= Composite(FieldTy, FP, Value);
-          }
-        }
-
-        for (unsigned I = 0; I < NB; ++I) {
-          const Record::Base *BD = Record->getBase(I);
-          QualType BaseTy = Ctx.getASTContext().getRecordType(BD->Decl);
-          const Pointer &BP = Ptr.atField(BD->Offset);
-          Ok &= Composite(BaseTy, BP, R.getStructBase(I));
-        }
-
-        for (unsigned I = 0; I < NV; ++I) {
-          const Record::Base *VD = Record->getVirtualBase(I);
-          QualType VirtBaseTy = Ctx.getASTContext().getRecordType(VD->Decl);
-          const Pointer &VP = Ptr.atField(VD->Offset);
-          Ok &= Composite(VirtBaseTy, VP, R.getStructBase(NB + I));
-        }
-      }
-      return Ok;
-    }
-
-    if (Ty->isIncompleteArrayType()) {
-      R = APValue(APValue::UninitArray(), 0, 0);
-      return true;
-    }
-
-    if (const auto *AT = Ty->getAsArrayTypeUnsafe()) {
-      const size_t NumElems = Ptr.getNumElems();
-      QualType ElemTy = AT->getElementType();
-      R = APValue(APValue::UninitArray{}, NumElems, NumElems);
-
-      bool Ok = true;
-      for (unsigned I = 0; I < NumElems; ++I) {
-        APValue &Slot = R.getArrayInitializedElt(I);
-        const Pointer &EP = Ptr.atIndex(I);
-        if (std::optional T = Ctx.classify(ElemTy)) {
-          TYPE_SWITCH(*T, Ok &= ReturnValue(EP.deref(), Slot));
-        } else {
-          Ok &= Composite(ElemTy, EP.narrow(), Slot);
-        }
-      }
-      return Ok;
-    }
-
-    // Complex types.
-    if (const auto *CT = Ty->getAs()) {
-      QualType ElemTy = CT->getElementType();
-      std::optional ElemT = Ctx.classify(ElemTy);
-      assert(ElemT);
-
-      if (ElemTy->isIntegerType()) {
-        INT_TYPE_SWITCH(*ElemT, {
-          auto V1 = Ptr.atIndex(0).deref();
-          auto V2 = Ptr.atIndex(1).deref();
-          Result = APValue(V1.toAPSInt(), V2.toAPSInt());
-          return true;
-        });
-      } else if (ElemTy->isFloatingType()) {
-        Result = APValue(Ptr.atIndex(0).deref().getAPFloat(),
-                         Ptr.atIndex(1).deref().getAPFloat());
-        return true;
-      }
-      return false;
-    }
-    llvm_unreachable("invalid value to return");
-  };
-
-  // Return the composite type.
   const auto &Ptr = S.Stk.pop();
-  return Composite(Ptr.getType(), Ptr, Result);
+  if (std::optional APV = Ptr.toRValue(S.getCtx())) {
+    EvalResult.setValue(*APV);
+    return true;
+  }
+
+  EvalResult.setInvalid();
+  return false;
 }
 
 bool EvalEmitter::emitGetPtrLocal(uint32_t I, const SourceInfo &Info) {
diff --git a/clang/lib/AST/Interp/EvalEmitter.h b/clang/lib/AST/Interp/EvalEmitter.h
index 5a9be18c34a0..deb2ebc4e61f 100644
--- a/clang/lib/AST/Interp/EvalEmitter.h
+++ b/clang/lib/AST/Interp/EvalEmitter.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_CLANG_AST_INTERP_EVALEMITTER_H
 #define LLVM_CLANG_AST_INTERP_EVALEMITTER_H
 
+#include "EvaluationResult.h"
 #include "InterpState.h"
 #include "PrimType.h"
 #include "Source.h"
@@ -33,8 +34,10 @@ class EvalEmitter : public SourceMapper {
   using AddrTy = uintptr_t;
   using Local = Scope::Local;
 
-  llvm::Expected interpretExpr(const Expr *E);
-  llvm::Expected interpretDecl(const VarDecl *VD);
+  EvaluationResult interpretExpr(const Expr *E);
+  EvaluationResult interpretDecl(const VarDecl *VD);
+
+  InterpState &getState() { return S; }
 
 protected:
   EvalEmitter(Context &Ctx, Program &P, State &Parent, InterpStack &Stk,
@@ -51,10 +54,6 @@ class EvalEmitter : public SourceMapper {
   virtual bool visitExpr(const Expr *E) = 0;
   virtual bool visitDecl(const VarDecl *VD) = 0;
 
-  bool bail(const Stmt *S) { return bail(S->getBeginLoc()); }
-  bool bail(const Decl *D) { return bail(D->getBeginLoc()); }
-  bool bail(const SourceLocation &Loc);
-
   /// Emits jumps.
   bool jumpTrue(const LabelTy &Label);
   bool jumpFalse(const LabelTy &Label);
@@ -86,7 +85,7 @@ class EvalEmitter : public SourceMapper {
   /// Callee evaluation state.
   InterpState S;
   /// Location to write the result to.
-  APValue &Result;
+  EvaluationResult EvalResult;
 
   /// Temporaries which require storage.
   llvm::DenseMap> Locals;
@@ -100,8 +99,6 @@ class EvalEmitter : public SourceMapper {
   // The emitter always tracks the current instruction and sets OpPC to a token
   // value which is mapped to the location of the opcode being evaluated.
   CodePtr OpPC;
-  /// Location of a failure.
-  std::optional BailLocation;
   /// Location of the current instruction.
   SourceInfo CurrentSource;
 
diff --git a/clang/lib/AST/Interp/EvaluationResult.cpp b/clang/lib/AST/Interp/EvaluationResult.cpp
new file mode 100644
index 000000000000..a14dc87f1dfd
--- /dev/null
+++ b/clang/lib/AST/Interp/EvaluationResult.cpp
@@ -0,0 +1,196 @@
+//===----- EvaluationResult.cpp - Result class  for the VM ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "EvaluationResult.h"
+#include "Context.h"
+#include "InterpState.h"
+#include "Record.h"
+#include "clang/AST/ExprCXX.h"
+
+namespace clang {
+namespace interp {
+
+APValue EvaluationResult::toAPValue() const {
+  assert(!empty());
+  switch (Kind) {
+  case LValue:
+    // Either a pointer or a function pointer.
+    if (const auto *P = std::get_if(&Value))
+      return P->toAPValue();
+    else if (const auto *FP = std::get_if(&Value))
+      return FP->toAPValue();
+    else
+      llvm_unreachable("Unhandled LValue type");
+    break;
+  case RValue:
+    return std::get(Value);
+  case Valid:
+    return APValue();
+  default:
+    llvm_unreachable("Unhandled result kind?");
+  }
+}
+
+std::optional EvaluationResult::toRValue() const {
+  if (Kind == RValue)
+    return toAPValue();
+
+  assert(Kind == LValue);
+
+  // We have a pointer and want an RValue.
+  if (const auto *P = std::get_if(&Value))
+    return P->toRValue(*Ctx);
+  else if (const auto *FP = std::get_if(&Value)) // Nope
+    return FP->toAPValue();
+  llvm_unreachable("Unhandled lvalue kind");
+}
+
+static void DiagnoseUninitializedSubobject(InterpState &S, SourceLocation Loc,
+                                           const FieldDecl *SubObjDecl) {
+  assert(SubObjDecl && "Subobject declaration does not exist");
+  S.FFDiag(Loc, diag::note_constexpr_uninitialized)
+      << /*(name)*/ 1 << SubObjDecl;
+  S.Note(SubObjDecl->getLocation(),
+         diag::note_constexpr_subobject_declared_here);
+}
+
+static bool CheckFieldsInitialized(InterpState &S, SourceLocation Loc,
+                                   const Pointer &BasePtr, const Record *R);
+
+static bool CheckArrayInitialized(InterpState &S, SourceLocation Loc,
+                                  const Pointer &BasePtr,
+                                  const ConstantArrayType *CAT) {
+  bool Result = true;
+  size_t NumElems = CAT->getSize().getZExtValue();
+  QualType ElemType = CAT->getElementType();
+
+  if (ElemType->isRecordType()) {
+    const Record *R = BasePtr.getElemRecord();
+    for (size_t I = 0; I != NumElems; ++I) {
+      Pointer ElemPtr = BasePtr.atIndex(I).narrow();
+      Result &= CheckFieldsInitialized(S, Loc, ElemPtr, R);
+    }
+  } else if (const auto *ElemCAT = dyn_cast(ElemType)) {
+    for (size_t I = 0; I != NumElems; ++I) {
+      Pointer ElemPtr = BasePtr.atIndex(I).narrow();
+      Result &= CheckArrayInitialized(S, Loc, ElemPtr, ElemCAT);
+    }
+  } else {
+    for (size_t I = 0; I != NumElems; ++I) {
+      if (!BasePtr.atIndex(I).isInitialized()) {
+        DiagnoseUninitializedSubobject(S, Loc, BasePtr.getField());
+        Result = false;
+      }
+    }
+  }
+
+  return Result;
+}
+
+static bool CheckFieldsInitialized(InterpState &S, SourceLocation Loc,
+                                   const Pointer &BasePtr, const Record *R) {
+  assert(R);
+  bool Result = true;
+  // Check all fields of this record are initialized.
+  for (const Record::Field &F : R->fields()) {
+    Pointer FieldPtr = BasePtr.atField(F.Offset);
+    QualType FieldType = F.Decl->getType();
+
+    if (FieldType->isRecordType()) {
+      Result &= CheckFieldsInitialized(S, Loc, FieldPtr, FieldPtr.getRecord());
+    } else if (FieldType->isIncompleteArrayType()) {
+      // Nothing to do here.
+    } else if (FieldType->isArrayType()) {
+      const auto *CAT =
+          cast(FieldType->getAsArrayTypeUnsafe());
+      Result &= CheckArrayInitialized(S, Loc, FieldPtr, CAT);
+    } else if (!FieldPtr.isInitialized()) {
+      DiagnoseUninitializedSubobject(S, Loc, F.Decl);
+      Result = false;
+    }
+  }
+
+  // Check Fields in all bases
+  for (const Record::Base &B : R->bases()) {
+    Pointer P = BasePtr.atField(B.Offset);
+    if (!P.isInitialized()) {
+      S.FFDiag(BasePtr.getDeclDesc()->asDecl()->getLocation(),
+               diag::note_constexpr_uninitialized_base)
+          << B.Desc->getType();
+      return false;
+    }
+    Result &= CheckFieldsInitialized(S, Loc, P, B.R);
+  }
+
+  // TODO: Virtual bases
+
+  return Result;
+}
+
+bool EvaluationResult::checkFullyInitialized(InterpState &S) const {
+  assert(Source);
+  assert(isLValue());
+
+  // Our Source must be a VarDecl.
+  const Decl *SourceDecl = Source.dyn_cast();
+  assert(SourceDecl);
+  const auto *VD = cast(SourceDecl);
+  assert(VD->getType()->isRecordType() || VD->getType()->isArrayType());
+  SourceLocation InitLoc = VD->getAnyInitializer()->getExprLoc();
+
+  const Pointer &Ptr = *std::get_if(&Value);
+  assert(!Ptr.isZero());
+
+  if (const Record *R = Ptr.getRecord())
+    return CheckFieldsInitialized(S, InitLoc, Ptr, R);
+  const auto *CAT =
+      cast(Ptr.getType()->getAsArrayTypeUnsafe());
+  return CheckArrayInitialized(S, InitLoc, Ptr, CAT);
+
+  return true;
+}
+
+void EvaluationResult::dump() const {
+  assert(Ctx);
+  auto &OS = llvm::errs();
+  const ASTContext &ASTCtx = Ctx->getASTContext();
+
+  switch (Kind) {
+  case Empty:
+    OS << "Empty\n";
+    break;
+  case RValue:
+    OS << "RValue: ";
+    std::get(Value).dump(OS, ASTCtx);
+    break;
+  case LValue: {
+    assert(Source);
+    QualType SourceType;
+    if (const auto *D = Source.dyn_cast()) {
+      if (const auto *VD = dyn_cast(D))
+        SourceType = VD->getType();
+    } else if (const auto *E = Source.dyn_cast()) {
+      SourceType = E->getType();
+    }
+
+    OS << "LValue: ";
+    if (const auto *P = std::get_if(&Value))
+      P->toAPValue().printPretty(OS, ASTCtx, SourceType);
+    else if (const auto *FP = std::get_if(&Value)) // Nope
+      FP->toAPValue().printPretty(OS, ASTCtx, SourceType);
+    OS << "\n";
+    break;
+  }
+
+  default:
+    llvm_unreachable("Can't print that.");
+  }
+}
+
+} // namespace interp
+} // namespace clang
diff --git a/clang/lib/AST/Interp/EvaluationResult.h b/clang/lib/AST/Interp/EvaluationResult.h
new file mode 100644
index 000000000000..2b9fc16f1a0a
--- /dev/null
+++ b/clang/lib/AST/Interp/EvaluationResult.h
@@ -0,0 +1,111 @@
+//===------ EvaluationResult.h - Result class  for the VM -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_AST_INTERP_EVALUATION_RESULT_H
+#define LLVM_CLANG_AST_INTERP_EVALUATION_RESULT_H
+
+#include "FunctionPointer.h"
+#include "Pointer.h"
+#include "clang/AST/APValue.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/Expr.h"
+#include 
+#include 
+
+namespace clang {
+namespace interp {
+class EvalEmitter;
+class Context;
+
+/// Defines the result of an evaluation.
+///
+/// The result might be in different forms--one of the pointer types,
+/// an APValue, or nothing.
+///
+/// We use this class to inspect and diagnose the result, as well as
+/// convert it to the requested form.
+class EvaluationResult final {
+public:
+  enum ResultKind {
+    Empty,   // Initial state.
+    LValue,  // Result is an lvalue/pointer.
+    RValue,  // Result is an rvalue.
+    Invalid, // Result is invalid.
+    Valid,   // Result is valid and empty.
+  };
+
+  using DeclTy = llvm::PointerUnion;
+
+private:
+  const Context *Ctx = nullptr;
+  std::variant Value;
+  ResultKind Kind = Empty;
+  DeclTy Source = nullptr; // Currently only needed for dump().
+
+  EvaluationResult(ResultKind Kind) : Kind(Kind) {
+    // Leave everything empty. Can be used as an
+    // error marker or for void return values.
+    assert(Kind == Valid || Kind == Invalid);
+  }
+
+  void setSource(DeclTy D) { Source = D; }
+
+  void setValue(const APValue &V) {
+    assert(empty());
+    assert(!V.isLValue());
+    Value = std::move(V);
+    Kind = RValue;
+  }
+  void setPointer(const Pointer P) {
+    assert(empty());
+    Value = P;
+    Kind = LValue;
+  }
+  void setFunctionPointer(const FunctionPointer &P) {
+    assert(empty());
+    Value = P;
+    Kind = LValue;
+  }
+  void setInvalid() {
+    assert(empty());
+    Kind = Invalid;
+  }
+  void setValid() {
+    assert(empty());
+    Kind = Valid;
+  }
+
+public:
+  EvaluationResult(const Context *Ctx) : Ctx(Ctx) {}
+
+  bool empty() const { return Kind == Empty; }
+  bool isInvalid() const { return Kind == Invalid; }
+  bool isLValue() const { return Kind == LValue; }
+  bool isRValue() const { return Kind == RValue; }
+
+  /// Returns an APValue for the evaluation result. The returned
+  /// APValue might be an LValue or RValue.
+  APValue toAPValue() const;
+
+  /// If the result is an LValue, convert that to an RValue
+  /// and return it. This may fail, e.g. if the result is an
+  /// LValue and we can't read from it.
+  std::optional toRValue() const;
+
+  bool checkFullyInitialized(InterpState &S) const;
+
+  /// Dump to stderr.
+  void dump() const;
+
+  friend class EvalEmitter;
+};
+
+} // namespace interp
+} // namespace clang
+
+#endif
diff --git a/clang/lib/AST/Interp/IntegralAP.h b/clang/lib/AST/Interp/IntegralAP.h
index d5f46409d231..55e29caa1cd7 100644
--- a/clang/lib/AST/Interp/IntegralAP.h
+++ b/clang/lib/AST/Interp/IntegralAP.h
@@ -204,14 +204,18 @@ template  class IntegralAP final {
   }
 
   static bool rem(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
-    // FIXME: Implement.
-    assert(false);
+    if constexpr (Signed)
+      *R = IntegralAP(A.V.srem(B.V));
+    else
+      *R = IntegralAP(A.V.urem(B.V));
     return false;
   }
 
   static bool div(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
-    // FIXME: Implement.
-    assert(false);
+    if constexpr (Signed)
+      *R = IntegralAP(A.V.sdiv(B.V));
+    else
+      *R = IntegralAP(A.V.udiv(B.V));
     return false;
   }
 
diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp
index a82d1c3c7c62..807b860f3565 100644
--- a/clang/lib/AST/Interp/Interp.cpp
+++ b/clang/lib/AST/Interp/Interp.cpp
@@ -53,6 +53,25 @@ static bool Jf(InterpState &S, CodePtr &PC, int32_t Offset) {
   return true;
 }
 
+static void diagnoseNonConstVariable(InterpState &S, CodePtr OpPC,
+                                     const ValueDecl *VD) {
+  if (!S.getLangOpts().CPlusPlus)
+    return;
+
+  const SourceInfo &Loc = S.Current->getSource(OpPC);
+
+  if (VD->getType()->isIntegralOrEnumerationType())
+    S.FFDiag(Loc, diag::note_constexpr_ltor_non_const_int, 1) << VD;
+  else
+    S.FFDiag(Loc,
+             S.getLangOpts().CPlusPlus11
+                 ? diag::note_constexpr_ltor_non_constexpr
+                 : diag::note_constexpr_ltor_non_integral,
+             1)
+        << VD << VD->getType();
+  S.Note(VD->getLocation(), diag::note_declared_at);
+}
+
 static bool CheckActive(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
                         AccessKinds AK) {
   if (Ptr.isActive())
@@ -134,6 +153,18 @@ void cleanupAfterFunctionCall(InterpState &S, CodePtr OpPC) {
   if (CurFunc->isUnevaluatedBuiltin())
     return;
 
+  // Some builtin functions require us to only look at the call site, since
+  // the classified parameter types do not match.
+  if (CurFunc->isBuiltin()) {
+    const auto *CE =
+        cast(S.Current->Caller->getExpr(S.Current->getRetPC()));
+    for (int32_t I = CE->getNumArgs() - 1; I >= 0; --I) {
+      const Expr *A = CE->getArg(I);
+      popArg(S, A);
+    }
+    return;
+  }
+
   if (S.Current->Caller && CurFunc->isVariadic()) {
     // CallExpr we're look for is at the return PC of the current function, i.e.
     // in the caller.
@@ -159,9 +190,7 @@ bool CheckExtern(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
 
   if (!S.checkingPotentialConstantExpression() && S.getLangOpts().CPlusPlus) {
     const auto *VD = Ptr.getDeclDesc()->asValueDecl();
-    const SourceInfo &Loc = S.Current->getSource(OpPC);
-    S.FFDiag(Loc, diag::note_constexpr_ltor_non_constexpr, 1) << VD;
-    S.Note(VD->getLocation(), diag::note_declared_at);
+    diagnoseNonConstVariable(S, OpPC, VD);
   }
   return false;
 }
@@ -204,6 +233,44 @@ bool CheckLive(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
   return true;
 }
 
+bool CheckConstant(InterpState &S, CodePtr OpPC, const Descriptor *Desc) {
+  assert(Desc);
+
+  auto IsConstType = [&S](const VarDecl *VD) -> bool {
+    if (VD->isConstexpr())
+      return true;
+
+    if (S.getLangOpts().CPlusPlus && !S.getLangOpts().CPlusPlus11)
+      return false;
+
+    QualType T = VD->getType();
+    if (T.isConstQualified())
+      return true;
+
+    if (const auto *RT = T->getAs())
+      return RT->getPointeeType().isConstQualified();
+
+    if (const auto *PT = T->getAs())
+      return PT->getPointeeType().isConstQualified();
+
+    return false;
+  };
+
+  if (const auto *D = Desc->asValueDecl()) {
+    if (const auto *VD = dyn_cast(D);
+        VD && VD->hasGlobalStorage() && !IsConstType(VD)) {
+      diagnoseNonConstVariable(S, OpPC, VD);
+      return S.inConstantContext();
+    }
+  }
+
+  return true;
+}
+
+static bool CheckConstant(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
+  return CheckConstant(S, OpPC, Ptr.getDeclDesc());
+}
+
 bool CheckDummy(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
   return !Ptr.isZero() && !Ptr.isDummy();
 }
@@ -290,10 +357,13 @@ bool CheckInitialized(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 }
 
 bool CheckLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
-  if (!CheckDummy(S, OpPC, Ptr))
-    return false;
   if (!CheckLive(S, OpPC, Ptr, AK_Read))
     return false;
+  if (!CheckConstant(S, OpPC, Ptr))
+    return false;
+
+  if (!CheckDummy(S, OpPC, Ptr))
+    return false;
   if (!CheckExtern(S, OpPC, Ptr))
     return false;
   if (!CheckRange(S, OpPC, Ptr, AK_Read))
@@ -420,7 +490,7 @@ bool CheckThis(InterpState &S, CodePtr OpPC, const Pointer &This) {
 }
 
 bool CheckPure(InterpState &S, CodePtr OpPC, const CXXMethodDecl *MD) {
-  if (!MD->isPure())
+  if (!MD->isPureVirtual())
     return true;
   const SourceInfo &E = S.Current->getSource(OpPC);
   S.FFDiag(E, diag::note_constexpr_pure_virtual_call, 1) << MD;
@@ -428,98 +498,6 @@ bool CheckPure(InterpState &S, CodePtr OpPC, const CXXMethodDecl *MD) {
   return false;
 }
 
-static void DiagnoseUninitializedSubobject(InterpState &S, const SourceInfo &SI,
-                                           const FieldDecl *SubObjDecl) {
-  assert(SubObjDecl && "Subobject declaration does not exist");
-  S.FFDiag(SI, diag::note_constexpr_uninitialized)
-      << /*(name)*/ 1 << SubObjDecl;
-  S.Note(SubObjDecl->getLocation(),
-         diag::note_constexpr_subobject_declared_here);
-}
-
-static bool CheckFieldsInitialized(InterpState &S, CodePtr OpPC,
-                                   const Pointer &BasePtr, const Record *R);
-
-static bool CheckArrayInitialized(InterpState &S, CodePtr OpPC,
-                                  const Pointer &BasePtr,
-                                  const ConstantArrayType *CAT) {
-  bool Result = true;
-  size_t NumElems = CAT->getSize().getZExtValue();
-  QualType ElemType = CAT->getElementType();
-
-  if (ElemType->isRecordType()) {
-    const Record *R = BasePtr.getElemRecord();
-    for (size_t I = 0; I != NumElems; ++I) {
-      Pointer ElemPtr = BasePtr.atIndex(I).narrow();
-      Result &= CheckFieldsInitialized(S, OpPC, ElemPtr, R);
-    }
-  } else if (const auto *ElemCAT = dyn_cast(ElemType)) {
-    for (size_t I = 0; I != NumElems; ++I) {
-      Pointer ElemPtr = BasePtr.atIndex(I).narrow();
-      Result &= CheckArrayInitialized(S, OpPC, ElemPtr, ElemCAT);
-    }
-  } else {
-    for (size_t I = 0; I != NumElems; ++I) {
-      if (!BasePtr.atIndex(I).isInitialized()) {
-        DiagnoseUninitializedSubobject(S, S.Current->getSource(OpPC),
-                                       BasePtr.getField());
-        Result = false;
-      }
-    }
-  }
-
-  return Result;
-}
-
-static bool CheckFieldsInitialized(InterpState &S, CodePtr OpPC,
-                                   const Pointer &BasePtr, const Record *R) {
-  assert(R);
-  bool Result = true;
-  // Check all fields of this record are initialized.
-  for (const Record::Field &F : R->fields()) {
-    Pointer FieldPtr = BasePtr.atField(F.Offset);
-    QualType FieldType = F.Decl->getType();
-
-    if (FieldType->isRecordType()) {
-      Result &= CheckFieldsInitialized(S, OpPC, FieldPtr, FieldPtr.getRecord());
-    } else if (FieldType->isIncompleteArrayType()) {
-      // Nothing to do here.
-    } else if (FieldType->isArrayType()) {
-      const auto *CAT =
-          cast(FieldType->getAsArrayTypeUnsafe());
-      Result &= CheckArrayInitialized(S, OpPC, FieldPtr, CAT);
-    } else if (!FieldPtr.isInitialized()) {
-      DiagnoseUninitializedSubobject(S, S.Current->getSource(OpPC), F.Decl);
-      Result = false;
-    }
-  }
-
-  // Check Fields in all bases
-  for (const Record::Base &B : R->bases()) {
-    Pointer P = BasePtr.atField(B.Offset);
-    if (!P.isInitialized()) {
-      S.FFDiag(BasePtr.getDeclDesc()->asDecl()->getLocation(),
-               diag::note_constexpr_uninitialized_base)
-          << B.Desc->getType();
-      return false;
-    }
-    Result &= CheckFieldsInitialized(S, OpPC, P, B.R);
-  }
-
-  // TODO: Virtual bases
-
-  return Result;
-}
-
-bool CheckCtorCall(InterpState &S, CodePtr OpPC, const Pointer &This) {
-  assert(!This.isZero());
-  if (const Record *R = This.getRecord())
-    return CheckFieldsInitialized(S, OpPC, This, R);
-  const auto *CAT =
-      cast(This.getType()->getAsArrayTypeUnsafe());
-  return CheckArrayInitialized(S, OpPC, This, CAT);
-}
-
 bool CheckPotentialReinterpretCast(InterpState &S, CodePtr OpPC,
                                    const Pointer &Ptr) {
   if (!S.inConstantContext())
@@ -593,13 +571,7 @@ bool CheckDeclRef(InterpState &S, CodePtr OpPC, const DeclRefExpr *DR) {
     }
   } else if (const auto *VD = dyn_cast(D)) {
     if (!VD->getType().isConstQualified()) {
-      S.FFDiag(E,
-               VD->getType()->isIntegralOrEnumerationType()
-                   ? diag::note_constexpr_ltor_non_const_int
-                   : diag::note_constexpr_ltor_non_constexpr,
-               1)
-          << VD;
-      S.Note(VD->getLocation(), diag::note_declared_at) << VD->getSourceRange();
+      diagnoseNonConstVariable(S, OpPC, VD);
       return false;
     }
 
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index a240d74d6342..65c54ed9c89b 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -37,7 +37,6 @@
 namespace clang {
 namespace interp {
 
-using APInt = llvm::APInt;
 using APSInt = llvm::APSInt;
 
 /// Convert a value to an APValue.
@@ -78,6 +77,9 @@ bool CheckSubobject(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 /// Checks if a pointer points to const storage.
 bool CheckConst(InterpState &S, CodePtr OpPC, const Pointer &Ptr);
 
+/// Checks if the Descriptor is of a constexpr or const global variable.
+bool CheckConstant(InterpState &S, CodePtr OpPC, const Descriptor *Desc);
+
 /// Checks if a pointer points to a mutable field.
 bool CheckMutable(InterpState &S, CodePtr OpPC, const Pointer &Ptr);
 
@@ -109,9 +111,6 @@ bool CheckThis(InterpState &S, CodePtr OpPC, const Pointer &This);
 /// Checks if a method is pure virtual.
 bool CheckPure(InterpState &S, CodePtr OpPC, const CXXMethodDecl *MD);
 
-/// Checks that all fields are initialized after a constructor call.
-bool CheckCtorCall(InterpState &S, CodePtr OpPC, const Pointer &This);
-
 /// Checks if reinterpret casts are legal in the current context.
 bool CheckPotentialReinterpretCast(InterpState &S, CodePtr OpPC,
                                    const Pointer &Ptr);
@@ -1005,12 +1004,23 @@ bool SetThisField(InterpState &S, CodePtr OpPC, uint32_t I) {
 template ::T>
 bool GetGlobal(InterpState &S, CodePtr OpPC, uint32_t I) {
   const Block *B = S.P.getGlobal(I);
+
+  if (!CheckConstant(S, OpPC, B->getDescriptor()))
+    return false;
   if (B->isExtern())
     return false;
   S.Stk.push(B->deref());
   return true;
 }
 
+/// Same as GetGlobal, but without the checks.
+template ::T>
+bool GetGlobalUnchecked(InterpState &S, CodePtr OpPC, uint32_t I) {
+  auto *B = S.P.getGlobal(I);
+  S.Stk.push(B->deref());
+  return true;
+}
+
 template ::T>
 bool SetGlobal(InterpState &S, CodePtr OpPC, uint32_t I) {
   // TODO: emit warning.
@@ -1048,8 +1058,12 @@ inline bool InitGlobalTempComp(InterpState &S, CodePtr OpPC,
   const Pointer &P = S.Stk.peek();
   APValue *Cached = Temp->getOrCreateValue(true);
 
-  *Cached = P.toRValue(S.getCtx());
-  return true;
+  if (std::optional APV = P.toRValue(S.getCtx())) {
+    *Cached = *APV;
+    return true;
+  }
+
+  return false;
 }
 
 template ::T>
@@ -1065,15 +1079,18 @@ bool InitThisField(InterpState &S, CodePtr OpPC, uint32_t I) {
   return true;
 }
 
+// FIXME: The Field pointer here is too much IMO and we could instead just
+// pass an Offset + BitWidth pair.
 template ::T>
-bool InitThisBitField(InterpState &S, CodePtr OpPC, const Record::Field *F) {
+bool InitThisBitField(InterpState &S, CodePtr OpPC, const Record::Field *F,
+                      uint32_t FieldOffset) {
   assert(F->isBitField());
   if (S.checkingPotentialConstantExpression())
     return false;
   const Pointer &This = S.Current->getThis();
   if (!CheckThis(S, OpPC, This))
     return false;
-  const Pointer &Field = This.atField(F->Offset);
+  const Pointer &Field = This.atField(FieldOffset);
   const auto &Value = S.Stk.pop();
   Field.deref() = Value.truncate(F->Decl->getBitWidthValue(S.getCtx()));
   Field.initialize();
@@ -1814,9 +1831,6 @@ inline bool ArrayElemPtr(InterpState &S, CodePtr OpPC) {
   const T &Offset = S.Stk.pop();
   const Pointer &Ptr = S.Stk.peek();
 
-  if (!CheckArray(S, OpPC, Ptr))
-    return false;
-
   if (!OffsetHelper(S, OpPC, Offset, Ptr))
     return false;
 
@@ -1844,20 +1858,12 @@ inline bool ArrayElemPtrPop(InterpState &S, CodePtr OpPC) {
   const T &Offset = S.Stk.pop();
   const Pointer &Ptr = S.Stk.pop();
 
-  if (!CheckArray(S, OpPC, Ptr))
-    return false;
-
   if (!OffsetHelper(S, OpPC, Offset, Ptr))
     return false;
 
   return NarrowPtr(S, OpPC);
 }
 
-inline bool CheckGlobalCtor(InterpState &S, CodePtr OpPC) {
-  const Pointer &Obj = S.Stk.peek();
-  return CheckCtorCall(S, OpPC, Obj);
-}
-
 inline bool Call(InterpState &S, CodePtr OpPC, const Function *Func) {
   if (Func->hasThisPointer()) {
     size_t ThisOffset =
diff --git a/clang/lib/AST/Interp/InterpBuiltin.cpp b/clang/lib/AST/Interp/InterpBuiltin.cpp
index b55b1569a259..754ca96b0c64 100644
--- a/clang/lib/AST/Interp/InterpBuiltin.cpp
+++ b/clang/lib/AST/Interp/InterpBuiltin.cpp
@@ -164,6 +164,8 @@ static bool retPrimValue(InterpState &S, CodePtr OpPC, APValue &Result,
   case X:                                                                      \
     return Ret(S, OpPC, Result);
   switch (*T) {
+    RET_CASE(PT_Ptr);
+    RET_CASE(PT_FnPtr);
     RET_CASE(PT_Float);
     RET_CASE(PT_Bool);
     RET_CASE(PT_Sint8);
@@ -613,15 +615,34 @@ static bool interp__builtin_ffs(InterpState &S, CodePtr OpPC,
   return true;
 }
 
+static bool interp__builtin_addressof(InterpState &S, CodePtr OpPC,
+                                      const InterpFrame *Frame,
+                                      const Function *Func,
+                                      const CallExpr *Call) {
+  PrimType PtrT =
+      S.getContext().classify(Call->getArg(0)->getType()).value_or(PT_Ptr);
+
+  if (PtrT == PT_FnPtr) {
+    const FunctionPointer &Arg = S.Stk.peek();
+    S.Stk.push(Arg);
+  } else if (PtrT == PT_Ptr) {
+    const Pointer &Arg = S.Stk.peek();
+    S.Stk.push(Arg);
+  } else {
+    assert(false && "Unsupported pointer type passed to __builtin_addressof()");
+  }
+  return true;
+}
+
 bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
                       const CallExpr *Call) {
   InterpFrame *Frame = S.Current;
   APValue Dummy;
 
-  QualType ReturnType = Call->getCallReturnType(S.getCtx());
-  std::optional ReturnT = S.getContext().classify(ReturnType);
+  std::optional ReturnT = S.getContext().classify(Call->getType());
+
   // If classify failed, we assume void.
-  assert(ReturnT || ReturnType->isVoidType());
+  assert(ReturnT || Call->getType()->isVoidType());
 
   switch (F->getBuiltinID()) {
   case Builtin::BI__builtin_is_constant_evaluated:
@@ -820,6 +841,12 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
     if (!interp__builtin_ffs(S, OpPC, Frame, F, Call))
       return false;
     break;
+  case Builtin::BIaddressof:
+  case Builtin::BI__addressof:
+  case Builtin::BI__builtin_addressof:
+    if (!interp__builtin_addressof(S, OpPC, Frame, F, Call))
+      return false;
+    break;
 
   default:
     return false;
diff --git a/clang/lib/AST/Interp/Opcodes.td b/clang/lib/AST/Interp/Opcodes.td
index 69068e87d572..24747b6b98c1 100644
--- a/clang/lib/AST/Interp/Opcodes.td
+++ b/clang/lib/AST/Interp/Opcodes.td
@@ -375,10 +375,9 @@ def GetLocal : AccessOpcode { let HasCustomEval = 1; }
 // [] -> [Pointer]
 def SetLocal : AccessOpcode { let HasCustomEval = 1; }
 
-def CheckGlobalCtor : Opcode {}
-
 // [] -> [Value]
 def GetGlobal : AccessOpcode;
+def GetGlobalUnchecked : AccessOpcode;
 // [Value] -> []
 def InitGlobal : AccessOpcode;
 // [Value] -> []
@@ -416,7 +415,11 @@ def InitThisField : AccessOpcode;
 // [Value] -> []
 def InitThisFieldActive : AccessOpcode;
 // [Value] -> []
-def InitThisBitField : BitFieldOpcode;
+def InitThisBitField : Opcode {
+  let Types = [AluTypeClass];
+  let Args = [ArgRecordField, ArgUint32];
+  let HasGroup = 1;
+}
 // [Pointer, Value] -> []
 def InitField : AccessOpcode;
 // [Pointer, Value] -> []
diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/Interp/Pointer.cpp
index e979b99b0fdd..5af1d6d52e93 100644
--- a/clang/lib/AST/Interp/Pointer.cpp
+++ b/clang/lib/AST/Interp/Pointer.cpp
@@ -231,33 +231,143 @@ bool Pointer::hasSameArray(const Pointer &A, const Pointer &B) {
   return hasSameBase(A, B) && A.Base == B.Base && A.getFieldDesc()->IsArray;
 }
 
-APValue Pointer::toRValue(const Context &Ctx) const {
-  // Primitives.
-  if (getFieldDesc()->isPrimitive()) {
-    PrimType PT = *Ctx.classify(getType());
-    TYPE_SWITCH(PT, return deref().toAPValue());
-    llvm_unreachable("Unhandled PrimType?");
-  }
+std::optional Pointer::toRValue(const Context &Ctx) const {
+  // Method to recursively traverse composites.
+  std::function Composite;
+  Composite = [&Composite, &Ctx](QualType Ty, const Pointer &Ptr, APValue &R) {
+    if (const auto *AT = Ty->getAs())
+      Ty = AT->getValueType();
+
+    // Invalid pointers.
+    if (Ptr.isDummy() || !Ptr.isLive() ||
+        (!Ptr.isUnknownSizeArray() && Ptr.isOnePastEnd()))
+      return false;
 
-  APValue Result;
-  // Records.
-  if (getFieldDesc()->isRecord()) {
-    const Record *R = getRecord();
-    Result =
-        APValue(APValue::UninitStruct(), R->getNumBases(), R->getNumFields());
-
-    for (unsigned I = 0; I != R->getNumFields(); ++I) {
-      const Pointer &FieldPtr = this->atField(R->getField(I)->Offset);
-      Result.getStructField(I) = FieldPtr.toRValue(Ctx);
+    // Primitive values.
+    if (std::optional T = Ctx.classify(Ty)) {
+      if (T == PT_Ptr || T == PT_FnPtr) {
+        R = Ptr.toAPValue();
+      } else {
+        TYPE_SWITCH(*T, R = Ptr.deref().toAPValue());
+      }
+      return true;
     }
 
-    for (unsigned I = 0; I != R->getNumBases(); ++I) {
-      const Pointer &BasePtr = this->atField(R->getBase(I)->Offset);
-      Result.getStructBase(I) = BasePtr.toRValue(Ctx);
+    if (const auto *RT = Ty->getAs()) {
+      const auto *Record = Ptr.getRecord();
+      assert(Record && "Missing record descriptor");
+
+      bool Ok = true;
+      if (RT->getDecl()->isUnion()) {
+        const FieldDecl *ActiveField = nullptr;
+        APValue Value;
+        for (const auto &F : Record->fields()) {
+          const Pointer &FP = Ptr.atField(F.Offset);
+          QualType FieldTy = F.Decl->getType();
+          if (FP.isActive()) {
+            if (std::optional T = Ctx.classify(FieldTy)) {
+              TYPE_SWITCH(*T, Value = FP.deref().toAPValue());
+            } else {
+              Ok &= Composite(FieldTy, FP, Value);
+            }
+            break;
+          }
+        }
+        R = APValue(ActiveField, Value);
+      } else {
+        unsigned NF = Record->getNumFields();
+        unsigned NB = Record->getNumBases();
+        unsigned NV = Ptr.isBaseClass() ? 0 : Record->getNumVirtualBases();
+
+        R = APValue(APValue::UninitStruct(), NB, NF);
+
+        for (unsigned I = 0; I < NF; ++I) {
+          const Record::Field *FD = Record->getField(I);
+          QualType FieldTy = FD->Decl->getType();
+          const Pointer &FP = Ptr.atField(FD->Offset);
+          APValue &Value = R.getStructField(I);
+
+          if (std::optional T = Ctx.classify(FieldTy)) {
+            TYPE_SWITCH(*T, Value = FP.deref().toAPValue());
+          } else {
+            Ok &= Composite(FieldTy, FP, Value);
+          }
+        }
+
+        for (unsigned I = 0; I < NB; ++I) {
+          const Record::Base *BD = Record->getBase(I);
+          QualType BaseTy = Ctx.getASTContext().getRecordType(BD->Decl);
+          const Pointer &BP = Ptr.atField(BD->Offset);
+          Ok &= Composite(BaseTy, BP, R.getStructBase(I));
+        }
+
+        for (unsigned I = 0; I < NV; ++I) {
+          const Record::Base *VD = Record->getVirtualBase(I);
+          QualType VirtBaseTy = Ctx.getASTContext().getRecordType(VD->Decl);
+          const Pointer &VP = Ptr.atField(VD->Offset);
+          Ok &= Composite(VirtBaseTy, VP, R.getStructBase(NB + I));
+        }
+      }
+      return Ok;
     }
-  }
 
-  // TODO: Arrays
+    if (Ty->isIncompleteArrayType()) {
+      R = APValue(APValue::UninitArray(), 0, 0);
+      return true;
+    }
+
+    if (const auto *AT = Ty->getAsArrayTypeUnsafe()) {
+      const size_t NumElems = Ptr.getNumElems();
+      QualType ElemTy = AT->getElementType();
+      R = APValue(APValue::UninitArray{}, NumElems, NumElems);
+
+      bool Ok = true;
+      for (unsigned I = 0; I < NumElems; ++I) {
+        APValue &Slot = R.getArrayInitializedElt(I);
+        const Pointer &EP = Ptr.atIndex(I);
+        if (std::optional T = Ctx.classify(ElemTy)) {
+          TYPE_SWITCH(*T, Slot = EP.deref().toAPValue());
+        } else {
+          Ok &= Composite(ElemTy, EP.narrow(), Slot);
+        }
+      }
+      return Ok;
+    }
 
+    // Complex types.
+    if (const auto *CT = Ty->getAs()) {
+      QualType ElemTy = CT->getElementType();
+      std::optional ElemT = Ctx.classify(ElemTy);
+      assert(ElemT);
+
+      if (ElemTy->isIntegerType()) {
+        INT_TYPE_SWITCH(*ElemT, {
+          auto V1 = Ptr.atIndex(0).deref();
+          auto V2 = Ptr.atIndex(1).deref();
+          R = APValue(V1.toAPSInt(), V2.toAPSInt());
+          return true;
+        });
+      } else if (ElemTy->isFloatingType()) {
+        R = APValue(Ptr.atIndex(0).deref().getAPFloat(),
+                    Ptr.atIndex(1).deref().getAPFloat());
+        return true;
+      }
+      return false;
+    }
+
+    llvm_unreachable("invalid value to return");
+  };
+
+  if (isZero())
+    return APValue(static_cast(nullptr), CharUnits::Zero(), {}, false,
+                   true);
+
+  if (isDummy() || !isLive())
+    return std::nullopt;
+
+  // Return the composite type.
+  APValue Result;
+  if (!Composite(getType(), *this, Result))
+    return std::nullopt;
   return Result;
 }
diff --git a/clang/lib/AST/Interp/Pointer.h b/clang/lib/AST/Interp/Pointer.h
index a8f6e62fa76d..8ccaff41ded8 100644
--- a/clang/lib/AST/Interp/Pointer.h
+++ b/clang/lib/AST/Interp/Pointer.h
@@ -98,7 +98,7 @@ class Pointer {
   }
 
   /// Converts the pointer to an APValue that is an rvalue.
-  APValue toRValue(const Context &Ctx) const;
+  std::optional toRValue(const Context &Ctx) const;
 
   /// Offsets a pointer inside an array.
   [[nodiscard]] Pointer atIndex(unsigned Idx) const {
@@ -379,6 +379,7 @@ class Pointer {
       return *reinterpret_cast(Pointee->rawData() + Base +
                                     sizeof(InitMapPtr));
 
+    assert(Offset + sizeof(T) <= Pointee->getDescriptor()->getAllocSize());
     return *reinterpret_cast(Pointee->rawData() + Offset);
   }
 
diff --git a/clang/lib/AST/Interp/Program.cpp b/clang/lib/AST/Interp/Program.cpp
index 52e13398163e..1daefab4dcda 100644
--- a/clang/lib/AST/Interp/Program.cpp
+++ b/clang/lib/AST/Interp/Program.cpp
@@ -315,14 +315,14 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
                                       bool IsConst, bool IsTemporary,
                                       bool IsMutable, const Expr *Init) {
   // Classes and structures.
-  if (auto *RT = Ty->getAs()) {
-    if (auto *Record = getOrCreateRecord(RT->getDecl()))
+  if (const auto *RT = Ty->getAs()) {
+    if (const auto *Record = getOrCreateRecord(RT->getDecl()))
       return allocateDescriptor(D, Record, MDSize, IsConst, IsTemporary,
                                 IsMutable);
   }
 
   // Arrays.
-  if (auto ArrayType = Ty->getAsArrayTypeUnsafe()) {
+  if (const auto ArrayType = Ty->getAsArrayTypeUnsafe()) {
     QualType ElemTy = ArrayType->getElementType();
     // Array of well-known bounds.
     if (auto CAT = dyn_cast(ArrayType)) {
@@ -338,7 +338,7 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
       } else {
         // Arrays of composites. In this case, the array is a list of pointers,
         // followed by the actual elements.
-        Descriptor *ElemDesc = createDescriptor(
+        const Descriptor *ElemDesc = createDescriptor(
             D, ElemTy.getTypePtr(), std::nullopt, IsConst, IsTemporary);
         if (!ElemDesc)
           return nullptr;
@@ -358,8 +358,8 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
         return allocateDescriptor(D, *T, IsTemporary,
                                   Descriptor::UnknownSize{});
       } else {
-        Descriptor *Desc = createDescriptor(D, ElemTy.getTypePtr(), MDSize,
-                                            IsConst, IsTemporary);
+        const Descriptor *Desc = createDescriptor(D, ElemTy.getTypePtr(),
+                                                  MDSize, IsConst, IsTemporary);
         if (!Desc)
           return nullptr;
         return allocateDescriptor(D, Desc, IsTemporary,
@@ -369,14 +369,14 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
   }
 
   // Atomic types.
-  if (auto *AT = Ty->getAs()) {
+  if (const auto *AT = Ty->getAs()) {
     const Type *InnerTy = AT->getValueType().getTypePtr();
     return createDescriptor(D, InnerTy, MDSize, IsConst, IsTemporary,
                             IsMutable);
   }
 
   // Complex types - represented as arrays of elements.
-  if (auto *CT = Ty->getAs()) {
+  if (const auto *CT = Ty->getAs()) {
     PrimType ElemTy = *Ctx.classify(CT->getElementType());
     return allocateDescriptor(D, ElemTy, MDSize, 2, IsConst, IsTemporary,
                               IsMutable);
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index eb7240033d74..e04b1efd3ca5 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -4842,9 +4842,23 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity,
     E = cast(E)->getSubExpr();
     goto recurse;
 
-  case Expr::SubstNonTypeTemplateParmExprClass:
+  case Expr::SubstNonTypeTemplateParmExprClass: {
+    // Mangle a substituted parameter the same way we mangle the template
+    // argument.
+    auto *SNTTPE = cast(E);
+    if (auto *CE = dyn_cast(SNTTPE->getReplacement())) {
+      // Pull out the constant value and mangle it as a template argument.
+      QualType ParamType = SNTTPE->getParameterType(Context.getASTContext());
+      assert(CE->hasAPValueResult() && "expected the NTTP to have an APValue");
+      mangleValueInTemplateArg(ParamType, CE->getAPValueResult(), false,
+                               /*NeedExactType=*/true);
+      break;
+    }
+    // The remaining cases all happen to be substituted with expressions that
+    // mangle the same as a corresponding template argument anyway.
     E = cast(E)->getReplacement();
     goto recurse;
+  }
 
   case Expr::UserDefinedLiteralClass:
     // We follow g++'s approach of mangling a UDL as a call to the literal
@@ -4892,7 +4906,7 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity,
     Out << '_';
     mangleType(New->getAllocatedType());
     if (New->hasInitializer()) {
-      if (New->getInitializationStyle() == CXXNewInitializationStyle::List)
+      if (New->getInitializationStyle() == CXXNewInitializationStyle::Braces)
         Out << "il";
       else
         Out << "pi";
@@ -4907,7 +4921,7 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity,
         for (unsigned i = 0, e = PLE->getNumExprs(); i != e; ++i)
           mangleExpression(PLE->getExpr(i));
       } else if (New->getInitializationStyle() ==
-                     CXXNewInitializationStyle::List &&
+                     CXXNewInitializationStyle::Braces &&
                  isa(Init)) {
         // Only take InitListExprs apart for list-initialization.
         mangleInitListElements(cast(Init));
@@ -6073,6 +6087,11 @@ void CXXNameMangler::mangleTemplateArg(TemplateArgument A, bool NeedExactType) {
     mangleNullPointer(A.getNullPtrType());
     break;
   }
+  case TemplateArgument::StructuralValue:
+    mangleValueInTemplateArg(A.getStructuralValueType(),
+                             A.getAsStructuralValue(),
+                             /*TopLevel=*/true, NeedExactType);
+    break;
   case TemplateArgument::Pack: {
     //   ::= J * E
     Out << 'J';
@@ -6481,7 +6500,20 @@ void CXXNameMangler::mangleValueInTemplateArg(QualType T, const APValue &V,
       Out << "plcvPcad";
       Kind = Offset;
     } else {
-      if (!V.getLValuePath().empty() || V.isLValueOnePastTheEnd()) {
+      // Clang 11 and before mangled an array subject to array-to-pointer decay
+      // as if it were the declaration itself.
+      bool IsArrayToPointerDecayMangledAsDecl = false;
+      if (TopLevel && Ctx.getLangOpts().getClangABICompat() <=
+                          LangOptions::ClangABI::Ver11) {
+        QualType BType = B.getType();
+        IsArrayToPointerDecayMangledAsDecl =
+            BType->isArrayType() && V.getLValuePath().size() == 1 &&
+            V.getLValuePath()[0].getAsArrayIndex() == 0 &&
+            Ctx.hasSimilarType(T, Ctx.getDecayedType(BType));
+      }
+
+      if ((!V.getLValuePath().empty() || V.isLValueOnePastTheEnd()) &&
+          !IsArrayToPointerDecayMangledAsDecl) {
         NotPrimaryExpr();
         // A final conversion to the template parameter's type is usually
         // folded into the 'so' mangling, but we can't do that for 'void*'
diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp
index 637d06cee78c..3daba13d0fce 100644
--- a/clang/lib/AST/JSONNodeDumper.cpp
+++ b/clang/lib/AST/JSONNodeDumper.cpp
@@ -930,7 +930,7 @@ void JSONNodeDumper::VisitFunctionDecl(const FunctionDecl *FD) {
     JOS.attribute("storageClass", VarDecl::getStorageClassSpecifierString(SC));
   attributeOnlyIfTrue("inline", FD->isInlineSpecified());
   attributeOnlyIfTrue("virtual", FD->isVirtualAsWritten());
-  attributeOnlyIfTrue("pure", FD->isPure());
+  attributeOnlyIfTrue("pure", FD->isPureVirtual());
   attributeOnlyIfTrue("explicitlyDeleted", FD->isDeletedAsWritten());
   attributeOnlyIfTrue("constexpr", FD->isConstexpr());
   attributeOnlyIfTrue("variadic", FD->isVariadic());
@@ -1356,12 +1356,11 @@ void JSONNodeDumper::VisitCXXNewExpr(const CXXNewExpr *NE) {
   attributeOnlyIfTrue("isPlacement", NE->getNumPlacementArgs() != 0);
   switch (NE->getInitializationStyle()) {
   case CXXNewInitializationStyle::None:
-  case CXXNewInitializationStyle::Implicit:
     break;
-  case CXXNewInitializationStyle::Call:
+  case CXXNewInitializationStyle::Parens:
     JOS.attribute("initStyle", "call");
     break;
-  case CXXNewInitializationStyle::List:
+  case CXXNewInitializationStyle::Braces:
     JOS.attribute("initStyle", "list");
     break;
   }
diff --git a/clang/lib/AST/Mangle.cpp b/clang/lib/AST/Mangle.cpp
index d3a6b61fd2be..30cff1ba2e6f 100644
--- a/clang/lib/AST/Mangle.cpp
+++ b/clang/lib/AST/Mangle.cpp
@@ -464,7 +464,7 @@ class ASTNameGenerator::Implementation {
       SmallString<40> Mangled;
       auto Prefix = getClassSymbolPrefix(Kind, OCD->getASTContext());
       llvm::Mangler::getNameWithPrefix(Mangled, Prefix + ClassName, DL);
-      return std::string(Mangled.str());
+      return std::string(Mangled);
     };
 
     return {
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index fbdc47fc6b12..988ebc18bb5a 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -340,6 +340,7 @@ class MicrosoftCXXNameMangler {
 
 public:
   enum QualifierMangleMode { QMM_Drop, QMM_Mangle, QMM_Escape, QMM_Result };
+  enum class TplArgKind { ClassNTTP, StructuralValue };
 
   MicrosoftCXXNameMangler(MicrosoftMangleContextImpl &C, raw_ostream &Out_)
       : Context(C), Out(Out_), Structor(nullptr), StructorType(-1),
@@ -456,7 +457,7 @@ class MicrosoftCXXNameMangler {
                           const TemplateArgumentList &TemplateArgs);
   void mangleTemplateArg(const TemplateDecl *TD, const TemplateArgument &TA,
                          const NamedDecl *Parm);
-  void mangleTemplateArgValue(QualType T, const APValue &V,
+  void mangleTemplateArgValue(QualType T, const APValue &V, TplArgKind,
                               bool WithScalarType = false);
 
   void mangleObjCProtocol(const ObjCProtocolDecl *PD);
@@ -1091,7 +1092,7 @@ void MicrosoftCXXNameMangler::mangleUnqualifiedName(GlobalDecl GD,
       if (const auto *TPO = dyn_cast(ND)) {
         Out << "?__N";
         mangleTemplateArgValue(TPO->getType().getUnqualifiedType(),
-                               TPO->getValue());
+                               TPO->getValue(), TplArgKind::ClassNTTP);
         break;
       }
 
@@ -1607,6 +1608,22 @@ void MicrosoftCXXNameMangler::mangleTemplateArgs(
   }
 }
 
+/// If value V (with type T) represents a decayed pointer to the first element
+/// of an array, return that array.
+static ValueDecl *getAsArrayToPointerDecayedDecl(QualType T, const APValue &V) {
+  // Must be a pointer...
+  if (!T->isPointerType() || !V.isLValue() || !V.hasLValuePath() ||
+      !V.getLValueBase())
+    return nullptr;
+  // ... to element 0 of an array.
+  QualType BaseT = V.getLValueBase().getType();
+  if (!BaseT->isArrayType() || V.getLValuePath().size() != 1 ||
+      V.getLValuePath()[0].getAsArrayIndex() != 0)
+    return nullptr;
+  return const_cast(
+      V.getLValueBase().dyn_cast());
+}
+
 void MicrosoftCXXNameMangler::mangleTemplateArg(const TemplateDecl *TD,
                                                 const TemplateArgument &TA,
                                                 const NamedDecl *Parm) {
@@ -1672,7 +1689,7 @@ void MicrosoftCXXNameMangler::mangleTemplateArg(const TemplateDecl *TD,
       Out << "$";
       auto *TPO = cast(ND);
       mangleTemplateArgValue(TPO->getType().getUnqualifiedType(),
-                             TPO->getValue());
+                             TPO->getValue(), TplArgKind::ClassNTTP);
     } else {
       mangle(ND, "$1?");
     }
@@ -1715,6 +1732,27 @@ void MicrosoftCXXNameMangler::mangleTemplateArg(const TemplateDecl *TD,
                          cast(Parm), T);
     break;
   }
+  case TemplateArgument::StructuralValue:
+    if (ValueDecl *D = getAsArrayToPointerDecayedDecl(
+            TA.getStructuralValueType(), TA.getAsStructuralValue())) {
+      // Mangle the result of array-to-pointer decay as if it were a reference
+      // to the original declaration, to match MSVC's behavior. This can result
+      // in mangling collisions in some cases!
+      return mangleTemplateArg(
+          TD, TemplateArgument(D, TA.getStructuralValueType()), Parm);
+    }
+    Out << "$";
+    if (cast(Parm)
+            ->getType()
+            ->getContainedDeducedType()) {
+      Out << "M";
+      mangleType(TA.getNonTypeTemplateArgumentType(), SourceRange(), QMM_Drop);
+    }
+    mangleTemplateArgValue(TA.getStructuralValueType(),
+                           TA.getAsStructuralValue(),
+                           TplArgKind::StructuralValue,
+                           /*WithScalarType=*/false);
+    break;
   case TemplateArgument::Expression:
     mangleExpression(TA.getAsExpr(), cast(Parm));
     break;
@@ -1757,6 +1795,7 @@ void MicrosoftCXXNameMangler::mangleTemplateArg(const TemplateDecl *TD,
 
 void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
                                                      const APValue &V,
+                                                     TplArgKind TAK,
                                                      bool WithScalarType) {
   switch (V.getKind()) {
   case APValue::None:
@@ -1809,7 +1848,7 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
         break;
       }
     } else {
-      if (T->isPointerType())
+      if (TAK == TplArgKind::ClassNTTP && T->isPointerType())
         Out << "5";
 
       SmallVector EntryTypes;
@@ -1853,12 +1892,12 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
       auto *VD = Base.dyn_cast();
       if (!VD)
         break;
-      Out << "E";
+      Out << (TAK == TplArgKind::ClassNTTP ? 'E' : '1');
       mangle(VD);
 
       for (const std::function &Mangler : EntryManglers)
         Mangler();
-      if (T->isPointerType())
+      if (TAK == TplArgKind::ClassNTTP && T->isPointerType())
         Out << '@';
     }
 
@@ -1872,11 +1911,18 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
     const CXXRecordDecl *RD =
         T->castAs()->getMostRecentCXXRecordDecl();
     const ValueDecl *D = V.getMemberPointerDecl();
-    if (T->isMemberDataPointerType())
-      mangleMemberDataPointerInClassNTTP(RD, D);
-    else
-      mangleMemberFunctionPointerInClassNTTP(RD,
-                                             cast_or_null(D));
+    if (TAK == TplArgKind::ClassNTTP) {
+      if (T->isMemberDataPointerType())
+        mangleMemberDataPointerInClassNTTP(RD, D);
+      else
+        mangleMemberFunctionPointerInClassNTTP(RD,
+                                               cast_or_null(D));
+    } else {
+      if (T->isMemberDataPointerType())
+        mangleMemberDataPointer(RD, D, "");
+      else
+        mangleMemberFunctionPointer(RD, cast_or_null(D), "");
+    }
     return;
   }
 
@@ -1888,11 +1934,11 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
 
     unsigned BaseIndex = 0;
     for (const CXXBaseSpecifier &B : RD->bases())
-      mangleTemplateArgValue(B.getType(), V.getStructBase(BaseIndex++));
+      mangleTemplateArgValue(B.getType(), V.getStructBase(BaseIndex++), TAK);
     for (const FieldDecl *FD : RD->fields())
       if (!FD->isUnnamedBitfield())
         mangleTemplateArgValue(FD->getType(),
-                               V.getStructField(FD->getFieldIndex()),
+                               V.getStructField(FD->getFieldIndex()), TAK,
                                /*WithScalarType*/ true);
     Out << '@';
     return;
@@ -1903,7 +1949,7 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
     mangleType(T, SourceRange(), QMM_Escape);
     if (const FieldDecl *FD = V.getUnionField()) {
       mangleUnqualifiedName(FD);
-      mangleTemplateArgValue(FD->getType(), V.getUnionValue());
+      mangleTemplateArgValue(FD->getType(), V.getUnionValue(), TAK);
     }
     Out << '@';
     return;
@@ -1935,7 +1981,7 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
       const APValue &ElemV = I < V.getArrayInitializedElts()
                                  ? V.getArrayInitializedElt(I)
                                  : V.getArrayFiller();
-      mangleTemplateArgValue(ElemT, ElemV);
+      mangleTemplateArgValue(ElemT, ElemV, TAK);
       Out << '@';
     }
     Out << '@';
@@ -1952,7 +1998,7 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
     mangleType(ElemT, SourceRange(), QMM_Escape);
     for (unsigned I = 0, N = V.getVectorLength(); I != N; ++I) {
       const APValue &ElemV = V.getVectorElt(I);
-      mangleTemplateArgValue(ElemT, ElemV);
+      mangleTemplateArgValue(ElemT, ElemV, TAK);
       Out << '@';
     }
     Out << "@@";
diff --git a/clang/lib/AST/ODRDiagsEmitter.cpp b/clang/lib/AST/ODRDiagsEmitter.cpp
index 9dcd2ed04f6f..5b1cdc16e2ea 100644
--- a/clang/lib/AST/ODRDiagsEmitter.cpp
+++ b/clang/lib/AST/ODRDiagsEmitter.cpp
@@ -1102,8 +1102,8 @@ bool ODRDiagsEmitter::diagnoseMismatch(
 
     const bool FirstVirtual = FirstMethod->isVirtualAsWritten();
     const bool SecondVirtual = SecondMethod->isVirtualAsWritten();
-    const bool FirstPure = FirstMethod->isPure();
-    const bool SecondPure = SecondMethod->isPure();
+    const bool FirstPure = FirstMethod->isPureVirtual();
+    const bool SecondPure = SecondMethod->isPureVirtual();
     if ((FirstVirtual || SecondVirtual) &&
         (FirstVirtual != SecondVirtual || FirstPure != SecondPure)) {
       DiagMethodError(MethodVirtual) << FirstPure << FirstVirtual;
diff --git a/clang/lib/AST/ODRHash.cpp b/clang/lib/AST/ODRHash.cpp
index aea1a93ae1fa..5b98646a1e8d 100644
--- a/clang/lib/AST/ODRHash.cpp
+++ b/clang/lib/AST/ODRHash.cpp
@@ -180,6 +180,10 @@ void ODRHash::AddTemplateArgument(TemplateArgument TA) {
       TA.getAsIntegral().Profile(ID);
       break;
     }
+    case TemplateArgument::StructuralValue:
+      AddQualType(TA.getStructuralValueType());
+      AddStructuralValue(TA.getAsStructuralValue());
+      break;
     case TemplateArgument::Template:
     case TemplateArgument::TemplateExpansion:
       AddTemplateName(TA.getAsTemplateOrTemplatePattern());
@@ -688,7 +692,7 @@ void ODRHash::AddFunctionDecl(const FunctionDecl *Function,
   ID.AddInteger(Function->getStorageClass());
   AddBoolean(Function->isInlineSpecified());
   AddBoolean(Function->isVirtualAsWritten());
-  AddBoolean(Function->isPure());
+  AddBoolean(Function->isPureVirtual());
   AddBoolean(Function->isDeletedAsWritten());
   AddBoolean(Function->isExplicitlyDefaulted());
 
@@ -741,8 +745,55 @@ void ODRHash::AddEnumDecl(const EnumDecl *Enum) {
   if (Enum->isScoped())
     AddBoolean(Enum->isScopedUsingClassTag());
 
-  if (Enum->getIntegerTypeSourceInfo())
-    AddQualType(Enum->getIntegerType());
+  if (Enum->getIntegerTypeSourceInfo()) {
+    // FIMXE: This allows two enums with different spellings to have the same
+    // hash.
+    //
+    //  // mod1.cppm
+    //  module;
+    //  extern "C" {
+    //      typedef unsigned __int64 size_t;
+    //  }
+    //  namespace std {
+    //      using :: size_t;
+    //  }
+    //
+    //  extern "C++" {
+    //      namespace std {
+    //          enum class align_val_t : std::size_t {};
+    //      }
+    //  }
+    //
+    //  export module mod1;
+    //  export using std::align_val_t;
+    //
+    //  // mod2.cppm
+    //  module;
+    //  extern "C" {
+    //      typedef unsigned __int64 size_t;
+    //  }
+    //
+    //  extern "C++" {
+    //      namespace std {
+    //          enum class align_val_t : size_t {};
+    //      }
+    //  }
+    //
+    //  export module mod2;
+    //  import mod1;
+    //  export using std::align_val_t;
+    //
+    // The above example should be disallowed since it violates
+    // [basic.def.odr]p14:
+    //
+    //    Each such definition shall consist of the same sequence of tokens
+    //
+    // The definitions of `std::align_val_t` in two module units have different
+    // spellings but we failed to give an error here.
+    //
+    // See https://github.com/llvm/llvm-project/issues/76638 for details.
+    AddQualType(Enum->getIntegerType().getCanonicalType());
+  }
 
   // Filter out sub-Decls which will not be processed in order to get an
   // accurate count of Decl's.
@@ -1249,3 +1300,66 @@ void ODRHash::AddQualType(QualType T) {
 void ODRHash::AddBoolean(bool Value) {
   Bools.push_back(Value);
 }
+
+void ODRHash::AddStructuralValue(const APValue &Value) {
+  ID.AddInteger(Value.getKind());
+
+  // 'APValue::Profile' uses pointer values to make hash for LValue and
+  // MemberPointer, but they differ from one compiler invocation to another.
+  // So, handle them explicitly here.
+
+  switch (Value.getKind()) {
+  case APValue::LValue: {
+    const APValue::LValueBase &Base = Value.getLValueBase();
+    if (!Base) {
+      ID.AddInteger(Value.getLValueOffset().getQuantity());
+      break;
+    }
+
+    assert(Base.is());
+    AddDecl(Base.get());
+    ID.AddInteger(Value.getLValueOffset().getQuantity());
+
+    bool OnePastTheEnd = Value.isLValueOnePastTheEnd();
+    if (Value.hasLValuePath()) {
+      QualType TypeSoFar = Base.getType();
+      for (APValue::LValuePathEntry E : Value.getLValuePath()) {
+        if (const auto *AT = TypeSoFar->getAsArrayTypeUnsafe()) {
+          if (const auto *CAT = dyn_cast(AT))
+            OnePastTheEnd |= CAT->getSize() == E.getAsArrayIndex();
+          TypeSoFar = AT->getElementType();
+        } else {
+          const Decl *D = E.getAsBaseOrMember().getPointer();
+          if (const auto *FD = dyn_cast(D)) {
+            if (FD->getParent()->isUnion())
+              ID.AddInteger(FD->getFieldIndex());
+            TypeSoFar = FD->getType();
+          } else {
+            TypeSoFar =
+                D->getASTContext().getRecordType(cast(D));
+          }
+        }
+      }
+    }
+    unsigned Val = 0;
+    if (Value.isNullPointer())
+      Val |= 1 << 0;
+    if (OnePastTheEnd)
+      Val |= 1 << 1;
+    if (Value.hasLValuePath())
+      Val |= 1 << 2;
+    ID.AddInteger(Val);
+    break;
+  }
+  case APValue::MemberPointer: {
+    const ValueDecl *D = Value.getMemberPointerDecl();
+    assert(D);
+    AddDecl(D);
+    ID.AddInteger(
+        D->getASTContext().getMemberPointerPathAdjustment(Value).getQuantity());
+    break;
+  }
+  default:
+    Value.Profile(ID);
+  }
+}
diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp
index 706991f4fb50..6dfaadd92e79 100644
--- a/clang/lib/AST/RecordLayoutBuilder.cpp
+++ b/clang/lib/AST/RecordLayoutBuilder.cpp
@@ -2349,7 +2349,7 @@ static const CXXMethodDecl *computeKeyFunction(ASTContext &Context,
     if (!MD->isVirtual())
       continue;
 
-    if (MD->isPure())
+    if (MD->isPureVirtual())
       continue;
 
     // Ignore implicit member functions, they are always marked as inline, but
@@ -3293,7 +3293,7 @@ void MicrosoftRecordLayoutBuilder::computeVtorDispSet(
   // Seed the working set with our non-destructor, non-pure virtual methods.
   for (const CXXMethodDecl *MD : RD->methods())
     if (MicrosoftVTableContext::hasVtableSlot(MD) &&
-        !isa(MD) && !MD->isPure())
+        !isa(MD) && !MD->isPureVirtual())
       Work.insert(MD);
   while (!Work.empty()) {
     const CXXMethodDecl *MD = *Work.begin();
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index c04cb313c338..9d4aa07ec4da 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -2300,9 +2300,8 @@ void StmtPrinter::VisitCXXNewExpr(CXXNewExpr *E) {
     OS << ")";
 
   CXXNewInitializationStyle InitStyle = E->getInitializationStyle();
-  if (InitStyle != CXXNewInitializationStyle::None &&
-      InitStyle != CXXNewInitializationStyle::Implicit) {
-    bool Bare = InitStyle == CXXNewInitializationStyle::Call &&
+  if (InitStyle != CXXNewInitializationStyle::None) {
+    bool Bare = InitStyle == CXXNewInitializationStyle::Parens &&
                 !isa(E->getInitializer());
     if (Bare)
       OS << "(";
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index d7b980a58570..dd0838edab7b 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -2416,6 +2416,12 @@ void StmtProfiler::VisitTemplateArgument(const TemplateArgument &Arg) {
     Arg.getAsIntegral().Profile(ID);
     break;
 
+  case TemplateArgument::StructuralValue:
+    VisitType(Arg.getStructuralValueType());
+    // FIXME: Do we need to recursively decompose this ourselves?
+    Arg.getAsStructuralValue().Profile(ID);
+    break;
+
   case TemplateArgument::Expression:
     Visit(Arg.getAsExpr());
     break;
diff --git a/clang/lib/AST/TemplateBase.cpp b/clang/lib/AST/TemplateBase.cpp
index 2ec0b2cce08c..2bdbeb08ef20 100644
--- a/clang/lib/AST/TemplateBase.cpp
+++ b/clang/lib/AST/TemplateBase.cpp
@@ -161,8 +161,25 @@ static bool needsAmpersandOnTemplateArg(QualType paramType, QualType argType) {
 // TemplateArgument Implementation
 //===----------------------------------------------------------------------===//
 
-TemplateArgument::TemplateArgument(ASTContext &Ctx, const llvm::APSInt &Value,
-                                   QualType Type, bool IsDefaulted) {
+void TemplateArgument::initFromType(QualType T, bool IsNullPtr,
+                                    bool IsDefaulted) {
+  TypeOrValue.Kind = IsNullPtr ? NullPtr : Type;
+  TypeOrValue.IsDefaulted = IsDefaulted;
+  TypeOrValue.V = reinterpret_cast(T.getAsOpaquePtr());
+}
+
+void TemplateArgument::initFromDeclaration(ValueDecl *D, QualType QT,
+                                           bool IsDefaulted) {
+  assert(D && "Expected decl");
+  DeclArg.Kind = Declaration;
+  DeclArg.IsDefaulted = IsDefaulted;
+  DeclArg.QT = QT.getAsOpaquePtr();
+  DeclArg.D = D;
+}
+
+void TemplateArgument::initFromIntegral(const ASTContext &Ctx,
+                                        const llvm::APSInt &Value,
+                                        QualType Type, bool IsDefaulted) {
   Integer.Kind = Integral;
   Integer.IsDefaulted = IsDefaulted;
   // Copy the APSInt value into our decomposed form.
@@ -181,6 +198,56 @@ TemplateArgument::TemplateArgument(ASTContext &Ctx, const llvm::APSInt &Value,
   Integer.Type = Type.getAsOpaquePtr();
 }
 
+void TemplateArgument::initFromStructural(const ASTContext &Ctx, QualType Type,
+                                          const APValue &V, bool IsDefaulted) {
+  Value.Kind = StructuralValue;
+  Value.IsDefaulted = IsDefaulted;
+  Value.Value = new (Ctx) APValue(V);
+  Ctx.addDestruction(Value.Value);
+  Value.Type = Type.getAsOpaquePtr();
+}
+
+TemplateArgument::TemplateArgument(const ASTContext &Ctx,
+                                   const llvm::APSInt &Value, QualType Type,
+                                   bool IsDefaulted) {
+  initFromIntegral(Ctx, Value, Type, IsDefaulted);
+}
+
+static const ValueDecl *getAsSimpleValueDeclRef(const ASTContext &Ctx,
+                                                QualType T, const APValue &V) {
+  // Pointers to members are relatively easy.
+  if (V.isMemberPointer() && V.getMemberPointerPath().empty())
+    return V.getMemberPointerDecl();
+
+  // We model class non-type template parameters as their template parameter
+  // object declaration.
+  if (V.isStruct() || V.isUnion())
+    return Ctx.getTemplateParamObjectDecl(T, V);
+
+  // Pointers and references with an empty path use the special 'Declaration'
+  // representation.
+  if (V.isLValue() && V.hasLValuePath() && V.getLValuePath().empty() &&
+      !V.isLValueOnePastTheEnd())
+    return V.getLValueBase().dyn_cast();
+
+  // Everything else uses the 'structural' representation.
+  return nullptr;
+}
+
+TemplateArgument::TemplateArgument(const ASTContext &Ctx, QualType Type,
+                                   const APValue &V, bool IsDefaulted) {
+  if (Type->isIntegralOrEnumerationType() && V.isInt())
+    initFromIntegral(Ctx, V.getInt(), Type, IsDefaulted);
+  else if ((V.isLValue() && V.isNullPointer()) ||
+           (V.isMemberPointer() && !V.getMemberPointerDecl()))
+    initFromType(Type, /*isNullPtr=*/true, IsDefaulted);
+  else if (const ValueDecl *VD = getAsSimpleValueDeclRef(Ctx, Type, V))
+    // FIXME: The Declaration form should expose a const ValueDecl*.
+    initFromDeclaration(const_cast(VD), Type, IsDefaulted);
+  else
+    initFromStructural(Ctx, Type, V, IsDefaulted);
+}
+
 TemplateArgument
 TemplateArgument::CreatePackCopy(ASTContext &Context,
                                  ArrayRef Args) {
@@ -221,6 +288,7 @@ TemplateArgumentDependence TemplateArgument::getDependence() const {
 
   case NullPtr:
   case Integral:
+  case StructuralValue:
     return TemplateArgumentDependence::None;
 
   case Expression:
@@ -251,6 +319,7 @@ bool TemplateArgument::isPackExpansion() const {
   case Null:
   case Declaration:
   case Integral:
+  case StructuralValue:
   case Pack:
   case Template:
   case NullPtr:
@@ -301,6 +370,9 @@ QualType TemplateArgument::getNonTypeTemplateArgumentType() const {
 
   case TemplateArgument::NullPtr:
     return getNullPtrType();
+
+  case TemplateArgument::StructuralValue:
+    return getStructuralValueType();
   }
 
   llvm_unreachable("Invalid TemplateArgument Kind!");
@@ -334,8 +406,13 @@ void TemplateArgument::Profile(llvm::FoldingSetNodeID &ID,
     break;
 
   case Integral:
-    getAsIntegral().Profile(ID);
     getIntegralType().Profile(ID);
+    getAsIntegral().Profile(ID);
+    break;
+
+  case StructuralValue:
+    getStructuralValueType().Profile(ID);
+    getAsStructuralValue().Profile(ID);
     break;
 
   case Expression:
@@ -372,6 +449,16 @@ bool TemplateArgument::structurallyEquals(const TemplateArgument &Other) const {
     return getIntegralType() == Other.getIntegralType() &&
            getAsIntegral() == Other.getAsIntegral();
 
+  case StructuralValue: {
+    if (getStructuralValueType() != Other.getStructuralValueType())
+      return false;
+
+    llvm::FoldingSetNodeID A, B;
+    getAsStructuralValue().Profile(A);
+    Other.getAsStructuralValue().Profile(B);
+    return A == B;
+  }
+
   case Pack:
     if (Args.NumArgs != Other.Args.NumArgs) return false;
     for (unsigned I = 0, E = Args.NumArgs; I != E; ++I)
@@ -398,6 +485,7 @@ TemplateArgument TemplateArgument::getPackExpansionPattern() const {
 
   case Declaration:
   case Integral:
+  case StructuralValue:
   case Pack:
   case Null:
   case Template:
@@ -440,6 +528,10 @@ void TemplateArgument::print(const PrintingPolicy &Policy, raw_ostream &Out,
     break;
   }
 
+  case StructuralValue:
+    getAsStructuralValue().printPretty(Out, Policy, getStructuralValueType());
+    break;
+
   case NullPtr:
     // FIXME: Include the type if it's not obvious from the context.
     Out << "nullptr";
@@ -523,6 +615,9 @@ SourceRange TemplateArgumentLoc::getSourceRange() const {
   case TemplateArgument::Integral:
     return getSourceIntegralExpression()->getSourceRange();
 
+  case TemplateArgument::StructuralValue:
+    return getSourceStructuralValueExpression()->getSourceRange();
+
   case TemplateArgument::Pack:
   case TemplateArgument::Null:
     return SourceRange();
@@ -551,6 +646,18 @@ static const T &DiagTemplateArg(const T &DB, const TemplateArgument &Arg) {
   case TemplateArgument::Integral:
     return DB << toString(Arg.getAsIntegral(), 10);
 
+  case TemplateArgument::StructuralValue: {
+    // FIXME: We're guessing at LangOptions!
+    SmallString<32> Str;
+    llvm::raw_svector_ostream OS(Str);
+    LangOptions LangOpts;
+    LangOpts.CPlusPlus = true;
+    PrintingPolicy Policy(LangOpts);
+    Arg.getAsStructuralValue().printPretty(OS, Policy,
+                                           Arg.getStructuralValueType());
+    return DB << OS.str();
+  }
+
   case TemplateArgument::Template:
     return DB << Arg.getAsTemplate();
 
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index e8274fcd5cfe..48c6729a6738 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -1094,6 +1094,16 @@ void clang::TextNodeDumper::VisitReturnStmt(const ReturnStmt *Node) {
   }
 }
 
+void clang::TextNodeDumper::VisitCoawaitExpr(const CoawaitExpr *Node) {
+  if (Node->isImplicit())
+    OS << " implicit";
+}
+
+void clang::TextNodeDumper::VisitCoreturnStmt(const CoreturnStmt *Node) {
+  if (Node->isImplicit())
+    OS << " implicit";
+}
+
 void TextNodeDumper::VisitConstantExpr(const ConstantExpr *Node) {
   if (Node->hasAPValueResult())
     AddChild("value",
@@ -1865,7 +1875,7 @@ void TextNodeDumper::VisitFunctionDecl(const FunctionDecl *D) {
   if (D->isModulePrivate())
     OS << " __module_private__";
 
-  if (D->isPure())
+  if (D->isPureVirtual())
     OS << " pure";
   if (D->isDefaulted()) {
     OS << " default";
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 94def751a72c..810d61a65910 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -2607,19 +2607,22 @@ bool QualType::isTrivialType(const ASTContext &Context) const {
   return false;
 }
 
-bool QualType::isTriviallyCopyableType(const ASTContext &Context) const {
-  if ((*this)->isArrayType())
-    return Context.getBaseElementType(*this).isTriviallyCopyableType(Context);
+static bool isTriviallyCopyableTypeImpl(const QualType &type,
+                                        const ASTContext &Context,
+                                        bool IsCopyConstructible) {
+  if (type->isArrayType())
+    return isTriviallyCopyableTypeImpl(Context.getBaseElementType(type),
+                                       Context, IsCopyConstructible);
 
-  if (hasNonTrivialObjCLifetime())
+  if (type.hasNonTrivialObjCLifetime())
     return false;
 
   // C++11 [basic.types]p9 - See Core 2094
   //   Scalar types, trivially copyable class types, arrays of such types, and
   //   cv-qualified versions of these types are collectively
-  //   called trivially copyable types.
+  //   called trivially copy constructible types.
 
-  QualType CanonicalType = getCanonicalType();
+  QualType CanonicalType = type.getCanonicalType();
   if (CanonicalType->isDependentType())
     return false;
 
@@ -2637,16 +2640,29 @@ bool QualType::isTriviallyCopyableType(const ASTContext &Context) const {
 
   if (const auto *RT = CanonicalType->getAs()) {
     if (const auto *ClassDecl = dyn_cast(RT->getDecl())) {
-      if (!ClassDecl->isTriviallyCopyable()) return false;
+      if (IsCopyConstructible) {
+        return ClassDecl->isTriviallyCopyConstructible();
+      } else {
+        return ClassDecl->isTriviallyCopyable();
+      }
     }
-
     return true;
   }
-
   // No other types can match.
   return false;
 }
 
+bool QualType::isTriviallyCopyableType(const ASTContext &Context) const {
+  return isTriviallyCopyableTypeImpl(*this, Context,
+                                     /*IsCopyConstructible=*/false);
+}
+
+bool QualType::isTriviallyCopyConstructibleType(
+    const ASTContext &Context) const {
+  return isTriviallyCopyableTypeImpl(*this, Context,
+                                     /*IsCopyConstructible=*/true);
+}
+
 bool QualType::isTriviallyRelocatableType(const ASTContext &Context) const {
   QualType BaseElementType = Context.getBaseElementType(*this);
 
@@ -3421,6 +3437,13 @@ StringRef FunctionType::getNameForCallConv(CallingConv CC) {
   llvm_unreachable("Invalid calling convention.");
 }
 
+void FunctionProtoType::ExceptionSpecInfo::instantiate() {
+  assert(Type == EST_Uninstantiated);
+  NoexceptExpr =
+      cast(SourceTemplate->getType())->getNoexceptExpr();
+  Type = EST_DependentNoexcept;
+}
+
 FunctionProtoType::FunctionProtoType(QualType result, ArrayRef params,
                                      QualType canonical,
                                      const ExtProtoInfo &epi)
@@ -3443,6 +3466,14 @@ FunctionProtoType::FunctionProtoType(QualType result, ArrayRef params,
     FunctionTypeBits.HasExtraBitfields = false;
   }
 
+  if (epi.requiresFunctionProtoTypeArmAttributes()) {
+    auto &ArmTypeAttrs = *getTrailingObjects();
+    ArmTypeAttrs = FunctionTypeArmAttributes();
+
+    // Also set the bit in FunctionTypeExtraBitfields
+    auto &ExtraBits = *getTrailingObjects();
+    ExtraBits.HasArmTypeAttributes = true;
+  }
 
   // Fill in the trailing argument array.
   auto *argSlot = getTrailingObjects();
@@ -3454,10 +3485,10 @@ FunctionProtoType::FunctionProtoType(QualType result, ArrayRef params,
 
   // Propagate the SME ACLE attributes.
   if (epi.AArch64SMEAttributes != SME_NormalFunction) {
-    auto &ExtraBits = *getTrailingObjects();
+    auto &ArmTypeAttrs = *getTrailingObjects();
     assert(epi.AArch64SMEAttributes <= SME_AttributeMask &&
            "Not enough bits to encode SME attributes");
-    ExtraBits.AArch64SMEAttributes = epi.AArch64SMEAttributes;
+    ArmTypeAttrs.AArch64SMEAttributes = epi.AArch64SMEAttributes;
   }
 
   // Fill in the exception type array if present.
diff --git a/clang/lib/AST/TypeLoc.cpp b/clang/lib/AST/TypeLoc.cpp
index 10deaa7673b0..b8cb86441173 100644
--- a/clang/lib/AST/TypeLoc.cpp
+++ b/clang/lib/AST/TypeLoc.cpp
@@ -592,6 +592,7 @@ void TemplateSpecializationTypeLoc::initializeArgLocs(
     case TemplateArgument::Integral:
     case TemplateArgument::Declaration:
     case TemplateArgument::NullPtr:
+    case TemplateArgument::StructuralValue:
       ArgInfos[i] = TemplateArgumentLocInfo();
       break;
 
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index f69412429273..80b42c8f84a0 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -937,15 +937,28 @@ void TypePrinter::printFunctionProtoAfter(const FunctionProtoType *T,
   OS << ')';
 
   FunctionType::ExtInfo Info = T->getExtInfo();
+  unsigned SMEBits = T->getAArch64SMEAttributes();
 
-  if ((T->getAArch64SMEAttributes() & FunctionType::SME_PStateSMCompatibleMask))
+  if (SMEBits & FunctionType::SME_PStateSMCompatibleMask)
     OS << " __arm_streaming_compatible";
-  if ((T->getAArch64SMEAttributes() & FunctionType::SME_PStateSMEnabledMask))
+  if (SMEBits & FunctionType::SME_PStateSMEnabledMask)
     OS << " __arm_streaming";
-  if ((T->getAArch64SMEAttributes() & FunctionType::SME_PStateZASharedMask))
-    OS << " __arm_shared_za";
-  if ((T->getAArch64SMEAttributes() & FunctionType::SME_PStateZAPreservedMask))
-    OS << " __arm_preserves_za";
+  if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_Preserves)
+    OS << " __arm_preserves(\"za\")";
+  if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_In)
+    OS << " __arm_in(\"za\")";
+  if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_Out)
+    OS << " __arm_out(\"za\")";
+  if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_InOut)
+    OS << " __arm_inout(\"za\")";
+  if (FunctionType::getArmZT0State(SMEBits) == FunctionType::ARM_Preserves)
+    OS << " __arm_preserves(\"zt0\")";
+  if (FunctionType::getArmZT0State(SMEBits) == FunctionType::ARM_In)
+    OS << " __arm_in(\"zt0\")";
+  if (FunctionType::getArmZT0State(SMEBits) == FunctionType::ARM_Out)
+    OS << " __arm_out(\"zt0\")";
+  if (FunctionType::getArmZT0State(SMEBits) == FunctionType::ARM_InOut)
+    OS << " __arm_inout(\"zt0\")";
 
   printFunctionAfter(Info, OS);
 
@@ -1788,14 +1801,6 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
     OS << "__arm_streaming_compatible";
     return;
   }
-  if (T->getAttrKind() == attr::ArmSharedZA) {
-    OS << "__arm_shared_za";
-    return;
-  }
-  if (T->getAttrKind() == attr::ArmPreservesZA) {
-    OS << "__arm_preserves_za";
-    return;
-  }
 
   OS << " __attribute__((";
   switch (T->getAttrKind()) {
@@ -1839,8 +1844,10 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
   case attr::WebAssemblyFuncref:
   case attr::ArmStreaming:
   case attr::ArmStreamingCompatible:
-  case attr::ArmSharedZA:
-  case attr::ArmPreservesZA:
+  case attr::ArmIn:
+  case attr::ArmOut:
+  case attr::ArmInOut:
+  case attr::ArmPreserves:
     llvm_unreachable("This attribute should have been handled already");
 
   case attr::NSReturnsRetained:
diff --git a/clang/lib/AST/VTableBuilder.cpp b/clang/lib/AST/VTableBuilder.cpp
index 69861c65f6e3..a956ca5b37ac 100644
--- a/clang/lib/AST/VTableBuilder.cpp
+++ b/clang/lib/AST/VTableBuilder.cpp
@@ -422,7 +422,7 @@ void FinalOverriders::dump(raw_ostream &Out, BaseSubobject Base,
     Out << ", " << Overrider.Offset.getQuantity() << ')';
 
     BaseOffset Offset;
-    if (!Overrider.Method->isPure())
+    if (!Overrider.Method->isPureVirtual())
       Offset = ComputeReturnAdjustmentBaseOffset(Context, Overrider.Method, MD);
 
     if (!Offset.isEmpty()) {
@@ -1261,7 +1261,7 @@ ThisAdjustment ItaniumVTableBuilder::ComputeThisAdjustment(
     const CXXMethodDecl *MD, CharUnits BaseOffsetInLayoutClass,
     FinalOverriders::OverriderInfo Overrider) {
   // Ignore adjustments for pure virtual member functions.
-  if (Overrider.Method->isPure())
+  if (Overrider.Method->isPureVirtual())
     return ThisAdjustment();
 
   BaseSubobject OverriddenBaseSubobject(MD->getParent(),
@@ -1607,7 +1607,7 @@ void ItaniumVTableBuilder::AddMethods(
     // Check if this overrider needs a return adjustment.
     // We don't want to do this for pure virtual member functions.
     BaseOffset ReturnAdjustmentOffset;
-    if (!OverriderMD->isPure()) {
+    if (!OverriderMD->isPureVirtual()) {
       ReturnAdjustmentOffset =
         ComputeReturnAdjustmentBaseOffset(Context, OverriderMD, MD);
     }
@@ -1959,7 +1959,7 @@ void ItaniumVTableBuilder::dumpLayout(raw_ostream &Out) {
       std::string Str = PredefinedExpr::ComputeName(
           PredefinedIdentKind::PrettyFunctionNoVirtual, MD);
       Out << Str;
-      if (MD->isPure())
+      if (MD->isPureVirtual())
         Out << " [pure]";
 
       if (MD->isDeleted())
@@ -2010,7 +2010,7 @@ void ItaniumVTableBuilder::dumpLayout(raw_ostream &Out) {
       else
         Out << "() [deleting]";
 
-      if (DD->isPure())
+      if (DD->isPureVirtual())
         Out << " [pure]";
 
       ThunkInfo Thunk = VTableThunks.lookup(I);
@@ -2038,7 +2038,7 @@ void ItaniumVTableBuilder::dumpLayout(raw_ostream &Out) {
       std::string Str = PredefinedExpr::ComputeName(
           PredefinedIdentKind::PrettyFunctionNoVirtual, MD);
       Out << "[unused] " << Str;
-      if (MD->isPure())
+      if (MD->isPureVirtual())
         Out << " [pure]";
     }
 
@@ -3076,7 +3076,7 @@ void VFTableBuilder::AddMethods(BaseSubobject Base, unsigned BaseDepth,
     // We don't want to do this for pure virtual member functions.
     BaseOffset ReturnAdjustmentOffset;
     ReturnAdjustment ReturnAdjustment;
-    if (!FinalOverriderMD->isPure()) {
+    if (!FinalOverriderMD->isPureVirtual()) {
       ReturnAdjustmentOffset =
           ComputeReturnAdjustmentBaseOffset(Context, FinalOverriderMD, MD);
     }
@@ -3175,7 +3175,7 @@ void VFTableBuilder::dumpLayout(raw_ostream &Out) {
       std::string Str = PredefinedExpr::ComputeName(
           PredefinedIdentKind::PrettyFunctionNoVirtual, MD);
       Out << Str;
-      if (MD->isPure())
+      if (MD->isPureVirtual())
         Out << " [pure]";
 
       if (MD->isDeleted())
@@ -3194,7 +3194,7 @@ void VFTableBuilder::dumpLayout(raw_ostream &Out) {
       DD->printQualifiedName(Out);
       Out << "() [scalar deleting]";
 
-      if (DD->isPure())
+      if (DD->isPureVirtual())
         Out << " [pure]";
 
       ThunkInfo Thunk = VTableThunks.lookup(I);
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 8ed213ca2ce0..bf87b1aa0992 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -893,6 +893,7 @@ const internal::VariadicDynCastAllOfMatcher
     cxxOperatorCallExpr;
 const internal::VariadicDynCastAllOfMatcher
     cxxRewrittenBinaryOperator;
+const internal::VariadicDynCastAllOfMatcher cxxFoldExpr;
 const internal::VariadicDynCastAllOfMatcher expr;
 const internal::VariadicDynCastAllOfMatcher declRefExpr;
 const internal::VariadicDynCastAllOfMatcher objcIvarRefExpr;
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index 2e43dec331b7..15dad022df5f 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -198,6 +198,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(cxxDependentScopeMemberExpr);
   REGISTER_MATCHER(cxxDestructorDecl);
   REGISTER_MATCHER(cxxDynamicCastExpr);
+  REGISTER_MATCHER(cxxFoldExpr);
   REGISTER_MATCHER(cxxForRangeStmt);
   REGISTER_MATCHER(cxxFunctionalCastExpr);
   REGISTER_MATCHER(cxxMemberCallExpr);
@@ -319,6 +320,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(hasExplicitSpecifier);
   REGISTER_MATCHER(hasExternalFormalLinkage);
   REGISTER_MATCHER(hasFalseExpression);
+  REGISTER_MATCHER(hasFoldInit);
   REGISTER_MATCHER(hasGlobalStorage);
   REGISTER_MATCHER(hasImplicitDestinationType);
   REGISTER_MATCHER(hasInClassInitializer);
@@ -344,6 +346,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(hasOverloadedOperatorName);
   REGISTER_MATCHER(hasParameter);
   REGISTER_MATCHER(hasParent);
+  REGISTER_MATCHER(hasPattern);
   REGISTER_MATCHER(hasPointeeLoc);
   REGISTER_MATCHER(hasQualifier);
   REGISTER_MATCHER(hasRHS);
@@ -404,6 +407,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(isAssignmentOperator);
   REGISTER_MATCHER(isAtPosition);
   REGISTER_MATCHER(isBaseInitializer);
+  REGISTER_MATCHER(isBinaryFold);
   REGISTER_MATCHER(isBitField);
   REGISTER_MATCHER(isCatchAll);
   REGISTER_MATCHER(isClass);
@@ -447,6 +451,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(isInteger);
   REGISTER_MATCHER(isIntegral);
   REGISTER_MATCHER(isLambda);
+  REGISTER_MATCHER(isLeftFold);
   REGISTER_MATCHER(isListInitialization);
   REGISTER_MATCHER(isMain);
   REGISTER_MATCHER(isMemberInitializer);
@@ -460,6 +465,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(isProtected);
   REGISTER_MATCHER(isPublic);
   REGISTER_MATCHER(isPure);
+  REGISTER_MATCHER(isRightFold);
   REGISTER_MATCHER(isScoped);
   REGISTER_MATCHER(isSharedKind);
   REGISTER_MATCHER(isSignedInteger);
@@ -469,6 +475,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(isStruct);
   REGISTER_MATCHER(isTemplateInstantiation);
   REGISTER_MATCHER(isTypeDependent);
+  REGISTER_MATCHER(isUnaryFold);
   REGISTER_MATCHER(isUnion);
   REGISTER_MATCHER(isUnsignedInteger);
   REGISTER_MATCHER(isUserProvided);
diff --git a/clang/lib/Analysis/ExprMutationAnalyzer.cpp b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
index 624a643cc60e..bb042760d297 100644
--- a/clang/lib/Analysis/ExprMutationAnalyzer.cpp
+++ b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
@@ -15,6 +15,81 @@
 namespace clang {
 using namespace ast_matchers;
 
+// Check if result of Source expression could be a Target expression.
+// Checks:
+//  - Implicit Casts
+//  - Binary Operators
+//  - ConditionalOperator
+//  - BinaryConditionalOperator
+static bool canExprResolveTo(const Expr *Source, const Expr *Target) {
+
+  const auto IgnoreDerivedToBase = [](const Expr *E, auto Matcher) {
+    if (Matcher(E))
+      return true;
+    if (const auto *Cast = dyn_cast(E)) {
+      if ((Cast->getCastKind() == CK_DerivedToBase ||
+           Cast->getCastKind() == CK_UncheckedDerivedToBase) &&
+          Matcher(Cast->getSubExpr()))
+        return true;
+    }
+    return false;
+  };
+
+  const auto EvalCommaExpr = [](const Expr *E, auto Matcher) {
+    const Expr *Result = E;
+    while (const auto *BOComma =
+               dyn_cast_or_null(Result->IgnoreParens())) {
+      if (!BOComma->isCommaOp())
+        break;
+      Result = BOComma->getRHS();
+    }
+
+    return Result != E && Matcher(Result);
+  };
+
+  // The 'ConditionalOperatorM' matches on ` ?  : `.
+  // This matching must be recursive because `` can be anything resolving
+  // to the `InnerMatcher`, for example another conditional operator.
+  // The edge-case `BaseClass &b =  ? DerivedVar1 : DerivedVar2;`
+  // is handled, too. The implicit cast happens outside of the conditional.
+  // This is matched by `IgnoreDerivedToBase(canResolveToExpr(InnerMatcher))`
+  // below.
+  const auto ConditionalOperatorM = [Target](const Expr *E) {
+    if (const auto *OP = dyn_cast(E)) {
+      if (const auto *TE = OP->getTrueExpr()->IgnoreParens())
+        if (canExprResolveTo(TE, Target))
+          return true;
+      if (const auto *FE = OP->getFalseExpr()->IgnoreParens())
+        if (canExprResolveTo(FE, Target))
+          return true;
+    }
+    return false;
+  };
+
+  const auto ElvisOperator = [Target](const Expr *E) {
+    if (const auto *OP = dyn_cast(E)) {
+      if (const auto *TE = OP->getTrueExpr()->IgnoreParens())
+        if (canExprResolveTo(TE, Target))
+          return true;
+      if (const auto *FE = OP->getFalseExpr()->IgnoreParens())
+        if (canExprResolveTo(FE, Target))
+          return true;
+    }
+    return false;
+  };
+
+  const Expr *SourceExprP = Source->IgnoreParens();
+  return IgnoreDerivedToBase(SourceExprP,
+                             [&](const Expr *E) {
+                               return E == Target || ConditionalOperatorM(E) ||
+                                      ElvisOperator(E);
+                             }) ||
+         EvalCommaExpr(SourceExprP, [&](const Expr *E) {
+           return IgnoreDerivedToBase(
+               E->IgnoreParens(), [&](const Expr *EE) { return EE == Target; });
+         });
+}
+
 namespace {
 
 AST_MATCHER_P(LambdaExpr, hasCaptureInit, const Expr *, E) {
@@ -27,56 +102,14 @@ AST_MATCHER_P(CXXForRangeStmt, hasRangeStmt,
   return InnerMatcher.matches(*Range, Finder, Builder);
 }
 
-AST_MATCHER_P(Expr, maybeEvalCommaExpr, ast_matchers::internal::Matcher,
-              InnerMatcher) {
-  const Expr *Result = &Node;
-  while (const auto *BOComma =
-             dyn_cast_or_null(Result->IgnoreParens())) {
-    if (!BOComma->isCommaOp())
-      break;
-    Result = BOComma->getRHS();
-  }
-  return InnerMatcher.matches(*Result, Finder, Builder);
-}
-
-AST_MATCHER_P(Stmt, canResolveToExpr, ast_matchers::internal::Matcher,
-              InnerMatcher) {
+AST_MATCHER_P(Stmt, canResolveToExpr, const Stmt *, Inner) {
   auto *Exp = dyn_cast(&Node);
-  if (!Exp) {
-    return stmt().matches(Node, Finder, Builder);
-  }
-
-  auto DerivedToBase = [](const ast_matchers::internal::Matcher &Inner) {
-    return implicitCastExpr(anyOf(hasCastKind(CK_DerivedToBase),
-                                  hasCastKind(CK_UncheckedDerivedToBase)),
-                            hasSourceExpression(Inner));
-  };
-  auto IgnoreDerivedToBase =
-      [&DerivedToBase](const ast_matchers::internal::Matcher &Inner) {
-        return ignoringParens(expr(anyOf(Inner, DerivedToBase(Inner))));
-      };
-
-  // The 'ConditionalOperator' matches on ` ?  : `.
-  // This matching must be recursive because `` can be anything resolving
-  // to the `InnerMatcher`, for example another conditional operator.
-  // The edge-case `BaseClass &b =  ? DerivedVar1 : DerivedVar2;`
-  // is handled, too. The implicit cast happens outside of the conditional.
-  // This is matched by `IgnoreDerivedToBase(canResolveToExpr(InnerMatcher))`
-  // below.
-  auto const ConditionalOperator = conditionalOperator(anyOf(
-      hasTrueExpression(ignoringParens(canResolveToExpr(InnerMatcher))),
-      hasFalseExpression(ignoringParens(canResolveToExpr(InnerMatcher)))));
-  auto const ElvisOperator = binaryConditionalOperator(anyOf(
-      hasTrueExpression(ignoringParens(canResolveToExpr(InnerMatcher))),
-      hasFalseExpression(ignoringParens(canResolveToExpr(InnerMatcher)))));
-
-  auto const ComplexMatcher = ignoringParens(
-      expr(anyOf(IgnoreDerivedToBase(InnerMatcher),
-                 maybeEvalCommaExpr(IgnoreDerivedToBase(InnerMatcher)),
-                 IgnoreDerivedToBase(ConditionalOperator),
-                 IgnoreDerivedToBase(ElvisOperator))));
-
-  return ComplexMatcher.matches(*Exp, Finder, Builder);
+  if (!Exp)
+    return true;
+  auto *Target = dyn_cast(Inner);
+  if (!Target)
+    return false;
+  return canExprResolveTo(Exp, Target);
 }
 
 // Similar to 'hasAnyArgument', but does not work because 'InitListExpr' does
@@ -121,6 +154,12 @@ AST_MATCHER_P(GenericSelectionExpr, hasControllingExpr,
   return InnerMatcher.matches(*Node.getControllingExpr(), Finder, Builder);
 }
 
+template 
+ast_matchers::internal::Matcher
+findFirst(const ast_matchers::internal::Matcher &Matcher) {
+  return anyOf(Matcher, hasDescendant(Matcher));
+}
+
 const auto nonConstReferenceType = [] {
   return hasUnqualifiedDesugaredType(
       referenceType(pointee(unless(isConstQualified()))));
@@ -204,9 +243,16 @@ const Stmt *ExprMutationAnalyzer::findMutationMemoized(
 
 const Stmt *ExprMutationAnalyzer::tryEachDeclRef(const Decl *Dec,
                                                  MutationFinder Finder) {
-  const auto Refs =
-      match(findAll(declRefExpr(to(equalsNode(Dec))).bind(NodeID::value)),
-            Stm, Context);
+  const auto Refs = match(
+      findAll(
+          declRefExpr(to(
+                          // `Dec` or a binding if `Dec` is a decomposition.
+                          anyOf(equalsNode(Dec),
+                                bindingDecl(forDecomposition(equalsNode(Dec))))
+                          //
+                          ))
+              .bind(NodeID::value)),
+      Stm, Context);
   for (const auto &RefNodes : Refs) {
     const auto *E = RefNodes.getNodeAs(NodeID::value);
     if ((this->*Finder)(E))
@@ -220,8 +266,8 @@ bool ExprMutationAnalyzer::isUnevaluated(const Stmt *Exp, const Stmt &Stm,
   return selectFirst(
              NodeID::value,
              match(
-                 findAll(
-                     stmt(canResolveToExpr(equalsNode(Exp)),
+                 findFirst(
+                     stmt(canResolveToExpr(Exp),
                           anyOf(
                               // `Exp` is part of the underlying expression of
                               // decltype/typeof if it has an ancestor of
@@ -275,44 +321,45 @@ const Stmt *ExprMutationAnalyzer::findDeclPointeeMutation(
 
 const Stmt *ExprMutationAnalyzer::findDirectMutation(const Expr *Exp) {
   // LHS of any assignment operators.
-  const auto AsAssignmentLhs = binaryOperator(
-      isAssignmentOperator(), hasLHS(canResolveToExpr(equalsNode(Exp))));
+  const auto AsAssignmentLhs =
+      binaryOperator(isAssignmentOperator(), hasLHS(canResolveToExpr(Exp)));
 
   // Operand of increment/decrement operators.
   const auto AsIncDecOperand =
       unaryOperator(anyOf(hasOperatorName("++"), hasOperatorName("--")),
-                    hasUnaryOperand(canResolveToExpr(equalsNode(Exp))));
+                    hasUnaryOperand(canResolveToExpr(Exp)));
 
   // Invoking non-const member function.
   // A member function is assumed to be non-const when it is unresolved.
   const auto NonConstMethod = cxxMethodDecl(unless(isConst()));
 
   const auto AsNonConstThis = expr(anyOf(
-      cxxMemberCallExpr(on(canResolveToExpr(equalsNode(Exp))),
-                        unless(isConstCallee())),
+      cxxMemberCallExpr(on(canResolveToExpr(Exp)), unless(isConstCallee())),
       cxxOperatorCallExpr(callee(NonConstMethod),
-                          hasArgument(0, canResolveToExpr(equalsNode(Exp)))),
+                          hasArgument(0, canResolveToExpr(Exp))),
       // In case of a templated type, calling overloaded operators is not
       // resolved and modelled as `binaryOperator` on a dependent type.
       // Such instances are considered a modification, because they can modify
       // in different instantiations of the template.
-      binaryOperator(
-          hasEitherOperand(ignoringImpCasts(canResolveToExpr(equalsNode(Exp)))),
-          isTypeDependent()),
+      binaryOperator(isTypeDependent(),
+                     hasEitherOperand(ignoringImpCasts(canResolveToExpr(Exp)))),
+      // A fold expression may contain `Exp` as it's initializer.
+      // We don't know if the operator modifies `Exp` because the
+      // operator is type dependent due to the parameter pack.
+      cxxFoldExpr(hasFoldInit(ignoringImpCasts(canResolveToExpr(Exp)))),
       // Within class templates and member functions the member expression might
       // not be resolved. In that case, the `callExpr` is considered to be a
       // modification.
-      callExpr(
-          callee(expr(anyOf(unresolvedMemberExpr(hasObjectExpression(
-                                canResolveToExpr(equalsNode(Exp)))),
-                            cxxDependentScopeMemberExpr(hasObjectExpression(
-                                canResolveToExpr(equalsNode(Exp)))))))),
+      callExpr(callee(expr(anyOf(
+          unresolvedMemberExpr(hasObjectExpression(canResolveToExpr(Exp))),
+          cxxDependentScopeMemberExpr(
+              hasObjectExpression(canResolveToExpr(Exp))))))),
       // Match on a call to a known method, but the call itself is type
       // dependent (e.g. `vector v; v.push(T{});` in a templated function).
-      callExpr(allOf(isTypeDependent(),
-                     callee(memberExpr(hasDeclaration(NonConstMethod),
-                                       hasObjectExpression(canResolveToExpr(
-                                           equalsNode(Exp)))))))));
+      callExpr(allOf(
+          isTypeDependent(),
+          callee(memberExpr(hasDeclaration(NonConstMethod),
+                            hasObjectExpression(canResolveToExpr(Exp))))))));
 
   // Taking address of 'Exp'.
   // We're assuming 'Exp' is mutated as soon as its address is taken, though in
@@ -322,11 +369,10 @@ const Stmt *ExprMutationAnalyzer::findDirectMutation(const Expr *Exp) {
       unaryOperator(hasOperatorName("&"),
                     // A NoOp implicit cast is adding const.
                     unless(hasParent(implicitCastExpr(hasCastKind(CK_NoOp)))),
-                    hasUnaryOperand(canResolveToExpr(equalsNode(Exp))));
-  const auto AsPointerFromArrayDecay =
-      castExpr(hasCastKind(CK_ArrayToPointerDecay),
-               unless(hasParent(arraySubscriptExpr())),
-               has(canResolveToExpr(equalsNode(Exp))));
+                    hasUnaryOperand(canResolveToExpr(Exp)));
+  const auto AsPointerFromArrayDecay = castExpr(
+      hasCastKind(CK_ArrayToPointerDecay),
+      unless(hasParent(arraySubscriptExpr())), has(canResolveToExpr(Exp)));
   // Treat calling `operator->()` of move-only classes as taking address.
   // These are typically smart pointers with unique ownership so we treat
   // mutation of pointee as mutation of the smart pointer itself.
@@ -334,7 +380,7 @@ const Stmt *ExprMutationAnalyzer::findDirectMutation(const Expr *Exp) {
       hasOverloadedOperatorName("->"),
       callee(
           cxxMethodDecl(ofClass(isMoveOnly()), returns(nonConstPointerType()))),
-      argumentCountIs(1), hasArgument(0, canResolveToExpr(equalsNode(Exp))));
+      argumentCountIs(1), hasArgument(0, canResolveToExpr(Exp)));
 
   // Used as non-const-ref argument when calling a function.
   // An argument is assumed to be non-const-ref when the function is unresolved.
@@ -342,8 +388,8 @@ const Stmt *ExprMutationAnalyzer::findDirectMutation(const Expr *Exp) {
   // findFunctionArgMutation which has additional smarts for handling forwarding
   // references.
   const auto NonConstRefParam = forEachArgumentWithParamType(
-      anyOf(canResolveToExpr(equalsNode(Exp)),
-            memberExpr(hasObjectExpression(canResolveToExpr(equalsNode(Exp))))),
+      anyOf(canResolveToExpr(Exp),
+            memberExpr(hasObjectExpression(canResolveToExpr(Exp)))),
       nonConstReferenceType());
   const auto NotInstantiated = unless(hasDeclaration(isInstantiated()));
   const auto TypeDependentCallee =
@@ -354,19 +400,17 @@ const Stmt *ExprMutationAnalyzer::findDirectMutation(const Expr *Exp) {
   const auto AsNonConstRefArg = anyOf(
       callExpr(NonConstRefParam, NotInstantiated),
       cxxConstructExpr(NonConstRefParam, NotInstantiated),
-      callExpr(TypeDependentCallee,
-               hasAnyArgument(canResolveToExpr(equalsNode(Exp)))),
-      cxxUnresolvedConstructExpr(
-          hasAnyArgument(canResolveToExpr(equalsNode(Exp)))),
+      callExpr(TypeDependentCallee, hasAnyArgument(canResolveToExpr(Exp))),
+      cxxUnresolvedConstructExpr(hasAnyArgument(canResolveToExpr(Exp))),
       // Previous False Positive in the following Code:
       // `template  void f() { int i = 42; new Type(i); }`
       // Where the constructor of `Type` takes its argument as reference.
       // The AST does not resolve in a `cxxConstructExpr` because it is
       // type-dependent.
-      parenListExpr(hasDescendant(expr(canResolveToExpr(equalsNode(Exp))))),
+      parenListExpr(hasDescendant(expr(canResolveToExpr(Exp)))),
       // If the initializer is for a reference type, there is no cast for
       // the variable. Values are cast to RValue first.
-      initListExpr(hasAnyInit(expr(canResolveToExpr(equalsNode(Exp))))));
+      initListExpr(hasAnyInit(expr(canResolveToExpr(Exp)))));
 
   // Captured by a lambda by reference.
   // If we're initializing a capture with 'Exp' directly then we're initializing
@@ -380,76 +424,72 @@ const Stmt *ExprMutationAnalyzer::findDirectMutation(const Expr *Exp) {
   // For returning by const-ref there will be an ImplicitCastExpr  (for
   // adding const.)
   const auto AsNonConstRefReturn =
-      returnStmt(hasReturnValue(canResolveToExpr(equalsNode(Exp))));
+      returnStmt(hasReturnValue(canResolveToExpr(Exp)));
 
   // It is used as a non-const-reference for initalizing a range-for loop.
-  const auto AsNonConstRefRangeInit = cxxForRangeStmt(
-      hasRangeInit(declRefExpr(allOf(canResolveToExpr(equalsNode(Exp)),
-                                     hasType(nonConstReferenceType())))));
+  const auto AsNonConstRefRangeInit = cxxForRangeStmt(hasRangeInit(declRefExpr(
+      allOf(canResolveToExpr(Exp), hasType(nonConstReferenceType())))));
 
   const auto Matches = match(
-      traverse(TK_AsIs,
-               findAll(stmt(anyOf(AsAssignmentLhs, AsIncDecOperand,
-                                  AsNonConstThis, AsAmpersandOperand,
-                                  AsPointerFromArrayDecay, AsOperatorArrowThis,
-                                  AsNonConstRefArg, AsLambdaRefCaptureInit,
-                                  AsNonConstRefReturn, AsNonConstRefRangeInit))
-                           .bind("stmt"))),
+      traverse(
+          TK_AsIs,
+          findFirst(stmt(anyOf(AsAssignmentLhs, AsIncDecOperand, AsNonConstThis,
+                               AsAmpersandOperand, AsPointerFromArrayDecay,
+                               AsOperatorArrowThis, AsNonConstRefArg,
+                               AsLambdaRefCaptureInit, AsNonConstRefReturn,
+                               AsNonConstRefRangeInit))
+                        .bind("stmt"))),
       Stm, Context);
   return selectFirst("stmt", Matches);
 }
 
 const Stmt *ExprMutationAnalyzer::findMemberMutation(const Expr *Exp) {
   // Check whether any member of 'Exp' is mutated.
-  const auto MemberExprs =
-      match(findAll(expr(anyOf(memberExpr(hasObjectExpression(
-                                   canResolveToExpr(equalsNode(Exp)))),
-                               cxxDependentScopeMemberExpr(hasObjectExpression(
-                                   canResolveToExpr(equalsNode(Exp)))),
-                               binaryOperator(hasOperatorName(".*"),
-                                              hasLHS(equalsNode(Exp)))))
-                        .bind(NodeID::value)),
-            Stm, Context);
+  const auto MemberExprs = match(
+      findAll(expr(anyOf(memberExpr(hasObjectExpression(canResolveToExpr(Exp))),
+                         cxxDependentScopeMemberExpr(
+                             hasObjectExpression(canResolveToExpr(Exp))),
+                         binaryOperator(hasOperatorName(".*"),
+                                        hasLHS(equalsNode(Exp)))))
+                  .bind(NodeID::value)),
+      Stm, Context);
   return findExprMutation(MemberExprs);
 }
 
 const Stmt *ExprMutationAnalyzer::findArrayElementMutation(const Expr *Exp) {
   // Check whether any element of an array is mutated.
-  const auto SubscriptExprs =
-      match(findAll(arraySubscriptExpr(
-                        anyOf(hasBase(canResolveToExpr(equalsNode(Exp))),
-                              hasBase(implicitCastExpr(
-                                  allOf(hasCastKind(CK_ArrayToPointerDecay),
-                                        hasSourceExpression(canResolveToExpr(
-                                            equalsNode(Exp))))))))
-                        .bind(NodeID::value)),
-            Stm, Context);
+  const auto SubscriptExprs = match(
+      findAll(arraySubscriptExpr(
+                  anyOf(hasBase(canResolveToExpr(Exp)),
+                        hasBase(implicitCastExpr(allOf(
+                            hasCastKind(CK_ArrayToPointerDecay),
+                            hasSourceExpression(canResolveToExpr(Exp)))))))
+                  .bind(NodeID::value)),
+      Stm, Context);
   return findExprMutation(SubscriptExprs);
 }
 
 const Stmt *ExprMutationAnalyzer::findCastMutation(const Expr *Exp) {
   // If the 'Exp' is explicitly casted to a non-const reference type the
   // 'Exp' is considered to be modified.
-  const auto ExplicitCast = match(
-      findAll(
-          stmt(castExpr(hasSourceExpression(canResolveToExpr(equalsNode(Exp))),
-                        explicitCastExpr(
-                            hasDestinationType(nonConstReferenceType()))))
-              .bind("stmt")),
-      Stm, Context);
+  const auto ExplicitCast =
+      match(findFirst(stmt(castExpr(hasSourceExpression(canResolveToExpr(Exp)),
+                                    explicitCastExpr(hasDestinationType(
+                                        nonConstReferenceType()))))
+                          .bind("stmt")),
+            Stm, Context);
 
   if (const auto *CastStmt = selectFirst("stmt", ExplicitCast))
     return CastStmt;
 
   // If 'Exp' is casted to any non-const reference type, check the castExpr.
   const auto Casts = match(
-      findAll(
-          expr(castExpr(hasSourceExpression(canResolveToExpr(equalsNode(Exp))),
-                        anyOf(explicitCastExpr(
-                                  hasDestinationType(nonConstReferenceType())),
-                              implicitCastExpr(hasImplicitDestinationType(
-                                  nonConstReferenceType())))))
-              .bind(NodeID::value)),
+      findAll(expr(castExpr(hasSourceExpression(canResolveToExpr(Exp)),
+                            anyOf(explicitCastExpr(hasDestinationType(
+                                      nonConstReferenceType())),
+                                  implicitCastExpr(hasImplicitDestinationType(
+                                      nonConstReferenceType())))))
+                  .bind(NodeID::value)),
       Stm, Context);
 
   if (const Stmt *S = findExprMutation(Casts))
@@ -458,7 +498,7 @@ const Stmt *ExprMutationAnalyzer::findCastMutation(const Expr *Exp) {
   const auto Calls =
       match(findAll(callExpr(callee(namedDecl(
                                  hasAnyName("::std::move", "::std::forward"))),
-                             hasArgument(0, canResolveToExpr(equalsNode(Exp))))
+                             hasArgument(0, canResolveToExpr(Exp)))
                         .bind("expr")),
             Stm, Context);
   return findExprMutation(Calls);
@@ -473,16 +513,16 @@ const Stmt *ExprMutationAnalyzer::findRangeLoopMutation(const Expr *Exp) {
   // array is considered modified if the loop-variable is a non-const reference.
   const auto DeclStmtToNonRefToArray = declStmt(hasSingleDecl(varDecl(hasType(
       hasUnqualifiedDesugaredType(referenceType(pointee(arrayType())))))));
-  const auto RefToArrayRefToElements =
-      match(findAll(stmt(cxxForRangeStmt(
-                             hasLoopVariable(
-                                 varDecl(anyOf(hasType(nonConstReferenceType()),
-                                               hasType(nonConstPointerType())))
-                                     .bind(NodeID::value)),
-                             hasRangeStmt(DeclStmtToNonRefToArray),
-                             hasRangeInit(canResolveToExpr(equalsNode(Exp)))))
-                        .bind("stmt")),
-            Stm, Context);
+  const auto RefToArrayRefToElements = match(
+      findFirst(stmt(cxxForRangeStmt(
+                         hasLoopVariable(
+                             varDecl(anyOf(hasType(nonConstReferenceType()),
+                                           hasType(nonConstPointerType())))
+                                 .bind(NodeID::value)),
+                         hasRangeStmt(DeclStmtToNonRefToArray),
+                         hasRangeInit(canResolveToExpr(Exp))))
+                    .bind("stmt")),
+      Stm, Context);
 
   if (const auto *BadRangeInitFromArray =
           selectFirst("stmt", RefToArrayRefToElements))
@@ -505,12 +545,12 @@ const Stmt *ExprMutationAnalyzer::findRangeLoopMutation(const Expr *Exp) {
       hasSingleDecl(varDecl(hasType(hasUnqualifiedDesugaredType(referenceType(
           pointee(hasDeclaration(cxxRecordDecl(HasAnyNonConstIterator)))))))));
 
-  const auto RefToContainerBadIterators =
-      match(findAll(stmt(cxxForRangeStmt(allOf(
-                             hasRangeStmt(DeclStmtToNonConstIteratorContainer),
-                             hasRangeInit(canResolveToExpr(equalsNode(Exp))))))
-                        .bind("stmt")),
-            Stm, Context);
+  const auto RefToContainerBadIterators = match(
+      findFirst(stmt(cxxForRangeStmt(allOf(
+                         hasRangeStmt(DeclStmtToNonConstIteratorContainer),
+                         hasRangeInit(canResolveToExpr(Exp)))))
+                    .bind("stmt")),
+      Stm, Context);
 
   if (const auto *BadIteratorsContainer =
           selectFirst("stmt", RefToContainerBadIterators))
@@ -522,7 +562,7 @@ const Stmt *ExprMutationAnalyzer::findRangeLoopMutation(const Expr *Exp) {
       match(findAll(cxxForRangeStmt(
                 hasLoopVariable(varDecl(hasType(nonConstReferenceType()))
                                     .bind(NodeID::value)),
-                hasRangeInit(canResolveToExpr(equalsNode(Exp))))),
+                hasRangeInit(canResolveToExpr(Exp)))),
             Stm, Context);
   return findDeclMutation(LoopVars);
 }
@@ -531,31 +571,29 @@ const Stmt *ExprMutationAnalyzer::findReferenceMutation(const Expr *Exp) {
   // Follow non-const reference returned by `operator*()` of move-only classes.
   // These are typically smart pointers with unique ownership so we treat
   // mutation of pointee as mutation of the smart pointer itself.
-  const auto Ref =
-      match(findAll(cxxOperatorCallExpr(
-                        hasOverloadedOperatorName("*"),
-                        callee(cxxMethodDecl(ofClass(isMoveOnly()),
-                                             returns(nonConstReferenceType()))),
-                        argumentCountIs(1),
-                        hasArgument(0, canResolveToExpr(equalsNode(Exp))))
-                        .bind(NodeID::value)),
-            Stm, Context);
+  const auto Ref = match(
+      findAll(cxxOperatorCallExpr(
+                  hasOverloadedOperatorName("*"),
+                  callee(cxxMethodDecl(ofClass(isMoveOnly()),
+                                       returns(nonConstReferenceType()))),
+                  argumentCountIs(1), hasArgument(0, canResolveToExpr(Exp)))
+                  .bind(NodeID::value)),
+      Stm, Context);
   if (const Stmt *S = findExprMutation(Ref))
     return S;
 
   // If 'Exp' is bound to a non-const reference, check all declRefExpr to that.
   const auto Refs = match(
       stmt(forEachDescendant(
-          varDecl(
-              hasType(nonConstReferenceType()),
-              hasInitializer(anyOf(canResolveToExpr(equalsNode(Exp)),
-                                   memberExpr(hasObjectExpression(
-                                       canResolveToExpr(equalsNode(Exp)))))),
-              hasParent(declStmt().bind("stmt")),
-              // Don't follow the reference in range statement, we've
-              // handled that separately.
-              unless(hasParent(declStmt(hasParent(
-                  cxxForRangeStmt(hasRangeStmt(equalsBoundNode("stmt"))))))))
+          varDecl(hasType(nonConstReferenceType()),
+                  hasInitializer(anyOf(
+                      canResolveToExpr(Exp),
+                      memberExpr(hasObjectExpression(canResolveToExpr(Exp))))),
+                  hasParent(declStmt().bind("stmt")),
+                  // Don't follow the reference in range statement, we've
+                  // handled that separately.
+                  unless(hasParent(declStmt(hasParent(cxxForRangeStmt(
+                      hasRangeStmt(equalsBoundNode("stmt"))))))))
               .bind(NodeID::value))),
       Stm, Context);
   return findDeclMutation(Refs);
@@ -563,7 +601,7 @@ const Stmt *ExprMutationAnalyzer::findReferenceMutation(const Expr *Exp) {
 
 const Stmt *ExprMutationAnalyzer::findFunctionArgMutation(const Expr *Exp) {
   const auto NonConstRefParam = forEachArgumentWithParam(
-      canResolveToExpr(equalsNode(Exp)),
+      canResolveToExpr(Exp),
       parmVarDecl(hasType(nonConstReferenceType())).bind("parm"));
   const auto IsInstantiated = hasDeclaration(isInstantiated());
   const auto FuncDecl = hasDeclaration(functionDecl().bind("func"));
diff --git a/clang/lib/Analysis/FlowSensitive/ControlFlowContext.cpp b/clang/lib/Analysis/FlowSensitive/ControlFlowContext.cpp
index 56246066e4aa..c9ebffe6f378 100644
--- a/clang/lib/Analysis/FlowSensitive/ControlFlowContext.cpp
+++ b/clang/lib/Analysis/FlowSensitive/ControlFlowContext.cpp
@@ -69,7 +69,7 @@ static llvm::BitVector findReachableBlocks(const CFG &Cfg) {
 
 llvm::Expected
 ControlFlowContext::build(const FunctionDecl &Func) {
-  if (!Func.hasBody())
+  if (!Func.doesThisDeclarationHaveABody())
     return llvm::createStringError(
         std::make_error_code(std::errc::invalid_argument),
         "Cannot analyze function without a body");
diff --git a/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp b/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
index fa114979c8e3..f4c4af022f51 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
@@ -174,6 +174,9 @@ Solver::Result DataflowAnalysisContext::querySolver(
 
 bool DataflowAnalysisContext::flowConditionImplies(Atom Token,
                                                    const Formula &F) {
+  if (F.isLiteral(true))
+    return true;
+
   // Returns true if and only if truth assignment of the flow condition implies
   // that `F` is also true. We prove whether or not this property holds by
   // reducing the problem to satisfiability checking. In other words, we attempt
@@ -188,6 +191,9 @@ bool DataflowAnalysisContext::flowConditionImplies(Atom Token,
 
 bool DataflowAnalysisContext::flowConditionAllows(Atom Token,
                                                   const Formula &F) {
+  if (F.isLiteral(false))
+    return false;
+
   llvm::SetVector Constraints;
   Constraints.insert(&arena().makeAtomRef(Token));
   Constraints.insert(&F);
@@ -292,7 +298,7 @@ DataflowAnalysisContext::getControlFlowContext(const FunctionDecl *F) {
   if (It != FunctionContexts.end())
     return &It->second;
 
-  if (F->hasBody()) {
+  if (F->doesThisDeclarationHaveABody()) {
     auto CFCtx = ControlFlowContext::build(*F);
     // FIXME: Handle errors.
     assert(CFCtx);
diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
index 96fe6df88dbb..196a1360a775 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
@@ -386,7 +386,7 @@ void Environment::initialize() {
     return;
 
   if (const auto *FuncDecl = dyn_cast(DeclCtx)) {
-    assert(FuncDecl->getBody() != nullptr);
+    assert(FuncDecl->doesThisDeclarationHaveABody());
 
     initFieldsGlobalsAndFuncs(FuncDecl);
 
@@ -426,7 +426,7 @@ void Environment::initialize() {
 // FIXME: Add support for resetting globals after function calls to enable
 // the implementation of sound analyses.
 void Environment::initFieldsGlobalsAndFuncs(const FunctionDecl *FuncDecl) {
-  assert(FuncDecl->getBody() != nullptr);
+  assert(FuncDecl->doesThisDeclarationHaveABody());
 
   FieldSet Fields;
   llvm::DenseSet Vars;
@@ -747,6 +747,7 @@ static bool isOriginalRecordConstructor(const Expr &RecordPRValue) {
     return !Init->isSemanticForm() || !Init->isTransparent();
   return isa(RecordPRValue) || isa(RecordPRValue) ||
          isa(RecordPRValue) ||
+         isa(RecordPRValue) ||
          // The framework currently does not propagate the objects created in
          // the two branches of a `ConditionalOperator` because there is no way
          // to reconcile their storage locations, which are different. We
@@ -781,8 +782,13 @@ Environment::getResultObjectLocation(const Expr &RecordPRValue) const {
     return Val->getLoc();
   }
 
-  // Expression nodes that propagate a record prvalue should have exactly one
-  // child.
+  if (auto *Op = dyn_cast(&RecordPRValue);
+      Op && Op->isCommaOp()) {
+    return getResultObjectLocation(*Op->getRHS());
+  }
+
+  // All other expression nodes that propagate a record prvalue should have
+  // exactly one child.
   llvm::SmallVector children(RecordPRValue.child_begin(),
                                            RecordPRValue.child_end());
   assert(children.size() == 1);
@@ -803,13 +809,15 @@ void Environment::setValue(const StorageLocation &Loc, Value &Val) {
 }
 
 void Environment::setValue(const Expr &E, Value &Val) {
+  const Expr &CanonE = ignoreCFGOmittedNodes(E);
+
   if (auto *RecordVal = dyn_cast(&Val)) {
-    assert(isOriginalRecordConstructor(E) ||
-           &RecordVal->getLoc() == &getResultObjectLocation(E));
+    assert(isOriginalRecordConstructor(CanonE) ||
+           &RecordVal->getLoc() == &getResultObjectLocation(CanonE));
   }
 
-  assert(E.isPRValue());
-  ExprToVal[&E] = &Val;
+  assert(CanonE.isPRValue());
+  ExprToVal[&CanonE] = &Val;
 }
 
 Value *Environment::getValue(const StorageLocation &Loc) const {
diff --git a/clang/lib/Analysis/FlowSensitive/SimplifyConstraints.cpp b/clang/lib/Analysis/FlowSensitive/SimplifyConstraints.cpp
index 00a3104a57b6..cc20202768b9 100644
--- a/clang/lib/Analysis/FlowSensitive/SimplifyConstraints.cpp
+++ b/clang/lib/Analysis/FlowSensitive/SimplifyConstraints.cpp
@@ -131,9 +131,9 @@ void simplifyConstraints(llvm::SetVector &Constraints,
     for (const auto *Constraint : Constraints) {
       const Formula &NewConstraint =
           substitute(*Constraint, Substitutions, arena);
-      if (&NewConstraint == &arena.makeLiteral(true))
+      if (NewConstraint.isLiteral(true))
         continue;
-      if (&NewConstraint == &arena.makeLiteral(false)) {
+      if (NewConstraint.isLiteral(false)) {
         contradiction();
         return;
       }
diff --git a/clang/lib/Analysis/FlowSensitive/Transfer.cpp b/clang/lib/Analysis/FlowSensitive/Transfer.cpp
index 55093c2e2cda..2271a75fbcaf 100644
--- a/clang/lib/Analysis/FlowSensitive/Transfer.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Transfer.cpp
@@ -42,6 +42,8 @@ const Environment *StmtToEnvMap::getEnvironment(const Stmt &S) const {
   assert(BlockIt != CFCtx.getStmtToBlock().end());
   if (!CFCtx.isBlockReachable(*BlockIt->getSecond()))
     return nullptr;
+  if (BlockIt->getSecond()->getBlockID() == CurBlockID)
+    return &CurState.Env;
   const auto &State = BlockToState[BlockIt->getSecond()->getBlockID()];
   if (!(State))
     return nullptr;
diff --git a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
index faf83a8920d4..49e425bde66a 100644
--- a/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
+++ b/clang/lib/Analysis/FlowSensitive/TypeErasedDataflowAnalysis.cpp
@@ -75,9 +75,8 @@ using TerminatorVisitorRetTy = std::pair;
 class TerminatorVisitor
     : public ConstStmtVisitor {
 public:
-  TerminatorVisitor(const StmtToEnvMap &StmtToEnv, Environment &Env,
-                    int BlockSuccIdx)
-      : StmtToEnv(StmtToEnv), Env(Env), BlockSuccIdx(BlockSuccIdx) {}
+  TerminatorVisitor(Environment &Env, int BlockSuccIdx)
+      : Env(Env), BlockSuccIdx(BlockSuccIdx) {}
 
   TerminatorVisitorRetTy VisitIfStmt(const IfStmt *S) {
     auto *Cond = S->getCond();
@@ -126,19 +125,12 @@ class TerminatorVisitor
 
 private:
   TerminatorVisitorRetTy extendFlowCondition(const Expr &Cond) {
-    // The terminator sub-expression might not be evaluated.
-    if (Env.getValue(Cond) == nullptr)
-      transfer(StmtToEnv, Cond, Env);
-
     auto *Val = Env.get(Cond);
-    // Value merging depends on flow conditions from different environments
-    // being mutually exclusive -- that is, they cannot both be true in their
-    // entirety (even if they may share some clauses). So, we need *some* value
-    // for the condition expression, even if just an atom.
-    if (Val == nullptr) {
-      Val = &Env.makeAtomicBoolValue();
-      Env.setValue(Cond, *Val);
-    }
+    // In transferCFGBlock(), we ensure that we always have a `Value` for the
+    // terminator condition, so assert this.
+    // We consciously assert ourselves instead of asserting via `cast()` so
+    // that we get a more meaningful line number if the assertion fails.
+    assert(Val != nullptr);
 
     bool ConditionValue = true;
     // The condition must be inverted for the successor that encompasses the
@@ -152,7 +144,6 @@ class TerminatorVisitor
     return {&Cond, ConditionValue};
   }
 
-  const StmtToEnvMap &StmtToEnv;
   Environment &Env;
   int BlockSuccIdx;
 };
@@ -335,10 +326,8 @@ computeBlockInputState(const CFGBlock &Block, AnalysisContext &AC) {
         // when the terminator is taken. Copy now.
         TypeErasedDataflowAnalysisState Copy = MaybePredState->fork();
 
-        const StmtToEnvMap StmtToEnv(AC.CFCtx, AC.BlockStates);
         auto [Cond, CondValue] =
-            TerminatorVisitor(StmtToEnv, Copy.Env,
-                              blockIndexInPredecessor(*Pred, Block))
+            TerminatorVisitor(Copy.Env, blockIndexInPredecessor(*Pred, Block))
                 .Visit(PredTerminatorStmt);
         if (Cond != nullptr)
           // FIXME: Call transferBranchTypeErased even if BuiltinTransferOpts
@@ -356,12 +345,13 @@ computeBlockInputState(const CFGBlock &Block, AnalysisContext &AC) {
 
 /// Built-in transfer function for `CFGStmt`.
 static void
-builtinTransferStatement(const CFGStmt &Elt,
+builtinTransferStatement(unsigned CurBlockID, const CFGStmt &Elt,
                          TypeErasedDataflowAnalysisState &InputState,
                          AnalysisContext &AC) {
   const Stmt *S = Elt.getStmt();
   assert(S != nullptr);
-  transfer(StmtToEnvMap(AC.CFCtx, AC.BlockStates), *S, InputState.Env);
+  transfer(StmtToEnvMap(AC.CFCtx, AC.BlockStates, CurBlockID, InputState), *S,
+           InputState.Env);
 }
 
 /// Built-in transfer function for `CFGInitializer`.
@@ -428,12 +418,12 @@ builtinTransferInitializer(const CFGInitializer &Elt,
   }
 }
 
-static void builtinTransfer(const CFGElement &Elt,
+static void builtinTransfer(unsigned CurBlockID, const CFGElement &Elt,
                             TypeErasedDataflowAnalysisState &State,
                             AnalysisContext &AC) {
   switch (Elt.getKind()) {
   case CFGElement::Statement:
-    builtinTransferStatement(Elt.castAs(), State, AC);
+    builtinTransferStatement(CurBlockID, Elt.castAs(), State, AC);
     break;
   case CFGElement::Initializer:
     builtinTransferInitializer(Elt.castAs(), State);
@@ -477,7 +467,7 @@ transferCFGBlock(const CFGBlock &Block, AnalysisContext &AC,
     AC.Log.enterElement(Element);
     // Built-in analysis
     if (AC.Analysis.builtinOptions()) {
-      builtinTransfer(Element, State, AC);
+      builtinTransfer(Block.getBlockID(), Element, State, AC);
     }
 
     // User-provided analysis
@@ -489,6 +479,32 @@ transferCFGBlock(const CFGBlock &Block, AnalysisContext &AC,
     }
     AC.Log.recordState(State);
   }
+
+  // If we have a terminator, evaluate its condition.
+  // This `Expr` may not appear as a `CFGElement` anywhere else, and it's
+  // important that we evaluate it here (rather than while processing the
+  // terminator) so that we put the corresponding value in the right
+  // environment.
+  if (const Expr *TerminatorCond =
+          dyn_cast_or_null(Block.getTerminatorCondition())) {
+    if (State.Env.getValue(*TerminatorCond) == nullptr)
+      // FIXME: This only runs the builtin transfer, not the analysis-specific
+      // transfer. Fixing this isn't trivial, as the analysis-specific transfer
+      // takes a `CFGElement` as input, but some expressions only show up as a
+      // terminator condition, but not as a `CFGElement`. The condition of an if
+      // statement is one such example.
+      transfer(
+          StmtToEnvMap(AC.CFCtx, AC.BlockStates, Block.getBlockID(), State),
+          *TerminatorCond, State.Env);
+
+    // If the transfer function didn't produce a value, create an atom so that
+    // we have *some* value for the condition expression. This ensures that
+    // when we extend the flow condition, it actually changes.
+    if (State.Env.getValue(*TerminatorCond) == nullptr)
+      State.Env.setValue(*TerminatorCond, State.Env.makeAtomicBoolValue());
+    AC.Log.recordState(State);
+  }
+
   return State;
 }
 
@@ -498,7 +514,8 @@ runTypeErasedDataflowAnalysis(
     const Environment &InitEnv,
     std::function
-        PostVisitCFG) {
+        PostVisitCFG,
+    std::int32_t MaxBlockVisits) {
   PrettyStackTraceAnalysis CrashInfo(CFCtx, "runTypeErasedDataflowAnalysis");
 
   std::optional MaybeStartingEnv;
@@ -524,27 +541,20 @@ runTypeErasedDataflowAnalysis(
 
   AnalysisContext AC(CFCtx, Analysis, StartingEnv, BlockStates);
 
-  // Bugs in lattices and transfer functions can prevent the analysis from
-  // converging. To limit the damage (infinite loops) that these bugs can cause,
-  // limit the number of iterations.
-  // FIXME: Consider making the maximum number of iterations configurable.
-  // FIXME: Consider restricting the number of backedges followed, rather than
-  // iterations.
-  // FIXME: Set up statistics (see llvm/ADT/Statistic.h) to count average number
-  // of iterations, number of functions that time out, etc.
-  static constexpr uint32_t MaxAverageVisitsPerBlock = 4;
-  static constexpr uint32_t AbsoluteMaxIterations = 1 << 16;
-  const uint32_t RelativeMaxIterations =
+  // FIXME: remove relative cap. There isn't really any good setting for
+  // `MaxAverageVisitsPerBlock`, so it has no clear value over using
+  // `MaxBlockVisits` directly.
+  static constexpr std::int32_t MaxAverageVisitsPerBlock = 4;
+  const std::int32_t RelativeMaxBlockVisits =
       MaxAverageVisitsPerBlock * BlockStates.size();
-  const uint32_t MaxIterations =
-      std::min(RelativeMaxIterations, AbsoluteMaxIterations);
-  uint32_t Iterations = 0;
+  MaxBlockVisits = std::min(RelativeMaxBlockVisits, MaxBlockVisits);
+  std::int32_t BlockVisits = 0;
   while (const CFGBlock *Block = Worklist.dequeue()) {
     LLVM_DEBUG(llvm::dbgs()
                << "Processing Block " << Block->getBlockID() << "\n");
-    if (++Iterations > MaxIterations) {
+    if (++BlockVisits > MaxBlockVisits) {
       return llvm::createStringError(std::errc::timed_out,
-                                     "maximum number of iterations reached");
+                                     "maximum number of blocks processed");
     }
 
     const std::optional &OldBlockState =
diff --git a/clang/lib/Analysis/FlowSensitive/Value.cpp b/clang/lib/Analysis/FlowSensitive/Value.cpp
index 349f873f1e6c..7fad6deb0e91 100644
--- a/clang/lib/Analysis/FlowSensitive/Value.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Value.cpp
@@ -27,9 +27,17 @@ static bool areEquivalentIndirectionValues(const Value &Val1,
 }
 
 bool areEquivalentValues(const Value &Val1, const Value &Val2) {
-  return &Val1 == &Val2 || (Val1.getKind() == Val2.getKind() &&
-                            (isa(&Val1) ||
-                             areEquivalentIndirectionValues(Val1, Val2)));
+  if (&Val1 == &Val2)
+    return true;
+  if (Val1.getKind() != Val2.getKind())
+    return false;
+  // If values are distinct and have properties, we don't consider them equal,
+  // leaving equality up to the user model.
+  if (!Val1.properties().empty() || !Val2.properties().empty())
+    return false;
+  if (isa(&Val1))
+    return true;
+  return areEquivalentIndirectionValues(Val1, Val2);
 }
 
 raw_ostream &operator<<(raw_ostream &OS, const Value &Val) {
diff --git a/clang/lib/Analysis/PathDiagnostic.cpp b/clang/lib/Analysis/PathDiagnostic.cpp
index 0cb03943c547..79f337a91ec8 100644
--- a/clang/lib/Analysis/PathDiagnostic.cpp
+++ b/clang/lib/Analysis/PathDiagnostic.cpp
@@ -50,12 +50,7 @@
 using namespace clang;
 using namespace ento;
 
-static StringRef StripTrailingDots(StringRef s) {
-  for (StringRef::size_type i = s.size(); i != 0; --i)
-    if (s[i - 1] != '.')
-      return s.substr(0, i);
-  return {};
-}
+static StringRef StripTrailingDots(StringRef s) { return s.rtrim('.'); }
 
 PathDiagnosticPiece::PathDiagnosticPiece(StringRef s,
                                          Kind k, DisplayHint hint)
diff --git a/clang/lib/Analysis/RetainSummaryManager.cpp b/clang/lib/Analysis/RetainSummaryManager.cpp
index 6f50d95b179f..8d279d969b61 100644
--- a/clang/lib/Analysis/RetainSummaryManager.cpp
+++ b/clang/lib/Analysis/RetainSummaryManager.cpp
@@ -1098,7 +1098,7 @@ RetainSummaryManager::getStandardMethodSummary(const ObjCMethodDecl *MD,
   if (S.isKeywordSelector()) {
     for (unsigned i = 0, e = S.getNumArgs(); i != e; ++i) {
       StringRef Slot = S.getNameForSlot(i);
-      if (Slot.substr(Slot.size() - 8).equals_insensitive("delegate")) {
+      if (Slot.ends_with_insensitive("delegate")) {
         if (ResultEff == ObjCInitRetE)
           ResultEff = RetEffect::MakeNoRetHard();
         else
diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index 70eec1cee57f..7df706beb226 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -721,6 +721,35 @@ class UnsafeBufferUsageAttrGadget : public WarningGadget {
   DeclUseList getClaimedVarUseSites() const override { return {}; }
 };
 
+// Warning gadget for unsafe invocation of span::data method.
+// Triggers when the pointer returned by the invocation is immediately
+// cast to a larger type.
+
+class DataInvocationGadget : public WarningGadget {
+  constexpr static const char *const OpTag = "data_invocation_expr";
+  const ExplicitCastExpr *Op;
+
+public:
+  DataInvocationGadget(const MatchFinder::MatchResult &Result)
+      : WarningGadget(Kind::DataInvocation),
+        Op(Result.Nodes.getNodeAs(OpTag)) {}
+
+  static bool classof(const Gadget *G) {
+    return G->getKind() == Kind::DataInvocation;
+  }
+
+  static Matcher matcher() {
+    Matcher callExpr = cxxMemberCallExpr(
+        callee(cxxMethodDecl(hasName("data"), ofClass(hasName("std::span")))));
+    return stmt(
+        explicitCastExpr(anyOf(has(callExpr), has(parenExpr(has(callExpr)))))
+            .bind(OpTag));
+  }
+  const Stmt *getBaseStmt() const override { return Op; }
+
+  DeclUseList getClaimedVarUseSites() const override { return {}; }
+};
+
 // Represents expressions of the form `DRE[*]` in the Unspecified Lvalue
 // Context (see `isInUnspecifiedLvalueContext`).
 // Note here `[]` is the built-in subscript operator.
@@ -2657,8 +2686,8 @@ void clang::checkUnsafeBufferUsage(const Decl *D,
     // every problematic operation and consider it done. No need to deal
     // with fixable gadgets, no need to group operations by variable.
     for (const auto &G : WarningGadgets) {
-      Handler.handleUnsafeOperation(G->getBaseStmt(),
-                                    /*IsRelatedToDecl=*/false);
+      Handler.handleUnsafeOperation(G->getBaseStmt(), /*IsRelatedToDecl=*/false,
+                                    D->getASTContext());
     }
 
     // This return guarantees that most of the machine doesn't run when
@@ -2893,7 +2922,8 @@ void clang::checkUnsafeBufferUsage(const Decl *D,
                   Tracker, Handler, VarGrpMgr);
 
   for (const auto &G : UnsafeOps.noVar) {
-    Handler.handleUnsafeOperation(G->getBaseStmt(), /*IsRelatedToDecl=*/false);
+    Handler.handleUnsafeOperation(G->getBaseStmt(), /*IsRelatedToDecl=*/false,
+                                  D->getASTContext());
   }
 
   for (const auto &[VD, WarningGadgets] : UnsafeOps.byVar) {
@@ -2904,7 +2934,8 @@ void clang::checkUnsafeBufferUsage(const Decl *D,
                                       : FixItList{},
                                       D);
     for (const auto &G : WarningGadgets) {
-      Handler.handleUnsafeOperation(G->getBaseStmt(), /*IsRelatedToDecl=*/true);
+      Handler.handleUnsafeOperation(G->getBaseStmt(), /*IsRelatedToDecl=*/true,
+                                    D->getASTContext());
     }
   }
 }
diff --git a/clang/lib/Basic/CodeGenOptions.cpp b/clang/lib/Basic/CodeGenOptions.cpp
index 182d0a2fa4d8..79d715305ef2 100644
--- a/clang/lib/Basic/CodeGenOptions.cpp
+++ b/clang/lib/Basic/CodeGenOptions.cpp
@@ -27,6 +27,8 @@ void CodeGenOptions::resetNonModularOptions(StringRef ModuleFormat) {
 #define ENUM_DEBUGOPT(Name, Type, Bits, Default)
 #define CODEGENOPT(Name, Bits, Default) Name = Default;
 #define ENUM_CODEGENOPT(Name, Type, Bits, Default) set##Name(Default);
+// Do not reset AST affecting code generation options.
+#define AFFECTING_VALUE_CODEGENOPT(Name, Bits, Default)
 #include "clang/Basic/CodeGenOptions.def"
 
   // Next reset all debug options that can always be reset, because they never
diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp
index d16626b10652..974c8c22598f 100644
--- a/clang/lib/Basic/FileManager.cpp
+++ b/clang/lib/Basic/FileManager.cpp
@@ -532,7 +532,7 @@ void FileManager::fillRealPathName(FileEntry *UFE, llvm::StringRef FileName) {
   // misleading. We need to clean up the interface here.
   makeAbsolutePath(AbsPath);
   llvm::sys::path::remove_dots(AbsPath, /*remove_dot_dot=*/true);
-  UFE->RealPathName = std::string(AbsPath.str());
+  UFE->RealPathName = std::string(AbsPath);
 }
 
 llvm::ErrorOr>
diff --git a/clang/lib/Basic/Targets.cpp b/clang/lib/Basic/Targets.cpp
index 286175252744..e2826890beb4 100644
--- a/clang/lib/Basic/Targets.cpp
+++ b/clang/lib/Basic/Targets.cpp
@@ -652,6 +652,8 @@ std::unique_ptr AllocateTarget(const llvm::Triple &Triple,
       return std::make_unique>(Triple, Opts);
     case llvm::Triple::PS5:
       return std::make_unique>(Triple, Opts);
+    case llvm::Triple::Hurd:
+      return std::make_unique>(Triple, Opts);
     default:
       return std::make_unique(Triple, Opts);
     }
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 2f8395cb8932..d47181bfca4f 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -226,6 +226,7 @@ bool AArch64TargetInfo::validateBranchProtection(StringRef Spec, StringRef,
 
   BPI.BranchTargetEnforcement = PBP.BranchTargetEnforcement;
   BPI.BranchProtectionPAuthLR = PBP.BranchProtectionPAuthLR;
+  BPI.GuardedControlStack = PBP.GuardedControlStack;
   return true;
 }
 
@@ -532,6 +533,9 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   if (Opts.BranchTargetEnforcement)
     Builder.defineMacro("__ARM_FEATURE_BTI_DEFAULT", "1");
 
+  if (Opts.GuardedControlStack)
+    Builder.defineMacro("__ARM_FEATURE_GCS_DEFAULT", "1");
+
   if (HasLS64)
     Builder.defineMacro("__ARM_FEATURE_LS64", "1");
 
@@ -544,6 +548,9 @@ void AArch64TargetInfo::getTargetDefines(const LangOptions &Opts,
   if (HasD128)
     Builder.defineMacro("__ARM_FEATURE_SYSREG128", "1");
 
+  if (HasGCS)
+    Builder.defineMacro("__ARM_FEATURE_GCS", "1");
+
   if (*ArchInfo == llvm::AArch64::ARMV8_1A)
     getTargetDefinesARMV81A(Opts, Builder);
   else if (*ArchInfo == llvm::AArch64::ARMV8_2A)
@@ -615,9 +622,8 @@ AArch64TargetInfo::getVScaleRange(const LangOptions &LangOpts) const {
 unsigned AArch64TargetInfo::multiVersionSortPriority(StringRef Name) const {
   if (Name == "default")
     return 0;
-  for (const auto &E : llvm::AArch64::Extensions)
-    if (Name == E.Name)
-      return E.FmvPriority;
+  if (auto Ext = llvm::AArch64::parseArchExtension(Name))
+    return Ext->FmvPriority;
   return 0;
 }
 
@@ -627,24 +633,19 @@ unsigned AArch64TargetInfo::multiVersionFeatureCost() const {
 }
 
 bool AArch64TargetInfo::doesFeatureAffectCodeGen(StringRef Name) const {
-  auto F = llvm::find_if(llvm::AArch64::Extensions, [&](const auto &E) {
-    return Name == E.Name && !E.DependentFeatures.empty();
-  });
-  return F != std::end(llvm::AArch64::Extensions);
+  if (auto Ext = llvm::AArch64::parseArchExtension(Name))
+    return !Ext->DependentFeatures.empty();
+  return false;
 }
 
 StringRef AArch64TargetInfo::getFeatureDependencies(StringRef Name) const {
-  auto F = llvm::find_if(llvm::AArch64::Extensions,
-                         [&](const auto &E) { return Name == E.Name; });
-  return F != std::end(llvm::AArch64::Extensions) ? F->DependentFeatures
-                                                  : StringRef();
+  if (auto Ext = llvm::AArch64::parseArchExtension(Name))
+    return Ext->DependentFeatures;
+  return StringRef();
 }
 
 bool AArch64TargetInfo::validateCpuSupports(StringRef FeatureStr) const {
-  for (const auto &E : llvm::AArch64::Extensions)
-    if (FeatureStr == E.Name)
-      return true;
-  return false;
+  return llvm::AArch64::parseArchExtension(FeatureStr).has_value();
 }
 
 bool AArch64TargetInfo::hasFeature(StringRef Feature) const {
@@ -1086,8 +1087,7 @@ ParsedTargetAttr AArch64TargetInfo::parseTargetAttr(StringRef Features) const {
       FoundArch = true;
       std::pair Split =
           Feature.split("=").second.trim().split("+");
-      const std::optional AI =
-          llvm::AArch64::parseArch(Split.first);
+      const llvm::AArch64::ArchInfo *AI = llvm::AArch64::parseArch(Split.first);
 
       // Parse the architecture version, adding the required features to
       // Ret.Features.
diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h
index 1819ba544ccf..90a1516ecdd2 100644
--- a/clang/lib/Basic/Targets/AMDGPU.h
+++ b/clang/lib/Basic/Targets/AMDGPU.h
@@ -169,10 +169,8 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
     }
 
     bool HasLeftParen = false;
-    if (S.front() == '{') {
+    if (S.consume_front("{"))
       HasLeftParen = true;
-      S = S.drop_front();
-    }
     if (S.empty())
       return false;
     if (S.front() != 'v' && S.front() != 's' && S.front() != 'a') {
@@ -199,29 +197,24 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
       return true;
     }
     bool HasLeftBracket = false;
-    if (!S.empty() && S.front() == '[') {
+    if (S.consume_front("["))
       HasLeftBracket = true;
-      S = S.drop_front();
-    }
     unsigned long long N;
     if (S.empty() || consumeUnsignedInteger(S, 10, N))
       return false;
-    if (!S.empty() && S.front() == ':') {
+    if (S.consume_front(":")) {
       if (!HasLeftBracket)
         return false;
-      S = S.drop_front();
       unsigned long long M;
       if (consumeUnsignedInteger(S, 10, M) || N >= M)
         return false;
     }
     if (HasLeftBracket) {
-      if (S.empty() || S.front() != ']')
+      if (!S.consume_front("]"))
         return false;
-      S = S.drop_front();
     }
-    if (S.empty() || S.front() != '}')
+    if (!S.consume_front("}"))
       return false;
-    S = S.drop_front();
     if (!S.empty())
       return false;
     // Found {vn}, {sn}, {an}, {v[n]}, {s[n]}, {a[n]}, {v[n:m]}, {s[n:m]}
diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp
index 01f9e844da12..55b71557452f 100644
--- a/clang/lib/Basic/Targets/ARM.cpp
+++ b/clang/lib/Basic/Targets/ARM.cpp
@@ -17,6 +17,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/TargetParser/ARMTargetParser.h"
 
 using namespace clang;
 using namespace clang::targets;
@@ -226,6 +227,8 @@ StringRef ARMTargetInfo::getCPUAttr() const {
     return "9_3A";
   case llvm::ARM::ArchKind::ARMV9_4A:
     return "9_4A";
+  case llvm::ARM::ArchKind::ARMV9_5A:
+    return "9_5A";
   case llvm::ARM::ArchKind::ARMV8MBaseline:
     return "8M_BASE";
   case llvm::ARM::ArchKind::ARMV8MMainline:
@@ -837,6 +840,70 @@ void ARMTargetInfo::getTargetDefines(const LangOptions &Opts,
   if (Opts.RWPI)
     Builder.defineMacro("__ARM_RWPI", "1");
 
+  // Macros for enabling co-proc intrinsics
+  uint64_t FeatureCoprocBF = 0;
+  switch (ArchKind) {
+  default:
+    break;
+  case llvm::ARM::ArchKind::ARMV4:
+  case llvm::ARM::ArchKind::ARMV4T:
+    // Filter __arm_ldcl and __arm_stcl in acle.h
+    FeatureCoprocBF = isThumb() ? 0 : FEATURE_COPROC_B1;
+    break;
+  case llvm::ARM::ArchKind::ARMV5T:
+    FeatureCoprocBF = isThumb() ? 0 : FEATURE_COPROC_B1 | FEATURE_COPROC_B2;
+    break;
+  case llvm::ARM::ArchKind::ARMV5TE:
+  case llvm::ARM::ArchKind::ARMV5TEJ:
+    if (!isThumb())
+      FeatureCoprocBF =
+          FEATURE_COPROC_B1 | FEATURE_COPROC_B2 | FEATURE_COPROC_B3;
+    break;
+  case llvm::ARM::ArchKind::ARMV6:
+  case llvm::ARM::ArchKind::ARMV6K:
+  case llvm::ARM::ArchKind::ARMV6KZ:
+  case llvm::ARM::ArchKind::ARMV6T2:
+    if (!isThumb() || ArchKind == llvm::ARM::ArchKind::ARMV6T2)
+      FeatureCoprocBF = FEATURE_COPROC_B1 | FEATURE_COPROC_B2 |
+                        FEATURE_COPROC_B3 | FEATURE_COPROC_B4;
+    break;
+  case llvm::ARM::ArchKind::ARMV7A:
+  case llvm::ARM::ArchKind::ARMV7R:
+  case llvm::ARM::ArchKind::ARMV7M:
+  case llvm::ARM::ArchKind::ARMV7S:
+  case llvm::ARM::ArchKind::ARMV7EM:
+    FeatureCoprocBF = FEATURE_COPROC_B1 | FEATURE_COPROC_B2 |
+                      FEATURE_COPROC_B3 | FEATURE_COPROC_B4;
+    break;
+  case llvm::ARM::ArchKind::ARMV8A:
+  case llvm::ARM::ArchKind::ARMV8R:
+  case llvm::ARM::ArchKind::ARMV8_1A:
+  case llvm::ARM::ArchKind::ARMV8_2A:
+  case llvm::ARM::ArchKind::ARMV8_3A:
+  case llvm::ARM::ArchKind::ARMV8_4A:
+  case llvm::ARM::ArchKind::ARMV8_5A:
+  case llvm::ARM::ArchKind::ARMV8_6A:
+  case llvm::ARM::ArchKind::ARMV8_7A:
+  case llvm::ARM::ArchKind::ARMV8_8A:
+  case llvm::ARM::ArchKind::ARMV8_9A:
+  case llvm::ARM::ArchKind::ARMV9A:
+  case llvm::ARM::ArchKind::ARMV9_1A:
+  case llvm::ARM::ArchKind::ARMV9_2A:
+  case llvm::ARM::ArchKind::ARMV9_3A:
+  case llvm::ARM::ArchKind::ARMV9_4A:
+  case llvm::ARM::ArchKind::ARMV9_5A:
+    // Filter __arm_cdp, __arm_ldcl, __arm_stcl in arm_acle.h
+    FeatureCoprocBF = FEATURE_COPROC_B1 | FEATURE_COPROC_B3;
+    break;
+  case llvm::ARM::ArchKind::ARMV8MMainline:
+  case llvm::ARM::ArchKind::ARMV8_1MMainline:
+    FeatureCoprocBF = FEATURE_COPROC_B1 | FEATURE_COPROC_B2 |
+                      FEATURE_COPROC_B3 | FEATURE_COPROC_B4;
+    break;
+  }
+  Builder.defineMacro("__ARM_FEATURE_COPROC",
+                      "0x" + Twine::utohexstr(FeatureCoprocBF));
+
   if (ArchKind == llvm::ARM::ArchKind::XSCALE)
     Builder.defineMacro("__XSCALE__");
 
@@ -993,6 +1060,7 @@ void ARMTargetInfo::getTargetDefines(const LangOptions &Opts,
   case llvm::ARM::ArchKind::ARMV9_2A:
   case llvm::ARM::ArchKind::ARMV9_3A:
   case llvm::ARM::ArchKind::ARMV9_4A:
+  case llvm::ARM::ArchKind::ARMV9_5A:
     getTargetDefinesARMV83A(Opts, Builder);
     break;
   }
diff --git a/clang/lib/Basic/Targets/ARM.h b/clang/lib/Basic/Targets/ARM.h
index b1aa2794c7e4..9802eb01abf3 100644
--- a/clang/lib/Basic/Targets/ARM.h
+++ b/clang/lib/Basic/Targets/ARM.h
@@ -100,6 +100,19 @@ class LLVM_LIBRARY_VISIBILITY ARMTargetInfo : public TargetInfo {
   };
   uint32_t HW_FP;
 
+  enum {
+    /// __arm_cdp __arm_ldc, __arm_ldcl, __arm_stc,
+    /// __arm_stcl, __arm_mcr and __arm_mrc
+    FEATURE_COPROC_B1 = (1 << 0),
+    /// __arm_cdp2, __arm_ldc2, __arm_stc2, __arm_ldc2l,
+    /// __arm_stc2l, __arm_mcr2 and __arm_mrc2
+    FEATURE_COPROC_B2 = (1 << 1),
+    /// __arm_mcrr, __arm_mrrc
+    FEATURE_COPROC_B3 = (1 << 2),
+    /// __arm_mcrr2,  __arm_mrrc2
+    FEATURE_COPROC_B4 = (1 << 3),
+  };
+
   void setABIAAPCS();
   void setABIAPCS(bool IsAAPCS16);
 
diff --git a/clang/lib/Basic/Targets/AVR.h b/clang/lib/Basic/Targets/AVR.h
index 854a51d78c39..9376c46cd98c 100644
--- a/clang/lib/Basic/Targets/AVR.h
+++ b/clang/lib/Basic/Targets/AVR.h
@@ -146,7 +146,9 @@ class LLVM_LIBRARY_VISIBILITY AVRTargetInfo : public TargetInfo {
     case 'R': // Integer constant (Range: -6 to 5)
       Info.setRequiresImmediate(-6, 5);
       return true;
-    case 'G': // Floating point constant
+    case 'G': // Floating point constant 0.0
+      Info.setRequiresImmediate(0);
+      return true;
     case 'Q': // A memory address based on Y or Z pointer with displacement.
       return true;
     }
diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h
index 342af4bbc42b..4366c1149e40 100644
--- a/clang/lib/Basic/Targets/OSTargets.h
+++ b/clang/lib/Basic/Targets/OSTargets.h
@@ -74,7 +74,8 @@ class LLVM_LIBRARY_VISIBILITY DarwinTargetInfo : public OSTargetInfo {
         this->TLSSupported = !Triple.isOSVersionLT(3);
     } else if (Triple.isDriverKit()) {
       // No TLS on DriverKit.
-    }
+    } else if (Triple.isXROS())
+      this->TLSSupported = true;
 
     this->MCountName = "\01mcount";
   }
@@ -109,6 +110,9 @@ class LLVM_LIBRARY_VISIBILITY DarwinTargetInfo : public OSTargetInfo {
     case llvm::Triple::WatchOS: // Earliest supporting version is 5.0.0.
       MinVersion = llvm::VersionTuple(5U);
       break;
+    case llvm::Triple::XROS:
+      MinVersion = llvm::VersionTuple(0);
+      break;
     default:
       // Conservatively return 8 bytes if OS is unknown.
       return 64;
diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index 045c273f03c7..41935abfb65d 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -212,6 +212,7 @@ static void defineXLCompatMacros(MacroBuilder &Builder) {
   Builder.defineMacro("__darn_32", "__builtin_darn_32");
   Builder.defineMacro("__darn_raw", "__builtin_darn_raw");
   Builder.defineMacro("__dcbf", "__builtin_dcbf");
+  Builder.defineMacro("__fence", "__builtin_ppc_fence");
   Builder.defineMacro("__fmadd", "__builtin_fma");
   Builder.defineMacro("__fmadds", "__builtin_fmaf");
   Builder.defineMacro("__abs", "__builtin_abs");
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index 6bc57a83a2d5..c71b2e9eeb6c 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -154,7 +154,7 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
   else
     Builder.defineMacro("__riscv_float_abi_soft");
 
-  if (ABIName == "ilp32e")
+  if (ABIName == "ilp32e" || ABIName == "lp64e")
     Builder.defineMacro("__riscv_abi_rve");
 
   Builder.defineMacro("__riscv_arch_test");
@@ -163,9 +163,8 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
     auto ExtName = Extension.first;
     auto ExtInfo = Extension.second;
 
-    Builder.defineMacro(
-        Twine("__riscv_", ExtName),
-        Twine(getVersionValue(ExtInfo.MajorVersion, ExtInfo.MinorVersion)));
+    Builder.defineMacro(Twine("__riscv_", ExtName),
+                        Twine(getVersionValue(ExtInfo.Major, ExtInfo.Minor)));
   }
 
   if (ISAInfo->hasExtension("m") || ISAInfo->hasExtension("zmmul"))
@@ -215,6 +214,13 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__riscv_misaligned_fast");
   else
     Builder.defineMacro("__riscv_misaligned_avoid");
+
+  if (ISAInfo->hasExtension("e")) {
+    if (Is64Bit)
+      Builder.defineMacro("__riscv_64e");
+    else
+      Builder.defineMacro("__riscv_32e");
+  }
 }
 
 static constexpr Builtin::Info BuiltinInfo[] = {
@@ -235,46 +241,6 @@ ArrayRef RISCVTargetInfo::getTargetBuiltins() const {
                         clang::RISCV::LastTSBuiltin - Builtin::FirstTSBuiltin);
 }
 
-static std::vector
-collectNonISAExtFeature(ArrayRef FeaturesNeedOverride, int XLen) {
-  auto ParseResult =
-      llvm::RISCVISAInfo::parseFeatures(XLen, FeaturesNeedOverride);
-
-  if (!ParseResult) {
-    consumeError(ParseResult.takeError());
-    return std::vector();
-  }
-
-  std::vector ImpliedFeatures = (*ParseResult)->toFeatureVector();
-
-  std::vector NonISAExtFeatureVec;
-
-  llvm::copy_if(FeaturesNeedOverride, std::back_inserter(NonISAExtFeatureVec),
-                [&](const std::string &Feat) {
-                  return !llvm::is_contained(ImpliedFeatures, Feat);
-                });
-
-  return NonISAExtFeatureVec;
-}
-
-static std::vector
-resolveTargetAttrOverride(const std::vector &FeaturesVec,
-                          int XLen) {
-  auto I = llvm::find(FeaturesVec, "__RISCV_TargetAttrNeedOverride");
-  if (I == FeaturesVec.end())
-    return FeaturesVec;
-
-  ArrayRef FeaturesNeedOverride(&*FeaturesVec.begin(), &*I);
-  std::vector NonISAExtFeature =
-      collectNonISAExtFeature(FeaturesNeedOverride, XLen);
-
-  std::vector ResolvedFeature(++I, FeaturesVec.end());
-  ResolvedFeature.insert(ResolvedFeature.end(), NonISAExtFeature.begin(),
-                         NonISAExtFeature.end());
-
-  return ResolvedFeature;
-}
-
 bool RISCVTargetInfo::initFeatureMap(
     llvm::StringMap &Features, DiagnosticsEngine &Diags, StringRef CPU,
     const std::vector &FeaturesVec) const {
@@ -288,10 +254,27 @@ bool RISCVTargetInfo::initFeatureMap(
     Features["32bit"] = true;
   }
 
-  std::vector NewFeaturesVec =
-      resolveTargetAttrOverride(FeaturesVec, XLen);
+  // If a target attribute specified a full arch string, override all the ISA
+  // extension target features.
+  const auto I = llvm::find(FeaturesVec, "__RISCV_TargetAttrNeedOverride");
+  if (I != FeaturesVec.end()) {
+    std::vector OverrideFeatures(std::next(I), FeaturesVec.end());
+
+    // Add back any non ISA extension features, e.g. +relax.
+    auto IsNonISAExtFeature = [](StringRef Feature) {
+      assert(Feature.size() > 1 && (Feature[0] == '+' || Feature[0] == '-'));
+      StringRef Ext = Feature.substr(1); // drop the +/-
+      return !llvm::RISCVISAInfo::isSupportedExtensionFeature(Ext);
+    };
+    llvm::copy_if(llvm::make_range(FeaturesVec.begin(), I),
+                  std::back_inserter(OverrideFeatures), IsNonISAExtFeature);
 
-  auto ParseResult = llvm::RISCVISAInfo::parseFeatures(XLen, NewFeaturesVec);
+    return TargetInfo::initFeatureMap(Features, Diags, CPU, OverrideFeatures);
+  }
+
+  // Otherwise, parse the features and add any implied extensions.
+  std::vector AllFeatures = FeaturesVec;
+  auto ParseResult = llvm::RISCVISAInfo::parseFeatures(XLen, FeaturesVec);
   if (!ParseResult) {
     std::string Buffer;
     llvm::raw_string_ostream OutputErrMsg(Buffer);
@@ -302,21 +285,9 @@ bool RISCVTargetInfo::initFeatureMap(
     return false;
   }
 
-  // RISCVISAInfo makes implications for ISA features
-  std::vector ImpliedFeatures = (*ParseResult)->toFeatureVector();
-
-  // parseFeatures normalizes the feature set by dropping any explicit
-  // negatives, and non-extension features.  We need to preserve the later
-  // for correctness and want to preserve the former for consistency.
-  for (auto &Feature : NewFeaturesVec) {
-     StringRef ExtName = Feature;
-     assert(ExtName.size() > 1 && (ExtName[0] == '+' || ExtName[0] == '-'));
-     ExtName = ExtName.drop_front(1); // Drop '+' or '-'
-     if (!llvm::is_contained(ImpliedFeatures, ("+" + ExtName).str()) &&
-         !llvm::is_contained(ImpliedFeatures, ("-" + ExtName).str()))
-       ImpliedFeatures.push_back(Feature);
-  }
-  return TargetInfo::initFeatureMap(Features, Diags, CPU, ImpliedFeatures);
+  // Append all features, not just new ones, so we override any negatives.
+  llvm::append_range(AllFeatures, (*ParseResult)->toFeatures());
+  return TargetInfo::initFeatureMap(Features, Diags, CPU, AllFeatures);
 }
 
 std::optional>
@@ -386,6 +357,11 @@ bool RISCVTargetInfo::handleTargetFeatures(std::vector &Features,
   if (llvm::is_contained(Features, "+experimental"))
     HasExperimental = true;
 
+  if (ABI == "ilp32e" && ISAInfo->hasExtension("d")) {
+    Diags.Report(diag::err_invalid_feature_combination)
+        << "ILP32E cannot be used with the D ISA extension";
+    return false;
+  }
   return true;
 }
 
@@ -420,7 +396,10 @@ static void handleFullArchString(StringRef FullArchStr,
     // Forward the invalid FullArchStr.
     Features.push_back("+" + FullArchStr.str());
   } else {
-    std::vector FeatStrings = (*RII)->toFeatureVector();
+    // Append a full list of features, including any negative extensions so that
+    // we override the CPU's features.
+    std::vector FeatStrings =
+        (*RII)->toFeatures(/* AddAllExtensions */ true);
     Features.insert(Features.end(), FeatStrings.begin(), FeatStrings.end());
   }
 }
diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h
index f98c88cd45f8..bfbdafb682c8 100644
--- a/clang/lib/Basic/Targets/RISCV.h
+++ b/clang/lib/Basic/Targets/RISCV.h
@@ -132,6 +132,12 @@ class LLVM_LIBRARY_VISIBILITY RISCV32TargetInfo : public RISCVTargetInfo {
   }
 
   bool setABI(const std::string &Name) override {
+    if (Name == "ilp32e") {
+      ABI = Name;
+      resetDataLayout("e-m:e-p:32:32-i64:64-n32-S32");
+      return true;
+    }
+
     if (Name == "ilp32" || Name == "ilp32f" || Name == "ilp32d") {
       ABI = Name;
       return true;
@@ -156,6 +162,12 @@ class LLVM_LIBRARY_VISIBILITY RISCV64TargetInfo : public RISCVTargetInfo {
   }
 
   bool setABI(const std::string &Name) override {
+    if (Name == "lp64e") {
+      ABI = Name;
+      resetDataLayout("e-m:e-p:64:64-i64:64-i128:128-n32:64-S64");
+      return true;
+    }
+
     if (Name == "lp64" || Name == "lp64f" || Name == "lp64d") {
       ABI = Name;
       return true;
diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index 9ab2b7c60936..fa4a3bb1c82e 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -17,6 +17,7 @@
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/TargetOptions.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/VersionTuple.h"
 #include "llvm/TargetParser/Triple.h"
 #include 
 
@@ -301,8 +302,9 @@ class LLVM_LIBRARY_VISIBILITY SPIRVTargetInfo : public BaseSPIRVTargetInfo {
       : BaseSPIRVTargetInfo(Triple, Opts) {
     assert(Triple.getArch() == llvm::Triple::spirv &&
            "Invalid architecture for Logical SPIR-V.");
-    assert(Triple.getOS() == llvm::Triple::ShaderModel &&
-           "Logical SPIR-V requires a valid ShaderModel.");
+    assert(Triple.getOS() == llvm::Triple::Vulkan &&
+           Triple.getVulkanVersion() != llvm::VersionTuple(0) &&
+           "Logical SPIR-V requires a valid Vulkan environment.");
     assert(Triple.getEnvironment() >= llvm::Triple::Pixel &&
            Triple.getEnvironment() <= llvm::Triple::Amplification &&
            "Logical SPIR-V environment must be a valid shader stage.");
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 3deaa19f8d4f..a68b662d9401 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -295,11 +295,13 @@ bool X86TargetInfo::handleTargetFeatures(std::vector &Features,
       HasAVX512BF16 = true;
     } else if (Feature == "+avx512er") {
       HasAVX512ER = true;
+      Diags.Report(diag::warn_knl_knm_isa_support_removed);
     } else if (Feature == "+avx512fp16") {
       HasAVX512FP16 = true;
       HasLegalHalfType = true;
     } else if (Feature == "+avx512pf") {
       HasAVX512PF = true;
+      Diags.Report(diag::warn_knl_knm_isa_support_removed);
     } else if (Feature == "+avx512dq") {
       HasAVX512DQ = true;
     } else if (Feature == "+avx512bitalg") {
@@ -358,6 +360,7 @@ bool X86TargetInfo::handleTargetFeatures(std::vector &Features,
       HasPREFETCHI = true;
     } else if (Feature == "+prefetchwt1") {
       HasPREFETCHWT1 = true;
+      Diags.Report(diag::warn_knl_knm_isa_support_removed);
     } else if (Feature == "+clzero") {
       HasCLZERO = true;
     } else if (Feature == "+cldemote") {
@@ -1415,6 +1418,14 @@ bool X86TargetInfo::validateAsmConstraint(
   case 'O':
     Info.setRequiresImmediate(0, 127);
     return true;
+  case 'W':
+    switch (*++Name) {
+    default:
+      return false;
+    case 's':
+      Info.setAllowsRegister();
+      return true;
+    }
   // Register constraints.
   case 'Y': // 'Y' is the first character for several 2-character constraints.
     // Shift the pointer to the second character of the constraint.
@@ -1712,6 +1723,9 @@ std::string X86TargetInfo::convertConstraint(const char *&Constraint) const {
     return std::string("{st}");
   case 'u':                        // second from top of floating point stack.
     return std::string("{st(1)}"); // second from top of floating point stack.
+  case 'W':
+    assert(Constraint[1] == 's');
+    return '^' + std::string(Constraint++, 2);
   case 'Y':
     switch (Constraint[1]) {
     default:
diff --git a/clang/lib/Basic/Warnings.cpp b/clang/lib/Basic/Warnings.cpp
index 92954cab6fb0..5a5ac5556338 100644
--- a/clang/lib/Basic/Warnings.cpp
+++ b/clang/lib/Basic/Warnings.cpp
@@ -199,8 +199,7 @@ void clang::ProcessWarningOptions(DiagnosticsEngine &Diags,
 
       // Check to see if this warning starts with "no-", if so, this is a
       // negative form of the option.
-      bool IsPositive = !Opt.starts_with("no-");
-      if (!IsPositive) Opt = Opt.substr(3);
+      bool IsPositive = !Opt.consume_front("no-");
 
       auto Severity = IsPositive ? diag::Severity::Remark
                                  : diag::Severity::Ignored;
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index a6142d99f3b6..ec203f6f28bc 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -1001,8 +1001,9 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
     }
 
     if (CodeGenOpts.FatLTO) {
-      assert(CodeGenOpts.UnifiedLTO && "FatLTO requires UnifiedLTO");
-      MPM.addPass(PB.buildFatLTODefaultPipeline(Level));
+      MPM.addPass(PB.buildFatLTODefaultPipeline(
+          Level, PrepareForThinLTO,
+          PrepareForThinLTO || shouldEmitRegularLTOSummary()));
     } else if (PrepareForThinLTO) {
       MPM.addPass(PB.buildThinLTOPreLinkDefaultPipeline(Level));
     } else if (PrepareForLTO) {
@@ -1073,8 +1074,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
     if (!TheModule->getModuleFlag("EnableSplitLTOUnit"))
       TheModule->addModuleFlag(llvm::Module::Error, "EnableSplitLTOUnit",
                                uint32_t(CodeGenOpts.EnableSplitLTOUnit));
-    // FatLTO always means UnifiedLTO
-    if (!TheModule->getModuleFlag("UnifiedLTO"))
+    if (CodeGenOpts.UnifiedLTO && !TheModule->getModuleFlag("UnifiedLTO"))
       TheModule->addModuleFlag(llvm::Module::Error, "UnifiedLTO", uint32_t(1));
   }
 
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 93509107d62e..7019e98d7e67 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -28,6 +28,7 @@
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/OSLog.h"
+#include "clang/AST/OperationKinds.h"
 #include "clang/Basic/TargetBuiltins.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/TargetOptions.h"
@@ -823,6 +824,238 @@ CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
   return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true);
 }
 
+const FieldDecl *CodeGenFunction::FindFlexibleArrayMemberField(
+    ASTContext &Ctx, const RecordDecl *RD, StringRef Name, uint64_t &Offset) {
+  const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
+      getLangOpts().getStrictFlexArraysLevel();
+  unsigned FieldNo = 0;
+  bool IsUnion = RD->isUnion();
+
+  for (const Decl *D : RD->decls()) {
+    if (const auto *Field = dyn_cast(D);
+        Field && (Name.empty() || Field->getNameAsString() == Name) &&
+        Decl::isFlexibleArrayMemberLike(
+            Ctx, Field, Field->getType(), StrictFlexArraysLevel,
+            /*IgnoreTemplateOrMacroSubstitution=*/true)) {
+      const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
+      Offset += Layout.getFieldOffset(FieldNo);
+      return Field;
+    }
+
+    if (const auto *Record = dyn_cast(D))
+      if (const FieldDecl *Field =
+              FindFlexibleArrayMemberField(Ctx, Record, Name, Offset)) {
+        const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
+        Offset += Layout.getFieldOffset(FieldNo);
+        return Field;
+      }
+
+    if (!IsUnion && isa(D))
+      ++FieldNo;
+  }
+
+  return nullptr;
+}
+
+static unsigned CountCountedByAttrs(const RecordDecl *RD) {
+  unsigned Num = 0;
+
+  for (const Decl *D : RD->decls()) {
+    if (const auto *FD = dyn_cast(D);
+        FD && FD->hasAttr()) {
+      return ++Num;
+    }
+
+    if (const auto *Rec = dyn_cast(D))
+      Num += CountCountedByAttrs(Rec);
+  }
+
+  return Num;
+}
+
+llvm::Value *
+CodeGenFunction::emitFlexibleArrayMemberSize(const Expr *E, unsigned Type,
+                                             llvm::IntegerType *ResType) {
+  // The code generated here calculates the size of a struct with a flexible
+  // array member that uses the counted_by attribute. There are two instances
+  // we handle:
+  //
+  //       struct s {
+  //         unsigned long flags;
+  //         int count;
+  //         int array[] __attribute__((counted_by(count)));
+  //       }
+  //
+  //   1) bdos of the flexible array itself:
+  //
+  //     __builtin_dynamic_object_size(p->array, 1) ==
+  //         p->count * sizeof(*p->array)
+  //
+  //   2) bdos of a pointer into the flexible array:
+  //
+  //     __builtin_dynamic_object_size(&p->array[42], 1) ==
+  //         (p->count - 42) * sizeof(*p->array)
+  //
+  //   2) bdos of the whole struct, including the flexible array:
+  //
+  //     __builtin_dynamic_object_size(p, 1) ==
+  //        max(sizeof(struct s),
+  //            offsetof(struct s, array) + p->count * sizeof(*p->array))
+  //
+  ASTContext &Ctx = getContext();
+  const Expr *Base = E->IgnoreParenImpCasts();
+  const Expr *Idx = nullptr;
+
+  if (const auto *UO = dyn_cast(Base);
+      UO && UO->getOpcode() == UO_AddrOf) {
+    Expr *SubExpr = UO->getSubExpr()->IgnoreParenImpCasts();
+    if (const auto *ASE = dyn_cast(SubExpr)) {
+      Base = ASE->getBase()->IgnoreParenImpCasts();
+      Idx = ASE->getIdx()->IgnoreParenImpCasts();
+
+      if (const auto *IL = dyn_cast(Idx)) {
+        int64_t Val = IL->getValue().getSExtValue();
+        if (Val < 0)
+          return getDefaultBuiltinObjectSizeResult(Type, ResType);
+
+        if (Val == 0)
+          // The index is 0, so we don't need to take it into account.
+          Idx = nullptr;
+      }
+    } else {
+      // Potential pointer to another element in the struct.
+      Base = SubExpr;
+    }
+  }
+
+  // Get the flexible array member Decl.
+  const RecordDecl *OuterRD = nullptr;
+  std::string FAMName;
+  if (const auto *ME = dyn_cast(Base)) {
+    // Check if \p Base is referencing the FAM itself.
+    const ValueDecl *VD = ME->getMemberDecl();
+    OuterRD = VD->getDeclContext()->getOuterLexicalRecordContext();
+    FAMName = VD->getNameAsString();
+  } else if (const auto *DRE = dyn_cast(Base)) {
+    // Check if we're pointing to the whole struct.
+    QualType Ty = DRE->getDecl()->getType();
+    if (Ty->isPointerType())
+      Ty = Ty->getPointeeType();
+    OuterRD = Ty->getAsRecordDecl();
+
+    // If we have a situation like this:
+    //
+    //     struct union_of_fams {
+    //         int flags;
+    //         union {
+    //             signed char normal_field;
+    //             struct {
+    //                 int count1;
+    //                 int arr1[] __counted_by(count1);
+    //             };
+    //             struct {
+    //                 signed char count2;
+    //                 int arr2[] __counted_by(count2);
+    //             };
+    //         };
+    //    };
+    //
+    // We don't konw which 'count' to use in this scenario:
+    //
+    //     size_t get_size(struct union_of_fams *p) {
+    //         return __builtin_dynamic_object_size(p, 1);
+    //     }
+    //
+    // Instead of calculating a wrong number, we give up.
+    if (OuterRD && CountCountedByAttrs(OuterRD) > 1)
+      return nullptr;
+  }
+
+  if (!OuterRD)
+    return nullptr;
+
+  uint64_t Offset = 0;
+  const FieldDecl *FAMDecl =
+      FindFlexibleArrayMemberField(Ctx, OuterRD, FAMName, Offset);
+  Offset = Ctx.toCharUnitsFromBits(Offset).getQuantity();
+
+  if (!FAMDecl || !FAMDecl->hasAttr())
+    // No flexible array member found or it doesn't have the "counted_by"
+    // attribute.
+    return nullptr;
+
+  const FieldDecl *CountedByFD = FindCountedByField(FAMDecl);
+  if (!CountedByFD)
+    // Can't find the field referenced by the "counted_by" attribute.
+    return nullptr;
+
+  // Build a load of the counted_by field.
+  bool IsSigned = CountedByFD->getType()->isSignedIntegerType();
+  Value *CountedByInst = EmitCountedByFieldExpr(Base, FAMDecl, CountedByFD);
+  if (!CountedByInst)
+    return getDefaultBuiltinObjectSizeResult(Type, ResType);
+
+  CountedByInst = Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
+
+  // Build a load of the index and subtract it from the count.
+  Value *IdxInst = nullptr;
+  if (Idx) {
+    if (Idx->HasSideEffects(getContext()))
+      // We can't have side-effects.
+      return getDefaultBuiltinObjectSizeResult(Type, ResType);
+
+    bool IdxSigned = Idx->getType()->isSignedIntegerType();
+    IdxInst = EmitAnyExprToTemp(Idx).getScalarVal();
+    IdxInst = Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
+
+    // We go ahead with the calculation here. If the index turns out to be
+    // negative, we'll catch it at the end.
+    CountedByInst =
+        Builder.CreateSub(CountedByInst, IdxInst, "", !IsSigned, IsSigned);
+  }
+
+  // Calculate how large the flexible array member is in bytes.
+  const ArrayType *ArrayTy = Ctx.getAsArrayType(FAMDecl->getType());
+  CharUnits Size = Ctx.getTypeSizeInChars(ArrayTy->getElementType());
+  llvm::Constant *ElemSize =
+      llvm::ConstantInt::get(ResType, Size.getQuantity(), IsSigned);
+  Value *FAMSize =
+      Builder.CreateMul(CountedByInst, ElemSize, "", !IsSigned, IsSigned);
+  FAMSize = Builder.CreateIntCast(FAMSize, ResType, IsSigned);
+  Value *Res = FAMSize;
+
+  if (isa(Base)) {
+    // The whole struct is specificed in the __bdos.
+    const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(OuterRD);
+
+    // Get the offset of the FAM.
+    llvm::Constant *FAMOffset = ConstantInt::get(ResType, Offset, IsSigned);
+    Value *OffsetAndFAMSize =
+        Builder.CreateAdd(FAMOffset, Res, "", !IsSigned, IsSigned);
+
+    // Get the full size of the struct.
+    llvm::Constant *SizeofStruct =
+        ConstantInt::get(ResType, Layout.getSize().getQuantity(), IsSigned);
+
+    // max(sizeof(struct s),
+    //     offsetof(struct s, array) + p->count * sizeof(*p->array))
+    Res = IsSigned
+              ? Builder.CreateBinaryIntrinsic(llvm::Intrinsic::smax,
+                                              OffsetAndFAMSize, SizeofStruct)
+              : Builder.CreateBinaryIntrinsic(llvm::Intrinsic::umax,
+                                              OffsetAndFAMSize, SizeofStruct);
+  }
+
+  // A negative \p IdxInst or \p CountedByInst means that the index lands
+  // outside of the flexible array member. If that's the case, we want to
+  // return 0.
+  Value *Cmp = Builder.CreateIsNotNeg(CountedByInst);
+  if (IdxInst)
+    Cmp = Builder.CreateAnd(Builder.CreateIsNotNeg(IdxInst), Cmp);
+
+  return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
+}
+
 /// Returns a Value corresponding to the size of the given expression.
 /// This Value may be either of the following:
 ///   - A llvm::Argument (if E is a param with the pass_object_size attribute on
@@ -855,6 +1088,13 @@ CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
     }
   }
 
+  if (IsDynamic) {
+    // Emit special code for a flexible array member with the "counted_by"
+    // attribute.
+    if (Value *V = emitFlexibleArrayMemberSize(E, Type, ResType))
+      return V;
+  }
+
   // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
   // evaluate E for side-effects. In either case, we shouldn't lower to
   // @llvm.objectsize.
@@ -9690,8 +9930,8 @@ Value *CodeGenFunction::EmitSVEMaskedStore(const CallExpr *E,
   bool IsQuadStore = false;
 
   switch (IntrinsicID) {
-  case Intrinsic::aarch64_sve_st1uwq:
-  case Intrinsic::aarch64_sve_st1udq:
+  case Intrinsic::aarch64_sve_st1wq:
+  case Intrinsic::aarch64_sve_st1dq:
     AddrMemoryTy = llvm::ScalableVectorType::get(MemEltTy, 1);
     PredTy =
         llvm::ScalableVectorType::get(IntegerType::get(getLLVMContext(), 1), 1);
@@ -9825,7 +10065,7 @@ CodeGenFunction::getSVEOverloadTypes(const SVETypeFlags &TypeFlags,
 
   llvm::Type *DefaultType = getSVEType(TypeFlags);
 
-  if (TypeFlags.isOverloadWhile())
+  if (TypeFlags.isOverloadWhileOrMultiVecCvt())
     return {DefaultType, Ops[1]->getType()};
 
   if (TypeFlags.isOverloadWhileRW())
@@ -10439,6 +10679,26 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
     return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
   }
 
+  if (BuiltinID == clang::AArch64::BI__builtin_arm_get_sme_state) {
+    // Create call to __arm_sme_state and store the results to the two pointers.
+    CallInst *CI = EmitRuntimeCall(CGM.CreateRuntimeFunction(
+        llvm::FunctionType::get(StructType::get(CGM.Int64Ty, CGM.Int64Ty), {},
+                                false),
+        "__arm_sme_state"));
+    auto Attrs =
+        AttributeList()
+            .addFnAttribute(getLLVMContext(), "aarch64_pstate_sm_compatible")
+            .addFnAttribute(getLLVMContext(), "aarch64_pstate_za_preserved");
+    CI->setAttributes(Attrs);
+    CI->setCallingConv(
+        llvm::CallingConv::
+            AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2);
+    Builder.CreateStore(Builder.CreateExtractValue(CI, 0),
+                        EmitPointerWithAlignment(E->getArg(0)));
+    return Builder.CreateStore(Builder.CreateExtractValue(CI, 1),
+                               EmitPointerWithAlignment(E->getArg(1)));
+  }
+
   if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
     assert((getContext().getTypeSize(E->getType()) == 32) &&
            "rbit of unusual size!");
@@ -13036,7 +13296,7 @@ Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID,
     const auto *DR = cast(CE->getSubExpr());
     const auto *Enumerator = cast(DR->getDecl());
 
-    auto &InitVal = Enumerator->getInitVal();
+    auto InitVal = Enumerator->getInitVal();
     std::string InitValStr;
     if (InitVal.isNegative() || InitVal > uint64_t(INT64_MAX))
       InitValStr = std::to_string(InitVal.getSExtValue());
@@ -17927,6 +18187,45 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     llvm::Function *F = CGM.getIntrinsic(IID, {ArgTy});
     return Builder.CreateCall(F, {Addr, Val, ZeroI32, ZeroI32, ZeroI1});
   }
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr_i32:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr_v2i32:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4f16:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4i16:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8f16:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8i16: {
+
+    llvm::Type *ArgTy;
+    switch (BuiltinID) {
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr_i32:
+      ArgTy = llvm::Type::getInt32Ty(getLLVMContext());
+      break;
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr_v2i32:
+      ArgTy = llvm::FixedVectorType::get(
+          llvm::Type::getInt32Ty(getLLVMContext()), 2);
+      break;
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4f16:
+      ArgTy = llvm::FixedVectorType::get(
+          llvm::Type::getHalfTy(getLLVMContext()), 4);
+      break;
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4i16:
+      ArgTy = llvm::FixedVectorType::get(
+          llvm::Type::getInt16Ty(getLLVMContext()), 4);
+      break;
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8f16:
+      ArgTy = llvm::FixedVectorType::get(
+          llvm::Type::getHalfTy(getLLVMContext()), 8);
+      break;
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8i16:
+      ArgTy = llvm::FixedVectorType::get(
+          llvm::Type::getInt16Ty(getLLVMContext()), 8);
+      break;
+    }
+
+    llvm::Value *Addr = EmitScalarExpr(E->getArg(0));
+    llvm::Function *F =
+        CGM.getIntrinsic(Intrinsic::amdgcn_global_load_tr, {ArgTy});
+    return Builder.CreateCall(F, {Addr});
+  }
   case AMDGPU::BI__builtin_amdgcn_read_exec:
     return EmitAMDGCNBallotForExec(*this, E, Int64Ty, Int64Ty, false);
   case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index 353370f1d761..5b43272bfa62 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -893,7 +893,7 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() {
     llvm::raw_svector_ostream OS(ModuleID);
     OS << ModuleIDPrefix << llvm::format("%" PRIx64, FatbinWrapper->getGUID());
     llvm::Constant *ModuleIDConstant = makeConstantArray(
-        std::string(ModuleID.str()), "", ModuleIDSectionName, 32, /*AddNull=*/true);
+        std::string(ModuleID), "", ModuleIDSectionName, 32, /*AddNull=*/true);
 
     // Create an alias for the FatbinWrapper that nvcc will look for.
     llvm::GlobalAlias::create(llvm::GlobalValue::ExternalLinkage,
diff --git a/clang/lib/CodeGen/CGCXX.cpp b/clang/lib/CodeGen/CGCXX.cpp
index 110e21f7cb6d..e95a735f92f7 100644
--- a/clang/lib/CodeGen/CGCXX.cpp
+++ b/clang/lib/CodeGen/CGCXX.cpp
@@ -40,6 +40,11 @@ bool CodeGenModule::TryEmitBaseDestructorAsAlias(const CXXDestructorDecl *D) {
   if (getCodeGenOpts().OptimizationLevel == 0)
     return true;
 
+  // Disable this optimization for ARM64EC.  FIXME: This probably should work,
+  // but getting the symbol table correct is complicated.
+  if (getTarget().getTriple().isWindowsArm64EC())
+    return true;
+
   // If sanitizing memory to check for use-after-dtor, do not emit as
   //  an alias, unless this class owns no members.
   if (getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 51a43b5f85b3..28c211aa631e 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -1767,14 +1767,31 @@ static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
       FPT->isNothrow())
     FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
 
-  if (FPT->getAArch64SMEAttributes() & FunctionType::SME_PStateSMEnabledMask)
+  unsigned SMEBits = FPT->getAArch64SMEAttributes();
+  if (SMEBits & FunctionType::SME_PStateSMEnabledMask)
     FuncAttrs.addAttribute("aarch64_pstate_sm_enabled");
-  if (FPT->getAArch64SMEAttributes() & FunctionType::SME_PStateSMCompatibleMask)
+  if (SMEBits & FunctionType::SME_PStateSMCompatibleMask)
     FuncAttrs.addAttribute("aarch64_pstate_sm_compatible");
-  if (FPT->getAArch64SMEAttributes() & FunctionType::SME_PStateZASharedMask)
+
+  // ZA
+  if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_Out ||
+      FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_InOut)
+    FuncAttrs.addAttribute("aarch64_pstate_za_shared");
+  if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_Preserves ||
+      FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_In) {
     FuncAttrs.addAttribute("aarch64_pstate_za_shared");
-  if (FPT->getAArch64SMEAttributes() & FunctionType::SME_PStateZAPreservedMask)
     FuncAttrs.addAttribute("aarch64_pstate_za_preserved");
+  }
+
+  // ZT0
+  if (FunctionType::getArmZT0State(SMEBits) == FunctionType::ARM_Preserves)
+    FuncAttrs.addAttribute("aarch64_preserves_zt0");
+  if (FunctionType::getArmZT0State(SMEBits) == FunctionType::ARM_In)
+    FuncAttrs.addAttribute("aarch64_in_zt0");
+  if (FunctionType::getArmZT0State(SMEBits) == FunctionType::ARM_Out)
+    FuncAttrs.addAttribute("aarch64_out_zt0");
+  if (FunctionType::getArmZT0State(SMEBits) == FunctionType::ARM_InOut)
+    FuncAttrs.addAttribute("aarch64_inout_zt0");
 }
 
 static void AddAttributesFromAssumes(llvm::AttrBuilder &FuncAttrs,
@@ -2446,9 +2463,6 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
 
     if (TargetDecl->hasAttr())
       FuncAttrs.addAttribute("aarch64_pstate_sm_body");
-
-    if (TargetDecl->hasAttr())
-      FuncAttrs.addAttribute("aarch64_pstate_za_new");
   }
 
   // Attach "no-builtins" attributes to:
@@ -2612,6 +2626,8 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
   if (IRFunctionArgs.hasSRetArg()) {
     llvm::AttrBuilder SRETAttrs(getLLVMContext());
     SRETAttrs.addStructRetAttr(getTypes().ConvertTypeForMem(RetTy));
+    SRETAttrs.addAttribute(llvm::Attribute::Writable);
+    SRETAttrs.addAttribute(llvm::Attribute::DeadOnUnwind);
     hasUsedSRet = true;
     if (RetAI.getInReg())
       SRETAttrs.addAttribute(llvm::Attribute::InReg);
diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp
index d18f186ce5b4..34319381901a 100644
--- a/clang/lib/CodeGen/CGClass.cpp
+++ b/clang/lib/CodeGen/CGClass.cpp
@@ -856,6 +856,7 @@ void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
     EnterCXXTryStmt(*cast(Body), true);
 
   incrementProfileCounter(Body);
+  maybeCreateMCDCCondBitmap();
 
   RunCleanupsScope RunCleanups(*this);
 
@@ -1444,8 +1445,10 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
   }
 
   Stmt *Body = Dtor->getBody();
-  if (Body)
+  if (Body) {
     incrementProfileCounter(Body);
+    maybeCreateMCDCCondBitmap();
+  }
 
   // The call to operator delete in a deleting destructor happens
   // outside of the function-try-block, which means it's always
@@ -1548,6 +1551,7 @@ void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args)
   LexicalScope Scope(*this, RootCS->getSourceRange());
 
   incrementProfileCounter(RootCS);
+  maybeCreateMCDCCondBitmap();
   AssignmentMemcpyizer AM(*this, AssignOp, Args);
   for (auto *I : RootCS->body())
     AM.emitAssignment(I);
diff --git a/clang/lib/CodeGen/CGCleanup.h b/clang/lib/CodeGen/CGCleanup.h
index 079a3e25d6dc..fcfbf41b0eaf 100644
--- a/clang/lib/CodeGen/CGCleanup.h
+++ b/clang/lib/CodeGen/CGCleanup.h
@@ -613,6 +613,7 @@ struct EHPersonality {
   static const EHPersonality MSVC_CxxFrameHandler3;
   static const EHPersonality GNU_Wasm_CPlusPlus;
   static const EHPersonality XL_CPlusPlus;
+  static const EHPersonality ZOS_CPlusPlus;
 
   /// Does this personality use landingpads or the family of pad instructions
   /// designed to form funclets?
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index df70b5a12e33..dddd84e98af7 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -1935,7 +1935,7 @@ llvm::DISubprogram *CGDebugInfo::CreateCXXMemberFunction(
   int ThisAdjustment = 0;
 
   if (VTableContextBase::hasVtableSlot(Method)) {
-    if (Method->isPure())
+    if (Method->isPureVirtual())
       SPFlags |= llvm::DISubprogram::SPFlagPureVirtual;
     else
       SPFlags |= llvm::DISubprogram::SPFlagVirtual;
@@ -2210,6 +2210,14 @@ CGDebugInfo::CollectTemplateParams(std::optional OArgs,
       TemplateParams.push_back(DBuilder.createTemplateValueParameter(
           TheCU, Name, TTy, defaultParameter, V));
     } break;
+    case TemplateArgument::StructuralValue: {
+      QualType T = TA.getStructuralValueType();
+      llvm::DIType *TTy = getOrCreateType(T, Unit);
+      llvm::Constant *V = ConstantEmitter(CGM).emitAbstract(
+          SourceLocation(), TA.getAsStructuralValue(), T);
+      TemplateParams.push_back(DBuilder.createTemplateValueParameter(
+          TheCU, Name, TTy, defaultParameter, V));
+    } break;
     case TemplateArgument::Template: {
       std::string QualName;
       llvm::raw_string_ostream OS(QualName);
@@ -5410,6 +5418,8 @@ std::string CGDebugInfo::GetName(const Decl *D, bool Qualified) const {
             // feasible some day.
             return TA.getAsIntegral().getBitWidth() <= 64 &&
                    IsReconstitutableType(TA.getIntegralType());
+          case TemplateArgument::StructuralValue:
+            return false;
           case TemplateArgument::Type:
             return IsReconstitutableType(TA.getAsType());
           default:
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index a5da0aa2965a..bbe14ef4c172 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -1759,20 +1759,34 @@ void CodeGenFunction::emitZeroOrPatternForAutoVarInit(QualType type,
                                                       const VarDecl &D,
                                                       Address Loc) {
   auto trivialAutoVarInit = getContext().getLangOpts().getTrivialAutoVarInit();
+  auto trivialAutoVarInitMaxSize =
+      getContext().getLangOpts().TrivialAutoVarInitMaxSize;
   CharUnits Size = getContext().getTypeSizeInChars(type);
   bool isVolatile = type.isVolatileQualified();
   if (!Size.isZero()) {
+    // We skip auto-init variables by their alloc size. Take this as an example:
+    // "struct Foo {int x; char buff[1024];}" Assume the max-size flag is 1023.
+    // All Foo type variables will be skipped. Ideally, we only skip the buff
+    // array and still auto-init X in this example.
+    // TODO: Improve the size filtering to by member size.
+    auto allocSize = CGM.getDataLayout().getTypeAllocSize(Loc.getElementType());
     switch (trivialAutoVarInit) {
     case LangOptions::TrivialAutoVarInitKind::Uninitialized:
       llvm_unreachable("Uninitialized handled by caller");
     case LangOptions::TrivialAutoVarInitKind::Zero:
       if (CGM.stopAutoInit())
         return;
+      if (trivialAutoVarInitMaxSize > 0 &&
+          allocSize > trivialAutoVarInitMaxSize)
+        return;
       emitStoresForZeroInit(CGM, D, Loc, isVolatile, Builder);
       break;
     case LangOptions::TrivialAutoVarInitKind::Pattern:
       if (CGM.stopAutoInit())
         return;
+      if (trivialAutoVarInitMaxSize > 0 &&
+          allocSize > trivialAutoVarInitMaxSize)
+        return;
       emitStoresForPatternInit(CGM, D, Loc, isVolatile, Builder);
       break;
     }
diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp
index 0d507da5c1ba..5a9d06da12de 100644
--- a/clang/lib/CodeGen/CGException.cpp
+++ b/clang/lib/CodeGen/CGException.cpp
@@ -127,6 +127,8 @@ const EHPersonality
 EHPersonality::GNU_Wasm_CPlusPlus = { "__gxx_wasm_personality_v0", nullptr };
 const EHPersonality EHPersonality::XL_CPlusPlus = {"__xlcxx_personality_v1",
                                                    nullptr};
+const EHPersonality EHPersonality::ZOS_CPlusPlus = {"__zos_cxx_personality_v2",
+                                                    nullptr};
 
 static const EHPersonality &getCPersonality(const TargetInfo &Target,
                                             const LangOptions &L) {
@@ -156,7 +158,9 @@ static const EHPersonality &getObjCPersonality(const TargetInfo &Target,
   case ObjCRuntime::WatchOS:
     return EHPersonality::NeXT_ObjC;
   case ObjCRuntime::GNUstep:
-    if (L.ObjCRuntime.getVersion() >= VersionTuple(1, 7))
+    if (T.isOSCygMing())
+      return EHPersonality::GNU_CPlusPlus_SEH;
+    else if (L.ObjCRuntime.getVersion() >= VersionTuple(1, 7))
       return EHPersonality::GNUstep_ObjC;
     [[fallthrough]];
   case ObjCRuntime::GCC:
@@ -185,6 +189,8 @@ static const EHPersonality &getCXXPersonality(const TargetInfo &Target,
     return EHPersonality::GNU_CPlusPlus_SEH;
   if (L.hasWasmExceptions())
     return EHPersonality::GNU_Wasm_CPlusPlus;
+  if (T.isOSzOS())
+    return EHPersonality::ZOS_CPlusPlus;
   return EHPersonality::GNU_CPlusPlus;
 }
 
@@ -210,7 +216,8 @@ static const EHPersonality &getObjCXXPersonality(const TargetInfo &Target,
     return getObjCPersonality(Target, L);
 
   case ObjCRuntime::GNUstep:
-    return EHPersonality::GNU_ObjCXX;
+    return Target.getTriple().isOSCygMing() ? EHPersonality::GNU_CPlusPlus_SEH
+                                            : EHPersonality::GNU_ObjCXX;
 
   // The GCC runtime's personality function inherently doesn't support
   // mixed EH.  Use the ObjC personality just to avoid returning null.
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 3f277725d9e7..c5f6b6d3a99f 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -26,10 +26,12 @@
 #include "clang/AST/Attr.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/NSAPI.h"
+#include "clang/AST/StmtVisitor.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/CodeGenOptions.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Intrinsics.h"
@@ -925,16 +927,21 @@ static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
     if (CE->getCastKind() == CK_ArrayToPointerDecay &&
         !CE->getSubExpr()->isFlexibleArrayMemberLike(CGF.getContext(),
                                                      StrictFlexArraysLevel)) {
+      CodeGenFunction::SanitizerScope SanScope(&CGF);
+
       IndexedType = CE->getSubExpr()->getType();
       const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
       if (const auto *CAT = dyn_cast(AT))
         return CGF.Builder.getInt(CAT->getSize());
-      else if (const auto *VAT = dyn_cast(AT))
+
+      if (const auto *VAT = dyn_cast(AT))
         return CGF.getVLASize(VAT).NumElts;
       // Ignore pass_object_size here. It's not applicable on decayed pointers.
     }
   }
 
+  CodeGenFunction::SanitizerScope SanScope(&CGF);
+
   QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
   if (llvm::Value *POS = CGF.LoadPassedObjectSize(Base, EltTy)) {
     IndexedType = Base->getType();
@@ -944,22 +951,248 @@ static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
   return nullptr;
 }
 
+namespace {
+
+/// \p StructAccessBase returns the base \p Expr of a field access. It returns
+/// either a \p DeclRefExpr, representing the base pointer to the struct, i.e.:
+///
+///     p in p-> a.b.c
+///
+/// or a \p MemberExpr, if the \p MemberExpr has the \p RecordDecl we're
+/// looking for:
+///
+///     struct s {
+///       struct s *ptr;
+///       int count;
+///       char array[] __attribute__((counted_by(count)));
+///     };
+///
+/// If we have an expression like \p p->ptr->array[index], we want the
+/// \p MemberExpr for \p p->ptr instead of \p p.
+class StructAccessBase
+    : public ConstStmtVisitor {
+  const RecordDecl *ExpectedRD;
+
+  bool IsExpectedRecordDecl(const Expr *E) const {
+    QualType Ty = E->getType();
+    if (Ty->isPointerType())
+      Ty = Ty->getPointeeType();
+    return ExpectedRD == Ty->getAsRecordDecl();
+  }
+
+public:
+  StructAccessBase(const RecordDecl *ExpectedRD) : ExpectedRD(ExpectedRD) {}
+
+  //===--------------------------------------------------------------------===//
+  //                            Visitor Methods
+  //===--------------------------------------------------------------------===//
+
+  // NOTE: If we build C++ support for counted_by, then we'll have to handle
+  // horrors like this:
+  //
+  //     struct S {
+  //       int x, y;
+  //       int blah[] __attribute__((counted_by(x)));
+  //     } s;
+  //
+  //     int foo(int index, int val) {
+  //       int (S::*IHatePMDs)[] = &S::blah;
+  //       (s.*IHatePMDs)[index] = val;
+  //     }
+
+  const Expr *Visit(const Expr *E) {
+    return ConstStmtVisitor::Visit(E);
+  }
+
+  const Expr *VisitStmt(const Stmt *S) { return nullptr; }
+
+  // These are the types we expect to return (in order of most to least
+  // likely):
+  //
+  //   1. DeclRefExpr - This is the expression for the base of the structure.
+  //      It's exactly what we want to build an access to the \p counted_by
+  //      field.
+  //   2. MemberExpr - This is the expression that has the same \p RecordDecl
+  //      as the flexble array member's lexical enclosing \p RecordDecl. This
+  //      allows us to catch things like: "p->p->array"
+  //   3. CompoundLiteralExpr - This is for people who create something
+  //      heretical like (struct foo has a flexible array member):
+  //
+  //        (struct foo){ 1, 2 }.blah[idx];
+  const Expr *VisitDeclRefExpr(const DeclRefExpr *E) {
+    return IsExpectedRecordDecl(E) ? E : nullptr;
+  }
+  const Expr *VisitMemberExpr(const MemberExpr *E) {
+    if (IsExpectedRecordDecl(E) && E->isArrow())
+      return E;
+    const Expr *Res = Visit(E->getBase());
+    return !Res && IsExpectedRecordDecl(E) ? E : Res;
+  }
+  const Expr *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
+    return IsExpectedRecordDecl(E) ? E : nullptr;
+  }
+  const Expr *VisitCallExpr(const CallExpr *E) {
+    return IsExpectedRecordDecl(E) ? E : nullptr;
+  }
+
+  const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
+    if (IsExpectedRecordDecl(E))
+      return E;
+    return Visit(E->getBase());
+  }
+  const Expr *VisitCastExpr(const CastExpr *E) {
+    return Visit(E->getSubExpr());
+  }
+  const Expr *VisitParenExpr(const ParenExpr *E) {
+    return Visit(E->getSubExpr());
+  }
+  const Expr *VisitUnaryAddrOf(const UnaryOperator *E) {
+    return Visit(E->getSubExpr());
+  }
+  const Expr *VisitUnaryDeref(const UnaryOperator *E) {
+    return Visit(E->getSubExpr());
+  }
+};
+
+} // end anonymous namespace
+
+using RecIndicesTy =
+    SmallVector, 8>;
+
+static bool getGEPIndicesToField(CodeGenFunction &CGF, const RecordDecl *RD,
+                                 const FieldDecl *FD, RecIndicesTy &Indices) {
+  const CGRecordLayout &Layout = CGF.CGM.getTypes().getCGRecordLayout(RD);
+  int64_t FieldNo = -1;
+  for (const Decl *D : RD->decls()) {
+    if (const auto *Field = dyn_cast(D)) {
+      FieldNo = Layout.getLLVMFieldNo(Field);
+      if (FD == Field) {
+        Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo)));
+        return true;
+      }
+    }
+
+    if (const auto *Record = dyn_cast(D)) {
+      ++FieldNo;
+      if (getGEPIndicesToField(CGF, Record, FD, Indices)) {
+        if (RD->isUnion())
+          FieldNo = 0;
+        Indices.emplace_back(std::make_pair(RD, CGF.Builder.getInt32(FieldNo)));
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+/// This method is typically called in contexts where we can't generate
+/// side-effects, like in __builtin_dynamic_object_size. When finding
+/// expressions, only choose those that have either already been emitted or can
+/// be loaded without side-effects.
+///
+/// - \p FAMDecl: the \p Decl for the flexible array member. It may not be
+///   within the top-level struct.
+/// - \p CountDecl: must be within the same non-anonymous struct as \p FAMDecl.
+llvm::Value *CodeGenFunction::EmitCountedByFieldExpr(
+    const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {
+  const RecordDecl *RD = CountDecl->getParent()->getOuterLexicalRecordContext();
+
+  // Find the base struct expr (i.e. p in p->a.b.c.d).
+  const Expr *StructBase = StructAccessBase(RD).Visit(Base);
+  if (!StructBase || StructBase->HasSideEffects(getContext()))
+    return nullptr;
+
+  llvm::Value *Res = nullptr;
+  if (const auto *DRE = dyn_cast(StructBase)) {
+    Res = EmitDeclRefLValue(DRE).getPointer(*this);
+    Res = Builder.CreateAlignedLoad(ConvertType(DRE->getType()), Res,
+                                    getPointerAlign(), "dre.load");
+  } else if (const MemberExpr *ME = dyn_cast(StructBase)) {
+    LValue LV = EmitMemberExpr(ME);
+    Address Addr = LV.getAddress(*this);
+    Res = Addr.getPointer();
+  } else if (StructBase->getType()->isPointerType()) {
+    LValueBaseInfo BaseInfo;
+    TBAAAccessInfo TBAAInfo;
+    Address Addr = EmitPointerWithAlignment(StructBase, &BaseInfo, &TBAAInfo);
+    Res = Addr.getPointer();
+  } else {
+    return nullptr;
+  }
+
+  llvm::Value *Zero = Builder.getInt32(0);
+  RecIndicesTy Indices;
+
+  getGEPIndicesToField(*this, RD, CountDecl, Indices);
+
+  for (auto I = Indices.rbegin(), E = Indices.rend(); I != E; ++I)
+    Res = Builder.CreateInBoundsGEP(
+        ConvertType(QualType(I->first->getTypeForDecl(), 0)), Res,
+        {Zero, I->second}, "..counted_by.gep");
+
+  return Builder.CreateAlignedLoad(ConvertType(CountDecl->getType()), Res,
+                                   getIntAlign(), "..counted_by.load");
+}
+
+const FieldDecl *CodeGenFunction::FindCountedByField(const FieldDecl *FD) {
+  if (!FD || !FD->hasAttr())
+    return nullptr;
+
+  const auto *CBA = FD->getAttr();
+  if (!CBA)
+    return nullptr;
+
+  auto GetNonAnonStructOrUnion =
+      [](const RecordDecl *RD) -> const RecordDecl * {
+    while (RD && RD->isAnonymousStructOrUnion()) {
+      const auto *R = dyn_cast(RD->getDeclContext());
+      if (!R)
+        return nullptr;
+      RD = R;
+    }
+    return RD;
+  };
+  const RecordDecl *EnclosingRD = GetNonAnonStructOrUnion(FD->getParent());
+  if (!EnclosingRD)
+    return nullptr;
+
+  DeclarationName DName(CBA->getCountedByField());
+  DeclContext::lookup_result Lookup = EnclosingRD->lookup(DName);
+
+  if (Lookup.empty())
+    return nullptr;
+
+  const NamedDecl *ND = Lookup.front();
+  if (const auto *IFD = dyn_cast(ND))
+    ND = IFD->getAnonField();
+
+  return dyn_cast(ND);
+}
+
 void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base,
                                       llvm::Value *Index, QualType IndexType,
                                       bool Accessed) {
   assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
          "should not be called unless adding bounds checks");
-  SanitizerScope SanScope(this);
-
   const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
-    getLangOpts().getStrictFlexArraysLevel();
-
+      getLangOpts().getStrictFlexArraysLevel();
   QualType IndexedType;
   llvm::Value *Bound =
       getArrayIndexingBound(*this, Base, IndexedType, StrictFlexArraysLevel);
+
+  EmitBoundsCheckImpl(E, Bound, Index, IndexType, IndexedType, Accessed);
+}
+
+void CodeGenFunction::EmitBoundsCheckImpl(const Expr *E, llvm::Value *Bound,
+                                          llvm::Value *Index,
+                                          QualType IndexType,
+                                          QualType IndexedType, bool Accessed) {
   if (!Bound)
     return;
 
+  SanitizerScope SanScope(this);
+
   bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
   llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned);
   llvm::Value *BoundVal = Builder.CreateIntCast(Bound, SizeTy, false);
@@ -975,7 +1208,6 @@ void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base,
             SanitizerHandler::OutOfBounds, StaticData, Index);
 }
 
-
 CodeGenFunction::ComplexPairTy CodeGenFunction::
 EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
                          bool isInc, bool isPre) {
@@ -1284,6 +1516,14 @@ LValue CodeGenFunction::EmitLValue(const Expr *E,
   return LV;
 }
 
+static QualType getConstantExprReferredType(const FullExpr *E,
+                                            const ASTContext &Ctx) {
+  const Expr *SE = E->getSubExpr()->IgnoreImplicit();
+  if (isa(SE))
+    return SE->getType();
+  return cast(SE)->getCallReturnType(Ctx)->getPointeeType();
+}
+
 LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
                                          KnownNonNull_t IsKnownNonNull) {
   ApplyDebugLocation DL(*this, E);
@@ -1322,9 +1562,7 @@ LValue CodeGenFunction::EmitLValueHelper(const Expr *E,
   case Expr::ConstantExprClass: {
     const ConstantExpr *CE = cast(E);
     if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) {
-      QualType RetType = cast(CE->getSubExpr()->IgnoreImplicit())
-                             ->getCallReturnType(getContext())
-                             ->getPointeeType();
+      QualType RetType = getConstantExprReferredType(CE, getContext());
       return MakeNaturalAlignAddrLValue(Result, RetType);
     }
     return EmitLValue(cast(E)->getSubExpr(), IsKnownNonNull);
@@ -3823,6 +4061,61 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
   return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign);
 }
 
+/// The offset of a field from the beginning of the record.
+static bool getFieldOffsetInBits(CodeGenFunction &CGF, const RecordDecl *RD,
+                                 const FieldDecl *FD, int64_t &Offset) {
+  ASTContext &Ctx = CGF.getContext();
+  const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
+  unsigned FieldNo = 0;
+
+  for (const Decl *D : RD->decls()) {
+    if (const auto *Record = dyn_cast(D))
+      if (getFieldOffsetInBits(CGF, Record, FD, Offset)) {
+        Offset += Layout.getFieldOffset(FieldNo);
+        return true;
+      }
+
+    if (const auto *Field = dyn_cast(D))
+      if (FD == Field) {
+        Offset += Layout.getFieldOffset(FieldNo);
+        return true;
+      }
+
+    if (isa(D))
+      ++FieldNo;
+  }
+
+  return false;
+}
+
+/// Returns the relative offset difference between \p FD1 and \p FD2.
+/// \code
+///   offsetof(struct foo, FD1) - offsetof(struct foo, FD2)
+/// \endcode
+/// Both fields must be within the same struct.
+static std::optional getOffsetDifferenceInBits(CodeGenFunction &CGF,
+                                                        const FieldDecl *FD1,
+                                                        const FieldDecl *FD2) {
+  const RecordDecl *FD1OuterRec =
+      FD1->getParent()->getOuterLexicalRecordContext();
+  const RecordDecl *FD2OuterRec =
+      FD2->getParent()->getOuterLexicalRecordContext();
+
+  if (FD1OuterRec != FD2OuterRec)
+    // Fields must be within the same RecordDecl.
+    return std::optional();
+
+  int64_t FD1Offset = 0;
+  if (!getFieldOffsetInBits(CGF, FD1OuterRec, FD1, FD1Offset))
+    return std::optional();
+
+  int64_t FD2Offset = 0;
+  if (!getFieldOffsetInBits(CGF, FD2OuterRec, FD2, FD2Offset))
+    return std::optional();
+
+  return std::make_optional(FD1Offset - FD2Offset);
+}
+
 LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
                                                bool Accessed) {
   // The index must always be an integer, which is not an aggregate.  Emit it
@@ -3950,6 +4243,47 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
       ArrayLV = EmitLValue(Array);
     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
 
+    if (SanOpts.has(SanitizerKind::ArrayBounds)) {
+      // If the array being accessed has a "counted_by" attribute, generate
+      // bounds checking code. The "count" field is at the top level of the
+      // struct or in an anonymous struct, that's also at the top level. Future
+      // expansions may allow the "count" to reside at any place in the struct,
+      // but the value of "counted_by" will be a "simple" path to the count,
+      // i.e. "a.b.count", so we shouldn't need the full force of EmitLValue or
+      // similar to emit the correct GEP.
+      const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
+          getLangOpts().getStrictFlexArraysLevel();
+
+      if (const auto *ME = dyn_cast(Array);
+          ME &&
+          ME->isFlexibleArrayMemberLike(getContext(), StrictFlexArraysLevel) &&
+          ME->getMemberDecl()->hasAttr()) {
+        const FieldDecl *FAMDecl = dyn_cast(ME->getMemberDecl());
+        if (const FieldDecl *CountFD = FindCountedByField(FAMDecl)) {
+          if (std::optional Diff =
+                  getOffsetDifferenceInBits(*this, CountFD, FAMDecl)) {
+            CharUnits OffsetDiff = CGM.getContext().toCharUnitsFromBits(*Diff);
+
+            // Create a GEP with a byte offset between the FAM and count and
+            // use that to load the count value.
+            Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(
+                ArrayLV.getAddress(*this), Int8PtrTy, Int8Ty);
+
+            llvm::Type *CountTy = ConvertType(CountFD->getType());
+            llvm::Value *Res = Builder.CreateInBoundsGEP(
+                Int8Ty, Addr.getPointer(),
+                Builder.getInt32(OffsetDiff.getQuantity()), ".counted_by.gep");
+            Res = Builder.CreateAlignedLoad(CountTy, Res, getIntAlign(),
+                                            ".counted_by.load");
+
+            // Now emit the bounds checking.
+            EmitBoundsCheckImpl(E, Res, Idx, E->getIdx()->getType(),
+                                Array->getType(), Accessed);
+          }
+        }
+      }
+    }
+
     // Propagate the alignment from the array itself to the result.
     QualType arrayType = Array->getType();
     Addr = emitArraySubscriptGEP(
diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp
index 98ae56e2df88..d136bfc37278 100644
--- a/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/clang/lib/CodeGen/CGExprCXX.cpp
@@ -1038,11 +1038,25 @@ void CodeGenFunction::EmitNewArrayInitializer(
     return true;
   };
 
+  const InitListExpr *ILE = dyn_cast(Init);
+  const CXXParenListInitExpr *CPLIE = nullptr;
+  const StringLiteral *SL = nullptr;
+  const ObjCEncodeExpr *OCEE = nullptr;
+  const Expr *IgnoreParen = nullptr;
+  if (!ILE) {
+    IgnoreParen = Init->IgnoreParenImpCasts();
+    CPLIE = dyn_cast(IgnoreParen);
+    SL = dyn_cast(IgnoreParen);
+    OCEE = dyn_cast(IgnoreParen);
+  }
+
   // If the initializer is an initializer list, first do the explicit elements.
-  if (const InitListExpr *ILE = dyn_cast(Init)) {
+  if (ILE || CPLIE || SL || OCEE) {
     // Initializing from a (braced) string literal is a special case; the init
     // list element does not initialize a (single) array element.
-    if (ILE->isStringLiteralInit()) {
+    if ((ILE && ILE->isStringLiteralInit()) || SL || OCEE) {
+      if (!ILE)
+        Init = IgnoreParen;
       // Initialize the initial portion of length equal to that of the string
       // literal. The allocation must be for at least this much; we emitted a
       // check for that earlier.
@@ -1054,12 +1068,13 @@ void CodeGenFunction::EmitNewArrayInitializer(
                                 AggValueSlot::DoesNotOverlap,
                                 AggValueSlot::IsNotZeroed,
                                 AggValueSlot::IsSanitizerChecked);
-      EmitAggExpr(ILE->getInit(0), Slot);
+      EmitAggExpr(ILE ? ILE->getInit(0) : Init, Slot);
 
       // Move past these elements.
       InitListElements =
-          cast(ILE->getType()->getAsArrayTypeUnsafe())
-              ->getSize().getZExtValue();
+          cast(Init->getType()->getAsArrayTypeUnsafe())
+              ->getSize()
+              .getZExtValue();
       CurPtr = Builder.CreateConstInBoundsGEP(
           CurPtr, InitListElements, "string.init.end");
 
@@ -1073,7 +1088,9 @@ void CodeGenFunction::EmitNewArrayInitializer(
       return;
     }
 
-    InitListElements = ILE->getNumInits();
+    ArrayRef InitExprs =
+        ILE ? ILE->inits() : CPLIE->getInitExprs();
+    InitListElements = InitExprs.size();
 
     // If this is a multi-dimensional array new, we will initialize multiple
     // elements with each init list element.
@@ -1101,7 +1118,8 @@ void CodeGenFunction::EmitNewArrayInitializer(
     }
 
     CharUnits StartAlign = CurPtr.getAlignment();
-    for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i) {
+    unsigned i = 0;
+    for (const Expr *IE : InitExprs) {
       // Tell the cleanup that it needs to destroy up to this
       // element.  TODO: some of these stores can be trivially
       // observed to be unnecessary.
@@ -1111,18 +1129,17 @@ void CodeGenFunction::EmitNewArrayInitializer(
       // FIXME: If the last initializer is an incomplete initializer list for
       // an array, and we have an array filler, we can fold together the two
       // initialization loops.
-      StoreAnyExprIntoOneUnit(*this, ILE->getInit(i),
-                              ILE->getInit(i)->getType(), CurPtr,
+      StoreAnyExprIntoOneUnit(*this, IE, IE->getType(), CurPtr,
                               AggValueSlot::DoesNotOverlap);
       CurPtr = Address(Builder.CreateInBoundsGEP(
                            CurPtr.getElementType(), CurPtr.getPointer(),
                            Builder.getSize(1), "array.exp.next"),
                        CurPtr.getElementType(),
-                       StartAlign.alignmentAtOffset((i + 1) * ElementSize));
+                       StartAlign.alignmentAtOffset((++i) * ElementSize));
     }
 
     // The remaining elements are filled with the array filler expression.
-    Init = ILE->getArrayFiller();
+    Init = ILE ? ILE->getArrayFiller() : CPLIE->getArrayFiller();
 
     // Extract the initializer for the individual array elements by pulling
     // out the array filler from all the nested initializer lists. This avoids
@@ -1561,16 +1578,23 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
   // 1. Build a call to the allocation function.
   FunctionDecl *allocator = E->getOperatorNew();
 
-  // If there is a brace-initializer, cannot allocate fewer elements than inits.
+  // If there is a brace-initializer or C++20 parenthesized initializer, cannot
+  // allocate fewer elements than inits.
   unsigned minElements = 0;
   if (E->isArray() && E->hasInitializer()) {
-    const InitListExpr *ILE = dyn_cast(E->getInitializer());
-    if (ILE && ILE->isStringLiteralInit())
+    const Expr *Init = E->getInitializer();
+    const InitListExpr *ILE = dyn_cast(Init);
+    const CXXParenListInitExpr *CPLIE = dyn_cast(Init);
+    const Expr *IgnoreParen = Init->IgnoreParenImpCasts();
+    if ((ILE && ILE->isStringLiteralInit()) ||
+        isa(IgnoreParen) || isa(IgnoreParen)) {
       minElements =
-          cast(ILE->getType()->getAsArrayTypeUnsafe())
-              ->getSize().getZExtValue();
-    else if (ILE)
-      minElements = ILE->getNumInits();
+          cast(Init->getType()->getAsArrayTypeUnsafe())
+              ->getSize()
+              .getZExtValue();
+    } else if (ILE || CPLIE) {
+      minElements = ILE ? ILE->getNumInits() : CPLIE->getInitExprs().size();
+    }
   }
 
   llvm::Value *numElements = nullptr;
diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp
index e532794b71bd..839fe16cd772 100644
--- a/clang/lib/CodeGen/CGExprComplex.cpp
+++ b/clang/lib/CodeGen/CGExprComplex.cpp
@@ -892,6 +892,9 @@ ComplexPairTy ComplexExprEmitter::EmitRangeReductionDiv(llvm::Value *LHSr,
                                                         llvm::Value *LHSi,
                                                         llvm::Value *RHSr,
                                                         llvm::Value *RHSi) {
+  // FIXME: This could eventually be replaced by an LLVM intrinsic to
+  // avoid this long IR sequence.
+
   // (a + ib) / (c + id) = (e + if)
   llvm::Value *FAbsRHSr = EmitllvmFAbs(CGF, RHSr); // |c|
   llvm::Value *FAbsRHSi = EmitllvmFAbs(CGF, RHSi); // |d|
@@ -936,7 +939,7 @@ ComplexPairTy ComplexExprEmitter::EmitRangeReductionDiv(llvm::Value *LHSr,
   llvm::Value *RC = Builder.CreateFMul(CdD, RHSr);  // rc
   llvm::Value *DpRC = Builder.CreateFAdd(RHSi, RC); // tmp=d+rc
 
-  llvm::Value *T7 = Builder.CreateFMul(LHSr, RC);    // ar
+  llvm::Value *T7 = Builder.CreateFMul(LHSr, CdD);   // ar
   llvm::Value *T8 = Builder.CreateFAdd(T7, LHSi);    // ar+b
   llvm::Value *DSTFr = Builder.CreateFDiv(T8, DpRC); // (ar+b)/tmp
 
@@ -978,7 +981,10 @@ ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) {
       return EmitRangeReductionDiv(LHSr, LHSi, RHSr, RHSi);
     else if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Limited)
       return EmitAlgebraicDiv(LHSr, LHSi, RHSr, RHSi);
-    else if (!CGF.getLangOpts().FastMath) {
+    else if (!CGF.getLangOpts().FastMath ||
+             // '-ffast-math' is used in the command line but followed by an
+             // '-fno-cx-limited-range'.
+             Op.FPFeatures.getComplexRange() == LangOptions::CX_Full) {
       LHSi = OrigLHSi;
       // If we have a complex operand on the RHS and FastMath is not allowed, we
       // delegate to a libcall to handle all of the complexities and minimize
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 85bacead7425..c99b781812a8 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -4646,6 +4646,12 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
     if (LHSCondVal) { // If we have 1 && X, just emit X.
       CGF.incrementProfileCounter(E);
 
+      // If the top of the logical operator nest, reset the MCDC temp to 0.
+      if (CGF.MCDCLogOpStack.empty())
+        CGF.maybeResetMCDCCondBitmap(E);
+
+      CGF.MCDCLogOpStack.push_back(E);
+
       Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
 
       // If we're generating for profiling or coverage, generate a branch to a
@@ -4654,6 +4660,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
       // "FalseBlock" after the increment is done.
       if (InstrumentRegions &&
           CodeGenFunction::isInstrumentedCondition(E->getRHS())) {
+        CGF.maybeUpdateMCDCCondBitmap(E->getRHS(), RHSCond);
         llvm::BasicBlock *FBlock = CGF.createBasicBlock("land.end");
         llvm::BasicBlock *RHSBlockCnt = CGF.createBasicBlock("land.rhscnt");
         Builder.CreateCondBr(RHSCond, RHSBlockCnt, FBlock);
@@ -4663,6 +4670,11 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
         CGF.EmitBlock(FBlock);
       }
 
+      CGF.MCDCLogOpStack.pop_back();
+      // If the top of the logical operator nest, update the MCDC bitmap.
+      if (CGF.MCDCLogOpStack.empty())
+        CGF.maybeUpdateMCDCTestVectorBitmap(E);
+
       // ZExt result to int or bool.
       return Builder.CreateZExtOrBitCast(RHSCond, ResTy, "land.ext");
     }
@@ -4672,6 +4684,12 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
       return llvm::Constant::getNullValue(ResTy);
   }
 
+  // If the top of the logical operator nest, reset the MCDC temp to 0.
+  if (CGF.MCDCLogOpStack.empty())
+    CGF.maybeResetMCDCCondBitmap(E);
+
+  CGF.MCDCLogOpStack.push_back(E);
+
   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("land.end");
   llvm::BasicBlock *RHSBlock  = CGF.createBasicBlock("land.rhs");
 
@@ -4704,6 +4722,7 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
   // condition coverage.
   if (InstrumentRegions &&
       CodeGenFunction::isInstrumentedCondition(E->getRHS())) {
+    CGF.maybeUpdateMCDCCondBitmap(E->getRHS(), RHSCond);
     llvm::BasicBlock *RHSBlockCnt = CGF.createBasicBlock("land.rhscnt");
     Builder.CreateCondBr(RHSCond, RHSBlockCnt, ContBlock);
     CGF.EmitBlock(RHSBlockCnt);
@@ -4721,6 +4740,11 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
   // Insert an entry into the phi node for the edge with the value of RHSCond.
   PN->addIncoming(RHSCond, RHSBlock);
 
+  CGF.MCDCLogOpStack.pop_back();
+  // If the top of the logical operator nest, update the MCDC bitmap.
+  if (CGF.MCDCLogOpStack.empty())
+    CGF.maybeUpdateMCDCTestVectorBitmap(E);
+
   // Artificial location to preserve the scope information
   {
     auto NL = ApplyDebugLocation::CreateArtificial(CGF);
@@ -4762,6 +4786,12 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
     if (!LHSCondVal) { // If we have 0 || X, just emit X.
       CGF.incrementProfileCounter(E);
 
+      // If the top of the logical operator nest, reset the MCDC temp to 0.
+      if (CGF.MCDCLogOpStack.empty())
+        CGF.maybeResetMCDCCondBitmap(E);
+
+      CGF.MCDCLogOpStack.push_back(E);
+
       Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
 
       // If we're generating for profiling or coverage, generate a branch to a
@@ -4770,6 +4800,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
       // "FalseBlock" after the increment is done.
       if (InstrumentRegions &&
           CodeGenFunction::isInstrumentedCondition(E->getRHS())) {
+        CGF.maybeUpdateMCDCCondBitmap(E->getRHS(), RHSCond);
         llvm::BasicBlock *FBlock = CGF.createBasicBlock("lor.end");
         llvm::BasicBlock *RHSBlockCnt = CGF.createBasicBlock("lor.rhscnt");
         Builder.CreateCondBr(RHSCond, FBlock, RHSBlockCnt);
@@ -4779,6 +4810,11 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
         CGF.EmitBlock(FBlock);
       }
 
+      CGF.MCDCLogOpStack.pop_back();
+      // If the top of the logical operator nest, update the MCDC bitmap.
+      if (CGF.MCDCLogOpStack.empty())
+        CGF.maybeUpdateMCDCTestVectorBitmap(E);
+
       // ZExt result to int or bool.
       return Builder.CreateZExtOrBitCast(RHSCond, ResTy, "lor.ext");
     }
@@ -4788,6 +4824,12 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
       return llvm::ConstantInt::get(ResTy, 1);
   }
 
+  // If the top of the logical operator nest, reset the MCDC temp to 0.
+  if (CGF.MCDCLogOpStack.empty())
+    CGF.maybeResetMCDCCondBitmap(E);
+
+  CGF.MCDCLogOpStack.push_back(E);
+
   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("lor.end");
   llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("lor.rhs");
 
@@ -4824,6 +4866,7 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
   // condition coverage.
   if (InstrumentRegions &&
       CodeGenFunction::isInstrumentedCondition(E->getRHS())) {
+    CGF.maybeUpdateMCDCCondBitmap(E->getRHS(), RHSCond);
     llvm::BasicBlock *RHSBlockCnt = CGF.createBasicBlock("lor.rhscnt");
     Builder.CreateCondBr(RHSCond, ContBlock, RHSBlockCnt);
     CGF.EmitBlock(RHSBlockCnt);
@@ -4837,6 +4880,11 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
   CGF.EmitBlock(ContBlock);
   PN->addIncoming(RHSCond, RHSBlock);
 
+  CGF.MCDCLogOpStack.pop_back();
+  // If the top of the logical operator nest, update the MCDC bitmap.
+  if (CGF.MCDCLogOpStack.empty())
+    CGF.maybeUpdateMCDCTestVectorBitmap(E);
+
   // ZExt result to int.
   return Builder.CreateZExtOrBitCast(PN, ResTy, "lor.ext");
 }
@@ -4981,6 +5029,10 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
     return Builder.CreateSelect(CondV, LHS, RHS, "cond");
   }
 
+  // If the top of the logical operator nest, reset the MCDC temp to 0.
+  if (CGF.MCDCLogOpStack.empty())
+    CGF.maybeResetMCDCCondBitmap(condExpr);
+
   llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true");
   llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false");
   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");
@@ -4990,6 +5042,13 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
                            CGF.getProfileCount(lhsExpr));
 
   CGF.EmitBlock(LHSBlock);
+
+  // If the top of the logical operator nest, update the MCDC bitmap for the
+  // ConditionalOperator prior to visiting its LHS and RHS blocks, since they
+  // may also contain a boolean expression.
+  if (CGF.MCDCLogOpStack.empty())
+    CGF.maybeUpdateMCDCTestVectorBitmap(condExpr);
+
   CGF.incrementProfileCounter(E);
   eval.begin(CGF);
   Value *LHS = Visit(lhsExpr);
@@ -4999,6 +5058,13 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
   Builder.CreateBr(ContBlock);
 
   CGF.EmitBlock(RHSBlock);
+
+  // If the top of the logical operator nest, update the MCDC bitmap for the
+  // ConditionalOperator prior to visiting its LHS and RHS blocks, since they
+  // may also contain a boolean expression.
+  if (CGF.MCDCLogOpStack.empty())
+    CGF.maybeUpdateMCDCTestVectorBitmap(condExpr);
+
   eval.begin(CGF);
   Value *RHS = Visit(rhsExpr);
   eval.end(CGF);
@@ -5016,6 +5082,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
   llvm::PHINode *PN = Builder.CreatePHI(LHS->getType(), 2, "cond");
   PN->addIncoming(LHS, LHSBlock);
   PN->addIncoming(RHS, RHSBlock);
+
   return PN;
 }
 
@@ -5374,8 +5441,8 @@ static GEPOffsetAndOverflow EmitGEPOffsetInBytes(Value *BasePtr, Value *GEPVal,
     } else {
       // Otherwise this is array-like indexing. The local offset is the index
       // multiplied by the element size.
-      auto *ElementSize = llvm::ConstantInt::get(
-          IntPtrTy, DL.getTypeAllocSize(GTI.getIndexedType()));
+      auto *ElementSize =
+          llvm::ConstantInt::get(IntPtrTy, GTI.getSequentialElementStride(DL));
       auto *IndexS = Builder.CreateIntCast(Index, IntPtrTy, /*isSigned=*/true);
       LocalOffset = eval(BO_Mul, ElementSize, IndexS);
     }
diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp
index acc85165a470..03fc0ec7ff54 100644
--- a/clang/lib/CodeGen/CGObjC.cpp
+++ b/clang/lib/CodeGen/CGObjC.cpp
@@ -3941,6 +3941,8 @@ static unsigned getBaseMachOPlatformID(const llvm::Triple &TT) {
     return llvm::MachO::PLATFORM_TVOS;
   case llvm::Triple::WatchOS:
     return llvm::MachO::PLATFORM_WATCHOS;
+  case llvm::Triple::XROS:
+    return llvm::MachO::PLATFORM_XROS;
   case llvm::Triple::DriverKit:
     return llvm::MachO::PLATFORM_DRIVERKIT;
   default:
@@ -4024,6 +4026,9 @@ static bool isFoundationNeededForDarwinAvailabilityCheck(
   case llvm::Triple::MacOSX:
     FoundationDroppedInVersion = VersionTuple(/*Major=*/10, /*Minor=*/15);
     break;
+  case llvm::Triple::XROS:
+    // XROS doesn't need Foundation.
+    return false;
   case llvm::Triple::DriverKit:
     // DriverKit doesn't need Foundation.
     return false;
diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp
index 4ca1a8cce64d..a36b0cdddaf0 100644
--- a/clang/lib/CodeGen/CGObjCGNU.cpp
+++ b/clang/lib/CodeGen/CGObjCGNU.cpp
@@ -18,6 +18,8 @@
 #include "CGObjCRuntime.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CodeGenTypes.h"
+#include "SanitizerMetadata.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
@@ -168,6 +170,8 @@ class CGObjCGNU : public CGObjCRuntime {
   /// Does the current target use SEH-based exceptions? False implies
   /// Itanium-style DWARF unwinding.
   bool usesSEHExceptions;
+  /// Does the current target uses C++-based exceptions?
+  bool usesCxxExceptions;
 
   /// Helper to check if we are targeting a specific runtime version or later.
   bool isRuntime(ObjCRuntime::Kind kind, unsigned major, unsigned minor=0) {
@@ -595,6 +599,10 @@ class CGObjCGNU : public CGObjCRuntime {
 
   llvm::Function *GenerateMethod(const ObjCMethodDecl *OMD,
                                  const ObjCContainerDecl *CD) override;
+
+  // Map to unify direct method definitions.
+  llvm::DenseMap
+      DirectMethodDefinitions;
   void GenerateDirectMethodPrologue(CodeGenFunction &CGF, llvm::Function *Fn,
                                     const ObjCMethodDecl *OMD,
                                     const ObjCContainerDecl *CD) override;
@@ -819,12 +827,18 @@ class CGObjCGNUstep : public CGObjCGNU {
       SlotLookupSuperFn.init(&CGM, "objc_slot_lookup_super", SlotTy,
                              PtrToObjCSuperTy, SelectorTy);
       // If we're in ObjC++ mode, then we want to make
-      if (usesSEHExceptions) {
-          llvm::Type *VoidTy = llvm::Type::getVoidTy(VMContext);
-          // void objc_exception_rethrow(void)
-          ExceptionReThrowFn.init(&CGM, "objc_exception_rethrow", VoidTy);
+      llvm::Type *VoidTy = llvm::Type::getVoidTy(VMContext);
+      if (usesCxxExceptions) {
+        // void *__cxa_begin_catch(void *e)
+        EnterCatchFn.init(&CGM, "__cxa_begin_catch", PtrTy, PtrTy);
+        // void __cxa_end_catch(void)
+        ExitCatchFn.init(&CGM, "__cxa_end_catch", VoidTy);
+        // void objc_exception_rethrow(void*)
+        ExceptionReThrowFn.init(&CGM, "__cxa_rethrow", PtrTy);
+      } else if (usesSEHExceptions) {
+        // void objc_exception_rethrow(void)
+        ExceptionReThrowFn.init(&CGM, "objc_exception_rethrow", VoidTy);
       } else if (CGM.getLangOpts().CPlusPlus) {
-        llvm::Type *VoidTy = llvm::Type::getVoidTy(VMContext);
         // void *__cxa_begin_catch(void *e)
         EnterCatchFn.init(&CGM, "__cxa_begin_catch", PtrTy, PtrTy);
         // void __cxa_end_catch(void)
@@ -833,7 +847,6 @@ class CGObjCGNUstep : public CGObjCGNU {
         ExceptionReThrowFn.init(&CGM, "_Unwind_Resume_or_Rethrow", VoidTy,
                                 PtrTy);
       } else if (R.getVersion() >= VersionTuple(1, 7)) {
-        llvm::Type *VoidTy = llvm::Type::getVoidTy(VMContext);
         // id objc_begin_catch(void *e)
         EnterCatchFn.init(&CGM, "objc_begin_catch", IdTy, PtrTy);
         // void objc_end_catch(void)
@@ -841,7 +854,6 @@ class CGObjCGNUstep : public CGObjCGNU {
         // void _Unwind_Resume_or_Rethrow(void*)
         ExceptionReThrowFn.init(&CGM, "objc_exception_rethrow", VoidTy, PtrTy);
       }
-      llvm::Type *VoidTy = llvm::Type::getVoidTy(VMContext);
       SetPropertyAtomic.init(&CGM, "objc_setProperty_atomic", VoidTy, IdTy,
                              SelectorTy, IdTy, PtrDiffTy);
       SetPropertyAtomicCopy.init(&CGM, "objc_setProperty_atomic_copy", VoidTy,
@@ -911,6 +923,14 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
     ClassAliasSection,
     ConstantStringSection
   };
+  /// The subset of `objc_class_flags` used at compile time.
+  enum ClassFlags {
+    /// This is a metaclass
+    ClassFlagMeta = (1 << 0),
+    /// This class has been initialised by the runtime (+initialize has been
+    /// sent if necessary).
+    ClassFlagInitialized = (1 << 8),
+  };
   static const char *const SectionsBaseNames[8];
   static const char *const PECOFFSectionsBaseNames[8];
   template
@@ -926,6 +946,8 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
   /// structure describing the receiver and the class, and a selector as
   /// arguments.  Returns the IMP for the corresponding method.
   LazyRuntimeFunction MsgLookupSuperFn;
+  /// Function to ensure that +initialize is sent to a class.
+  LazyRuntimeFunction SentInitializeFn;
   /// A flag indicating if we've emitted at least one protocol.
   /// If we haven't, then we need to emit an empty protocol, to ensure that the
   /// __start__objc_protocols and __stop__objc_protocols sections exist.
@@ -1425,12 +1447,24 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
                                 const std::string &TypeEncoding) override {
     return GetConstantSelector(Sel, TypeEncoding);
   }
+  std::string GetSymbolNameForTypeEncoding(const std::string &TypeEncoding) {
+    std::string MangledTypes = std::string(TypeEncoding);
+    // @ is used as a special character in ELF symbol names (used for symbol
+    // versioning), so mangle the name to not include it.  Replace it with a
+    // character that is not a valid type encoding character (and, being
+    // non-printable, never will be!)
+    if (CGM.getTriple().isOSBinFormatELF())
+      std::replace(MangledTypes.begin(), MangledTypes.end(), '@', '\1');
+    // = in dll exported names causes lld to fail when linking on Windows.
+    if (CGM.getTriple().isOSWindows())
+      std::replace(MangledTypes.begin(), MangledTypes.end(), '=', '\2');
+    return MangledTypes;
+  }
   llvm::Constant  *GetTypeString(llvm::StringRef TypeEncoding) {
     if (TypeEncoding.empty())
       return NULLPtr;
-    std::string MangledTypes = std::string(TypeEncoding);
-    std::replace(MangledTypes.begin(), MangledTypes.end(),
-      '@', '\1');
+    std::string MangledTypes =
+        GetSymbolNameForTypeEncoding(std::string(TypeEncoding));
     std::string TypesVarName = ".objc_sel_types_" + MangledTypes;
     auto *TypesGlobal = TheModule.getGlobalVariable(TypesVarName);
     if (!TypesGlobal) {
@@ -1447,13 +1481,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
   }
   llvm::Constant *GetConstantSelector(Selector Sel,
                                       const std::string &TypeEncoding) override {
-    // @ is used as a special character in symbol names (used for symbol
-    // versioning), so mangle the name to not include it.  Replace it with a
-    // character that is not a valid type encoding character (and, being
-    // non-printable, never will be!)
-    std::string MangledTypes = TypeEncoding;
-    std::replace(MangledTypes.begin(), MangledTypes.end(),
-      '@', '\1');
+    std::string MangledTypes = GetSymbolNameForTypeEncoding(TypeEncoding);
     auto SelVarName = (StringRef(".objc_selector_") + Sel.getAsString() + "_" +
       MangledTypes).str();
     if (auto *GV = TheModule.getNamedGlobal(SelVarName))
@@ -1665,9 +1693,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
                                         const ObjCIvarDecl *Ivar) override {
     std::string TypeEncoding;
     CGM.getContext().getObjCEncodingForType(Ivar->getType(), TypeEncoding);
-    // Prevent the @ from being interpreted as a symbol version.
-    std::replace(TypeEncoding.begin(), TypeEncoding.end(),
-      '@', '\1');
+    TypeEncoding = GetSymbolNameForTypeEncoding(TypeEncoding);
     const std::string Name = "__objc_ivar_offset_" + ID->getNameAsString()
       + '.' + Ivar->getNameAsString() + '.' + TypeEncoding;
     return Name;
@@ -1709,7 +1735,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
     metaclassFields.addInt(LongTy, 0);
     // unsigned long info;
     // objc_class_flag_meta
-    metaclassFields.addInt(LongTy, 1);
+    metaclassFields.addInt(LongTy, ClassFlags::ClassFlagMeta);
     // long instance_size;
     // Setting this to zero is consistent with the older ABI, but it might be
     // more sensible to set this to sizeof(struct objc_class)
@@ -1851,6 +1877,8 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
                     llvm::GlobalValue::HiddenVisibility :
                     llvm::GlobalValue::DefaultVisibility;
         OffsetVar->setVisibility(ivarVisibility);
+        if (ivarVisibility != llvm::GlobalValue::HiddenVisibility)
+          CGM.setGVProperties(OffsetVar, OID->getClassInterface());
         ivarBuilder.add(OffsetVar);
         // Ivar size
         ivarBuilder.addInt(Int32Ty,
@@ -1981,6 +2009,8 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
     CGObjCGNUstep2(CodeGenModule &Mod) : CGObjCGNUstep(Mod, 10, 4, 2) {
       MsgLookupSuperFn.init(&CGM, "objc_msg_lookup_super", IMPTy,
                             PtrToObjCSuperTy, SelectorTy);
+      SentInitializeFn.init(&CGM, "objc_send_initialize",
+                            llvm::Type::getVoidTy(VMContext), IdTy);
       // struct objc_property
       // {
       //   const char *name;
@@ -1994,6 +2024,106 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
             { PtrToInt8Ty, PtrToInt8Ty, PtrToInt8Ty, PtrToInt8Ty, PtrToInt8Ty });
     }
 
+    void GenerateDirectMethodPrologue(CodeGenFunction &CGF, llvm::Function *Fn,
+                                      const ObjCMethodDecl *OMD,
+                                      const ObjCContainerDecl *CD) override {
+      auto &Builder = CGF.Builder;
+      bool ReceiverCanBeNull = true;
+      auto selfAddr = CGF.GetAddrOfLocalVar(OMD->getSelfDecl());
+      auto selfValue = Builder.CreateLoad(selfAddr);
+
+      // Generate:
+      //
+      // /* unless the receiver is never NULL */
+      // if (self == nil) {
+      //     return (ReturnType){ };
+      // }
+      //
+      // /* for class methods only to force class lazy initialization */
+      // if (!__objc_{class}_initialized)
+      // {
+      //   objc_send_initialize(class);
+      //   __objc_{class}_initialized = 1;
+      // }
+      //
+      // _cmd = @selector(...)
+      // ...
+
+      if (OMD->isClassMethod()) {
+        const ObjCInterfaceDecl *OID = cast(CD);
+
+        // Nullable `Class` expressions cannot be messaged with a direct method
+        // so the only reason why the receive can be null would be because
+        // of weak linking.
+        ReceiverCanBeNull = isWeakLinkedClass(OID);
+      }
+
+      llvm::MDBuilder MDHelper(CGM.getLLVMContext());
+      if (ReceiverCanBeNull) {
+        llvm::BasicBlock *SelfIsNilBlock =
+            CGF.createBasicBlock("objc_direct_method.self_is_nil");
+        llvm::BasicBlock *ContBlock =
+            CGF.createBasicBlock("objc_direct_method.cont");
+
+        // if (self == nil) {
+        auto selfTy = cast(selfValue->getType());
+        auto Zero = llvm::ConstantPointerNull::get(selfTy);
+
+        Builder.CreateCondBr(Builder.CreateICmpEQ(selfValue, Zero),
+                             SelfIsNilBlock, ContBlock,
+                             MDHelper.createBranchWeights(1, 1 << 20));
+
+        CGF.EmitBlock(SelfIsNilBlock);
+
+        //   return (ReturnType){ };
+        auto retTy = OMD->getReturnType();
+        Builder.SetInsertPoint(SelfIsNilBlock);
+        if (!retTy->isVoidType()) {
+          CGF.EmitNullInitialization(CGF.ReturnValue, retTy);
+        }
+        CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
+        // }
+
+        // rest of the body
+        CGF.EmitBlock(ContBlock);
+        Builder.SetInsertPoint(ContBlock);
+      }
+
+      if (OMD->isClassMethod()) {
+        // Prefix of the class type.
+        auto *classStart =
+            llvm::StructType::get(PtrTy, PtrTy, PtrTy, LongTy, LongTy);
+        auto &astContext = CGM.getContext();
+        auto flags = Builder.CreateLoad(
+            Address{Builder.CreateStructGEP(classStart, selfValue, 4), LongTy,
+                    CharUnits::fromQuantity(
+                        astContext.getTypeAlign(astContext.UnsignedLongTy))});
+        auto isInitialized =
+            Builder.CreateAnd(flags, ClassFlags::ClassFlagInitialized);
+        llvm::BasicBlock *notInitializedBlock =
+            CGF.createBasicBlock("objc_direct_method.class_uninitialized");
+        llvm::BasicBlock *initializedBlock =
+            CGF.createBasicBlock("objc_direct_method.class_initialized");
+        Builder.CreateCondBr(Builder.CreateICmpEQ(isInitialized, Zeros[0]),
+                             notInitializedBlock, initializedBlock,
+                             MDHelper.createBranchWeights(1, 1 << 20));
+        CGF.EmitBlock(notInitializedBlock);
+        Builder.SetInsertPoint(notInitializedBlock);
+        CGF.EmitRuntimeCall(SentInitializeFn, selfValue);
+        Builder.CreateBr(initializedBlock);
+        CGF.EmitBlock(initializedBlock);
+        Builder.SetInsertPoint(initializedBlock);
+      }
+
+      // only synthesize _cmd if it's referenced
+      if (OMD->getCmdDecl()->isUsed()) {
+        // `_cmd` is not a parameter to direct methods, so storage must be
+        // explicitly declared for it.
+        CGF.EmitVarDecl(*OMD->getCmdDecl());
+        Builder.CreateStore(GetSelector(CGF, OMD),
+                            CGF.GetAddrOfLocalVar(OMD->getCmdDecl()));
+      }
+    }
 };
 
 const char *const CGObjCGNUstep2::SectionsBaseNames[8] =
@@ -2124,6 +2254,9 @@ CGObjCGNU::CGObjCGNU(CodeGenModule &cgm, unsigned runtimeABIVersion,
   msgSendMDKind = VMContext.getMDKindID("GNUObjCMessageSend");
   usesSEHExceptions =
       cgm.getContext().getTargetInfo().getTriple().isWindowsMSVCEnvironment();
+  usesCxxExceptions =
+      cgm.getContext().getTargetInfo().getTriple().isOSCygMing() &&
+      isRuntime(ObjCRuntime::GNUstep, 2);
 
   CodeGenTypes &Types = CGM.getTypes();
   IntTy = cast(
@@ -2210,7 +2343,10 @@ CGObjCGNU::CGObjCGNU(CodeGenModule &cgm, unsigned runtimeABIVersion,
 
   // void objc_exception_throw(id);
   ExceptionThrowFn.init(&CGM, "objc_exception_throw", VoidTy, IdTy);
-  ExceptionReThrowFn.init(&CGM, "objc_exception_throw", VoidTy, IdTy);
+  ExceptionReThrowFn.init(&CGM,
+                          usesCxxExceptions ? "objc_exception_rethrow"
+                                            : "objc_exception_throw",
+                          VoidTy, IdTy);
   // int objc_sync_enter(id);
   SyncEnterFn.init(&CGM, "objc_sync_enter", IntTy, IdTy);
   // int objc_sync_exit(id);
@@ -2387,7 +2523,7 @@ llvm::Constant *CGObjCGNUstep::GetEHType(QualType T) {
   if (usesSEHExceptions)
     return CGM.getCXXABI().getAddrOfRTTIDescriptor(T);
 
-  if (!CGM.getLangOpts().CPlusPlus)
+  if (!CGM.getLangOpts().CPlusPlus && !usesCxxExceptions)
     return CGObjCGNU::GetEHType(T);
 
   // For Objective-C++, we want to provide the ability to catch both C++ and
@@ -2631,13 +2767,18 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF,
     }
   }
 
+  bool isDirect = Method && Method->isDirectMethod();
+
   IdTy = cast(CGM.getTypes().ConvertType(ASTIdTy));
   llvm::Value *cmd;
-  if (Method)
-    cmd = GetSelector(CGF, Method);
-  else
-    cmd = GetSelector(CGF, Sel);
-  cmd = EnforceType(Builder, cmd, SelectorTy);
+  if (!isDirect) {
+    if (Method)
+      cmd = GetSelector(CGF, Method);
+    else
+      cmd = GetSelector(CGF, Sel);
+    cmd = EnforceType(Builder, cmd, SelectorTy);
+  }
+
   Receiver = EnforceType(Builder, Receiver, IdTy);
 
   llvm::Metadata *impMD[] = {
@@ -2649,7 +2790,8 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF,
 
   CallArgList ActualArgs;
   ActualArgs.add(RValue::get(Receiver), ASTIdTy);
-  ActualArgs.add(RValue::get(cmd), CGF.getContext().getObjCSelType());
+  if (!isDirect)
+    ActualArgs.add(RValue::get(cmd), CGF.getContext().getObjCSelType());
   ActualArgs.addFrom(CallArgs);
 
   MessageSendInfo MSI = getMessageSendInfo(Method, ResultType, ActualArgs);
@@ -2668,7 +2810,7 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF,
   // Rather than doing a whole target-specific analysis, we assume it
   // only works for void, integer, and pointer types, and in all
   // other cases we do an explicit nil check is emitted code.  In
-  // addition to ensuring we produe a zero value for other types, this
+  // addition to ensuring we produce a zero value for other types, this
   // sidesteps the few outright CC incompatibilities we know about that
   // could otherwise lead to crashes, like when a method is expected to
   // return on the x87 floating point stack or adjust the stack pointer
@@ -2702,8 +2844,9 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF,
         // FIXME: we probably need a size limit here, but we've
         // never imposed one before
       } else {
-        // Otherwise, use an explicit check just to be sure.
-        requiresExplicitZeroResult = true;
+        // Otherwise, use an explicit check just to be sure, unless we're
+        // calling a direct method, where the implementation does this for us.
+        requiresExplicitZeroResult = !isDirect;
       }
     }
 
@@ -2747,10 +2890,14 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF,
   // Get the IMP to call
   llvm::Value *imp;
 
-  // If we have non-legacy dispatch specified, we try using the objc_msgSend()
-  // functions.  These are not supported on all platforms (or all runtimes on a
-  // given platform), so we
-  switch (CGM.getCodeGenOpts().getObjCDispatchMethod()) {
+  // If this is a direct method, just emit it here.
+  if (isDirect)
+    imp = GenerateMethod(Method, Method->getClassInterface());
+  else
+    // If we have non-legacy dispatch specified, we try using the
+    // objc_msgSend() functions.  These are not supported on all platforms
+    // (or all runtimes on a given platform), so we
+    switch (CGM.getCodeGenOpts().getObjCDispatchMethod()) {
     case CodeGenOptions::Legacy:
       imp = LookupIMP(CGF, Receiver, cmd, node, MSI);
       break;
@@ -2773,7 +2920,7 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF,
                      llvm::FunctionType::get(IdTy, IdTy, true), "objc_msgSend")
                   .getCallee();
       }
-  }
+    }
 
   // Reset the receiver in case the lookup modified it
   ActualArgs[0] = CallArg(RValue::get(Receiver), ASTIdTy);
@@ -2783,7 +2930,8 @@ CGObjCGNU::GenerateMessageSend(CodeGenFunction &CGF,
   llvm::CallBase *call;
   CGCallee callee(CGCalleeInfo(), imp);
   RValue msgRet = CGF.EmitCall(MSI.CallInfo, callee, Return, ActualArgs, &call);
-  call->setMetadata(msgSendMDKind, node);
+  if (!isDirect)
+    call->setMetadata(msgSendMDKind, node);
 
   if (requiresNilReceiverCheck) {
     llvm::BasicBlock *nonNilPathBB = CGF.Builder.GetInsertBlock();
@@ -3906,14 +4054,50 @@ llvm::Function *CGObjCGNU::GenerateMethod(const ObjCMethodDecl *OMD,
   CodeGenTypes &Types = CGM.getTypes();
   llvm::FunctionType *MethodTy =
     Types.GetFunctionType(Types.arrangeObjCMethodDeclaration(OMD));
-  std::string FunctionName = getSymbolNameForMethod(OMD);
-
-  llvm::Function *Method
-    = llvm::Function::Create(MethodTy,
-                             llvm::GlobalValue::InternalLinkage,
-                             FunctionName,
-                             &TheModule);
-  return Method;
+
+  bool isDirect = OMD->isDirectMethod();
+  std::string FunctionName =
+      getSymbolNameForMethod(OMD, /*include category*/ !isDirect);
+
+  if (!isDirect)
+    return llvm::Function::Create(MethodTy,
+                                  llvm::GlobalVariable::InternalLinkage,
+                                  FunctionName, &TheModule);
+
+  auto *COMD = OMD->getCanonicalDecl();
+  auto I = DirectMethodDefinitions.find(COMD);
+  llvm::Function *OldFn = nullptr, *Fn = nullptr;
+
+  if (I == DirectMethodDefinitions.end()) {
+    auto *F =
+        llvm::Function::Create(MethodTy, llvm::GlobalVariable::ExternalLinkage,
+                               FunctionName, &TheModule);
+    DirectMethodDefinitions.insert(std::make_pair(COMD, F));
+    return F;
+  }
+
+  // Objective-C allows for the declaration and implementation types
+  // to differ slightly.
+  //
+  // If we're being asked for the Function associated for a method
+  // implementation, a previous value might have been cached
+  // based on the type of the canonical declaration.
+  //
+  // If these do not match, then we'll replace this function with
+  // a new one that has the proper type below.
+  if (!OMD->getBody() || COMD->getReturnType() == OMD->getReturnType())
+    return I->second;
+
+  OldFn = I->second;
+  Fn = llvm::Function::Create(MethodTy, llvm::GlobalValue::ExternalLinkage, "",
+                              &CGM.getModule());
+  Fn->takeName(OldFn);
+  OldFn->replaceAllUsesWith(Fn);
+  OldFn->eraseFromParent();
+
+  // Replace the cached function in the map.
+  I->second = Fn;
+  return Fn;
 }
 
 void CGObjCGNU::GenerateDirectMethodPrologue(CodeGenFunction &CGF,
@@ -3993,7 +4177,7 @@ void CGObjCGNU::EmitThrowStmt(CodeGenFunction &CGF,
     ExceptionAsObject = CGF.ObjCEHValueStack.back();
     isRethrow = true;
   }
-  if (isRethrow && usesSEHExceptions) {
+  if (isRethrow && (usesSEHExceptions || usesCxxExceptions)) {
     // For SEH, ExceptionAsObject may be undef, because the catch handler is
     // not passed it for catchalls and so it is not visible to the catch
     // funclet.  The real thrown object will still be live on the stack at this
@@ -4003,8 +4187,7 @@ void CGObjCGNU::EmitThrowStmt(CodeGenFunction &CGF,
     // argument.
     llvm::CallBase *Throw = CGF.EmitRuntimeCallOrInvoke(ExceptionReThrowFn);
     Throw->setDoesNotReturn();
-  }
-  else {
+  } else {
     ExceptionAsObject = CGF.Builder.CreateBitCast(ExceptionAsObject, IdTy);
     llvm::CallBase *Throw =
         CGF.EmitRuntimeCallOrInvoke(ExceptionThrowFn, ExceptionAsObject);
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index ea6645a39e83..4855e7410a01 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -1044,6 +1044,13 @@ CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
                                          ? CGM.getLangOpts().OMPHostIRFile
                                          : StringRef{});
   OMPBuilder.setConfig(Config);
+
+  // The user forces the compiler to behave as if omp requires
+  // unified_shared_memory was given.
+  if (CGM.getLangOpts().OpenMPForceUSM) {
+    HasRequiresUnifiedSharedMemory = true;
+    OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);
+  }
 }
 
 void CGOpenMPRuntime::clear() {
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 0f79a2e861d2..beff0ad9da27 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -837,7 +837,19 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
   if (!ThenCount && !getCurrentProfileCount() &&
       CGM.getCodeGenOpts().OptimizationLevel)
     LH = Stmt::getLikelihood(S.getThen(), S.getElse());
-  EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, ThenCount, LH);
+
+  // When measuring MC/DC, always fully evaluate the condition up front using
+  // EvaluateExprAsBool() so that the test vector bitmap can be updated prior to
+  // executing the body of the if.then or if.else. This is useful for when
+  // there is a 'return' within the body, but this is particularly beneficial
+  // when one if-stmt is nested within another if-stmt so that all of the MC/DC
+  // updates are kept linear and consistent.
+  if (!CGM.getCodeGenOpts().MCDCCoverage)
+    EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, ThenCount, LH);
+  else {
+    llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
+    Builder.CreateCondBr(BoolCondVal, ThenBlock, ElseBlock);
+  }
 
   // Emit the 'then' code.
   EmitBlock(ThenBlock);
@@ -2387,9 +2399,9 @@ EmitAsmStores(CodeGenFunction &CGF, const AsmStmt &S,
         Tmp = Builder.CreatePtrToInt(
             Tmp, llvm::IntegerType::get(CTX, (unsigned)TmpSize));
         Tmp = Builder.CreateTrunc(Tmp, TruncTy);
-      } else if (TruncTy->isIntegerTy()) {
+      } else if (Tmp->getType()->isIntegerTy() && TruncTy->isIntegerTy()) {
         Tmp = Builder.CreateZExtOrTrunc(Tmp, TruncTy);
-      } else if (TruncTy->isVectorTy()) {
+      } else if (Tmp->getType()->isVectorTy() || TruncTy->isVectorTy()) {
         Tmp = Builder.CreateBitCast(Tmp, TruncTy);
       }
     }
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index ed426098ac69..e362c9da51fe 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -6406,13 +6406,11 @@ static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF,
   }
 }
 
-static void emitOMPAtomicCompareExpr(CodeGenFunction &CGF,
-                                     llvm::AtomicOrdering AO, const Expr *X,
-                                     const Expr *V, const Expr *R,
-                                     const Expr *E, const Expr *D,
-                                     const Expr *CE, bool IsXBinopExpr,
-                                     bool IsPostfixUpdate, bool IsFailOnly,
-                                     SourceLocation Loc) {
+static void emitOMPAtomicCompareExpr(
+    CodeGenFunction &CGF, llvm::AtomicOrdering AO, llvm::AtomicOrdering FailAO,
+    const Expr *X, const Expr *V, const Expr *R, const Expr *E, const Expr *D,
+    const Expr *CE, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly,
+    SourceLocation Loc) {
   llvm::OpenMPIRBuilder &OMPBuilder =
       CGF.CGM.getOpenMPRuntime().getOMPBuilder();
 
@@ -6477,13 +6475,21 @@ static void emitOMPAtomicCompareExpr(CodeGenFunction &CGF,
               R->getType().isVolatileQualified()};
   }
 
-  CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
-      CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
-      IsPostfixUpdate, IsFailOnly));
+  if (FailAO == llvm::AtomicOrdering::NotAtomic) {
+    // fail clause was not mentionend on the
+    // "#pragma omp atomic compare" construct.
+    CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
+        CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
+        IsPostfixUpdate, IsFailOnly));
+  } else
+    CGF.Builder.restoreIP(OMPBuilder.createAtomicCompare(
+        CGF.Builder, XOpVal, VOpVal, ROpVal, EVal, DVal, AO, Op, IsXBinopExpr,
+        IsPostfixUpdate, IsFailOnly, FailAO));
 }
 
 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
-                              llvm::AtomicOrdering AO, bool IsPostfixUpdate,
+                              llvm::AtomicOrdering AO,
+                              llvm::AtomicOrdering FailAO, bool IsPostfixUpdate,
                               const Expr *X, const Expr *V, const Expr *R,
                               const Expr *E, const Expr *UE, const Expr *D,
                               const Expr *CE, bool IsXLHSInRHSPart,
@@ -6504,12 +6510,8 @@ static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
                              IsXLHSInRHSPart, Loc);
     break;
   case OMPC_compare: {
-    emitOMPAtomicCompareExpr(CGF, AO, X, V, R, E, D, CE, IsXLHSInRHSPart,
-                             IsPostfixUpdate, IsFailOnly, Loc);
-    break;
-  }
-  case OMPC_fail: {
-    //TODO
+    emitOMPAtomicCompareExpr(CGF, AO, FailAO, X, V, R, E, D, CE,
+                             IsXLHSInRHSPart, IsPostfixUpdate, IsFailOnly, Loc);
     break;
   }
   default:
@@ -6519,6 +6521,8 @@ static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
 
 void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
   llvm::AtomicOrdering AO = llvm::AtomicOrdering::Monotonic;
+  // Fail Memory Clause Ordering.
+  llvm::AtomicOrdering FailAO = llvm::AtomicOrdering::NotAtomic;
   bool MemOrderingSpecified = false;
   if (S.getSingleClause()) {
     AO = llvm::AtomicOrdering::SequentiallyConsistent;
@@ -6572,12 +6576,27 @@ void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
     }
   }
 
+  if (KindsEncountered.contains(OMPC_compare) &&
+      KindsEncountered.contains(OMPC_fail)) {
+    Kind = OMPC_compare;
+    const auto *FailClause = S.getSingleClause();
+    if (FailClause) {
+      OpenMPClauseKind FailParameter = FailClause->getFailParameter();
+      if (FailParameter == llvm::omp::OMPC_relaxed)
+        FailAO = llvm::AtomicOrdering::Monotonic;
+      else if (FailParameter == llvm::omp::OMPC_acquire)
+        FailAO = llvm::AtomicOrdering::Acquire;
+      else if (FailParameter == llvm::omp::OMPC_seq_cst)
+        FailAO = llvm::AtomicOrdering::SequentiallyConsistent;
+    }
+  }
+
   LexicalScope Scope(*this, S.getSourceRange());
   EmitStopPoint(S.getAssociatedStmt());
-  emitOMPAtomicExpr(*this, Kind, AO, S.isPostfixUpdate(), S.getX(), S.getV(),
-                    S.getR(), S.getExpr(), S.getUpdateExpr(), S.getD(),
-                    S.getCondExpr(), S.isXLHSInRHSPart(), S.isFailOnly(),
-                    S.getBeginLoc());
+  emitOMPAtomicExpr(*this, Kind, AO, FailAO, S.isPostfixUpdate(), S.getX(),
+                    S.getV(), S.getR(), S.getExpr(), S.getUpdateExpr(),
+                    S.getD(), S.getCondExpr(), S.isXLHSInRHSPart(),
+                    S.isFailOnly(), S.getBeginLoc());
 }
 
 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp
index 27a2cab4f753..8dee3f74b44b 100644
--- a/clang/lib/CodeGen/CGVTables.cpp
+++ b/clang/lib/CodeGen/CGVTables.cpp
@@ -793,7 +793,7 @@ void CodeGenVTables::addVTableComponent(ConstantArrayBuilder &builder,
     llvm::Constant *fnPtr;
 
     // Pure virtual member functions.
-    if (cast(GD.getDecl())->isPure()) {
+    if (cast(GD.getDecl())->isPureVirtual()) {
       if (!PureVirtualFn)
         PureVirtualFn =
             getSpecialVirtualFn(CGM.getCXXABI().GetPureVirtualCallName());
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 2199d7b58fb9..2673e4a5cee7 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -1256,6 +1256,7 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
 
 void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
   incrementProfileCounter(Body);
+  maybeCreateMCDCCondBitmap();
   if (const CompoundStmt *S = dyn_cast(Body))
     EmitCompoundStmtWithoutScope(*S);
   else
@@ -1601,6 +1602,13 @@ bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
 bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
                                                    bool &ResultBool,
                                                    bool AllowLabels) {
+  // If MC/DC is enabled, disable folding so that we can instrument all
+  // conditions to yield complete test vectors. We still keep track of
+  // folded conditions during region mapping and visualization.
+  if (!AllowLabels && CGM.getCodeGenOpts().hasProfileClangInstr() &&
+      CGM.getCodeGenOpts().MCDCCoverage)
+    return false;
+
   llvm::APSInt ResultInt;
   if (!ConstantFoldsToSimpleInteger(Cond, ResultInt, AllowLabels))
     return false;
@@ -1629,16 +1637,20 @@ bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
   return true;
 }
 
+/// Strip parentheses and simplistic logical-NOT operators.
+const Expr *CodeGenFunction::stripCond(const Expr *C) {
+  while (const UnaryOperator *Op = dyn_cast(C->IgnoreParens())) {
+    if (Op->getOpcode() != UO_LNot)
+      break;
+    C = Op->getSubExpr();
+  }
+  return C->IgnoreParens();
+}
+
 /// Determine whether the given condition is an instrumentable condition
 /// (i.e. no "&&" or "||").
 bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {
-  // Bypass simplistic logical-NOT operator before determining whether the
-  // condition contains any other logical operator.
-  if (const UnaryOperator *UnOp = dyn_cast(C->IgnoreParens()))
-    if (UnOp->getOpcode() == UO_LNot)
-      C = UnOp->getSubExpr();
-
-  const BinaryOperator *BOp = dyn_cast(C->IgnoreParens());
+  const BinaryOperator *BOp = dyn_cast(stripCond(C));
   return (!BOp || !BOp->isLogicalOp());
 }
 
@@ -1717,17 +1729,19 @@ void CodeGenFunction::EmitBranchToCounterBlock(
 /// statement) to the specified blocks.  Based on the condition, this might try
 /// to simplify the codegen of the conditional based on the branch.
 /// \param LH The value of the likelihood attribute on the True branch.
-void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
-                                           llvm::BasicBlock *TrueBlock,
-                                           llvm::BasicBlock *FalseBlock,
-                                           uint64_t TrueCount,
-                                           Stmt::Likelihood LH) {
+/// \param ConditionalOp Used by MC/DC code coverage to track the result of the
+/// ConditionalOperator (ternary) through a recursive call for the operator's
+/// LHS and RHS nodes.
+void CodeGenFunction::EmitBranchOnBoolExpr(
+    const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock,
+    uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp) {
   Cond = Cond->IgnoreParens();
 
   if (const BinaryOperator *CondBOp = dyn_cast(Cond)) {
-
     // Handle X && Y in a condition.
     if (CondBOp->getOpcode() == BO_LAnd) {
+      MCDCLogOpStack.push_back(CondBOp);
+
       // If we have "1 && X", simplify the code.  "0 && X" would have constant
       // folded if the case was simple enough.
       bool ConstantBool = false;
@@ -1735,8 +1749,10 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
           ConstantBool) {
         // br(1 && X) -> br(X).
         incrementProfileCounter(CondBOp);
-        return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,
-                                        FalseBlock, TrueCount, LH);
+        EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,
+                                 FalseBlock, TrueCount, LH);
+        MCDCLogOpStack.pop_back();
+        return;
       }
 
       // If we have "X && 1", simplify the code to use an uncond branch.
@@ -1744,8 +1760,10 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
       if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
           ConstantBool) {
         // br(X && 1) -> br(X).
-        return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LAnd, TrueBlock,
-                                        FalseBlock, TrueCount, LH, CondBOp);
+        EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LAnd, TrueBlock,
+                                 FalseBlock, TrueCount, LH, CondBOp);
+        MCDCLogOpStack.pop_back();
+        return;
       }
 
       // Emit the LHS as a conditional.  If the LHS conditional is false, we
@@ -1774,11 +1792,13 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
       EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,
                                FalseBlock, TrueCount, LH);
       eval.end(*this);
-
+      MCDCLogOpStack.pop_back();
       return;
     }
 
     if (CondBOp->getOpcode() == BO_LOr) {
+      MCDCLogOpStack.push_back(CondBOp);
+
       // If we have "0 || X", simplify the code.  "1 || X" would have constant
       // folded if the case was simple enough.
       bool ConstantBool = false;
@@ -1786,8 +1806,10 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
           !ConstantBool) {
         // br(0 || X) -> br(X).
         incrementProfileCounter(CondBOp);
-        return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock,
-                                        FalseBlock, TrueCount, LH);
+        EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock,
+                                 FalseBlock, TrueCount, LH);
+        MCDCLogOpStack.pop_back();
+        return;
       }
 
       // If we have "X || 0", simplify the code to use an uncond branch.
@@ -1795,10 +1817,11 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
       if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
           !ConstantBool) {
         // br(X || 0) -> br(X).
-        return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LOr, TrueBlock,
-                                        FalseBlock, TrueCount, LH, CondBOp);
+        EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LOr, TrueBlock,
+                                 FalseBlock, TrueCount, LH, CondBOp);
+        MCDCLogOpStack.pop_back();
+        return;
       }
-
       // Emit the LHS as a conditional.  If the LHS conditional is true, we
       // want to jump to the TrueBlock.
       llvm::BasicBlock *LHSFalse = createBasicBlock("lor.lhs.false");
@@ -1829,14 +1852,20 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
                                RHSCount, LH);
 
       eval.end(*this);
-
+      MCDCLogOpStack.pop_back();
       return;
     }
   }
 
   if (const UnaryOperator *CondUOp = dyn_cast(Cond)) {
     // br(!x, t, f) -> br(x, f, t)
-    if (CondUOp->getOpcode() == UO_LNot) {
+    // Avoid doing this optimization when instrumenting a condition for MC/DC.
+    // LNot is taken as part of the condition for simplicity, and changing its
+    // sense negatively impacts test vector tracking.
+    bool MCDCCondition = CGM.getCodeGenOpts().hasProfileClangInstr() &&
+                         CGM.getCodeGenOpts().MCDCCoverage &&
+                         isInstrumentedCondition(Cond);
+    if (CondUOp->getOpcode() == UO_LNot && !MCDCCondition) {
       // Negate the count.
       uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
       // The values of the enum are chosen to make this negation possible.
@@ -1876,14 +1905,14 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
     {
       ApplyDebugLocation DL(*this, Cond);
       EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock,
-                           LHSScaledTrueCount, LH);
+                           LHSScaledTrueCount, LH, CondOp);
     }
     cond.end(*this);
 
     cond.begin(*this);
     EmitBlock(RHSBlock);
     EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock,
-                         TrueCount - LHSScaledTrueCount, LH);
+                         TrueCount - LHSScaledTrueCount, LH, CondOp);
     cond.end(*this);
 
     return;
@@ -1906,6 +1935,21 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
     CondV = EvaluateExprAsBool(Cond);
   }
 
+  // If not at the top of the logical operator nest, update MCDC temp with the
+  // boolean result of the evaluated condition.
+  if (!MCDCLogOpStack.empty()) {
+    const Expr *MCDCBaseExpr = Cond;
+    // When a nested ConditionalOperator (ternary) is encountered in a boolean
+    // expression, MC/DC tracks the result of the ternary, and this is tied to
+    // the ConditionalOperator expression and not the ternary's LHS or RHS. If
+    // this is the case, the ConditionalOperator expression is passed through
+    // the ConditionalOp parameter and then used as the MCDC base expression.
+    if (ConditionalOp)
+      MCDCBaseExpr = ConditionalOp;
+
+    maybeUpdateMCDCCondBitmap(MCDCBaseExpr, CondV);
+  }
+
   llvm::MDNode *Weights = nullptr;
   llvm::MDNode *Unpredictable = nullptr;
 
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 0ea58f18fa93..3ea4ba89711e 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -290,6 +290,9 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// nest would extend.
   SmallVector OMPLoopNestStack;
 
+  /// Stack to track the Logical Operator recursion nest for MC/DC.
+  SmallVector MCDCLogOpStack;
+
   /// Number of nested loop to be consumed by the last surrounding
   /// loop-associated directive.
   int ExpectedOMPLoopDepth = 0;
@@ -1524,6 +1527,9 @@ class CodeGenFunction : public CodeGenTypeCache {
 
   CodeGenPGO PGO;
 
+  /// Bitmap used by MC/DC to track condition outcomes of a boolean expression.
+  Address MCDCCondBitmapAddr = Address::invalid();
+
   /// Calculate branch weights appropriate for PGO data
   llvm::MDNode *createProfileWeights(uint64_t TrueCount,
                                      uint64_t FalseCount) const;
@@ -1542,6 +1548,52 @@ class CodeGenFunction : public CodeGenTypeCache {
     PGO.setCurrentStmt(S);
   }
 
+  bool isMCDCCoverageEnabled() const {
+    return (CGM.getCodeGenOpts().hasProfileClangInstr() &&
+            CGM.getCodeGenOpts().MCDCCoverage &&
+            !CurFn->hasFnAttribute(llvm::Attribute::NoProfile));
+  }
+
+  /// Allocate a temp value on the stack that MCDC can use to track condition
+  /// results.
+  void maybeCreateMCDCCondBitmap() {
+    if (isMCDCCoverageEnabled()) {
+      PGO.emitMCDCParameters(Builder);
+      MCDCCondBitmapAddr =
+          CreateIRTemp(getContext().UnsignedIntTy, "mcdc.addr");
+    }
+  }
+
+  bool isBinaryLogicalOp(const Expr *E) const {
+    const BinaryOperator *BOp = dyn_cast(E->IgnoreParens());
+    return (BOp && BOp->isLogicalOp());
+  }
+
+  /// Zero-init the MCDC temp value.
+  void maybeResetMCDCCondBitmap(const Expr *E) {
+    if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {
+      PGO.emitMCDCCondBitmapReset(Builder, E, MCDCCondBitmapAddr);
+      PGO.setCurrentStmt(E);
+    }
+  }
+
+  /// Increment the profiler's counter for the given expression by \p StepV.
+  /// If \p StepV is null, the default increment is 1.
+  void maybeUpdateMCDCTestVectorBitmap(const Expr *E) {
+    if (isMCDCCoverageEnabled() && isBinaryLogicalOp(E)) {
+      PGO.emitMCDCTestVectorBitmapUpdate(Builder, E, MCDCCondBitmapAddr);
+      PGO.setCurrentStmt(E);
+    }
+  }
+
+  /// Update the MCDC temp value with the condition's evaluated result.
+  void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val) {
+    if (isMCDCCoverageEnabled()) {
+      PGO.emitMCDCCondBitmapUpdate(Builder, E, MCDCCondBitmapAddr, Val);
+      PGO.setCurrentStmt(E);
+    }
+  }
+
   /// Get the profiler's count for the given statement.
   uint64_t getProfileCount(const Stmt *S) {
     return PGO.getStmtCount(S).value_or(0);
@@ -3024,6 +3076,25 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// this expression is used as an lvalue, for instance in "&Arr[Idx]".
   void EmitBoundsCheck(const Expr *E, const Expr *Base, llvm::Value *Index,
                        QualType IndexType, bool Accessed);
+  void EmitBoundsCheckImpl(const Expr *E, llvm::Value *Bound,
+                           llvm::Value *Index, QualType IndexType,
+                           QualType IndexedType, bool Accessed);
+
+  // Find a struct's flexible array member. It may be embedded inside multiple
+  // sub-structs, but must still be the last field.
+  const FieldDecl *FindFlexibleArrayMemberField(ASTContext &Ctx,
+                                                const RecordDecl *RD,
+                                                StringRef Name,
+                                                uint64_t &Offset);
+
+  /// Find the FieldDecl specified in a FAM's "counted_by" attribute. Returns
+  /// \p nullptr if either the attribute or the field doesn't exist.
+  const FieldDecl *FindCountedByField(const FieldDecl *FD);
+
+  /// Build an expression accessing the "counted_by" field.
+  llvm::Value *EmitCountedByFieldExpr(const Expr *Base,
+                                      const FieldDecl *FAMDecl,
+                                      const FieldDecl *CountDecl);
 
   llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
                                        bool isInc, bool isPre);
@@ -4631,6 +4702,9 @@ class CodeGenFunction : public CodeGenTypeCache {
   bool ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &Result,
                                     bool AllowLabels = false);
 
+  /// Ignore parentheses and logical-NOT to track conditions consistently.
+  static const Expr *stripCond(const Expr *C);
+
   /// isInstrumentedCondition - Determine whether the given condition is an
   /// instrumentable condition (i.e. no "&&" or "||").
   static bool isInstrumentedCondition(const Expr *C);
@@ -4653,7 +4727,8 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// evaluate to true based on PGO data.
   void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,
                             llvm::BasicBlock *FalseBlock, uint64_t TrueCount,
-                            Stmt::Likelihood LH = Stmt::LH_None);
+                            Stmt::Likelihood LH = Stmt::LH_None,
+                            const Expr *ConditionalOp = nullptr);
 
   /// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is
   /// nonnull, if \p LHS is marked _Nonnull.
@@ -4822,6 +4897,9 @@ class CodeGenFunction : public CodeGenTypeCache {
                                      llvm::Value *EmittedE,
                                      bool IsDynamic);
 
+  llvm::Value *emitFlexibleArrayMemberSize(const Expr *E, unsigned Type,
+                                           llvm::IntegerType *ResType);
+
   void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D,
                                        Address Loc);
 
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 40a457e5c217..8421dd7819c5 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -232,7 +232,8 @@ createTargetCodeGenInfo(CodeGenModule &CGM) {
       ABIFLen = 32;
     else if (ABIStr.ends_with("d"))
       ABIFLen = 64;
-    return createRISCVTargetCodeGenInfo(CGM, XLen, ABIFLen);
+    bool EABI = ABIStr.ends_with("e");
+    return createRISCVTargetCodeGenInfo(CGM, XLen, ABIFLen, EABI);
   }
 
   case llvm::Triple::systemz: {
@@ -727,43 +728,70 @@ void InstrProfStats::reportDiagnostics(DiagnosticsEngine &Diags,
   }
 }
 
+static std::optional
+getLLVMVisibility(clang::LangOptions::VisibilityFromDLLStorageClassKinds K) {
+  // Map to LLVM visibility.
+  switch (K) {
+  case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Keep:
+    return std::nullopt;
+  case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Default:
+    return llvm::GlobalValue::DefaultVisibility;
+  case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Hidden:
+    return llvm::GlobalValue::HiddenVisibility;
+  case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Protected:
+    return llvm::GlobalValue::ProtectedVisibility;
+  }
+  llvm_unreachable("unknown option value!");
+}
+
+void setLLVMVisibility(llvm::GlobalValue &GV,
+                       std::optional V) {
+  if (!V)
+    return;
+
+  // Reset DSO locality before setting the visibility. This removes
+  // any effects that visibility options and annotations may have
+  // had on the DSO locality. Setting the visibility will implicitly set
+  // appropriate globals to DSO Local; however, this will be pessimistic
+  // w.r.t. to the normal compiler IRGen.
+  GV.setDSOLocal(false);
+  GV.setVisibility(*V);
+}
+
 static void setVisibilityFromDLLStorageClass(const clang::LangOptions &LO,
                                              llvm::Module &M) {
   if (!LO.VisibilityFromDLLStorageClass)
     return;
 
-  llvm::GlobalValue::VisibilityTypes DLLExportVisibility =
-      CodeGenModule::GetLLVMVisibility(LO.getDLLExportVisibility());
-  llvm::GlobalValue::VisibilityTypes NoDLLStorageClassVisibility =
-      CodeGenModule::GetLLVMVisibility(LO.getNoDLLStorageClassVisibility());
-  llvm::GlobalValue::VisibilityTypes ExternDeclDLLImportVisibility =
-      CodeGenModule::GetLLVMVisibility(LO.getExternDeclDLLImportVisibility());
-  llvm::GlobalValue::VisibilityTypes ExternDeclNoDLLStorageClassVisibility =
-      CodeGenModule::GetLLVMVisibility(
-          LO.getExternDeclNoDLLStorageClassVisibility());
+  std::optional DLLExportVisibility =
+      getLLVMVisibility(LO.getDLLExportVisibility());
+
+  std::optional
+      NoDLLStorageClassVisibility =
+          getLLVMVisibility(LO.getNoDLLStorageClassVisibility());
+
+  std::optional
+      ExternDeclDLLImportVisibility =
+          getLLVMVisibility(LO.getExternDeclDLLImportVisibility());
+
+  std::optional
+      ExternDeclNoDLLStorageClassVisibility =
+          getLLVMVisibility(LO.getExternDeclNoDLLStorageClassVisibility());
 
   for (llvm::GlobalValue &GV : M.global_values()) {
     if (GV.hasAppendingLinkage() || GV.hasLocalLinkage())
       continue;
 
-    // Reset DSO locality before setting the visibility. This removes
-    // any effects that visibility options and annotations may have
-    // had on the DSO locality. Setting the visibility will implicitly set
-    // appropriate globals to DSO Local; however, this will be pessimistic
-    // w.r.t. to the normal compiler IRGen.
-    GV.setDSOLocal(false);
-
-    if (GV.isDeclarationForLinker()) {
-      GV.setVisibility(GV.getDLLStorageClass() ==
-                               llvm::GlobalValue::DLLImportStorageClass
-                           ? ExternDeclDLLImportVisibility
-                           : ExternDeclNoDLLStorageClassVisibility);
-    } else {
-      GV.setVisibility(GV.getDLLStorageClass() ==
-                               llvm::GlobalValue::DLLExportStorageClass
-                           ? DLLExportVisibility
-                           : NoDLLStorageClassVisibility);
-    }
+    if (GV.isDeclarationForLinker())
+      setLLVMVisibility(GV, GV.getDLLStorageClass() ==
+                                    llvm::GlobalValue::DLLImportStorageClass
+                                ? ExternDeclDLLImportVisibility
+                                : ExternDeclNoDLLStorageClassVisibility);
+    else
+      setLLVMVisibility(GV, GV.getDLLStorageClass() ==
+                                    llvm::GlobalValue::DLLExportStorageClass
+                                ? DLLExportVisibility
+                                : NoDLLStorageClassVisibility);
 
     GV.setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
   }
@@ -1115,6 +1143,8 @@ void CodeGenModule::Release() {
     if (LangOpts.BranchProtectionPAuthLR)
       getModule().addModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr",
                                 1);
+    if (LangOpts.GuardedControlStack)
+      getModule().addModuleFlag(llvm::Module::Min, "guarded-control-stack", 1);
     if (LangOpts.hasSignReturnAddress())
       getModule().addModuleFlag(llvm::Module::Min, "sign-return-address", 1);
     if (LangOpts.isSignReturnAddressScopeAll())
@@ -1205,7 +1235,7 @@ void CodeGenModule::Release() {
       llvm::CodeModel::Model codeModel = static_cast(CM);
       getModule().setCodeModel(codeModel);
 
-      if (CM == llvm::CodeModel::Medium &&
+      if ((CM == llvm::CodeModel::Medium || CM == llvm::CodeModel::Large) &&
           Context.getTargetInfo().getTriple().getArch() ==
               llvm::Triple::x86_64) {
         getModule().setLargeDataThreshold(getCodeGenOpts().LargeDataThreshold);
@@ -1692,8 +1722,10 @@ static void AppendCPUSpecificCPUDispatchMangling(const CodeGenModule &CGM,
 static void AppendTargetVersionMangling(const CodeGenModule &CGM,
                                         const TargetVersionAttr *Attr,
                                         raw_ostream &Out) {
-  if (Attr->isDefaultVersion())
+  if (Attr->isDefaultVersion()) {
+    Out << ".default";
     return;
+  }
   Out << "._";
   const TargetInfo &TI = CGM.getTarget();
   llvm::SmallVector Feats;
@@ -1756,8 +1788,10 @@ static void AppendTargetClonesMangling(const CodeGenModule &CGM,
   const TargetInfo &TI = CGM.getTarget();
   if (TI.getTriple().isAArch64()) {
     StringRef FeatureStr = Attr->getFeatureStr(VersionIndex);
-    if (FeatureStr == "default")
+    if (FeatureStr == "default") {
+      Out << ".default";
       return;
+    }
     Out << "._";
     SmallVector Features;
     FeatureStr.split(Features, "+");
@@ -2384,8 +2418,12 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
   if (D->hasAttr())
     B.addAttribute("aarch64_pstate_sm_body");
 
-  if (D->hasAttr())
-    B.addAttribute("aarch64_pstate_za_new");
+  if (auto *Attr = D->getAttr()) {
+    if (Attr->isNewZA())
+      B.addAttribute("aarch64_pstate_za_new");
+    if (Attr->isNewZT0())
+      B.addAttribute("aarch64_new_zt0");
+  }
 
   // Track whether we need to add the optnone LLVM attribute,
   // starting with the default for this optimization level.
@@ -4003,6 +4041,8 @@ void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD,
         EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr);
     // Ensure that the resolver function is also emitted.
     GetOrCreateMultiVersionResolver(GD);
+  } else if (FD->hasAttr()) {
+    GetOrCreateMultiVersionResolver(GD);
   } else
     EmitGlobalFunctionDefinition(GD, GV);
 }
@@ -4184,14 +4224,7 @@ void CodeGenModule::emitMultiVersionFunctions() {
     llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD);
     if (auto *IFunc = dyn_cast(ResolverConstant)) {
       ResolverConstant = IFunc->getResolver();
-      // In Aarch64, default versions of multiversioned functions are mangled to
-      // their 'normal' assembly name. This deviates from other targets which
-      // append a '.default' string. As a result we need to continue appending
-      // .ifunc in Aarch64.
-      // FIXME: Should Aarch64 mangling for 'default' multiversion function and
-      // in turn ifunc function match that of other targets?
-      if (FD->isTargetClonesMultiVersion() &&
-          !getTarget().getTriple().isAArch64()) {
+      if (FD->isTargetClonesMultiVersion()) {
         const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
         llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI);
         std::string MangledName = getMangledNameImpl(
@@ -4372,14 +4405,7 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) {
   // a separate resolver).
   std::string ResolverName = MangledName;
   if (getTarget().supportsIFunc()) {
-    // In Aarch64, default versions of multiversioned functions are mangled to
-    // their 'normal' assembly name. This deviates from other targets which
-    // append a '.default' string. As a result we need to continue appending
-    // .ifunc in Aarch64.
-    // FIXME: Should Aarch64 mangling for 'default' multiversion function and
-    // in turn ifunc function match that of other targets?
-    if (!FD->isTargetClonesMultiVersion() ||
-        getTarget().getTriple().isAArch64())
+    if (!FD->isTargetClonesMultiVersion())
       ResolverName += ".ifunc";
   } else if (FD->isTargetMultiVersion()) {
     ResolverName += ".resolver";
@@ -4875,6 +4901,10 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty,
         isExternallyVisible(D->getLinkageAndVisibility().getLinkage()))
       GV->setSection(".cp.rodata");
 
+    // Handle code model attribute
+    if (const auto *CMA = D->getAttr())
+      GV->setCodeModel(CMA->getModel());
+
     // Check if we a have a const declaration with an initializer, we may be
     // able to emit it as available_externally to expose it's value to the
     // optimizer.
diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp
index 81bf8ea696b1..5d7c38477457 100644
--- a/clang/lib/CodeGen/CodeGenPGO.cpp
+++ b/clang/lib/CodeGen/CodeGenPGO.cpp
@@ -28,6 +28,8 @@ static llvm::cl::opt
                          llvm::cl::desc("Enable value profiling"),
                          llvm::cl::Hidden, llvm::cl::init(false));
 
+extern llvm::cl::opt SystemHeadersCoverage;
+
 using namespace clang;
 using namespace CodeGen;
 
@@ -161,13 +163,24 @@ struct MapRegionCounters : public RecursiveASTVisitor {
   PGOHash Hash;
   /// The map of statements to counters.
   llvm::DenseMap &CounterMap;
+  /// The next bitmap byte index to assign.
+  unsigned NextMCDCBitmapIdx;
+  /// The map of statements to MC/DC bitmap coverage objects.
+  llvm::DenseMap &MCDCBitmapMap;
+  /// Maximum number of supported MC/DC conditions in a boolean expression.
+  unsigned MCDCMaxCond;
   /// The profile version.
   uint64_t ProfileVersion;
+  /// Diagnostics Engine used to report warnings.
+  DiagnosticsEngine &Diag;
 
   MapRegionCounters(PGOHashVersion HashVersion, uint64_t ProfileVersion,
-                    llvm::DenseMap &CounterMap)
+                    llvm::DenseMap &CounterMap,
+                    llvm::DenseMap &MCDCBitmapMap,
+                    unsigned MCDCMaxCond, DiagnosticsEngine &Diag)
       : NextCounter(0), Hash(HashVersion), CounterMap(CounterMap),
-        ProfileVersion(ProfileVersion) {}
+        NextMCDCBitmapIdx(0), MCDCBitmapMap(MCDCBitmapMap),
+        MCDCMaxCond(MCDCMaxCond), ProfileVersion(ProfileVersion), Diag(Diag) {}
 
   // Blocks and lambdas are handled as separate functions, so we need not
   // traverse them in the parent context.
@@ -207,15 +220,126 @@ struct MapRegionCounters : public RecursiveASTVisitor {
     return Type;
   }
 
+  /// The following stacks are used with dataTraverseStmtPre() and
+  /// dataTraverseStmtPost() to track the depth of nested logical operators in a
+  /// boolean expression in a function.  The ultimate purpose is to keep track
+  /// of the number of leaf-level conditions in the boolean expression so that a
+  /// profile bitmap can be allocated based on that number.
+  ///
+  /// The stacks are also used to find error cases and notify the user.  A
+  /// standard logical operator nest for a boolean expression could be in a form
+  /// similar to this: "x = a && b && c && (d || f)"
+  unsigned NumCond = 0;
+  bool SplitNestedLogicalOp = false;
+  SmallVector NonLogOpStack;
+  SmallVector LogOpStack;
+
+  // Hook: dataTraverseStmtPre() is invoked prior to visiting an AST Stmt node.
+  bool dataTraverseStmtPre(Stmt *S) {
+    /// If MC/DC is not enabled, MCDCMaxCond will be set to 0. Do nothing.
+    if (MCDCMaxCond == 0)
+      return true;
+
+    /// At the top of the logical operator nest, reset the number of conditions.
+    if (LogOpStack.empty())
+      NumCond = 0;
+
+    if (const Expr *E = dyn_cast(S)) {
+      const BinaryOperator *BinOp = dyn_cast(E->IgnoreParens());
+      if (BinOp && BinOp->isLogicalOp()) {
+        /// Check for "split-nested" logical operators. This happens when a new
+        /// boolean expression logical-op nest is encountered within an existing
+        /// boolean expression, separated by a non-logical operator.  For
+        /// example, in "x = (a && b && c && foo(d && f))", the "d && f" case
+        /// starts a new boolean expression that is separated from the other
+        /// conditions by the operator foo(). Split-nested cases are not
+        /// supported by MC/DC.
+        SplitNestedLogicalOp = SplitNestedLogicalOp || !NonLogOpStack.empty();
+
+        LogOpStack.push_back(BinOp);
+        return true;
+      }
+    }
+
+    /// Keep track of non-logical operators. These are OK as long as we don't
+    /// encounter a new logical operator after seeing one.
+    if (!LogOpStack.empty())
+      NonLogOpStack.push_back(S);
+
+    return true;
+  }
+
+  // Hook: dataTraverseStmtPost() is invoked by the AST visitor after visiting
+  // an AST Stmt node.  MC/DC will use it to to signal when the top of a
+  // logical operation (boolean expression) nest is encountered.
+  bool dataTraverseStmtPost(Stmt *S) {
+    /// If MC/DC is not enabled, MCDCMaxCond will be set to 0. Do nothing.
+    if (MCDCMaxCond == 0)
+      return true;
+
+    if (const Expr *E = dyn_cast(S)) {
+      const BinaryOperator *BinOp = dyn_cast(E->IgnoreParens());
+      if (BinOp && BinOp->isLogicalOp()) {
+        assert(LogOpStack.back() == BinOp);
+        LogOpStack.pop_back();
+
+        /// At the top of logical operator nest:
+        if (LogOpStack.empty()) {
+          /// Was the "split-nested" logical operator case encountered?
+          if (SplitNestedLogicalOp) {
+            unsigned DiagID = Diag.getCustomDiagID(
+                DiagnosticsEngine::Warning,
+                "unsupported MC/DC boolean expression; "
+                "contains an operation with a nested boolean expression. "
+                "Expression will not be covered");
+            Diag.Report(S->getBeginLoc(), DiagID);
+            return false;
+          }
+
+          /// Was the maximum number of conditions encountered?
+          if (NumCond > MCDCMaxCond) {
+            unsigned DiagID = Diag.getCustomDiagID(
+                DiagnosticsEngine::Warning,
+                "unsupported MC/DC boolean expression; "
+                "number of conditions (%0) exceeds max (%1). "
+                "Expression will not be covered");
+            Diag.Report(S->getBeginLoc(), DiagID) << NumCond << MCDCMaxCond;
+            return false;
+          }
+
+          // Otherwise, allocate the number of bytes required for the bitmap
+          // based on the number of conditions. Must be at least 1-byte long.
+          MCDCBitmapMap[BinOp] = NextMCDCBitmapIdx;
+          unsigned SizeInBits = std::max(1L << NumCond, CHAR_BIT);
+          NextMCDCBitmapIdx += SizeInBits / CHAR_BIT;
+        }
+        return true;
+      }
+    }
+
+    if (!LogOpStack.empty())
+      NonLogOpStack.pop_back();
+
+    return true;
+  }
+
   /// The RHS of all logical operators gets a fresh counter in order to count
   /// how many times the RHS evaluates to true or false, depending on the
   /// semantics of the operator. This is only valid for ">= v7" of the profile
-  /// version so that we facilitate backward compatibility.
+  /// version so that we facilitate backward compatibility. In addition, in
+  /// order to use MC/DC, count the number of total LHS and RHS conditions.
   bool VisitBinaryOperator(BinaryOperator *S) {
-    if (ProfileVersion >= llvm::IndexedInstrProf::Version7)
-      if (S->isLogicalOp() &&
-          CodeGenFunction::isInstrumentedCondition(S->getRHS()))
-        CounterMap[S->getRHS()] = NextCounter++;
+    if (S->isLogicalOp()) {
+      if (CodeGenFunction::isInstrumentedCondition(S->getLHS()))
+        NumCond++;
+
+      if (CodeGenFunction::isInstrumentedCondition(S->getRHS())) {
+        if (ProfileVersion >= llvm::IndexedInstrProf::Version7)
+          CounterMap[S->getRHS()] = NextCounter++;
+
+        NumCond++;
+      }
+    }
     return Base::VisitBinaryOperator(S);
   }
 
@@ -851,8 +975,22 @@ void CodeGenPGO::mapRegionCounters(const Decl *D) {
     ProfileVersion = PGOReader->getVersion();
   }
 
+  // If MC/DC is enabled, set the MaxConditions to a preset value. Otherwise,
+  // set it to zero. This value impacts the number of conditions accepted in a
+  // given boolean expression, which impacts the size of the bitmap used to
+  // track test vector execution for that boolean expression.  Because the
+  // bitmap scales exponentially (2^n) based on the number of conditions seen,
+  // the maximum value is hard-coded at 6 conditions, which is more than enough
+  // for most embedded applications. Setting a maximum value prevents the
+  // bitmap footprint from growing too large without the user's knowledge. In
+  // the future, this value could be adjusted with a command-line option.
+  unsigned MCDCMaxConditions = (CGM.getCodeGenOpts().MCDCCoverage) ? 6 : 0;
+
   RegionCounterMap.reset(new llvm::DenseMap);
-  MapRegionCounters Walker(HashVersion, ProfileVersion, *RegionCounterMap);
+  RegionMCDCBitmapMap.reset(new llvm::DenseMap);
+  MapRegionCounters Walker(HashVersion, ProfileVersion, *RegionCounterMap,
+                           *RegionMCDCBitmapMap, MCDCMaxConditions,
+                           CGM.getDiags());
   if (const FunctionDecl *FD = dyn_cast_or_null(D))
     Walker.TraverseDecl(const_cast(FD));
   else if (const ObjCMethodDecl *MD = dyn_cast_or_null(D))
@@ -863,6 +1001,7 @@ void CodeGenPGO::mapRegionCounters(const Decl *D) {
     Walker.TraverseDecl(const_cast(CD));
   assert(Walker.NextCounter > 0 && "no entry counter mapped for decl");
   NumRegionCounters = Walker.NextCounter;
+  MCDCBitmapBytes = Walker.NextMCDCBitmapIdx;
   FunctionHash = Walker.Hash.finalize();
 }
 
@@ -885,7 +1024,7 @@ bool CodeGenPGO::skipRegionMappingForDecl(const Decl *D) {
   // Don't map the functions in system headers.
   const auto &SM = CGM.getContext().getSourceManager();
   auto Loc = D->getBody()->getBeginLoc();
-  return SM.isInSystemHeader(Loc);
+  return !SystemHeadersCoverage && SM.isInSystemHeader(Loc);
 }
 
 void CodeGenPGO::emitCounterRegionMapping(const Decl *D) {
@@ -894,9 +1033,11 @@ void CodeGenPGO::emitCounterRegionMapping(const Decl *D) {
 
   std::string CoverageMapping;
   llvm::raw_string_ostream OS(CoverageMapping);
-  CoverageMappingGen MappingGen(*CGM.getCoverageMapping(),
-                                CGM.getContext().getSourceManager(),
-                                CGM.getLangOpts(), RegionCounterMap.get());
+  RegionCondIDMap.reset(new llvm::DenseMap);
+  CoverageMappingGen MappingGen(
+      *CGM.getCoverageMapping(), CGM.getContext().getSourceManager(),
+      CGM.getLangOpts(), RegionCounterMap.get(), RegionMCDCBitmapMap.get(),
+      RegionCondIDMap.get());
   MappingGen.emitCounterMapping(D, OS);
   OS.flush();
 
@@ -972,6 +1113,108 @@ void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S,
         ArrayRef(Args));
 }
 
+bool CodeGenPGO::canEmitMCDCCoverage(const CGBuilderTy &Builder) {
+  return (CGM.getCodeGenOpts().hasProfileClangInstr() &&
+          CGM.getCodeGenOpts().MCDCCoverage && Builder.GetInsertBlock());
+}
+
+void CodeGenPGO::emitMCDCParameters(CGBuilderTy &Builder) {
+  if (!canEmitMCDCCoverage(Builder) || !RegionMCDCBitmapMap)
+    return;
+
+  auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.getLLVMContext());
+
+  // Emit intrinsic representing MCDC bitmap parameters at function entry.
+  // This is used by the instrumentation pass, but it isn't actually lowered to
+  // anything.
+  llvm::Value *Args[3] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
+                          Builder.getInt64(FunctionHash),
+                          Builder.getInt32(MCDCBitmapBytes)};
+  Builder.CreateCall(
+      CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_parameters), Args);
+}
+
+void CodeGenPGO::emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder,
+                                                const Expr *S,
+                                                Address MCDCCondBitmapAddr) {
+  if (!canEmitMCDCCoverage(Builder) || !RegionMCDCBitmapMap)
+    return;
+
+  S = S->IgnoreParens();
+
+  auto ExprMCDCBitmapMapIterator = RegionMCDCBitmapMap->find(S);
+  if (ExprMCDCBitmapMapIterator == RegionMCDCBitmapMap->end())
+    return;
+
+  // Extract the ID of the global bitmap associated with this expression.
+  unsigned MCDCTestVectorBitmapID = ExprMCDCBitmapMapIterator->second;
+  auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.getLLVMContext());
+
+  // Emit intrinsic responsible for updating the global bitmap corresponding to
+  // a boolean expression. The index being set is based on the value loaded
+  // from a pointer to a dedicated temporary value on the stack that is itself
+  // updated via emitMCDCCondBitmapReset() and emitMCDCCondBitmapUpdate(). The
+  // index represents an executed test vector.
+  llvm::Value *Args[5] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
+                          Builder.getInt64(FunctionHash),
+                          Builder.getInt32(MCDCBitmapBytes),
+                          Builder.getInt32(MCDCTestVectorBitmapID),
+                          MCDCCondBitmapAddr.getPointer()};
+  Builder.CreateCall(
+      CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_tvbitmap_update), Args);
+}
+
+void CodeGenPGO::emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S,
+                                         Address MCDCCondBitmapAddr) {
+  if (!canEmitMCDCCoverage(Builder) || !RegionMCDCBitmapMap)
+    return;
+
+  S = S->IgnoreParens();
+
+  if (RegionMCDCBitmapMap->find(S) == RegionMCDCBitmapMap->end())
+    return;
+
+  // Emit intrinsic that resets a dedicated temporary value on the stack to 0.
+  Builder.CreateStore(Builder.getInt32(0), MCDCCondBitmapAddr);
+}
+
+void CodeGenPGO::emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S,
+                                          Address MCDCCondBitmapAddr,
+                                          llvm::Value *Val) {
+  if (!canEmitMCDCCoverage(Builder) || !RegionCondIDMap)
+    return;
+
+  // Even though, for simplicity, parentheses and unary logical-NOT operators
+  // are considered part of their underlying condition for both MC/DC and
+  // branch coverage, the condition IDs themselves are assigned and tracked
+  // using the underlying condition itself.  This is done solely for
+  // consistency since parentheses and logical-NOTs are ignored when checking
+  // whether the condition is actually an instrumentable condition. This can
+  // also make debugging a bit easier.
+  S = CodeGenFunction::stripCond(S);
+
+  auto ExprMCDCConditionIDMapIterator = RegionCondIDMap->find(S);
+  if (ExprMCDCConditionIDMapIterator == RegionCondIDMap->end())
+    return;
+
+  // Extract the ID of the condition we are setting in the bitmap.
+  unsigned CondID = ExprMCDCConditionIDMapIterator->second;
+  assert(CondID > 0 && "Condition has no ID!");
+
+  auto *I8PtrTy = llvm::PointerType::getUnqual(CGM.getLLVMContext());
+
+  // Emit intrinsic that updates a dedicated temporary value on the stack after
+  // a condition is evaluated. After the set of conditions has been updated,
+  // the resulting value is used to update the boolean expression's bitmap.
+  llvm::Value *Args[5] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
+                          Builder.getInt64(FunctionHash),
+                          Builder.getInt32(CondID - 1),
+                          MCDCCondBitmapAddr.getPointer(), Val};
+  Builder.CreateCall(
+      CGM.getIntrinsic(llvm::Intrinsic::instrprof_mcdc_condbitmap_update),
+      Args);
+}
+
 void CodeGenPGO::setValueProfilingFlag(llvm::Module &M) {
   if (CGM.getCodeGenOpts().hasProfileClangInstr())
     M.addModuleFlag(llvm::Module::Warning, "EnableValueProfiling",
diff --git a/clang/lib/CodeGen/CodeGenPGO.h b/clang/lib/CodeGen/CodeGenPGO.h
index 392ec5a144fe..6596b6c35277 100644
--- a/clang/lib/CodeGen/CodeGenPGO.h
+++ b/clang/lib/CodeGen/CodeGenPGO.h
@@ -33,8 +33,11 @@ class CodeGenPGO {
 
   std::array  NumValueSites;
   unsigned NumRegionCounters;
+  unsigned MCDCBitmapBytes;
   uint64_t FunctionHash;
   std::unique_ptr> RegionCounterMap;
+  std::unique_ptr> RegionMCDCBitmapMap;
+  std::unique_ptr> RegionCondIDMap;
   std::unique_ptr> StmtCountMap;
   std::unique_ptr ProfRecord;
   std::vector RegionCounts;
@@ -43,7 +46,8 @@ class CodeGenPGO {
 public:
   CodeGenPGO(CodeGenModule &CGModule)
       : CGM(CGModule), FuncNameVar(nullptr), NumValueSites({{0}}),
-        NumRegionCounters(0), FunctionHash(0), CurrentRegionCount(0) {}
+        NumRegionCounters(0), MCDCBitmapBytes(0), FunctionHash(0),
+        CurrentRegionCount(0) {}
 
   /// Whether or not we have PGO region data for the current function. This is
   /// false both when we have no data at all and when our data has been
@@ -103,10 +107,18 @@ class CodeGenPGO {
                         bool IsInMainFile);
   bool skipRegionMappingForDecl(const Decl *D);
   void emitCounterRegionMapping(const Decl *D);
+  bool canEmitMCDCCoverage(const CGBuilderTy &Builder);
 
 public:
   void emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S,
                             llvm::Value *StepV);
+  void emitMCDCTestVectorBitmapUpdate(CGBuilderTy &Builder, const Expr *S,
+                                      Address MCDCCondBitmapAddr);
+  void emitMCDCParameters(CGBuilderTy &Builder);
+  void emitMCDCCondBitmapReset(CGBuilderTy &Builder, const Expr *S,
+                               Address MCDCCondBitmapAddr);
+  void emitMCDCCondBitmapUpdate(CGBuilderTy &Builder, const Expr *S,
+                                Address MCDCCondBitmapAddr, llvm::Value *Val);
 
   /// Return the region count for the counter at the given index.
   uint64_t getRegionCount(const Stmt *S) {
diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp
index 56411e2240e5..5eca00f22bb8 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.cpp
+++ b/clang/lib/CodeGen/CoverageMappingGen.cpp
@@ -37,7 +37,7 @@ static llvm::cl::opt EmptyLineCommentCoverage(
                    "disable it on test)"),
     llvm::cl::init(true), llvm::cl::Hidden);
 
-static llvm::cl::opt SystemHeadersCoverage(
+llvm::cl::opt SystemHeadersCoverage(
     "system-headers-coverage",
     llvm::cl::desc("Enable collecting coverage from system headers"),
     llvm::cl::init(false), llvm::cl::Hidden);
@@ -95,6 +95,8 @@ void CoverageSourceInfo::updateNextTokLoc(SourceLocation Loc) {
 }
 
 namespace {
+using MCDCConditionID = CounterMappingRegion::MCDCConditionID;
+using MCDCParameters = CounterMappingRegion::MCDCParameters;
 
 /// A region of source code that can be mapped to a counter.
 class SourceMappingRegion {
@@ -104,6 +106,9 @@ class SourceMappingRegion {
   /// Secondary Counter used for Branch Regions for "False" branches.
   std::optional FalseCount;
 
+  /// Parameters used for Modified Condition/Decision Coverage
+  MCDCParameters MCDCParams;
+
   /// The region's starting location.
   std::optional LocStart;
 
@@ -114,19 +119,31 @@ class SourceMappingRegion {
   /// as the line execution count if there are no other regions on the line.
   bool GapRegion;
 
+  /// Whetever this region is skipped ('if constexpr' or 'if consteval' untaken
+  /// branch, or anything skipped but not empty line / comments)
+  bool SkippedRegion;
+
 public:
   SourceMappingRegion(Counter Count, std::optional LocStart,
                       std::optional LocEnd,
                       bool GapRegion = false)
-      : Count(Count), LocStart(LocStart), LocEnd(LocEnd), GapRegion(GapRegion) {
-  }
+      : Count(Count), LocStart(LocStart), LocEnd(LocEnd), GapRegion(GapRegion),
+        SkippedRegion(false) {}
 
   SourceMappingRegion(Counter Count, std::optional FalseCount,
+                      MCDCParameters MCDCParams,
                       std::optional LocStart,
                       std::optional LocEnd,
                       bool GapRegion = false)
-      : Count(Count), FalseCount(FalseCount), LocStart(LocStart),
-        LocEnd(LocEnd), GapRegion(GapRegion) {}
+      : Count(Count), FalseCount(FalseCount), MCDCParams(MCDCParams),
+        LocStart(LocStart), LocEnd(LocEnd), GapRegion(GapRegion),
+        SkippedRegion(false) {}
+
+  SourceMappingRegion(MCDCParameters MCDCParams,
+                      std::optional LocStart,
+                      std::optional LocEnd)
+      : MCDCParams(MCDCParams), LocStart(LocStart), LocEnd(LocEnd),
+        GapRegion(false), SkippedRegion(false) {}
 
   const Counter &getCounter() const { return Count; }
 
@@ -162,7 +179,15 @@ class SourceMappingRegion {
 
   void setGap(bool Gap) { GapRegion = Gap; }
 
+  bool isSkipped() const { return SkippedRegion; }
+
+  void setSkipped(bool Skipped) { SkippedRegion = Skipped; }
+
   bool isBranch() const { return FalseCount.has_value(); }
+
+  bool isMCDCDecision() const { return MCDCParams.NumConditions != 0; }
+
+  const MCDCParameters &getMCDCParams() const { return MCDCParams; }
 };
 
 /// Spelling locations for the start and end of a source region.
@@ -452,10 +477,19 @@ class CoverageMappingBuilder {
         MappingRegions.push_back(CounterMappingRegion::makeGapRegion(
             Region.getCounter(), *CovFileID, SR.LineStart, SR.ColumnStart,
             SR.LineEnd, SR.ColumnEnd));
+      } else if (Region.isSkipped()) {
+        MappingRegions.push_back(CounterMappingRegion::makeSkipped(
+            *CovFileID, SR.LineStart, SR.ColumnStart, SR.LineEnd,
+            SR.ColumnEnd));
       } else if (Region.isBranch()) {
         MappingRegions.push_back(CounterMappingRegion::makeBranchRegion(
-            Region.getCounter(), Region.getFalseCounter(), *CovFileID,
-            SR.LineStart, SR.ColumnStart, SR.LineEnd, SR.ColumnEnd));
+            Region.getCounter(), Region.getFalseCounter(),
+            Region.getMCDCParams(), *CovFileID, SR.LineStart, SR.ColumnStart,
+            SR.LineEnd, SR.ColumnEnd));
+      } else if (Region.isMCDCDecision()) {
+        MappingRegions.push_back(CounterMappingRegion::makeDecisionRegion(
+            Region.getMCDCParams(), *CovFileID, SR.LineStart, SR.ColumnStart,
+            SR.LineEnd, SR.ColumnEnd));
       } else {
         MappingRegions.push_back(CounterMappingRegion::makeRegion(
             Region.getCounter(), *CovFileID, SR.LineStart, SR.ColumnStart,
@@ -542,6 +576,232 @@ struct EmptyCoverageMappingBuilder : public CoverageMappingBuilder {
   }
 };
 
+/// A wrapper object for maintaining stacks to track the resursive AST visitor
+/// walks for the purpose of assigning IDs to leaf-level conditions measured by
+/// MC/DC. The object is created with a reference to the MCDCBitmapMap that was
+/// created during the initial AST walk. The presence of a bitmap associated
+/// with a boolean expression (top-level logical operator nest) indicates that
+/// the boolean expression qualified for MC/DC.  The resulting condition IDs
+/// are preserved in a map reference that is also provided during object
+/// creation.
+struct MCDCCoverageBuilder {
+
+  struct DecisionIDPair {
+    MCDCConditionID TrueID = 0;
+    MCDCConditionID FalseID = 0;
+  };
+
+  /// The AST walk recursively visits nested logical-AND or logical-OR binary
+  /// operator nodes and then visits their LHS and RHS children nodes.  As this
+  /// happens, the algorithm will assign IDs to each operator's LHS and RHS side
+  /// as the walk moves deeper into the nest.  At each level of the recursive
+  /// nest, the LHS and RHS may actually correspond to larger subtrees (not
+  /// leaf-conditions). If this is the case, when that node is visited, the ID
+  /// assigned to the subtree is re-assigned to its LHS, and a new ID is given
+  /// to its RHS. At the end of the walk, all leaf-level conditions will have a
+  /// unique ID -- keep in mind that the final set of IDs may not be in
+  /// numerical order from left to right.
+  ///
+  /// Example: "x = (A && B) || (C && D) || (D && F)"
+  ///
+  ///      Visit Depth1:
+  ///              (A && B) || (C && D) || (D && F)
+  ///              ^-------LHS--------^    ^-RHS--^
+  ///                      ID=1              ID=2
+  ///
+  ///      Visit LHS-Depth2:
+  ///              (A && B) || (C && D)
+  ///              ^-LHS--^    ^-RHS--^
+  ///                ID=1        ID=3
+  ///
+  ///      Visit LHS-Depth3:
+  ///               (A && B)
+  ///               LHS   RHS
+  ///               ID=1  ID=4
+  ///
+  ///      Visit RHS-Depth3:
+  ///                         (C && D)
+  ///                         LHS   RHS
+  ///                         ID=3  ID=5
+  ///
+  ///      Visit RHS-Depth2:              (D && F)
+  ///                                     LHS   RHS
+  ///                                     ID=2  ID=6
+  ///
+  ///      Visit Depth1:
+  ///              (A && B)  || (C && D)  || (D && F)
+  ///              ID=1  ID=4   ID=3  ID=5   ID=2  ID=6
+  ///
+  /// A node ID of '0' always means MC/DC isn't being tracked.
+  ///
+  /// As the AST walk proceeds recursively, the algorithm will also use a stack
+  /// to track the IDs of logical-AND and logical-OR operations on the RHS so
+  /// that it can be determined which nodes are executed next, depending on how
+  /// a LHS or RHS of a logical-AND or logical-OR is evaluated.  This
+  /// information relies on the assigned IDs and are embedded within the
+  /// coverage region IDs of each branch region associated with a leaf-level
+  /// condition. This information helps the visualization tool reconstruct all
+  /// possible test vectors for the purposes of MC/DC analysis. If a "next" node
+  /// ID is '0', it means it's the end of the test vector. The following rules
+  /// are used:
+  ///
+  /// For logical-AND ("LHS && RHS"):
+  /// - If LHS is TRUE, execution goes to the RHS node.
+  /// - If LHS is FALSE, execution goes to the LHS node of the next logical-OR.
+  ///   If that does not exist, execution exits (ID == 0).
+  ///
+  /// - If RHS is TRUE, execution goes to LHS node of the next logical-AND.
+  ///   If that does not exist, execution exits (ID == 0).
+  /// - If RHS is FALSE, execution goes to the LHS node of the next logical-OR.
+  ///   If that does not exist, execution exits (ID == 0).
+  ///
+  /// For logical-OR ("LHS || RHS"):
+  /// - If LHS is TRUE, execution goes to the LHS node of the next logical-AND.
+  ///   If that does not exist, execution exits (ID == 0).
+  /// - If LHS is FALSE, execution goes to the RHS node.
+  ///
+  /// - If RHS is TRUE, execution goes to LHS node of the next logical-AND.
+  ///   If that does not exist, execution exits (ID == 0).
+  /// - If RHS is FALSE, execution goes to the LHS node of the next logical-OR.
+  ///   If that does not exist, execution exits (ID == 0).
+  ///
+  /// Finally, the condition IDs are also used when instrumenting the code to
+  /// indicate a unique offset into a temporary bitmap that represents the true
+  /// or false evaluation of that particular condition.
+  ///
+  /// NOTE regarding the use of CodeGenFunction::stripCond(). Even though, for
+  /// simplicity, parentheses and unary logical-NOT operators are considered
+  /// part of their underlying condition for both MC/DC and branch coverage, the
+  /// condition IDs themselves are assigned and tracked using the underlying
+  /// condition itself.  This is done solely for consistency since parentheses
+  /// and logical-NOTs are ignored when checking whether the condition is
+  /// actually an instrumentable condition. This can also make debugging a bit
+  /// easier.
+
+private:
+  CodeGenModule &CGM;
+
+  llvm::SmallVector DecisionStack;
+  llvm::DenseMap &CondIDs;
+  llvm::DenseMap &MCDCBitmapMap;
+  MCDCConditionID NextID = 1;
+  bool NotMapped = false;
+
+  /// Represent a sentinel value of [0,0] for the bottom of DecisionStack.
+  static constexpr DecisionIDPair DecisionStackSentinel{0, 0};
+
+  /// Is this a logical-AND operation?
+  bool isLAnd(const BinaryOperator *E) const {
+    return E->getOpcode() == BO_LAnd;
+  }
+
+public:
+  MCDCCoverageBuilder(CodeGenModule &CGM,
+                      llvm::DenseMap &CondIDMap,
+                      llvm::DenseMap &MCDCBitmapMap)
+      : CGM(CGM), DecisionStack(1, DecisionStackSentinel), CondIDs(CondIDMap),
+        MCDCBitmapMap(MCDCBitmapMap) {}
+
+  /// Return whether the build of the control flow map is at the top-level
+  /// (root) of a logical operator nest in a boolean expression prior to the
+  /// assignment of condition IDs.
+  bool isIdle() const { return (NextID == 1 && !NotMapped); }
+
+  /// Return whether any IDs have been assigned in the build of the control
+  /// flow map, indicating that the map is being generated for this boolean
+  /// expression.
+  bool isBuilding() const { return (NextID > 1); }
+
+  /// Set the given condition's ID.
+  void setCondID(const Expr *Cond, MCDCConditionID ID) {
+    CondIDs[CodeGenFunction::stripCond(Cond)] = ID;
+  }
+
+  /// Return the ID of a given condition.
+  MCDCConditionID getCondID(const Expr *Cond) const {
+    auto I = CondIDs.find(CodeGenFunction::stripCond(Cond));
+    if (I == CondIDs.end())
+      return 0;
+    else
+      return I->second;
+  }
+
+  /// Return the LHS Decision ([0,0] if not set).
+  const DecisionIDPair &back() const { return DecisionStack.back(); }
+
+  /// Push the binary operator statement to track the nest level and assign IDs
+  /// to the operator's LHS and RHS.  The RHS may be a larger subtree that is
+  /// broken up on successive levels.
+  void pushAndAssignIDs(const BinaryOperator *E) {
+    if (!CGM.getCodeGenOpts().MCDCCoverage)
+      return;
+
+    // If binary expression is disqualified, don't do mapping.
+    if (!isBuilding() && !MCDCBitmapMap.contains(CodeGenFunction::stripCond(E)))
+      NotMapped = true;
+
+    // Don't go any further if we don't need to map condition IDs.
+    if (NotMapped)
+      return;
+
+    const DecisionIDPair &ParentDecision = DecisionStack.back();
+
+    // If the operator itself has an assigned ID, this means it represents a
+    // larger subtree.  In this case, assign that ID to its LHS node.  Its RHS
+    // will receive a new ID below. Otherwise, assign ID+1 to LHS.
+    if (CondIDs.contains(CodeGenFunction::stripCond(E)))
+      setCondID(E->getLHS(), getCondID(E));
+    else
+      setCondID(E->getLHS(), NextID++);
+
+    // Assign a ID+1 for the RHS.
+    MCDCConditionID RHSid = NextID++;
+    setCondID(E->getRHS(), RHSid);
+
+    // Push the LHS decision IDs onto the DecisionStack.
+    if (isLAnd(E))
+      DecisionStack.push_back({RHSid, ParentDecision.FalseID});
+    else
+      DecisionStack.push_back({ParentDecision.TrueID, RHSid});
+  }
+
+  /// Pop and return the LHS Decision ([0,0] if not set).
+  DecisionIDPair pop() {
+    if (!CGM.getCodeGenOpts().MCDCCoverage || NotMapped)
+      return DecisionStack.front();
+
+    assert(DecisionStack.size() > 1);
+    DecisionIDPair D = DecisionStack.back();
+    DecisionStack.pop_back();
+    return D;
+  }
+
+  /// Return the total number of conditions and reset the state. The number of
+  /// conditions is zero if the expression isn't mapped.
+  unsigned getTotalConditionsAndReset(const BinaryOperator *E) {
+    if (!CGM.getCodeGenOpts().MCDCCoverage)
+      return 0;
+
+    assert(!isIdle());
+    assert(DecisionStack.size() == 1);
+
+    // Reset state if not doing mapping.
+    if (NotMapped) {
+      NotMapped = false;
+      assert(NextID == 1);
+      return 0;
+    }
+
+    // Set number of conditions and reset.
+    unsigned TotalConds = NextID - 1;
+
+    // Reset ID back to beginning.
+    NextID = 1;
+
+    return TotalConds;
+  }
+};
+
 /// A StmtVisitor that creates coverage mapping regions which map
 /// from the source code locations to the PGO counters.
 struct CounterCoverageMappingBuilder
@@ -550,8 +810,14 @@ struct CounterCoverageMappingBuilder
   /// The map of statements to count values.
   llvm::DenseMap &CounterMap;
 
+  /// The map of statements to bitmap coverage object values.
+  llvm::DenseMap &MCDCBitmapMap;
+
   /// A stack of currently live regions.
-  std::vector RegionStack;
+  llvm::SmallVector RegionStack;
+
+  /// An object to manage MCDC regions.
+  MCDCCoverageBuilder MCDCBuilder;
 
   CounterExpressionBuilder Builder;
 
@@ -589,6 +855,8 @@ struct CounterCoverageMappingBuilder
     return Counter::getCounter(CounterMap[S]);
   }
 
+  unsigned getRegionBitmap(const Stmt *S) { return MCDCBitmapMap[S]; }
+
   /// Push a region onto the stack.
   ///
   /// Returns the index on the stack where the region was pushed. This can be
@@ -596,7 +864,9 @@ struct CounterCoverageMappingBuilder
   size_t pushRegion(Counter Count,
                     std::optional StartLoc = std::nullopt,
                     std::optional EndLoc = std::nullopt,
-                    std::optional FalseCount = std::nullopt) {
+                    std::optional FalseCount = std::nullopt,
+                    MCDCConditionID ID = 0, MCDCConditionID TrueID = 0,
+                    MCDCConditionID FalseID = 0) {
 
     if (StartLoc && !FalseCount) {
       MostRecentLocation = *StartLoc;
@@ -615,7 +885,19 @@ struct CounterCoverageMappingBuilder
       StartLoc = std::nullopt;
     if (EndLoc && EndLoc->isInvalid())
       EndLoc = std::nullopt;
-    RegionStack.emplace_back(Count, FalseCount, StartLoc, EndLoc);
+    RegionStack.emplace_back(Count, FalseCount,
+                             MCDCParameters{0, 0, ID, TrueID, FalseID},
+                             StartLoc, EndLoc);
+
+    return RegionStack.size() - 1;
+  }
+
+  size_t pushRegion(unsigned BitmapIdx, unsigned Conditions,
+                    std::optional StartLoc = std::nullopt,
+                    std::optional EndLoc = std::nullopt) {
+
+    RegionStack.emplace_back(MCDCParameters{BitmapIdx, Conditions}, StartLoc,
+                             EndLoc);
 
     return RegionStack.size() - 1;
   }
@@ -742,11 +1024,15 @@ struct CounterCoverageMappingBuilder
     return (Cond->EvaluateAsInt(Result, CVM.getCodeGenModule().getContext()));
   }
 
+  using MCDCDecisionIDPair = MCDCCoverageBuilder::DecisionIDPair;
+
   /// Create a Branch Region around an instrumentable condition for coverage
   /// and add it to the function's SourceRegions.  A branch region tracks a
   /// "True" counter and a "False" counter for boolean expressions that
   /// result in the generation of a branch.
-  void createBranchRegion(const Expr *C, Counter TrueCnt, Counter FalseCnt) {
+  void
+  createBranchRegion(const Expr *C, Counter TrueCnt, Counter FalseCnt,
+                     const MCDCDecisionIDPair &IDPair = MCDCDecisionIDPair()) {
     // Check for NULL conditions.
     if (!C)
       return;
@@ -756,6 +1042,10 @@ struct CounterCoverageMappingBuilder
     // function's SourceRegions) because it doesn't apply to any other source
     // code other than the Condition.
     if (CodeGenFunction::isInstrumentedCondition(C)) {
+      MCDCConditionID ID = MCDCBuilder.getCondID(C);
+      MCDCConditionID TrueID = IDPair.TrueID;
+      MCDCConditionID FalseID = IDPair.FalseID;
+
       // If a condition can fold to true or false, the corresponding branch
       // will be removed.  Create a region with both counters hard-coded to
       // zero. This allows us to visualize them in a special way.
@@ -764,13 +1054,21 @@ struct CounterCoverageMappingBuilder
       // CodeGenFunction.c always returns false, but that is very heavy-handed.
       if (ConditionFoldsToBool(C))
         popRegions(pushRegion(Counter::getZero(), getStart(C), getEnd(C),
-                              Counter::getZero()));
+                              Counter::getZero(), ID, TrueID, FalseID));
       else
         // Otherwise, create a region with the True counter and False counter.
-        popRegions(pushRegion(TrueCnt, getStart(C), getEnd(C), FalseCnt));
+        popRegions(pushRegion(TrueCnt, getStart(C), getEnd(C), FalseCnt, ID,
+                              TrueID, FalseID));
     }
   }
 
+  /// Create a Decision Region with a BitmapIdx and number of Conditions. This
+  /// type of region "contains" branch regions, one for each of the conditions.
+  /// The visualization tool will group everything together.
+  void createDecisionRegion(const Expr *C, unsigned BitmapIdx, unsigned Conds) {
+    popRegions(pushRegion(BitmapIdx, Conds, getStart(C), getEnd(C)));
+  }
+
   /// Create a Branch Region around a SwitchCase for code coverage
   /// and add it to the function's SourceRegions.
   void createSwitchCaseRegion(const SwitchCase *SC, Counter TrueCnt,
@@ -851,8 +1149,12 @@ struct CounterCoverageMappingBuilder
         // we've seen this region.
         if (StartLocs.insert(Loc).second) {
           if (I.isBranch())
-            SourceRegions.emplace_back(I.getCounter(), I.getFalseCounter(), Loc,
-                                       getEndOfFileOrMacro(Loc), I.isBranch());
+            SourceRegions.emplace_back(
+                I.getCounter(), I.getFalseCounter(),
+                MCDCParameters{0, 0, I.getMCDCParams().ID,
+                               I.getMCDCParams().TrueID,
+                               I.getMCDCParams().FalseID},
+                Loc, getEndOfFileOrMacro(Loc), I.isBranch());
           else
             SourceRegions.emplace_back(I.getCounter(), Loc,
                                        getEndOfFileOrMacro(Loc));
@@ -962,6 +1264,69 @@ struct CounterCoverageMappingBuilder
     popRegions(Index);
   }
 
+  /// Find a valid range starting with \p StartingLoc and ending before \p
+  /// BeforeLoc.
+  std::optional findAreaStartingFromTo(SourceLocation StartingLoc,
+                                                    SourceLocation BeforeLoc) {
+    // If StartingLoc is in function-like macro, use its start location.
+    if (StartingLoc.isMacroID()) {
+      FileID FID = SM.getFileID(StartingLoc);
+      const SrcMgr::ExpansionInfo *EI = &SM.getSLocEntry(FID).getExpansion();
+      if (EI->isFunctionMacroExpansion())
+        StartingLoc = EI->getExpansionLocStart();
+    }
+
+    size_t StartDepth = locationDepth(StartingLoc);
+    size_t EndDepth = locationDepth(BeforeLoc);
+    while (!SM.isWrittenInSameFile(StartingLoc, BeforeLoc)) {
+      bool UnnestStart = StartDepth >= EndDepth;
+      bool UnnestEnd = EndDepth >= StartDepth;
+      if (UnnestEnd) {
+        assert(SM.isWrittenInSameFile(getStartOfFileOrMacro(BeforeLoc),
+                                      BeforeLoc));
+
+        BeforeLoc = getIncludeOrExpansionLoc(BeforeLoc);
+        assert(BeforeLoc.isValid());
+        EndDepth--;
+      }
+      if (UnnestStart) {
+        assert(SM.isWrittenInSameFile(StartingLoc,
+                                      getStartOfFileOrMacro(StartingLoc)));
+
+        StartingLoc = getIncludeOrExpansionLoc(StartingLoc);
+        assert(StartingLoc.isValid());
+        StartDepth--;
+      }
+    }
+    // If the start and end locations of the gap are both within the same macro
+    // file, the range may not be in source order.
+    if (StartingLoc.isMacroID() || BeforeLoc.isMacroID())
+      return std::nullopt;
+    if (!SM.isWrittenInSameFile(StartingLoc, BeforeLoc) ||
+        !SpellingRegion(SM, StartingLoc, BeforeLoc).isInSourceOrder())
+      return std::nullopt;
+    return {{StartingLoc, BeforeLoc}};
+  }
+
+  void markSkipped(SourceLocation StartLoc, SourceLocation BeforeLoc) {
+    const auto Skipped = findAreaStartingFromTo(StartLoc, BeforeLoc);
+
+    if (!Skipped)
+      return;
+
+    const auto NewStartLoc = Skipped->getBegin();
+    const auto EndLoc = Skipped->getEnd();
+
+    if (NewStartLoc == EndLoc)
+      return;
+    assert(SpellingRegion(SM, NewStartLoc, EndLoc).isInSourceOrder());
+    handleFileExit(NewStartLoc);
+    size_t Index = pushRegion({}, NewStartLoc, EndLoc);
+    getRegion().setSkipped(true);
+    handleFileExit(EndLoc);
+    popRegions(Index);
+  }
+
   /// Keep counts of breaks and continues inside loops.
   struct BreakContinue {
     Counter BreakCount;
@@ -971,9 +1336,13 @@ struct CounterCoverageMappingBuilder
 
   CounterCoverageMappingBuilder(
       CoverageMappingModuleGen &CVM,
-      llvm::DenseMap &CounterMap, SourceManager &SM,
-      const LangOptions &LangOpts)
-      : CoverageMappingBuilder(CVM, SM, LangOpts), CounterMap(CounterMap) {}
+      llvm::DenseMap &CounterMap,
+      llvm::DenseMap &MCDCBitmapMap,
+      llvm::DenseMap &CondIDMap,
+      SourceManager &SM, const LangOptions &LangOpts)
+      : CoverageMappingBuilder(CVM, SM, LangOpts), CounterMap(CounterMap),
+        MCDCBitmapMap(MCDCBitmapMap),
+        MCDCBuilder(CVM.getCodeGenModule(), CondIDMap, MCDCBitmapMap) {}
 
   /// Write the mapping data to the output stream
   void write(llvm::raw_ostream &OS) {
@@ -1407,35 +1776,120 @@ struct CounterCoverageMappingBuilder
     Visit(S->getSubStmt());
   }
 
+  void coverIfConsteval(const IfStmt *S) {
+    assert(S->isConsteval());
+
+    const auto *Then = S->getThen();
+    const auto *Else = S->getElse();
+
+    // It's better for llvm-cov to create a new region with same counter
+    // so line-coverage can be properly calculated for lines containing
+    // a skipped region (without it the line is marked uncovered)
+    const Counter ParentCount = getRegion().getCounter();
+
+    extendRegion(S);
+
+    if (S->isNegatedConsteval()) {
+      // ignore 'if consteval'
+      markSkipped(S->getIfLoc(), getStart(Then));
+      propagateCounts(ParentCount, Then);
+
+      if (Else) {
+        // ignore 'else '
+        markSkipped(getEnd(Then), getEnd(Else));
+      }
+    } else {
+      assert(S->isNonNegatedConsteval());
+      // ignore 'if consteval  [else]'
+      markSkipped(S->getIfLoc(), Else ? getStart(Else) : getEnd(Then));
+
+      if (Else)
+        propagateCounts(ParentCount, Else);
+    }
+  }
+
+  void coverIfConstexpr(const IfStmt *S) {
+    assert(S->isConstexpr());
+
+    // evaluate constant condition...
+    const auto *E = cast(S->getCond());
+    const bool isTrue = E->getResultAsAPSInt().getExtValue();
+
+    extendRegion(S);
+
+    // I'm using 'propagateCounts' later as new region is better and allows me
+    // to properly calculate line coverage in llvm-cov utility
+    const Counter ParentCount = getRegion().getCounter();
+
+    // ignore 'if constexpr ('
+    SourceLocation startOfSkipped = S->getIfLoc();
+
+    if (const auto *Init = S->getInit()) {
+      const auto start = getStart(Init);
+      const auto end = getEnd(Init);
+
+      // this check is to make sure typedef here which doesn't have valid source
+      // location won't crash it
+      if (start.isValid() && end.isValid()) {
+        markSkipped(startOfSkipped, start);
+        propagateCounts(ParentCount, Init);
+        startOfSkipped = getEnd(Init);
+      }
+    }
+
+    const auto *Then = S->getThen();
+    const auto *Else = S->getElse();
+
+    if (isTrue) {
+      // ignore ')'
+      markSkipped(startOfSkipped, getStart(Then));
+      propagateCounts(ParentCount, Then);
+
+      if (Else)
+        // ignore 'else '
+        markSkipped(getEnd(Then), getEnd(Else));
+    } else {
+      // ignore ')  [else]'
+      markSkipped(startOfSkipped, Else ? getStart(Else) : getEnd(Then));
+
+      if (Else)
+        propagateCounts(ParentCount, Else);
+    }
+  }
+
   void VisitIfStmt(const IfStmt *S) {
+    // "if constexpr" and "if consteval" are not normal conditional statements,
+    // their discarded statement should be skipped
+    if (S->isConsteval())
+      return coverIfConsteval(S);
+    else if (S->isConstexpr())
+      return coverIfConstexpr(S);
+
     extendRegion(S);
     if (S->getInit())
       Visit(S->getInit());
 
     // Extend into the condition before we propagate through it below - this is
     // needed to handle macros that generate the "if" but not the condition.
-    if (!S->isConsteval())
-      extendRegion(S->getCond());
+    extendRegion(S->getCond());
 
     Counter ParentCount = getRegion().getCounter();
     Counter ThenCount = getRegionCounter(S);
 
-    if (!S->isConsteval()) {
-      // Emitting a counter for the condition makes it easier to interpret the
-      // counter for the body when looking at the coverage.
-      propagateCounts(ParentCount, S->getCond());
+    // Emitting a counter for the condition makes it easier to interpret the
+    // counter for the body when looking at the coverage.
+    propagateCounts(ParentCount, S->getCond());
 
-      // The 'then' count applies to the area immediately after the condition.
-      std::optional Gap =
-          findGapAreaBetween(S->getRParenLoc(), getStart(S->getThen()));
-      if (Gap)
-        fillGapAreaWithCount(Gap->getBegin(), Gap->getEnd(), ThenCount);
-    }
+    // The 'then' count applies to the area immediately after the condition.
+    std::optional Gap =
+        findGapAreaBetween(S->getRParenLoc(), getStart(S->getThen()));
+    if (Gap)
+      fillGapAreaWithCount(Gap->getBegin(), Gap->getEnd(), ThenCount);
 
     extendRegion(S->getThen());
     Counter OutCount = propagateCounts(ThenCount, S->getThen());
-
     Counter ElseCount = subtractCounters(ParentCount, ThenCount);
+
     if (const Stmt *Else = S->getElse()) {
       bool ThenHasTerminateStmt = HasTerminateStmt;
       HasTerminateStmt = false;
@@ -1457,11 +1911,9 @@ struct CounterCoverageMappingBuilder
       GapRegionCounter = OutCount;
     }
 
-    if (!S->isConsteval()) {
-      // Create Branch Region around condition.
-      createBranchRegion(S->getCond(), ThenCount,
-                         subtractCounters(ParentCount, ThenCount));
-    }
+    // Create Branch Region around condition.
+    createBranchRegion(S->getCond(), ThenCount,
+                       subtractCounters(ParentCount, ThenCount));
   }
 
   void VisitCXXTryStmt(const CXXTryStmt *S) {
@@ -1519,14 +1971,30 @@ struct CounterCoverageMappingBuilder
   }
 
   void VisitBinLAnd(const BinaryOperator *E) {
+    bool IsRootNode = MCDCBuilder.isIdle();
+
+    // Keep track of Binary Operator and assign MCDC condition IDs.
+    MCDCBuilder.pushAndAssignIDs(E);
+
     extendRegion(E->getLHS());
     propagateCounts(getRegion().getCounter(), E->getLHS());
     handleFileExit(getEnd(E->getLHS()));
 
+    // Track LHS True/False Decision.
+    const auto DecisionLHS = MCDCBuilder.pop();
+
     // Counter tracks the right hand side of a logical and operator.
     extendRegion(E->getRHS());
     propagateCounts(getRegionCounter(E), E->getRHS());
 
+    // Track RHS True/False Decision.
+    const auto DecisionRHS = MCDCBuilder.back();
+
+    // Create MCDC Decision Region if at top-level (root).
+    unsigned NumConds = 0;
+    if (IsRootNode && (NumConds = MCDCBuilder.getTotalConditionsAndReset(E)))
+      createDecisionRegion(E, getRegionBitmap(E), NumConds);
+
     // Extract the RHS's Execution Counter.
     Counter RHSExecCnt = getRegionCounter(E);
 
@@ -1538,11 +2006,11 @@ struct CounterCoverageMappingBuilder
 
     // Create Branch Region around LHS condition.
     createBranchRegion(E->getLHS(), RHSExecCnt,
-                       subtractCounters(ParentCnt, RHSExecCnt));
+                       subtractCounters(ParentCnt, RHSExecCnt), DecisionLHS);
 
     // Create Branch Region around RHS condition.
     createBranchRegion(E->getRHS(), RHSTrueCnt,
-                       subtractCounters(RHSExecCnt, RHSTrueCnt));
+                       subtractCounters(RHSExecCnt, RHSTrueCnt), DecisionRHS);
   }
 
   // Determine whether the right side of OR operation need to be visited.
@@ -1556,14 +2024,30 @@ struct CounterCoverageMappingBuilder
   }
 
   void VisitBinLOr(const BinaryOperator *E) {
+    bool IsRootNode = MCDCBuilder.isIdle();
+
+    // Keep track of Binary Operator and assign MCDC condition IDs.
+    MCDCBuilder.pushAndAssignIDs(E);
+
     extendRegion(E->getLHS());
     Counter OutCount = propagateCounts(getRegion().getCounter(), E->getLHS());
     handleFileExit(getEnd(E->getLHS()));
 
+    // Track LHS True/False Decision.
+    const auto DecisionLHS = MCDCBuilder.pop();
+
     // Counter tracks the right hand side of a logical or operator.
     extendRegion(E->getRHS());
     propagateCounts(getRegionCounter(E), E->getRHS());
 
+    // Track RHS True/False Decision.
+    const auto DecisionRHS = MCDCBuilder.back();
+
+    // Create MCDC Decision Region if at top-level (root).
+    unsigned NumConds = 0;
+    if (IsRootNode && (NumConds = MCDCBuilder.getTotalConditionsAndReset(E)))
+      createDecisionRegion(E, getRegionBitmap(E), NumConds);
+
     // Extract the RHS's Execution Counter.
     Counter RHSExecCnt = getRegionCounter(E);
 
@@ -1579,11 +2063,11 @@ struct CounterCoverageMappingBuilder
 
     // Create Branch Region around LHS condition.
     createBranchRegion(E->getLHS(), subtractCounters(ParentCnt, RHSExecCnt),
-                       RHSExecCnt);
+                       RHSExecCnt, DecisionLHS);
 
     // Create Branch Region around RHS condition.
     createBranchRegion(E->getRHS(), subtractCounters(RHSExecCnt, RHSFalseCnt),
-                       RHSFalseCnt);
+                       RHSFalseCnt, DecisionRHS);
   }
 
   void VisitLambdaExpr(const LambdaExpr *LE) {
@@ -1633,11 +2117,23 @@ static void dump(llvm::raw_ostream &OS, StringRef FunctionName,
 
     OS << "File " << R.FileID << ", " << R.LineStart << ":" << R.ColumnStart
        << " -> " << R.LineEnd << ":" << R.ColumnEnd << " = ";
-    Ctx.dump(R.Count, OS);
 
-    if (R.Kind == CounterMappingRegion::BranchRegion) {
-      OS << ", ";
-      Ctx.dump(R.FalseCount, OS);
+    if (R.Kind == CounterMappingRegion::MCDCDecisionRegion) {
+      OS << "M:" << R.MCDCParams.BitmapIdx;
+      OS << ", C:" << R.MCDCParams.NumConditions;
+    } else {
+      Ctx.dump(R.Count, OS);
+
+      if (R.Kind == CounterMappingRegion::BranchRegion ||
+          R.Kind == CounterMappingRegion::MCDCBranchRegion) {
+        OS << ", ";
+        Ctx.dump(R.FalseCount, OS);
+      }
+    }
+
+    if (R.Kind == CounterMappingRegion::MCDCBranchRegion) {
+      OS << " [" << R.MCDCParams.ID << "," << R.MCDCParams.TrueID;
+      OS << "," << R.MCDCParams.FalseID << "] ";
     }
 
     if (R.Kind == CounterMappingRegion::ExpansionRegion)
@@ -1846,8 +2342,9 @@ unsigned CoverageMappingModuleGen::getFileID(FileEntryRef File) {
 
 void CoverageMappingGen::emitCounterMapping(const Decl *D,
                                             llvm::raw_ostream &OS) {
-  assert(CounterMap);
-  CounterCoverageMappingBuilder Walker(CVM, *CounterMap, SM, LangOpts);
+  assert(CounterMap && MCDCBitmapMap);
+  CounterCoverageMappingBuilder Walker(CVM, *CounterMap, *MCDCBitmapMap,
+                                       *CondIDMap, SM, LangOpts);
   Walker.VisitDecl(D);
   Walker.write(OS);
 }
diff --git a/clang/lib/CodeGen/CoverageMappingGen.h b/clang/lib/CodeGen/CoverageMappingGen.h
index 77d7c6cd87cf..62cea173c9fc 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.h
+++ b/clang/lib/CodeGen/CoverageMappingGen.h
@@ -150,16 +150,22 @@ class CoverageMappingGen {
   SourceManager &SM;
   const LangOptions &LangOpts;
   llvm::DenseMap *CounterMap;
+  llvm::DenseMap *MCDCBitmapMap;
+  llvm::DenseMap *CondIDMap;
 
 public:
   CoverageMappingGen(CoverageMappingModuleGen &CVM, SourceManager &SM,
                      const LangOptions &LangOpts)
-      : CVM(CVM), SM(SM), LangOpts(LangOpts), CounterMap(nullptr) {}
+      : CVM(CVM), SM(SM), LangOpts(LangOpts), CounterMap(nullptr),
+        MCDCBitmapMap(nullptr), CondIDMap(nullptr) {}
 
   CoverageMappingGen(CoverageMappingModuleGen &CVM, SourceManager &SM,
                      const LangOptions &LangOpts,
-                     llvm::DenseMap *CounterMap)
-      : CVM(CVM), SM(SM), LangOpts(LangOpts), CounterMap(CounterMap) {}
+                     llvm::DenseMap *CounterMap,
+                     llvm::DenseMap *MCDCBitmapMap,
+                     llvm::DenseMap *CondIDMap)
+      : CVM(CVM), SM(SM), LangOpts(LangOpts), CounterMap(CounterMap),
+        MCDCBitmapMap(MCDCBitmapMap), CondIDMap(CondIDMap) {}
 
   /// Emit the coverage mapping data which maps the regions of
   /// code to counters that will be used to find the execution
diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h
index cd31e3bd0e10..5a79e0af2207 100644
--- a/clang/lib/CodeGen/TargetInfo.h
+++ b/clang/lib/CodeGen/TargetInfo.h
@@ -502,7 +502,8 @@ createPPC64_SVR4_TargetCodeGenInfo(CodeGenModule &CGM, PPC64_SVR4_ABIKind Kind,
                                    bool SoftFloatABI);
 
 std::unique_ptr
-createRISCVTargetCodeGenInfo(CodeGenModule &CGM, unsigned XLen, unsigned FLen);
+createRISCVTargetCodeGenInfo(CodeGenModule &CGM, unsigned XLen, unsigned FLen,
+                             bool EABI);
 
 std::unique_ptr
 createCommonSPIRTargetCodeGenInfo(CodeGenModule &CGM);
diff --git a/clang/lib/CodeGen/Targets/AArch64.cpp b/clang/lib/CodeGen/Targets/AArch64.cpp
index 7102d190fe00..ee7f95084d2e 100644
--- a/clang/lib/CodeGen/Targets/AArch64.cpp
+++ b/clang/lib/CodeGen/Targets/AArch64.cpp
@@ -138,6 +138,8 @@ class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
                   BPI.BranchTargetEnforcement ? "true" : "false");
     Fn->addFnAttr("branch-protection-pauth-lr",
                   BPI.BranchProtectionPAuthLR ? "true" : "false");
+    Fn->addFnAttr("guarded-control-stack",
+                  BPI.GuardedControlStack ? "true" : "false");
   }
 
   bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF,
diff --git a/clang/lib/CodeGen/Targets/LoongArch.cpp b/clang/lib/CodeGen/Targets/LoongArch.cpp
index 7b2c31139b0b..63b9a1fdb988 100644
--- a/clang/lib/CodeGen/Targets/LoongArch.cpp
+++ b/clang/lib/CodeGen/Targets/LoongArch.cpp
@@ -324,13 +324,6 @@ ABIArgInfo LoongArchABIInfo::classifyArgumentType(QualType Ty, bool IsFixed,
     return ABIArgInfo::getDirect();
   }
 
-  // Pass 128-bit/256-bit vector values via vector registers directly.
-  if (Ty->isVectorType() && (((getContext().getTypeSize(Ty) == 128) &&
-                              (getTarget().hasFeature("lsx"))) ||
-                             ((getContext().getTypeSize(Ty) == 256) &&
-                              getTarget().hasFeature("lasx"))))
-    return ABIArgInfo::getDirect();
-
   // Complex types for the *f or *d ABI must be passed directly rather than
   // using CoerceAndExpand.
   if (IsFixed && Ty->isComplexType() && FRLen && FARsLeft >= 2) {
diff --git a/clang/lib/CodeGen/Targets/RISCV.cpp b/clang/lib/CodeGen/Targets/RISCV.cpp
index 1e1d249b37ac..0851d1993d0c 100644
--- a/clang/lib/CodeGen/Targets/RISCV.cpp
+++ b/clang/lib/CodeGen/Targets/RISCV.cpp
@@ -25,8 +25,9 @@ class RISCVABIInfo : public DefaultABIInfo {
   // ISA might have a wider FLen than the selected ABI (e.g. an RV32IF target
   // with soft float ABI has FLen==0).
   unsigned FLen;
-  static const int NumArgGPRs = 8;
-  static const int NumArgFPRs = 8;
+  const int NumArgGPRs;
+  const int NumArgFPRs;
+  const bool EABI;
   bool detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff,
                                       llvm::Type *&Field1Ty,
                                       CharUnits &Field1Off,
@@ -34,8 +35,10 @@ class RISCVABIInfo : public DefaultABIInfo {
                                       CharUnits &Field2Off) const;
 
 public:
-  RISCVABIInfo(CodeGen::CodeGenTypes &CGT, unsigned XLen, unsigned FLen)
-      : DefaultABIInfo(CGT), XLen(XLen), FLen(FLen) {}
+  RISCVABIInfo(CodeGen::CodeGenTypes &CGT, unsigned XLen, unsigned FLen,
+               bool EABI)
+      : DefaultABIInfo(CGT), XLen(XLen), FLen(FLen), NumArgGPRs(EABI ? 6 : 8),
+        NumArgFPRs(FLen != 0 ? 8 : 0), EABI(EABI) {}
 
   // DefaultABIInfo's classifyReturnType and classifyArgumentType are
   // non-virtual, but computeInfo is virtual, so we overload it.
@@ -86,7 +89,7 @@ void RISCVABIInfo::computeInfo(CGFunctionInfo &FI) const {
   }
 
   int ArgGPRsLeft = IsRetIndirect ? NumArgGPRs - 1 : NumArgGPRs;
-  int ArgFPRsLeft = FLen ? NumArgFPRs : 0;
+  int ArgFPRsLeft = NumArgFPRs;
   int NumFixedArgs = FI.getNumRequiredArgs();
 
   int ArgNum = 0;
@@ -396,9 +399,12 @@ ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed,
   // Determine the number of GPRs needed to pass the current argument
   // according to the ABI. 2*XLen-aligned varargs are passed in "aligned"
   // register pairs, so may consume 3 registers.
+  // TODO: To be compatible with GCC's behaviors, we don't align registers
+  // currently if we are using ILP32E calling convention. This behavior may be
+  // changed when RV32E/ILP32E is ratified.
   int NeededArgGPRs = 1;
   if (!IsFixed && NeededAlign == 2 * XLen)
-    NeededArgGPRs = 2 + (ArgGPRsLeft % 2);
+    NeededArgGPRs = 2 + (EABI && XLen == 32 ? 0 : (ArgGPRsLeft % 2));
   else if (Size > XLen && Size <= 2 * XLen)
     NeededArgGPRs = 2;
 
@@ -480,6 +486,13 @@ Address RISCVABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
 
   auto TInfo = getContext().getTypeInfoInChars(Ty);
 
+  // TODO: To be compatible with GCC's behaviors, we force arguments with
+  // 2×XLEN-bit alignment and size at most 2×XLEN bits like `long long`,
+  // `unsigned long long` and `double` to have 4-byte alignment. This
+  // behavior may be changed when RV32E/ILP32E is ratified.
+  if (EABI && XLen == 32)
+    TInfo.Align = std::min(TInfo.Align, CharUnits::fromQuantity(4));
+
   // Arguments bigger than 2*Xlen bytes are passed indirectly.
   bool IsIndirect = TInfo.Width > 2 * SlotSize;
 
@@ -499,8 +512,9 @@ namespace {
 class RISCVTargetCodeGenInfo : public TargetCodeGenInfo {
 public:
   RISCVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, unsigned XLen,
-                         unsigned FLen)
-      : TargetCodeGenInfo(std::make_unique(CGT, XLen, FLen)) {}
+                         unsigned FLen, bool EABI)
+      : TargetCodeGenInfo(
+            std::make_unique(CGT, XLen, FLen, EABI)) {}
 
   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
                            CodeGen::CodeGenModule &CGM) const override {
@@ -526,6 +540,7 @@ class RISCVTargetCodeGenInfo : public TargetCodeGenInfo {
 
 std::unique_ptr
 CodeGen::createRISCVTargetCodeGenInfo(CodeGenModule &CGM, unsigned XLen,
-                                      unsigned FLen) {
-  return std::make_unique(CGM.getTypes(), XLen, FLen);
+                                      unsigned FLen, bool EABI) {
+  return std::make_unique(CGM.getTypes(), XLen, FLen,
+                                                  EABI);
 }
diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 2af240350438..2291c991fb11 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -40,6 +40,11 @@ static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
     return llvm::Type::getX86_MMXTy(CGF.getLLVMContext());
   }
 
+  if (Constraint == "k") {
+    llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGF.getLLVMContext());
+    return llvm::FixedVectorType::get(Int1Ty, Ty->getScalarSizeInBits());
+  }
+
   // No operation needed
   return Ty;
 }
@@ -1797,6 +1802,9 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
     } else if (k == BuiltinType::Float || k == BuiltinType::Double ||
                k == BuiltinType::Float16 || k == BuiltinType::BFloat16) {
       Current = SSE;
+    } else if (k == BuiltinType::Float128) {
+      Lo = SSE;
+      Hi = SSEUp;
     } else if (k == BuiltinType::LongDouble) {
       const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
       if (LDF == &llvm::APFloat::IEEEquad()) {
diff --git a/clang/lib/CrossTU/CrossTranslationUnit.cpp b/clang/lib/CrossTU/CrossTranslationUnit.cpp
index 94c10e50d7d0..986470042bd8 100644
--- a/clang/lib/CrossTU/CrossTranslationUnit.cpp
+++ b/clang/lib/CrossTU/CrossTranslationUnit.cpp
@@ -247,7 +247,7 @@ CrossTranslationUnitContext::getLookupName(const NamedDecl *ND) {
   bool Ret = index::generateUSRForDecl(ND, DeclUSR);
   if (Ret)
     return {};
-  return std::string(DeclUSR.str());
+  return std::string(DeclUSR);
 }
 
 /// Recursively visits the decls of a DeclContext, and returns one with the
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index fb8069139b40..adde4b2faaa0 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -190,7 +190,7 @@ std::string Driver::GetResourcesPath(StringRef BinaryPath,
                             CLANG_VERSION_MAJOR_STRING);
   }
 
-  return std::string(P.str());
+  return std::string(P);
 }
 
 Driver::Driver(StringRef ClangExecutable, StringRef TargetTriple,
@@ -1314,10 +1314,23 @@ Compilation *Driver::BuildCompilation(ArrayRef ArgList) {
 
       A->claim();
 
-      // TODO: Specify Vulkan target environment somewhere in the triple.
       if (Args.hasArg(options::OPT_spirv)) {
         llvm::Triple T(TargetTriple);
         T.setArch(llvm::Triple::spirv);
+        T.setOS(llvm::Triple::Vulkan);
+
+        // Set specific Vulkan version if applicable.
+        if (const Arg *A = Args.getLastArg(options::OPT_fspv_target_env_EQ)) {
+          const llvm::StringSet<> ValidValues = {"vulkan1.2", "vulkan1.3"};
+          if (ValidValues.contains(A->getValue())) {
+            T.setOSName(A->getValue());
+          } else {
+            Diag(diag::err_drv_invalid_value)
+                << A->getAsString(Args) << A->getValue();
+          }
+          A->claim();
+        }
+
         TargetTriple = T.str();
       }
     } else {
@@ -1434,6 +1447,17 @@ Compilation *Driver::BuildCompilation(ArrayRef ArgList) {
   const ToolChain &TC = getToolChain(
       *UArgs, computeTargetTriple(*this, TargetTriple, *UArgs));
 
+  if (TC.getTriple().isAndroid()) {
+    llvm::Triple Triple = TC.getTriple();
+    StringRef TripleVersionName = Triple.getEnvironmentVersionString();
+
+    if (Triple.getEnvironmentVersion().empty() && TripleVersionName != "") {
+      Diags.Report(diag::err_drv_triple_version_invalid)
+          << TripleVersionName << TC.getTripleString();
+      ContainsError = true;
+    }
+  }
+
   // Report warning when arm64EC option is overridden by specified target
   if ((TC.getTriple().getArch() != llvm::Triple::aarch64 ||
        TC.getTriple().getSubArch() != llvm::Triple::AArch64SubArch_arm64ec) &&
@@ -1824,7 +1848,7 @@ void Driver::generateCompilationDiagnostics(
       ScriptOS << "\n# Additional information: " << AdditionalInformation
                << "\n";
     if (Report)
-      Report->TemporaryFiles.push_back(std::string(Script.str()));
+      Report->TemporaryFiles.push_back(std::string(Script));
     Diag(clang::diag::note_drv_command_failed_diag_msg) << Script;
   }
 
@@ -4744,10 +4768,11 @@ Action *Driver::ConstructPhaseAction(
   case phases::Backend: {
     if (isUsingLTO() && TargetDeviceOffloadKind == Action::OFK_None) {
       types::ID Output;
-      if (Args.hasArg(options::OPT_S))
+      if (Args.hasArg(options::OPT_ffat_lto_objects))
+        Output = Args.hasArg(options::OPT_emit_llvm) ? types::TY_LTO_IR
+                                                     : types::TY_PP_Asm;
+      else if (Args.hasArg(options::OPT_S))
         Output = types::TY_LTO_IR;
-      else if (Args.hasArg(options::OPT_ffat_lto_objects))
-        Output = types::TY_PP_Asm;
       else
         Output = types::TY_LTO_BC;
       return C.MakeAction(Input, Output);
@@ -6066,17 +6091,17 @@ std::string Driver::GetFilePath(StringRef Name, const ToolChain &TC) const {
   SmallString<128> R(ResourceDir);
   llvm::sys::path::append(R, Name);
   if (llvm::sys::fs::exists(Twine(R)))
-    return std::string(R.str());
+    return std::string(R);
 
   SmallString<128> P(TC.getCompilerRTPath());
   llvm::sys::path::append(P, Name);
   if (llvm::sys::fs::exists(Twine(P)))
-    return std::string(P.str());
+    return std::string(P);
 
   SmallString<128> D(Dir);
   llvm::sys::path::append(D, "..", Name);
   if (llvm::sys::fs::exists(Twine(D)))
-    return std::string(D.str());
+    return std::string(D);
 
   if (auto P = SearchPaths(TC.getLibraryPaths()))
     return *P;
@@ -6113,11 +6138,11 @@ std::string Driver::GetProgramPath(StringRef Name, const ToolChain &TC) const {
     if (llvm::sys::fs::is_directory(PrefixDir)) {
       SmallString<128> P(PrefixDir);
       if (ScanDirForExecutable(P, Name))
-        return std::string(P.str());
+        return std::string(P);
     } else {
       SmallString<128> P((PrefixDir + Name).str());
       if (llvm::sys::fs::can_execute(Twine(P)))
-        return std::string(P.str());
+        return std::string(P);
     }
   }
 
@@ -6133,7 +6158,7 @@ std::string Driver::GetProgramPath(StringRef Name, const ToolChain &TC) const {
     for (const auto &Path : List) {
       SmallString<128> P(Path);
       if (ScanDirForExecutable(P, TargetSpecificExecutable))
-        return std::string(P.str());
+        return std::string(P);
     }
 
     // Fall back to the path
@@ -6153,7 +6178,7 @@ std::string Driver::GetTemporaryPath(StringRef Prefix, StringRef Suffix) const {
     return "";
   }
 
-  return std::string(Path.str());
+  return std::string(Path);
 }
 
 std::string Driver::GetTemporaryDirectory(StringRef Prefix) const {
@@ -6164,7 +6189,7 @@ std::string Driver::GetTemporaryDirectory(StringRef Prefix) const {
     return "";
   }
 
-  return std::string(Path.str());
+  return std::string(Path);
 }
 
 std::string Driver::GetClPchPath(Compilation &C, StringRef BaseName) const {
@@ -6186,7 +6211,7 @@ std::string Driver::GetClPchPath(Compilation &C, StringRef BaseName) const {
       Output = BaseName;
     llvm::sys::path::replace_extension(Output, ".pch");
   }
-  return std::string(Output.str());
+  return std::string(Output);
 }
 
 const ToolChain &Driver::getToolChain(const ArgList &Args,
@@ -6206,6 +6231,7 @@ const ToolChain &Driver::getToolChain(const ArgList &Args,
     case llvm::Triple::IOS:
     case llvm::Triple::TvOS:
     case llvm::Triple::WatchOS:
+    case llvm::Triple::XROS:
     case llvm::Triple::DriverKit:
       TC = std::make_unique(*this, Target, Args);
       break;
diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp
index ad68c086b717..56d497eb4c32 100644
--- a/clang/lib/Driver/SanitizerArgs.cpp
+++ b/clang/lib/Driver/SanitizerArgs.cpp
@@ -39,8 +39,6 @@ static const SanitizerMask NotAllowedWithTrap = SanitizerKind::Vptr;
 static const SanitizerMask NotAllowedWithMinimalRuntime = SanitizerKind::Vptr;
 static const SanitizerMask NotAllowedWithExecuteOnly =
     SanitizerKind::Function | SanitizerKind::KCFI;
-static const SanitizerMask RequiresPIE =
-    SanitizerKind::DataFlow | SanitizerKind::Scudo;
 static const SanitizerMask NeedsUnwindTables =
     SanitizerKind::Address | SanitizerKind::HWAddress | SanitizerKind::Thread |
     SanitizerKind::Memory | SanitizerKind::DataFlow;
@@ -192,7 +190,7 @@ static void addDefaultIgnorelists(const Driver &D, SanitizerMask Kinds,
     clang::SmallString<64> Path(D.ResourceDir);
     llvm::sys::path::append(Path, "share", BL.File);
     if (D.getVFS().exists(Path))
-      IgnorelistFiles.push_back(std::string(Path.str()));
+      IgnorelistFiles.push_back(std::string(Path));
     else if (BL.Mask == SanitizerKind::CFI && DiagnoseErrors)
       // If cfi_ignorelist.txt cannot be found in the resource dir, driver
       // should fail.
@@ -303,9 +301,7 @@ bool SanitizerArgs::needsCfiDiagRt() const {
          CfiCrossDso && !ImplicitCfiRuntime;
 }
 
-bool SanitizerArgs::requiresPIE() const {
-  return NeedPIE || (Sanitizers.Mask & RequiresPIE);
-}
+bool SanitizerArgs::requiresPIE() const { return NeedPIE; }
 
 bool SanitizerArgs::needsUnwindTables() const {
   return static_cast(Sanitizers.Mask & NeedsUnwindTables);
@@ -699,8 +695,6 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC,
     MsanParamRetval = Args.hasFlag(
         options::OPT_fsanitize_memory_param_retval,
         options::OPT_fno_sanitize_memory_param_retval, MsanParamRetval);
-    NeedPIE |= !(TC.getTriple().isOSLinux() &&
-                 TC.getTriple().getArch() == llvm::Triple::x86_64);
   } else if (AllAddedKinds & SanitizerKind::KernelMemory) {
     MsanUseAfterDtor = false;
     MsanParamRetval = Args.hasFlag(
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 96a57927339a..388030592b48 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -606,7 +606,7 @@ std::string ToolChain::getCompilerRTPath() const {
   } else {
     llvm::sys::path::append(Path, "lib", getOSLibName());
   }
-  return std::string(Path.str());
+  return std::string(Path);
 }
 
 std::string ToolChain::getCompilerRTBasename(const ArgList &Args,
@@ -659,7 +659,7 @@ std::string ToolChain::getCompilerRT(const ArgList &Args, StringRef Component,
     SmallString<128> P(LibPath);
     llvm::sys::path::append(P, CRTBasename);
     if (getVFS().exists(P))
-      return std::string(P.str());
+      return std::string(P);
   }
 
   // Fall back to the old expected compiler-rt name if the new one does not
@@ -668,7 +668,7 @@ std::string ToolChain::getCompilerRT(const ArgList &Args, StringRef Component,
       buildCompilerRTBasename(Args, Component, Type, /*AddArch=*/true);
   SmallString<128> Path(getCompilerRTPath());
   llvm::sys::path::append(Path, CRTBasename);
-  return std::string(Path.str());
+  return std::string(Path);
 }
 
 const char *ToolChain::getCompilerRTArgString(const llvm::opt::ArgList &Args,
@@ -783,7 +783,7 @@ ToolChain::path_list ToolChain::getArchSpecificLibPaths() const {
     llvm::sys::path::append(Path, "lib");
     for (auto &S : SS)
       llvm::sys::path::append(Path, S);
-    Paths.push_back(std::string(Path.str()));
+    Paths.push_back(std::string(Path));
   };
 
   AddPath({getTriple().str()});
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index 56f06fc5fccb..b3c9d5908654 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -545,26 +545,35 @@ void RocmInstallationDetector::AddHIPIncludeArgs(const ArgList &DriverArgs,
   }
 
   const auto HandleHipStdPar = [=, &DriverArgs, &CC1Args]() {
-    if (!hasHIPStdParLibrary()) {
-      D.Diag(diag::err_drv_no_hipstdpar_lib);
-      return;
-    }
-    if (!HasRocThrustLibrary &&
-        !D.getVFS().exists(getIncludePath() + "/thrust")) {
+    StringRef Inc = getIncludePath();
+    auto &FS = D.getVFS();
+
+    if (!hasHIPStdParLibrary())
+      if (!HIPStdParPathArg.empty() ||
+          !FS.exists(Inc + "/thrust/system/hip/hipstdpar/hipstdpar_lib.hpp")) {
+        D.Diag(diag::err_drv_no_hipstdpar_lib);
+        return;
+      }
+    if (!HasRocThrustLibrary && !FS.exists(Inc + "/thrust")) {
       D.Diag(diag::err_drv_no_hipstdpar_thrust_lib);
       return;
     }
-    if (!HasRocPrimLibrary &&
-        !D.getVFS().exists(getIncludePath() + "/rocprim")) {
+    if (!HasRocPrimLibrary && !FS.exists(Inc + "/rocprim")) {
       D.Diag(diag::err_drv_no_hipstdpar_prim_lib);
       return;
     }
-
     const char *ThrustPath;
     if (HasRocThrustLibrary)
       ThrustPath = DriverArgs.MakeArgString(HIPRocThrustPathArg);
     else
-      ThrustPath = DriverArgs.MakeArgString(getIncludePath() + "/thrust");
+      ThrustPath = DriverArgs.MakeArgString(Inc + "/thrust");
+
+    const char *HIPStdParPath;
+    if (hasHIPStdParLibrary())
+      HIPStdParPath = DriverArgs.MakeArgString(HIPStdParPathArg);
+    else
+      HIPStdParPath = DriverArgs.MakeArgString(StringRef(ThrustPath) +
+                                               "/system/hip/hipstdpar");
 
     const char *PrimPath;
     if (HasRocPrimLibrary)
@@ -573,8 +582,8 @@ void RocmInstallationDetector::AddHIPIncludeArgs(const ArgList &DriverArgs,
       PrimPath = DriverArgs.MakeArgString(getIncludePath() + "/rocprim");
 
     CC1Args.append({"-idirafter", ThrustPath, "-idirafter", PrimPath,
-                    "-idirafter", DriverArgs.MakeArgString(HIPStdParPathArg),
-                    "-include", "hipstdpar_lib.hpp"});
+                    "-idirafter", HIPStdParPath, "-include",
+                    "hipstdpar_lib.hpp"});
   };
 
   if (DriverArgs.hasArg(options::OPT_nogpuinc)) {
diff --git a/clang/lib/Driver/ToolChains/AVR.cpp b/clang/lib/Driver/ToolChains/AVR.cpp
index a4ce2258c3d8..bb5c0e6db997 100644
--- a/clang/lib/Driver/ToolChains/AVR.cpp
+++ b/clang/lib/Driver/ToolChains/AVR.cpp
@@ -430,7 +430,7 @@ AVRToolChain::getCompilerRT(const llvm::opt::ArgList &Args, StringRef Component,
   SmallString<256> Path(ToolChain::getCompilerRTPath());
   llvm::sys::path::append(Path, "avr");
   llvm::sys::path::append(Path, File.str());
-  return std::string(Path.str());
+  return std::string(Path);
 }
 
 void AVR::Linker::ConstructJob(Compilation &C, const JobAction &JA,
diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index 912df79417ae..0cf96bb5c9cb 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -53,6 +53,11 @@ std::string aarch64::getAArch64TargetCPU(const ArgList &Args,
     return "apple-m1";
   }
 
+  if (Triple.isXROS()) {
+    // The xrOS simulator runs on M1 as well, it should have been covered above.
+    assert(!Triple.isSimulatorEnvironment() && "xrossim should be mac-like");
+    return "apple-a12";
+  }
   // arm64e requires v8.3a and only runs on apple-a12 and later CPUs.
   if (Triple.isArm64e())
     return "apple-a12";
@@ -67,103 +72,45 @@ std::string aarch64::getAArch64TargetCPU(const ArgList &Args,
 
 // Decode AArch64 features from string like +[no]featureA+[no]featureB+...
 static bool DecodeAArch64Features(const Driver &D, StringRef text,
-                                  std::vector &Features,
-                                  const llvm::AArch64::ArchInfo &ArchInfo) {
+                                  llvm::AArch64::ExtensionSet &Extensions) {
   SmallVector Split;
   text.split(Split, StringRef("+"), -1, false);
 
   for (StringRef Feature : Split) {
-    StringRef FeatureName = llvm::AArch64::getArchExtFeature(Feature);
-    if (!FeatureName.empty())
-      Features.push_back(FeatureName);
-    else if (Feature == "neon" || Feature == "noneon")
+    if (Feature == "neon" || Feature == "noneon") {
       D.Diag(clang::diag::err_drv_no_neon_modifier);
-    else
-      return false;
-
-    // +sme implies +bf16.
-    // +sme-f64f64 and +sme-i16i64 both imply +sme.
-    if (Feature == "sme") {
-      Features.push_back("+bf16");
-    } else if (Feature == "nosme") {
-      Features.push_back("-sme-f64f64");
-      Features.push_back("-sme-i16i64");
-    } else if (Feature == "sme-f64f64") {
-      Features.push_back("+sme");
-      Features.push_back("+bf16");
-    } else if (Feature == "sme-i16i64") {
-      Features.push_back("+sme");
-      Features.push_back("+bf16");
-    } else if (Feature == "nobf16") {
-      Features.push_back("-sme");
-      Features.push_back("-sme-f64f64");
-      Features.push_back("-sme-i16i64");
+      continue;
     }
-
-    if (Feature == "sve2")
-      Features.push_back("+sve");
-    else if (Feature == "sve2-bitperm" || Feature == "sve2-sha3" ||
-             Feature == "sve2-aes" || Feature == "sve2-sm4") {
-      Features.push_back("+sve");
-      Features.push_back("+sve2");
-    } else if (Feature == "nosve") {
-      Features.push_back("-sve2");
-      Features.push_back("-sve2-bitperm");
-      Features.push_back("-sve2-sha3");
-      Features.push_back("-sve2-aes");
-      Features.push_back("-sve2-sm4");
-    } else if (Feature == "nosve2") {
-      Features.push_back("-sve2-bitperm");
-      Features.push_back("-sve2-sha3");
-      Features.push_back("-sve2-aes");
-      Features.push_back("-sve2-sm4");
-    }
-
-    // +sve implies +f32mm if the base architecture is >= v8.6A (except v9A)
-    // It isn't the case in general that sve implies both f64mm and f32mm
-    if ((ArchInfo == llvm::AArch64::ARMV8_6A ||
-         ArchInfo == llvm::AArch64::ARMV8_7A ||
-         ArchInfo == llvm::AArch64::ARMV8_8A ||
-         ArchInfo == llvm::AArch64::ARMV8_9A ||
-         ArchInfo == llvm::AArch64::ARMV9_1A ||
-         ArchInfo == llvm::AArch64::ARMV9_2A ||
-         ArchInfo == llvm::AArch64::ARMV9_3A ||
-         ArchInfo == llvm::AArch64::ARMV9_4A) &&
-        Feature == "sve")
-      Features.push_back("+f32mm");
+    if (!Extensions.parseModifier(Feature))
+      return false;
   }
+
   return true;
 }
 
 // Check if the CPU name and feature modifiers in -mcpu are legal. If yes,
 // decode CPU and feature.
 static bool DecodeAArch64Mcpu(const Driver &D, StringRef Mcpu, StringRef &CPU,
-                              std::vector &Features) {
+                              llvm::AArch64::ExtensionSet &Extensions) {
   std::pair Split = Mcpu.split("+");
   CPU = Split.first;
-  const llvm::AArch64::ArchInfo *ArchInfo = &llvm::AArch64::ARMV8A;
 
   if (CPU == "native")
     CPU = llvm::sys::getHostCPUName();
 
   if (CPU == "generic") {
-    Features.push_back("+neon");
+    Extensions.enable(llvm::AArch64::AEK_SIMD);
   } else {
     const std::optional CpuInfo =
         llvm::AArch64::parseCpu(CPU);
     if (!CpuInfo)
       return false;
-    ArchInfo = &CpuInfo->Arch;
-
-    Features.push_back(ArchInfo->ArchFeature);
 
-    auto Extension = CpuInfo->getImpliedExtensions();
-    if (!llvm::AArch64::getExtensionFeatures(Extension, Features))
-      return false;
+    Extensions.addCPUDefaults(*CpuInfo);
   }
 
   if (Split.second.size() &&
-      !DecodeAArch64Features(D, Split.second, Features, *ArchInfo))
+      !DecodeAArch64Features(D, Split.second, Extensions))
     return false;
 
   return true;
@@ -172,30 +119,21 @@ static bool DecodeAArch64Mcpu(const Driver &D, StringRef Mcpu, StringRef &CPU,
 static bool
 getAArch64ArchFeaturesFromMarch(const Driver &D, StringRef March,
                                 const ArgList &Args,
-                                std::vector &Features) {
+                                llvm::AArch64::ExtensionSet &Extensions) {
   std::string MarchLowerCase = March.lower();
   std::pair Split = StringRef(MarchLowerCase).split("+");
 
-  std::optional  ArchInfo =
+  const llvm::AArch64::ArchInfo *ArchInfo =
       llvm::AArch64::parseArch(Split.first);
   if (Split.first == "native")
     ArchInfo = llvm::AArch64::getArchForCpu(llvm::sys::getHostCPUName().str());
   if (!ArchInfo)
     return false;
-  Features.push_back(ArchInfo->ArchFeature);
-
-  // Enable SVE2 by default on Armv9-A.
-  // It can still be disabled if +nosve2 is present.
-  // We must do this early so that DecodeAArch64Features has the correct state
-  if ((*ArchInfo == llvm::AArch64::ARMV9A ||
-       *ArchInfo == llvm::AArch64::ARMV9_1A ||
-       *ArchInfo == llvm::AArch64::ARMV9_2A)) {
-    Features.push_back("+sve");
-    Features.push_back("+sve2");
-  }
+
+  Extensions.addArchDefaults(*ArchInfo);
 
   if ((Split.second.size() &&
-       !DecodeAArch64Features(D, Split.second, Features, *ArchInfo)))
+       !DecodeAArch64Features(D, Split.second, Extensions)))
     return false;
 
   return true;
@@ -204,10 +142,10 @@ getAArch64ArchFeaturesFromMarch(const Driver &D, StringRef March,
 static bool
 getAArch64ArchFeaturesFromMcpu(const Driver &D, StringRef Mcpu,
                                const ArgList &Args,
-                               std::vector &Features) {
+                               llvm::AArch64::ExtensionSet &Extensions) {
   StringRef CPU;
   std::string McpuLowerCase = Mcpu.lower();
-  if (!DecodeAArch64Mcpu(D, McpuLowerCase, CPU, Features))
+  if (!DecodeAArch64Mcpu(D, McpuLowerCase, CPU, Extensions))
     return false;
 
   return true;
@@ -218,10 +156,10 @@ getAArch64MicroArchFeaturesFromMtune(const Driver &D, StringRef Mtune,
                                      const ArgList &Args,
                                      std::vector &Features) {
   std::string MtuneLowerCase = Mtune.lower();
-  // Check CPU name is valid
-  std::vector MtuneFeatures;
+  // Check CPU name is valid, but ignore any extensions on it.
+  llvm::AArch64::ExtensionSet Extensions;
   StringRef Tune;
-  if (!DecodeAArch64Mcpu(D, MtuneLowerCase, Tune, MtuneFeatures))
+  if (!DecodeAArch64Mcpu(D, MtuneLowerCase, Tune, Extensions))
     return false;
 
   // Handle CPU name is 'native'.
@@ -240,7 +178,8 @@ getAArch64MicroArchFeaturesFromMcpu(const Driver &D, StringRef Mcpu,
                                     const ArgList &Args,
                                     std::vector &Features) {
   StringRef CPU;
-  std::vector DecodedFeature;
+  // Check CPU name is valid, but ignore any extensions on it.
+  llvm::AArch64::ExtensionSet DecodedFeature;
   std::string McpuLowerCase = Mcpu.lower();
   if (!DecodeAArch64Mcpu(D, McpuLowerCase, CPU, DecodedFeature))
     return false;
@@ -255,9 +194,8 @@ void aarch64::getAArch64TargetFeatures(const Driver &D,
                                        bool ForAS) {
   Arg *A;
   bool success = true;
-  // Enable NEON by default.
-  Features.push_back("+neon");
   llvm::StringRef WaMArch;
+  llvm::AArch64::ExtensionSet Extensions;
   if (ForAS)
     for (const auto *A :
          Args.filtered(options::OPT_Wa_COMMA, options::OPT_Xassembler))
@@ -268,17 +206,19 @@ void aarch64::getAArch64TargetFeatures(const Driver &D,
   // "-Xassembler -march" is detected. Otherwise it may return false
   // and causes Clang to error out.
   if (!WaMArch.empty())
-    success = getAArch64ArchFeaturesFromMarch(D, WaMArch, Args, Features);
+    success = getAArch64ArchFeaturesFromMarch(D, WaMArch, Args, Extensions);
   else if ((A = Args.getLastArg(options::OPT_march_EQ)))
-    success = getAArch64ArchFeaturesFromMarch(D, A->getValue(), Args, Features);
+    success =
+        getAArch64ArchFeaturesFromMarch(D, A->getValue(), Args, Extensions);
   else if ((A = Args.getLastArg(options::OPT_mcpu_EQ)))
-    success = getAArch64ArchFeaturesFromMcpu(D, A->getValue(), Args, Features);
+    success =
+        getAArch64ArchFeaturesFromMcpu(D, A->getValue(), Args, Extensions);
   else if (isCPUDeterminedByTriple(Triple))
     success = getAArch64ArchFeaturesFromMcpu(
-        D, getAArch64TargetCPU(Args, Triple, A), Args, Features);
+        D, getAArch64TargetCPU(Args, Triple, A), Args, Extensions);
   else
     // Default to 'A' profile if the architecture is not specified.
-    success = getAArch64ArchFeaturesFromMarch(D, "armv8-a", Args, Features);
+    success = getAArch64ArchFeaturesFromMarch(D, "armv8-a", Args, Extensions);
 
   if (success && (A = Args.getLastArg(clang::driver::options::OPT_mtune_EQ)))
     success =
@@ -300,12 +240,23 @@ void aarch64::getAArch64TargetFeatures(const Driver &D,
       Diag << A->getSpelling() << A->getValue();
   }
 
+  // -mgeneral-regs-only disables all floating-point features.
   if (Args.getLastArg(options::OPT_mgeneral_regs_only)) {
-    Features.push_back("-fp-armv8");
-    Features.push_back("-crypto");
-    Features.push_back("-neon");
+    Extensions.disable(llvm::AArch64::AEK_FP);
   }
 
+  // En/disable crc
+  if (Arg *A = Args.getLastArg(options::OPT_mcrc, options::OPT_mnocrc)) {
+    if (A->getOption().matches(options::OPT_mcrc))
+      Extensions.enable(llvm::AArch64::AEK_CRC);
+    else
+      Extensions.disable(llvm::AArch64::AEK_CRC);
+  }
+
+  // At this point all hardware features are decided, so convert the extensions
+  // set to a feature list.
+  Extensions.toLLVMFeatureList(Features);
+
   if (Arg *A = Args.getLastArg(options::OPT_mtp_mode_EQ)) {
     StringRef Mtp = A->getValue();
     if (Mtp == "el3" || Mtp == "tpidr_el3")
@@ -367,147 +318,6 @@ void aarch64::getAArch64TargetFeatures(const Driver &D,
     }
   }
 
-  // En/disable crc
-  if (Arg *A = Args.getLastArg(options::OPT_mcrc, options::OPT_mnocrc)) {
-    if (A->getOption().matches(options::OPT_mcrc))
-      Features.push_back("+crc");
-    else
-      Features.push_back("-crc");
-  }
-
-  int V8Version = -1;
-  int V9Version = -1;
-  bool HasNoSM4 = false;
-  bool HasNoSHA3 = false;
-  bool HasNoSHA2 = false;
-  bool HasNoAES = false;
-  bool HasSM4 = false;
-  bool HasSHA3 = false;
-  bool HasSHA2 = false;
-  bool HasAES = false;
-  bool HasCrypto = false;
-  bool HasNoCrypto = false;
-  int FullFP16Pos = -1;
-  int NoFullFP16Pos = -1;
-  int FP16FMLPos = -1;
-  int NoFP16FMLPos = -1;
-  int ArchFeatPos = -1;
-
-  for (auto I = Features.begin(), E = Features.end(); I != E; I++) {
-    if (*I == "+v8a")   V8Version = 0;
-    else if (*I == "+v8.1a") V8Version = 1;
-    else if (*I == "+v8.2a") V8Version = 2;
-    else if (*I == "+v8.3a") V8Version = 3;
-    else if (*I == "+v8.4a") V8Version = 4;
-    else if (*I == "+v8.5a") V8Version = 5;
-    else if (*I == "+v8.6a") V8Version = 6;
-    else if (*I == "+v8.7a") V8Version = 7;
-    else if (*I == "+v8.8a") V8Version = 8;
-    else if (*I == "+v8.9a") V8Version = 9;
-    else if (*I == "+v9a")   V9Version = 0;
-    else if (*I == "+v9.1a") V9Version = 1;
-    else if (*I == "+v9.2a") V9Version = 2;
-    else if (*I == "+v9.3a") V9Version = 3;
-    else if (*I == "+v9.4a") V9Version = 4;
-    else if (*I == "+v9.5a") V9Version = 5;
-    else if (*I == "+sm4")  HasSM4 = true;
-    else if (*I == "+sha3") HasSHA3 = true;
-    else if (*I == "+sha2") HasSHA2 = true;
-    else if (*I == "+aes")  HasAES = true;
-    else if (*I == "-sm4")  HasNoSM4 = true;
-    else if (*I == "-sha3") HasNoSHA3 = true;
-    else if (*I == "-sha2") HasNoSHA2 = true;
-    else if (*I == "-aes")  HasNoAES = true;
-    else if (*I == "+fp16fml")  FP16FMLPos = I - Features.begin();
-    else if (*I == "-fp16fml")  NoFP16FMLPos = I - Features.begin();
-    else if (*I == "-fullfp16") NoFullFP16Pos = I - Features.begin();
-    else if (*I == "+fullfp16") FullFP16Pos = I - Features.begin();
-    // Whichever option comes after (right-most option) will win
-    else if (*I == "+crypto") {
-      HasCrypto = true;
-      HasNoCrypto = false;
-    } else if (*I == "-crypto" || *I == "-neon") {
-      HasCrypto = false;
-      HasNoCrypto = true;
-      HasSM4 = HasSHA2 = HasSHA3 = HasAES = false;
-    }
-    // Register the iterator position if this is an architecture feature
-    if (ArchFeatPos == -1 && (V8Version != -1 || V9Version != -1))
-      ArchFeatPos = I - Features.begin();
-  }
-
-  // Handle (arch-dependent) fp16fml/fullfp16 relationship.
-  // FIXME: this fp16fml option handling will be reimplemented after the
-  // TargetParser rewrite.
-  if (V8Version >= 4) {
-    // "-fullfp16" "+fullfp16" && "+fp16fml" "+fullfp16" && no "+fullfp16" "-fp16fml" = "+fp16fml"
-    if (FullFP16Pos > NoFullFP16Pos && FullFP16Pos > FP16FMLPos && FullFP16Pos > NoFP16FMLPos)
-      // Only entangled feature that can be to the right of this +fullfp16 is -fp16fml.
-      // Only append the +fp16fml if there is no -fp16fml after the +fullfp16.
-      Features.push_back("+fp16fml");
-    else
-      goto fp16_fml_fallthrough;
-  } else {
-fp16_fml_fallthrough:
-    // In both of these cases, putting the 'other' feature on the end of the vector will
-    // result in the same effect as placing it immediately after the current feature.
-    // "+fp16fml"  "-fullfp16" = "-fp16fml"
-    if (NoFullFP16Pos > FP16FMLPos)
-      Features.push_back("-fp16fml");
-    // "-fullfp16" "+fp16fml" = "+fullfp16"
-    else if (NoFullFP16Pos < FP16FMLPos)
-      Features.push_back("+fullfp16");
-  }
-
-  // FIXME: this needs reimplementation too after the TargetParser rewrite
-  //
-  // Context sensitive meaning of Crypto:
-  // 1) For Arch >= ARMv8.4a:  crypto = sm4 + sha3 + sha2 + aes
-  // 2) For Arch <= ARMv8.3a:  crypto = sha2 + aes
-  if (V8Version >= 4 || V9Version >= 0) {
-    if (HasCrypto && !HasNoCrypto) {
-      // Check if we have NOT disabled an algorithm with something like:
-      //   +crypto, -algorithm
-      // And if "-algorithm" does not occur, we enable that crypto algorithm.
-      if (!HasNoSM4)
-        Features.push_back("+sm4");
-      if (!HasNoSHA3)
-        Features.push_back("+sha3");
-      if (!HasNoSHA2)
-        Features.push_back("+sha2");
-      if (!HasNoAES)
-        Features.push_back("+aes");
-    } else if (HasNoCrypto) {
-      // Check if we have NOT enabled a crypto algorithm with something like:
-      //   -crypto, +algorithm
-      // And if "+algorithm" does not occur, we disable that crypto algorithm.
-      if (!HasSM4)
-        Features.push_back("-sm4");
-      if (!HasSHA3)
-        Features.push_back("-sha3");
-      if (!HasSHA2)
-        Features.push_back("-sha2");
-      if (!HasAES)
-        Features.push_back("-aes");
-    }
-  } else {
-    if (HasCrypto && !HasNoCrypto) {
-      if (!HasNoSHA2)
-        Features.push_back("+sha2");
-      if (!HasNoAES)
-        Features.push_back("+aes");
-    } else if (HasNoCrypto) {
-      if (!HasSHA2)
-        Features.push_back("-sha2");
-      if (!HasAES)
-        Features.push_back("-aes");
-      if (V8Version == 2 || V8Version == 3) {
-        Features.push_back("-sm4");
-        Features.push_back("-sha3");
-      }
-    }
-  }
-
   if (Arg *A = Args.getLastArg(options::OPT_mno_unaligned_access,
                                options::OPT_munaligned_access)) {
     if (A->getOption().matches(options::OPT_mno_unaligned_access))
diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.cpp b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
index 25470db2b6ce..e6ee2f88a84e 100644
--- a/clang/lib/Driver/ToolChains/Arch/ARM.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
@@ -367,6 +367,7 @@ arm::FloatABI arm::getDefaultFloatABI(const llvm::Triple &Triple) {
   case llvm::Triple::IOS:
   case llvm::Triple::TvOS:
   case llvm::Triple::DriverKit:
+  case llvm::Triple::XROS:
     // Darwin defaults to "softfp" for v6 and v7.
     if (Triple.isWatchABI())
       return FloatABI::Hard;
@@ -836,8 +837,8 @@ llvm::ARM::FPUKind arm::getARMTargetFeatures(const Driver &D,
     if (A->getOption().matches(options::OPT_mlong_calls))
       Features.push_back("+long-calls");
   } else if (KernelOrKext && (!Triple.isiOS() || Triple.isOSVersionLT(6)) &&
-             !Triple.isWatchOS()) {
-      Features.push_back("+long-calls");
+             !Triple.isWatchOS() && !Triple.isXROS()) {
+    Features.push_back("+long-calls");
   }
 
   // Generate execute-only output (no data access to code sections).
diff --git a/clang/lib/Driver/ToolChains/Arch/Mips.cpp b/clang/lib/Driver/ToolChains/Arch/Mips.cpp
index f9f14c01b2b9..fe9d112b8800 100644
--- a/clang/lib/Driver/ToolChains/Arch/Mips.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/Mips.cpp
@@ -221,6 +221,7 @@ void mips::getMIPSTargetFeatures(const Driver &D, const llvm::Triple &Triple,
   bool IsN64 = ABIName == "64";
   bool IsPIC = false;
   bool NonPIC = false;
+  bool HasNaN2008Opt = false;
 
   Arg *LastPICArg = Args.getLastArg(options::OPT_fPIC, options::OPT_fno_PIC,
                                     options::OPT_fpic, options::OPT_fno_pic,
@@ -285,9 +286,10 @@ void mips::getMIPSTargetFeatures(const Driver &D, const llvm::Triple &Triple,
   if (Arg *A = Args.getLastArg(options::OPT_mnan_EQ)) {
     StringRef Val = StringRef(A->getValue());
     if (Val == "2008") {
-      if (mips::getIEEE754Standard(CPUName) & mips::Std2008)
+      if (mips::getIEEE754Standard(CPUName) & mips::Std2008) {
         Features.push_back("+nan2008");
-      else {
+        HasNaN2008Opt = true;
+      } else {
         Features.push_back("-nan2008");
         D.Diag(diag::warn_target_unsupported_nan2008) << CPUName;
       }
@@ -323,6 +325,8 @@ void mips::getMIPSTargetFeatures(const Driver &D, const llvm::Triple &Triple,
       D.Diag(diag::err_drv_unsupported_option_argument)
           << A->getSpelling() << Val;
     }
+  } else if (HasNaN2008Opt) {
+    Features.push_back("+abs2008");
   }
 
   AddTargetFeature(Args, Features, options::OPT_msingle_float,
diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index 0717e3b813e1..a46b44f9ad2b 100644
--- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -42,9 +42,9 @@ static bool getArchFeatures(const Driver &D, StringRef Arch,
     return false;
   }
 
-  (*ISAInfo)->toFeatures(
-      Features, [&Args](const Twine &Str) { return Args.MakeArgString(Str); },
-      /*AddAllExtensions=*/true);
+  for (const std::string &Str : (*ISAInfo)->toFeatures(/*AddAllExtension=*/true,
+                                                       /*IgnoreUnknown=*/false))
+    Features.push_back(Args.MakeArgString(Str));
 
   if (EnableExperimentalExtensions)
     Features.push_back(Args.MakeArgString("+experimental"));
@@ -210,6 +210,7 @@ StringRef riscv::getRISCVABI(const ArgList &Args, const llvm::Triple &Triple) {
   // rv32e -> ilp32e
   // rv32* -> ilp32
   // rv64g | rv64*d -> lp64d
+  // rv64e -> lp64e
   // rv64* -> lp64
   StringRef Arch = getRISCVArch(Args, Triple);
 
@@ -285,6 +286,7 @@ StringRef riscv::getRISCVArch(const llvm::opt::ArgList &Args,
   // 3. Choose a default based on `-mabi=`
   //
   // ilp32e -> rv32e
+  // lp64e -> rv64e
   // ilp32 | ilp32f | ilp32d -> rv32imafdc
   // lp64 | lp64f | lp64d -> rv64imafdc
   if (const Arg *A = Args.getLastArg(options::OPT_mabi_EQ)) {
@@ -292,6 +294,8 @@ StringRef riscv::getRISCVArch(const llvm::opt::ArgList &Args,
 
     if (MABI.equals_insensitive("ilp32e"))
       return "rv32e";
+    else if (MABI.equals_insensitive("lp64e"))
+      return "rv64e";
     else if (MABI.starts_with_insensitive("ilp32"))
       return "rv32imafdc";
     else if (MABI.starts_with_insensitive("lp64")) {
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index 42c8336e626c..391c47f88bde 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -293,9 +293,8 @@ void BareMetal::addClangTargetOptions(const ArgList &DriverArgs,
 
 void BareMetal::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
                                              ArgStringList &CC1Args) const {
-  if (DriverArgs.hasArg(options::OPT_nostdinc) ||
-      DriverArgs.hasArg(options::OPT_nostdlibinc) ||
-      DriverArgs.hasArg(options::OPT_nostdincxx))
+  if (DriverArgs.hasArg(options::OPT_nostdinc, options::OPT_nostdlibinc,
+                        options::OPT_nostdincxx))
     return;
 
   const Driver &D = getDriver();
diff --git a/clang/lib/Driver/ToolChains/CSKYToolChain.cpp b/clang/lib/Driver/ToolChains/CSKYToolChain.cpp
index 0c280347b2af..feb3bc922920 100644
--- a/clang/lib/Driver/ToolChains/CSKYToolChain.cpp
+++ b/clang/lib/Driver/ToolChains/CSKYToolChain.cpp
@@ -125,7 +125,7 @@ std::string CSKYToolChain::computeSysRoot() const {
   if (!llvm::sys::fs::exists(SysRootDir))
     return std::string();
 
-  return std::string(SysRootDir.str());
+  return std::string(SysRootDir);
 }
 
 void CSKY::Linker::ConstructJob(Compilation &C, const JobAction &JA,
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index acfa11980506..5dc614e11aab 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -43,7 +43,9 @@
 #include "clang/Driver/XRayArgs.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/BinaryFormat/Magic.h"
 #include "llvm/Config/llvm-config.h"
+#include "llvm/Object/ObjectFile.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/Compiler.h"
@@ -698,6 +700,17 @@ static void addPGOAndCoverageFlags(const ToolChain &TC, Compilation &C,
     CmdArgs.push_back("-fcoverage-mapping");
   }
 
+  if (Args.hasFlag(options::OPT_fmcdc_coverage, options::OPT_fno_mcdc_coverage,
+                   false)) {
+    if (!Args.hasFlag(options::OPT_fcoverage_mapping,
+                      options::OPT_fno_coverage_mapping, false))
+      D.Diag(clang::diag::err_drv_argument_only_allowed_with)
+          << "-fcoverage-mcdc"
+          << "-fcoverage-mapping";
+
+    CmdArgs.push_back("-fcoverage-mcdc");
+  }
+
   if (Arg *A = Args.getLastArg(options::OPT_ffile_compilation_dir_EQ,
                                options::OPT_fcoverage_compilation_dir_EQ)) {
     if (A->getOption().matches(options::OPT_ffile_compilation_dir_EQ))
@@ -937,11 +950,21 @@ static void handleAMDGPUCodeObjectVersionOptions(const Driver &D,
   }
 }
 
-static bool hasClangPchSignature(const Driver &D, StringRef Path) {
-  if (llvm::ErrorOr> MemBuf =
-          D.getVFS().getBufferForFile(Path))
-    return (*MemBuf)->getBuffer().starts_with("CPCH");
-  return false;
+static bool maybeHasClangPchSignature(const Driver &D, StringRef Path) {
+  llvm::ErrorOr> MemBuf =
+      D.getVFS().getBufferForFile(Path);
+  if (!MemBuf)
+    return false;
+  llvm::file_magic Magic = llvm::identify_magic((*MemBuf)->getBuffer());
+  if (Magic == llvm::file_magic::unknown)
+    return false;
+  // Return true for both raw Clang AST files and object files which may
+  // contain a __clangast section.
+  if (Magic == llvm::file_magic::clang_ast)
+    return true;
+  Expected> Obj =
+      llvm::object::ObjectFile::createObjectFile(**MemBuf, Magic);
+  return !Obj.takeError();
 }
 
 static bool gchProbe(const Driver &D, StringRef Path) {
@@ -953,14 +976,14 @@ static bool gchProbe(const Driver &D, StringRef Path) {
     std::error_code EC;
     for (llvm::vfs::directory_iterator DI = D.getVFS().dir_begin(Path, EC), DE;
          !EC && DI != DE; DI = DI.increment(EC)) {
-      if (hasClangPchSignature(D, DI->path()))
+      if (maybeHasClangPchSignature(D, DI->path()))
         return true;
     }
     D.Diag(diag::warn_drv_pch_ignoring_gch_dir) << Path;
     return false;
   }
 
-  if (hasClangPchSignature(D, Path))
+  if (maybeHasClangPchSignature(D, Path))
     return true;
   D.Diag(diag::warn_drv_pch_ignoring_gch_file) << Path;
   return false;
@@ -1497,7 +1520,7 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args,
         << Triple.getArchName();
 
   StringRef Scope, Key;
-  bool IndirectBranches, BranchProtectionPAuthLR;
+  bool IndirectBranches, BranchProtectionPAuthLR, GuardedControlStack;
 
   if (A->getOption().matches(options::OPT_msign_return_address_EQ)) {
     Scope = A->getValue();
@@ -1507,6 +1530,7 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args,
     Key = "a_key";
     IndirectBranches = false;
     BranchProtectionPAuthLR = false;
+    GuardedControlStack = false;
   } else {
     StringRef DiagMsg;
     llvm::ARM::ParsedBranchProtection PBP;
@@ -1520,6 +1544,7 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args,
     Key = PBP.Key;
     BranchProtectionPAuthLR = PBP.BranchProtectionPAuthLR;
     IndirectBranches = PBP.BranchTargetEnforcement;
+    GuardedControlStack = PBP.GuardedControlStack;
   }
 
   CmdArgs.push_back(
@@ -1532,6 +1557,8 @@ static void CollectARMPACBTIOptions(const ToolChain &TC, const ArgList &Args,
         Args.MakeArgString(Twine("-mbranch-protection-pauth-lr")));
   if (IndirectBranches)
     CmdArgs.push_back("-mbranch-target-enforce");
+  if (GuardedControlStack)
+    CmdArgs.push_back("-mguarded-control-stack");
 }
 
 void Clang::AddARMTargetArgs(const llvm::Triple &Triple, const ArgList &Args,
@@ -2685,9 +2712,22 @@ static void EmitComplexRangeDiag(const Driver &D,
         << EnumComplexRangeToStr(Range1) << EnumComplexRangeToStr(Range2);
 }
 
-static std::string RenderComplexRangeOption(std::string Range) {
+static std::string
+RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) {
   std::string ComplexRangeStr = "-complex-range=";
-  ComplexRangeStr += Range;
+  switch (Range) {
+  case LangOptions::ComplexRangeKind::CX_Full:
+    ComplexRangeStr += "full";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Limited:
+    ComplexRangeStr += "limited";
+    break;
+  case LangOptions::ComplexRangeKind::CX_Fortran:
+    ComplexRangeStr += "fortran";
+    break;
+  default:
+    assert(0 && "Unexpected range option");
+  }
   return ComplexRangeStr;
 }
 
@@ -2737,7 +2777,8 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
   bool StrictFPModel = false;
   StringRef Float16ExcessPrecision = "";
   StringRef BFloat16ExcessPrecision = "";
-  LangOptions::ComplexRangeKind Range = LangOptions::ComplexRangeKind::CX_Full;
+  LangOptions::ComplexRangeKind Range = LangOptions::ComplexRangeKind::CX_None;
+  std::string ComplexRangeStr = "";
 
   if (const Arg *A = Args.getLastArg(options::OPT_flimited_precision_EQ)) {
     CmdArgs.push_back("-mlimit-float-precision");
@@ -2753,23 +2794,19 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
     case options::OPT_fcx_limited_range: {
       EmitComplexRangeDiag(D, Range, LangOptions::ComplexRangeKind::CX_Limited);
       Range = LangOptions::ComplexRangeKind::CX_Limited;
-      std::string ComplexRangeStr = RenderComplexRangeOption("limited");
-      if (!ComplexRangeStr.empty())
-        CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr));
       break;
     }
     case options::OPT_fno_cx_limited_range:
+      EmitComplexRangeDiag(D, Range, LangOptions::ComplexRangeKind::CX_Full);
       Range = LangOptions::ComplexRangeKind::CX_Full;
       break;
     case options::OPT_fcx_fortran_rules: {
       EmitComplexRangeDiag(D, Range, LangOptions::ComplexRangeKind::CX_Fortran);
       Range = LangOptions::ComplexRangeKind::CX_Fortran;
-      std::string ComplexRangeStr = RenderComplexRangeOption("fortran");
-      if (!ComplexRangeStr.empty())
-        CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr));
       break;
     }
     case options::OPT_fno_cx_fortran_rules:
+      EmitComplexRangeDiag(D, Range, LangOptions::ComplexRangeKind::CX_Full);
       Range = LangOptions::ComplexRangeKind::CX_Full;
       break;
     case options::OPT_ffp_model_EQ: {
@@ -3041,9 +3078,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
       SeenUnsafeMathModeOption = true;
       // ffast-math enables fortran rules for complex multiplication and
       // division.
-      std::string ComplexRangeStr = RenderComplexRangeOption("limited");
-      if (!ComplexRangeStr.empty())
-        CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr));
+      Range = LangOptions::ComplexRangeKind::CX_Limited;
       break;
     }
     case options::OPT_fno_fast_math:
@@ -3200,6 +3235,10 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D,
                    options::OPT_fstrict_float_cast_overflow, false))
     CmdArgs.push_back("-fno-strict-float-cast-overflow");
 
+  if (Range != LangOptions::ComplexRangeKind::CX_None)
+    ComplexRangeStr = RenderComplexRangeOption(Range);
+  if (!ComplexRangeStr.empty())
+    CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr));
   if (Args.hasArg(options::OPT_fcx_limited_range))
     CmdArgs.push_back("-fcx-limited-range");
   if (Args.hasArg(options::OPT_fcx_fortran_rules))
@@ -3508,6 +3547,20 @@ static void RenderTrivialAutoVarInitOptions(const Driver &D,
     CmdArgs.push_back(
         Args.MakeArgString("-ftrivial-auto-var-init-stop-after=" + Val));
   }
+
+  if (Arg *A = Args.getLastArg(options::OPT_ftrivial_auto_var_init_max_size)) {
+    if (!Args.hasArg(options::OPT_ftrivial_auto_var_init) ||
+        StringRef(
+            Args.getLastArg(options::OPT_ftrivial_auto_var_init)->getValue()) ==
+            "uninitialized")
+      D.Diag(diag::err_drv_trivial_auto_var_init_max_size_missing_dependency);
+    A->claim();
+    StringRef Val = A->getValue();
+    if (std::stoi(Val.str()) <= 0)
+      D.Diag(diag::err_drv_trivial_auto_var_init_max_size_invalid_value);
+    CmdArgs.push_back(
+        Args.MakeArgString("-ftrivial-auto-var-init-max-size=" + Val));
+  }
 }
 
 static void RenderOpenCLOptions(const ArgList &Args, ArgStringList &CmdArgs,
@@ -4801,9 +4854,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   bool UnifiedLTO = false;
   if (IsUsingLTO) {
     UnifiedLTO = Args.hasFlag(options::OPT_funified_lto,
-                              options::OPT_fno_unified_lto, Triple.isPS()) ||
-                 Args.hasFlag(options::OPT_ffat_lto_objects,
-                              options::OPT_fno_fat_lto_objects, false);
+                              options::OPT_fno_unified_lto, Triple.isPS());
     if (UnifiedLTO)
       CmdArgs.push_back("-funified-lto");
   }
@@ -5737,20 +5788,24 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
-  if (Arg *A = Args.getLastArg(options::OPT_mlarge_data_threshold_EQ)) {
-    if (!Triple.isX86()) {
-      D.Diag(diag::err_drv_unsupported_opt_for_target)
-          << A->getOption().getName() << TripleStr;
-    } else {
-      bool IsMediumCM = false;
-      if (Arg *A = Args.getLastArg(options::OPT_mcmodel_EQ))
-        IsMediumCM = StringRef(A->getValue()) == "medium";
-      if (!IsMediumCM) {
+  if (Triple.getArch() == llvm::Triple::x86_64) {
+    bool IsMediumCM = false;
+    bool IsLargeCM = false;
+    if (Arg *A = Args.getLastArg(options::OPT_mcmodel_EQ)) {
+      IsMediumCM = StringRef(A->getValue()) == "medium";
+      IsLargeCM = StringRef(A->getValue()) == "large";
+    }
+    if (Arg *A = Args.getLastArg(options::OPT_mlarge_data_threshold_EQ)) {
+      if (!IsMediumCM && !IsLargeCM) {
         D.Diag(diag::warn_drv_large_data_threshold_invalid_code_model)
             << A->getOption().getRenderName();
       } else {
         A->render(Args, CmdArgs);
       }
+    } else if (IsMediumCM) {
+      CmdArgs.push_back("-mlarge-data-threshold=65536");
+    } else if (IsLargeCM) {
+      CmdArgs.push_back("-mlarge-data-threshold=0");
     }
   }
 
@@ -6280,7 +6335,28 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddLastArg(CmdArgs, options::OPT_fvisibility_inlines_hidden_static_local_var,
                            options::OPT_fno_visibility_inlines_hidden_static_local_var);
-  Args.AddLastArg(CmdArgs, options::OPT_fvisibility_global_new_delete_hidden);
+
+  // -fvisibility-global-new-delete-hidden is a deprecated spelling of
+  // -fvisibility-global-new-delete=force-hidden.
+  if (const Arg *A =
+          Args.getLastArg(options::OPT_fvisibility_global_new_delete_hidden)) {
+    D.Diag(diag::warn_drv_deprecated_arg)
+        << A->getAsString(Args)
+        << "-fvisibility-global-new-delete=force-hidden";
+  }
+
+  if (const Arg *A =
+          Args.getLastArg(options::OPT_fvisibility_global_new_delete_EQ,
+                          options::OPT_fvisibility_global_new_delete_hidden)) {
+    if (A->getOption().matches(options::OPT_fvisibility_global_new_delete_EQ)) {
+      A->render(Args, CmdArgs);
+    } else {
+      assert(A->getOption().matches(
+          options::OPT_fvisibility_global_new_delete_hidden));
+      CmdArgs.push_back("-fvisibility-global-new-delete=force-hidden");
+    }
+  }
+
   Args.AddLastArg(CmdArgs, options::OPT_ftlsmodel_EQ);
 
   if (Args.hasFlag(options::OPT_fnew_infallible,
@@ -6382,6 +6458,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
         CmdArgs.push_back("-fopenmp-assume-no-nested-parallelism");
       if (Args.hasArg(options::OPT_fopenmp_offload_mandatory))
         CmdArgs.push_back("-fopenmp-offload-mandatory");
+      if (Args.hasArg(options::OPT_fopenmp_force_usm))
+        CmdArgs.push_back("-fopenmp-force-usm");
       break;
     default:
       // By default, if Clang doesn't know how to generate useful OpenMP code
@@ -8669,7 +8747,7 @@ void OffloadPackager::ConstructJob(Compilation &C, const JobAction &JA,
     SmallVector Parts{
         "file=" + File.str(),
         "triple=" + TC->getTripleString(),
-        "arch=" + getProcessorFromTargetID(TC->getTriple(), Arch).str(),
+        "arch=" + Arch.str(),
         "kind=" + Kind.str(),
     };
 
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index e431c31af1b6..eb03560afc76 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -741,6 +741,8 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
   const bool IsAMDGCN = ToolChain.getTriple().isAMDGCN();
   const char *Linker = Args.MakeArgString(ToolChain.GetLinkerPath());
   const Driver &D = ToolChain.getDriver();
+  const bool IsFatLTO = Args.hasArg(options::OPT_ffat_lto_objects);
+  const bool IsUnifiedLTO = Args.hasArg(options::OPT_funified_lto);
   if (llvm::sys::path::filename(Linker) != "ld.lld" &&
       llvm::sys::path::stem(Linker) != "ld.lld" &&
       !ToolChain.getTriple().isOSOpenBSD()) {
@@ -770,7 +772,7 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
   } else {
     // Tell LLD to find and use .llvm.lto section in regular relocatable object
     // files
-    if (Args.hasArg(options::OPT_ffat_lto_objects))
+    if (IsFatLTO)
       CmdArgs.push_back("--fat-lto-objects");
   }
 
@@ -830,7 +832,8 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
   // Matrix intrinsic lowering happens at link time with ThinLTO. Enable
   // LowerMatrixIntrinsicsPass, which is transitively called by
   // buildThinLTODefaultPipeline under EnableMatrix.
-  if (IsThinLTO && Args.hasArg(options::OPT_fenable_matrix))
+  if ((IsThinLTO || IsFatLTO || IsUnifiedLTO) &&
+        Args.hasArg(options::OPT_fenable_matrix))
     CmdArgs.push_back(
         Args.MakeArgString(Twine(PluginOptPrefix) + "-enable-matrix"));
 
@@ -1196,6 +1199,18 @@ static void addFortranMain(const ToolChain &TC, const ArgList &Args,
   }
 
   // 2. GNU and similar
+  const Driver &D = TC.getDriver();
+  const char *FortranMainLinkFlag = "-lFortran_main";
+
+  // Warn if the user added `-lFortran_main` - this library is an implementation
+  // detail of Flang and should be handled automaticaly by the driver.
+  for (const char *arg : CmdArgs) {
+    if (strncmp(arg, FortranMainLinkFlag, strlen(FortranMainLinkFlag)) == 0)
+      D.Diag(diag::warn_drv_deprecated_custom)
+          << FortranMainLinkFlag
+          << "see the Flang driver documentation for correct usage";
+  }
+
   // The --whole-archive option needs to be part of the link line to make
   // sure that the main() function from Fortran_main.a is pulled in by the
   // linker. However, it shouldn't be used if it's already active.
@@ -1203,12 +1218,12 @@ static void addFortranMain(const ToolChain &TC, const ArgList &Args,
   if (!isWholeArchivePresent(Args) && !TC.getTriple().isMacOSX() &&
       !TC.getTriple().isOSAIX()) {
     CmdArgs.push_back("--whole-archive");
-    CmdArgs.push_back("-lFortran_main");
+    CmdArgs.push_back(FortranMainLinkFlag);
     CmdArgs.push_back("--no-whole-archive");
     return;
   }
 
-  CmdArgs.push_back("-lFortran_main");
+  CmdArgs.push_back(FortranMainLinkFlag);
 }
 
 /// Add Fortran runtime libs
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index f7a208575cb0..1462576ca870 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -894,7 +894,7 @@ std::string CudaToolChain::getInputFilename(const InputInfo &Input) const {
   // these particular file names.
   SmallString<256> Filename(ToolChain::getInputFilename(Input));
   llvm::sys::path::replace_extension(Filename, "cubin");
-  return std::string(Filename.str());
+  return std::string(Filename);
 }
 
 llvm::opt::DerivedArgList *
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index 65846cace461..fae8ad1a958a 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -219,7 +219,8 @@ static bool shouldLinkerNotDedup(bool IsLinkerOnlyAction, const ArgList &Args) {
 void darwin::Linker::AddLinkArgs(Compilation &C, const ArgList &Args,
                                  ArgStringList &CmdArgs,
                                  const InputInfoList &Inputs,
-                                 VersionTuple Version, bool LinkerIsLLD) const {
+                                 VersionTuple Version, bool LinkerIsLLD,
+                                 bool UsePlatformVersion) const {
   const Driver &D = getToolChain().getDriver();
   const toolchains::MachO &MachOTC = getMachOToolChain();
 
@@ -355,7 +356,7 @@ void darwin::Linker::AddLinkArgs(Compilation &C, const ArgList &Args,
   Args.AddAllArgs(CmdArgs, options::OPT_init);
 
   // Add the deployment target.
-  if (Version >= VersionTuple(520) || LinkerIsLLD)
+  if (Version >= VersionTuple(520) || LinkerIsLLD || UsePlatformVersion)
     MachOTC.addPlatformVersionArgs(Args, CmdArgs);
   else
     MachOTC.addMinVersionArgs(Args, CmdArgs);
@@ -596,9 +597,13 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   const char *Exec =
       Args.MakeArgString(getToolChain().GetLinkerPath(&LinkerIsLLD));
 
+  // xrOS always uses -platform-version.
+  bool UsePlatformVersion = getToolChain().getTriple().isXROS();
+
   // I'm not sure why this particular decomposition exists in gcc, but
   // we follow suite for ease of comparison.
-  AddLinkArgs(C, Args, CmdArgs, Inputs, Version, LinkerIsLLD);
+  AddLinkArgs(C, Args, CmdArgs, Inputs, Version, LinkerIsLLD,
+              UsePlatformVersion);
 
   if (willEmitRemarks(Args) &&
       checkRemarksOptions(getToolChain().getDriver(), Args,
@@ -954,6 +959,13 @@ ObjCRuntime Darwin::getDefaultObjCRuntime(bool isNonFragile) const {
     return ObjCRuntime(ObjCRuntime::WatchOS, TargetVersion);
   if (isTargetIOSBased())
     return ObjCRuntime(ObjCRuntime::iOS, TargetVersion);
+  if (isTargetXROS()) {
+    // XROS uses the iOS runtime.
+    auto T = llvm::Triple(Twine("arm64-apple-") +
+                          llvm::Triple::getOSTypeName(llvm::Triple::XROS) +
+                          TargetVersion.getAsString());
+    return ObjCRuntime(ObjCRuntime::iOS, T.getiOSVersion());
+  }
   if (isNonFragile)
     return ObjCRuntime(ObjCRuntime::MacOSX, TargetVersion);
   return ObjCRuntime(ObjCRuntime::FragileMacOSX, TargetVersion);
@@ -961,7 +973,7 @@ ObjCRuntime Darwin::getDefaultObjCRuntime(bool isNonFragile) const {
 
 /// Darwin provides a blocks runtime starting in MacOS X 10.6 and iOS 3.2.
 bool Darwin::hasBlocksRuntime() const {
-  if (isTargetWatchOSBased() || isTargetDriverKit())
+  if (isTargetWatchOSBased() || isTargetDriverKit() || isTargetXROS())
     return true;
   else if (isTargetIOSBased())
     return !isIPhoneOSVersionLT(3, 2);
@@ -973,12 +985,12 @@ bool Darwin::hasBlocksRuntime() const {
 
 void Darwin::AddCudaIncludeArgs(const ArgList &DriverArgs,
                                 ArgStringList &CC1Args) const {
-  CudaInstallation.AddCudaIncludeArgs(DriverArgs, CC1Args);
+  CudaInstallation->AddCudaIncludeArgs(DriverArgs, CC1Args);
 }
 
 void Darwin::AddHIPIncludeArgs(const ArgList &DriverArgs,
                                ArgStringList &CC1Args) const {
-  RocmInstallation.AddHIPIncludeArgs(DriverArgs, CC1Args);
+  RocmInstallation->AddHIPIncludeArgs(DriverArgs, CC1Args);
 }
 
 // This is just a MachO name translation routine and there's no
@@ -1092,6 +1104,8 @@ std::string Darwin::ComputeEffectiveClangTriple(const ArgList &Args,
     Str += "driverkit";
   else if (isTargetIOSBased() || isTargetMacCatalyst())
     Str += "ios";
+  else if (isTargetXROS())
+    Str += llvm::Triple::getOSTypeName(llvm::Triple::XROS);
   else
     Str += "macosx";
   Str += getTripleTargetVersion().getAsString();
@@ -1174,6 +1188,8 @@ void DarwinClang::AddLinkARCArgs(const ArgList &Args,
   // ARC runtime is supported everywhere on arm64e.
   if (getTriple().isArm64e())
     return;
+  if (isTargetXROS())
+    return;
 
   ObjCRuntime runtime = getDefaultObjCRuntime(/*nonfragile*/ true);
 
@@ -1309,6 +1325,8 @@ StringRef Darwin::getPlatformFamily() const {
       return "Watch";
     case DarwinPlatformKind::DriverKit:
       return "DriverKit";
+    case DarwinPlatformKind::XROS:
+      return "XR";
   }
   llvm_unreachable("Unsupported platform");
 }
@@ -1340,6 +1358,9 @@ StringRef Darwin::getOSLibraryNameSuffix(bool IgnoreSim) const {
   case DarwinPlatformKind::WatchOS:
     return TargetEnvironment == NativeEnvironment || IgnoreSim ? "watchos"
                                                                : "watchossim";
+  case DarwinPlatformKind::XROS:
+    return TargetEnvironment == NativeEnvironment || IgnoreSim ? "xros"
+                                                               : "xrossim";
   case DarwinPlatformKind::DriverKit:
     return "driverkit";
   }
@@ -1649,6 +1670,9 @@ struct DarwinPlatform {
     case DarwinPlatformKind::WatchOS:
       Opt = options::OPT_mwatchos_version_min_EQ;
       break;
+    case DarwinPlatformKind::XROS:
+      // xrOS always explicitly provides a version in the triple.
+      return;
     case DarwinPlatformKind::DriverKit:
       // DriverKit always explicitly provides a version in the triple.
       return;
@@ -1794,6 +1818,8 @@ struct DarwinPlatform {
       return DarwinPlatformKind::TvOS;
     case llvm::Triple::WatchOS:
       return DarwinPlatformKind::WatchOS;
+    case llvm::Triple::XROS:
+      return DarwinPlatformKind::XROS;
     case llvm::Triple::DriverKit:
       return DarwinPlatformKind::DriverKit;
     default:
@@ -1975,6 +2001,10 @@ inferDeploymentTargetFromSDK(DerivedArgList &Args,
       return DarwinPlatform::createFromSDK(
           Darwin::TvOS, Version,
           /*IsSimulator=*/SDK.starts_with("AppleTVSimulator"));
+    else if (SDK.starts_with("XR"))
+      return DarwinPlatform::createFromSDK(
+          Darwin::XROS, Version,
+          /*IsSimulator=*/SDK.contains("Simulator"));
     else if (SDK.starts_with("DriverKit"))
       return DarwinPlatform::createFromSDK(Darwin::DriverKit, Version);
     return std::nullopt;
@@ -2013,6 +2043,11 @@ std::string getOSVersion(llvm::Triple::OSType OS, const llvm::Triple &Triple,
   case llvm::Triple::WatchOS:
     OsVersion = Triple.getWatchOSVersion();
     break;
+  case llvm::Triple::XROS:
+    OsVersion = Triple.getOSVersion();
+    if (!OsVersion.getMajor())
+      OsVersion = OsVersion.withMajorReplaced(1);
+    break;
   case llvm::Triple::DriverKit:
     OsVersion = Triple.getDriverKitVersion();
     break;
@@ -2104,6 +2139,7 @@ std::optional getDeploymentTargetFromMTargetOSArg(
   case llvm::Triple::IOS:
   case llvm::Triple::TvOS:
   case llvm::Triple::WatchOS:
+  case llvm::Triple::XROS:
     break;
   default:
     TheDriver.Diag(diag::err_drv_invalid_os_in_arg)
@@ -2319,6 +2355,13 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
         Micro >= 100)
       getDriver().Diag(diag::err_drv_invalid_version_number)
           << OSTarget->getAsString(Args, Opts);
+  } else if (Platform == XROS) {
+    if (!Driver::GetReleaseVersion(OSTarget->getOSVersion(), Major, Minor,
+                                   Micro, HadExtra) ||
+        HadExtra || Major < 1 || Major >= MajorVersionLimit || Minor >= 100 ||
+        Micro >= 100)
+      getDriver().Diag(diag::err_drv_invalid_version_number)
+          << OSTarget->getAsString(Args, Opts);
   } else
     llvm_unreachable("unknown kind of Darwin platform");
 
@@ -2641,6 +2684,10 @@ void DarwinClang::AddCCKextLibArgs(const ArgList &Args,
     llvm::sys::path::append(P, "libclang_rt.cc_kext_ios.a");
   } else if (isTargetDriverKit()) {
     // DriverKit doesn't want extra runtime support.
+  } else if (isTargetXROSDevice()) {
+    llvm::sys::path::append(
+        P, llvm::Twine("libclang_rt.cc_kext_") +
+               llvm::Triple::getOSTypeName(llvm::Triple::XROS) + ".a");
   } else {
     llvm::sys::path::append(P, "libclang_rt.cc_kext.a");
   }
@@ -2855,6 +2902,8 @@ bool Darwin::isAlignedAllocationUnavailable() const {
   case WatchOS: // Earlier than 4.0.
     OS = llvm::Triple::WatchOS;
     break;
+  case XROS: // Always available.
+    return false;
   case DriverKit: // Always available.
     return false;
   }
@@ -2876,6 +2925,8 @@ static bool sdkSupportsBuiltinModules(const Darwin::DarwinPlatformKind &TargetPl
     return SDKVersion >= VersionTuple(99U);
   case Darwin::WatchOS:
     return SDKVersion >= VersionTuple(99U);
+  case Darwin::XROS:
+    return SDKVersion >= VersionTuple(99U);
   default:
     return true;
   }
@@ -2917,6 +2968,10 @@ void Darwin::addClangTargetOptions(const llvm::opt::ArgList &DriverArgs,
   // to fix the same problem with C++ headers, and is generally fragile.
   if (!sdkSupportsBuiltinModules(TargetPlatform, SDKInfo))
     CC1Args.push_back("-fbuiltin-headers-in-system-modules");
+
+  if (!DriverArgs.hasArgNoClaim(options::OPT_fdefine_target_os_macros,
+                                options::OPT_fno_define_target_os_macros))
+    CC1Args.push_back("-fdefine-target-os-macros");
 }
 
 void Darwin::addClangCC1ASTargetOptions(
@@ -2994,7 +3049,7 @@ Darwin::TranslateArgs(const DerivedArgList &Args, StringRef BoundArch,
   // FIXME: It would be far better to avoid inserting those -static arguments,
   // but we can't check the deployment target in the translation code until
   // it is set here.
-  if (isTargetWatchOSBased() || isTargetDriverKit() ||
+  if (isTargetWatchOSBased() || isTargetDriverKit() || isTargetXROS() ||
       (isTargetIOSBased() && !isIPhoneOSVersionLT(6, 0))) {
     for (ArgList::iterator it = DAL->begin(), ie = DAL->end(); it != ie; ) {
       Arg *A = *it;
@@ -3088,6 +3143,8 @@ void Darwin::addMinVersionArgs(const ArgList &Args,
                                ArgStringList &CmdArgs) const {
   VersionTuple TargetVersion = getTripleTargetVersion();
 
+  assert(!isTargetXROS() && "xrOS always uses -platform-version");
+
   if (isTargetWatchOS())
     CmdArgs.push_back("-watchos_version_min");
   else if (isTargetWatchOSSimulator())
@@ -3147,6 +3204,8 @@ static const char *getPlatformName(Darwin::DarwinPlatformKind Platform,
     return "tvos";
   case Darwin::WatchOS:
     return "watchos";
+  case Darwin::XROS:
+    return "xros";
   case Darwin::DriverKit:
     return "driverkit";
   }
@@ -3343,7 +3402,7 @@ void Darwin::addStartObjectFileArgs(const ArgList &Args,
 }
 
 void Darwin::CheckObjCARC() const {
-  if (isTargetIOSBased() || isTargetWatchOSBased() ||
+  if (isTargetIOSBased() || isTargetWatchOSBased() || isTargetXROS() ||
       (isTargetMacOSBased() && !isMacosxVersionLT(10, 6)))
     return;
   getDriver().Diag(diag::err_arc_unsupported_on_toolchain);
@@ -3377,6 +3436,6 @@ SanitizerMask Darwin::getSupportedSanitizers() const {
 }
 
 void Darwin::printVerboseInfo(raw_ostream &OS) const {
-  CudaInstallation.print(OS);
-  RocmInstallation.print(OS);
+  CudaInstallation->print(OS);
+  RocmInstallation->print(OS);
 }
diff --git a/clang/lib/Driver/ToolChains/Darwin.h b/clang/lib/Driver/ToolChains/Darwin.h
index 815449ae8f37..5e60b0841d6d 100644
--- a/clang/lib/Driver/ToolChains/Darwin.h
+++ b/clang/lib/Driver/ToolChains/Darwin.h
@@ -10,6 +10,7 @@
 #define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_DARWIN_H
 
 #include "Cuda.h"
+#include "LazyDetector.h"
 #include "ROCm.h"
 #include "clang/Basic/DarwinSDKInfo.h"
 #include "clang/Basic/LangOptions.h"
@@ -65,7 +66,7 @@ class LLVM_LIBRARY_VISIBILITY Linker : public MachOTool {
   void AddLinkArgs(Compilation &C, const llvm::opt::ArgList &Args,
                    llvm::opt::ArgStringList &CmdArgs,
                    const InputInfoList &Inputs, VersionTuple Version,
-                   bool LinkerIsLLD) const;
+                   bool LinkerIsLLD, bool UsePlatformVersion) const;
 
 public:
   Linker(const ToolChain &TC) : MachOTool("darwin::Linker", "linker", TC) {}
@@ -298,6 +299,7 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public MachO {
     TvOS,
     WatchOS,
     DriverKit,
+    XROS,
     LastDarwinPlatform = DriverKit
   };
   enum DarwinEnvironmentKind {
@@ -320,8 +322,8 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public MachO {
   /// The target variant triple that was specified (if any).
   mutable std::optional TargetVariantTriple;
 
-  CudaInstallationDetector CudaInstallation;
-  RocmInstallationDetector RocmInstallation;
+  LazyDetector CudaInstallation;
+  LazyDetector RocmInstallation;
 
 private:
   void AddDeploymentTarget(llvm::opt::DerivedArgList &Args) const;
@@ -404,6 +406,16 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public MachO {
     return isTargetIPhoneOS() || isTargetIOSSimulator();
   }
 
+  bool isTargetXROSDevice() const {
+    return TargetPlatform == XROS && TargetEnvironment == NativeEnvironment;
+  }
+
+  bool isTargetXROSSimulator() const {
+    return TargetPlatform == XROS && TargetEnvironment == Simulator;
+  }
+
+  bool isTargetXROS() const { return TargetPlatform == XROS; }
+
   bool isTargetTvOS() const {
     assert(TargetInitialized && "Target not initialized!");
     return TargetPlatform == TvOS && TargetEnvironment == NativeEnvironment;
@@ -546,7 +558,8 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public MachO {
   GetDefaultStackProtectorLevel(bool KernelOrKext) const override {
     // Stack protectors default to on for user code on 10.5,
     // and for everything in 10.6 and beyond
-    if (isTargetIOSBased() || isTargetWatchOSBased() || isTargetDriverKit())
+    if (isTargetIOSBased() || isTargetWatchOSBased() || isTargetDriverKit() ||
+        isTargetXROS())
       return LangOptions::SSPOn;
     else if (isTargetMacOSBased() && !isMacosxVersionLT(10, 6))
       return LangOptions::SSPOn;
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 41eaad3bbad0..03d68c3df7fb 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "Flang.h"
+#include "Arch/RISCV.h"
 #include "CommonArgs.h"
 
 #include "clang/Basic/CodeGenOptions.h"
@@ -14,6 +15,8 @@
 #include "llvm/Frontend/Debug/Options.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/RISCVISAInfo.h"
+#include "llvm/TargetParser/RISCVTargetParser.h"
 
 #include 
 
@@ -203,6 +206,51 @@ void Flang::AddAArch64TargetArgs(const ArgList &Args,
   }
 }
 
+void Flang::AddRISCVTargetArgs(const ArgList &Args,
+                               ArgStringList &CmdArgs) const {
+  const llvm::Triple &Triple = getToolChain().getTriple();
+  // Handle -mrvv-vector-bits=
+  if (Arg *A = Args.getLastArg(options::OPT_mrvv_vector_bits_EQ)) {
+    StringRef Val = A->getValue();
+    const Driver &D = getToolChain().getDriver();
+
+    // Get minimum VLen from march.
+    unsigned MinVLen = 0;
+    StringRef Arch = riscv::getRISCVArch(Args, Triple);
+    auto ISAInfo = llvm::RISCVISAInfo::parseArchString(
+        Arch, /*EnableExperimentalExtensions*/ true);
+    // Ignore parsing error.
+    if (!errorToBool(ISAInfo.takeError()))
+      MinVLen = (*ISAInfo)->getMinVLen();
+
+    // If the value is "zvl", use MinVLen from march. Otherwise, try to parse
+    // as integer as long as we have a MinVLen.
+    unsigned Bits = 0;
+    if (Val.equals("zvl") && MinVLen >= llvm::RISCV::RVVBitsPerBlock) {
+      Bits = MinVLen;
+    } else if (!Val.getAsInteger(10, Bits)) {
+      // Only accept power of 2 values beteen RVVBitsPerBlock and 65536 that
+      // at least MinVLen.
+      if (Bits < MinVLen || Bits < llvm::RISCV::RVVBitsPerBlock ||
+          Bits > 65536 || !llvm::isPowerOf2_32(Bits))
+        Bits = 0;
+    }
+
+    // If we got a valid value try to use it.
+    if (Bits != 0) {
+      unsigned VScaleMin = Bits / llvm::RISCV::RVVBitsPerBlock;
+      CmdArgs.push_back(
+          Args.MakeArgString("-mvscale-max=" + llvm::Twine(VScaleMin)));
+      CmdArgs.push_back(
+          Args.MakeArgString("-mvscale-min=" + llvm::Twine(VScaleMin)));
+    } else if (!Val.equals("scalable")) {
+      // Handle the unsupported values passed to mrvv-vector-bits.
+      D.Diag(diag::err_drv_unsupported_option_argument)
+          << A->getSpelling() << Val;
+    }
+  }
+}
+
 static void addVSDefines(const ToolChain &TC, const ArgList &Args,
                          ArgStringList &CmdArgs) {
 
@@ -321,6 +369,9 @@ void Flang::addTargetOptions(const ArgList &Args,
     AddAMDGPUTargetArgs(Args, CmdArgs);
     break;
   case llvm::Triple::riscv64:
+    getTargetFeatures(D, Triple, Args, CmdArgs, /*ForAs*/ false);
+    AddRISCVTargetArgs(Args, CmdArgs);
+    break;
   case llvm::Triple::x86_64:
     getTargetFeatures(D, Triple, Args, CmdArgs, /*ForAs*/ false);
     break;
@@ -352,12 +403,10 @@ void Flang::addTargetOptions(const ArgList &Args,
         if (A->getValue() == StringRef{"Accelerate"}) {
           CmdArgs.push_back("-framework");
           CmdArgs.push_back("Accelerate");
-          A->render(Args, CmdArgs);
         }
       }
-    } else {
-      A->render(Args, CmdArgs);
     }
+    A->render(Args, CmdArgs);
   }
 
   if (Triple.isKnownWindowsMSVCEnvironment()) {
@@ -428,6 +477,8 @@ void Flang::addOffloadOptions(Compilation &C, const InputInfoList &Inputs,
       CmdArgs.push_back("-fopenmp-assume-no-thread-state");
     if (Args.hasArg(options::OPT_fopenmp_assume_no_nested_parallelism))
       CmdArgs.push_back("-fopenmp-assume-no-nested-parallelism");
+    if (Args.hasArg(options::OPT_nogpulib))
+      CmdArgs.push_back("-nogpulib");
   }
 }
 
diff --git a/clang/lib/Driver/ToolChains/Flang.h b/clang/lib/Driver/ToolChains/Flang.h
index 8d35080e1c0c..ec2e545a1d0b 100644
--- a/clang/lib/Driver/ToolChains/Flang.h
+++ b/clang/lib/Driver/ToolChains/Flang.h
@@ -70,6 +70,13 @@ class LLVM_LIBRARY_VISIBILITY Flang : public Tool {
   void AddAMDGPUTargetArgs(const llvm::opt::ArgList &Args,
                            llvm::opt::ArgStringList &CmdArgs) const;
 
+  /// Add specific options for RISC-V target.
+  ///
+  /// \param [in] Args The list of input driver arguments
+  /// \param [out] CmdArgs The list of output command arguments
+  void AddRISCVTargetArgs(const llvm::opt::ArgList &Args,
+                          llvm::opt::ArgStringList &CmdArgs) const;
+
   /// Extract offload options from the driver arguments and add them to
   /// the command arguments.
   /// \param [in] C The current compilation for the driver invocation
diff --git a/clang/lib/Driver/ToolChains/Fuchsia.cpp b/clang/lib/Driver/ToolChains/Fuchsia.cpp
index 6525cf8356d1..14b838500bec 100644
--- a/clang/lib/Driver/ToolChains/Fuchsia.cpp
+++ b/clang/lib/Driver/ToolChains/Fuchsia.cpp
@@ -258,7 +258,7 @@ Fuchsia::Fuchsia(const Driver &D, const llvm::Triple &Triple,
   if (!D.SysRoot.empty()) {
     SmallString<128> P(D.SysRoot);
     llvm::sys::path::append(P, "lib");
-    getFilePaths().push_back(std::string(P.str()));
+    getFilePaths().push_back(std::string(P));
   }
 
   auto FilePaths = [&](const Multilib &M) -> std::vector {
@@ -266,7 +266,7 @@ Fuchsia::Fuchsia(const Driver &D, const llvm::Triple &Triple,
     if (std::optional Path = getStdlibPath()) {
       SmallString<128> P(*Path);
       llvm::sys::path::append(P, M.gccSuffix());
-      FP.push_back(std::string(P.str()));
+      FP.push_back(std::string(P));
     }
     return FP;
   };
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index a610a94a39a2..e5e1b1d77269 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -508,6 +508,11 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
     // Add crtfastmath.o if available and fast math is enabled.
     ToolChain.addFastMathRuntimeIfAvailable(Args, CmdArgs);
+
+    if (isAndroid && Args.hasFlag(options::OPT_fandroid_pad_segment,
+                                  options::OPT_fno_android_pad_segment, false))
+      CmdArgs.push_back(
+          Args.MakeArgString(ToolChain.GetFilePath("crt_pad_segment.o")));
   }
 
   Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_u});
@@ -2251,6 +2256,15 @@ void Generic_GCC::GCCInstallationDetector::init(
     return;
   }
 
+  // If --gcc-triple is specified use this instead of trying to
+  // auto-detect a triple.
+  if (const Arg *A =
+          Args.getLastArg(clang::driver::options::OPT_gcc_triple_EQ)) {
+    StringRef GCCTriple = A->getValue();
+    CandidateTripleAliases.clear();
+    CandidateTripleAliases.push_back(GCCTriple);
+  }
+
   // Compute the set of prefixes for our search.
   SmallVector Prefixes;
   StringRef GCCToolchainDir = getGCCToolchainDir(Args, D.SysRoot);
@@ -2476,7 +2490,7 @@ void Generic_GCC::GCCInstallationDetector::AddDefaultGCCPrefixes(
   static const char *const X86Triples[] = {
       "i586-linux-gnu",      "i686-linux-gnu",        "i686-pc-linux-gnu",
       "i386-redhat-linux6E", "i686-redhat-linux",     "i386-redhat-linux",
-      "i586-suse-linux",     "i686-montavista-linux", "i686-gnu",
+      "i586-suse-linux",     "i686-montavista-linux",
   };
 
   static const char *const LoongArch64LibDirs[] = {"/lib64", "/lib"};
@@ -2643,6 +2657,23 @@ void Generic_GCC::GCCInstallationDetector::AddDefaultGCCPrefixes(
     return;
   }
 
+  if (TargetTriple.isOSHurd()) {
+    switch (TargetTriple.getArch()) {
+    case llvm::Triple::x86_64:
+      LibDirs.append(begin(X86_64LibDirs), end(X86_64LibDirs));
+      TripleAliases.push_back("x86_64-gnu");
+      break;
+    case llvm::Triple::x86:
+      LibDirs.append(begin(X86LibDirs), end(X86LibDirs));
+      TripleAliases.push_back("i686-gnu");
+      break;
+    default:
+      break;
+    }
+
+    return;
+  }
+
   switch (TargetTriple.getArch()) {
   case llvm::Triple::aarch64:
     LibDirs.append(begin(AArch64LibDirs), end(AArch64LibDirs));
@@ -2659,7 +2690,9 @@ void Generic_GCC::GCCInstallationDetector::AddDefaultGCCPrefixes(
   case llvm::Triple::arm:
   case llvm::Triple::thumb:
     LibDirs.append(begin(ARMLibDirs), end(ARMLibDirs));
-    if (TargetTriple.getEnvironment() == llvm::Triple::GNUEABIHF) {
+    if (TargetTriple.getEnvironment() == llvm::Triple::GNUEABIHF ||
+        TargetTriple.getEnvironment() == llvm::Triple::MuslEABIHF ||
+        TargetTriple.getEnvironment() == llvm::Triple::EABIHF) {
       TripleAliases.append(begin(ARMHFTriples), end(ARMHFTriples));
     } else {
       TripleAliases.append(begin(ARMTriples), end(ARMTriples));
@@ -2668,7 +2701,9 @@ void Generic_GCC::GCCInstallationDetector::AddDefaultGCCPrefixes(
   case llvm::Triple::armeb:
   case llvm::Triple::thumbeb:
     LibDirs.append(begin(ARMebLibDirs), end(ARMebLibDirs));
-    if (TargetTriple.getEnvironment() == llvm::Triple::GNUEABIHF) {
+    if (TargetTriple.getEnvironment() == llvm::Triple::GNUEABIHF ||
+        TargetTriple.getEnvironment() == llvm::Triple::MuslEABIHF ||
+        TargetTriple.getEnvironment() == llvm::Triple::EABIHF) {
       TripleAliases.append(begin(ARMebHFTriples), end(ARMebHFTriples));
     } else {
       TripleAliases.append(begin(ARMebTriples), end(ARMebTriples));
diff --git a/clang/lib/Driver/ToolChains/Hexagon.cpp b/clang/lib/Driver/ToolChains/Hexagon.cpp
index 6a2a105bee32..d1eed931be5f 100644
--- a/clang/lib/Driver/ToolChains/Hexagon.cpp
+++ b/clang/lib/Driver/ToolChains/Hexagon.cpp
@@ -550,7 +550,7 @@ std::string HexagonToolChain::getCompilerRTPath() const {
   if (!SelectedMultilibs.empty()) {
     Dir += SelectedMultilibs.back().gccSuffix();
   }
-  return std::string(Dir.str());
+  return std::string(Dir);
 }
 
 void HexagonToolChain::getHexagonLibraryPaths(const ArgList &Args,
diff --git a/clang/lib/Driver/ToolChains/Hurd.cpp b/clang/lib/Driver/ToolChains/Hurd.cpp
index 7a4c2bb7ede1..0bc114b90ffc 100644
--- a/clang/lib/Driver/ToolChains/Hurd.cpp
+++ b/clang/lib/Driver/ToolChains/Hurd.cpp
@@ -30,13 +30,21 @@ using tools::addPathIfExists;
 std::string Hurd::getMultiarchTriple(const Driver &D,
                                      const llvm::Triple &TargetTriple,
                                      StringRef SysRoot) const {
-  if (TargetTriple.getArch() == llvm::Triple::x86) {
+  switch (TargetTriple.getArch()) {
+  default:
+    break;
+
+  case llvm::Triple::x86:
     // We use the existence of '/lib/' as a directory to detect some
     // common hurd triples that don't quite match the Clang triple for both
     // 32-bit and 64-bit targets. Multiarch fixes its install triples to these
     // regardless of what the actual target triple is.
     if (D.getVFS().exists(SysRoot + "/lib/i386-gnu"))
       return "i386-gnu";
+    break;
+
+  case llvm::Triple::x86_64:
+    return "x86_64-gnu";
   }
 
   // For most architectures, just use whatever we have rather than trying to be
@@ -126,8 +134,14 @@ Tool *Hurd::buildAssembler() const {
 }
 
 std::string Hurd::getDynamicLinker(const ArgList &Args) const {
-  if (getArch() == llvm::Triple::x86)
+  switch (getArch()) {
+  case llvm::Triple::x86:
     return "/lib/ld.so";
+  case llvm::Triple::x86_64:
+    return "/lib/ld-x86-64.so.1";
+  default:
+    break;
+  }
 
   llvm_unreachable("unsupported architecture");
 }
diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp
index 735af54f114c..4300a2bdff17 100644
--- a/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/clang/lib/Driver/ToolChains/Linux.cpp
@@ -61,12 +61,16 @@ std::string Linux::getMultiarchTriple(const Driver &D,
   case llvm::Triple::thumb:
     if (IsAndroid)
       return "arm-linux-androideabi";
-    if (TargetEnvironment == llvm::Triple::GNUEABIHF)
+    if (TargetEnvironment == llvm::Triple::GNUEABIHF ||
+        TargetEnvironment == llvm::Triple::MuslEABIHF ||
+        TargetEnvironment == llvm::Triple::EABIHF)
       return "arm-linux-gnueabihf";
     return "arm-linux-gnueabi";
   case llvm::Triple::armeb:
   case llvm::Triple::thumbeb:
-    if (TargetEnvironment == llvm::Triple::GNUEABIHF)
+    if (TargetEnvironment == llvm::Triple::GNUEABIHF ||
+        TargetEnvironment == llvm::Triple::MuslEABIHF ||
+        TargetEnvironment == llvm::Triple::EABIHF)
       return "armeb-linux-gnueabihf";
     return "armeb-linux-gnueabi";
   case llvm::Triple::x86:
diff --git a/clang/lib/Driver/ToolChains/MSP430.cpp b/clang/lib/Driver/ToolChains/MSP430.cpp
index 8dc23521f400..07e875c64960 100644
--- a/clang/lib/Driver/ToolChains/MSP430.cpp
+++ b/clang/lib/Driver/ToolChains/MSP430.cpp
@@ -142,7 +142,7 @@ std::string MSP430ToolChain::computeSysRoot() const {
   else
     llvm::sys::path::append(Dir, getDriver().Dir, "..");
 
-  return std::string(Dir.str());
+  return std::string(Dir);
 }
 
 void MSP430ToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp
index 8e1e95173836..396522225158 100644
--- a/clang/lib/Driver/ToolChains/MSVC.cpp
+++ b/clang/lib/Driver/ToolChains/MSVC.cpp
@@ -495,24 +495,24 @@ bool MSVCToolChain::isPICDefaultForced() const {
 
 void MSVCToolChain::AddCudaIncludeArgs(const ArgList &DriverArgs,
                                        ArgStringList &CC1Args) const {
-  CudaInstallation.AddCudaIncludeArgs(DriverArgs, CC1Args);
+  CudaInstallation->AddCudaIncludeArgs(DriverArgs, CC1Args);
 }
 
 void MSVCToolChain::AddHIPIncludeArgs(const ArgList &DriverArgs,
                                       ArgStringList &CC1Args) const {
-  RocmInstallation.AddHIPIncludeArgs(DriverArgs, CC1Args);
+  RocmInstallation->AddHIPIncludeArgs(DriverArgs, CC1Args);
 }
 
 void MSVCToolChain::AddHIPRuntimeLibArgs(const ArgList &Args,
                                          ArgStringList &CmdArgs) const {
   CmdArgs.append({Args.MakeArgString(StringRef("-libpath:") +
-                                     RocmInstallation.getLibPath()),
+                                     RocmInstallation->getLibPath()),
                   "amdhip64.lib"});
 }
 
 void MSVCToolChain::printVerboseInfo(raw_ostream &OS) const {
-  CudaInstallation.print(OS);
-  RocmInstallation.print(OS);
+  CudaInstallation->print(OS);
+  RocmInstallation->print(OS);
 }
 
 std::string
@@ -585,7 +585,7 @@ bool MSVCToolChain::getUniversalCRTLibraryPath(const ArgList &Args,
   llvm::SmallString<128> LibPath(UniversalCRTSdkPath);
   llvm::sys::path::append(LibPath, "Lib", UCRTVersion, "ucrt", ArchName);
 
-  Path = std::string(LibPath.str());
+  Path = std::string(LibPath);
   return true;
 }
 
diff --git a/clang/lib/Driver/ToolChains/MSVC.h b/clang/lib/Driver/ToolChains/MSVC.h
index fb27c621f2d1..48369e030aad 100644
--- a/clang/lib/Driver/ToolChains/MSVC.h
+++ b/clang/lib/Driver/ToolChains/MSVC.h
@@ -11,6 +11,7 @@
 
 #include "AMDGPU.h"
 #include "Cuda.h"
+#include "LazyDetector.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Tool.h"
 #include "clang/Driver/ToolChain.h"
@@ -136,8 +137,8 @@ class LLVM_LIBRARY_VISIBILITY MSVCToolChain : public ToolChain {
   std::optional WinSdkDir, WinSdkVersion, WinSysRoot;
   std::string VCToolChainPath;
   llvm::ToolsetLayout VSLayout = llvm::ToolsetLayout::OlderVS;
-  CudaInstallationDetector CudaInstallation;
-  RocmInstallationDetector RocmInstallation;
+  LazyDetector CudaInstallation;
+  LazyDetector RocmInstallation;
 };
 
 } // end namespace toolchains
diff --git a/clang/lib/Driver/ToolChains/MinGW.cpp b/clang/lib/Driver/ToolChains/MinGW.cpp
index 65512f16357d..067758c05e97 100644
--- a/clang/lib/Driver/ToolChains/MinGW.cpp
+++ b/clang/lib/Driver/ToolChains/MinGW.cpp
@@ -471,12 +471,23 @@ findClangRelativeSysroot(const Driver &D, const llvm::Triple &LiteralTriple,
   return make_error_code(std::errc::no_such_file_or_directory);
 }
 
+static bool looksLikeMinGWSysroot(const std::string &Directory) {
+  StringRef Sep = llvm::sys::path::get_separator();
+  if (!llvm::sys::fs::exists(Directory + Sep + "include" + Sep + "_mingw.h"))
+    return false;
+  if (!llvm::sys::fs::exists(Directory + Sep + "lib" + Sep + "libkernel32.a"))
+    return false;
+  return true;
+}
+
 toolchains::MinGW::MinGW(const Driver &D, const llvm::Triple &Triple,
                          const ArgList &Args)
     : ToolChain(D, Triple, Args), CudaInstallation(D, Triple, Args),
       RocmInstallation(D, Triple, Args) {
   getProgramPaths().push_back(getDriver().getInstalledDir());
 
+  std::string InstallBase =
+      std::string(llvm::sys::path::parent_path(getDriver().getInstalledDir()));
   // The sequence for detecting a sysroot here should be kept in sync with
   // the testTriple function below.
   llvm::Triple LiteralTriple = getLiteralTriple(D, getTriple());
@@ -487,13 +498,17 @@ toolchains::MinGW::MinGW(const Driver &D, const llvm::Triple &Triple,
   else if (llvm::ErrorOr TargetSubdir = findClangRelativeSysroot(
                getDriver(), LiteralTriple, getTriple(), SubdirName))
     Base = std::string(llvm::sys::path::parent_path(TargetSubdir.get()));
+  // If the install base of Clang seems to have mingw sysroot files directly
+  // in the toplevel include and lib directories, use this as base instead of
+  // looking for a triple prefixed GCC in the path.
+  else if (looksLikeMinGWSysroot(InstallBase))
+    Base = InstallBase;
   else if (llvm::ErrorOr GPPName =
                findGcc(LiteralTriple, getTriple()))
     Base = std::string(llvm::sys::path::parent_path(
         llvm::sys::path::parent_path(GPPName.get())));
   else
-    Base = std::string(
-        llvm::sys::path::parent_path(getDriver().getInstalledDir()));
+    Base = InstallBase;
 
   Base += llvm::sys::path::get_separator();
   findGccLibDir(LiteralTriple);
@@ -599,17 +614,17 @@ SanitizerMask toolchains::MinGW::getSupportedSanitizers() const {
 
 void toolchains::MinGW::AddCudaIncludeArgs(const ArgList &DriverArgs,
                                            ArgStringList &CC1Args) const {
-  CudaInstallation.AddCudaIncludeArgs(DriverArgs, CC1Args);
+  CudaInstallation->AddCudaIncludeArgs(DriverArgs, CC1Args);
 }
 
 void toolchains::MinGW::AddHIPIncludeArgs(const ArgList &DriverArgs,
                                           ArgStringList &CC1Args) const {
-  RocmInstallation.AddHIPIncludeArgs(DriverArgs, CC1Args);
+  RocmInstallation->AddHIPIncludeArgs(DriverArgs, CC1Args);
 }
 
 void toolchains::MinGW::printVerboseInfo(raw_ostream &OS) const {
-  CudaInstallation.print(OS);
-  RocmInstallation.print(OS);
+  CudaInstallation->print(OS);
+  RocmInstallation->print(OS);
 }
 
 // Include directories for various hosts:
@@ -778,9 +793,15 @@ static bool testTriple(const Driver &D, const llvm::Triple &Triple,
   if (D.SysRoot.size())
     return true;
   llvm::Triple LiteralTriple = getLiteralTriple(D, Triple);
+  std::string InstallBase =
+      std::string(llvm::sys::path::parent_path(D.getInstalledDir()));
   if (llvm::ErrorOr TargetSubdir =
           findClangRelativeSysroot(D, LiteralTriple, Triple, SubdirName))
     return true;
+  // If the install base itself looks like a mingw sysroot, we'll use that
+  // - don't use any potentially unrelated gcc to influence what triple to use.
+  if (looksLikeMinGWSysroot(InstallBase))
+    return false;
   if (llvm::ErrorOr GPPName = findGcc(LiteralTriple, Triple))
     return true;
   // If we neither found a colocated sysroot or a matching gcc executable,
diff --git a/clang/lib/Driver/ToolChains/MinGW.h b/clang/lib/Driver/ToolChains/MinGW.h
index 75d629d5d1cb..a9963d8d06c2 100644
--- a/clang/lib/Driver/ToolChains/MinGW.h
+++ b/clang/lib/Driver/ToolChains/MinGW.h
@@ -11,6 +11,7 @@
 
 #include "Cuda.h"
 #include "Gnu.h"
+#include "LazyDetector.h"
 #include "ROCm.h"
 #include "clang/Driver/Tool.h"
 #include "clang/Driver/ToolChain.h"
@@ -102,8 +103,8 @@ class LLVM_LIBRARY_VISIBILITY MinGW : public ToolChain {
   Tool *buildAssembler() const override;
 
 private:
-  CudaInstallationDetector CudaInstallation;
-  RocmInstallationDetector RocmInstallation;
+  LazyDetector CudaInstallation;
+  LazyDetector RocmInstallation;
 
   std::string Base;
   std::string GccLibDir;
diff --git a/clang/lib/Driver/ToolChains/MipsLinux.cpp b/clang/lib/Driver/ToolChains/MipsLinux.cpp
index eacdcbf730b6..4183eccceedb 100644
--- a/clang/lib/Driver/ToolChains/MipsLinux.cpp
+++ b/clang/lib/Driver/ToolChains/MipsLinux.cpp
@@ -138,5 +138,5 @@ std::string MipsLLVMToolChain::getCompilerRT(const ArgList &Args,
   }
   llvm::sys::path::append(
       Path, Twine("libclang_rt." + Component + "-" + "mips" + Suffix));
-  return std::string(Path.str());
+  return std::string(Path);
 }
diff --git a/clang/lib/Driver/ToolChains/OpenBSD.cpp b/clang/lib/Driver/ToolChains/OpenBSD.cpp
index 21b9004ef2d5..fd6aa4d7e684 100644
--- a/clang/lib/Driver/ToolChains/OpenBSD.cpp
+++ b/clang/lib/Driver/ToolChains/OpenBSD.cpp
@@ -371,7 +371,7 @@ std::string OpenBSD::getCompilerRT(const ArgList &Args, StringRef Component,
   if (Component == "builtins") {
     SmallString<128> Path(getDriver().SysRoot);
     llvm::sys::path::append(Path, "/usr/lib/libcompiler_rt.a");
-    return std::string(Path.str());
+    return std::string(Path);
   }
   SmallString<128> P(getDriver().ResourceDir);
   std::string CRTBasename =
@@ -379,7 +379,7 @@ std::string OpenBSD::getCompilerRT(const ArgList &Args, StringRef Component,
   llvm::sys::path::append(P, "lib", CRTBasename);
   // Checks if this is the base system case which uses a different location.
   if (getVFS().exists(P))
-    return std::string(P.str());
+    return std::string(P);
   return ToolChain::getCompilerRT(Args, Component, Type);
 }
 
diff --git a/clang/lib/Driver/ToolChains/PS4CPU.cpp b/clang/lib/Driver/ToolChains/PS4CPU.cpp
index 5fd82d1da199..8ba8b80cfec7 100644
--- a/clang/lib/Driver/ToolChains/PS4CPU.cpp
+++ b/clang/lib/Driver/ToolChains/PS4CPU.cpp
@@ -292,7 +292,7 @@ toolchains::PS4PS5Base::PS4PS5Base(const Driver &D, const llvm::Triple &Triple,
         << Twine(Platform, " system libraries").str() << SDKLibDir << Whence;
     return;
   }
-  getFilePaths().push_back(std::string(SDKLibDir.str()));
+  getFilePaths().push_back(std::string(SDKLibDir));
 }
 
 void toolchains::PS4PS5Base::AddClangSystemIncludeArgs(
@@ -359,6 +359,12 @@ void toolchains::PS4PS5Base::addClangTargetOptions(
 
   CC1Args.push_back("-fno-use-init-array");
 
+  // Default to -fvisibility-global-new-delete=source for PS5.
+  if (getTriple().isPS5() &&
+      !DriverArgs.hasArg(options::OPT_fvisibility_global_new_delete_EQ,
+                         options::OPT_fvisibility_global_new_delete_hidden))
+    CC1Args.push_back("-fvisibility-global-new-delete=source");
+
   const Arg *A =
       DriverArgs.getLastArg(options::OPT_fvisibility_from_dllstorageclass,
                             options::OPT_fno_visibility_from_dllstorageclass);
diff --git a/clang/lib/Driver/ToolChains/RISCVToolchain.cpp b/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
index 7e6abd144428..5e4fa4d5331f 100644
--- a/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
+++ b/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
@@ -141,7 +141,7 @@ std::string RISCVToolChain::computeSysRoot() const {
   if (!llvm::sys::fs::exists(SysRootDir))
     return std::string();
 
-  return std::string(SysRootDir.str());
+  return std::string(SysRootDir);
 }
 
 void RISCV::Linker::ConstructJob(Compilation &C, const JobAction &JA,
diff --git a/clang/lib/Driver/ToolChains/Solaris.cpp b/clang/lib/Driver/ToolChains/Solaris.cpp
index 9a9792d019d5..200ac46aa534 100644
--- a/clang/lib/Driver/ToolChains/Solaris.cpp
+++ b/clang/lib/Driver/ToolChains/Solaris.cpp
@@ -295,13 +295,12 @@ static StringRef getSolarisLibSuffix(const llvm::Triple &Triple) {
   switch (Triple.getArch()) {
   case llvm::Triple::x86:
   case llvm::Triple::sparc:
+  default:
     break;
   case llvm::Triple::x86_64:
     return "/amd64";
   case llvm::Triple::sparcv9:
     return "/sparcv9";
-  default:
-    llvm_unreachable("Unsupported architecture");
   }
   return "";
 }
diff --git a/clang/lib/Driver/Types.cpp b/clang/lib/Driver/Types.cpp
index 08df34ade7b6..a7b6b9000e1d 100644
--- a/clang/lib/Driver/Types.cpp
+++ b/clang/lib/Driver/Types.cpp
@@ -133,7 +133,7 @@ bool types::isAcceptedByClang(ID Id) {
 
   case TY_Asm:
   case TY_C: case TY_PP_C:
-  case TY_CL: case TY_CLCXX:
+  case TY_CL: case TY_PP_CL: case TY_CLCXX: case TY_PP_CLCXX:
   case TY_CUDA: case TY_PP_CUDA:
   case TY_CUDA_DEVICE:
   case TY_HIP:
@@ -181,7 +181,9 @@ bool types::isDerivedFromC(ID Id) {
   case TY_PP_C:
   case TY_C:
   case TY_CL:
+  case TY_PP_CL:
   case TY_CLCXX:
+  case TY_PP_CLCXX:
   case TY_PP_CUDA:
   case TY_CUDA:
   case TY_CUDA_DEVICE:
@@ -241,6 +243,7 @@ bool types::isCXX(ID Id) {
   case TY_PP_CXXHeaderUnit:
   case TY_ObjCXXHeader: case TY_PP_ObjCXXHeader:
   case TY_CXXModule: case TY_PP_CXXModule:
+  case TY_PP_CLCXX:
   case TY_CUDA: case TY_PP_CUDA: case TY_CUDA_DEVICE:
   case TY_HIP:
   case TY_PP_HIP:
@@ -310,7 +313,9 @@ types::ID types::lookupTypeForExtension(llvm::StringRef Ext) {
       .Case("cc", TY_CXX)
       .Case("CC", TY_CXX)
       .Case("cl", TY_CL)
+      .Case("cli", TY_PP_CL)
       .Case("clcpp", TY_CLCXX)
+      .Case("clii", TY_PP_CLCXX)
       .Case("cp", TY_CXX)
       .Case("cu", TY_CUDA)
       .Case("hh", TY_CXXHeader)
diff --git a/clang/lib/ExtractAPI/API.cpp b/clang/lib/ExtractAPI/API.cpp
index 71c655ba5b5b..aa7a1e9360f4 100644
--- a/clang/lib/ExtractAPI/API.cpp
+++ b/clang/lib/ExtractAPI/API.cpp
@@ -45,7 +45,7 @@ RecordTy *addTopLevelRecord(DenseMap &USRLookupTable,
 
 NamespaceRecord *
 APISet::addNamespace(APIRecord *Parent, StringRef Name, StringRef USR,
-                     PresumedLoc Loc, AvailabilitySet Availability,
+                     PresumedLoc Loc, AvailabilityInfo Availability,
                      LinkageInfo Linkage, const DocComment &Comment,
                      DeclarationFragments Declaration,
                      DeclarationFragments SubHeading, bool IsFromSystemHeader) {
@@ -61,17 +61,17 @@ APISet::addNamespace(APIRecord *Parent, StringRef Name, StringRef USR,
 
 GlobalVariableRecord *
 APISet::addGlobalVar(StringRef Name, StringRef USR, PresumedLoc Loc,
-                     AvailabilitySet Availabilities, LinkageInfo Linkage,
+                     AvailabilityInfo Availability, LinkageInfo Linkage,
                      const DocComment &Comment, DeclarationFragments Fragments,
                      DeclarationFragments SubHeading, bool IsFromSystemHeader) {
   return addTopLevelRecord(USRBasedLookupTable, GlobalVariables, USR, Name, Loc,
-                           std::move(Availabilities), Linkage, Comment,
-                           Fragments, SubHeading, IsFromSystemHeader);
+                           std::move(Availability), Linkage, Comment, Fragments,
+                           SubHeading, IsFromSystemHeader);
 }
 
 GlobalVariableTemplateRecord *APISet::addGlobalVariableTemplate(
     StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availability, LinkageInfo Linkage,
+    AvailabilityInfo Availability, LinkageInfo Linkage,
     const DocComment &Comment, DeclarationFragments Declaration,
     DeclarationFragments SubHeading, Template Template,
     bool IsFromSystemHeader) {
@@ -83,19 +83,18 @@ GlobalVariableTemplateRecord *APISet::addGlobalVariableTemplate(
 
 GlobalFunctionRecord *APISet::addGlobalFunction(
     StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availabilities, LinkageInfo Linkage,
+    AvailabilityInfo Availability, LinkageInfo Linkage,
     const DocComment &Comment, DeclarationFragments Fragments,
     DeclarationFragments SubHeading, FunctionSignature Signature,
     bool IsFromSystemHeader) {
   return addTopLevelRecord(USRBasedLookupTable, GlobalFunctions, USR, Name, Loc,
-                           std::move(Availabilities), Linkage, Comment,
-                           Fragments, SubHeading, Signature,
-                           IsFromSystemHeader);
+                           std::move(Availability), Linkage, Comment, Fragments,
+                           SubHeading, Signature, IsFromSystemHeader);
 }
 
 GlobalFunctionTemplateRecord *APISet::addGlobalFunctionTemplate(
     StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availability, LinkageInfo Linkage,
+    AvailabilityInfo Availability, LinkageInfo Linkage,
     const DocComment &Comment, DeclarationFragments Declaration,
     DeclarationFragments SubHeading, FunctionSignature Signature,
     Template Template, bool IsFromSystemHeader) {
@@ -108,7 +107,7 @@ GlobalFunctionTemplateRecord *APISet::addGlobalFunctionTemplate(
 GlobalFunctionTemplateSpecializationRecord *
 APISet::addGlobalFunctionTemplateSpecialization(
     StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availability, LinkageInfo Linkage,
+    AvailabilityInfo Availability, LinkageInfo Linkage,
     const DocComment &Comment, DeclarationFragments Declaration,
     DeclarationFragments SubHeading, FunctionSignature Signature,
     bool IsFromSystemHeader) {
@@ -120,14 +119,14 @@ APISet::addGlobalFunctionTemplateSpecialization(
 
 EnumConstantRecord *APISet::addEnumConstant(EnumRecord *Enum, StringRef Name,
                                             StringRef USR, PresumedLoc Loc,
-                                            AvailabilitySet Availabilities,
+                                            AvailabilityInfo Availability,
                                             const DocComment &Comment,
                                             DeclarationFragments Declaration,
                                             DeclarationFragments SubHeading,
                                             bool IsFromSystemHeader) {
   auto Record = std::make_unique(
-      USR, Name, Loc, std::move(Availabilities), Comment, Declaration,
-      SubHeading, IsFromSystemHeader);
+      USR, Name, Loc, std::move(Availability), Comment, Declaration, SubHeading,
+      IsFromSystemHeader);
   Record->ParentInformation = APIRecord::HierarchyInformation(
       Enum->USR, Enum->Name, Enum->getKind(), Enum);
   USRBasedLookupTable.insert({USR, Record.get()});
@@ -135,64 +134,63 @@ EnumConstantRecord *APISet::addEnumConstant(EnumRecord *Enum, StringRef Name,
 }
 
 EnumRecord *APISet::addEnum(StringRef Name, StringRef USR, PresumedLoc Loc,
-                            AvailabilitySet Availabilities,
+                            AvailabilityInfo Availability,
                             const DocComment &Comment,
                             DeclarationFragments Declaration,
                             DeclarationFragments SubHeading,
                             bool IsFromSystemHeader) {
   return addTopLevelRecord(USRBasedLookupTable, Enums, USR, Name, Loc,
-                           std::move(Availabilities), Comment, Declaration,
+                           std::move(Availability), Comment, Declaration,
                            SubHeading, IsFromSystemHeader);
 }
 
-StructFieldRecord *APISet::addStructField(StructRecord *Struct, StringRef Name,
-                                          StringRef USR, PresumedLoc Loc,
-                                          AvailabilitySet Availabilities,
-                                          const DocComment &Comment,
-                                          DeclarationFragments Declaration,
-                                          DeclarationFragments SubHeading,
-                                          bool IsFromSystemHeader) {
-  auto Record = std::make_unique(
-      USR, Name, Loc, std::move(Availabilities), Comment, Declaration,
-      SubHeading, IsFromSystemHeader);
-  Record->ParentInformation = APIRecord::HierarchyInformation(
-      Struct->USR, Struct->Name, Struct->getKind(), Struct);
-  USRBasedLookupTable.insert({USR, Record.get()});
-  return Struct->Fields.emplace_back(std::move(Record)).get();
-}
-
-StructRecord *APISet::addStruct(StringRef Name, StringRef USR, PresumedLoc Loc,
-                                AvailabilitySet Availabilities,
+RecordFieldRecord *APISet::addRecordField(
+    RecordRecord *Record, StringRef Name, StringRef USR, PresumedLoc Loc,
+    AvailabilityInfo Availability, const DocComment &Comment,
+    DeclarationFragments Declaration, DeclarationFragments SubHeading,
+    APIRecord::RecordKind Kind, bool IsFromSystemHeader) {
+  auto RecordField = std::make_unique(
+      USR, Name, Loc, std::move(Availability), Comment, Declaration, SubHeading,
+      Kind, IsFromSystemHeader);
+  RecordField->ParentInformation = APIRecord::HierarchyInformation(
+      Record->USR, Record->Name, Record->getKind(), Record);
+  USRBasedLookupTable.insert({USR, RecordField.get()});
+  return Record->Fields.emplace_back(std::move(RecordField)).get();
+}
+
+RecordRecord *APISet::addRecord(StringRef Name, StringRef USR, PresumedLoc Loc,
+                                AvailabilityInfo Availability,
                                 const DocComment &Comment,
                                 DeclarationFragments Declaration,
                                 DeclarationFragments SubHeading,
+                                APIRecord::RecordKind Kind,
                                 bool IsFromSystemHeader) {
-  return addTopLevelRecord(USRBasedLookupTable, Structs, USR, Name, Loc,
-                           std::move(Availabilities), Comment, Declaration,
-                           SubHeading, IsFromSystemHeader);
+  return addTopLevelRecord(USRBasedLookupTable, Records, USR, Name, Loc,
+                           std::move(Availability), Comment, Declaration,
+                           SubHeading, Kind, IsFromSystemHeader);
 }
 
 StaticFieldRecord *
 APISet::addStaticField(StringRef Name, StringRef USR, PresumedLoc Loc,
-                       AvailabilitySet Availabilities, LinkageInfo Linkage,
+                       AvailabilityInfo Availability, LinkageInfo Linkage,
                        const DocComment &Comment,
                        DeclarationFragments Declaration,
                        DeclarationFragments SubHeading, SymbolReference Context,
                        AccessControl Access, bool IsFromSystemHeader) {
   return addTopLevelRecord(USRBasedLookupTable, StaticFields, USR, Name, Loc,
-                           std::move(Availabilities), Linkage, Comment,
+                           std::move(Availability), Linkage, Comment,
                            Declaration, SubHeading, Context, Access,
                            IsFromSystemHeader);
 }
 
 CXXFieldRecord *
 APISet::addCXXField(APIRecord *CXXClass, StringRef Name, StringRef USR,
-                    PresumedLoc Loc, AvailabilitySet Availabilities,
+                    PresumedLoc Loc, AvailabilityInfo Availability,
                     const DocComment &Comment, DeclarationFragments Declaration,
                     DeclarationFragments SubHeading, AccessControl Access,
                     bool IsFromSystemHeader) {
   auto *Record = addTopLevelRecord(
-      USRBasedLookupTable, CXXFields, USR, Name, Loc, std::move(Availabilities),
+      USRBasedLookupTable, CXXFields, USR, Name, Loc, std::move(Availability),
       Comment, Declaration, SubHeading, Access, IsFromSystemHeader);
   Record->ParentInformation = APIRecord::HierarchyInformation(
       CXXClass->USR, CXXClass->Name, CXXClass->getKind(), CXXClass);
@@ -201,7 +199,7 @@ APISet::addCXXField(APIRecord *CXXClass, StringRef Name, StringRef USR,
 
 CXXFieldTemplateRecord *APISet::addCXXFieldTemplate(
     APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availability, const DocComment &Comment,
+    AvailabilityInfo Availability, const DocComment &Comment,
     DeclarationFragments Declaration, DeclarationFragments SubHeading,
     AccessControl Access, Template Template, bool IsFromSystemHeader) {
   auto *Record =
@@ -216,14 +214,13 @@ CXXFieldTemplateRecord *APISet::addCXXFieldTemplate(
 
 CXXClassRecord *
 APISet::addCXXClass(APIRecord *Parent, StringRef Name, StringRef USR,
-                    PresumedLoc Loc, AvailabilitySet Availabilities,
+                    PresumedLoc Loc, AvailabilityInfo Availability,
                     const DocComment &Comment, DeclarationFragments Declaration,
                     DeclarationFragments SubHeading, APIRecord::RecordKind Kind,
                     AccessControl Access, bool IsFromSystemHeader) {
-  auto *Record =
-      addTopLevelRecord(USRBasedLookupTable, CXXClasses, USR, Name, Loc,
-                        std::move(Availabilities), Comment, Declaration,
-                        SubHeading, Kind, Access, IsFromSystemHeader);
+  auto *Record = addTopLevelRecord(
+      USRBasedLookupTable, CXXClasses, USR, Name, Loc, std::move(Availability),
+      Comment, Declaration, SubHeading, Kind, Access, IsFromSystemHeader);
   if (Parent)
     Record->ParentInformation = APIRecord::HierarchyInformation(
         Parent->USR, Parent->Name, Parent->getKind(), Parent);
@@ -232,7 +229,7 @@ APISet::addCXXClass(APIRecord *Parent, StringRef Name, StringRef USR,
 
 ClassTemplateRecord *APISet::addClassTemplate(
     APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availability, const DocComment &Comment,
+    AvailabilityInfo Availability, const DocComment &Comment,
     DeclarationFragments Declaration, DeclarationFragments SubHeading,
     Template Template, AccessControl Access, bool IsFromSystemHeader) {
   auto *Record =
@@ -247,7 +244,7 @@ ClassTemplateRecord *APISet::addClassTemplate(
 
 ClassTemplateSpecializationRecord *APISet::addClassTemplateSpecialization(
     APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availability, const DocComment &Comment,
+    AvailabilityInfo Availability, const DocComment &Comment,
     DeclarationFragments Declaration, DeclarationFragments SubHeading,
     AccessControl Access, bool IsFromSystemHeader) {
   auto *Record =
@@ -263,7 +260,7 @@ ClassTemplateSpecializationRecord *APISet::addClassTemplateSpecialization(
 ClassTemplatePartialSpecializationRecord *
 APISet::addClassTemplatePartialSpecialization(
     APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availability, const DocComment &Comment,
+    AvailabilityInfo Availability, const DocComment &Comment,
     DeclarationFragments Declaration, DeclarationFragments SubHeading,
     Template Template, AccessControl Access, bool IsFromSystemHeader) {
   auto *Record = addTopLevelRecord(
@@ -279,7 +276,7 @@ APISet::addClassTemplatePartialSpecialization(
 GlobalVariableTemplateSpecializationRecord *
 APISet::addGlobalVariableTemplateSpecialization(
     StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availability, LinkageInfo Linkage,
+    AvailabilityInfo Availability, LinkageInfo Linkage,
     const DocComment &Comment, DeclarationFragments Declaration,
     DeclarationFragments SubHeading, bool IsFromSystemHeader) {
   return addTopLevelRecord(USRBasedLookupTable,
@@ -291,7 +288,7 @@ APISet::addGlobalVariableTemplateSpecialization(
 GlobalVariableTemplatePartialSpecializationRecord *
 APISet::addGlobalVariableTemplatePartialSpecialization(
     StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availability, LinkageInfo Linkage,
+    AvailabilityInfo Availability, LinkageInfo Linkage,
     const DocComment &Comment, DeclarationFragments Declaration,
     DeclarationFragments SubHeading, Template Template,
     bool IsFromSystemHeader) {
@@ -302,7 +299,8 @@ APISet::addGlobalVariableTemplatePartialSpecialization(
 }
 
 ConceptRecord *APISet::addConcept(StringRef Name, StringRef USR,
-                                  PresumedLoc Loc, AvailabilitySet Availability,
+                                  PresumedLoc Loc,
+                                  AvailabilityInfo Availability,
                                   const DocComment &Comment,
                                   DeclarationFragments Declaration,
                                   DeclarationFragments SubHeading,
@@ -314,7 +312,7 @@ ConceptRecord *APISet::addConcept(StringRef Name, StringRef USR,
 
 CXXMethodRecord *APISet::addCXXInstanceMethod(
     APIRecord *CXXClassRecord, StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availability, const DocComment &Comment,
+    AvailabilityInfo Availability, const DocComment &Comment,
     DeclarationFragments Declaration, DeclarationFragments SubHeading,
     FunctionSignature Signature, AccessControl Access,
     bool IsFromSystemHeader) {
@@ -331,7 +329,7 @@ CXXMethodRecord *APISet::addCXXInstanceMethod(
 
 CXXMethodRecord *APISet::addCXXStaticMethod(
     APIRecord *CXXClassRecord, StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availability, const DocComment &Comment,
+    AvailabilityInfo Availability, const DocComment &Comment,
     DeclarationFragments Declaration, DeclarationFragments SubHeading,
     FunctionSignature Signature, AccessControl Access,
     bool IsFromSystemHeader) {
@@ -348,7 +346,7 @@ CXXMethodRecord *APISet::addCXXStaticMethod(
 
 CXXMethodTemplateRecord *APISet::addCXXMethodTemplate(
     APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availability, const DocComment &Comment,
+    AvailabilityInfo Availability, const DocComment &Comment,
     DeclarationFragments Declaration, DeclarationFragments SubHeading,
     FunctionSignature Signature, AccessControl Access, Template Template,
     bool IsFromSystemHeader) {
@@ -364,7 +362,7 @@ CXXMethodTemplateRecord *APISet::addCXXMethodTemplate(
 
 CXXMethodTemplateSpecializationRecord *APISet::addCXXMethodTemplateSpec(
     APIRecord *Parent, StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availability, const DocComment &Comment,
+    AvailabilityInfo Availability, const DocComment &Comment,
     DeclarationFragments Declaration, DeclarationFragments SubHeading,
     FunctionSignature Signature, AccessControl Access,
     bool IsFromSystemHeader) {
@@ -381,14 +379,14 @@ CXXMethodTemplateSpecializationRecord *APISet::addCXXMethodTemplateSpec(
 
 ObjCCategoryRecord *APISet::addObjCCategory(
     StringRef Name, StringRef USR, PresumedLoc Loc,
-    AvailabilitySet Availabilities, const DocComment &Comment,
+    AvailabilityInfo Availability, const DocComment &Comment,
     DeclarationFragments Declaration, DeclarationFragments SubHeading,
     SymbolReference Interface, bool IsFromSystemHeader,
     bool IsFromExternalModule) {
   // Create the category record.
   auto *Record =
       addTopLevelRecord(USRBasedLookupTable, ObjCCategories, USR, Name, Loc,
-                        std::move(Availabilities), Comment, Declaration,
+                        std::move(Availability), Comment, Declaration,
                         SubHeading, Interface, IsFromSystemHeader);
 
   Record->IsFromExternalModule = IsFromExternalModule;
@@ -402,31 +400,31 @@ ObjCCategoryRecord *APISet::addObjCCategory(
 
 ObjCInterfaceRecord *
 APISet::addObjCInterface(StringRef Name, StringRef USR, PresumedLoc Loc,
-                         AvailabilitySet Availabilities, LinkageInfo Linkage,
+                         AvailabilityInfo Availability, LinkageInfo Linkage,
                          const DocComment &Comment,
                          DeclarationFragments Declaration,
                          DeclarationFragments SubHeading,
                          SymbolReference SuperClass, bool IsFromSystemHeader) {
   return addTopLevelRecord(USRBasedLookupTable, ObjCInterfaces, USR, Name, Loc,
-                           std::move(Availabilities), Linkage, Comment,
+                           std::move(Availability), Linkage, Comment,
                            Declaration, SubHeading, SuperClass,
                            IsFromSystemHeader);
 }
 
 ObjCMethodRecord *APISet::addObjCMethod(
     ObjCContainerRecord *Container, StringRef Name, StringRef USR,
-    PresumedLoc Loc, AvailabilitySet Availabilities, const DocComment &Comment,
+    PresumedLoc Loc, AvailabilityInfo Availability, const DocComment &Comment,
     DeclarationFragments Declaration, DeclarationFragments SubHeading,
     FunctionSignature Signature, bool IsInstanceMethod,
     bool IsFromSystemHeader) {
   std::unique_ptr Record;
   if (IsInstanceMethod)
     Record = std::make_unique(
-        USR, Name, Loc, std::move(Availabilities), Comment, Declaration,
+        USR, Name, Loc, std::move(Availability), Comment, Declaration,
         SubHeading, Signature, IsFromSystemHeader);
   else
     Record = std::make_unique(
-        USR, Name, Loc, std::move(Availabilities), Comment, Declaration,
+        USR, Name, Loc, std::move(Availability), Comment, Declaration,
         SubHeading, Signature, IsFromSystemHeader);
 
   Record->ParentInformation = APIRecord::HierarchyInformation(
@@ -437,7 +435,7 @@ ObjCMethodRecord *APISet::addObjCMethod(
 
 ObjCPropertyRecord *APISet::addObjCProperty(
     ObjCContainerRecord *Container, StringRef Name, StringRef USR,
-    PresumedLoc Loc, AvailabilitySet Availabilities, const DocComment &Comment,
+    PresumedLoc Loc, AvailabilityInfo Availability, const DocComment &Comment,
     DeclarationFragments Declaration, DeclarationFragments SubHeading,
     ObjCPropertyRecord::AttributeKind Attributes, StringRef GetterName,
     StringRef SetterName, bool IsOptional, bool IsInstanceProperty,
@@ -445,12 +443,12 @@ ObjCPropertyRecord *APISet::addObjCProperty(
   std::unique_ptr Record;
   if (IsInstanceProperty)
     Record = std::make_unique(
-        USR, Name, Loc, std::move(Availabilities), Comment, Declaration,
+        USR, Name, Loc, std::move(Availability), Comment, Declaration,
         SubHeading, Attributes, GetterName, SetterName, IsOptional,
         IsFromSystemHeader);
   else
     Record = std::make_unique(
-        USR, Name, Loc, std::move(Availabilities), Comment, Declaration,
+        USR, Name, Loc, std::move(Availability), Comment, Declaration,
         SubHeading, Attributes, GetterName, SetterName, IsOptional,
         IsFromSystemHeader);
   Record->ParentInformation = APIRecord::HierarchyInformation(
@@ -461,12 +459,12 @@ ObjCPropertyRecord *APISet::addObjCProperty(
 
 ObjCInstanceVariableRecord *APISet::addObjCInstanceVariable(
     ObjCContainerRecord *Container, StringRef Name, StringRef USR,
-    PresumedLoc Loc, AvailabilitySet Availabilities, const DocComment &Comment,
+    PresumedLoc Loc, AvailabilityInfo Availability, const DocComment &Comment,
     DeclarationFragments Declaration, DeclarationFragments SubHeading,
     ObjCInstanceVariableRecord::AccessControl Access, bool IsFromSystemHeader) {
   auto Record = std::make_unique(
-      USR, Name, Loc, std::move(Availabilities), Comment, Declaration,
-      SubHeading, Access, IsFromSystemHeader);
+      USR, Name, Loc, std::move(Availability), Comment, Declaration, SubHeading,
+      Access, IsFromSystemHeader);
   Record->ParentInformation = APIRecord::HierarchyInformation(
       Container->USR, Container->Name, Container->getKind(), Container);
   USRBasedLookupTable.insert({USR, Record.get()});
@@ -475,13 +473,13 @@ ObjCInstanceVariableRecord *APISet::addObjCInstanceVariable(
 
 ObjCProtocolRecord *APISet::addObjCProtocol(StringRef Name, StringRef USR,
                                             PresumedLoc Loc,
-                                            AvailabilitySet Availabilities,
+                                            AvailabilityInfo Availability,
                                             const DocComment &Comment,
                                             DeclarationFragments Declaration,
                                             DeclarationFragments SubHeading,
                                             bool IsFromSystemHeader) {
   return addTopLevelRecord(USRBasedLookupTable, ObjCProtocols, USR, Name, Loc,
-                           std::move(Availabilities), Comment, Declaration,
+                           std::move(Availability), Comment, Declaration,
                            SubHeading, IsFromSystemHeader);
 }
 
@@ -496,12 +494,12 @@ APISet::addMacroDefinition(StringRef Name, StringRef USR, PresumedLoc Loc,
 
 TypedefRecord *
 APISet::addTypedef(StringRef Name, StringRef USR, PresumedLoc Loc,
-                   AvailabilitySet Availabilities, const DocComment &Comment,
+                   AvailabilityInfo Availability, const DocComment &Comment,
                    DeclarationFragments Declaration,
                    DeclarationFragments SubHeading,
                    SymbolReference UnderlyingType, bool IsFromSystemHeader) {
   return addTopLevelRecord(USRBasedLookupTable, Typedefs, USR, Name, Loc,
-                           std::move(Availabilities), Comment, Declaration,
+                           std::move(Availability), Comment, Declaration,
                            SubHeading, UnderlyingType, IsFromSystemHeader);
 }
 
@@ -548,8 +546,8 @@ void GlobalFunctionRecord::anchor() {}
 void GlobalVariableRecord::anchor() {}
 void EnumConstantRecord::anchor() {}
 void EnumRecord::anchor() {}
-void StructFieldRecord::anchor() {}
-void StructRecord::anchor() {}
+void RecordFieldRecord::anchor() {}
+void RecordRecord::anchor() {}
 void CXXFieldRecord::anchor() {}
 void CXXClassRecord::anchor() {}
 void CXXConstructorRecord::anchor() {}
diff --git a/clang/lib/ExtractAPI/AvailabilityInfo.cpp b/clang/lib/ExtractAPI/AvailabilityInfo.cpp
index 1df852fdbf93..18e4d16b45bb 100644
--- a/clang/lib/ExtractAPI/AvailabilityInfo.cpp
+++ b/clang/lib/ExtractAPI/AvailabilityInfo.cpp
@@ -1,50 +1,35 @@
 #include "clang/ExtractAPI/AvailabilityInfo.h"
+#include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
+#include "clang/Basic/TargetInfo.h"
 #include "llvm/ADT/STLExtras.h"
 
-using namespace clang;
-using namespace extractapi;
+using namespace clang::extractapi;
+using namespace llvm;
 
-AvailabilitySet::AvailabilitySet(const Decl *Decl) {
-  // Collect availability attributes from all redeclrations.
-  for (const auto *RD : Decl->redecls()) {
-    if (const auto *A = RD->getAttr()) {
-      if (!A->isImplicit()) {
-        this->Availabilities.clear();
-        UnconditionallyUnavailable = true;
-      }
-    }
+AvailabilityInfo AvailabilityInfo::createFromDecl(const Decl *Decl) {
+  ASTContext &Context = Decl->getASTContext();
+  StringRef PlatformName = Context.getTargetInfo().getPlatformName();
+  AvailabilityInfo Availability;
 
-    if (const auto *A = RD->getAttr()) {
-      if (!A->isImplicit()) {
-        this->Availabilities.clear();
-        UnconditionallyDeprecated = true;
-      }
+  // Collect availability attributes from all redeclarations.
+  for (const auto *RD : Decl->redecls()) {
+    for (const auto *A : RD->specific_attrs()) {
+      if (A->getPlatform()->getName() != PlatformName)
+        continue;
+      Availability =
+          AvailabilityInfo(A->getPlatform()->getName(), A->getIntroduced(),
+                           A->getDeprecated(), A->getObsoleted(), false, false);
+      break;
     }
 
-    for (const auto *Attr : RD->specific_attrs()) {
-      StringRef Domain = Attr->getPlatform()->getName();
-      auto *Availability =
-          llvm::find_if(Availabilities, [Domain](const AvailabilityInfo &Info) {
-            return Domain.equals(Info.Domain);
-          });
-      if (Availability != Availabilities.end()) {
-        // Get the highest introduced version for all redeclarations.
-        if (Availability->Introduced < Attr->getIntroduced())
-          Availability->Introduced = Attr->getIntroduced();
+    if (const auto *A = RD->getAttr())
+      if (!A->isImplicit())
+        Availability.UnconditionallyUnavailable = true;
 
-        // Get the lowest deprecated version for all redeclarations.
-        if (Availability->Deprecated > Attr->getDeprecated())
-          Availability->Deprecated = Attr->getDeprecated();
-
-        // Get the lowest obsoleted version for all redeclarations.
-        if (Availability->Obsoleted > Attr->getObsoleted())
-          Availability->Obsoleted = Attr->getObsoleted();
-      } else {
-        Availabilities.emplace_back(Domain, Attr->getIntroduced(),
-                                    Attr->getDeprecated(), Attr->getObsoleted(),
-                                    Attr->getUnavailable());
-      }
-    }
+    if (const auto *A = RD->getAttr())
+      if (!A->isImplicit())
+        Availability.UnconditionallyDeprecated = true;
   }
+  return Availability;
 }
diff --git a/clang/lib/ExtractAPI/DeclarationFragments.cpp b/clang/lib/ExtractAPI/DeclarationFragments.cpp
index eb6eea0aaf54..56c1f5bf5eab 100644
--- a/clang/lib/ExtractAPI/DeclarationFragments.cpp
+++ b/clang/lib/ExtractAPI/DeclarationFragments.cpp
@@ -252,6 +252,46 @@ DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForType(
 
   DeclarationFragments Fragments;
 
+  // An ElaboratedType is a sugar for types that are referred to using an
+  // elaborated keyword, e.g., `struct S`, `enum E`, or (in C++) via a
+  // qualified name, e.g., `N::M::type`, or both.
+  if (const ElaboratedType *ET = dyn_cast(T)) {
+    ElaboratedTypeKeyword Keyword = ET->getKeyword();
+    if (Keyword != ElaboratedTypeKeyword::None) {
+      Fragments
+          .append(ElaboratedType::getKeywordName(Keyword),
+                  DeclarationFragments::FragmentKind::Keyword)
+          .appendSpace();
+    }
+
+    if (const NestedNameSpecifier *NNS = ET->getQualifier())
+      Fragments.append(getFragmentsForNNS(NNS, Context, After));
+
+    // After handling the elaborated keyword or qualified name, build
+    // declaration fragments for the desugared underlying type.
+    return Fragments.append(getFragmentsForType(ET->desugar(), Context, After));
+  }
+
+  // If the type is a typedefed type, get the underlying TypedefNameDecl for a
+  // direct reference to the typedef instead of the wrapped type.
+
+  // 'id' type is a typedef for an ObjCObjectPointerType
+  //  we treat it as a typedef
+  if (const TypedefType *TypedefTy = dyn_cast(T)) {
+    const TypedefNameDecl *Decl = TypedefTy->getDecl();
+    TypedefUnderlyingTypeResolver TypedefResolver(Context);
+    std::string USR = TypedefResolver.getUSRForType(QualType(T, 0));
+
+    if (T->isObjCIdType()) {
+      return Fragments.append(Decl->getName(),
+                              DeclarationFragments::FragmentKind::Keyword);
+    }
+
+    return Fragments.append(
+        Decl->getName(), DeclarationFragments::FragmentKind::TypeIdentifier,
+        USR, TypedefResolver.getUnderlyingTypeDecl(QualType(T, 0)));
+  }
+
   // Declaration fragments of a pointer type is the declaration fragments of
   // the pointee type followed by a `*`,
   if (T->isPointerType() && !T->isFunctionPointerType())
@@ -328,46 +368,6 @@ DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForType(
         getFragmentsForType(AT->getElementType(), Context, After));
   }
 
-  // An ElaboratedType is a sugar for types that are referred to using an
-  // elaborated keyword, e.g., `struct S`, `enum E`, or (in C++) via a
-  // qualified name, e.g., `N::M::type`, or both.
-  if (const ElaboratedType *ET = dyn_cast(T)) {
-    ElaboratedTypeKeyword Keyword = ET->getKeyword();
-    if (Keyword != ElaboratedTypeKeyword::None) {
-      Fragments
-          .append(ElaboratedType::getKeywordName(Keyword),
-                  DeclarationFragments::FragmentKind::Keyword)
-          .appendSpace();
-    }
-
-    if (const NestedNameSpecifier *NNS = ET->getQualifier())
-      Fragments.append(getFragmentsForNNS(NNS, Context, After));
-
-    // After handling the elaborated keyword or qualified name, build
-    // declaration fragments for the desugared underlying type.
-    return Fragments.append(getFragmentsForType(ET->desugar(), Context, After));
-  }
-
-  // If the type is a typedefed type, get the underlying TypedefNameDecl for a
-  // direct reference to the typedef instead of the wrapped type.
-
-  // 'id' type is a typedef for an ObjCObjectPointerType
-  //  we treat it as a typedef
-  if (const TypedefType *TypedefTy = dyn_cast(T)) {
-    const TypedefNameDecl *Decl = TypedefTy->getDecl();
-    TypedefUnderlyingTypeResolver TypedefResolver(Context);
-    std::string USR = TypedefResolver.getUSRForType(QualType(T, 0));
-
-    if (T->isObjCIdType()) {
-      return Fragments.append(Decl->getName(),
-                              DeclarationFragments::FragmentKind::Keyword);
-    }
-
-    return Fragments.append(
-        Decl->getName(), DeclarationFragments::FragmentKind::TypeIdentifier,
-        USR, TypedefResolver.getUnderlyingTypeDecl(QualType(T, 0)));
-  }
-
   // Everything we care about has been handled now, reduce to the canonical
   // unqualified base type.
   QualType Base = T->getCanonicalTypeUnqualified();
@@ -760,13 +760,16 @@ DeclarationFragmentsBuilder::getFragmentsForField(const FieldDecl *Field) {
       .append(";", DeclarationFragments::FragmentKind::Text);
 }
 
-DeclarationFragments
-DeclarationFragmentsBuilder::getFragmentsForStruct(const RecordDecl *Record) {
+DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForRecordDecl(
+    const RecordDecl *Record) {
   if (const auto *TypedefNameDecl = Record->getTypedefNameForAnonDecl())
     return getFragmentsForTypedef(TypedefNameDecl);
 
   DeclarationFragments Fragments;
-  Fragments.append("struct", DeclarationFragments::FragmentKind::Keyword);
+  if (Record->isUnion())
+    Fragments.append("union", DeclarationFragments::FragmentKind::Keyword);
+  else
+    Fragments.append("struct", DeclarationFragments::FragmentKind::Keyword);
 
   if (!Record->getName().empty())
     Fragments.appendSpace().append(
diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
index 53b22297ee0e..349b93e2a232 100644
--- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
+++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
@@ -147,46 +147,40 @@ Object serializeSourceRange(const PresumedLoc &BeginLoc,
 /// Serialize the availability attributes of a symbol.
 ///
 /// Availability information contains the introduced, deprecated, and obsoleted
-/// versions of the symbol for a given domain (roughly corresponds to a
-/// platform) as semantic versions, if not default.  Availability information
-/// also contains flags to indicate if the symbol is unconditionally unavailable
-/// or deprecated, i.e. \c __attribute__((unavailable)) and \c
-/// __attribute__((deprecated)).
+/// versions of the symbol as semantic versions, if not default.
+/// Availability information also contains flags to indicate if the symbol is
+/// unconditionally unavailable or deprecated,
+/// i.e. \c __attribute__((unavailable)) and \c __attribute__((deprecated)).
 ///
 /// \returns \c std::nullopt if the symbol has default availability attributes,
-/// or an \c Array containing the formatted availability information.
-std::optional
-serializeAvailability(const AvailabilitySet &Availabilities) {
-  if (Availabilities.isDefault())
+/// or an \c Array containing an object with the formatted availability
+/// information.
+std::optional serializeAvailability(const AvailabilityInfo &Avail) {
+  if (Avail.isDefault())
     return std::nullopt;
 
+  Object Availability;
   Array AvailabilityArray;
-
-  if (Availabilities.isUnconditionallyDeprecated()) {
+  Availability["domain"] = Avail.Domain;
+  serializeObject(Availability, "introduced",
+                  serializeSemanticVersion(Avail.Introduced));
+  serializeObject(Availability, "deprecated",
+                  serializeSemanticVersion(Avail.Deprecated));
+  serializeObject(Availability, "obsoleted",
+                  serializeSemanticVersion(Avail.Obsoleted));
+  if (Avail.isUnconditionallyDeprecated()) {
     Object UnconditionallyDeprecated;
     UnconditionallyDeprecated["domain"] = "*";
     UnconditionallyDeprecated["isUnconditionallyDeprecated"] = true;
     AvailabilityArray.emplace_back(std::move(UnconditionallyDeprecated));
   }
-
-  // Note unconditionally unavailable records are skipped.
-
-  for (const auto &AvailInfo : Availabilities) {
-    Object Availability;
-    Availability["domain"] = AvailInfo.Domain;
-    if (AvailInfo.Unavailable)
-      Availability["isUnconditionallyUnavailable"] = true;
-    else {
-      serializeObject(Availability, "introduced",
-                      serializeSemanticVersion(AvailInfo.Introduced));
-      serializeObject(Availability, "deprecated",
-                      serializeSemanticVersion(AvailInfo.Deprecated));
-      serializeObject(Availability, "obsoleted",
-                      serializeSemanticVersion(AvailInfo.Obsoleted));
-    }
-    AvailabilityArray.emplace_back(std::move(Availability));
+  if (Avail.isUnconditionallyUnavailable()) {
+    Object UnconditionallyUnavailable;
+    UnconditionallyUnavailable["domain"] = "*";
+    UnconditionallyUnavailable["isUnconditionallyUnavailable"] = true;
+    AvailabilityArray.emplace_back(std::move(UnconditionallyUnavailable));
   }
-
+  AvailabilityArray.emplace_back(std::move(Availability));
   return AvailabilityArray;
 }
 
@@ -406,7 +400,7 @@ Object serializeSymbolKind(APIRecord::RecordKind RK, Language Lang) {
     Kind["identifier"] = AddLangPrefix("struct");
     Kind["displayName"] = "Structure";
     break;
-  case APIRecord::RK_CXXField:
+  case APIRecord::RK_UnionField:
     Kind["identifier"] = AddLangPrefix("property");
     Kind["displayName"] = "Instance Property";
     break;
@@ -414,6 +408,10 @@ Object serializeSymbolKind(APIRecord::RecordKind RK, Language Lang) {
     Kind["identifier"] = AddLangPrefix("union");
     Kind["displayName"] = "Union";
     break;
+  case APIRecord::RK_CXXField:
+    Kind["identifier"] = AddLangPrefix("property");
+    Kind["displayName"] = "Instance Property";
+    break;
   case APIRecord::RK_StaticField:
     Kind["identifier"] = AddLangPrefix("type.property");
     Kind["displayName"] = "Type Property";
@@ -738,7 +736,7 @@ bool SymbolGraphSerializer::shouldSkip(const APIRecord &Record) const {
     return true;
 
   // Skip unconditionally unavailable symbols
-  if (Record.Availabilities.isUnconditionallyUnavailable())
+  if (Record.Availability.isUnconditionallyUnavailable())
     return true;
 
   // Filter out symbols prefixed with an underscored as they are understood to
@@ -764,7 +762,7 @@ SymbolGraphSerializer::serializeAPIRecord(const RecordTy &Record) const {
       Obj, "location",
       serializeSourceLocation(Record.Location, /*IncludeFileURI=*/true));
   serializeArray(Obj, "availability",
-                 serializeAvailability(Record.Availabilities));
+                 serializeAvailability(Record.Availability));
   serializeObject(Obj, "docComment", serializeDocComment(Record.Comment));
   serializeArray(Obj, "declarationFragments",
                  serializeDeclarationFragments(Record.Declaration));
@@ -877,12 +875,12 @@ void SymbolGraphSerializer::visitEnumRecord(const EnumRecord &Record) {
   serializeMembers(Record, Record.Constants);
 }
 
-void SymbolGraphSerializer::visitStructRecord(const StructRecord &Record) {
-  auto Struct = serializeAPIRecord(Record);
-  if (!Struct)
+void SymbolGraphSerializer::visitRecordRecord(const RecordRecord &Record) {
+  auto SerializedRecord = serializeAPIRecord(Record);
+  if (!SerializedRecord)
     return;
 
-  Symbols.emplace_back(std::move(*Struct));
+  Symbols.emplace_back(std::move(*SerializedRecord));
   serializeMembers(Record, Record.Fields);
 }
 
@@ -1173,7 +1171,9 @@ void SymbolGraphSerializer::serializeSingleRecord(const APIRecord *Record) {
     visitEnumRecord(*cast(Record));
     break;
   case APIRecord::RK_Struct:
-    visitStructRecord(*cast(Record));
+    LLVM_FALLTHROUGH;
+  case APIRecord::RK_Union:
+    visitRecordRecord(*cast(Record));
     break;
   case APIRecord::RK_StaticField:
     visitStaticFieldRecord(*cast(Record));
diff --git a/clang/lib/Format/.clang-format b/clang/lib/Format/.clang-format
index d7331b3c8cf0..f95602cab0f7 100644
--- a/clang/lib/Format/.clang-format
+++ b/clang/lib/Format/.clang-format
@@ -1 +1,6 @@
-BasedOnStyle: clang-format
+BasedOnStyle: LLVM
+InsertBraces: true
+InsertNewlineAtEOF: true
+LineEnding: LF
+RemoveBracesLLVM: true
+RemoveParentheses: ReturnStatement
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index 102504182c45..a3eb9138b218 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -703,7 +703,9 @@ void ContinuationIndenter::addTokenOnCurrentLine(LineState &State, bool DryRun,
 
   if (Current.is(tok::equal) &&
       (State.Line->First->is(tok::kw_for) || Current.NestingLevel == 0) &&
-      CurrentState.VariablePos == 0) {
+      CurrentState.VariablePos == 0 &&
+      (!Previous.Previous ||
+       Previous.Previous->isNot(TT_DesignatedInitializerPeriod))) {
     CurrentState.VariablePos = State.Column;
     // Move over * and & if they are bound to the variable name.
     const FormatToken *Tok = &Previous;
@@ -738,7 +740,8 @@ void ContinuationIndenter::addTokenOnCurrentLine(LineState &State, bool DryRun,
 
   if (!DryRun) {
     Whitespaces.replaceWhitespace(Current, /*Newlines=*/0, Spaces,
-                                  State.Column + Spaces + PPColumnCorrection);
+                                  State.Column + Spaces + PPColumnCorrection,
+                                  /*IsAligned=*/false, State.Line->InMacroBody);
   }
 
   // If "BreakBeforeInheritanceComma" mode, don't break within the inheritance
@@ -768,15 +771,25 @@ void ContinuationIndenter::addTokenOnCurrentLine(LineState &State, bool DryRun,
   // parenthesis by disallowing any further line breaks if there is no line
   // break after the opening parenthesis. Don't break if it doesn't conserve
   // columns.
+  auto IsOpeningBracket = [&](const FormatToken &Tok) {
+    auto IsStartOfBracedList = [&]() {
+      return Tok.is(tok::l_brace) && Tok.isNot(BK_Block) &&
+             Style.Cpp11BracedListStyle;
+    };
+    if (!Tok.isOneOf(tok::l_paren, TT_TemplateOpener, tok::l_square) &&
+        !IsStartOfBracedList()) {
+      return false;
+    }
+    if (!Tok.Previous)
+      return true;
+    if (Tok.Previous->isIf())
+      return Style.AlignAfterOpenBracket == FormatStyle::BAS_AlwaysBreak;
+    return !Tok.Previous->isOneOf(TT_CastRParen, tok::kw_for, tok::kw_while,
+                                  tok::kw_switch);
+  };
   if ((Style.AlignAfterOpenBracket == FormatStyle::BAS_AlwaysBreak ||
        Style.AlignAfterOpenBracket == FormatStyle::BAS_BlockIndent) &&
-      (Previous.isOneOf(tok::l_paren, TT_TemplateOpener, tok::l_square) ||
-       (Previous.is(tok::l_brace) && Previous.isNot(BK_Block) &&
-        Style.Cpp11BracedListStyle)) &&
-      State.Column > getNewLineColumn(State) &&
-      (!Previous.Previous ||
-       !Previous.Previous->isOneOf(TT_CastRParen, tok::kw_for, tok::kw_while,
-                                   tok::kw_switch)) &&
+      IsOpeningBracket(Previous) && State.Column > getNewLineColumn(State) &&
       // Don't do this for simple (no expressions) one-argument function calls
       // as that feels like needlessly wasting whitespace, e.g.:
       //
@@ -1591,6 +1604,9 @@ unsigned ContinuationIndenter::moveStateToNextToken(LineState &State,
     State.StartOfStringLiteral = State.Column + 1;
   if (Current.is(TT_CSharpStringLiteral) && State.StartOfStringLiteral == 0) {
     State.StartOfStringLiteral = State.Column + 1;
+  } else if (Current.is(TT_TableGenMultiLineString) &&
+             State.StartOfStringLiteral == 0) {
+    State.StartOfStringLiteral = State.Column + 1;
   } else if (Current.isStringLiteral() && State.StartOfStringLiteral == 0) {
     State.StartOfStringLiteral = State.Column;
   } else if (!Current.isOneOf(tok::comment, tok::identifier, tok::hash) &&
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index f798d555bf99..ff326dc78478 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -76,41 +76,39 @@ template <> struct MappingTraits {
                 FormatStyle::AlignConsecutiveStyle(
                     {/*Enabled=*/false, /*AcrossEmptyLines=*/false,
                      /*AcrossComments=*/false, /*AlignCompound=*/false,
-                     /*PadOperators=*/true}));
+                     /*AlignFunctionPointers=*/false, /*PadOperators=*/true}));
     IO.enumCase(Value, "Consecutive",
                 FormatStyle::AlignConsecutiveStyle(
                     {/*Enabled=*/true, /*AcrossEmptyLines=*/false,
                      /*AcrossComments=*/false, /*AlignCompound=*/false,
-                     /*PadOperators=*/true}));
+                     /*AlignFunctionPointers=*/false, /*PadOperators=*/true}));
     IO.enumCase(Value, "AcrossEmptyLines",
                 FormatStyle::AlignConsecutiveStyle(
                     {/*Enabled=*/true, /*AcrossEmptyLines=*/true,
                      /*AcrossComments=*/false, /*AlignCompound=*/false,
-                     /*PadOperators=*/true}));
+                     /*AlignFunctionPointers=*/false, /*PadOperators=*/true}));
     IO.enumCase(Value, "AcrossComments",
-                FormatStyle::AlignConsecutiveStyle({/*Enabled=*/true,
-                                                    /*AcrossEmptyLines=*/false,
-                                                    /*AcrossComments=*/true,
-                                                    /*AlignCompound=*/false,
-                                                    /*PadOperators=*/true}));
+                FormatStyle::AlignConsecutiveStyle(
+                    {/*Enabled=*/true, /*AcrossEmptyLines=*/false,
+                     /*AcrossComments=*/true, /*AlignCompound=*/false,
+                     /*AlignFunctionPointers=*/false, /*PadOperators=*/true}));
     IO.enumCase(Value, "AcrossEmptyLinesAndComments",
-                FormatStyle::AlignConsecutiveStyle({/*Enabled=*/true,
-                                                    /*AcrossEmptyLines=*/true,
-                                                    /*AcrossComments=*/true,
-                                                    /*AlignCompound=*/false,
-                                                    /*PadOperators=*/true}));
+                FormatStyle::AlignConsecutiveStyle(
+                    {/*Enabled=*/true, /*AcrossEmptyLines=*/true,
+                     /*AcrossComments=*/true, /*AlignCompound=*/false,
+                     /*AlignFunctionPointers=*/false, /*PadOperators=*/true}));
 
     // For backward compatibility.
     IO.enumCase(Value, "true",
                 FormatStyle::AlignConsecutiveStyle(
                     {/*Enabled=*/true, /*AcrossEmptyLines=*/false,
                      /*AcrossComments=*/false, /*AlignCompound=*/false,
-                     /*PadOperators=*/true}));
+                     /*AlignFunctionPointers=*/false, /*PadOperators=*/true}));
     IO.enumCase(Value, "false",
                 FormatStyle::AlignConsecutiveStyle(
                     {/*Enabled=*/false, /*AcrossEmptyLines=*/false,
                      /*AcrossComments=*/false, /*AlignCompound=*/false,
-                     /*PadOperators=*/true}));
+                     /*AlignFunctionPointers=*/false, /*PadOperators=*/true}));
   }
 
   static void mapping(IO &IO, FormatStyle::AlignConsecutiveStyle &Value) {
@@ -118,6 +116,7 @@ template <> struct MappingTraits {
     IO.mapOptional("AcrossEmptyLines", Value.AcrossEmptyLines);
     IO.mapOptional("AcrossComments", Value.AcrossComments);
     IO.mapOptional("AlignCompound", Value.AlignCompound);
+    IO.mapOptional("AlignFunctionPointers", Value.AlignFunctionPointers);
     IO.mapOptional("PadOperators", Value.PadOperators);
   }
 };
@@ -1055,6 +1054,8 @@ template <> struct MappingTraits {
                    Style.PenaltyBreakFirstLessLess);
     IO.mapOptional("PenaltyBreakOpenParenthesis",
                    Style.PenaltyBreakOpenParenthesis);
+    IO.mapOptional("PenaltyBreakScopeResolution",
+                   Style.PenaltyBreakScopeResolution);
     IO.mapOptional("PenaltyBreakString", Style.PenaltyBreakString);
     IO.mapOptional("PenaltyBreakTemplateDeclaration",
                    Style.PenaltyBreakTemplateDeclaration);
@@ -1084,6 +1085,7 @@ template <> struct MappingTraits {
                    Style.RequiresExpressionIndentation);
     IO.mapOptional("SeparateDefinitionBlocks", Style.SeparateDefinitionBlocks);
     IO.mapOptional("ShortNamespaceLines", Style.ShortNamespaceLines);
+    IO.mapOptional("SkipMacroDefinitionBody", Style.SkipMacroDefinitionBody);
     IO.mapOptional("SortIncludes", Style.SortIncludes);
     IO.mapOptional("SortJavaStaticImport", Style.SortJavaStaticImport);
     IO.mapOptional("SortUsingDeclarations", Style.SortUsingDeclarations);
@@ -1432,6 +1434,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.AlignConsecutiveAssignments.AcrossEmptyLines = false;
   LLVMStyle.AlignConsecutiveAssignments.AcrossComments = false;
   LLVMStyle.AlignConsecutiveAssignments.AlignCompound = false;
+  LLVMStyle.AlignConsecutiveAssignments.AlignFunctionPointers = false;
   LLVMStyle.AlignConsecutiveAssignments.PadOperators = true;
   LLVMStyle.AlignConsecutiveBitFields = {};
   LLVMStyle.AlignConsecutiveDeclarations = {};
@@ -1554,6 +1557,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.RequiresExpressionIndentation = FormatStyle::REI_OuterScope;
   LLVMStyle.SeparateDefinitionBlocks = FormatStyle::SDS_Leave;
   LLVMStyle.ShortNamespaceLines = 1;
+  LLVMStyle.SkipMacroDefinitionBody = false;
   LLVMStyle.SortIncludes = FormatStyle::SI_CaseSensitive;
   LLVMStyle.SortJavaStaticImport = FormatStyle::SJSIO_Before;
   LLVMStyle.SortUsingDeclarations = FormatStyle::SUD_LexicographicNumeric;
@@ -1602,6 +1606,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.PenaltyReturnTypeOnItsOwnLine = 60;
   LLVMStyle.PenaltyBreakBeforeFirstCallParameter = 19;
   LLVMStyle.PenaltyBreakOpenParenthesis = 0;
+  LLVMStyle.PenaltyBreakScopeResolution = 500;
   LLVMStyle.PenaltyBreakTemplateDeclaration = prec::Relational;
   LLVMStyle.PenaltyIndentedWhitespace = 0;
 
@@ -1698,8 +1703,6 @@ FormatStyle getGoogleStyle(FormatStyle::LanguageKind Language) {
           /*BasedOnStyle=*/"google",
       },
   };
-  GoogleStyle.AttributeMacros.push_back("GUARDED_BY");
-  GoogleStyle.AttributeMacros.push_back("ABSL_GUARDED_BY");
 
   GoogleStyle.SpacesBeforeTrailingComments = 2;
   GoogleStyle.Standard = FormatStyle::LS_Auto;
diff --git a/clang/lib/Format/FormatToken.cpp b/clang/lib/Format/FormatToken.cpp
index 7a2df8c53952..b791c5a26bbe 100644
--- a/clang/lib/Format/FormatToken.cpp
+++ b/clang/lib/Format/FormatToken.cpp
@@ -113,8 +113,8 @@ unsigned CommaSeparatedList::formatAfterToken(LineState &State,
   if (!State.NextToken || !State.NextToken->Previous)
     return 0;
 
-  if (Formats.size() == 1)
-    return 0; // Handled by formatFromToken
+  if (Formats.size() <= 1)
+    return 0; // Handled by formatFromToken (1) or avoid severe penalty (0).
 
   // Ensure that we start on the opening brace.
   const FormatToken *LBrace =
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 3f9664f8f78a..dede89f26001 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -148,6 +148,7 @@ namespace format {
   TYPE(StructLBrace)                                                           \
   TYPE(StructRBrace)                                                           \
   TYPE(StructuredBindingLSquare)                                               \
+  TYPE(TableGenMultiLineString)                                                \
   TYPE(TemplateCloser)                                                         \
   TYPE(TemplateOpener)                                                         \
   TYPE(TemplateString)                                                         \
@@ -275,14 +276,15 @@ class AnnotatedLine;
 struct FormatToken {
   FormatToken()
       : HasUnescapedNewline(false), IsMultiline(false), IsFirst(false),
-        MustBreakBefore(false), IsUnterminatedLiteral(false),
-        CanBreakBefore(false), ClosesTemplateDeclaration(false),
-        StartsBinaryExpression(false), EndsBinaryExpression(false),
-        PartOfMultiVariableDeclStmt(false), ContinuesLineCommentSection(false),
-        Finalized(false), ClosesRequiresClause(false),
-        EndsCppAttributeGroup(false), BlockKind(BK_Unknown),
-        Decision(FD_Unformatted), PackingKind(PPK_Inconclusive),
-        TypeIsFinalized(false), Type(TT_Unknown) {}
+        MustBreakBefore(false), MustBreakBeforeFinalized(false),
+        IsUnterminatedLiteral(false), CanBreakBefore(false),
+        ClosesTemplateDeclaration(false), StartsBinaryExpression(false),
+        EndsBinaryExpression(false), PartOfMultiVariableDeclStmt(false),
+        ContinuesLineCommentSection(false), Finalized(false),
+        ClosesRequiresClause(false), EndsCppAttributeGroup(false),
+        BlockKind(BK_Unknown), Decision(FD_Unformatted),
+        PackingKind(PPK_Inconclusive), TypeIsFinalized(false),
+        Type(TT_Unknown) {}
 
   /// The \c Token.
   Token Tok;
@@ -318,6 +320,10 @@ struct FormatToken {
   /// before the token.
   unsigned MustBreakBefore : 1;
 
+  /// Whether MustBreakBefore is finalized during parsing and must not
+  /// be reset between runs.
+  unsigned MustBreakBeforeFinalized : 1;
+
   /// Set to \c true if this token is an unterminated literal.
   unsigned IsUnterminatedLiteral : 1;
 
@@ -416,10 +422,14 @@ struct FormatToken {
   /// to another one please use overwriteFixedType, or even better remove the
   /// need to reassign the type.
   void setFinalizedType(TokenType T) {
+    if (MacroCtx && MacroCtx->Role == MR_UnexpandedArg)
+      return;
     Type = T;
     TypeIsFinalized = true;
   }
   void overwriteFixedType(TokenType T) {
+    if (MacroCtx && MacroCtx->Role == MR_UnexpandedArg)
+      return;
     TypeIsFinalized = false;
     setType(T);
   }
@@ -1202,6 +1212,21 @@ struct AdditionalKeywords {
     kw_verilogHashHash = &IdentTable.get("##");
     kw_apostrophe = &IdentTable.get("\'");
 
+    // TableGen keywords
+    kw_bit = &IdentTable.get("bit");
+    kw_bits = &IdentTable.get("bits");
+    kw_code = &IdentTable.get("code");
+    kw_dag = &IdentTable.get("dag");
+    kw_def = &IdentTable.get("def");
+    kw_defm = &IdentTable.get("defm");
+    kw_defset = &IdentTable.get("defset");
+    kw_defvar = &IdentTable.get("defvar");
+    kw_dump = &IdentTable.get("dump");
+    kw_include = &IdentTable.get("include");
+    kw_list = &IdentTable.get("list");
+    kw_multiclass = &IdentTable.get("multiclass");
+    kw_then = &IdentTable.get("then");
+
     // Keep this at the end of the constructor to make sure everything here
     // is
     // already initialized.
@@ -1294,6 +1319,27 @@ struct AdditionalKeywords {
          kw_wildcard,     kw_wire,
          kw_with,         kw_wor,
          kw_verilogHash,  kw_verilogHashHash});
+
+    TableGenExtraKeywords = std::unordered_set({
+        kw_assert,
+        kw_bit,
+        kw_bits,
+        kw_code,
+        kw_dag,
+        kw_def,
+        kw_defm,
+        kw_defset,
+        kw_defvar,
+        kw_dump,
+        kw_foreach,
+        kw_in,
+        kw_include,
+        kw_let,
+        kw_list,
+        kw_multiclass,
+        kw_string,
+        kw_then,
+    });
   }
 
   // Context sensitive keywords.
@@ -1539,6 +1585,21 @@ struct AdditionalKeywords {
   // Symbols in Verilog that don't exist in C++.
   IdentifierInfo *kw_apostrophe;
 
+  // TableGen keywords
+  IdentifierInfo *kw_bit;
+  IdentifierInfo *kw_bits;
+  IdentifierInfo *kw_code;
+  IdentifierInfo *kw_dag;
+  IdentifierInfo *kw_def;
+  IdentifierInfo *kw_defm;
+  IdentifierInfo *kw_defset;
+  IdentifierInfo *kw_defvar;
+  IdentifierInfo *kw_dump;
+  IdentifierInfo *kw_include;
+  IdentifierInfo *kw_list;
+  IdentifierInfo *kw_multiclass;
+  IdentifierInfo *kw_then;
+
   /// Returns \c true if \p Tok is a keyword or an identifier.
   bool isWordLike(const FormatToken &Tok) const {
     // getIdentifierinfo returns non-null for keywords as well as identifiers.
@@ -1811,6 +1872,27 @@ struct AdditionalKeywords {
     }
   }
 
+  bool isTableGenDefinition(const FormatToken &Tok) const {
+    return Tok.isOneOf(kw_def, kw_defm, kw_defset, kw_defvar, kw_multiclass,
+                       kw_let, tok::kw_class);
+  }
+
+  bool isTableGenKeyword(const FormatToken &Tok) const {
+    switch (Tok.Tok.getKind()) {
+    case tok::kw_class:
+    case tok::kw_else:
+    case tok::kw_false:
+    case tok::kw_if:
+    case tok::kw_int:
+    case tok::kw_true:
+      return true;
+    default:
+      return Tok.is(tok::identifier) &&
+             TableGenExtraKeywords.find(Tok.Tok.getIdentifierInfo()) !=
+                 TableGenExtraKeywords.end();
+    }
+  }
+
 private:
   /// The JavaScript keywords beyond the C++ keyword set.
   std::unordered_set JsExtraKeywords;
@@ -1820,6 +1902,9 @@ struct AdditionalKeywords {
 
   /// The Verilog keywords beyond the C++ keyword set.
   std::unordered_set VerilogExtraKeywords;
+
+  /// The TableGen keywords beyond the C++ keyword set.
+  std::unordered_set TableGenExtraKeywords;
 };
 
 inline bool isLineComment(const FormatToken &FormatTok) {
diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index 61430282c6f8..52a55ea23b5f 100644
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -93,6 +93,10 @@ ArrayRef FormatTokenLexer::lex() {
       // string literals are correctly identified.
       handleCSharpVerbatimAndInterpolatedStrings();
     }
+    if (Style.isTableGen()) {
+      handleTableGenMultilineString();
+      handleTableGenNumericLikeIdentifier();
+    }
     if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline)
       FirstInLineIndex = Tokens.size() - 1;
   } while (Tokens.back()->isNot(tok::eof));
@@ -272,6 +276,14 @@ void FormatTokenLexer::tryMergePreviousTokens() {
       return;
     }
   }
+  // TableGen's Multi line string starts with [{
+  if (Style.isTableGen() && tryMergeTokens({tok::l_square, tok::l_brace},
+                                           TT_TableGenMultiLineString)) {
+    // Set again with finalizing. This must never be annotated as other types.
+    Tokens.back()->setFinalizedType(TT_TableGenMultiLineString);
+    Tokens.back()->Tok.setKind(tok::string_literal);
+    return;
+  }
 }
 
 bool FormatTokenLexer::tryMergeNSStringLiteral() {
@@ -763,6 +775,75 @@ void FormatTokenLexer::handleCSharpVerbatimAndInterpolatedStrings() {
   resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset + 1)));
 }
 
+void FormatTokenLexer::handleTableGenMultilineString() {
+  FormatToken *MultiLineString = Tokens.back();
+  if (MultiLineString->isNot(TT_TableGenMultiLineString))
+    return;
+
+  auto OpenOffset = Lex->getCurrentBufferOffset() - 2 /* "[{" */;
+  // "}]" is the end of multi line string.
+  auto CloseOffset = Lex->getBuffer().find("}]", OpenOffset);
+  if (CloseOffset == StringRef::npos)
+    return;
+  auto Text = Lex->getBuffer().substr(OpenOffset, CloseOffset + 2);
+  MultiLineString->TokenText = Text;
+  resetLexer(SourceMgr.getFileOffset(
+      Lex->getSourceLocation(Lex->getBufferLocation() - 2 + Text.size())));
+  auto FirstLineText = Text;
+  auto FirstBreak = Text.find('\n');
+  // Set ColumnWidth and LastLineColumnWidth when it has multiple lines.
+  if (FirstBreak != StringRef::npos) {
+    MultiLineString->IsMultiline = true;
+    FirstLineText = Text.substr(0, FirstBreak + 1);
+    // LastLineColumnWidth holds the width of the last line.
+    auto LastBreak = Text.rfind('\n');
+    MultiLineString->LastLineColumnWidth = encoding::columnWidthWithTabs(
+        Text.substr(LastBreak + 1), MultiLineString->OriginalColumn,
+        Style.TabWidth, Encoding);
+  }
+  // ColumnWidth holds only the width of the first line.
+  MultiLineString->ColumnWidth = encoding::columnWidthWithTabs(
+      FirstLineText, MultiLineString->OriginalColumn, Style.TabWidth, Encoding);
+}
+
+void FormatTokenLexer::handleTableGenNumericLikeIdentifier() {
+  FormatToken *Tok = Tokens.back();
+  // TableGen identifiers can begin with digits. Such tokens are lexed as
+  // numeric_constant now.
+  if (Tok->isNot(tok::numeric_constant))
+    return;
+  StringRef Text = Tok->TokenText;
+  // The following check is based on llvm::TGLexer::LexToken.
+  // That lexes the token as a number if any of the following holds:
+  // 1. It starts with '+', '-'.
+  // 2. All the characters are digits.
+  // 3. The first non-digit character is 'b', and the next is '0' or '1'.
+  // 4. The first non-digit character is 'x', and the next is a hex digit.
+  // Note that in the case 3 and 4, if the next character does not exists in
+  // this token, the token is an identifier.
+  if (Text.size() < 1 || Text[0] == '+' || Text[0] == '-')
+    return;
+  const auto NonDigitPos = Text.find_if([](char C) { return !isdigit(C); });
+  // All the characters are digits
+  if (NonDigitPos == StringRef::npos)
+    return;
+  char FirstNonDigit = Text[NonDigitPos];
+  if (NonDigitPos < Text.size() - 1) {
+    char TheNext = Text[NonDigitPos + 1];
+    // Regarded as a binary number.
+    if (FirstNonDigit == 'b' && (TheNext == '0' || TheNext == '1'))
+      return;
+    // Regarded as hex number.
+    if (FirstNonDigit == 'x' && isxdigit(TheNext))
+      return;
+  }
+  if (isalpha(FirstNonDigit) || FirstNonDigit == '_') {
+    // This is actually an identifier in TableGen.
+    Tok->Tok.setKind(tok::identifier);
+    Tok->Tok.setIdentifierInfo(nullptr);
+  }
+}
+
 void FormatTokenLexer::handleTemplateStrings() {
   FormatToken *BacktickToken = Tokens.back();
 
@@ -1182,6 +1263,9 @@ FormatToken *FormatTokenLexer::getNextToken() {
                                   tok::kw_operator)) {
       FormatTok->Tok.setKind(tok::identifier);
       FormatTok->Tok.setIdentifierInfo(nullptr);
+    } else if (Style.isTableGen() && !Keywords.isTableGenKeyword(*FormatTok)) {
+      FormatTok->Tok.setKind(tok::identifier);
+      FormatTok->Tok.setIdentifierInfo(nullptr);
     }
   } else if (FormatTok->is(tok::greatergreater)) {
     FormatTok->Tok.setKind(tok::greater);
diff --git a/clang/lib/Format/FormatTokenLexer.h b/clang/lib/Format/FormatTokenLexer.h
index bb6a8ab69c1b..65dd733bd533 100644
--- a/clang/lib/Format/FormatTokenLexer.h
+++ b/clang/lib/Format/FormatTokenLexer.h
@@ -95,6 +95,13 @@ class FormatTokenLexer {
 
   void handleCSharpVerbatimAndInterpolatedStrings();
 
+  // Handles TableGen multiline strings. It has the form [{ ... }].
+  void handleTableGenMultilineString();
+  // Handles TableGen numeric like identifiers.
+  // They have a forms of [0-9]*[_a-zA-Z]([_a-zA-Z0-9]*). But limited to the
+  // case it is not lexed as an integer.
+  void handleTableGenNumericLikeIdentifier();
+
   void tryParsePythonComment();
 
   bool tryMerge_TMacro();
diff --git a/clang/lib/Format/MatchFilePath.cpp b/clang/lib/Format/MatchFilePath.cpp
index 412ee4954587..062b334dcdd8 100644
--- a/clang/lib/Format/MatchFilePath.cpp
+++ b/clang/lib/Format/MatchFilePath.cpp
@@ -19,8 +19,8 @@ using namespace llvm;
 namespace clang {
 namespace format {
 
-// Check whether `FilePath` matches `Pattern` based on POSIX (1003.1-2008)
-// 2.13.1, 2.13.2, and Rule 1 of 2.13.3.
+// Check whether `FilePath` matches `Pattern` based on POSIX 2.13.1, 2.13.2, and
+// Rule 1 of 2.13.3.
 bool matchFilePath(StringRef Pattern, StringRef FilePath) {
   assert(!Pattern.empty());
   assert(!FilePath.empty());
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 3ac3aa3c5e3a..25fcceb87864 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -2209,7 +2209,8 @@ class AnnotatingParser {
         (!NextNonComment && !Line.InMacroBody) ||
         (NextNonComment &&
          (NextNonComment->isPointerOrReference() ||
-          NextNonComment->isOneOf(tok::identifier, tok::string_literal)))) {
+          NextNonComment->is(tok::string_literal) ||
+          (Line.InPragmaDirective && NextNonComment->is(tok::identifier))))) {
       return false;
     }
 
@@ -2234,6 +2235,12 @@ class AnnotatingParser {
     if (PreviousNotConst->ClosesRequiresClause)
       return false;
 
+    if (Style.isTableGen()) {
+      // keywords such as let and def* defines names.
+      if (Keywords.isTableGenDefinition(*PreviousNotConst))
+        return true;
+    }
+
     bool IsPPKeyword = PreviousNotConst->is(tok::identifier) &&
                        PreviousNotConst->Previous &&
                        PreviousNotConst->Previous->is(tok::hash);
@@ -2369,7 +2376,7 @@ class AnnotatingParser {
       }
     }
 
-    if (Tok.Next->is(tok::question))
+    if (Tok.Next->isOneOf(tok::question, tok::ampamp))
       return false;
 
     // `foreach((A a, B b) in someList)` should not be seen as a cast.
@@ -2769,13 +2776,6 @@ class ExpressionParser {
       // Consume operators with higher precedence.
       parse(Precedence + 1);
 
-      // Do not assign fake parenthesis to tokens that are part of an
-      // unexpanded macro call. The line within the macro call contains
-      // the parenthesis and commas, and we will not find operators within
-      // that structure.
-      if (Current && Current->MacroParent)
-        break;
-
       int CurrentPrecedence = getCurrentPrecedence();
 
       if (Precedence == CurrentPrecedence && Current &&
@@ -2919,6 +2919,13 @@ class ExpressionParser {
 
   void addFakeParenthesis(FormatToken *Start, prec::Level Precedence,
                           FormatToken *End = nullptr) {
+    // Do not assign fake parenthesis to tokens that are part of an
+    // unexpanded macro call. The line within the macro call contains
+    // the parenthesis and commas, and we will not find operators within
+    // that structure.
+    if (Start->MacroParent)
+      return;
+
     Start->FakeLParens.push_back(Precedence);
     if (Precedence > prec::Unknown)
       Start->StartsBinaryExpression = true;
@@ -3759,7 +3766,7 @@ unsigned TokenAnnotator::splitPenalty(const AnnotatedLine &Line,
   }
 
   if (Left.is(tok::coloncolon))
-    return 500;
+    return Style.PenaltyBreakScopeResolution;
   if (Right.isOneOf(TT_StartOfName, TT_FunctionDeclarationName) ||
       Right.is(tok::kw_operator)) {
     if (Line.startsWith(tok::kw_for) && Right.PartOfMultiVariableDeclStmt)
@@ -4668,6 +4675,10 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
   } else if (Style.Language == FormatStyle::LK_Java) {
     if (Left.is(tok::r_square) && Right.is(tok::l_brace))
       return true;
+    // spaces inside square brackets.
+    if (Left.is(tok::l_square) || Right.is(tok::r_square))
+      return Style.SpacesInSquareBrackets;
+
     if (Left.is(Keywords.kw_synchronized) && Right.is(tok::l_paren)) {
       return Style.SpaceBeforeParensOptions.AfterControlStatements ||
              spaceRequiredBeforeParens(Right);
@@ -5151,6 +5162,14 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
     return true;
   if (Left.IsUnterminatedLiteral)
     return true;
+  // FIXME: Breaking after newlines seems useful in general. Turn this into an
+  // option and recognize more cases like endl etc, and break independent of
+  // what comes after operator lessless.
+  if (Right.is(tok::lessless) && Right.Next &&
+      Right.Next->is(tok::string_literal) && Left.is(tok::string_literal) &&
+      Left.TokenText.ends_with("\\n\"")) {
+    return true;
+  }
   if (Right.is(TT_RequiresClause)) {
     switch (Style.RequiresClausePosition) {
     case FormatStyle::RCPS_OwnLine:
diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp
index 56077499c39d..adeb07243487 100644
--- a/clang/lib/Format/UnwrappedLineFormatter.cpp
+++ b/clang/lib/Format/UnwrappedLineFormatter.cpp
@@ -95,7 +95,7 @@ class LevelIndentTracker {
   /// level to the same indent.
   /// Note that \c nextLine must have been called before this method.
   void adjustToUnmodifiedLine(const AnnotatedLine &Line) {
-    if (Line.InPPDirective)
+    if (Line.InPPDirective || Line.IsContinuation)
       return;
     assert(Line.Level < IndentForLevel.size());
     if (Line.First->is(tok::comment) && IndentForLevel[Line.Level] != -1)
@@ -954,13 +954,15 @@ static void markFinalized(FormatToken *Tok) {
       // will be modified as unexpanded arguments (as part of the macro call
       // formatting) in the next pass.
       Tok->MacroCtx->Role = MR_UnexpandedArg;
-      // Reset whether spaces are required before this token, as that is context
-      // dependent, and that context may change when formatting the macro call.
-      // For example, given M(x) -> 2 * x, and the macro call M(var),
-      // the token 'var' will have SpacesRequiredBefore = 1 after being
+      // Reset whether spaces or a line break are required before this token, as
+      // that is context dependent, and that context may change when formatting
+      // the macro call.  For example, given M(x) -> 2 * x, and the macro call
+      // M(var), the token 'var' will have SpacesRequiredBefore = 1 after being
       // formatted as part of the expanded macro, but SpacesRequiredBefore = 0
       // for its position within the macro call.
       Tok->SpacesRequiredBefore = 0;
+      if (!Tok->MustBreakBeforeFinalized)
+        Tok->MustBreakBefore = 0;
     } else {
       Tok->Finalized = true;
     }
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 684609747a55..b904e0e56d9e 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -1170,6 +1170,15 @@ void UnwrappedLineParser::parsePPDefine() {
   assert((int)Line->PPLevel >= 0);
   Line->InMacroBody = true;
 
+  if (Style.SkipMacroDefinitionBody) {
+    do {
+      FormatTok->Finalized = true;
+      nextToken();
+    } while (!eof());
+    addUnwrappedLine();
+    return;
+  }
+
   if (FormatTok->is(tok::identifier) &&
       Tokens->peekNextToken()->is(tok::colon)) {
     nextToken();
@@ -1652,7 +1661,8 @@ void UnwrappedLineParser::parseStructuralElement(
     // In Verilog labels can be any expression, so we don't do them here.
     // JS doesn't have macros, and within classes colons indicate fields, not
     // labels.
-    if (!Style.isJavaScript() && !Style.isVerilog() &&
+    // TableGen doesn't have labels.
+    if (!Style.isJavaScript() && !Style.isVerilog() && !Style.isTableGen() &&
         Tokens->peekNextToken()->is(tok::colon) && !Line->MustBeDeclaration) {
       nextToken();
       Line->Tokens.begin()->Tok->MustBreakBefore = true;
@@ -1781,6 +1791,12 @@ void UnwrappedLineParser::parseStructuralElement(
         addUnwrappedLine();
         return;
       }
+      if (Style.isTableGen()) {
+        // Do nothing special. In this case the l_brace becomes FunctionLBrace.
+        // This is same as def and so on.
+        nextToken();
+        break;
+      }
       [[fallthrough]];
     case tok::kw_struct:
     case tok::kw_union:
@@ -2019,6 +2035,16 @@ void UnwrappedLineParser::parseStructuralElement(
         // initialisers are indented the same way.
         if (Style.isCSharp())
           FormatTok->setBlockKind(BK_BracedInit);
+        // TableGen's defset statement has syntax of the form,
+        // `defset   = { ... }`
+        if (Style.isTableGen() &&
+            Line->Tokens.begin()->Tok->is(Keywords.kw_defset)) {
+          FormatTok->setFinalizedType(TT_FunctionLBrace);
+          parseBlock(/*MustBeDeclaration=*/false, /*AddLevels=*/1u,
+                     /*MunchSemi=*/false);
+          addUnwrappedLine();
+          break;
+        }
         nextToken();
         parseBracedList();
       } else if (Style.Language == FormatStyle::LK_Proto &&
@@ -2308,7 +2334,7 @@ bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
       LeftSquare->isCppStructuredBinding(Style)) {
     return false;
   }
-  if (FormatTok->is(tok::l_square))
+  if (FormatTok->is(tok::l_square) || tok::isLiteral(FormatTok->Tok.getKind()))
     return false;
   if (FormatTok->is(tok::r_square)) {
     const FormatToken *Next = Tokens->peekNextToken(/*SkipComment=*/true);
@@ -2734,6 +2760,14 @@ FormatToken *UnwrappedLineParser::parseIfThenElse(IfStmtKind *IfKind,
     }
   }
 
+  // TableGen's if statement has the form of `if  then { ... }`.
+  if (Style.isTableGen()) {
+    while (!eof() && FormatTok->isNot(Keywords.kw_then)) {
+      // Simply skip until then. This range only contains a value.
+      nextToken();
+    }
+  }
+
   // Handle `if !consteval`.
   if (FormatTok->is(tok::exclaim))
     nextToken();
@@ -3873,6 +3907,9 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
   const FormatToken &InitialToken = *FormatTok;
   nextToken();
 
+  auto IsNonMacroIdentifier = [](const FormatToken *Tok) {
+    return Tok->is(tok::identifier) && Tok->TokenText != Tok->TokenText.upper();
+  };
   // The actual identifier can be a nested name specifier, and in macros
   // it is often token-pasted.
   // An [[attribute]] can be before the identifier.
@@ -3894,27 +3931,26 @@ void UnwrappedLineParser::parseRecord(bool ParseAsExpr) {
     }
     if (FormatTok->is(tok::l_square) && handleCppAttributes())
       continue;
-    bool IsNonMacroIdentifier =
-        FormatTok->is(tok::identifier) &&
-        FormatTok->TokenText != FormatTok->TokenText.upper();
     nextToken();
     // We can have macros in between 'class' and the class name.
-    if (!IsNonMacroIdentifier && FormatTok->is(tok::l_paren))
+    if (!IsNonMacroIdentifier(FormatTok->Previous) &&
+        FormatTok->is(tok::l_paren)) {
       parseParens();
+    }
   }
 
-  // Note that parsing away template declarations here leads to incorrectly
-  // accepting function declarations as record declarations.
-  // In general, we cannot solve this problem. Consider:
-  // class A B() {}
-  // which can be a function definition or a class definition when B() is a
-  // macro. If we find enough real-world cases where this is a problem, we
-  // can parse for the 'template' keyword in the beginning of the statement,
-  // and thus rule out the record production in case there is no template
-  // (this would still leave us with an ambiguity between template function
-  // and class declarations).
   if (FormatTok->isOneOf(tok::colon, tok::less)) {
+    int AngleNestingLevel = 0;
     do {
+      if (FormatTok->is(tok::less))
+        ++AngleNestingLevel;
+      else if (FormatTok->is(tok::greater))
+        --AngleNestingLevel;
+
+      if (AngleNestingLevel == 0 && FormatTok->is(tok::l_paren) &&
+          IsNonMacroIdentifier(FormatTok->Previous)) {
+        break;
+      }
       if (FormatTok->is(tok::l_brace)) {
         calculateBraceTypes(/*ExpectClassBody=*/true);
         if (!tryToParseBracedList())
@@ -4675,6 +4711,7 @@ void UnwrappedLineParser::readToken(int LevelDifference) {
         conditionalCompilationEnd();
       FormatTok = Tokens->getNextToken();
       FormatTok->MustBreakBefore = true;
+      FormatTok->MustBreakBeforeFinalized = true;
     }
 
     auto IsFirstNonCommentOnLine = [](bool FirstNonCommentOnLine,
@@ -4891,6 +4928,7 @@ void UnwrappedLineParser::pushToken(FormatToken *Tok) {
   Line->Tokens.push_back(UnwrappedLineNode(Tok));
   if (MustBreakBeforeNextToken) {
     Line->Tokens.back().Tok->MustBreakBefore = true;
+    Line->Tokens.back().Tok->MustBreakBeforeFinalized = true;
     MustBreakBeforeNextToken = false;
   }
 }
diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp
index 3bc6915b8df0..df84f97a8e8a 100644
--- a/clang/lib/Format/WhitespaceManager.cpp
+++ b/clang/lib/Format/WhitespaceManager.cpp
@@ -978,7 +978,14 @@ void WhitespaceManager::alignConsecutiveDeclarations() {
 
   AlignTokens(
       Style,
-      [](Change const &C) {
+      [&](Change const &C) {
+        if (Style.AlignConsecutiveDeclarations.AlignFunctionPointers) {
+          for (const auto *Prev = C.Tok->Previous; Prev; Prev = Prev->Previous)
+            if (Prev->is(tok::equal))
+              return false;
+          if (C.Tok->is(TT_FunctionTypeLParen))
+            return true;
+        }
         if (C.Tok->is(TT_FunctionDeclarationName))
           return true;
         if (C.Tok->isNot(TT_StartOfName))
@@ -1359,11 +1366,12 @@ void WhitespaceManager::alignArrayInitializersLeftJustified(
   auto &Cells = CellDescs.Cells;
   // Now go through and fixup the spaces.
   auto *CellIter = Cells.begin();
-  // The first cell needs to be against the left brace.
-  if (Changes[CellIter->Index].NewlinesBefore == 0)
-    Changes[CellIter->Index].Spaces = BracePadding;
-  else
-    Changes[CellIter->Index].Spaces = CellDescs.InitialSpaces;
+  // The first cell of every row needs to be against the left brace.
+  for (const auto *Next = CellIter; Next; Next = Next->NextColumnElement) {
+    auto &Change = Changes[Next->Index];
+    Change.Spaces =
+        Change.NewlinesBefore == 0 ? BracePadding : CellDescs.InitialSpaces;
+  }
   ++CellIter;
   for (auto i = 1U; i < CellDescs.CellCounts[0]; i++, ++CellIter) {
     auto MaxNetWidth = getMaximumNetWidth(
@@ -1444,8 +1452,10 @@ WhitespaceManager::CellDescriptions WhitespaceManager::getCells(unsigned Start,
       } else if (C.Tok->is(tok::comma)) {
         if (!Cells.empty())
           Cells.back().EndIndex = i;
-        if (C.Tok->getNextNonComment()->isNot(tok::r_brace)) // dangling comma
+        if (const auto *Next = C.Tok->getNextNonComment();
+            Next && Next->isNot(tok::r_brace)) { // dangling comma
           ++Cell;
+        }
       }
     } else if (Depth == 1) {
       if (C.Tok == MatchingParen) {
diff --git a/clang/lib/Format/WhitespaceManager.h b/clang/lib/Format/WhitespaceManager.h
index 24fe492dcb02..8ac73305871a 100644
--- a/clang/lib/Format/WhitespaceManager.h
+++ b/clang/lib/Format/WhitespaceManager.h
@@ -55,7 +55,7 @@ class WhitespaceManager {
   /// this replacement. It is needed for determining how \p Spaces is turned
   /// into tabs and spaces for some format styles.
   void replaceWhitespace(FormatToken &Tok, unsigned Newlines, unsigned Spaces,
-                         unsigned StartOfTokenColumn, bool isAligned = false,
+                         unsigned StartOfTokenColumn, bool IsAligned = false,
                          bool InPPDirective = false);
 
   /// Adds information about an unchangeable token's whitespace.
@@ -282,6 +282,7 @@ class WhitespaceManager {
     for (auto PrevIter = Start; PrevIter != End; ++PrevIter) {
       // If we broke the line the initial spaces are already
       // accounted for.
+      assert(PrevIter->Index < Changes.size());
       if (Changes[PrevIter->Index].NewlinesBefore > 0)
         NetWidth = 0;
       NetWidth +=
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 56bbef9697b6..a25aa88bd85e 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -470,7 +470,7 @@ void CompilerInstance::createPreprocessor(TranslationUnitKind TUKind) {
 
   // Predefine macros and configure the preprocessor.
   InitializePreprocessor(*PP, PPOpts, getPCHContainerReader(),
-                         getFrontendOpts());
+                         getFrontendOpts(), getCodeGenOpts());
 
   // Initialize the header search object.  In CUDA compilations, we use the aux
   // triple (the host triple) to initialize our header search, since we need to
@@ -543,7 +543,7 @@ std::string CompilerInstance::getSpecificModuleCachePath(StringRef ModuleHash) {
   SmallString<256> SpecificModuleCache(getHeaderSearchOpts().ModuleCachePath);
   if (!SpecificModuleCache.empty() && !getHeaderSearchOpts().DisableModuleHash)
     llvm::sys::path::append(SpecificModuleCache, ModuleHash);
-  return std::string(SpecificModuleCache.str());
+  return std::string(SpecificModuleCache);
 }
 
 // ASTContext
@@ -2204,7 +2204,7 @@ void CompilerInstance::createModuleFromSource(SourceLocation ImportLoc,
   // Build the module, inheriting any modules that we've built locally.
   if (compileModuleImpl(*this, ImportLoc, ModuleName, Input, StringRef(),
                         ModuleFileName, PreBuildStep, PostBuildStep)) {
-    BuiltModules[std::string(ModuleName)] = std::string(ModuleFileName.str());
+    BuiltModules[std::string(ModuleName)] = std::string(ModuleFileName);
     llvm::sys::RemoveFileOnSignal(ModuleFileName);
   }
 }
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 11f3f2c2d642..feb4de2084b8 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -1861,20 +1861,6 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
     if (Args.hasArg(OPT_funified_lto))
       Opts.PrepareForThinLTO = true;
   }
-  if (Arg *A = Args.getLastArg(options::OPT_ffat_lto_objects,
-                               options::OPT_fno_fat_lto_objects)) {
-    if (A->getOption().matches(options::OPT_ffat_lto_objects)) {
-      if (Arg *Uni = Args.getLastArg(options::OPT_funified_lto,
-                                     options::OPT_fno_unified_lto)) {
-        if (Uni->getOption().matches(options::OPT_fno_unified_lto))
-          Diags.Report(diag::err_drv_incompatible_options)
-              << A->getAsString(Args) << "-fno-unified-lto";
-      } else
-        Diags.Report(diag::err_drv_argument_only_allowed_with)
-            << A->getAsString(Args) << "-funified-lto";
-    }
-  }
-
   if (Arg *A = Args.getLastArg(OPT_fthinlto_index_EQ)) {
     if (IK.getLanguage() != Language::LLVM_IR)
       Diags.Report(diag::err_drv_argument_only_allowed_with)
@@ -3176,7 +3162,7 @@ static bool ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args,
       llvm::sys::fs::make_absolute(WorkingDir, P);
   }
   llvm::sys::path::remove_dots(P);
-  Opts.ModuleCachePath = std::string(P.str());
+  Opts.ModuleCachePath = std::string(P);
 
   // Only the -fmodule-file== form.
   for (const auto *A : Args.filtered(OPT_fmodule_file)) {
@@ -3217,7 +3203,7 @@ static bool ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args,
       SmallString<32> Buffer;
       llvm::sys::path::append(Buffer, Opts.Sysroot,
                               llvm::StringRef(A->getValue()).substr(1));
-      Path = std::string(Buffer.str());
+      Path = std::string(Buffer);
     }
 
     Opts.AddPath(Path, Group, IsFramework,
@@ -3480,7 +3466,8 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
                 Twine(Major) + "." + Twine(Minor) + "." + Twine(Subminor));
   }
 
-  if ((!Opts.GNUMode && !Opts.MSVCCompat && !Opts.CPlusPlus17) || T.isOSzOS()) {
+  if ((!Opts.GNUMode && !Opts.MSVCCompat && !Opts.CPlusPlus17 && !Opts.C23) ||
+      T.isOSzOS()) {
     if (!Opts.Trigraphs)
       GenerateArg(Consumer, OPT_fno_trigraphs);
   } else {
@@ -3876,10 +3863,11 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
 
   // Mimicking gcc's behavior, trigraphs are only enabled if -trigraphs
   // is specified, or -std is set to a conforming mode.
-  // Trigraphs are disabled by default in c++1z onwards.
+  // Trigraphs are disabled by default in C++17 and C23 onwards.
   // For z/OS, trigraphs are enabled by default (without regard to the above).
   Opts.Trigraphs =
-      (!Opts.GNUMode && !Opts.MSVCCompat && !Opts.CPlusPlus17) || T.isOSzOS();
+      (!Opts.GNUMode && !Opts.MSVCCompat && !Opts.CPlusPlus17 && !Opts.C23) ||
+      T.isOSzOS();
   Opts.Trigraphs =
       Args.hasFlag(OPT_ftrigraphs, OPT_fno_trigraphs, Opts.Trigraphs);
 
@@ -4236,20 +4224,35 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
     // TODO: Revisit restricting SPIR-V to logical once we've figured out how to
     // handle PhysicalStorageBuffer64 memory model
     if (T.isDXIL() || T.isSPIRVLogical()) {
-      enum { ShaderModel, ShaderStage };
+      enum { ShaderModel, VulkanEnv, ShaderStage };
+      enum { OS, Environment };
+
+      int ExpectedOS = T.isSPIRVLogical() ? VulkanEnv : ShaderModel;
+
       if (T.getOSName().empty()) {
         Diags.Report(diag::err_drv_hlsl_bad_shader_required_in_target)
-            << ShaderModel << T.str();
-      } else if (!T.isShaderModelOS() || T.getOSVersion() == VersionTuple(0)) {
-        Diags.Report(diag::err_drv_hlsl_bad_shader_unsupported)
-            << ShaderModel << T.getOSName() << T.str();
+            << ExpectedOS << OS << T.str();
       } else if (T.getEnvironmentName().empty()) {
         Diags.Report(diag::err_drv_hlsl_bad_shader_required_in_target)
-            << ShaderStage << T.str();
+            << ShaderStage << Environment << T.str();
       } else if (!T.isShaderStageEnvironment()) {
         Diags.Report(diag::err_drv_hlsl_bad_shader_unsupported)
             << ShaderStage << T.getEnvironmentName() << T.str();
       }
+
+      if (T.isDXIL()) {
+        if (!T.isShaderModelOS() || T.getOSVersion() == VersionTuple(0)) {
+          Diags.Report(diag::err_drv_hlsl_bad_shader_unsupported)
+              << ShaderModel << T.getOSName() << T.str();
+        }
+      } else if (T.isSPIRVLogical()) {
+        if (!T.isVulkanOS() || T.getVulkanVersion() == VersionTuple(0)) {
+          Diags.Report(diag::err_drv_hlsl_bad_shader_unsupported)
+              << VulkanEnv << T.getOSName() << T.str();
+        }
+      } else {
+        llvm_unreachable("expected DXIL or SPIR-V target");
+      }
     } else
       Diags.Report(diag::err_drv_hlsl_unsupported_target) << T.str();
   }
diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index d83128adb511..1b91c86f9139 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -728,6 +728,7 @@ static void InitializeCPlusPlusFeatureTestMacros(const LangOptions &LangOpts,
     Builder.defineMacro("__cpp_size_t_suffix", "202011L");
     Builder.defineMacro("__cpp_if_consteval", "202106L");
     Builder.defineMacro("__cpp_multidimensional_subscript", "202211L");
+    Builder.defineMacro("__cpp_auto_cast", "202110L");
   }
 
   // We provide those C++23 features as extensions in earlier language modes, so
@@ -1364,12 +1365,22 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
   TI.getTargetDefines(LangOpts, Builder);
 }
 
+static void InitializePGOProfileMacros(const CodeGenOptions &CodeGenOpts,
+                                       MacroBuilder &Builder) {
+  if (CodeGenOpts.hasProfileInstr())
+    Builder.defineMacro("__LLVM_INSTR_PROFILE_GENERATE");
+
+  if (CodeGenOpts.hasProfileIRUse() || CodeGenOpts.hasProfileClangUse())
+    Builder.defineMacro("__LLVM_INSTR_PROFILE_USE");
+}
+
 /// InitializePreprocessor - Initialize the preprocessor getting it and the
 /// environment ready to process a single file.
-void clang::InitializePreprocessor(
-    Preprocessor &PP, const PreprocessorOptions &InitOpts,
-    const PCHContainerReader &PCHContainerRdr,
-    const FrontendOptions &FEOpts) {
+void clang::InitializePreprocessor(Preprocessor &PP,
+                                   const PreprocessorOptions &InitOpts,
+                                   const PCHContainerReader &PCHContainerRdr,
+                                   const FrontendOptions &FEOpts,
+                                   const CodeGenOptions &CodeGenOpts) {
   const LangOptions &LangOpts = PP.getLangOpts();
   std::string PredefineBuffer;
   PredefineBuffer.reserve(4080);
@@ -1416,6 +1427,11 @@ void clang::InitializePreprocessor(
   InitializeStandardPredefinedMacros(PP.getTargetInfo(), PP.getLangOpts(),
                                      FEOpts, Builder);
 
+  // The PGO instrumentation profile macros are driven by options
+  // -fprofile[-instr]-generate/-fcs-profile-generate/-fprofile[-instr]-use,
+  // hence they are not guarded by InitOpts.UsePredefines.
+  InitializePGOProfileMacros(CodeGenOpts, Builder);
+
   // Add on the predefines from the driver.  Wrap in a #line directive to report
   // that they come from the command line.
   Builder.append("# 1 \"\" 1");
diff --git a/clang/lib/Frontend/LogDiagnosticPrinter.cpp b/clang/lib/Frontend/LogDiagnosticPrinter.cpp
index 32fc6cb2acd8..469d1c22633a 100644
--- a/clang/lib/Frontend/LogDiagnosticPrinter.cpp
+++ b/clang/lib/Frontend/LogDiagnosticPrinter.cpp
@@ -134,7 +134,7 @@ void LogDiagnosticPrinter::HandleDiagnostic(DiagnosticsEngine::Level Level,
   // Format the message.
   SmallString<100> MessageStr;
   Info.FormatDiagnostic(MessageStr);
-  DE.Message = std::string(MessageStr.str());
+  DE.Message = std::string(MessageStr);
 
   // Set the location information.
   DE.Filename = "";
diff --git a/clang/lib/Frontend/Rewrite/FrontendActions.cpp b/clang/lib/Frontend/Rewrite/FrontendActions.cpp
index 14569013b92c..cf5a9437e89e 100644
--- a/clang/lib/Frontend/Rewrite/FrontendActions.cpp
+++ b/clang/lib/Frontend/Rewrite/FrontendActions.cpp
@@ -77,7 +77,7 @@ class FixItActionSuffixInserter : public FixItOptions {
     SmallString<128> Path(Filename);
     llvm::sys::path::replace_extension(Path,
       NewSuffix + llvm::sys::path::extension(Path));
-    return std::string(Path.str());
+    return std::string(Path);
   }
 };
 
@@ -88,7 +88,7 @@ class FixItRewriteToTemp : public FixItOptions {
     llvm::sys::fs::createTemporaryFile(llvm::sys::path::filename(Filename),
                                        llvm::sys::path::extension(Filename).drop_front(), fd,
                                        Path);
-    return std::string(Path.str());
+    return std::string(Path);
   }
 };
 } // end anonymous namespace
diff --git a/clang/lib/Frontend/TextDiagnosticBuffer.cpp b/clang/lib/Frontend/TextDiagnosticBuffer.cpp
index 90f273e65f88..681bc25f46b8 100644
--- a/clang/lib/Frontend/TextDiagnosticBuffer.cpp
+++ b/clang/lib/Frontend/TextDiagnosticBuffer.cpp
@@ -32,20 +32,20 @@ void TextDiagnosticBuffer::HandleDiagnostic(DiagnosticsEngine::Level Level,
                          "Diagnostic not handled during diagnostic buffering!");
   case DiagnosticsEngine::Note:
     All.emplace_back(Level, Notes.size());
-    Notes.emplace_back(Info.getLocation(), std::string(Buf.str()));
+    Notes.emplace_back(Info.getLocation(), std::string(Buf));
     break;
   case DiagnosticsEngine::Warning:
     All.emplace_back(Level, Warnings.size());
-    Warnings.emplace_back(Info.getLocation(), std::string(Buf.str()));
+    Warnings.emplace_back(Info.getLocation(), std::string(Buf));
     break;
   case DiagnosticsEngine::Remark:
     All.emplace_back(Level, Remarks.size());
-    Remarks.emplace_back(Info.getLocation(), std::string(Buf.str()));
+    Remarks.emplace_back(Info.getLocation(), std::string(Buf));
     break;
   case DiagnosticsEngine::Error:
   case DiagnosticsEngine::Fatal:
     All.emplace_back(Level, Errors.size());
-    Errors.emplace_back(Info.getLocation(), std::string(Buf.str()));
+    Errors.emplace_back(Info.getLocation(), std::string(Buf));
     break;
   }
 }
diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
index 8a3d2286cd16..f508408ba706 100644
--- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
+++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
@@ -611,12 +611,19 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
                    diag::err_verify_missing_start) << KindStr;
       continue;
     }
+    llvm::SmallString<8> CloseBrace("}}");
+    const char *const DelimBegin = PH.C;
     PH.Advance();
+    // Count the number of opening braces for `string` kinds
+    for (; !D.RegexKind && PH.Next("{"); PH.Advance())
+      CloseBrace += '}';
     const char* const ContentBegin = PH.C; // mark content begin
-    // Search for token: }}
-    if (!PH.SearchClosingBrace("{{", "}}")) {
-      Diags.Report(Pos.getLocWithOffset(PH.C-PH.Begin),
-                   diag::err_verify_missing_end) << KindStr;
+    // Search for closing brace
+    StringRef OpenBrace(DelimBegin, ContentBegin - DelimBegin);
+    if (!PH.SearchClosingBrace(OpenBrace, CloseBrace)) {
+      Diags.Report(Pos.getLocWithOffset(PH.C - PH.Begin),
+                   diag::err_verify_missing_end)
+          << KindStr << CloseBrace;
       continue;
     }
     const char* const ContentEnd = PH.P; // mark content end
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index 47c47ef8732b..c3a35cd262d2 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -420,8 +420,8 @@ if(ARM IN_LIST LLVM_TARGETS_TO_BUILD OR AArch64 IN_LIST LLVM_TARGETS_TO_BUILD)
   clang_generate_header(-gen-arm-fp16 arm_fp16.td arm_fp16.h)
   # Generate arm_sve.h
   clang_generate_header(-gen-arm-sve-header arm_sve.td arm_sve.h)
-  # Generate arm_sme_draft_spec_subject_to_change.h
-  clang_generate_header(-gen-arm-sme-header arm_sme.td arm_sme_draft_spec_subject_to_change.h)
+  # Generate arm_sme.h
+  clang_generate_header(-gen-arm-sme-header arm_sme.td arm_sme.h)
   # Generate arm_bf16.h
   clang_generate_header(-gen-arm-bf16 arm_bf16.td arm_bf16.h)
   # Generate arm_mve.h
@@ -444,7 +444,7 @@ if(ARM IN_LIST LLVM_TARGETS_TO_BUILD OR AArch64 IN_LIST LLVM_TARGETS_TO_BUILD)
 
   list(APPEND aarch64_only_generated_files
     "${CMAKE_CURRENT_BINARY_DIR}/arm_sve.h"
-    "${CMAKE_CURRENT_BINARY_DIR}/arm_sme_draft_spec_subject_to_change.h"
+    "${CMAKE_CURRENT_BINARY_DIR}/arm_sme.h"
     "${CMAKE_CURRENT_BINARY_DIR}/arm_bf16.h"
     "${CMAKE_CURRENT_BINARY_DIR}/arm_vector_types.h"
     )
diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h
index 61d80258d166..9cd34948e3c5 100644
--- a/clang/lib/Headers/arm_acle.h
+++ b/clang/lib/Headers/arm_acle.h
@@ -4,6 +4,13 @@
  * See https://llvm.org/LICENSE.txt for license information.
  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
+ * The Arm C Language Extensions specifications can be found in the following
+ * link: https://github.com/ARM-software/acle/releases
+ *
+ * The ACLE section numbers are subject to change. When consulting the
+ * specifications, it is recommended to search using section titles if
+ * the section numbers look outdated.
+ *
  *===-----------------------------------------------------------------------===
  */
 
@@ -20,8 +27,8 @@
 extern "C" {
 #endif
 
-/* 8 SYNCHRONIZATION, BARRIER AND HINT INTRINSICS */
-/* 8.3 Memory barriers */
+/* 7 SYNCHRONIZATION, BARRIER AND HINT INTRINSICS */
+/* 7.3 Memory barriers */
 #if !__has_builtin(__dmb)
 #define __dmb(i) __builtin_arm_dmb(i)
 #endif
@@ -32,7 +39,7 @@ extern "C" {
 #define __isb(i) __builtin_arm_isb(i)
 #endif
 
-/* 8.4 Hints */
+/* 7.4 Hints */
 
 #if !__has_builtin(__wfi)
 static __inline__ void __attribute__((__always_inline__, __nodebug__)) __wfi(void) {
@@ -68,7 +75,7 @@ static __inline__ void __attribute__((__always_inline__, __nodebug__)) __yield(v
 #define __dbg(t) __builtin_arm_dbg(t)
 #endif
 
-/* 8.5 Swap */
+/* 7.5 Swap */
 static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
 __swp(uint32_t __x, volatile uint32_t *__p) {
   uint32_t v;
@@ -78,8 +85,8 @@ __swp(uint32_t __x, volatile uint32_t *__p) {
   return v;
 }
 
-/* 8.6 Memory prefetch intrinsics */
-/* 8.6.1 Data prefetch */
+/* 7.6 Memory prefetch intrinsics */
+/* 7.6.1 Data prefetch */
 #define __pld(addr) __pldx(0, 0, 0, addr)
 
 #if defined(__ARM_32BIT_STATE) && __ARM_32BIT_STATE
@@ -90,7 +97,7 @@ __swp(uint32_t __x, volatile uint32_t *__p) {
   __builtin_arm_prefetch(addr, access_kind, cache_level, retention_policy, 1)
 #endif
 
-/* 8.6.2 Instruction prefetch */
+/* 7.6.2 Instruction prefetch */
 #define __pli(addr) __plix(0, 0, addr)
 
 #if defined(__ARM_32BIT_STATE) && __ARM_32BIT_STATE
@@ -101,15 +108,15 @@ __swp(uint32_t __x, volatile uint32_t *__p) {
   __builtin_arm_prefetch(addr, 0, cache_level, retention_policy, 0)
 #endif
 
-/* 8.7 NOP */
+/* 7.7 NOP */
 #if !defined(_MSC_VER) || !defined(__aarch64__)
 static __inline__ void __attribute__((__always_inline__, __nodebug__)) __nop(void) {
   __builtin_arm_nop();
 }
 #endif
 
-/* 9 DATA-PROCESSING INTRINSICS */
-/* 9.2 Miscellaneous data-processing intrinsics */
+/* 8 DATA-PROCESSING INTRINSICS */
+/* 8.2 Miscellaneous data-processing intrinsics */
 /* ROR */
 static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
 __ror(uint32_t __x, uint32_t __y) {
@@ -248,9 +255,7 @@ __rbitl(unsigned long __t) {
 #endif
 }
 
-/*
- * 9.3 16-bit multiplications
- */
+/* 8.3 16-bit multiplications */
 #if defined(__ARM_FEATURE_DSP) && __ARM_FEATURE_DSP
 static __inline__ int32_t __attribute__((__always_inline__,__nodebug__))
 __smulbb(int32_t __a, int32_t __b) {
@@ -279,18 +284,18 @@ __smulwt(int32_t __a, int32_t __b) {
 #endif
 
 /*
- * 9.4 Saturating intrinsics
+ * 8.4 Saturating intrinsics
  *
  * FIXME: Change guard to their corresponding __ARM_FEATURE flag when Q flag
  * intrinsics are implemented and the flag is enabled.
  */
-/* 9.4.1 Width-specified saturation intrinsics */
+/* 8.4.1 Width-specified saturation intrinsics */
 #if defined(__ARM_FEATURE_SAT) && __ARM_FEATURE_SAT
 #define __ssat(x, y) __builtin_arm_ssat(x, y)
 #define __usat(x, y) __builtin_arm_usat(x, y)
 #endif
 
-/* 9.4.2 Saturating addition and subtraction intrinsics */
+/* 8.4.2 Saturating addition and subtraction intrinsics */
 #if defined(__ARM_FEATURE_DSP) && __ARM_FEATURE_DSP
 static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
 __qadd(int32_t __t, int32_t __v) {
@@ -308,7 +313,7 @@ __qdbl(int32_t __t) {
 }
 #endif
 
-/* 9.4.3 Accumultating multiplications */
+/* 8.4.3 Accumultating multiplications */
 #if defined(__ARM_FEATURE_DSP) && __ARM_FEATURE_DSP
 static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
 __smlabb(int32_t __a, int32_t __b, int32_t __c) {
@@ -337,13 +342,13 @@ __smlawt(int32_t __a, int32_t __b, int32_t __c) {
 #endif
 
 
-/* 9.5.4 Parallel 16-bit saturation */
+/* 8.5.4 Parallel 16-bit saturation */
 #if defined(__ARM_FEATURE_SIMD32) && __ARM_FEATURE_SIMD32
 #define __ssat16(x, y) __builtin_arm_ssat16(x, y)
 #define __usat16(x, y) __builtin_arm_usat16(x, y)
 #endif
 
-/* 9.5.5 Packing and unpacking */
+/* 8.5.5 Packing and unpacking */
 #if defined(__ARM_FEATURE_SIMD32) && __ARM_FEATURE_SIMD32
 typedef int32_t int8x4_t;
 typedef int32_t int16x2_t;
@@ -368,7 +373,7 @@ __uxtb16(int8x4_t __a) {
 }
 #endif
 
-/* 9.5.6 Parallel selection */
+/* 8.5.6 Parallel selection */
 #if defined(__ARM_FEATURE_SIMD32) && __ARM_FEATURE_SIMD32
 static __inline__ uint8x4_t __attribute__((__always_inline__, __nodebug__))
 __sel(uint8x4_t __a, uint8x4_t __b) {
@@ -376,7 +381,7 @@ __sel(uint8x4_t __a, uint8x4_t __b) {
 }
 #endif
 
-/* 9.5.7 Parallel 8-bit addition and subtraction */
+/* 8.5.7 Parallel 8-bit addition and subtraction */
 #if defined(__ARM_FEATURE_SIMD32) && __ARM_FEATURE_SIMD32
 static __inline__ int8x4_t __attribute__((__always_inline__, __nodebug__))
 __qadd8(int8x4_t __a, int8x4_t __b) {
@@ -428,7 +433,7 @@ __usub8(uint8x4_t __a, uint8x4_t __b) {
 }
 #endif
 
-/* 9.5.8 Sum of 8-bit absolute differences */
+/* 8.5.8 Sum of 8-bit absolute differences */
 #if defined(__ARM_FEATURE_SIMD32) && __ARM_FEATURE_SIMD32
 static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
 __usad8(uint8x4_t __a, uint8x4_t __b) {
@@ -440,7 +445,7 @@ __usada8(uint8x4_t __a, uint8x4_t __b, uint32_t __c) {
 }
 #endif
 
-/* 9.5.9 Parallel 16-bit addition and subtraction */
+/* 8.5.9 Parallel 16-bit addition and subtraction */
 #if defined(__ARM_FEATURE_SIMD32) && __ARM_FEATURE_SIMD32
 static __inline__ int16x2_t __attribute__((__always_inline__, __nodebug__))
 __qadd16(int16x2_t __a, int16x2_t __b) {
@@ -540,7 +545,7 @@ __usub16(uint16x2_t __a, uint16x2_t __b) {
 }
 #endif
 
-/* 9.5.10 Parallel 16-bit multiplications */
+/* 8.5.10 Parallel 16-bit multiplications */
 #if defined(__ARM_FEATURE_SIMD32) && __ARM_FEATURE_SIMD32
 static __inline__ int32_t __attribute__((__always_inline__, __nodebug__))
 __smlad(int16x2_t __a, int16x2_t __b, int32_t __c) {
@@ -607,7 +612,7 @@ __rintnf(float __a) {
 }
 #endif
 
-/* 9.7 CRC32 intrinsics */
+/* 8.8 CRC32 intrinsics */
 #if (defined(__ARM_FEATURE_CRC32) && __ARM_FEATURE_CRC32) ||                   \
     (defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE)
 static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__, target("crc")))
@@ -651,6 +656,7 @@ __crc32cd(uint32_t __a, uint64_t __b) {
 }
 #endif
 
+/* 8.6 Floating-point data-processing intrinsics */
 /* Armv8.3-A Javascript conversion intrinsic */
 #if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE
 static __inline__ int32_t __attribute__((__always_inline__, __nodebug__, target("v8.3a")))
@@ -702,7 +708,7 @@ __rint64x(double __a) {
 }
 #endif
 
-/* Armv8.7-A load/store 64-byte intrinsics */
+/* 8.9 Armv8.7-A load/store 64-byte intrinsics */
 #if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE
 typedef struct {
     uint64_t val[8];
@@ -728,7 +734,7 @@ __arm_st64bv0(void *__addr, data512_t __value) {
 }
 #endif
 
-/* 10.1 Special register intrinsics */
+/* 11.1 Special register intrinsics */
 #define __arm_rsr(sysreg) __builtin_arm_rsr(sysreg)
 #define __arm_rsr64(sysreg) __builtin_arm_rsr64(sysreg)
 #define __arm_rsr128(sysreg) __builtin_arm_rsr128(sysreg)
@@ -742,7 +748,7 @@ __arm_st64bv0(void *__addr, data512_t __value) {
 #define __arm_wsrf(sysreg, v) __arm_wsr(sysreg, __builtin_bit_cast(uint32_t, v))
 #define __arm_wsrf64(sysreg, v) __arm_wsr64(sysreg, __builtin_bit_cast(uint64_t, v))
 
-/* Memory Tagging Extensions (MTE) Intrinsics */
+/* 10.3 Memory Tagging Extensions (MTE) Intrinsics */
 #if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE
 #define __arm_mte_create_random_tag(__ptr, __mask)  __builtin_arm_irg(__ptr, __mask)
 #define __arm_mte_increment_tag(__ptr, __tag_offset)  __builtin_arm_addg(__ptr, __tag_offset)
@@ -751,12 +757,71 @@ __arm_st64bv0(void *__addr, data512_t __value) {
 #define __arm_mte_set_tag(__ptr) __builtin_arm_stg(__ptr)
 #define __arm_mte_ptrdiff(__ptra, __ptrb) __builtin_arm_subp(__ptra, __ptrb)
 
-/* Memory Operations Intrinsics */
+/* 18 Memory Operations Intrinsics */
 #define __arm_mops_memset_tag(__tagged_address, __value, __size)    \
   __builtin_arm_mops_memset_tag(__tagged_address, __value, __size)
 #endif
 
-/* Transactional Memory Extension (TME) Intrinsics */
+/* 11.3 Coprocessor Intrinsics */
+#if defined(__ARM_FEATURE_COPROC)
+
+#if (__ARM_FEATURE_COPROC & 0x1)
+
+#if (__ARM_ARCH < 8)
+#define __arm_cdp(coproc, opc1, CRd, CRn, CRm, opc2)                           \
+  __builtin_arm_cdp(coproc, opc1, CRd, CRn, CRm, opc2)
+#endif /* __ARM_ARCH < 8 */
+
+#define __arm_ldc(coproc, CRd, p) __builtin_arm_ldc(coproc, CRd, p)
+#define __arm_stc(coproc, CRd, p) __builtin_arm_stc(coproc, CRd, p)
+
+#define __arm_mcr(coproc, opc1, value, CRn, CRm, opc2)                         \
+  __builtin_arm_mcr(coproc, opc1, value, CRn, CRm, opc2)
+#define __arm_mrc(coproc, opc1, CRn, CRm, opc2)                                \
+  __builtin_arm_mrc(coproc, opc1, CRn, CRm, opc2)
+
+#if (__ARM_ARCH != 4) && (__ARM_ARCH < 8)
+#define __arm_ldcl(coproc, CRd, p) __builtin_arm_ldcl(coproc, CRd, p)
+#define __arm_stcl(coproc, CRd, p) __builtin_arm_stcl(coproc, CRd, p)
+#endif /* (__ARM_ARCH != 4) && (__ARM_ARCH != 8) */
+
+#if (__ARM_ARCH_8M_MAIN__) || (__ARM_ARCH_8_1M_MAIN__)
+#define __arm_cdp(coproc, opc1, CRd, CRn, CRm, opc2)                           \
+  __builtin_arm_cdp(coproc, opc1, CRd, CRn, CRm, opc2)
+#define __arm_ldcl(coproc, CRd, p) __builtin_arm_ldcl(coproc, CRd, p)
+#define __arm_stcl(coproc, CRd, p) __builtin_arm_stcl(coproc, CRd, p)
+#endif /* ___ARM_ARCH_8M_MAIN__ */
+
+#endif /* __ARM_FEATURE_COPROC & 0x1 */
+
+#if (__ARM_FEATURE_COPROC & 0x2)
+#define __arm_cdp2(coproc, opc1, CRd, CRn, CRm, opc2)                          \
+  __builtin_arm_cdp2(coproc, opc1, CRd, CRn, CRm, opc2)
+#define __arm_ldc2(coproc, CRd, p) __builtin_arm_ldc2(coproc, CRd, p)
+#define __arm_stc2(coproc, CRd, p) __builtin_arm_stc2(coproc, CRd, p)
+#define __arm_ldc2l(coproc, CRd, p) __builtin_arm_ldc2l(coproc, CRd, p)
+#define __arm_stc2l(coproc, CRd, p) __builtin_arm_stc2l(coproc, CRd, p)
+#define __arm_mcr2(coproc, opc1, value, CRn, CRm, opc2)                        \
+  __builtin_arm_mcr2(coproc, opc1, value, CRn, CRm, opc2)
+#define __arm_mrc2(coproc, opc1, CRn, CRm, opc2)                               \
+  __builtin_arm_mrc2(coproc, opc1, CRn, CRm, opc2)
+#endif
+
+#if (__ARM_FEATURE_COPROC & 0x4)
+#define __arm_mcrr(coproc, opc1, value, CRm)                                   \
+  __builtin_arm_mcrr(coproc, opc1, value, CRm)
+#define __arm_mrrc(coproc, opc1, CRm) __builtin_arm_mrrc(coproc, opc1, CRm)
+#endif
+
+#if (__ARM_FEATURE_COPROC & 0x8)
+#define __arm_mcrr2(coproc, opc1, value, CRm)                                  \
+  __builtin_arm_mcrr2(coproc, opc1, value, CRm)
+#define __arm_mrrc2(coproc, opc1, CRm) __builtin_arm_mrrc2(coproc, opc1, CRm)
+#endif
+
+#endif // __ARM_FEATURE_COPROC
+
+/* 17 Transactional Memory Extension (TME) Intrinsics */
 #if defined(__ARM_FEATURE_TME) && __ARM_FEATURE_TME
 
 #define _TMFAILURE_REASON  0x00007fffu
@@ -778,7 +843,7 @@ __arm_st64bv0(void *__addr, data512_t __value) {
 
 #endif /* __ARM_FEATURE_TME */
 
-/* Armv8.5-A Random number generation intrinsics */
+/* 8.7 Armv8.5-A Random number generation intrinsics */
 #if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE
 static __inline__ int __attribute__((__always_inline__, __nodebug__, target("rand")))
 __rndr(uint64_t *__p) {
diff --git a/clang/lib/Headers/bmiintrin.h b/clang/lib/Headers/bmiintrin.h
index bc7c8a03c5e2..d8e57c0cb494 100644
--- a/clang/lib/Headers/bmiintrin.h
+++ b/clang/lib/Headers/bmiintrin.h
@@ -19,18 +19,17 @@
    to use it as a potentially faster version of BSF. */
 #define __RELAXED_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
 
-#define _tzcnt_u16 __tzcnt_u16
-
 /// Counts the number of trailing zero bits in the operand.
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  TZCNT  instruction.
+/// This intrinsic corresponds to the \c TZCNT instruction.
 ///
 /// \param __X
 ///    An unsigned 16-bit integer whose trailing zeros are to be counted.
 /// \returns An unsigned 16-bit integer containing the number of trailing zero
 ///    bits in the operand.
+/// \see _tzcnt_u16
 static __inline__ unsigned short __RELAXED_FN_ATTRS
 __tzcnt_u16(unsigned short __X)
 {
@@ -41,13 +40,30 @@ __tzcnt_u16(unsigned short __X)
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  TZCNT  instruction.
+/// \code
+/// unsigned short _tzcnt_u16(unsigned short __X);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TZCNT instruction.
+///
+/// \param __X
+///    An unsigned 16-bit integer whose trailing zeros are to be counted.
+/// \returns An unsigned 16-bit integer containing the number of trailing zero
+///    bits in the operand.
+/// \see __tzcnt_u16
+#define _tzcnt_u16 __tzcnt_u16
+
+/// Counts the number of trailing zero bits in the operand.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c TZCNT instruction.
 ///
 /// \param __X
 ///    An unsigned 32-bit integer whose trailing zeros are to be counted.
 /// \returns An unsigned 32-bit integer containing the number of trailing zero
 ///    bits in the operand.
-/// \see _mm_tzcnt_32
+/// \see { _mm_tzcnt_32 _tzcnt_u32 }
 static __inline__ unsigned int __RELAXED_FN_ATTRS
 __tzcnt_u32(unsigned int __X)
 {
@@ -58,19 +74,34 @@ __tzcnt_u32(unsigned int __X)
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  TZCNT  instruction.
+/// This intrinsic corresponds to the \c TZCNT instruction.
 ///
 /// \param __X
 ///    An unsigned 32-bit integer whose trailing zeros are to be counted.
-/// \returns An 32-bit integer containing the number of trailing zero bits in
+/// \returns A 32-bit integer containing the number of trailing zero bits in
 ///    the operand.
-/// \see __tzcnt_u32
+/// \see { __tzcnt_u32 _tzcnt_u32 }
 static __inline__ int __RELAXED_FN_ATTRS
 _mm_tzcnt_32(unsigned int __X)
 {
   return (int)__builtin_ia32_tzcnt_u32(__X);
 }
 
+/// Counts the number of trailing zero bits in the operand.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned int _tzcnt_u32(unsigned int __X);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TZCNT instruction.
+///
+/// \param __X
+///    An unsigned 32-bit integer whose trailing zeros are to be counted.
+/// \returns An unsigned 32-bit integer containing the number of trailing zero
+///    bits in the operand.
+/// \see { _mm_tzcnt_32 __tzcnt_u32 }
 #define _tzcnt_u32 __tzcnt_u32
 
 #ifdef __x86_64__
@@ -79,13 +110,13 @@ _mm_tzcnt_32(unsigned int __X)
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  TZCNT  instruction.
+/// This intrinsic corresponds to the \c TZCNT instruction.
 ///
 /// \param __X
 ///    An unsigned 64-bit integer whose trailing zeros are to be counted.
 /// \returns An unsigned 64-bit integer containing the number of trailing zero
 ///    bits in the operand.
-/// \see _mm_tzcnt_64
+/// \see { _mm_tzcnt_64 _tzcnt_u64 }
 static __inline__ unsigned long long __RELAXED_FN_ATTRS
 __tzcnt_u64(unsigned long long __X)
 {
@@ -96,19 +127,34 @@ __tzcnt_u64(unsigned long long __X)
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  TZCNT  instruction.
+/// This intrinsic corresponds to the \c TZCNT instruction.
 ///
 /// \param __X
 ///    An unsigned 64-bit integer whose trailing zeros are to be counted.
 /// \returns An 64-bit integer containing the number of trailing zero bits in
 ///    the operand.
-/// \see __tzcnt_u64
+/// \see { __tzcnt_u64 _tzcnt_u64 }
 static __inline__ long long __RELAXED_FN_ATTRS
 _mm_tzcnt_64(unsigned long long __X)
 {
   return (long long)__builtin_ia32_tzcnt_u64(__X);
 }
 
+/// Counts the number of trailing zero bits in the operand.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned long long _tzcnt_u64(unsigned long long __X);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TZCNT instruction.
+///
+/// \param __X
+///    An unsigned 64-bit integer whose trailing zeros are to be counted.
+/// \returns An unsigned 64-bit integer containing the number of trailing zero
+///    bits in the operand.
+/// \see { _mm_tzcnt_64 __tzcnt_u64
 #define _tzcnt_u64 __tzcnt_u64
 
 #endif /* __x86_64__ */
@@ -121,21 +167,12 @@ _mm_tzcnt_64(unsigned long long __X)
 /* Define the default attributes for the functions in this file. */
 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("bmi")))
 
-#define _andn_u32 __andn_u32
-
-/* _bextr_u32 != __bextr_u32 */
-#define _blsi_u32 __blsi_u32
-
-#define _blsmsk_u32 __blsmsk_u32
-
-#define _blsr_u32 __blsr_u32
-
 /// Performs a bitwise AND of the second operand with the one's
 ///    complement of the first operand.
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  ANDN  instruction.
+/// This intrinsic corresponds to the \c ANDN instruction.
 ///
 /// \param __X
 ///    An unsigned integer containing one of the operands.
@@ -143,19 +180,40 @@ _mm_tzcnt_64(unsigned long long __X)
 ///    An unsigned integer containing one of the operands.
 /// \returns An unsigned integer containing the bitwise AND of the second
 ///    operand with the one's complement of the first operand.
+/// \see _andn_u32
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
 __andn_u32(unsigned int __X, unsigned int __Y)
 {
   return ~__X & __Y;
 }
 
+/// Performs a bitwise AND of the second operand with the one's
+///    complement of the first operand.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned int _andn_u32(unsigned int __X, unsigned int __Y);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c ANDN instruction.
+///
+/// \param __X
+///    An unsigned integer containing one of the operands.
+/// \param __Y
+///    An unsigned integer containing one of the operands.
+/// \returns An unsigned integer containing the bitwise AND of the second
+///    operand with the one's complement of the first operand.
+/// \see __andn_u32
+#define _andn_u32 __andn_u32
+
 /* AMD-specified, double-leading-underscore version of BEXTR */
 /// Extracts the specified bits from the first operand and returns them
 ///    in the least significant bits of the result.
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  BEXTR  instruction.
+/// This intrinsic corresponds to the \c BEXTR instruction.
 ///
 /// \param __X
 ///    An unsigned integer whose bits are to be extracted.
@@ -178,7 +236,7 @@ __bextr_u32(unsigned int __X, unsigned int __Y)
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  BEXTR  instruction.
+/// This intrinsic corresponds to the \c BEXTR instruction.
 ///
 /// \param __X
 ///    An unsigned integer whose bits are to be extracted.
@@ -203,7 +261,7 @@ _bextr_u32(unsigned int __X, unsigned int __Y, unsigned int __Z)
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  BEXTR  instruction.
+/// This intrinsic corresponds to the \c BEXTR instruction.
 ///
 /// \param __X
 ///    An unsigned integer whose bits are to be extracted.
@@ -224,69 +282,117 @@ _bextr2_u32(unsigned int __X, unsigned int __Y) {
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  BLSI  instruction.
+/// This intrinsic corresponds to the \c BLSI instruction.
 ///
 /// \param __X
 ///    An unsigned integer whose bits are to be cleared.
 /// \returns An unsigned integer containing the result of clearing the bits from
 ///    the source operand.
+/// \see _blsi_u32
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
 __blsi_u32(unsigned int __X)
 {
   return __X & -__X;
 }
 
+/// Clears all bits in the source except for the least significant bit
+///    containing a value of 1 and returns the result.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned int _blsi_u32(unsigned int __X);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c BLSI instruction.
+///
+/// \param __X
+///    An unsigned integer whose bits are to be cleared.
+/// \returns An unsigned integer containing the result of clearing the bits from
+///    the source operand.
+/// \see __blsi_u32
+#define _blsi_u32 __blsi_u32
+
 /// Creates a mask whose bits are set to 1, using bit 0 up to and
 ///    including the least significant bit that is set to 1 in the source
 ///    operand and returns the result.
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  BLSMSK  instruction.
+/// This intrinsic corresponds to the \c BLSMSK instruction.
 ///
 /// \param __X
 ///    An unsigned integer used to create the mask.
 /// \returns An unsigned integer containing the newly created mask.
+/// \see _blsmsk_u32
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
 __blsmsk_u32(unsigned int __X)
 {
   return __X ^ (__X - 1);
 }
 
+/// Creates a mask whose bits are set to 1, using bit 0 up to and
+///    including the least significant bit that is set to 1 in the source
+///    operand and returns the result.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned int _blsmsk_u32(unsigned int __X);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c BLSMSK instruction.
+///
+/// \param __X
+///    An unsigned integer used to create the mask.
+/// \returns An unsigned integer containing the newly created mask.
+/// \see __blsmsk_u32
+#define _blsmsk_u32 __blsmsk_u32
+
 /// Clears the least significant bit that is set to 1 in the source
 ///    operand and returns the result.
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  BLSR  instruction.
+/// This intrinsic corresponds to the \c BLSR instruction.
 ///
 /// \param __X
 ///    An unsigned integer containing the operand to be cleared.
 /// \returns An unsigned integer containing the result of clearing the source
 ///    operand.
+/// \see _blsr_u32
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
 __blsr_u32(unsigned int __X)
 {
   return __X & (__X - 1);
 }
 
-#ifdef __x86_64__
-
-#define _andn_u64 __andn_u64
-
-/* _bextr_u64 != __bextr_u64 */
-#define _blsi_u64 __blsi_u64
-
-#define _blsmsk_u64 __blsmsk_u64
+/// Clears the least significant bit that is set to 1 in the source
+///    operand and returns the result.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned int _bls4_u32(unsigned int __X);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c BLSR instruction.
+///
+/// \param __X
+///    An unsigned integer containing the operand to be cleared.
+/// \returns An unsigned integer containing the result of clearing the source
+///    operand.
+/// \see __blsr_u32
+#define _blsr_u32 __blsr_u32
 
-#define _blsr_u64 __blsr_u64
+#ifdef __x86_64__
 
 /// Performs a bitwise AND of the second operand with the one's
 ///    complement of the first operand.
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  ANDN  instruction.
+/// This intrinsic corresponds to the \c ANDN instruction.
 ///
 /// \param __X
 ///    An unsigned 64-bit integer containing one of the operands.
@@ -294,19 +400,41 @@ __blsr_u32(unsigned int __X)
 ///    An unsigned 64-bit integer containing one of the operands.
 /// \returns An unsigned 64-bit integer containing the bitwise AND of the second
 ///    operand with the one's complement of the first operand.
+/// \see _andn_u64
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 __andn_u64 (unsigned long long __X, unsigned long long __Y)
 {
   return ~__X & __Y;
 }
 
+/// Performs a bitwise AND of the second operand with the one's
+///    complement of the first operand.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned long long _andn_u64(unsigned long long __X,
+///                              unsigned long long __Y);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c ANDN instruction.
+///
+/// \param __X
+///    An unsigned 64-bit integer containing one of the operands.
+/// \param __Y
+///    An unsigned 64-bit integer containing one of the operands.
+/// \returns An unsigned 64-bit integer containing the bitwise AND of the second
+///    operand with the one's complement of the first operand.
+/// \see __andn_u64
+#define _andn_u64 __andn_u64
+
 /* AMD-specified, double-leading-underscore version of BEXTR */
 /// Extracts the specified bits from the first operand and returns them
 ///    in the least significant bits of the result.
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  BEXTR  instruction.
+/// This intrinsic corresponds to the \c BEXTR instruction.
 ///
 /// \param __X
 ///    An unsigned 64-bit integer whose bits are to be extracted.
@@ -329,7 +457,7 @@ __bextr_u64(unsigned long long __X, unsigned long long __Y)
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  BEXTR  instruction.
+/// This intrinsic corresponds to the \c BEXTR instruction.
 ///
 /// \param __X
 ///    An unsigned 64-bit integer whose bits are to be extracted.
@@ -354,7 +482,7 @@ _bextr_u64(unsigned long long __X, unsigned int __Y, unsigned int __Z)
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  BEXTR  instruction.
+/// This intrinsic corresponds to the \c BEXTR instruction.
 ///
 /// \param __X
 ///    An unsigned 64-bit integer whose bits are to be extracted.
@@ -375,52 +503,109 @@ _bextr2_u64(unsigned long long __X, unsigned long long __Y) {
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  BLSI  instruction.
+/// This intrinsic corresponds to the \c BLSI instruction.
 ///
 /// \param __X
 ///    An unsigned 64-bit integer whose bits are to be cleared.
 /// \returns An unsigned 64-bit integer containing the result of clearing the
 ///    bits from the source operand.
+/// \see _blsi_u64
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 __blsi_u64(unsigned long long __X)
 {
   return __X & -__X;
 }
 
+/// Clears all bits in the source except for the least significant bit
+///    containing a value of 1 and returns the result.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned long long _blsi_u64(unsigned long long __X);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c BLSI instruction.
+///
+/// \param __X
+///    An unsigned 64-bit integer whose bits are to be cleared.
+/// \returns An unsigned 64-bit integer containing the result of clearing the
+///    bits from the source operand.
+/// \see __blsi_u64
+#define _blsi_u64 __blsi_u64
+
 /// Creates a mask whose bits are set to 1, using bit 0 up to and
 ///    including the least significant bit that is set to 1 in the source
 ///    operand and returns the result.
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  BLSMSK  instruction.
+/// This intrinsic corresponds to the \c BLSMSK instruction.
 ///
 /// \param __X
 ///    An unsigned 64-bit integer used to create the mask.
 /// \returns An unsigned 64-bit integer containing the newly created mask.
+/// \see _blsmsk_u64
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 __blsmsk_u64(unsigned long long __X)
 {
   return __X ^ (__X - 1);
 }
 
+/// Creates a mask whose bits are set to 1, using bit 0 up to and
+///    including the least significant bit that is set to 1 in the source
+///    operand and returns the result.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned long long _blsmsk_u64(unsigned long long __X);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c BLSMSK instruction.
+///
+/// \param __X
+///    An unsigned 64-bit integer used to create the mask.
+/// \returns An unsigned 64-bit integer containing the newly created mask.
+/// \see __blsmsk_u64
+#define _blsmsk_u64 __blsmsk_u64
+
 /// Clears the least significant bit that is set to 1 in the source
 ///    operand and returns the result.
 ///
 /// \headerfile 
 ///
-/// This intrinsic corresponds to the  BLSR  instruction.
+/// This intrinsic corresponds to the \c BLSR instruction.
 ///
 /// \param __X
 ///    An unsigned 64-bit integer containing the operand to be cleared.
 /// \returns An unsigned 64-bit integer containing the result of clearing the
 ///    source operand.
+/// \see _blsr_u64
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 __blsr_u64(unsigned long long __X)
 {
   return __X & (__X - 1);
 }
 
+/// Clears the least significant bit that is set to 1 in the source
+///    operand and returns the result.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned long long _blsr_u64(unsigned long long __X);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c BLSR instruction.
+///
+/// \param __X
+///    An unsigned 64-bit integer containing the operand to be cleared.
+/// \returns An unsigned 64-bit integer containing the result of clearing the
+///    source operand.
+/// \see __blsr_u64
+#define _blsr_u64 __blsr_u64
+
 #endif /* __x86_64__ */
 
 #undef __DEFAULT_FN_ATTRS
diff --git a/clang/lib/Headers/ia32intrin.h b/clang/lib/Headers/ia32intrin.h
index f1904efd71c4..1b979770e196 100644
--- a/clang/lib/Headers/ia32intrin.h
+++ b/clang/lib/Headers/ia32intrin.h
@@ -26,167 +26,271 @@
 #define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
 #endif
 
-/** Find the first set bit starting from the lsb. Result is undefined if
- *  input is 0.
- *
- *  \headerfile 
- *
- *  This intrinsic corresponds to the  BSF  instruction or the
- *   TZCNT  instruction.
- *
- *  \param __A
- *     A 32-bit integer operand.
- *  \returns A 32-bit integer containing the bit number.
- */
+/// Find the first set bit starting from the lsb. Result is undefined if
+///    input is 0.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c BSF instruction or the
+///    \c TZCNT instruction.
+///
+/// \param __A
+///    A 32-bit integer operand.
+/// \returns A 32-bit integer containing the bit number.
+/// \see _bit_scan_forward
 static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 __bsfd(int __A) {
   return __builtin_ctz((unsigned int)__A);
 }
 
-/** Find the first set bit starting from the msb. Result is undefined if
- *  input is 0.
- *
- *  \headerfile 
- *
- *  This intrinsic corresponds to the  BSR  instruction or the
- *   LZCNT  instruction and an  XOR .
- *
- *  \param __A
- *     A 32-bit integer operand.
- *  \returns A 32-bit integer containing the bit number.
- */
+/// Find the first set bit starting from the msb. Result is undefined if
+///    input is 0.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c BSR instruction or the
+///    \c LZCNT instruction and an \c XOR.
+///
+/// \param __A
+///    A 32-bit integer operand.
+/// \returns A 32-bit integer containing the bit number.
+/// \see _bit_scan_reverse
 static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 __bsrd(int __A) {
   return 31 - __builtin_clz((unsigned int)__A);
 }
 
-/** Swaps the bytes in the input. Converting little endian to big endian or
- *  vice versa.
- *
- *  \headerfile 
- *
- *  This intrinsic corresponds to the  BSWAP  instruction.
- *
- *  \param __A
- *     A 32-bit integer operand.
- *  \returns A 32-bit integer containing the swapped bytes.
- */
+/// Swaps the bytes in the input, converting little endian to big endian or
+///    vice versa.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c BSWAP instruction.
+///
+/// \param __A
+///    A 32-bit integer operand.
+/// \returns A 32-bit integer containing the swapped bytes.
 static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 __bswapd(int __A) {
   return (int)__builtin_bswap32((unsigned int)__A);
 }
 
+/// Swaps the bytes in the input, converting little endian to big endian or
+///    vice versa.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c BSWAP instruction.
+///
+/// \param __A
+///    A 32-bit integer operand.
+/// \returns A 32-bit integer containing the swapped bytes.
 static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 _bswap(int __A) {
   return (int)__builtin_bswap32((unsigned int)__A);
 }
 
+/// Find the first set bit starting from the lsb. Result is undefined if
+///    input is 0.
+///
+/// \headerfile 
+///
+/// \code
+/// int _bit_scan_forward(int A);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c BSF instruction or the
+///    \c TZCNT instruction.
+///
+/// \param A
+///    A 32-bit integer operand.
+/// \returns A 32-bit integer containing the bit number.
+/// \see __bsfd
 #define _bit_scan_forward(A) __bsfd((A))
+
+/// Find the first set bit starting from the msb. Result is undefined if
+///    input is 0.
+///
+/// \headerfile 
+///
+/// \code
+/// int _bit_scan_reverse(int A);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c BSR instruction or the
+///    \c LZCNT instruction and an \c XOR.
+///
+/// \param A
+///    A 32-bit integer operand.
+/// \returns A 32-bit integer containing the bit number.
+/// \see __bsrd
 #define _bit_scan_reverse(A) __bsrd((A))
 
 #ifdef __x86_64__
-/** Find the first set bit starting from the lsb. Result is undefined if
- *  input is 0.
- *
- *  \headerfile 
- *
- *  This intrinsic corresponds to the  BSF  instruction or the
- *   TZCNT  instruction.
- *
- *  \param __A
- *     A 64-bit integer operand.
- *  \returns A 32-bit integer containing the bit number.
- */
+/// Find the first set bit starting from the lsb. Result is undefined if
+///    input is 0.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c BSF instruction or the
+///    \c TZCNT instruction.
+///
+/// \param __A
+///    A 64-bit integer operand.
+/// \returns A 32-bit integer containing the bit number.
 static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 __bsfq(long long __A) {
   return (long long)__builtin_ctzll((unsigned long long)__A);
 }
 
-/** Find the first set bit starting from the msb. Result is undefined if
- *  input is 0.
- *
- *  \headerfile 
- *
- *  This intrinsic corresponds to the  BSR  instruction or the
- *   LZCNT  instruction and an  XOR .
- *
- *  \param __A
- *     A 64-bit integer operand.
- *  \returns A 32-bit integer containing the bit number.
- */
+/// Find the first set bit starting from the msb. Result is undefined if
+///    input is 0.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c BSR instruction or the
+///    \c LZCNT instruction and an \c XOR.
+///
+/// \param __A
+///    A 64-bit integer operand.
+/// \returns A 32-bit integer containing the bit number.
 static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 __bsrq(long long __A) {
   return 63 - __builtin_clzll((unsigned long long)__A);
 }
 
-/** Swaps the bytes in the input. Converting little endian to big endian or
- *  vice versa.
- *
- *  \headerfile 
- *
- *  This intrinsic corresponds to the  BSWAP  instruction.
- *
- *  \param __A
- *     A 64-bit integer operand.
- *  \returns A 64-bit integer containing the swapped bytes.
- */
+/// Swaps the bytes in the input. Converting little endian to big endian or
+///    vice versa.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c BSWAP instruction.
+///
+/// \param __A
+///    A 64-bit integer operand.
+/// \returns A 64-bit integer containing the swapped bytes.
+/// \see _bswap64
 static __inline__ long long __DEFAULT_FN_ATTRS_CONSTEXPR
 __bswapq(long long __A) {
   return (long long)__builtin_bswap64((unsigned long long)__A);
 }
 
+/// Swaps the bytes in the input. Converting little endian to big endian or
+///    vice versa.
+///
+/// \headerfile 
+///
+/// \code
+/// long long _bswap64(long long A);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c BSWAP instruction.
+///
+/// \param A
+///    A 64-bit integer operand.
+/// \returns A 64-bit integer containing the swapped bytes.
+/// \see __bswapq
 #define _bswap64(A) __bswapq((A))
-#endif
+#endif /* __x86_64__ */
 
-/** Counts the number of bits in the source operand having a value of 1.
- *
- *  \headerfile 
- *
- *  This intrinsic corresponds to the  POPCNT  instruction or a
- *  a sequence of arithmetic and logic ops to calculate it.
- *
- *  \param __A
- *     An unsigned 32-bit integer operand.
- *  \returns A 32-bit integer containing the number of bits with value 1 in the
- *     source operand.
- */
+/// Counts the number of bits in the source operand having a value of 1.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c POPCNT instruction or a
+///    a sequence of arithmetic and logic ops to calculate it.
+///
+/// \param __A
+///    An unsigned 32-bit integer operand.
+/// \returns A 32-bit integer containing the number of bits with value 1 in the
+///    source operand.
+/// \see _popcnt32
 static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
 __popcntd(unsigned int __A)
 {
   return __builtin_popcount(__A);
 }
 
+/// Counts the number of bits in the source operand having a value of 1.
+///
+/// \headerfile 
+///
+/// \code
+/// int _popcnt32(int A);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c POPCNT instruction or a
+///    a sequence of arithmetic and logic ops to calculate it.
+///
+/// \param A
+///    An unsigned 32-bit integer operand.
+/// \returns A 32-bit integer containing the number of bits with value 1 in the
+///    source operand.
+/// \see __popcntd
 #define _popcnt32(A) __popcntd((A))
 
 #ifdef __x86_64__
-/** Counts the number of bits in the source operand having a value of 1.
- *
- *  \headerfile 
- *
- *  This intrinsic corresponds to the  POPCNT  instruction or a
- *  a sequence of arithmetic and logic ops to calculate it.
- *
- *  \param __A
- *     An unsigned 64-bit integer operand.
- *  \returns A 64-bit integer containing the number of bits with value 1 in the
- *     source operand.
- */
+/// Counts the number of bits in the source operand having a value of 1.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c POPCNT instruction or a
+///    a sequence of arithmetic and logic ops to calculate it.
+///
+/// \param __A
+///    An unsigned 64-bit integer operand.
+/// \returns A 64-bit integer containing the number of bits with value 1 in the
+///    source operand.
+/// \see _popcnt64
 static __inline__ long long __DEFAULT_FN_ATTRS_CONSTEXPR
 __popcntq(unsigned long long __A)
 {
   return __builtin_popcountll(__A);
 }
 
+/// Counts the number of bits in the source operand having a value of 1.
+///
+/// \headerfile 
+///
+/// \code
+/// long long _popcnt64(unsigned long long A);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c POPCNT instruction or a
+///    a sequence of arithmetic and logic ops to calculate it.
+///
+/// \param A
+///    An unsigned 64-bit integer operand.
+/// \returns A 64-bit integer containing the number of bits with value 1 in the
+///    source operand.
+/// \see __popcntq
 #define _popcnt64(A) __popcntq((A))
 #endif /* __x86_64__ */
 
 #ifdef __x86_64__
+/// Returns the program status and control \c RFLAGS register with the \c VM
+///    and \c RF flags cleared.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c PUSHFQ + \c POP instruction sequence.
+///
+/// \returns The 64-bit value of the RFLAGS register.
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 __readeflags(void)
 {
   return __builtin_ia32_readeflags_u64();
 }
 
+/// Writes the specified value to the program status and control \c RFLAGS
+///    register. Reserved bits are not affected.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c PUSH + \c POPFQ instruction sequence.
+///
+/// \param __f
+///    The 64-bit value to write to \c RFLAGS.
 static __inline__ void __DEFAULT_FN_ATTRS
 __writeeflags(unsigned long long __f)
 {
@@ -194,12 +298,29 @@ __writeeflags(unsigned long long __f)
 }
 
 #else /* !__x86_64__ */
+/// Returns the program status and control \c EFLAGS register with the \c VM
+///    and \c RF flags cleared.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c PUSHFD + \c POP instruction sequence.
+///
+/// \returns The 32-bit value of the EFLAGS register.
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
 __readeflags(void)
 {
   return __builtin_ia32_readeflags_u32();
 }
 
+/// Writes the specified value to the program status and control \c EFLAGS
+///    register. Reserved bits are not affected.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c PUSH + \c POPFD instruction sequence.
+///
+/// \param __f
+///    The 32-bit value to write to \c EFLAGS.
 static __inline__ void __DEFAULT_FN_ATTRS
 __writeeflags(unsigned int __f)
 {
@@ -207,123 +328,120 @@ __writeeflags(unsigned int __f)
 }
 #endif /* !__x86_64__ */
 
-/** Cast a 32-bit float value to a 32-bit unsigned integer value
- *
- *  \headerfile 
- *  This intrinsic corresponds to the  VMOVD / MOVD  instruction in x86_64,
- *  and corresponds to the  VMOVL / MOVL  instruction in ia32.
- *
- *  \param __A
- *     A 32-bit float value.
- *  \returns a 32-bit unsigned integer containing the converted value.
- */
+/// Cast a 32-bit float value to a 32-bit unsigned integer value.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c VMOVD / \c MOVD instruction in x86_64,
+///    and corresponds to the \c VMOVL / \c MOVL instruction in ia32.
+///
+/// \param __A
+///    A 32-bit float value.
+/// \returns a 32-bit unsigned integer containing the converted value.
 static __inline__ unsigned int __DEFAULT_FN_ATTRS_CAST
 _castf32_u32(float __A) {
   return __builtin_bit_cast(unsigned int, __A);
 }
 
-/** Cast a 64-bit float value to a 64-bit unsigned integer value
- *
- *  \headerfile 
- *  This intrinsic corresponds to the  VMOVQ / MOVQ  instruction in x86_64,
- *  and corresponds to the  VMOVL / MOVL  instruction in ia32.
- *
- *  \param __A
- *     A 64-bit float value.
- *  \returns a 64-bit unsigned integer containing the converted value.
- */
+/// Cast a 64-bit float value to a 64-bit unsigned integer value.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c VMOVQ / \c MOVQ instruction in x86_64,
+///    and corresponds to the \c VMOVL / \c MOVL instruction in ia32.
+///
+/// \param __A
+///    A 64-bit float value.
+/// \returns a 64-bit unsigned integer containing the converted value.
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS_CAST
 _castf64_u64(double __A) {
   return __builtin_bit_cast(unsigned long long, __A);
 }
 
-/** Cast a 32-bit unsigned integer value to a 32-bit float value
- *
- *  \headerfile 
- *  This intrinsic corresponds to the  VMOVQ / MOVQ  instruction in x86_64,
- *  and corresponds to the  FLDS  instruction in ia32.
- *
- *  \param __A
- *     A 32-bit unsigned integer value.
- *  \returns a 32-bit float value containing the converted value.
- */
+/// Cast a 32-bit unsigned integer value to a 32-bit float value.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c VMOVQ / \c MOVQ instruction in x86_64,
+///    and corresponds to the \c FLDS instruction in ia32.
+///
+/// \param __A
+///    A 32-bit unsigned integer value.
+/// \returns a 32-bit float value containing the converted value.
 static __inline__ float __DEFAULT_FN_ATTRS_CAST
 _castu32_f32(unsigned int __A) {
   return __builtin_bit_cast(float, __A);
 }
 
-/** Cast a 64-bit unsigned integer value to a 64-bit float value
- *
- *  \headerfile 
- *  This intrinsic corresponds to the  VMOVQ / MOVQ  instruction in x86_64,
- *  and corresponds to the  FLDL  instruction in ia32.
- *
- *  \param __A
- *     A 64-bit unsigned integer value.
- *  \returns a 64-bit float value containing the converted value.
- */
+/// Cast a 64-bit unsigned integer value to a 64-bit float value.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c VMOVQ / \c MOVQ instruction in x86_64,
+///    and corresponds to the \c FLDL instruction in ia32.
+///
+/// \param __A
+///    A 64-bit unsigned integer value.
+/// \returns a 64-bit float value containing the converted value.
 static __inline__ double __DEFAULT_FN_ATTRS_CAST
 _castu64_f64(unsigned long long __A) {
   return __builtin_bit_cast(double, __A);
 }
 
-/** Adds the unsigned integer operand to the CRC-32C checksum of the
- *     unsigned char operand.
- *
- *  \headerfile 
- *
- *  This intrinsic corresponds to the  CRC32B  instruction.
- *
- *  \param __C
- *     An unsigned integer operand to add to the CRC-32C checksum of operand
- *     \a  __D.
- *  \param __D
- *     An unsigned 8-bit integer operand used to compute the CRC-32C checksum.
- *  \returns The result of adding operand \a __C to the CRC-32C checksum of
- *     operand \a __D.
- */
+/// Adds the unsigned integer operand to the CRC-32C checksum of the
+///     unsigned char operand.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c CRC32B instruction.
+///
+/// \param __C
+///    An unsigned integer operand to add to the CRC-32C checksum of operand
+///    \a  __D.
+/// \param __D
+///    An unsigned 8-bit integer operand used to compute the CRC-32C checksum.
+/// \returns The result of adding operand \a __C to the CRC-32C checksum of
+///    operand \a __D.
 static __inline__ unsigned int __DEFAULT_FN_ATTRS_CRC32
 __crc32b(unsigned int __C, unsigned char __D)
 {
   return __builtin_ia32_crc32qi(__C, __D);
 }
 
-/** Adds the unsigned integer operand to the CRC-32C checksum of the
- *     unsigned short operand.
- *
- *  \headerfile 
- *
- *  This intrinsic corresponds to the  CRC32W  instruction.
- *
- *  \param __C
- *     An unsigned integer operand to add to the CRC-32C checksum of operand
- *     \a  __D.
- *  \param __D
- *     An unsigned 16-bit integer operand used to compute the CRC-32C checksum.
- *  \returns The result of adding operand \a __C to the CRC-32C checksum of
- *     operand \a __D.
- */
+/// Adds the unsigned integer operand to the CRC-32C checksum of the
+///    unsigned short operand.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c CRC32W instruction.
+///
+/// \param __C
+///    An unsigned integer operand to add to the CRC-32C checksum of operand
+///    \a  __D.
+/// \param __D
+///    An unsigned 16-bit integer operand used to compute the CRC-32C checksum.
+/// \returns The result of adding operand \a __C to the CRC-32C checksum of
+///    operand \a __D.
 static __inline__ unsigned int __DEFAULT_FN_ATTRS_CRC32
 __crc32w(unsigned int __C, unsigned short __D)
 {
   return __builtin_ia32_crc32hi(__C, __D);
 }
 
-/** Adds the unsigned integer operand to the CRC-32C checksum of the
- *     second unsigned integer operand.
- *
- *  \headerfile 
- *
- *  This intrinsic corresponds to the  CRC32D  instruction.
- *
- *  \param __C
- *     An unsigned integer operand to add to the CRC-32C checksum of operand
- *     \a  __D.
- *  \param __D
- *     An unsigned 32-bit integer operand used to compute the CRC-32C checksum.
- *  \returns The result of adding operand \a __C to the CRC-32C checksum of
- *     operand \a __D.
- */
+/// Adds the unsigned integer operand to the CRC-32C checksum of the
+///    second unsigned integer operand.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c CRC32D instruction.
+///
+/// \param __C
+///    An unsigned integer operand to add to the CRC-32C checksum of operand
+///    \a  __D.
+/// \param __D
+///    An unsigned 32-bit integer operand used to compute the CRC-32C checksum.
+/// \returns The result of adding operand \a __C to the CRC-32C checksum of
+///    operand \a __D.
 static __inline__ unsigned int __DEFAULT_FN_ATTRS_CRC32
 __crc32d(unsigned int __C, unsigned int __D)
 {
@@ -331,21 +449,20 @@ __crc32d(unsigned int __C, unsigned int __D)
 }
 
 #ifdef __x86_64__
-/** Adds the unsigned integer operand to the CRC-32C checksum of the
- *     unsigned 64-bit integer operand.
- *
- *  \headerfile 
- *
- *  This intrinsic corresponds to the  CRC32Q  instruction.
- *
- *  \param __C
- *     An unsigned integer operand to add to the CRC-32C checksum of operand
- *     \a  __D.
- *  \param __D
- *     An unsigned 64-bit integer operand used to compute the CRC-32C checksum.
- *  \returns The result of adding operand \a __C to the CRC-32C checksum of
- *     operand \a __D.
- */
+/// Adds the unsigned integer operand to the CRC-32C checksum of the
+///    unsigned 64-bit integer operand.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c CRC32Q instruction.
+///
+/// \param __C
+///    An unsigned integer operand to add to the CRC-32C checksum of operand
+///    \a  __D.
+/// \param __D
+///    An unsigned 64-bit integer operand used to compute the CRC-32C checksum.
+/// \returns The result of adding operand \a __C to the CRC-32C checksum of
+///    operand \a __D.
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS_CRC32
 __crc32q(unsigned long long __C, unsigned long long __D)
 {
@@ -353,19 +470,67 @@ __crc32q(unsigned long long __C, unsigned long long __D)
 }
 #endif /* __x86_64__ */
 
+/// Reads the specified performance monitoring counter. Refer to your
+///    processor's documentation to determine which performance counters are
+///    supported.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c RDPMC instruction.
+///
+/// \param __A
+///    The performance counter to read.
+/// \returns The 64-bit value read from the performance counter.
+/// \see _rdpmc
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 __rdpmc(int __A) {
   return __builtin_ia32_rdpmc(__A);
 }
 
-/* __rdtscp */
+/// Reads the processor's time stamp counter and the \c IA32_TSC_AUX MSR
+///    \c (0xc0000103).
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c RDTSCP instruction.
+///
+/// \param __A
+///    Address of where to store the 32-bit \c IA32_TSC_AUX value.
+/// \returns The 64-bit value of the time stamp counter.
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 __rdtscp(unsigned int *__A) {
   return __builtin_ia32_rdtscp(__A);
 }
 
+/// Reads the processor's time stamp counter.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned long long _rdtsc();
+/// \endcode
+///
+/// This intrinsic corresponds to the \c RDTSC instruction.
+///
+/// \returns The 64-bit value of the time stamp counter.
 #define _rdtsc() __rdtsc()
 
+/// Reads the specified performance monitoring counter. Refer to your
+///    processor's documentation to determine which performance counters are
+///    supported.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned long long _rdpmc(int A);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c RDPMC instruction.
+///
+/// \param A
+///    The performance counter to read.
+/// \returns The 64-bit value read from the performance counter.
+/// \see __rdpmc
 #define _rdpmc(A) __rdpmc(A)
 
 static __inline__ void __DEFAULT_FN_ATTRS
@@ -373,42 +538,150 @@ _wbinvd(void) {
   __builtin_ia32_wbinvd();
 }
 
+/// Rotates an 8-bit value to the left by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c ROL instruction.
+///
+/// \param __X
+///    The unsigned 8-bit value to be rotated.
+/// \param __C
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
 static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
 __rolb(unsigned char __X, int __C) {
   return __builtin_rotateleft8(__X, __C);
 }
 
+/// Rotates an 8-bit value to the right by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c ROR instruction.
+///
+/// \param __X
+///    The unsigned 8-bit value to be rotated.
+/// \param __C
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
 static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
 __rorb(unsigned char __X, int __C) {
   return __builtin_rotateright8(__X, __C);
 }
 
+/// Rotates a 16-bit value to the left by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c ROL instruction.
+///
+/// \param __X
+///    The unsigned 16-bit value to be rotated.
+/// \param __C
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
+/// \see _rotwl
 static __inline__ unsigned short __DEFAULT_FN_ATTRS_CONSTEXPR
 __rolw(unsigned short __X, int __C) {
   return __builtin_rotateleft16(__X, __C);
 }
 
+/// Rotates a 16-bit value to the right by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c ROR instruction.
+///
+/// \param __X
+///    The unsigned 16-bit value to be rotated.
+/// \param __C
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
+/// \see _rotwr
 static __inline__ unsigned short __DEFAULT_FN_ATTRS_CONSTEXPR
 __rorw(unsigned short __X, int __C) {
   return __builtin_rotateright16(__X, __C);
 }
 
+/// Rotates a 32-bit value to the left by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c ROL instruction.
+///
+/// \param __X
+///    The unsigned 32-bit value to be rotated.
+/// \param __C
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
+/// \see _rotl
 static __inline__ unsigned int __DEFAULT_FN_ATTRS_CONSTEXPR
 __rold(unsigned int __X, int __C) {
   return __builtin_rotateleft32(__X, (unsigned int)__C);
 }
 
+/// Rotates a 32-bit value to the right by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c ROR instruction.
+///
+/// \param __X
+///    The unsigned 32-bit value to be rotated.
+/// \param __C
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
+/// \see _rotr
 static __inline__ unsigned int __DEFAULT_FN_ATTRS_CONSTEXPR
 __rord(unsigned int __X, int __C) {
   return __builtin_rotateright32(__X, (unsigned int)__C);
 }
 
 #ifdef __x86_64__
+/// Rotates a 64-bit value to the left by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c ROL instruction.
+///
+/// \param __X
+///    The unsigned 64-bit value to be rotated.
+/// \param __C
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS_CONSTEXPR
 __rolq(unsigned long long __X, int __C) {
   return __builtin_rotateleft64(__X, (unsigned long long)__C);
 }
 
+/// Rotates a 64-bit value to the right by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the \c ROR instruction.
+///
+/// \param __X
+///    The unsigned 64-bit value to be rotated.
+/// \param __C
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS_CONSTEXPR
 __rorq(unsigned long long __X, int __C) {
   return __builtin_rotateright64(__X, (unsigned long long)__C);
@@ -419,18 +692,167 @@ __rorq(unsigned long long __X, int __C) {
 /* These are already provided as builtins for MSVC. */
 /* Select the correct function based on the size of long. */
 #ifdef __LP64__
+/// Rotates a 64-bit value to the left by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned long long _lrotl(unsigned long long a, int b);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c ROL instruction.
+///
+/// \param a
+///    The unsigned 64-bit value to be rotated.
+/// \param b
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
+/// \see __rolq
 #define _lrotl(a,b) __rolq((a), (b))
+
+/// Rotates a 64-bit value to the right by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned long long _lrotr(unsigned long long a, int b);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c ROR instruction.
+///
+/// \param a
+///    The unsigned 64-bit value to be rotated.
+/// \param b
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
+/// \see __rorq
 #define _lrotr(a,b) __rorq((a), (b))
-#else
+#else // __LP64__
+/// Rotates a 32-bit value to the left by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned int _lrotl(unsigned int a, int b);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c ROL instruction.
+///
+/// \param a
+///    The unsigned 32-bit value to be rotated.
+/// \param b
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
+/// \see __rold
 #define _lrotl(a,b) __rold((a), (b))
+
+/// Rotates a 32-bit value to the right by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned int _lrotr(unsigned int a, int b);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c ROR instruction.
+///
+/// \param a
+///    The unsigned 32-bit value to be rotated.
+/// \param b
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
+/// \see __rord
 #define _lrotr(a,b) __rord((a), (b))
-#endif
+#endif // __LP64__
+
+/// Rotates a 32-bit value to the left by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned int _rotl(unsigned int a, int b);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c ROL instruction.
+///
+/// \param a
+///    The unsigned 32-bit value to be rotated.
+/// \param b
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
+/// \see __rold
 #define _rotl(a,b) __rold((a), (b))
+
+/// Rotates a 32-bit value to the right by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned int _rotr(unsigned int a, int b);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c ROR instruction.
+///
+/// \param a
+///    The unsigned 32-bit value to be rotated.
+/// \param b
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
+/// \see __rord
 #define _rotr(a,b) __rord((a), (b))
 #endif // _MSC_VER
 
 /* These are not builtins so need to be provided in all modes. */
+/// Rotates a 16-bit value to the left by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned short _rotwl(unsigned short a, int b);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c ROL instruction.
+///
+/// \param a
+///    The unsigned 16-bit value to be rotated.
+/// \param b
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
+/// \see __rolw
 #define _rotwl(a,b) __rolw((a), (b))
+
+/// Rotates a 16-bit value to the right by the specified number of bits.
+///    This operation is undefined if the number of bits exceeds the size of
+///    the value.
+///
+/// \headerfile 
+///
+/// \code
+/// unsigned short _rotwr(unsigned short a, int b);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c ROR instruction.
+///
+/// \param a
+///    The unsigned 16-bit value to be rotated.
+/// \param b
+///    The number of bits to rotate the value.
+/// \returns The rotated value.
+/// \see __rorw
 #define _rotwr(a,b) __rorw((a), (b))
 
 #undef __DEFAULT_FN_ATTRS
diff --git a/clang/lib/Headers/immintrin.h b/clang/lib/Headers/immintrin.h
index 0149a1cdea63..27800f7a8202 100644
--- a/clang/lib/Headers/immintrin.h
+++ b/clang/lib/Headers/immintrin.h
@@ -490,6 +490,15 @@ _writegsbase_u64(unsigned long long __V)
  * field inside of it.
  */
 
+/// Load a 16-bit value from memory and swap its bytes.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the MOVBE instruction.
+///
+/// \param __P
+///    A pointer to the 16-bit value to load.
+/// \returns The byte-swapped value.
 static __inline__ short __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 _loadbe_i16(void const * __P) {
   struct __loadu_i16 {
@@ -498,6 +507,16 @@ _loadbe_i16(void const * __P) {
   return (short)__builtin_bswap16(((const struct __loadu_i16*)__P)->__v);
 }
 
+/// Swap the bytes of a 16-bit value and store it to memory.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the MOVBE instruction.
+///
+/// \param __P
+///    A pointer to the memory for storing the swapped value.
+/// \param __D
+///    The 16-bit value to be byte-swapped.
 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 _storebe_i16(void * __P, short __D) {
   struct __storeu_i16 {
@@ -506,6 +525,15 @@ _storebe_i16(void * __P, short __D) {
   ((struct __storeu_i16*)__P)->__v = __builtin_bswap16((unsigned short)__D);
 }
 
+/// Load a 32-bit value from memory and swap its bytes.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the MOVBE instruction.
+///
+/// \param __P
+///    A pointer to the 32-bit value to load.
+/// \returns The byte-swapped value.
 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 _loadbe_i32(void const * __P) {
   struct __loadu_i32 {
@@ -514,6 +542,16 @@ _loadbe_i32(void const * __P) {
   return (int)__builtin_bswap32(((const struct __loadu_i32*)__P)->__v);
 }
 
+/// Swap the bytes of a 32-bit value and store it to memory.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the MOVBE instruction.
+///
+/// \param __P
+///    A pointer to the memory for storing the swapped value.
+/// \param __D
+///    The 32-bit value to be byte-swapped.
 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 _storebe_i32(void * __P, int __D) {
   struct __storeu_i32 {
@@ -523,6 +561,15 @@ _storebe_i32(void * __P, int __D) {
 }
 
 #ifdef __x86_64__
+/// Load a 64-bit value from memory and swap its bytes.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the MOVBE instruction.
+///
+/// \param __P
+///    A pointer to the 64-bit value to load.
+/// \returns The byte-swapped value.
 static __inline__ long long __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 _loadbe_i64(void const * __P) {
   struct __loadu_i64 {
@@ -531,6 +578,16 @@ _loadbe_i64(void const * __P) {
   return (long long)__builtin_bswap64(((const struct __loadu_i64*)__P)->__v);
 }
 
+/// Swap the bytes of a 64-bit value and store it to memory.
+///
+/// \headerfile 
+///
+/// This intrinsic corresponds to the MOVBE instruction.
+///
+/// \param __P
+///    A pointer to the memory for storing the swapped value.
+/// \param __D
+///    The 64-bit value to be byte-swapped.
 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 _storebe_i64(void * __P, long long __D) {
   struct __storeu_i64 {
diff --git a/clang/lib/Headers/larchintrin.h b/clang/lib/Headers/larchintrin.h
index c5c533ee0b8c..a613e5ca0e5e 100644
--- a/clang/lib/Headers/larchintrin.h
+++ b/clang/lib/Headers/larchintrin.h
@@ -228,6 +228,18 @@ extern __inline void
   ((void)__builtin_loongarch_ldpte_d((long int)(_1), (_2)))
 #endif
 
+#define __frecipe_s(/*float*/ _1)                                              \
+  (float)__builtin_loongarch_frecipe_s((float)_1)
+
+#define __frecipe_d(/*double*/ _1)                                             \
+  (double)__builtin_loongarch_frecipe_d((double)_1)
+
+#define __frsqrte_s(/*float*/ _1)                                              \
+  (float)__builtin_loongarch_frsqrte_s((float)_1)
+
+#define __frsqrte_d(/*double*/ _1)                                             \
+  (double)__builtin_loongarch_frsqrte_d((double)_1)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/clang/lib/Headers/lasxintrin.h b/clang/lib/Headers/lasxintrin.h
index 6b4d5012a24b..dafc2a2f3e6a 100644
--- a/clang/lib/Headers/lasxintrin.h
+++ b/clang/lib/Headers/lasxintrin.h
@@ -1726,6 +1726,18 @@ extern __inline
   return (__m256d)__builtin_lasx_xvfrecip_d((v4f64)_1);
 }
 
+extern __inline
+    __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
+    __lasx_xvfrecipe_s(__m256 _1) {
+  return (__m256)__builtin_lasx_xvfrecipe_s((v8f32)_1);
+}
+
+extern __inline
+    __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
+    __lasx_xvfrecipe_d(__m256d _1) {
+  return (__m256d)__builtin_lasx_xvfrecipe_d((v4f64)_1);
+}
+
 extern __inline
     __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
     __lasx_xvfrint_s(__m256 _1) {
@@ -1750,6 +1762,18 @@ extern __inline
   return (__m256d)__builtin_lasx_xvfrsqrt_d((v4f64)_1);
 }
 
+extern __inline
+    __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
+    __lasx_xvfrsqrte_s(__m256 _1) {
+  return (__m256)__builtin_lasx_xvfrsqrte_s((v8f32)_1);
+}
+
+extern __inline
+    __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
+    __lasx_xvfrsqrte_d(__m256d _1) {
+  return (__m256d)__builtin_lasx_xvfrsqrte_d((v4f64)_1);
+}
+
 extern __inline
     __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
     __lasx_xvflogb_s(__m256 _1) {
diff --git a/clang/lib/Headers/llvm_libc_wrappers/stdio.h b/clang/lib/Headers/llvm_libc_wrappers/stdio.h
index 0870f3e741ec..950f91b3763e 100644
--- a/clang/lib/Headers/llvm_libc_wrappers/stdio.h
+++ b/clang/lib/Headers/llvm_libc_wrappers/stdio.h
@@ -6,15 +6,41 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
-#define __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
-
 #if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
 #error "This file is for GPU offloading compilation only"
 #endif
 
 #include_next 
 
+// In some old versions of glibc, other standard headers sometimes define
+// special macros (e.g., __need_FILE) before including stdio.h to cause stdio.h
+// to produce special definitions.  Future includes of stdio.h when those
+// special macros are undefined are expected to produce the normal definitions
+// from stdio.h.
+//
+// We do not apply our include guard (__CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__)
+// unconditionally to the above include_next.  Otherwise, after an occurrence of
+// the first glibc stdio.h use case described above, the include_next would be
+// skipped for remaining includes of stdio.h, leaving required symbols
+// undefined.
+//
+// We make the following assumptions to handle all use cases:
+//
+// 1. If the above include_next produces special glibc definitions, then (a) it
+//    does not produce the normal definitions that we must intercept below, (b)
+//    the current file was included from a glibc header that already defined
+//    __GLIBC__ (usually by including glibc's ), and (c) the above
+//    include_next does not define _STDIO_H.  In that case, we skip the rest of
+//    the current file and don't guard against future includes.
+// 2. If the above include_next produces the normal stdio.h definitions, then
+//    either (a) __GLIBC__ is not defined because C headers are from some other
+//    libc implementation or (b) the above include_next defines _STDIO_H to
+//    prevent the above include_next from having any effect in the future.
+#if !defined(__GLIBC__) || defined(_STDIO_H)
+
+#ifndef __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
+#define __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
+
 #if __has_include()
 
 #if defined(__HIP__) || defined(__CUDA__)
@@ -50,3 +76,5 @@
 #endif
 
 #endif // __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
+
+#endif
diff --git a/clang/lib/Headers/llvm_libc_wrappers/string.h b/clang/lib/Headers/llvm_libc_wrappers/string.h
index b4fbf17c7e42..0ea49cb13760 100644
--- a/clang/lib/Headers/llvm_libc_wrappers/string.h
+++ b/clang/lib/Headers/llvm_libc_wrappers/string.h
@@ -51,13 +51,13 @@ char *strcpy(char *__restrict, const char *__restrict) __LIBC_ATTRS;
 size_t strcspn(const char *, const char *) __LIBC_ATTRS;
 char *strdup(const char *) __LIBC_ATTRS;
 size_t strlen(const char *) __LIBC_ATTRS;
-char *strncat(char *, const char *, size_t) __LIBC_ATTRS;
+char *strncat(char *__restrict, const char *__restrict, size_t) __LIBC_ATTRS;
 int strncmp(const char *, const char *, size_t) __LIBC_ATTRS;
 char *strncpy(char *__restrict, const char *__restrict, size_t) __LIBC_ATTRS;
 char *strndup(const char *, size_t) __LIBC_ATTRS;
 size_t strnlen(const char *, size_t) __LIBC_ATTRS;
 size_t strspn(const char *, const char *) __LIBC_ATTRS;
-char *strtok(char *__restrict, const char *) __LIBC_ATTRS;
+char *strtok(char *__restrict, const char *__restrict) __LIBC_ATTRS;
 char *strtok_r(char *__restrict, const char *__restrict,
                char **__restrict) __LIBC_ATTRS;
 size_t strxfrm(char *__restrict, const char *__restrict, size_t) __LIBC_ATTRS;
diff --git a/clang/lib/Headers/lsxintrin.h b/clang/lib/Headers/lsxintrin.h
index a29bc7757ab5..f347955ce6fb 100644
--- a/clang/lib/Headers/lsxintrin.h
+++ b/clang/lib/Headers/lsxintrin.h
@@ -1776,6 +1776,18 @@ extern __inline
   return (__m128d)__builtin_lsx_vfrecip_d((v2f64)_1);
 }
 
+extern __inline
+    __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
+    __lsx_vfrecipe_s(__m128 _1) {
+  return (__m128)__builtin_lsx_vfrecipe_s((v4f32)_1);
+}
+
+extern __inline
+    __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128d
+    __lsx_vfrecipe_d(__m128d _1) {
+  return (__m128d)__builtin_lsx_vfrecipe_d((v2f64)_1);
+}
+
 extern __inline
     __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
     __lsx_vfrint_s(__m128 _1) {
@@ -1800,6 +1812,18 @@ extern __inline
   return (__m128d)__builtin_lsx_vfrsqrt_d((v2f64)_1);
 }
 
+extern __inline
+    __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
+    __lsx_vfrsqrte_s(__m128 _1) {
+  return (__m128)__builtin_lsx_vfrsqrte_s((v4f32)_1);
+}
+
+extern __inline
+    __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128d
+    __lsx_vfrsqrte_d(__m128d _1) {
+  return (__m128d)__builtin_lsx_vfrsqrte_d((v2f64)_1);
+}
+
 extern __inline
     __attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
     __lsx_vflogb_s(__m128 _1) {
diff --git a/clang/lib/Index/USRGeneration.cpp b/clang/lib/Index/USRGeneration.cpp
index eb4ec1098638..1e3024f51d5f 100644
--- a/clang/lib/Index/USRGeneration.cpp
+++ b/clang/lib/Index/USRGeneration.cpp
@@ -15,6 +15,7 @@
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclVisitor.h"
+#include "clang/AST/ODRHash.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Lex/PreprocessingRecord.h"
 #include "llvm/Support/Path.h"
@@ -1058,6 +1059,15 @@ void USRGenerator::VisitTemplateArgument(const TemplateArgument &Arg) {
     VisitType(Arg.getIntegralType());
     Out << Arg.getAsIntegral();
     break;
+
+  case TemplateArgument::StructuralValue: {
+    Out << 'S';
+    VisitType(Arg.getStructuralValueType());
+    ODRHash Hash{};
+    Hash.AddStructuralValue(Arg.getAsStructuralValue());
+    Out << Hash.CalculateHash();
+    break;
+  }
   }
 }
 
diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp
index c9fcef5b5b5a..9f97a3c6b0be 100644
--- a/clang/lib/Interpreter/Interpreter.cpp
+++ b/clang/lib/Interpreter/Interpreter.cpp
@@ -248,7 +248,7 @@ Interpreter::~Interpreter() {
 // can't find the precise resource directory in unittests so we have to hard
 // code them.
 const char *const Runtimes = R"(
-    void* operator new(__SIZE_TYPE__, void* __p) noexcept;
+#ifdef __cplusplus
     void *__clang_Interpreter_SetValueWithAlloc(void*, void*, void*);
     void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*);
     void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, void*);
@@ -256,15 +256,18 @@ const char *const Runtimes = R"(
     void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, double);
     void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, long double);
     void __clang_Interpreter_SetValueNoAlloc(void*,void*,void*,unsigned long long);
+    struct __clang_Interpreter_NewTag{} __ci_newtag;
+    void* operator new(__SIZE_TYPE__, void* __p, __clang_Interpreter_NewTag) noexcept;
     template 
     void __clang_Interpreter_SetValueCopyArr(T* Src, void* Placement, unsigned long Size) {
       for (auto Idx = 0; Idx < Size; ++Idx)
-        new ((void*)(((T*)Placement) + Idx)) T(Src[Idx]);
+        new ((void*)(((T*)Placement) + Idx), __ci_newtag) T(Src[Idx]);
     }
     template 
     void __clang_Interpreter_SetValueCopyArr(const T (*Src)[N], void* Placement, unsigned long Size) {
       __clang_Interpreter_SetValueCopyArr(Src[0], Placement, Size);
     }
+#endif // __cplusplus
 )";
 
 llvm::Expected>
@@ -279,7 +282,7 @@ Interpreter::create(std::unique_ptr CI) {
   if (!PTU)
     return PTU.takeError();
 
-  Interp->ValuePrintingInfo.resize(3);
+  Interp->ValuePrintingInfo.resize(4);
   // FIXME: This is a ugly hack. Undo command checks its availability by looking
   // at the size of the PTU list. However we have parsed something in the
   // beginning of the REPL so we have to mark them as 'Irrevocable'.
@@ -500,7 +503,7 @@ Interpreter::CompileDtorCall(CXXRecordDecl *CXXRD) {
 static constexpr llvm::StringRef MagicRuntimeInterface[] = {
     "__clang_Interpreter_SetValueNoAlloc",
     "__clang_Interpreter_SetValueWithAlloc",
-    "__clang_Interpreter_SetValueCopyArr"};
+    "__clang_Interpreter_SetValueCopyArr", "__ci_newtag"};
 
 bool Interpreter::FindRuntimeInterface() {
   if (llvm::all_of(ValuePrintingInfo, [](Expr *E) { return E != nullptr; }))
@@ -530,6 +533,9 @@ bool Interpreter::FindRuntimeInterface() {
   if (!LookupInterface(ValuePrintingInfo[CopyArray],
                        MagicRuntimeInterface[CopyArray]))
     return false;
+  if (!LookupInterface(ValuePrintingInfo[NewTag],
+                       MagicRuntimeInterface[NewTag]))
+    return false;
   return true;
 }
 
@@ -607,7 +613,9 @@ class RuntimeInterfaceBuilder
                 .getValuePrintingInfo()[Interpreter::InterfaceKind::CopyArray],
             SourceLocation(), Args, SourceLocation());
       }
-      Expr *Args[] = {AllocCall.get()};
+      Expr *Args[] = {
+          AllocCall.get(),
+          Interp.getValuePrintingInfo()[Interpreter::InterfaceKind::NewTag]};
       ExprResult CXXNewCall = S.BuildCXXNew(
           E->getSourceRange(),
           /*UseGlobal=*/true, /*PlacementLParen=*/SourceLocation(), Args,
@@ -628,8 +636,9 @@ class RuntimeInterfaceBuilder
           Interp.getValuePrintingInfo()[Interpreter::InterfaceKind::NoAlloc],
           E->getBeginLoc(), Args, E->getEndLoc());
     }
+    default:
+      llvm_unreachable("Unhandled Interpreter::InterfaceKind");
     }
-    llvm_unreachable("Unhandled Interpreter::InterfaceKind");
   }
 
   Interpreter::InterfaceKind VisitRecordType(const RecordType *Ty) {
@@ -814,3 +823,15 @@ __clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType,
   VRef = Value(static_cast(This), OpaqueType);
   VRef.setLongDouble(Val);
 }
+
+// A trampoline to work around the fact that operator placement new cannot
+// really be forward declared due to libc++ and libstdc++ declaration mismatch.
+// FIXME: __clang_Interpreter_NewTag is ODR violation because we get the same
+// definition in the interpreter runtime. We should move it in a runtime header
+// which gets included by the interpreter and here.
+struct __clang_Interpreter_NewTag {};
+REPL_EXTERNAL_VISIBILITY void *
+operator new(size_t __sz, void *__p, __clang_Interpreter_NewTag) noexcept {
+  // Just forward to the standard operator placement new.
+  return operator new(__sz, __p);
+}
diff --git a/clang/lib/Lex/HeaderSearch.cpp b/clang/lib/Lex/HeaderSearch.cpp
index 0f1090187734..dfa974e9a67e 100644
--- a/clang/lib/Lex/HeaderSearch.cpp
+++ b/clang/lib/Lex/HeaderSearch.cpp
@@ -1408,57 +1408,57 @@ bool HeaderSearch::ShouldEnterIncludeFile(Preprocessor &PP,
   // Get information about this file.
   HeaderFileInfo &FileInfo = getFileInfo(File);
 
-  // FIXME: this is a workaround for the lack of proper modules-aware support
-  // for #import / #pragma once
-  auto TryEnterImported = [&]() -> bool {
-    if (!ModulesEnabled)
-      return false;
-    // Ensure FileInfo bits are up to date.
-    ModMap.resolveHeaderDirectives(File);
-    // Modules with builtins are special; multiple modules use builtins as
-    // modular headers, example:
-    //
-    //    module stddef { header "stddef.h" export * }
-    //
-    // After module map parsing, this expands to:
-    //
-    //    module stddef {
-    //      header "/path_to_builtin_dirs/stddef.h"
-    //      textual "stddef.h"
-    //    }
-    //
-    // It's common that libc++ and system modules will both define such
-    // submodules. Make sure cached results for a builtin header won't
-    // prevent other builtin modules from potentially entering the builtin
-    // header. Note that builtins are header guarded and the decision to
-    // actually enter them is postponed to the controlling macros logic below.
-    bool TryEnterHdr = false;
-    if (FileInfo.isCompilingModuleHeader && FileInfo.isModuleHeader)
-      TryEnterHdr = ModMap.isBuiltinHeader(File);
-
-    // Textual headers can be #imported from different modules. Since ObjC
-    // headers find in the wild might rely only on #import and do not contain
-    // controlling macros, be conservative and only try to enter textual headers
-    // if such macro is present.
-    if (!FileInfo.isModuleHeader &&
-        FileInfo.getControllingMacro(ExternalLookup))
-      TryEnterHdr = true;
-    return TryEnterHdr;
-  };
-
   // If this is a #import directive, check that we have not already imported
   // this header.
   if (isImport) {
     // If this has already been imported, don't import it again.
     FileInfo.isImport = true;
 
+    // FIXME: this is a workaround for the lack of proper modules-aware support
+    // for #import / #pragma once
+    auto TryEnterImported = [&]() -> bool {
+      if (!ModulesEnabled)
+        return false;
+      // Ensure FileInfo bits are up to date.
+      ModMap.resolveHeaderDirectives(File);
+      // Modules with builtins are special; multiple modules use builtins as
+      // modular headers, example:
+      //
+      //    module stddef { header "stddef.h" export * }
+      //
+      // After module map parsing, this expands to:
+      //
+      //    module stddef {
+      //      header "/path_to_builtin_dirs/stddef.h"
+      //      textual "stddef.h"
+      //    }
+      //
+      // It's common that libc++ and system modules will both define such
+      // submodules. Make sure cached results for a builtin header won't
+      // prevent other builtin modules from potentially entering the builtin
+      // header. Note that builtins are header guarded and the decision to
+      // actually enter them is postponed to the controlling macros logic below.
+      bool TryEnterHdr = false;
+      if (FileInfo.isCompilingModuleHeader && FileInfo.isModuleHeader)
+        TryEnterHdr = ModMap.isBuiltinHeader(File);
+
+      // Textual headers can be #imported from different modules. Since ObjC
+      // headers find in the wild might rely only on #import and do not contain
+      // controlling macros, be conservative and only try to enter textual
+      // headers if such macro is present.
+      if (!FileInfo.isModuleHeader &&
+          FileInfo.getControllingMacro(ExternalLookup))
+        TryEnterHdr = true;
+      return TryEnterHdr;
+    };
+
     // Has this already been #import'ed or #include'd?
     if (PP.alreadyIncluded(File) && !TryEnterImported())
       return false;
   } else {
     // Otherwise, if this is a #include of a file that was previously #import'd
     // or if this is the second #include of a #pragma once file, ignore it.
-    if ((FileInfo.isPragmaOnce || FileInfo.isImport) && !TryEnterImported())
+    if (FileInfo.isPragmaOnce || FileInfo.isImport)
       return false;
   }
 
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index ea5d13deb114..afb2948f05ae 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -308,7 +308,7 @@ void ModuleMap::resolveHeader(Module *Mod,
         setUmbrellaHeaderAsWritten(Mod, *File, Header.FileName,
                                    RelativePathName.str());
     } else {
-      Module::Header H = {Header.FileName, std::string(RelativePathName.str()),
+      Module::Header H = {Header.FileName, std::string(RelativePathName),
                           *File};
       addHeader(Mod, H, headerKindToRole(Header.Kind));
     }
@@ -984,7 +984,9 @@ static void inferFrameworkLink(Module *Mod) {
   assert(!Mod->isSubFramework() &&
          "Can only infer linking for top-level frameworks");
 
-  Mod->LinkLibraries.push_back(Module::LinkLibrary(Mod->Name,
+  StringRef FrameworkName(Mod->Name);
+  FrameworkName.consume_back("_Private");
+  Mod->LinkLibraries.push_back(Module::LinkLibrary(FrameworkName.str(),
                                                    /*IsFramework=*/true));
 }
 
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 64f54c6fc638..7fdb5d4c0d7b 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -1457,6 +1457,11 @@ void Preprocessor::emitRestrictExpansionWarning(const Token &Identifier) const {
   Diag(Info.Location, diag::note_pp_macro_annotation) << 1;
 }
 
+void Preprocessor::emitRestrictInfNaNWarning(const Token &Identifier,
+                                             unsigned DiagSelection) const {
+  Diag(Identifier, diag::warn_fp_nan_inf_when_disabled) << DiagSelection << 1;
+}
+
 void Preprocessor::emitFinalMacroWarning(const Token &Identifier,
                                          bool IsUndef) const {
   const MacroAnnotations &A =
diff --git a/clang/lib/Lex/UnicodeCharSets.h b/clang/lib/Lex/UnicodeCharSets.h
index 5316d2540b76..b63908024e5a 100644
--- a/clang/lib/Lex/UnicodeCharSets.h
+++ b/clang/lib/Lex/UnicodeCharSets.h
@@ -10,7 +10,7 @@
 
 #include "llvm/Support/UnicodeCharRanges.h"
 
-// Unicode 15.0 XID_Start
+// Unicode 15.1 XID_Start
 static const llvm::sys::UnicodeCharRange XIDStartRanges[] = {
     {0x0041, 0x005A},   {0x0061, 0x007A},   {0x00AA, 0x00AA},
     {0x00B5, 0x00B5},   {0x00BA, 0x00BA},   {0x00C0, 0x00D6},
@@ -233,9 +233,10 @@ static const llvm::sys::UnicodeCharRange XIDStartRanges[] = {
     {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9},
     {0x1EEAB, 0x1EEBB}, {0x20000, 0x2A6DF}, {0x2A700, 0x2B739},
     {0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, {0x2CEB0, 0x2EBE0},
-    {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, {0x31350, 0x323AF}};
+    {0x2EBF0, 0x2EE5D}, {0x2F800, 0x2FA1D}, {0x30000, 0x3134A},
+    {0x31350, 0x323AF}};
 
-// Unicode 15.0 XID_Continue, excluding XID_Start
+// Unicode 15.1 XID_Continue, excluding XID_Start
 // The Unicode Property XID_Continue is a super set of XID_Start.
 // To save Space, the table below only contains the codepoints
 // that are not also in XID_Start.
@@ -299,24 +300,25 @@ static const llvm::sys::UnicodeCharRange XIDContinueRanges[] = {
     {0x1C24, 0x1C37},   {0x1C40, 0x1C49},   {0x1C50, 0x1C59},
     {0x1CD0, 0x1CD2},   {0x1CD4, 0x1CE8},   {0x1CED, 0x1CED},
     {0x1CF4, 0x1CF4},   {0x1CF7, 0x1CF9},   {0x1DC0, 0x1DFF},
-    {0x203F, 0x2040},   {0x2054, 0x2054},   {0x20D0, 0x20DC},
-    {0x20E1, 0x20E1},   {0x20E5, 0x20F0},   {0x2CEF, 0x2CF1},
-    {0x2D7F, 0x2D7F},   {0x2DE0, 0x2DFF},   {0x302A, 0x302F},
-    {0x3099, 0x309A},   {0xA620, 0xA629},   {0xA66F, 0xA66F},
-    {0xA674, 0xA67D},   {0xA69E, 0xA69F},   {0xA6F0, 0xA6F1},
-    {0xA802, 0xA802},   {0xA806, 0xA806},   {0xA80B, 0xA80B},
-    {0xA823, 0xA827},   {0xA82C, 0xA82C},   {0xA880, 0xA881},
-    {0xA8B4, 0xA8C5},   {0xA8D0, 0xA8D9},   {0xA8E0, 0xA8F1},
-    {0xA8FF, 0xA909},   {0xA926, 0xA92D},   {0xA947, 0xA953},
-    {0xA980, 0xA983},   {0xA9B3, 0xA9C0},   {0xA9D0, 0xA9D9},
-    {0xA9E5, 0xA9E5},   {0xA9F0, 0xA9F9},   {0xAA29, 0xAA36},
-    {0xAA43, 0xAA43},   {0xAA4C, 0xAA4D},   {0xAA50, 0xAA59},
-    {0xAA7B, 0xAA7D},   {0xAAB0, 0xAAB0},   {0xAAB2, 0xAAB4},
-    {0xAAB7, 0xAAB8},   {0xAABE, 0xAABF},   {0xAAC1, 0xAAC1},
-    {0xAAEB, 0xAAEF},   {0xAAF5, 0xAAF6},   {0xABE3, 0xABEA},
-    {0xABEC, 0xABED},   {0xABF0, 0xABF9},   {0xFB1E, 0xFB1E},
-    {0xFE00, 0xFE0F},   {0xFE20, 0xFE2F},   {0xFE33, 0xFE34},
-    {0xFE4D, 0xFE4F},   {0xFF10, 0xFF19},   {0xFF3F, 0xFF3F},
+    {0x200C, 0x200D},   {0x203F, 0x2040},   {0x2054, 0x2054},
+    {0x20D0, 0x20DC},   {0x20E1, 0x20E1},   {0x20E5, 0x20F0},
+    {0x2CEF, 0x2CF1},   {0x2D7F, 0x2D7F},   {0x2DE0, 0x2DFF},
+    {0x302A, 0x302F},   {0x3099, 0x309A},   {0x30FB, 0x30FB},
+    {0xA620, 0xA629},   {0xA66F, 0xA66F},   {0xA674, 0xA67D},
+    {0xA69E, 0xA69F},   {0xA6F0, 0xA6F1},   {0xA802, 0xA802},
+    {0xA806, 0xA806},   {0xA80B, 0xA80B},   {0xA823, 0xA827},
+    {0xA82C, 0xA82C},   {0xA880, 0xA881},   {0xA8B4, 0xA8C5},
+    {0xA8D0, 0xA8D9},   {0xA8E0, 0xA8F1},   {0xA8FF, 0xA909},
+    {0xA926, 0xA92D},   {0xA947, 0xA953},   {0xA980, 0xA983},
+    {0xA9B3, 0xA9C0},   {0xA9D0, 0xA9D9},   {0xA9E5, 0xA9E5},
+    {0xA9F0, 0xA9F9},   {0xAA29, 0xAA36},   {0xAA43, 0xAA43},
+    {0xAA4C, 0xAA4D},   {0xAA50, 0xAA59},   {0xAA7B, 0xAA7D},
+    {0xAAB0, 0xAAB0},   {0xAAB2, 0xAAB4},   {0xAAB7, 0xAAB8},
+    {0xAABE, 0xAABF},   {0xAAC1, 0xAAC1},   {0xAAEB, 0xAAEF},
+    {0xAAF5, 0xAAF6},   {0xABE3, 0xABEA},   {0xABEC, 0xABED},
+    {0xABF0, 0xABF9},   {0xFB1E, 0xFB1E},   {0xFE00, 0xFE0F},
+    {0xFE20, 0xFE2F},   {0xFE33, 0xFE34},   {0xFE4D, 0xFE4F},
+    {0xFF10, 0xFF19},   {0xFF3F, 0xFF3F},   {0xFF65, 0xFF65},
     {0xFF9E, 0xFF9F},   {0x101FD, 0x101FD}, {0x102E0, 0x102E0},
     {0x10376, 0x1037A}, {0x104A0, 0x104A9}, {0x10A01, 0x10A03},
     {0x10A05, 0x10A06}, {0x10A0C, 0x10A0F}, {0x10A38, 0x10A3A},
@@ -364,7 +366,8 @@ static const llvm::sys::UnicodeCharRange XIDContinueRanges[] = {
     {0x1E026, 0x1E02A}, {0x1E08F, 0x1E08F}, {0x1E130, 0x1E136},
     {0x1E140, 0x1E149}, {0x1E2AE, 0x1E2AE}, {0x1E2EC, 0x1E2F9},
     {0x1E4EC, 0x1E4F9}, {0x1E8D0, 0x1E8D6}, {0x1E944, 0x1E94A},
-    {0x1E950, 0x1E959}, {0x1FBF0, 0x1FBF9}, {0xE0100, 0xE01EF}};
+    {0x1E950, 0x1E959}, {0x1FBF0, 0x1FBF9}, {0xE0100, 0xE01EF},
+};
 
 // Clang supports the "Mathematical notation profile" as an extension,
 // as described in https://www.unicode.org/L2/L2022/22230-math-profile.pdf
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index ed006f9d67de..356e7851ec63 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -79,7 +79,7 @@ TypeResult Parser::ParseTypeName(SourceRange *Range, DeclaratorContext Context,
   if (DeclaratorInfo.isInvalidType())
     return true;
 
-  return Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
+  return Actions.ActOnTypeName(DeclaratorInfo);
 }
 
 /// Normalizes an attribute name by dropping prefixed and suffixed __.
@@ -2661,7 +2661,12 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
       // ProduceConstructorSignatureHelp only on VarDecls.
       ExpressionStarts = SetPreferredType;
     }
-    if (ParseExpressionList(Exprs, ExpressionStarts)) {
+
+    bool SawError = ParseExpressionList(Exprs, ExpressionStarts);
+
+    InitScope.pop();
+
+    if (SawError) {
       if (ThisVarDecl && PP.isCodeCompletionReached() && !CalledSignatureHelp) {
         Actions.ProduceConstructorSignatureHelp(
             ThisVarDecl->getType()->getCanonicalTypeInternal(),
@@ -2674,7 +2679,6 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
     } else {
       // Match the ')'.
       T.consumeClose();
-      InitScope.pop();
 
       ExprResult Initializer = Actions.ActOnParenListExpr(T.getOpenLocation(),
                                                           T.getCloseLocation(),
@@ -3483,7 +3487,8 @@ void Parser::ParseDeclarationSpecifiers(
 
     case tok::coloncolon: // ::foo::bar
       // C++ scope specifier.  Annotate and loop, or bail out on error.
-      if (TryAnnotateCXXScopeToken(EnteringContext)) {
+      if (getLangOpts().CPlusPlus &&
+          TryAnnotateCXXScopeToken(EnteringContext)) {
         if (!DS.hasTypeSpecifier())
           DS.SetTypeSpecError();
         goto DoneWithDeclSpec;
@@ -5006,7 +5011,7 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
                                   DeclSpecContext::DSC_type_specifier);
       Declarator DeclaratorInfo(DS, ParsedAttributesView::none(),
                                 DeclaratorContext::TypeName);
-      BaseType = Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
+      BaseType = Actions.ActOnTypeName(DeclaratorInfo);
 
       BaseRange = SourceRange(ColonLoc, DeclaratorInfo.getSourceRange().getEnd());
 
@@ -6786,7 +6791,13 @@ void Parser::ParseDirectDeclarator(Declarator &D) {
     } else if (Tok.isRegularKeywordAttribute()) {
       // For consistency with attribute parsing.
       Diag(Tok, diag::err_keyword_not_allowed) << Tok.getIdentifierInfo();
+      bool TakesArgs = doesKeywordAttributeTakeArgs(Tok.getKind());
       ConsumeToken();
+      if (TakesArgs) {
+        BalancedDelimiterTracker T(*this, tok::l_paren);
+        if (!T.consumeOpen())
+          T.skipToEnd();
+      }
     } else if (Tok.is(tok::kw_requires) && D.hasGroupingParens()) {
       // This declarator is declaring a function, but the requires clause is
       // in the wrong place:
@@ -8040,6 +8051,71 @@ bool Parser::TryAltiVecTokenOutOfLine(DeclSpec &DS, SourceLocation Loc,
   return false;
 }
 
+TypeResult Parser::ParseTypeFromString(StringRef TypeStr, StringRef Context,
+                                       SourceLocation IncludeLoc) {
+  // Consume (unexpanded) tokens up to the end-of-directive.
+  SmallVector Tokens;
+  {
+    // Create a new buffer from which we will parse the type.
+    auto &SourceMgr = PP.getSourceManager();
+    FileID FID = SourceMgr.createFileID(
+        llvm::MemoryBuffer::getMemBufferCopy(TypeStr, Context), SrcMgr::C_User,
+        0, 0, IncludeLoc);
+
+    // Form a new lexer that references the buffer.
+    Lexer L(FID, SourceMgr.getBufferOrFake(FID), PP);
+    L.setParsingPreprocessorDirective(true);
+
+    // Lex the tokens from that buffer.
+    Token Tok;
+    do {
+      L.Lex(Tok);
+      Tokens.push_back(Tok);
+    } while (Tok.isNot(tok::eod));
+  }
+
+  // Replace the "eod" token with an "eof" token identifying the end of
+  // the provided string.
+  Token &EndToken = Tokens.back();
+  EndToken.startToken();
+  EndToken.setKind(tok::eof);
+  EndToken.setLocation(Tok.getLocation());
+  EndToken.setEofData(TypeStr.data());
+
+  // Add the current token back.
+  Tokens.push_back(Tok);
+
+  // Enter the tokens into the token stream.
+  PP.EnterTokenStream(Tokens, /*DisableMacroExpansion=*/false,
+                      /*IsReinject=*/false);
+
+  // Consume the current token so that we'll start parsing the tokens we
+  // added to the stream.
+  ConsumeAnyToken();
+
+  // Enter a new scope.
+  ParseScope LocalScope(this, 0);
+
+  // Parse the type.
+  TypeResult Result = ParseTypeName(nullptr);
+
+  // Check if we parsed the whole thing.
+  if (Result.isUsable() &&
+      (Tok.isNot(tok::eof) || Tok.getEofData() != TypeStr.data())) {
+    Diag(Tok.getLocation(), diag::err_type_unparsed);
+  }
+
+  // There could be leftover tokens (e.g. because of an error).
+  // Skip through until we reach the 'end of directive' token.
+  while (Tok.isNot(tok::eof))
+    ConsumeAnyToken();
+
+  // Consume the end token.
+  if (Tok.is(tok::eof) && Tok.getEofData() == TypeStr.data())
+    ConsumeAnyToken();
+  return Result;
+}
+
 void Parser::DiagnoseBitIntUse(const Token &Tok) {
   // If the token is for _ExtInt, diagnose it as being deprecated. Otherwise,
   // the token is about _BitInt and gets (potentially) diagnosed as use of an
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index 910112ecae96..c0d771dc93da 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -1290,7 +1290,7 @@ TypeResult Parser::ParseBaseTypeSpecifier(SourceLocation &BaseLoc,
 
     Declarator DeclaratorInfo(DS, ParsedAttributesView::none(),
                               DeclaratorContext::TypeName);
-    return Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
+    return Actions.ActOnTypeName(DeclaratorInfo);
   }
 
   // Check whether we have a template-id that names a type.
@@ -1385,7 +1385,7 @@ TypeResult Parser::ParseBaseTypeSpecifier(SourceLocation &BaseLoc,
 
   Declarator DeclaratorInfo(DS, ParsedAttributesView::none(),
                             DeclaratorContext::TypeName);
-  return Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
+  return Actions.ActOnTypeName(DeclaratorInfo);
 }
 
 void Parser::ParseMicrosoftInheritanceClassAttributes(ParsedAttributes &attrs) {
@@ -1890,7 +1890,13 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
         if (!SkipUntil(tok::r_paren, StopAtSemi))
           break;
       } else if (Tok.isRegularKeywordAttribute()) {
+        bool TakesArgs = doesKeywordAttributeTakeArgs(Tok.getKind());
         ConsumeToken();
+        if (TakesArgs) {
+          BalancedDelimiterTracker T(*this, tok::l_paren);
+          if (!T.consumeOpen())
+            T.skipToEnd();
+        }
       } else {
         break;
       }
@@ -2679,6 +2685,8 @@ Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
                                        ParsedAttributes &AccessAttrs,
                                        const ParsedTemplateInfo &TemplateInfo,
                                        ParsingDeclRAIIObject *TemplateDiags) {
+  assert(getLangOpts().CPlusPlus &&
+         "ParseCXXClassMemberDeclaration should only be called in C++ mode");
   if (Tok.is(tok::at)) {
     if (getLangOpts().ObjC && NextToken().isObjCAtKeyword(tok::objc_defs))
       Diag(Tok, diag::err_at_defs_cxx);
@@ -4537,8 +4545,18 @@ void Parser::ParseCXX11AttributeSpecifierInternal(ParsedAttributes &Attrs,
   if (Tok.isRegularKeywordAttribute()) {
     SourceLocation Loc = Tok.getLocation();
     IdentifierInfo *AttrName = Tok.getIdentifierInfo();
-    Attrs.addNew(AttrName, Loc, nullptr, Loc, nullptr, 0, Tok.getKind());
+    ParsedAttr::Form Form = ParsedAttr::Form(Tok.getKind());
+    bool TakesArgs = doesKeywordAttributeTakeArgs(Tok.getKind());
     ConsumeToken();
+    if (TakesArgs) {
+      if (!Tok.is(tok::l_paren))
+        Diag(Tok.getLocation(), diag::err_expected_lparen_after) << AttrName;
+      else
+        ParseAttributeArgsCommon(AttrName, Loc, Attrs, EndLoc,
+                                 /*ScopeName*/ nullptr,
+                                 /*ScopeLoc*/ Loc, Form);
+    } else
+      Attrs.addNew(AttrName, Loc, nullptr, Loc, nullptr, 0, Form);
     return;
   }
 
@@ -4704,11 +4722,13 @@ SourceLocation Parser::SkipCXX11Attributes() {
       T.consumeOpen();
       T.skipToEnd();
       EndLoc = T.getCloseLocation();
-    } else if (Tok.isRegularKeywordAttribute()) {
+    } else if (Tok.isRegularKeywordAttribute() &&
+               !doesKeywordAttributeTakeArgs(Tok.getKind())) {
       EndLoc = Tok.getLocation();
       ConsumeToken();
     } else {
-      assert(Tok.is(tok::kw_alignas) && "not an attribute specifier");
+      assert((Tok.is(tok::kw_alignas) || Tok.isRegularKeywordAttribute()) &&
+             "not an attribute specifier");
       ConsumeToken();
       BalancedDelimiterTracker T(*this, tok::l_paren);
       if (!T.consumeOpen())
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 897810557976..e862856a08ca 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -1243,8 +1243,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
 
             Declarator DeclaratorInfo(DS, ParsedAttributesView::none(),
                                       DeclaratorContext::TypeName);
-            TypeResult Ty = Actions.ActOnTypeName(getCurScope(),
-                                                  DeclaratorInfo);
+            TypeResult Ty = Actions.ActOnTypeName(DeclaratorInfo);
             if (Ty.isInvalid())
               break;
 
@@ -1538,7 +1537,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
 
       Declarator DeclaratorInfo(DS, ParsedAttributesView::none(),
                                 DeclaratorContext::TypeName);
-      TypeResult Ty = Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
+      TypeResult Ty = Actions.ActOnTypeName(DeclaratorInfo);
       if (Ty.isInvalid())
         break;
 
@@ -1974,10 +1973,11 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       PreferredType.enterSubscript(Actions, Tok.getLocation(), LHS.get());
 
       // We try to parse a list of indexes in all language mode first
-      // and, in we find 0 or one index, we try to parse an OpenMP array
+      // and, in we find 0 or one index, we try to parse an OpenMP/OpenACC array
       // section. This allow us to support C++23 multi dimensional subscript and
-      // OpenMp sections in the same language mode.
-      if (!getLangOpts().OpenMP || Tok.isNot(tok::colon)) {
+      // OpenMP/OpenACC sections in the same language mode.
+      if ((!getLangOpts().OpenMP && !AllowOpenACCArraySections) ||
+          Tok.isNot(tok::colon)) {
         if (!getLangOpts().CPlusPlus23) {
           ExprResult Idx;
           if (getLangOpts().CPlusPlus11 && Tok.is(tok::l_brace)) {
@@ -2001,7 +2001,18 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         }
       }
 
-      if (ArgExprs.size() <= 1 && getLangOpts().OpenMP) {
+      // Handle OpenACC first, since 'AllowOpenACCArraySections' is only enabled
+      // when actively parsing a 'var' in a 'var-list' during clause/'cache'
+      // parsing, so it is the most specific, and best allows us to handle
+      // OpenACC and OpenMP at the same time.
+      if (ArgExprs.size() <= 1 && AllowOpenACCArraySections) {
+        ColonProtectionRAIIObject RAII(*this);
+        if (Tok.is(tok::colon)) {
+          // Consume ':'
+          ColonLocFirst = ConsumeToken();
+          Length = Actions.CorrectDelayedTyposInExpr(ParseExpression());
+        }
+      } else if (ArgExprs.size() <= 1 && getLangOpts().OpenMP) {
         ColonProtectionRAIIObject RAII(*this);
         if (Tok.is(tok::colon)) {
           // Consume ':'
@@ -2031,6 +2042,12 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       if (!LHS.isInvalid() && !HasError && !Length.isInvalid() &&
           !Stride.isInvalid() && Tok.is(tok::r_square)) {
         if (ColonLocFirst.isValid() || ColonLocSecond.isValid()) {
+          // FIXME: OpenACC hasn't implemented Sema/Array section handling at a
+          // semantic level yet. For now, just reuse the OpenMP implementation
+          // as it gets the parsing/type management mostly right, and we can
+          // replace this call to ActOnOpenACCArraySectionExpr in the future.
+          // Eventually we'll genericize the OPenMPArraySectionExpr type as
+          // well.
           LHS = Actions.ActOnOMPArraySectionExpr(
               LHS.get(), Loc, ArgExprs.empty() ? nullptr : ArgExprs[0],
               ColonLocFirst, ColonLocSecond, Length.get(), Stride.get(), RLoc);
@@ -3078,7 +3095,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
       TypeResult Ty;
       {
         InMessageExpressionRAIIObject InMessage(*this, false);
-        Ty = Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
+        Ty = Actions.ActOnTypeName(DeclaratorInfo);
       }
       Result = ParseObjCMessageExpressionBody(SourceLocation(),
                                               SourceLocation(),
@@ -3093,7 +3110,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
         TypeResult Ty;
         {
           InMessageExpressionRAIIObject InMessage(*this, false);
-          Ty = Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
+          Ty = Actions.ActOnTypeName(DeclaratorInfo);
         }
         return ParseCompoundLiteralExpression(Ty.get(), OpenLoc, RParenLoc);
       }
@@ -3105,7 +3122,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
           TypeResult Ty;
           {
             InMessageExpressionRAIIObject InMessage(*this, false);
-            Ty = Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
+            Ty = Actions.ActOnTypeName(DeclaratorInfo);
           }
           if(Ty.isInvalid())
           {
@@ -3152,7 +3169,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
           TypeResult Ty;
           {
             InMessageExpressionRAIIObject InMessage(*this, false);
-            Ty = Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
+            Ty = Actions.ActOnTypeName(DeclaratorInfo);
           }
           CastTy = Ty.get();
           return ExprResult();
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index ef9ea6575205..d61f414406f0 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -1916,7 +1916,7 @@ ExprResult
 Parser::ParseCXXTypeConstructExpression(const DeclSpec &DS) {
   Declarator DeclaratorInfo(DS, ParsedAttributesView::none(),
                             DeclaratorContext::FunctionalCast);
-  ParsedType TypeRep = Actions.ActOnTypeName(getCurScope(), DeclaratorInfo).get();
+  ParsedType TypeRep = Actions.ActOnTypeName(DeclaratorInfo).get();
 
   assert((Tok.is(tok::l_paren) ||
           (getLangOpts().CPlusPlus11 && Tok.is(tok::l_brace)))
@@ -2815,7 +2815,7 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
   ParseDeclaratorInternal(D, /*DirectDeclParser=*/nullptr);
 
   // Finish up the type.
-  TypeResult Ty = Actions.ActOnTypeName(getCurScope(), D);
+  TypeResult Ty = Actions.ActOnTypeName(D);
   if (Ty.isInvalid())
     return true;
 
@@ -3252,8 +3252,7 @@ Parser::ParseCXXNewExpression(bool UseGlobal, SourceLocation Start) {
     ConstructorLParen = T.getOpenLocation();
     if (Tok.isNot(tok::r_paren)) {
       auto RunSignatureHelp = [&]() {
-        ParsedType TypeRep =
-            Actions.ActOnTypeName(getCurScope(), DeclaratorInfo).get();
+        ParsedType TypeRep = Actions.ActOnTypeName(DeclaratorInfo).get();
         QualType PreferredType;
         // ActOnTypeName might adjust DeclaratorInfo and return a null type even
         // the passing DeclaratorInfo is valid, e.g. running SignatureHelp on
@@ -4037,7 +4036,7 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
       if (DeclaratorInfo.isInvalidType())
         return ExprError();
 
-      TypeResult Ty = Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
+      TypeResult Ty = Actions.ActOnTypeName(DeclaratorInfo);
       return ParseCompoundLiteralExpression(Ty.get(),
                                             Tracker.getOpenLocation(),
                                             Tracker.getCloseLocation());
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index c0261c462b88..849fd1ac95a4 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -1332,7 +1332,7 @@ ParsedType Parser::ParseObjCTypeName(ObjCDeclSpec &DS,
                                            DS.getNullabilityLoc(),
                                            addedToDeclSpec);
 
-      TypeResult type = Actions.ActOnTypeName(getCurScope(), declarator);
+      TypeResult type = Actions.ActOnTypeName(declarator);
       if (!type.isInvalid())
         Ty = type.get();
 
@@ -1747,7 +1747,7 @@ void Parser::parseObjCTypeArgsOrProtocolQualifiers(
       // Form a declarator to turn this into a type.
       Declarator D(DS, ParsedAttributesView::none(),
                    DeclaratorContext::TypeName);
-      TypeResult fullTypeArg = Actions.ActOnTypeName(getCurScope(), D);
+      TypeResult fullTypeArg = Actions.ActOnTypeName(D);
       if (fullTypeArg.isUsable()) {
         typeArgs.push_back(fullTypeArg.get());
         if (!foundValidTypeId) {
@@ -3024,7 +3024,7 @@ bool Parser::ParseObjCXXMessageReceiver(bool &IsExpr, void *&TypeOrExpr) {
   // remainder of the class message.
   Declarator DeclaratorInfo(DS, ParsedAttributesView::none(),
                             DeclaratorContext::TypeName);
-  TypeResult Type = Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
+  TypeResult Type = Actions.ActOnTypeName(DeclaratorInfo);
   if (Type.isInvalid())
     return true;
 
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 67325f0a286a..9f7e63ecdc95 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -76,18 +76,56 @@ OpenACCClauseKind getOpenACCClauseKind(Token Tok) {
   if (Tok.is(tok::kw_auto))
     return OpenACCClauseKind::Auto;
 
+  // default is a keyword, so make sure we parse it correctly.
+  if (Tok.is(tok::kw_default))
+    return OpenACCClauseKind::Default;
+
+  // if is also a keyword, make sure we parse it correctly.
+  if (Tok.is(tok::kw_if))
+    return OpenACCClauseKind::If;
+
   if (!Tok.is(tok::identifier))
     return OpenACCClauseKind::Invalid;
 
   return llvm::StringSwitch(
              Tok.getIdentifierInfo()->getName())
+      .Case("attach", OpenACCClauseKind::Attach)
       .Case("auto", OpenACCClauseKind::Auto)
+      .Case("bind", OpenACCClauseKind::Bind)
+      .Case("create", OpenACCClauseKind::Create)
+      .Case("collapse", OpenACCClauseKind::Collapse)
+      .Case("copy", OpenACCClauseKind::Copy)
+      .Case("copyin", OpenACCClauseKind::CopyIn)
+      .Case("copyout", OpenACCClauseKind::CopyOut)
+      .Case("default", OpenACCClauseKind::Default)
+      .Case("default_async", OpenACCClauseKind::DefaultAsync)
+      .Case("delete", OpenACCClauseKind::Delete)
+      .Case("detach", OpenACCClauseKind::Detach)
+      .Case("device", OpenACCClauseKind::Device)
+      .Case("device_num", OpenACCClauseKind::DeviceNum)
+      .Case("device_resident", OpenACCClauseKind::DeviceResident)
+      .Case("device_type", OpenACCClauseKind::DeviceType)
+      .Case("deviceptr", OpenACCClauseKind::DevicePtr)
+      .Case("dtype", OpenACCClauseKind::DType)
       .Case("finalize", OpenACCClauseKind::Finalize)
+      .Case("firstprivate", OpenACCClauseKind::FirstPrivate)
+      .Case("host", OpenACCClauseKind::Host)
+      .Case("if", OpenACCClauseKind::If)
       .Case("if_present", OpenACCClauseKind::IfPresent)
       .Case("independent", OpenACCClauseKind::Independent)
+      .Case("link", OpenACCClauseKind::Link)
+      .Case("no_create", OpenACCClauseKind::NoCreate)
+      .Case("num_gangs", OpenACCClauseKind::NumGangs)
+      .Case("num_workers", OpenACCClauseKind::NumWorkers)
       .Case("nohost", OpenACCClauseKind::NoHost)
+      .Case("present", OpenACCClauseKind::Present)
+      .Case("private", OpenACCClauseKind::Private)
+      .Case("reduction", OpenACCClauseKind::Reduction)
+      .Case("self", OpenACCClauseKind::Self)
       .Case("seq", OpenACCClauseKind::Seq)
+      .Case("use_device", OpenACCClauseKind::UseDevice)
       .Case("vector", OpenACCClauseKind::Vector)
+      .Case("vector_length", OpenACCClauseKind::VectorLength)
       .Case("worker", OpenACCClauseKind::Worker)
       .Default(OpenACCClauseKind::Invalid);
 }
@@ -106,10 +144,25 @@ OpenACCAtomicKind getOpenACCAtomicKind(Token Tok) {
       .Default(OpenACCAtomicKind::Invalid);
 }
 
+OpenACCDefaultClauseKind getOpenACCDefaultClauseKind(Token Tok) {
+  if (!Tok.is(tok::identifier))
+    return OpenACCDefaultClauseKind::Invalid;
+
+  return llvm::StringSwitch(
+             Tok.getIdentifierInfo()->getName())
+      .Case("none", OpenACCDefaultClauseKind::None)
+      .Case("present", OpenACCDefaultClauseKind::Present)
+      .Default(OpenACCDefaultClauseKind::Invalid);
+}
+
 enum class OpenACCSpecialTokenKind {
   ReadOnly,
   DevNum,
   Queues,
+  Zero,
+  Force,
+  Num,
+  Length,
 };
 
 bool isOpenACCSpecialToken(OpenACCSpecialTokenKind Kind, Token Tok) {
@@ -123,10 +176,60 @@ bool isOpenACCSpecialToken(OpenACCSpecialTokenKind Kind, Token Tok) {
     return Tok.getIdentifierInfo()->isStr("devnum");
   case OpenACCSpecialTokenKind::Queues:
     return Tok.getIdentifierInfo()->isStr("queues");
+  case OpenACCSpecialTokenKind::Zero:
+    return Tok.getIdentifierInfo()->isStr("zero");
+  case OpenACCSpecialTokenKind::Force:
+    return Tok.getIdentifierInfo()->isStr("force");
+  case OpenACCSpecialTokenKind::Num:
+    return Tok.getIdentifierInfo()->isStr("num");
+  case OpenACCSpecialTokenKind::Length:
+    return Tok.getIdentifierInfo()->isStr("length");
   }
   llvm_unreachable("Unknown 'Kind' Passed");
 }
 
+/// Used for cases where we have a token we want to check against an
+/// 'identifier-like' token, but don't want to give awkward error messages in
+/// cases where it is accidentially a keyword.
+bool isTokenIdentifierOrKeyword(Parser &P, Token Tok) {
+  if (Tok.is(tok::identifier))
+    return true;
+
+  if (!Tok.isAnnotation() && Tok.getIdentifierInfo() &&
+      Tok.getIdentifierInfo()->isKeyword(P.getLangOpts()))
+    return true;
+
+  return false;
+}
+
+/// Parses and consumes an identifer followed immediately by a single colon, and
+/// diagnoses if it is not the 'special token' kind that we require. Used when
+/// the tag is the only valid value.
+/// Return 'true' if the special token was matched, false if no special token,
+/// or an invalid special token was found.
+template 
+bool tryParseAndConsumeSpecialTokenKind(Parser &P, OpenACCSpecialTokenKind Kind,
+                                        DirOrClauseTy DirOrClause) {
+  Token IdentTok = P.getCurToken();
+  // If this is an identifier-like thing followed by ':', it is one of the
+  // OpenACC 'special' name tags, so consume it.
+  if (isTokenIdentifierOrKeyword(P, IdentTok) && P.NextToken().is(tok::colon)) {
+    P.ConsumeToken();
+    P.ConsumeToken();
+
+    if (!isOpenACCSpecialToken(Kind, IdentTok)) {
+      P.Diag(IdentTok, diag::err_acc_invalid_tag_kind)
+          << IdentTok.getIdentifierInfo() << DirOrClause
+          << std::is_same_v;
+      return false;
+    }
+
+    return true;
+  }
+
+  return false;
+}
+
 bool isOpenACCDirectiveKind(OpenACCDirectiveKind Kind, Token Tok) {
   if (!Tok.is(tok::identifier))
     return false;
@@ -176,6 +279,59 @@ bool isOpenACCDirectiveKind(OpenACCDirectiveKind Kind, Token Tok) {
   llvm_unreachable("Unknown 'Kind' Passed");
 }
 
+OpenACCReductionOperator ParseReductionOperator(Parser &P) {
+  // If there is no colon, treat as if the reduction operator was missing, else
+  // we probably will not recover from it in the case where an expression starts
+  // with one of the operator tokens.
+  if (P.NextToken().isNot(tok::colon)) {
+    P.Diag(P.getCurToken(), diag::err_acc_expected_reduction_operator);
+    return OpenACCReductionOperator::Invalid;
+  }
+  Token ReductionKindTok = P.getCurToken();
+  // Consume both the kind and the colon.
+  P.ConsumeToken();
+  P.ConsumeToken();
+
+  switch (ReductionKindTok.getKind()) {
+  case tok::plus:
+    return OpenACCReductionOperator::Addition;
+  case tok::star:
+    return OpenACCReductionOperator::Multiplication;
+  case tok::amp:
+    return OpenACCReductionOperator::BitwiseAnd;
+  case tok::pipe:
+    return OpenACCReductionOperator::BitwiseOr;
+  case tok::caret:
+    return OpenACCReductionOperator::BitwiseXOr;
+  case tok::ampamp:
+    return OpenACCReductionOperator::And;
+  case tok::pipepipe:
+    return OpenACCReductionOperator::Or;
+  case tok::identifier:
+    if (ReductionKindTok.getIdentifierInfo()->isStr("max"))
+      return OpenACCReductionOperator::Max;
+    if (ReductionKindTok.getIdentifierInfo()->isStr("min"))
+      return OpenACCReductionOperator::Min;
+    LLVM_FALLTHROUGH;
+  default:
+    P.Diag(ReductionKindTok, diag::err_acc_invalid_reduction_operator);
+    return OpenACCReductionOperator::Invalid;
+  }
+  llvm_unreachable("Reduction op token kind not caught by 'default'?");
+}
+
+/// Used for cases where we expect an identifier-like token, but don't want to
+/// give awkward error messages in cases where it is accidentially a keyword.
+bool expectIdentifierOrKeyword(Parser &P) {
+  Token Tok = P.getCurToken();
+
+  if (isTokenIdentifierOrKeyword(P, Tok))
+    return false;
+
+  P.Diag(P.getCurToken(), diag::err_expected) << tok::identifier;
+  return true;
+}
+
 OpenACCDirectiveKind
 ParseOpenACCEnterExitDataDirective(Parser &P, Token FirstTok,
                                    OpenACCDirectiveKindEx ExtDirKind) {
@@ -291,26 +447,81 @@ OpenACCDirectiveKind ParseOpenACCDirectiveKind(Parser &P) {
   return DirKind;
 }
 
-// The OpenACC Clause List is a comma or space-delimited list of clauses (see
-// the comment on ParseOpenACCClauseList).  The concept of a 'clause' doesn't
-// really have its owner grammar and each individual one has its own definition.
-// However, they all are named with a single-identifier (or auto!) token,
-// followed in some cases by either braces or parens.
-bool ParseOpenACCClause(Parser &P) {
-  if (!P.getCurToken().isOneOf(tok::identifier, tok::kw_auto))
-    return P.Diag(P.getCurToken(), diag::err_expected) << tok::identifier;
+enum ClauseParensKind {
+  None,
+  Optional,
+  Required
+};
 
-  OpenACCClauseKind Kind = getOpenACCClauseKind(P.getCurToken());
+ClauseParensKind getClauseParensKind(OpenACCDirectiveKind DirKind,
+                                     OpenACCClauseKind Kind) {
+  switch (Kind) {
+  case OpenACCClauseKind::Self:
+    return DirKind == OpenACCDirectiveKind::Update ? ClauseParensKind::Required
+                                                   : ClauseParensKind::Optional;
+  case OpenACCClauseKind::Worker:
+  case OpenACCClauseKind::Vector:
+    return ClauseParensKind::Optional;
+
+  case OpenACCClauseKind::Default:
+  case OpenACCClauseKind::If:
+  case OpenACCClauseKind::Create:
+  case OpenACCClauseKind::Copy:
+  case OpenACCClauseKind::CopyIn:
+  case OpenACCClauseKind::CopyOut:
+  case OpenACCClauseKind::UseDevice:
+  case OpenACCClauseKind::NoCreate:
+  case OpenACCClauseKind::Present:
+  case OpenACCClauseKind::DevicePtr:
+  case OpenACCClauseKind::Attach:
+  case OpenACCClauseKind::Detach:
+  case OpenACCClauseKind::Private:
+  case OpenACCClauseKind::FirstPrivate:
+  case OpenACCClauseKind::Delete:
+  case OpenACCClauseKind::DeviceResident:
+  case OpenACCClauseKind::Device:
+  case OpenACCClauseKind::Link:
+  case OpenACCClauseKind::Host:
+  case OpenACCClauseKind::Reduction:
+  case OpenACCClauseKind::Collapse:
+  case OpenACCClauseKind::Bind:
+  case OpenACCClauseKind::VectorLength:
+  case OpenACCClauseKind::NumGangs:
+  case OpenACCClauseKind::NumWorkers:
+  case OpenACCClauseKind::DeviceNum:
+  case OpenACCClauseKind::DefaultAsync:
+  case OpenACCClauseKind::DeviceType:
+  case OpenACCClauseKind::DType:
+    return ClauseParensKind::Required;
+
+  case OpenACCClauseKind::Auto:
+  case OpenACCClauseKind::Finalize:
+  case OpenACCClauseKind::IfPresent:
+  case OpenACCClauseKind::Independent:
+  case OpenACCClauseKind::Invalid:
+  case OpenACCClauseKind::NoHost:
+  case OpenACCClauseKind::Seq:
+    return ClauseParensKind::None;
+  }
+  llvm_unreachable("Unhandled clause kind");
+}
 
-  if (Kind == OpenACCClauseKind::Invalid)
-    return P.Diag(P.getCurToken(), diag::err_acc_invalid_clause)
-           << P.getCurToken().getIdentifierInfo();
+bool ClauseHasOptionalParens(OpenACCDirectiveKind DirKind,
+                             OpenACCClauseKind Kind) {
+  return getClauseParensKind(DirKind, Kind) == ClauseParensKind::Optional;
+}
 
-  // Consume the clause name.
-  P.ConsumeToken();
+bool ClauseHasRequiredParens(OpenACCDirectiveKind DirKind,
+                             OpenACCClauseKind Kind) {
+  return getClauseParensKind(DirKind, Kind) == ClauseParensKind::Required;
+}
 
-  // FIXME: For future clauses, we need to handle parens/etc below.
-  return false;
+ExprResult ParseOpenACCConditionalExpr(Parser &P) {
+  // FIXME: It isn't clear if the spec saying 'condition' means the same as
+  // it does in an if/while/etc (See ParseCXXCondition), however as it was
+  // written with Fortran/C in mind, we're going to assume it just means an
+  // 'expression evaluating to boolean'.
+  return P.getActions().CorrectDelayedTyposInExpr(P.ParseExpression());
 }
 
 // Skip until we see the end of pragma token, but don't consume it. This is us
@@ -322,29 +533,265 @@ void SkipUntilEndOfDirective(Parser &P) {
     P.ConsumeAnyToken();
 }
 
+} // namespace
+
 // OpenACC 3.3, section 1.7:
 // To simplify the specification and convey appropriate constraint information,
 // a pqr-list is a comma-separated list of pdr items. The one exception is a
 // clause-list, which is a list of one or more clauses optionally separated by
 // commas.
-void ParseOpenACCClauseList(Parser &P) {
+void Parser::ParseOpenACCClauseList(OpenACCDirectiveKind DirKind) {
   bool FirstClause = true;
-  while (P.getCurToken().isNot(tok::annot_pragma_openacc_end)) {
+  while (getCurToken().isNot(tok::annot_pragma_openacc_end)) {
     // Comma is optional in a clause-list.
-    if (!FirstClause && P.getCurToken().is(tok::comma))
-      P.ConsumeToken();
+    if (!FirstClause && getCurToken().is(tok::comma))
+      ConsumeToken();
     FirstClause = false;
 
     // Recovering from a bad clause is really difficult, so we just give up on
     // error.
-    if (ParseOpenACCClause(P)) {
-      SkipUntilEndOfDirective(P);
+    if (ParseOpenACCClause(DirKind)) {
+      SkipUntilEndOfDirective(*this);
       return;
     }
   }
 }
 
-} // namespace
+ExprResult Parser::ParseOpenACCIntExpr() {
+  // FIXME: this is required to be an integer expression (or dependent), so we
+  // should ensure that is the case by passing this to SEMA here.
+  return getActions().CorrectDelayedTyposInExpr(ParseAssignmentExpression());
+}
+
+bool Parser::ParseOpenACCClauseVarList(OpenACCClauseKind Kind) {
+  // FIXME: Future clauses will require 'special word' parsing, check for one,
+  // then parse it based on whether it is a clause that requires a 'special
+  // word'.
+  (void)Kind;
+
+  // If the var parsing fails, skip until the end of the directive as this is
+  // an expression and gets messy if we try to continue otherwise.
+  if (ParseOpenACCVar())
+    return true;
+
+  while (!getCurToken().isOneOf(tok::r_paren, tok::annot_pragma_openacc_end)) {
+    ExpectAndConsume(tok::comma);
+
+    // If the var parsing fails, skip until the end of the directive as this is
+    // an expression and gets messy if we try to continue otherwise.
+    if (ParseOpenACCVar())
+      return true;
+  }
+  return false;
+}
+
+/// OpenACC 3.3 Section 2.4:
+/// The argument to the device_type clause is a comma-separated list of one or
+/// more device architecture name identifiers, or an asterisk.
+///
+/// The syntax of the device_type clause is
+/// device_type( * )
+/// device_type( device-type-list )
+///
+/// The device_type clause may be abbreviated to dtype.
+bool Parser::ParseOpenACCDeviceTypeList() {
+
+  if (expectIdentifierOrKeyword(*this)) {
+    SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
+              Parser::StopBeforeMatch);
+    return false;
+  }
+  ConsumeToken();
+
+  while (!getCurToken().isOneOf(tok::r_paren, tok::annot_pragma_openacc_end)) {
+    ExpectAndConsume(tok::comma);
+
+    if (expectIdentifierOrKeyword(*this)) {
+      SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end,
+                Parser::StopBeforeMatch);
+      return false;
+    }
+    ConsumeToken();
+  }
+  return false;
+}
+
+// The OpenACC Clause List is a comma or space-delimited list of clauses (see
+// the comment on ParseOpenACCClauseList).  The concept of a 'clause' doesn't
+// really have its owner grammar and each individual one has its own definition.
+// However, they all are named with a single-identifier (or auto/default!)
+// token, followed in some cases by either braces or parens.
+bool Parser::ParseOpenACCClause(OpenACCDirectiveKind DirKind) {
+  // A number of clause names are actually keywords, so accept a keyword that
+  // can be converted to a name.
+  if (expectIdentifierOrKeyword(*this))
+    return true;
+
+  OpenACCClauseKind Kind = getOpenACCClauseKind(getCurToken());
+
+  if (Kind == OpenACCClauseKind::Invalid)
+    return Diag(getCurToken(), diag::err_acc_invalid_clause)
+           << getCurToken().getIdentifierInfo();
+
+  // Consume the clause name.
+  ConsumeToken();
+
+  return ParseOpenACCClauseParams(DirKind, Kind);
+}
+
+bool Parser::ParseOpenACCClauseParams(OpenACCDirectiveKind DirKind,
+                                      OpenACCClauseKind Kind) {
+  BalancedDelimiterTracker Parens(*this, tok::l_paren,
+                                  tok::annot_pragma_openacc_end);
+
+  if (ClauseHasRequiredParens(DirKind, Kind)) {
+    if (Parens.expectAndConsume()) {
+      // We are missing a paren, so assume that the person just forgot the
+      // parameter.  Return 'false' so we try to continue on and parse the next
+      // clause.
+      SkipUntil(tok::comma, tok::r_paren, tok::annot_pragma_openacc_end,
+                Parser::StopBeforeMatch);
+      return false;
+    }
+
+    switch (Kind) {
+    case OpenACCClauseKind::Default: {
+      Token DefKindTok = getCurToken();
+
+      if (expectIdentifierOrKeyword(*this))
+        break;
+
+      ConsumeToken();
+
+      if (getOpenACCDefaultClauseKind(DefKindTok) ==
+          OpenACCDefaultClauseKind::Invalid)
+        Diag(DefKindTok, diag::err_acc_invalid_default_clause_kind);
+
+      break;
+    }
+    case OpenACCClauseKind::If: {
+      ExprResult CondExpr = ParseOpenACCConditionalExpr(*this);
+      // An invalid expression can be just about anything, so just give up on
+      // this clause list.
+      if (CondExpr.isInvalid())
+        return true;
+      break;
+    }
+    case OpenACCClauseKind::CopyIn:
+      tryParseAndConsumeSpecialTokenKind(
+          *this, OpenACCSpecialTokenKind::ReadOnly, Kind);
+      if (ParseOpenACCClauseVarList(Kind))
+        return true;
+      break;
+    case OpenACCClauseKind::Create:
+    case OpenACCClauseKind::CopyOut:
+      tryParseAndConsumeSpecialTokenKind(*this, OpenACCSpecialTokenKind::Zero,
+                                         Kind);
+      if (ParseOpenACCClauseVarList(Kind))
+        return true;
+      break;
+    case OpenACCClauseKind::Reduction:
+      // If we're missing a clause-kind (or it is invalid), see if we can parse
+      // the var-list anyway.
+      ParseReductionOperator(*this);
+      if (ParseOpenACCClauseVarList(Kind))
+        return true;
+      break;
+    case OpenACCClauseKind::Self:
+      // The 'self' clause is a var-list instead of a 'condition' in the case of
+      // the 'update' clause, so we have to handle it here.  U se an assert to
+      // make sure we get the right differentiator.
+      assert(DirKind == OpenACCDirectiveKind::Update);
+      LLVM_FALLTHROUGH;
+    case OpenACCClauseKind::Attach:
+    case OpenACCClauseKind::Copy:
+    case OpenACCClauseKind::Delete:
+    case OpenACCClauseKind::Detach:
+    case OpenACCClauseKind::Device:
+    case OpenACCClauseKind::DeviceResident:
+    case OpenACCClauseKind::DevicePtr:
+    case OpenACCClauseKind::FirstPrivate:
+    case OpenACCClauseKind::Host:
+    case OpenACCClauseKind::Link:
+    case OpenACCClauseKind::NoCreate:
+    case OpenACCClauseKind::Present:
+    case OpenACCClauseKind::Private:
+    case OpenACCClauseKind::UseDevice:
+      if (ParseOpenACCClauseVarList(Kind))
+        return true;
+      break;
+    case OpenACCClauseKind::Collapse: {
+      tryParseAndConsumeSpecialTokenKind(*this, OpenACCSpecialTokenKind::Force,
+                                         Kind);
+      ExprResult NumLoops =
+          getActions().CorrectDelayedTyposInExpr(ParseConstantExpression());
+      if (NumLoops.isInvalid())
+        return true;
+      break;
+    }
+    case OpenACCClauseKind::Bind: {
+      ExprResult BindArg = ParseOpenACCBindClauseArgument();
+      if (BindArg.isInvalid())
+        return true;
+      break;
+    }
+    case OpenACCClauseKind::NumGangs:
+    case OpenACCClauseKind::NumWorkers:
+    case OpenACCClauseKind::DeviceNum:
+    case OpenACCClauseKind::DefaultAsync:
+    case OpenACCClauseKind::VectorLength: {
+      ExprResult IntExpr = ParseOpenACCIntExpr();
+      if (IntExpr.isInvalid())
+        return true;
+      break;
+    }
+    case OpenACCClauseKind::DType:
+    case OpenACCClauseKind::DeviceType:
+      if (getCurToken().is(tok::star)) {
+        // FIXME: We want to mark that this is an 'everything else' type of
+        // device_type in Sema.
+        ConsumeToken();
+      } else if (ParseOpenACCDeviceTypeList()) {
+        return true;
+      }
+      break;
+    default:
+      llvm_unreachable("Not a required parens type?");
+    }
+
+    return Parens.consumeClose();
+  } else if (ClauseHasOptionalParens(DirKind, Kind)) {
+    if (!Parens.consumeOpen()) {
+      switch (Kind) {
+      case OpenACCClauseKind::Self: {
+        assert(DirKind != OpenACCDirectiveKind::Update);
+        ExprResult CondExpr = ParseOpenACCConditionalExpr(*this);
+        // An invalid expression can be just about anything, so just give up on
+        // this clause list.
+        if (CondExpr.isInvalid())
+          return true;
+        break;
+      }
+      case OpenACCClauseKind::Vector:
+      case OpenACCClauseKind::Worker: {
+        tryParseAndConsumeSpecialTokenKind(*this,
+                                           Kind == OpenACCClauseKind::Vector
+                                               ? OpenACCSpecialTokenKind::Length
+                                               : OpenACCSpecialTokenKind::Num,
+                                           Kind);
+        ExprResult IntExpr = ParseOpenACCIntExpr();
+        if (IntExpr.isInvalid())
+          return true;
+        break;
+      }
+      default:
+        llvm_unreachable("Not an optional parens type?");
+      }
+      Parens.consumeClose();
+    }
+  }
+  return false;
+}
 
 /// OpenACC 3.3, section 2.16:
 /// In this section and throughout the specification, the term wait-argument
@@ -410,7 +857,7 @@ bool Parser::ParseOpenACCWaitArgument() {
 ExprResult Parser::ParseOpenACCIDExpression() {
   ExprResult Res;
   if (getLangOpts().CPlusPlus) {
-    Res = ParseCXXIdExpression(/*isAddressOfOperand=*/false);
+    Res = ParseCXXIdExpression(/*isAddressOfOperand=*/true);
   } else {
     // There isn't anything quite the same as ParseCXXIdExpression for C, so we
     // need to get the identifier, then call into Sema ourselves.
@@ -437,49 +884,37 @@ ExprResult Parser::ParseOpenACCIDExpression() {
   return getActions().CorrectDelayedTyposInExpr(Res);
 }
 
-/// OpenACC 3.3, section 2.10:
-/// A 'var' in a cache directive must be a single array element or a simple
-/// subarray.  In C and C++, a simple subarray is an array name followed by an
-/// extended array range specification in brackets, with a start and length such
-/// as:
-///
-/// arr[lower:length]
-///
-bool Parser::ParseOpenACCCacheVar() {
-  ExprResult ArrayName = ParseOpenACCIDExpression();
-  if (ArrayName.isInvalid())
-    return true;
-
-  // If the expression is invalid, just continue parsing the brackets, there
-  // is likely other useful diagnostics we can emit inside of those.
-
-  BalancedDelimiterTracker SquareBrackets(*this, tok::l_square,
-                                          tok::annot_pragma_openacc_end);
-
-  // Square brackets are required, so error here, and try to recover by moving
-  // until the next comma, or the close paren/end of pragma.
-  if (SquareBrackets.expectAndConsume()) {
-    SkipUntil(tok::comma, tok::r_paren, tok::annot_pragma_openacc_end,
-              Parser::StopBeforeMatch);
-    return true;
+ExprResult Parser::ParseOpenACCBindClauseArgument() {
+  // OpenACC 3.3 section 2.15:
+  // The bind clause specifies the name to use when calling the procedure on a
+  // device other than the host. If the name is specified as an identifier, it
+  // is called as if that name were specified in the language being compiled. If
+  // the name is specified as a string, the string is used for the procedure
+  // name unmodified.
+  if (getCurToken().is(tok::r_paren)) {
+    Diag(getCurToken(), diag::err_acc_incorrect_bind_arg);
+    return ExprError();
   }
 
-  ExprResult Lower = getActions().CorrectDelayedTyposInExpr(ParseExpression());
-  if (Lower.isInvalid())
-    return true;
+  if (tok::isStringLiteral(getCurToken().getKind()))
+    return getActions().CorrectDelayedTyposInExpr(ParseStringLiteralExpression(
+        /*AllowUserDefinedLiteral=*/false, /*Unevaluated=*/true));
 
-  // The 'length' expression is optional, as this could be a single array
-  // element. If there is no colon, we can treat it as that.
-  if (getCurToken().is(tok::colon)) {
-    ConsumeToken();
-    ExprResult Length =
-        getActions().CorrectDelayedTyposInExpr(ParseExpression());
-    if (Length.isInvalid())
-      return true;
-  }
+  return ParseOpenACCIDExpression();
+}
 
-  // Diagnose the square bracket being in the wrong place and continue.
-  return SquareBrackets.consumeClose();
+/// OpenACC 3.3, section 1.6:
+/// In this spec, a 'var' (in italics) is one of the following:
+/// - a variable name (a scalar, array, or compisite variable name)
+/// - a subarray specification with subscript ranges
+/// - an array element
+/// - a member of a composite variable
+/// - a common block name between slashes (fortran only)
+bool Parser::ParseOpenACCVar() {
+  OpenACCArraySectionRAII ArraySections(*this);
+  ExprResult Res =
+      getActions().CorrectDelayedTyposInExpr(ParseAssignmentExpression());
+  return Res.isInvalid();
 }
 
 /// OpenACC 3.3, section 2.10:
@@ -493,14 +928,11 @@ void Parser::ParseOpenACCCacheVarList() {
     return;
 
   // The VarList is an optional `readonly:` followed by a list of a variable
-  // specifications.  First, see if we have `readonly:`, else we back-out and
-  // treat it like the beginning of a reference to a potentially-existing
-  // `readonly` variable.
-  if (isOpenACCSpecialToken(OpenACCSpecialTokenKind::ReadOnly, Tok) &&
-      NextToken().is(tok::colon)) {
-    // Consume both tokens.
-    ConsumeToken();
-    ConsumeToken();
+  // specifications. Consume something that looks like a 'tag', and diagnose if
+  // it isn't 'readonly'.
+  if (tryParseAndConsumeSpecialTokenKind(*this,
+                                         OpenACCSpecialTokenKind::ReadOnly,
+                                         OpenACCDirectiveKind::Cache)) {
     // FIXME: Record that this is a 'readonly' so that we can use that during
     // Sema/AST generation.
   }
@@ -510,7 +942,16 @@ void Parser::ParseOpenACCCacheVarList() {
     if (!FirstArray)
       ExpectAndConsume(tok::comma);
     FirstArray = false;
-    if (ParseOpenACCCacheVar())
+
+    // OpenACC 3.3, section 2.10:
+    // A 'var' in a cache directive must be a single array element or a simple
+    // subarray.  In C and C++, a simple subarray is an array name followed by
+    // an extended array range specification in brackets, with a start and
+    // length such as:
+    //
+    // arr[lower:length]
+    //
+    if (ParseOpenACCVar())
       SkipUntil(tok::r_paren, tok::annot_pragma_openacc_end, tok::comma,
                 StopBeforeMatch);
   }
@@ -570,7 +1011,7 @@ void Parser::ParseOpenACCDirective() {
   }
 
   // Parses the list of clauses, if present.
-  ParseOpenACCClauseList(*this);
+  ParseOpenACCClauseList(DirKind);
 
   Diag(getCurToken(), diag::warn_pragma_acc_unimplemented);
   assert(Tok.is(tok::annot_pragma_openacc_end) &&
diff --git a/clang/lib/Parse/ParseTentative.cpp b/clang/lib/Parse/ParseTentative.cpp
index 242741c15b5f..5bfabf55f50c 100644
--- a/clang/lib/Parse/ParseTentative.cpp
+++ b/clang/lib/Parse/ParseTentative.cpp
@@ -894,7 +894,8 @@ bool Parser::TrySkipAttributes() {
       // Note that explicitly checking for `[[` and `]]` allows to fail as
       // expected in the case of the Objective-C message send syntax.
       ConsumeBracket();
-    } else if (Tok.isRegularKeywordAttribute()) {
+    } else if (Tok.isRegularKeywordAttribute() &&
+               !doesKeywordAttributeTakeArgs(Tok.getKind())) {
       ConsumeToken();
     } else {
       ConsumeToken();
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index b703c2d9b8e0..0b092181bca7 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -70,6 +70,11 @@ Parser::Parser(Preprocessor &pp, Sema &actions, bool skipFunctionBodies)
   PP.addCommentHandler(CommentSemaHandler.get());
 
   PP.setCodeCompletionHandler(*this);
+
+  Actions.ParseTypeFromStringCallback =
+      [this](StringRef TypeStr, StringRef Context, SourceLocation IncludeLoc) {
+        return this->ParseTypeFromString(TypeStr, Context, IncludeLoc);
+      };
 }
 
 DiagnosticBuilder Parser::Diag(SourceLocation Loc, unsigned DiagID) {
diff --git a/clang/lib/Sema/AnalysisBasedWarnings.cpp b/clang/lib/Sema/AnalysisBasedWarnings.cpp
index 0947e8b0f526..9e9294572df9 100644
--- a/clang/lib/Sema/AnalysisBasedWarnings.cpp
+++ b/clang/lib/Sema/AnalysisBasedWarnings.cpp
@@ -2226,8 +2226,8 @@ class UnsafeBufferUsageReporter : public UnsafeBufferUsageHandler {
   UnsafeBufferUsageReporter(Sema &S, bool SuggestSuggestions)
     : S(S), SuggestSuggestions(SuggestSuggestions) {}
 
-  void handleUnsafeOperation(const Stmt *Operation,
-                             bool IsRelatedToDecl) override {
+  void handleUnsafeOperation(const Stmt *Operation, bool IsRelatedToDecl,
+                             ASTContext &Ctx) override {
     SourceLocation Loc;
     SourceRange Range;
     unsigned MsgParam = 0;
@@ -2261,6 +2261,21 @@ class UnsafeBufferUsageReporter : public UnsafeBufferUsageHandler {
         // note_unsafe_buffer_operation doesn't have this mode yet.
         assert(!IsRelatedToDecl && "Not implemented yet!");
         MsgParam = 3;
+      } else if (const auto *ECE = dyn_cast(Operation)) {
+        QualType destType = ECE->getType();
+        if (!isa(destType))
+          return;
+
+        const uint64_t dSize =
+            Ctx.getTypeSize(destType.getTypePtr()->getPointeeType());
+
+        QualType srcType = ECE->getSubExpr()->getType();
+        const uint64_t sSize =
+            Ctx.getTypeSize(srcType.getTypePtr()->getPointeeType());
+        if (sSize >= dSize)
+          return;
+
+        MsgParam = 4;
       }
       Loc = Operation->getBeginLoc();
       Range = Operation->getSourceRange();
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index ed3bb685f548..2a233880c1a6 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -966,7 +966,7 @@ static bool MethodsAndNestedClassesComplete(const CXXRecordDecl *RD,
        I != E && Complete; ++I) {
     if (const CXXMethodDecl *M = dyn_cast(*I))
       Complete = M->isDefined() || M->isDefaulted() ||
-                 (M->isPure() && !isa(M));
+                 (M->isPureVirtual() && !isa(M));
     else if (const FunctionTemplateDecl *F = dyn_cast(*I))
       // If the template function is marked as late template parsed at this
       // point, it has not been instantiated and therefore we have not
@@ -2767,7 +2767,7 @@ bool Sema::isDeclaratorFunctionLike(Declarator &D) {
     return false;
 
   LookupQualifiedName(LR, DC);
-  bool Result = std::all_of(LR.begin(), LR.end(), [](Decl *Dcl) {
+  bool Result = llvm::all_of(LR, [](Decl *Dcl) {
     if (NamedDecl *ND = dyn_cast(Dcl)) {
       ND = ND->getUnderlyingDecl();
       return isa(ND) || isa(ND) ||
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index da0570b7b0f1..7833d5a2ea20 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -2169,6 +2169,7 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
     ICEArguments &= ~(1 << ArgNo);
   }
 
+  FPOptions FPO;
   switch (BuiltinID) {
   case Builtin::BI__builtin___CFStringMakeConstantString:
     // CFStringMakeConstantString is currently not implemented for GOFF (i.e.,
@@ -2245,15 +2246,15 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
   case Builtin::BI__builtin_islessequal:
   case Builtin::BI__builtin_islessgreater:
   case Builtin::BI__builtin_isunordered:
-    if (SemaBuiltinUnorderedCompare(TheCall))
+    if (SemaBuiltinUnorderedCompare(TheCall, BuiltinID))
       return ExprError();
     break;
   case Builtin::BI__builtin_fpclassify:
-    if (SemaBuiltinFPClassification(TheCall, 6))
+    if (SemaBuiltinFPClassification(TheCall, 6, BuiltinID))
       return ExprError();
     break;
   case Builtin::BI__builtin_isfpclass:
-    if (SemaBuiltinFPClassification(TheCall, 2))
+    if (SemaBuiltinFPClassification(TheCall, 2, BuiltinID))
       return ExprError();
     break;
   case Builtin::BI__builtin_isfinite:
@@ -2267,7 +2268,7 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
   case Builtin::BI__builtin_signbit:
   case Builtin::BI__builtin_signbitf:
   case Builtin::BI__builtin_signbitl:
-    if (SemaBuiltinFPClassification(TheCall, 1))
+    if (SemaBuiltinFPClassification(TheCall, 1, BuiltinID))
       return ExprError();
     break;
   case Builtin::BI__builtin_shufflevector:
@@ -2998,7 +2999,26 @@ static QualType getNeonEltType(NeonTypeFlags Flags, ASTContext &Context,
   llvm_unreachable("Invalid NeonTypeFlag!");
 }
 
-enum ArmStreamingType { ArmNonStreaming, ArmStreaming, ArmStreamingCompatible };
+enum ArmStreamingType {
+  ArmNonStreaming,
+  ArmStreaming,
+  ArmStreamingCompatible,
+  ArmStreamingOrSVE2p1
+};
+
+enum ArmSMEState : unsigned {
+  ArmNoState = 0,
+
+  ArmInZA = 0b01,
+  ArmOutZA = 0b10,
+  ArmInOutZA = 0b11,
+  ArmZAMask = 0b11,
+
+  ArmInZT0 = 0b01 << 2,
+  ArmOutZT0 = 0b10 << 2,
+  ArmInOutZT0 = 0b11 << 2,
+  ArmZT0Mask = 0b11 << 2
+};
 
 bool Sema::ParseSVEImmChecks(
     CallExpr *TheCall, SmallVector, 3> &ImmChecks) {
@@ -3156,6 +3176,16 @@ static void checkArmStreamingBuiltin(Sema &S, CallExpr *TheCall,
                                      const FunctionDecl *FD,
                                      ArmStreamingType BuiltinType) {
   ArmStreamingType FnType = getArmStreamingFnType(FD);
+  if (BuiltinType == ArmStreamingOrSVE2p1) {
+    // Check intrinsics that are available in [sve2p1 or sme/sme2].
+    llvm::StringMap CallerFeatureMap;
+    S.Context.getFunctionFeatureMap(CallerFeatureMap, FD);
+    if (Builtin::evaluateRequiredTargetFeatures("sve2p1", CallerFeatureMap))
+      BuiltinType = ArmStreamingCompatible;
+    else
+      BuiltinType = ArmStreaming;
+  }
+
   if (FnType == ArmStreaming && BuiltinType == ArmNonStreaming) {
     S.Diag(TheCall->getBeginLoc(), diag::warn_attribute_arm_sm_incompat_builtin)
         << TheCall->getSourceRange() << "streaming";
@@ -3174,22 +3204,27 @@ static void checkArmStreamingBuiltin(Sema &S, CallExpr *TheCall,
   }
 }
 
-static bool hasSMEZAState(const FunctionDecl *FD) {
-  if (FD->hasAttr())
-    return true;
-  if (const auto *T = FD->getType()->getAs())
-    if (T->getAArch64SMEAttributes() & FunctionType::SME_PStateZASharedMask)
-      return true;
-  return false;
+static bool hasArmZAState(const FunctionDecl *FD) {
+  const auto *T = FD->getType()->getAs();
+  return (T && FunctionType::getArmZAState(T->getAArch64SMEAttributes()) !=
+                   FunctionType::ARM_None) ||
+         (FD->hasAttr() && FD->getAttr()->isNewZA());
 }
 
-static bool hasSMEZAState(unsigned BuiltinID) {
+static bool hasArmZT0State(const FunctionDecl *FD) {
+  const auto *T = FD->getType()->getAs();
+  return (T && FunctionType::getArmZT0State(T->getAArch64SMEAttributes()) !=
+                   FunctionType::ARM_None) ||
+         (FD->hasAttr() && FD->getAttr()->isNewZT0());
+}
+
+static ArmSMEState getSMEState(unsigned BuiltinID) {
   switch (BuiltinID) {
   default:
-    return false;
-#define GET_SME_BUILTIN_HAS_ZA_STATE
+    return ArmNoState;
+#define GET_SME_BUILTIN_GET_STATE
 #include "clang/Basic/arm_sme_builtins_za_state.inc"
-#undef GET_SME_BUILTIN_HAS_ZA_STATE
+#undef GET_SME_BUILTIN_GET_STATE
   }
 }
 
@@ -3206,10 +3241,15 @@ bool Sema::CheckSMEBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
     if (BuiltinType)
       checkArmStreamingBuiltin(*this, TheCall, FD, *BuiltinType);
 
-    if (hasSMEZAState(BuiltinID) && !hasSMEZAState(FD))
+    if ((getSMEState(BuiltinID) & ArmZAMask) && !hasArmZAState(FD))
       Diag(TheCall->getBeginLoc(),
            diag::warn_attribute_arm_za_builtin_no_za_state)
           << TheCall->getSourceRange();
+
+    if ((getSMEState(BuiltinID) & ArmZT0Mask) && !hasArmZT0State(FD))
+      Diag(TheCall->getBeginLoc(),
+           diag::warn_attribute_arm_zt0_builtin_no_zt0_state)
+          << TheCall->getSourceRange();
   }
 
   // Range check SME intrinsics that take immediate values.
@@ -7507,19 +7547,46 @@ void Sema::checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto,
 
     // If the callee uses AArch64 SME ZA state but the caller doesn't define
     // any, then this is an error.
-    if (ExtInfo.AArch64SMEAttributes & FunctionType::SME_PStateZASharedMask) {
+    FunctionType::ArmStateValue ArmZAState =
+        FunctionType::getArmZAState(ExtInfo.AArch64SMEAttributes);
+    if (ArmZAState != FunctionType::ARM_None) {
       bool CallerHasZAState = false;
       if (const auto *CallerFD = dyn_cast(CurContext)) {
-        if (CallerFD->hasAttr())
+        auto *Attr = CallerFD->getAttr();
+        if (Attr && Attr->isNewZA())
           CallerHasZAState = true;
-        else if (const auto *FPT = CallerFD->getType()->getAs())
-          CallerHasZAState = FPT->getExtProtoInfo().AArch64SMEAttributes &
-                             FunctionType::SME_PStateZASharedMask;
+        else if (const auto *FPT =
+                     CallerFD->getType()->getAs())
+          CallerHasZAState = FunctionType::getArmZAState(
+                                 FPT->getExtProtoInfo().AArch64SMEAttributes) !=
+                             FunctionType::ARM_None;
       }
 
       if (!CallerHasZAState)
         Diag(Loc, diag::err_sme_za_call_no_za_state);
     }
+
+    // If the callee uses AArch64 SME ZT0 state but the caller doesn't define
+    // any, then this is an error.
+    FunctionType::ArmStateValue ArmZT0State =
+        FunctionType::getArmZT0State(ExtInfo.AArch64SMEAttributes);
+    if (ArmZT0State != FunctionType::ARM_None) {
+      bool CallerHasZT0State = false;
+      if (const auto *CallerFD = dyn_cast(CurContext)) {
+        auto *Attr = CallerFD->getAttr();
+        if (Attr && Attr->isNewZT0())
+          CallerHasZT0State = true;
+        else if (const auto *FPT =
+                     CallerFD->getType()->getAs())
+          CallerHasZT0State =
+              FunctionType::getArmZT0State(
+                  FPT->getExtProtoInfo().AArch64SMEAttributes) !=
+              FunctionType::ARM_None;
+      }
+
+      if (!CallerHasZT0State)
+        Diag(Loc, diag::err_sme_zt0_call_no_zt0_state);
+    }
   }
 
   if (FDecl && FDecl->hasAttr()) {
@@ -7621,6 +7688,7 @@ bool Sema::CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall,
 
   CheckAbsoluteValueFunction(TheCall, FDecl);
   CheckMaxUnsignedZero(TheCall, FDecl);
+  CheckInfNaNFunction(TheCall, FDecl);
 
   if (getLangOpts().ObjC)
     DiagnoseCStringFormatDirectiveInCFAPI(*this, FDecl, Args, NumArgs);
@@ -9090,10 +9158,15 @@ bool Sema::SemaBuiltinVAStartARMMicrosoft(CallExpr *Call) {
 
 /// SemaBuiltinUnorderedCompare - Handle functions like __builtin_isgreater and
 /// friends.  This is declared to take (...), so we have to check everything.
-bool Sema::SemaBuiltinUnorderedCompare(CallExpr *TheCall) {
+bool Sema::SemaBuiltinUnorderedCompare(CallExpr *TheCall, unsigned BuiltinID) {
   if (checkArgCount(*this, TheCall, 2))
     return true;
 
+  if (BuiltinID == Builtin::BI__builtin_isunordered &&
+      TheCall->getFPFeaturesInEffect(getLangOpts()).getNoHonorNaNs())
+    Diag(TheCall->getBeginLoc(), diag::warn_fp_nan_inf_when_disabled)
+        << 1 << 0 << TheCall->getSourceRange();
+
   ExprResult OrigArg0 = TheCall->getArg(0);
   ExprResult OrigArg1 = TheCall->getArg(1);
 
@@ -9128,10 +9201,23 @@ bool Sema::SemaBuiltinUnorderedCompare(CallExpr *TheCall) {
 /// SemaBuiltinSemaBuiltinFPClassification - Handle functions like
 /// __builtin_isnan and friends.  This is declared to take (...), so we have
 /// to check everything.
-bool Sema::SemaBuiltinFPClassification(CallExpr *TheCall, unsigned NumArgs) {
+bool Sema::SemaBuiltinFPClassification(CallExpr *TheCall, unsigned NumArgs,
+                                       unsigned BuiltinID) {
   if (checkArgCount(*this, TheCall, NumArgs))
     return true;
 
+  FPOptions FPO = TheCall->getFPFeaturesInEffect(getLangOpts());
+  if (FPO.getNoHonorInfs() && (BuiltinID == Builtin::BI__builtin_isfinite ||
+                               BuiltinID == Builtin::BI__builtin_isinf ||
+                               BuiltinID == Builtin::BI__builtin_isinf_sign))
+    Diag(TheCall->getBeginLoc(), diag::warn_fp_nan_inf_when_disabled)
+        << 0 << 0 << TheCall->getSourceRange();
+
+  if (FPO.getNoHonorNaNs() && (BuiltinID == Builtin::BI__builtin_isnan ||
+                               BuiltinID == Builtin::BI__builtin_isunordered))
+    Diag(TheCall->getBeginLoc(), diag::warn_fp_nan_inf_when_disabled)
+        << 1 << 0 << TheCall->getSourceRange();
+
   bool IsFPClass = NumArgs == 2;
 
   // Find out position of floating-point argument.
@@ -12878,6 +12964,22 @@ static bool IsStdFunction(const FunctionDecl *FDecl,
   return true;
 }
 
+void Sema::CheckInfNaNFunction(const CallExpr *Call,
+                               const FunctionDecl *FDecl) {
+  FPOptions FPO = Call->getFPFeaturesInEffect(getLangOpts());
+  if ((IsStdFunction(FDecl, "isnan") || IsStdFunction(FDecl, "isunordered") ||
+       (Call->getBuiltinCallee() == Builtin::BI__builtin_nanf)) &&
+      FPO.getNoHonorNaNs())
+    Diag(Call->getBeginLoc(), diag::warn_fp_nan_inf_when_disabled)
+        << 1 << 0 << Call->getSourceRange();
+  else if ((IsStdFunction(FDecl, "isinf") ||
+            (IsStdFunction(FDecl, "isfinite") ||
+             (FDecl->getIdentifier() && FDecl->getName() == "infinity"))) &&
+           FPO.getNoHonorInfs())
+    Diag(Call->getBeginLoc(), diag::warn_fp_nan_inf_when_disabled)
+        << 0 << 0 << Call->getSourceRange();
+}
+
 // Warn when using the wrong abs() function.
 void Sema::CheckAbsoluteValueFunction(const CallExpr *Call,
                                       const FunctionDecl *FDecl) {
@@ -16677,7 +16779,7 @@ class SequenceChecker : public ConstEvaluatedExprVisitor {
     /// Have we issued a diagnostic for this object already?
     bool Diagnosed = false;
 
-    UsageInfo() = default;
+    UsageInfo();
   };
   using UsageInfoMap = llvm::SmallDenseMap;
 
@@ -17436,6 +17538,8 @@ class SequenceChecker : public ConstEvaluatedExprVisitor {
   }
 };
 
+SequenceChecker::UsageInfo::UsageInfo() = default;
+
 } // namespace
 
 void Sema::CheckUnsequencedOperations(const Expr *E) {
@@ -18359,7 +18463,7 @@ static bool isSetterLikeSelector(Selector sel) {
   if (sel.isUnarySelector()) return false;
 
   StringRef str = sel.getNameForSlot(0);
-  while (!str.empty() && str.front() == '_') str = str.substr(1);
+  str = str.ltrim('_');
   if (str.starts_with("set"))
     str = str.substr(3);
   else if (str.starts_with("add")) {
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 719c6aab74e0..acfc00f41254 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -771,10 +771,9 @@ namespace {
   };
 } // namespace
 
-static const Expr *
-SubstituteConstraintExpression(Sema &S,
-                               const Sema::TemplateCompareNewDeclInfo &DeclInfo,
-                               const Expr *ConstrExpr) {
+static const Expr *SubstituteConstraintExpressionWithoutSatisfaction(
+    Sema &S, const Sema::TemplateCompareNewDeclInfo &DeclInfo,
+    const Expr *ConstrExpr) {
   MultiLevelTemplateArgumentList MLTAL = S.getTemplateInstantiationArgs(
       DeclInfo.getDecl(), DeclInfo.getLexicalDeclContext(), /*Final=*/false,
       /*Innermost=*/nullptr,
@@ -797,8 +796,8 @@ SubstituteConstraintExpression(Sema &S,
   std::optional ThisScope;
   if (auto *RD = dyn_cast(DeclInfo.getDeclContext()))
     ThisScope.emplace(S, const_cast(RD), Qualifiers());
-  ExprResult SubstConstr =
-      S.SubstConstraintExpr(const_cast(ConstrExpr), MLTAL);
+  ExprResult SubstConstr = S.SubstConstraintExprWithoutSatisfaction(
+      const_cast(ConstrExpr), MLTAL);
   if (SFINAE.hasErrorOccurred() || !SubstConstr.isUsable())
     return nullptr;
   return SubstConstr.get();
@@ -814,12 +813,14 @@ bool Sema::AreConstraintExpressionsEqual(const NamedDecl *Old,
   if (Old && !New.isInvalid() && !New.ContainsDecl(Old) &&
       Old->getLexicalDeclContext() != New.getLexicalDeclContext()) {
     if (const Expr *SubstConstr =
-            SubstituteConstraintExpression(*this, Old, OldConstr))
+            SubstituteConstraintExpressionWithoutSatisfaction(*this, Old,
+                                                              OldConstr))
       OldConstr = SubstConstr;
     else
       return false;
     if (const Expr *SubstConstr =
-            SubstituteConstraintExpression(*this, New, NewConstr))
+            SubstituteConstraintExpressionWithoutSatisfaction(*this, New,
+                                                              NewConstr))
       NewConstr = SubstConstr;
     else
       return false;
diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp
index bee80db8d166..4e600fd29ee7 100644
--- a/clang/lib/Sema/SemaCoroutine.cpp
+++ b/clang/lib/Sema/SemaCoroutine.cpp
@@ -16,10 +16,12 @@
 #include "CoroutineStmtBuilder.h"
 #include "clang/AST/ASTLambda.h"
 #include "clang/AST/Decl.h"
+#include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Overload.h"
 #include "clang/Sema/ScopeInfo.h"
@@ -772,6 +774,9 @@ bool Sema::checkFinalSuspendNoThrow(const Stmt *FinalSuspend) {
 
 bool Sema::ActOnCoroutineBodyStart(Scope *SC, SourceLocation KWLoc,
                                    StringRef Keyword) {
+  // Ignore previous expr evaluation contexts.
+  EnterExpressionEvaluationContext PotentiallyEvaluated(
+      *this, Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
   if (!checkCoroutineContext(*this, KWLoc, Keyword))
     return false;
   auto *ScopeInfo = getCurFunction();
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index ffbe317d5599..f9bf1d14bdc4 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -2315,6 +2315,12 @@ void Sema::ActOnPopScope(SourceLocation Loc, Scope *S) {
       }
       ShadowingDecls.erase(ShadowI);
     }
+
+    if (!getLangOpts().CPlusPlus && S->isClassScope()) {
+      if (auto *FD = dyn_cast(TmpD);
+          FD && FD->hasAttr())
+        CheckCountedByAttr(S, FD);
+    }
   }
 
   llvm::sort(DeclDiags,
@@ -3813,8 +3819,7 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, NamedDecl *&OldD, Scope *S,
 
   // It is not permitted to redeclare an SME function with different SME
   // attributes.
-  if (IsInvalidSMECallConversion(Old->getType(), New->getType(),
-                                 AArch64SMECallConversionKind::MatchExactly)) {
+  if (IsInvalidSMECallConversion(Old->getType(), New->getType())) {
     Diag(New->getLocation(), diag::err_sme_attr_mismatch)
         << New->getType() << Old->getType();
     Diag(OldLocation, diag::note_previous_declaration);
@@ -4385,8 +4390,8 @@ bool Sema::MergeCompatibleFunctionDecls(FunctionDecl *New, FunctionDecl *Old,
   mergeDeclAttributes(New, Old);
 
   // Merge "pure" flag.
-  if (Old->isPure())
-    New->setPure();
+  if (Old->isPureVirtual())
+    New->setIsPureVirtual();
 
   // Merge "used" flag.
   if (Old->getMostRecentDecl()->isUsed(false))
@@ -5798,7 +5803,7 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
   // Mock up a declarator.
   Declarator Dc(DS, ParsedAttributesView::none(), DeclaratorContext::Member);
   StorageClass SC = StorageClassSpecToVarDeclStorageClass(DS);
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(Dc, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(Dc);
   assert(TInfo && "couldn't build declarator info for anonymous struct/union");
 
   // Create a declaration for this anonymous struct/union.
@@ -5895,7 +5900,7 @@ Decl *Sema::BuildMicrosoftCAnonymousStruct(Scope *S, DeclSpec &DS,
 
   // Mock up a declarator.
   Declarator Dc(DS, ParsedAttributesView::none(), DeclaratorContext::TypeName);
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(Dc, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(Dc);
   assert(TInfo && "couldn't build declarator info for anonymous struct");
 
   auto *ParentDecl = cast(CurContext);
@@ -6433,7 +6438,7 @@ NamedDecl *Sema::HandleDeclarator(Scope *S, Declarator &D,
     }
   }
 
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
   QualType R = TInfo->getType();
 
   if (DiagnoseUnexpandedParameterPack(D.getIdentifierLoc(), TInfo,
@@ -7063,8 +7068,7 @@ static void checkAttributesAfterMerging(Sema &S, NamedDecl &ND) {
   if (WeakRefAttr *Attr = ND.getAttr()) {
     if (ND.isExternallyVisible()) {
       S.Diag(Attr->getLocation(), diag::err_attribute_weakref_not_static);
-      ND.dropAttr();
-      ND.dropAttr();
+      ND.dropAttrs();
     }
   }
 
@@ -9874,7 +9878,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
     if (const CXXRecordDecl *Parent =
           dyn_cast(NewFD->getDeclContext())) {
       if (Parent->isInterface() && cast(NewFD)->isUserProvided())
-        NewFD->setPure(true);
+        NewFD->setIsPureVirtual(true);
 
       // C++ [class.union]p2
       //   A union can have member functions, but not virtual functions.
@@ -9900,15 +9904,15 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
     // Match up the template parameter lists with the scope specifier, then
     // determine whether we have a template or a template specialization.
     bool Invalid = false;
+    TemplateIdAnnotation *TemplateId =
+        D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId
+            ? D.getName().TemplateId
+            : nullptr;
     TemplateParameterList *TemplateParams =
         MatchTemplateParametersToScopeSpecifier(
             D.getDeclSpec().getBeginLoc(), D.getIdentifierLoc(),
-            D.getCXXScopeSpec(),
-            D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId
-                ? D.getName().TemplateId
-                : nullptr,
-            TemplateParamLists, isFriend, isMemberSpecialization,
-            Invalid);
+            D.getCXXScopeSpec(), TemplateId, TemplateParamLists, isFriend,
+            isMemberSpecialization, Invalid);
     if (TemplateParams) {
       // Check that we can declare a template here.
       if (CheckTemplateDeclScope(S, TemplateParams))
@@ -9921,6 +9925,11 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
         if (Name.getNameKind() == DeclarationName::CXXDestructorName) {
           Diag(NewFD->getLocation(), diag::err_destructor_template);
           NewFD->setInvalidDecl();
+          // Function template with explicit template arguments.
+        } else if (TemplateId) {
+          Diag(D.getIdentifierLoc(), diag::err_function_template_partial_spec)
+              << SourceRange(TemplateId->LAngleLoc, TemplateId->RAngleLoc);
+          NewFD->setInvalidDecl();
         }
 
         // If we're adding a template to a dependent context, we may need to
@@ -9973,6 +9982,11 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
             << FixItHint::CreateRemoval(RemoveRange)
             << FixItHint::CreateInsertion(InsertLoc, "<>");
           Invalid = true;
+
+          // Recover by faking up an empty template argument list.
+          HasExplicitTemplateArgs = true;
+          TemplateArgs.setLAngleLoc(InsertLoc);
+          TemplateArgs.setRAngleLoc(InsertLoc);
         }
       }
     } else {
@@ -9986,6 +10000,33 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
       if (TemplateParamLists.size() > 0)
         // For source fidelity, store all the template param lists.
         NewFD->setTemplateParameterListsInfo(Context, TemplateParamLists);
+
+      // "friend void foo<>(int);" is an implicit specialization decl.
+      if (isFriend && TemplateId)
+        isFunctionTemplateSpecialization = true;
+    }
+
+    // If this is a function template specialization and the unqualified-id of
+    // the declarator-id is a template-id, convert the template argument list
+    // into our AST format and check for unexpanded packs.
+    if (isFunctionTemplateSpecialization && TemplateId) {
+      HasExplicitTemplateArgs = true;
+
+      TemplateArgs.setLAngleLoc(TemplateId->LAngleLoc);
+      TemplateArgs.setRAngleLoc(TemplateId->RAngleLoc);
+      ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
+                                         TemplateId->NumArgs);
+      translateTemplateArguments(TemplateArgsPtr, TemplateArgs);
+
+      // FIXME: Should we check for unexpanded packs if this was an (invalid)
+      // declaration of a function template partial specialization? Should we
+      // consider the unexpanded pack context to be a partial specialization?
+      for (const TemplateArgumentLoc &ArgLoc : TemplateArgs.arguments()) {
+        if (DiagnoseUnexpandedParameterPack(
+                ArgLoc, isFriend ? UPPC_FriendDeclaration
+                                 : UPPC_ExplicitSpecialization))
+          NewFD->setInvalidDecl();
+      }
     }
 
     if (Invalid) {
@@ -10438,46 +10479,6 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
            diag::ext_operator_new_delete_declared_inline)
         << NewFD->getDeclName();
 
-    // If the declarator is a template-id, translate the parser's template
-    // argument list into our AST format.
-    if (D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId) {
-      TemplateIdAnnotation *TemplateId = D.getName().TemplateId;
-      TemplateArgs.setLAngleLoc(TemplateId->LAngleLoc);
-      TemplateArgs.setRAngleLoc(TemplateId->RAngleLoc);
-      ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
-                                         TemplateId->NumArgs);
-      translateTemplateArguments(TemplateArgsPtr,
-                                 TemplateArgs);
-
-      HasExplicitTemplateArgs = true;
-
-      if (NewFD->isInvalidDecl()) {
-        HasExplicitTemplateArgs = false;
-      } else if (FunctionTemplate) {
-        // Function template with explicit template arguments.
-        Diag(D.getIdentifierLoc(), diag::err_function_template_partial_spec)
-          << SourceRange(TemplateId->LAngleLoc, TemplateId->RAngleLoc);
-
-        HasExplicitTemplateArgs = false;
-      } else if (isFriend) {
-        // "friend void foo<>(int);" is an implicit specialization decl.
-        isFunctionTemplateSpecialization = true;
-      } else {
-        assert(isFunctionTemplateSpecialization &&
-               "should have a 'template<>' for this decl");
-      }
-    } else if (isFriend && isFunctionTemplateSpecialization) {
-      // This combination is only possible in a recovery case;  the user
-      // wrote something like:
-      //   template <> friend void foo(int);
-      // which we're recovering from as if the user had written:
-      //   friend void foo<>(int);
-      // Go ahead and fake up a template id.
-      HasExplicitTemplateArgs = true;
-      TemplateArgs.setLAngleLoc(D.getIdentifierLoc());
-      TemplateArgs.setRAngleLoc(D.getIdentifierLoc());
-    }
-
     // We do not add HD attributes to specializations here because
     // they may have different constexpr-ness compared to their
     // templates and, after maybeAddCUDAHostDeviceAttrs() is applied,
@@ -10844,9 +10845,19 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
     // Precalculate whether this is a friend function template with a constraint
     // that depends on an enclosing template, per [temp.friend]p9.
     if (isFriend && FunctionTemplate &&
-        FriendConstraintsDependOnEnclosingTemplate(NewFD))
+        FriendConstraintsDependOnEnclosingTemplate(NewFD)) {
       NewFD->setFriendConstraintRefersToEnclosingTemplate(true);
 
+      // C++ [temp.friend]p9:
+      //    A friend function template with a constraint that depends on a
+      //    template parameter from an enclosing template shall be a definition.
+      if (!D.isFunctionDefinition()) {
+        Diag(NewFD->getBeginLoc(),
+             diag::err_friend_decl_with_enclosing_temp_constraint_must_be_def);
+        NewFD->setInvalidDecl();
+      }
+    }
+
     if (FunctionTemplate) {
       if (NewFD->isInvalidDecl())
         FunctionTemplate->setInvalidDecl();
@@ -11790,6 +11801,32 @@ static bool CheckMultiVersionFunction(Sema &S, FunctionDecl *NewFD,
                                          OldDecl, Previous);
 }
 
+static void CheckConstPureAttributesUsage(Sema &S, FunctionDecl *NewFD) {
+  bool IsPure = NewFD->hasAttr();
+  bool IsConst = NewFD->hasAttr();
+
+  // If there are no pure or const attributes, there's nothing to check.
+  if (!IsPure && !IsConst)
+    return;
+
+  // If the function is marked both pure and const, we retain the const
+  // attribute because it makes stronger guarantees than the pure attribute, and
+  // we drop the pure attribute explicitly to prevent later confusion about
+  // semantics.
+  if (IsPure && IsConst) {
+    S.Diag(NewFD->getLocation(), diag::warn_const_attr_with_pure_attr);
+    NewFD->dropAttrs();
+  }
+
+  // Constructors and destructors are functions which return void, so are
+  // handled here as well.
+  if (NewFD->getReturnType()->isVoidType()) {
+    S.Diag(NewFD->getLocation(), diag::warn_pure_function_returns_void)
+        << IsConst;
+    NewFD->dropAttrs();
+  }
+}
+
 /// Perform semantic checking of a new function declaration.
 ///
 /// Performs semantic analysis of the new function declaration
@@ -11887,6 +11924,8 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
     NewFD->setInvalidDecl();
   }
 
+  CheckConstPureAttributesUsage(*this, NewFD);
+
   // C++11 [dcl.constexpr]p8:
   //   A constexpr specifier for a non-static member function that is not
   //   a constructor declares that member function to be const.
@@ -12063,11 +12102,12 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
         checkThisInStaticMemberFunctionType(Method);
     }
 
-    // C++20: dcl.decl.general p4:
-    // The optional requires-clause ([temp.pre]) in an init-declarator or
-    // member-declarator shall be present only if the declarator declares a
-    // templated function ([dcl.fct]).
     if (Expr *TRC = NewFD->getTrailingRequiresClause()) {
+      // C++20: dcl.decl.general p4:
+      // The optional requires-clause ([temp.pre]) in an init-declarator or
+      // member-declarator shall be present only if the declarator declares a
+      // templated function ([dcl.fct]).
+      //
       // [temp.pre]/8:
       // An entity is templated if it is
       // - a template,
@@ -12085,15 +12125,29 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
       // templated. A templated variable is a variable template or a variable
       // that is templated.
 
-      if (!NewFD->getDescribedFunctionTemplate() && // -a template
-          // defined... in a templated entity
+      bool IsTemplate = NewFD->getDescribedFunctionTemplate();
+      bool IsFriend = NewFD->getFriendObjectKind();
+      if (!IsTemplate && // -a template
+                         // defined... in a templated entity
           !(DeclIsDefn && NewFD->isTemplated()) &&
           // a member of a templated entity
           !(isa(NewFD) && NewFD->isTemplated()) &&
           // Don't complain about instantiations, they've already had these
           // rules + others enforced.
-          !NewFD->isTemplateInstantiation()) {
+          !NewFD->isTemplateInstantiation() &&
+          // If the function violates [temp.friend]p9 because it is missing
+          // a definition, and adding a definition would make it templated,
+          // then let that error take precedence.
+          !(!DeclIsDefn && IsFriend && NewFD->isTemplated())) {
         Diag(TRC->getBeginLoc(), diag::err_constrained_non_templated_function);
+      } else if (!DeclIsDefn && !IsTemplate && IsFriend &&
+                 !NewFD->isTemplateInstantiation()) {
+        // C++ [temp.friend]p9:
+        //   A non-template friend declaration with a requires-clause shall be a
+        //   definition.
+        Diag(NewFD->getBeginLoc(),
+             diag::err_non_temp_friend_decl_with_requires_clause_must_be_def);
+        NewFD->setInvalidDecl();
       }
     }
 
@@ -12177,13 +12231,18 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
   // Check if the function definition uses any AArch64 SME features without
   // having the '+sme' feature enabled.
   if (DeclIsDefn) {
+    const auto *Attr = NewFD->getAttr();
     bool UsesSM = NewFD->hasAttr();
-    bool UsesZA = NewFD->hasAttr();
+    bool UsesZA = Attr && Attr->isNewZA();
+    bool UsesZT0 = Attr && Attr->isNewZT0();
     if (const auto *FPT = NewFD->getType()->getAs()) {
       FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
       UsesSM |=
           EPI.AArch64SMEAttributes & FunctionType::SME_PStateSMEnabledMask;
-      UsesZA |= EPI.AArch64SMEAttributes & FunctionType::SME_PStateZASharedMask;
+      UsesZA |= FunctionType::getArmZAState(EPI.AArch64SMEAttributes) !=
+                FunctionType::ARM_None;
+      UsesZT0 |= FunctionType::getArmZT0State(EPI.AArch64SMEAttributes) !=
+                 FunctionType::ARM_None;
     }
 
     if (UsesSM || UsesZA) {
@@ -12198,6 +12257,14 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
                diag::err_sme_definition_using_za_in_non_sme_target);
       }
     }
+    if (UsesZT0) {
+      llvm::StringMap FeatureMap;
+      Context.getFunctionFeatureMap(FeatureMap, NewFD);
+      if (!FeatureMap.contains("sme2")) {
+        Diag(NewFD->getLocation(),
+             diag::err_sme_definition_using_zt0_in_non_sme2_target);
+      }
+    }
   }
 
   return Redeclaration;
@@ -13573,6 +13640,15 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
           CreateRecoveryExpr(Init->getBeginLoc(), Init->getEndLoc(), Args);
       if (RecoveryExpr.get())
         VDecl->setInit(RecoveryExpr.get());
+      // In general, for error recovery purposes, the initalizer doesn't play
+      // part in the valid bit of the declaration. There are a few exceptions:
+      //  1) if the var decl has a deduced auto type, and the type cannot be
+      //     deduced by an invalid initializer;
+      //  2) if the var decl is decompsition decl with a non-deduced type, and
+      //     the initialization fails (e.g. `int [a] = {1, 2};`);
+      // Case 1) was already handled elsewhere.
+      if (isa(VDecl)) // Case 2)
+        VDecl->setInvalidDecl();
       return;
     }
 
@@ -15043,7 +15119,7 @@ Decl *Sema::ActOnParamDeclarator(Scope *S, Declarator &D,
 
   CheckFunctionOrTemplateParamDeclarator(S, D);
 
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
   QualType parmDeclType = TInfo->getType();
 
   // Check for redeclaration of parameters, e.g. int foo(int x, int x);
@@ -15167,6 +15243,37 @@ void Sema::DiagnoseSizeOfParametersAndReturnValue(
   }
 }
 
+QualType Sema::AdjustParameterTypeForObjCAutoRefCount(QualType T,
+                                                      SourceLocation NameLoc,
+                                                      TypeSourceInfo *TSInfo) {
+  // In ARC, infer a lifetime qualifier for appropriate parameter types.
+  if (!getLangOpts().ObjCAutoRefCount ||
+      T.getObjCLifetime() != Qualifiers::OCL_None || !T->isObjCLifetimeType())
+    return T;
+
+  Qualifiers::ObjCLifetime Lifetime;
+
+  // Special cases for arrays:
+  //   - if it's const, use __unsafe_unretained
+  //   - otherwise, it's an error
+  if (T->isArrayType()) {
+    if (!T.isConstQualified()) {
+      if (DelayedDiagnostics.shouldDelayDiagnostics())
+        DelayedDiagnostics.add(sema::DelayedDiagnostic::makeForbiddenType(
+            NameLoc, diag::err_arc_array_param_no_ownership, T, false));
+      else
+        Diag(NameLoc, diag::err_arc_array_param_no_ownership)
+            << TSInfo->getTypeLoc().getSourceRange();
+    }
+    Lifetime = Qualifiers::OCL_ExplicitNone;
+  } else {
+    Lifetime = T->getObjCARCImplicitLifetime();
+  }
+  T = Context.getLifetimeQualifiedType(T, Lifetime);
+
+  return T;
+}
+
 ParmVarDecl *Sema::CheckParameter(DeclContext *DC, SourceLocation StartLoc,
                                   SourceLocation NameLoc, IdentifierInfo *Name,
                                   QualType T, TypeSourceInfo *TSInfo,
@@ -15844,15 +15951,26 @@ static void diagnoseImplicitlyRetainedSelf(Sema &S) {
           << FixItHint::CreateInsertion(P.first, "self->");
 }
 
+static bool methodHasName(const FunctionDecl *FD, StringRef Name) {
+  return isa(FD) && FD->param_empty() &&
+         FD->getDeclName().isIdentifier() && FD->getName().equals(Name);
+}
+
+bool Sema::CanBeGetReturnObject(const FunctionDecl *FD) {
+  return methodHasName(FD, "get_return_object");
+}
+
+bool Sema::CanBeGetReturnTypeOnAllocFailure(const FunctionDecl *FD) {
+  return FD->isStatic() &&
+         methodHasName(FD, "get_return_object_on_allocation_failure");
+}
+
 void Sema::CheckCoroutineWrapper(FunctionDecl *FD) {
-  if (!FD)
-    return;
   RecordDecl *RD = FD->getReturnType()->getAsRecordDecl();
   if (!RD || !RD->getUnderlyingDecl()->hasAttr())
     return;
-  // Allow `get_return_object()`.
-  if (FD->getDeclName().isIdentifier() &&
-      FD->getName().equals("get_return_object") && FD->param_empty())
+  // Allow some_promise_type::get_return_object().
+  if (CanBeGetReturnObject(FD) || CanBeGetReturnTypeOnAllocFailure(FD))
     return;
   if (!FD->hasAttr())
     Diag(FD->getLocation(), diag::err_coroutine_return_type) << RD;
@@ -15869,7 +15987,8 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
   sema::AnalysisBasedWarnings::Policy WP = AnalysisWarnings.getDefaultPolicy();
   sema::AnalysisBasedWarnings::Policy *ActivePolicy = nullptr;
 
-  if (getLangOpts().Coroutines) {
+  // If we skip function body, we can't tell if a function is a coroutine.
+  if (getLangOpts().Coroutines && FD && !FD->hasSkippedBody()) {
     if (FSI->isCoroutine())
       CheckCompletedCoroutineBody(FD, Body);
     else
@@ -15932,7 +16051,8 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
 
       // MSVC permits the use of pure specifier (=0) on function definition,
       // defined at class scope, warn about this non-standard construct.
-      if (getLangOpts().MicrosoftExt && FD->isPure() && !FD->isOutOfLine())
+      if (getLangOpts().MicrosoftExt && FD->isPureVirtual() &&
+          !FD->isOutOfLine())
         Diag(FD->getLocation(), diag::ext_pure_function_definition);
 
       if (!FD->isInvalidDecl()) {
@@ -18308,7 +18428,7 @@ FieldDecl *Sema::HandleField(Scope *S, RecordDecl *Record,
   SourceLocation Loc = DeclStart;
   if (II) Loc = D.getIdentifierLoc();
 
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
   QualType T = TInfo->getType();
   if (getLangOpts().CPlusPlus) {
     CheckExtraCXXDefaultArguments(D);
@@ -18672,7 +18792,7 @@ Decl *Sema::ActOnIvar(Scope *S, SourceLocation DeclStart, Declarator &D,
   // FIXME: Unnamed fields can be handled in various different ways, for
   // example, unnamed unions inject all members into the struct namespace!
 
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
   QualType T = TInfo->getType();
 
   if (BitWidth) {
@@ -20217,7 +20337,7 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceRange BraceRange,
     // Adjust the APSInt value.
     InitVal = InitVal.extOrTrunc(NewWidth);
     InitVal.setIsSigned(NewSign);
-    ECD->setInitVal(InitVal);
+    ECD->setInitVal(Context, InitVal);
 
     // Adjust the Expr initializer and type.
     if (ECD->getInitExpr() &&
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 3f4d4a49d4d6..55112803bde6 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -3398,6 +3398,22 @@ static void handleSectionAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   }
 }
 
+static void handleCodeModelAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
+  StringRef Str;
+  SourceLocation LiteralLoc;
+  // Check that it is a string.
+  if (!S.checkStringLiteralArgumentAttr(AL, 0, Str, &LiteralLoc))
+    return;
+
+  llvm::CodeModel::Model CM;
+  if (!CodeModelAttr::ConvertStrToModel(Str, CM)) {
+    S.Diag(LiteralLoc, diag::err_attr_codemodel_arg) << Str;
+    return;
+  }
+
+  D->addAttr(::new (S.Context) CodeModelAttr(S.Context, AL, CM));
+}
+
 // This is used for `__declspec(code_seg("segname"))` on a decl.
 // `#pragma code_seg("segname")` uses checkSectionName() instead.
 static bool checkCodeSegName(Sema &S, SourceLocation LiteralLoc,
@@ -6249,29 +6265,35 @@ static void handleObjCRequiresSuperAttr(Sema &S, Decl *D,
   D->addAttr(::new (S.Context) ObjCRequiresSuperAttr(S.Context, Attrs));
 }
 
-static void handleNSErrorDomain(Sema &S, Decl *D, const ParsedAttr &AL) {
-  auto *E = AL.getArgAsExpr(0);
-  auto Loc = E ? E->getBeginLoc() : AL.getLoc();
-
-  auto *DRE = dyn_cast(AL.getArgAsExpr(0));
-  if (!DRE) {
-    S.Diag(Loc, diag::err_nserrordomain_invalid_decl) << 0;
+static void handleNSErrorDomain(Sema &S, Decl *D, const ParsedAttr &Attr) {
+  if (!isa(D)) {
+    S.Diag(D->getBeginLoc(), diag::err_nserrordomain_invalid_decl) << 0;
     return;
   }
 
-  auto *VD = dyn_cast(DRE->getDecl());
-  if (!VD) {
-    S.Diag(Loc, diag::err_nserrordomain_invalid_decl) << 1 << DRE->getDecl();
+  IdentifierLoc *IdentLoc =
+      Attr.isArgIdent(0) ? Attr.getArgAsIdent(0) : nullptr;
+  if (!IdentLoc || !IdentLoc->Ident) {
+    // Try to locate the argument directly.
+    SourceLocation Loc = Attr.getLoc();
+    if (Attr.isArgExpr(0) && Attr.getArgAsExpr(0))
+      Loc = Attr.getArgAsExpr(0)->getBeginLoc();
+
+    S.Diag(Loc, diag::err_nserrordomain_invalid_decl) << 0;
     return;
   }
 
-  if (!isNSStringType(VD->getType(), S.Context) &&
-      !isCFStringType(VD->getType(), S.Context)) {
-    S.Diag(Loc, diag::err_nserrordomain_wrong_type) << VD;
+  // Verify that the identifier is a valid decl in the C decl namespace.
+  LookupResult Result(S, DeclarationName(IdentLoc->Ident), SourceLocation(),
+                      Sema::LookupNameKind::LookupOrdinaryName);
+  if (!S.LookupName(Result, S.TUScope) || !Result.getAsSingle()) {
+    S.Diag(IdentLoc->Loc, diag::err_nserrordomain_invalid_decl)
+        << 1 << IdentLoc->Ident;
     return;
   }
 
-  D->addAttr(::new (S.Context) NSErrorDomainAttr(S.Context, AL, VD));
+  D->addAttr(::new (S.Context)
+                 NSErrorDomainAttr(S.Context, Attr, IdentLoc->Ident));
 }
 
 static void handleObjCBridgeAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
@@ -8473,6 +8495,135 @@ static void handleZeroCallUsedRegsAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   D->addAttr(ZeroCallUsedRegsAttr::Create(S.Context, Kind, AL));
 }
 
+static void handleCountedByAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
+  if (!AL.isArgIdent(0)) {
+    S.Diag(AL.getLoc(), diag::err_attribute_argument_type)
+        << AL << AANT_ArgumentIdentifier;
+    return;
+  }
+
+  IdentifierLoc *IL = AL.getArgAsIdent(0);
+  CountedByAttr *CBA =
+      ::new (S.Context) CountedByAttr(S.Context, AL, IL->Ident);
+  CBA->setCountedByFieldLoc(IL->Loc);
+  D->addAttr(CBA);
+}
+
+static const FieldDecl *
+FindFieldInTopLevelOrAnonymousStruct(const RecordDecl *RD,
+                                     const IdentifierInfo *FieldName) {
+  for (const Decl *D : RD->decls()) {
+    if (const auto *FD = dyn_cast(D))
+      if (FD->getName() == FieldName->getName())
+        return FD;
+
+    if (const auto *R = dyn_cast(D))
+      if (const FieldDecl *FD =
+              FindFieldInTopLevelOrAnonymousStruct(R, FieldName))
+        return FD;
+  }
+
+  return nullptr;
+}
+
+bool Sema::CheckCountedByAttr(Scope *S, const FieldDecl *FD) {
+  LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
+      LangOptions::StrictFlexArraysLevelKind::IncompleteOnly;
+  if (!Decl::isFlexibleArrayMemberLike(Context, FD, FD->getType(),
+                                       StrictFlexArraysLevel, true)) {
+    // The "counted_by" attribute must be on a flexible array member.
+    SourceRange SR = FD->getLocation();
+    Diag(SR.getBegin(), diag::err_counted_by_attr_not_on_flexible_array_member)
+        << SR;
+    return true;
+  }
+
+  const auto *CBA = FD->getAttr();
+  const IdentifierInfo *FieldName = CBA->getCountedByField();
+
+  auto GetNonAnonStructOrUnion = [](const RecordDecl *RD) {
+    while (RD && !RD->getDeclName())
+      if (const auto *R = dyn_cast(RD->getDeclContext()))
+        RD = R;
+      else
+        break;
+
+    return RD;
+  };
+
+  const RecordDecl *EnclosingRD = GetNonAnonStructOrUnion(FD->getParent());
+  const FieldDecl *CountFD =
+      FindFieldInTopLevelOrAnonymousStruct(EnclosingRD, FieldName);
+
+  if (!CountFD) {
+    DeclarationNameInfo NameInfo(FieldName,
+                                 CBA->getCountedByFieldLoc().getBegin());
+    LookupResult MemResult(*this, NameInfo, Sema::LookupMemberName);
+    LookupName(MemResult, S);
+
+    if (!MemResult.empty()) {
+      SourceRange SR = CBA->getCountedByFieldLoc();
+      Diag(SR.getBegin(), diag::err_flexible_array_count_not_in_same_struct)
+          << CBA->getCountedByField() << SR;
+
+      if (auto *ND = MemResult.getAsSingle()) {
+        SR = ND->getLocation();
+        Diag(SR.getBegin(), diag::note_flexible_array_counted_by_attr_field)
+            << ND << SR;
+      }
+
+      return true;
+    } else {
+      // The "counted_by" field needs to exist in the struct.
+      LookupResult OrdResult(*this, NameInfo, Sema::LookupOrdinaryName);
+      LookupName(OrdResult, S);
+
+      if (!OrdResult.empty()) {
+        SourceRange SR = FD->getLocation();
+        Diag(SR.getBegin(), diag::err_counted_by_must_be_in_structure)
+            << FieldName << SR;
+
+        if (auto *ND = OrdResult.getAsSingle()) {
+          SR = ND->getLocation();
+          Diag(SR.getBegin(), diag::note_flexible_array_counted_by_attr_field)
+              << ND << SR;
+        }
+
+        return true;
+      }
+    }
+
+    CXXScopeSpec SS;
+    DeclFilterCCC Filter(FieldName);
+    return DiagnoseEmptyLookup(S, SS, MemResult, Filter, nullptr, std::nullopt,
+                               const_cast(FD->getDeclContext()));
+  }
+
+  if (CountFD->hasAttr()) {
+    // The "counted_by" field can't point to the flexible array member.
+    SourceRange SR = CBA->getCountedByFieldLoc();
+    Diag(SR.getBegin(), diag::err_counted_by_attr_refers_to_flexible_array)
+        << CBA->getCountedByField() << SR;
+    return true;
+  }
+
+  if (!CountFD->getType()->isIntegerType() ||
+      CountFD->getType()->isBooleanType()) {
+    // The "counted_by" field must have an integer type.
+    SourceRange SR = CBA->getCountedByFieldLoc();
+    Diag(SR.getBegin(),
+         diag::err_flexible_array_counted_by_attr_field_not_integer)
+        << CBA->getCountedByField() << SR;
+
+    SR = CountFD->getLocation();
+    Diag(SR.getBegin(), diag::note_flexible_array_counted_by_attr_field)
+        << CountFD << SR;
+    return true;
+  }
+
+  return false;
+}
+
 static void handleFunctionReturnThunksAttr(Sema &S, Decl *D,
                                            const ParsedAttr &AL) {
   StringRef KindStr;
@@ -8835,26 +8986,80 @@ static bool MustDelayAttributeArguments(const ParsedAttr &AL) {
   return false;
 }
 
+static bool checkArmNewAttrMutualExclusion(
+    Sema &S, const ParsedAttr &AL, const FunctionProtoType *FPT,
+    FunctionType::ArmStateValue CurrentState, StringRef StateName) {
+  auto CheckForIncompatibleAttr =
+      [&](FunctionType::ArmStateValue IncompatibleState,
+          StringRef IncompatibleStateName) {
+        if (CurrentState == IncompatibleState) {
+          S.Diag(AL.getLoc(), diag::err_attributes_are_not_compatible)
+              << (std::string("'__arm_new(\"") + StateName.str() + "\")'")
+              << (std::string("'") + IncompatibleStateName.str() + "(\"" +
+                  StateName.str() + "\")'")
+              << true;
+          AL.setInvalid();
+        }
+      };
 
-static void handleArmNewZaAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
-  if (auto *FPT = dyn_cast(D->getFunctionType())) {
-    if (FPT->getAArch64SMEAttributes() &
-        FunctionType::SME_PStateZASharedMask) {
-      S.Diag(AL.getLoc(), diag::err_attributes_are_not_compatible)
-          << AL << "'__arm_shared_za'" << true;
-      AL.setInvalid();
-    }
-    if (FPT->getAArch64SMEAttributes() &
-        FunctionType::SME_PStateZAPreservedMask) {
-      S.Diag(AL.getLoc(), diag::err_attributes_are_not_compatible)
-          << AL << "'__arm_preserves_za'" << true;
+  CheckForIncompatibleAttr(FunctionType::ARM_In, "__arm_in");
+  CheckForIncompatibleAttr(FunctionType::ARM_Out, "__arm_out");
+  CheckForIncompatibleAttr(FunctionType::ARM_InOut, "__arm_inout");
+  CheckForIncompatibleAttr(FunctionType::ARM_Preserves, "__arm_preserves");
+  return AL.isInvalid();
+}
+
+static void handleArmNewAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
+  if (!AL.getNumArgs()) {
+    S.Diag(AL.getLoc(), diag::err_missing_arm_state) << AL;
+    AL.setInvalid();
+    return;
+  }
+
+  std::vector NewState;
+  if (const auto *ExistingAttr = D->getAttr()) {
+    for (StringRef S : ExistingAttr->newArgs())
+      NewState.push_back(S);
+  }
+
+  bool HasZA = false;
+  bool HasZT0 = false;
+  for (unsigned I = 0, E = AL.getNumArgs(); I != E; ++I) {
+    StringRef StateName;
+    SourceLocation LiteralLoc;
+    if (!S.checkStringLiteralArgumentAttr(AL, I, StateName, &LiteralLoc))
+      return;
+
+    if (StateName == "za")
+      HasZA = true;
+    else if (StateName == "zt0")
+      HasZT0 = true;
+    else {
+      S.Diag(LiteralLoc, diag::err_unknown_arm_state) << StateName;
       AL.setInvalid();
+      return;
     }
-    if (AL.isInvalid())
+
+    if (!llvm::is_contained(NewState, StateName)) // Avoid adding duplicates.
+      NewState.push_back(StateName);
+  }
+
+  if (auto *FPT = dyn_cast(D->getFunctionType())) {
+    FunctionType::ArmStateValue ZAState =
+        FunctionType::getArmZAState(FPT->getAArch64SMEAttributes());
+    if (HasZA && ZAState != FunctionType::ARM_None &&
+        checkArmNewAttrMutualExclusion(S, AL, FPT, ZAState, "za"))
+      return;
+    FunctionType::ArmStateValue ZT0State =
+        FunctionType::getArmZT0State(FPT->getAArch64SMEAttributes());
+    if (HasZT0 && ZT0State != FunctionType::ARM_None &&
+        checkArmNewAttrMutualExclusion(S, AL, FPT, ZT0State, "zt0"))
       return;
   }
 
-  handleSimpleAttribute(S, D, AL);
+  D->dropAttr();
+  D->addAttr(::new (S.Context)
+                 ArmNewAttr(S.Context, AL, NewState.data(), NewState.size()));
 }
 
 /// ProcessDeclAttribute - Apply the specific attribute to the specified decl if
@@ -9288,6 +9493,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_Section:
     handleSectionAttr(S, D, AL);
     break;
+  case ParsedAttr::AT_CodeModel:
+    handleCodeModelAttr(S, D, AL);
+    break;
   case ParsedAttr::AT_RandomizeLayout:
     handleRandomizeLayoutAttr(S, D, AL);
     break;
@@ -9436,6 +9644,10 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
     handleAvailableOnlyInDefaultEvalMethod(S, D, AL);
     break;
 
+  case ParsedAttr::AT_CountedBy:
+    handleCountedByAttr(S, D, AL);
+    break;
+
   // Microsoft attributes:
   case ParsedAttr::AT_LayoutVersion:
     handleLayoutVersion(S, D, AL);
@@ -9635,8 +9847,8 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
     handleSimpleAttribute(S, D, AL);
     break;
 
-  case ParsedAttr::AT_ArmNewZA:
-    handleArmNewZaAttr(S, D, AL);
+  case ParsedAttr::AT_ArmNew:
+    handleArmNewAttr(S, D, AL);
     break;
 
   case ParsedAttr::AT_AcquireHandle:
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 36e53c684ac4..df5bd55e7c28 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -839,7 +839,7 @@ Sema::ActOnDecompositionDeclarator(Scope *S, Declarator &D,
     Diag(DS.getVolatileSpecLoc(),
          diag::warn_deprecated_volatile_structured_binding);
 
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
   QualType R = TInfo->getType();
 
   if (DiagnoseUnexpandedParameterPack(D.getIdentifierLoc(), TInfo,
@@ -6062,7 +6062,7 @@ void Sema::DiagnoseAbstractType(const CXXRecordDecl *RD) {
       if (SO->second.size() != 1)
         continue;
 
-      if (!SO->second.front().Method->isPure())
+      if (!SO->second.front().Method->isPureVirtual())
         continue;
 
       if (!SeenPureMethods.insert(SO->second.front().Method).second)
@@ -6545,8 +6545,7 @@ void Sema::checkClassLevelDLLAttribute(CXXRecordDecl *Class) {
   if ((Context.getTargetInfo().getCXXABI().isMicrosoft() ||
        Context.getTargetInfo().getTriple().isPS()) &&
       (!Class->isExternallyVisible() && Class->hasExternalFormalLinkage())) {
-    Class->dropAttr();
-    Class->dropAttr();
+    Class->dropAttrs();
     return;
   }
 
@@ -11322,9 +11321,20 @@ Decl *Sema::ActOnConversionDeclarator(CXXConversionDecl *Conversion) {
       << ClassType << ConvType;
   }
 
-  if (FunctionTemplateDecl *ConversionTemplate
-                                = Conversion->getDescribedFunctionTemplate())
+  if (FunctionTemplateDecl *ConversionTemplate =
+          Conversion->getDescribedFunctionTemplate()) {
+    if (const auto *ConvTypePtr = ConvType->getAs()) {
+      ConvType = ConvTypePtr->getPointeeType();
+    }
+    if (ConvType->isUndeducedAutoType()) {
+      Diag(Conversion->getTypeSpecStartLoc(), diag::err_auto_not_allowed)
+          << getReturnTypeLoc(Conversion).getSourceRange()
+          << llvm::to_underlying(ConvType->getAs()->getKeyword())
+          << /* in declaration of conversion function template= */ 24;
+    }
+
     return ConversionTemplate;
+  }
 
   return Conversion;
 }
@@ -17011,7 +17021,7 @@ VarDecl *Sema::BuildExceptionDeclaration(Scope *S,
 /// ActOnExceptionDeclarator - Parsed the exception-declarator in a C++ catch
 /// handler.
 Decl *Sema::ActOnExceptionDeclarator(Scope *S, Declarator &D) {
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
   bool Invalid = D.isInvalidType();
 
   // Check for unexpanded parameter packs.
@@ -17762,7 +17772,7 @@ Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
   // for a TUK_Friend.
   Declarator TheDeclarator(DS, ParsedAttributesView::none(),
                            DeclaratorContext::Member);
-  TypeSourceInfo *TSI = GetTypeForDeclarator(TheDeclarator, S);
+  TypeSourceInfo *TSI = GetTypeForDeclarator(TheDeclarator);
   QualType T = TSI->getType();
   if (TheDeclarator.isInvalidType())
     return nullptr;
@@ -17827,7 +17837,7 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D,
   assert(DS.getStorageClassSpec() == DeclSpec::SCS_unspecified);
 
   SourceLocation Loc = D.getIdentifierLoc();
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
 
   // C++ [class.friend]p1
   //   A friend of a class is a function or class....
@@ -18328,9 +18338,7 @@ bool Sema::CheckOverridingFunctionAttributes(const CXXMethodDecl *New,
   }
 
   // SME attributes must match when overriding a function declaration.
-  if (IsInvalidSMECallConversion(
-          Old->getType(), New->getType(),
-          AArch64SMECallConversionKind::MayAddPreservesZA)) {
+  if (IsInvalidSMECallConversion(Old->getType(), New->getType())) {
     Diag(New->getLocation(), diag::err_conflicting_overriding_attributes)
         << New << New->getType() << Old->getType();
     Diag(Old->getLocation(), diag::note_overridden_virtual_function);
@@ -18498,7 +18506,7 @@ bool Sema::CheckPureMethod(CXXMethodDecl *Method, SourceRange InitRange) {
     Method->setRangeEnd(EndLoc);
 
   if (Method->isVirtual() || Method->getParent()->isDependentContext()) {
-    Method->setPure();
+    Method->setIsPureVirtual();
     return false;
   }
 
@@ -18769,7 +18777,7 @@ bool Sema::DefineUsedVTables() {
 void Sema::MarkVirtualMemberExceptionSpecsNeeded(SourceLocation Loc,
                                                  const CXXRecordDecl *RD) {
   for (const auto *I : RD->methods())
-    if (I->isVirtual() && !I->isPure())
+    if (I->isVirtual() && !I->isPureVirtual())
       ResolveExceptionSpec(Loc, I->getType()->castAs());
 }
 
@@ -18790,7 +18798,8 @@ void Sema::MarkVirtualMembersReferenced(SourceLocation Loc,
 
       // C++ [basic.def.odr]p2:
       //   [...] A virtual member function is used if it is not pure. [...]
-      if (!Overrider->isPure() && (!ConstexprOnly || Overrider->isConstexpr()))
+      if (!Overrider->isPureVirtual() &&
+          (!ConstexprOnly || Overrider->isConstexpr()))
         MarkFunctionReferenced(Loc, Overrider);
     }
   }
@@ -19184,7 +19193,7 @@ MSPropertyDecl *Sema::HandleMSProperty(Scope *S, RecordDecl *Record,
   }
   SourceLocation Loc = D.getIdentifierLoc();
 
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
   QualType T = TInfo->getType();
   if (getLangOpts().CPlusPlus) {
     CheckExtraCXXDefaultArguments(D);
diff --git a/clang/lib/Sema/SemaDeclObjC.cpp b/clang/lib/Sema/SemaDeclObjC.cpp
index c3b95e168a60..bb0d0cd2030b 100644
--- a/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/clang/lib/Sema/SemaDeclObjC.cpp
@@ -1623,7 +1623,7 @@ void Sema::actOnObjCTypeArgsOrProtocolQualifiers(
     }
 
     // Convert this to a type.
-    return ActOnTypeName(S, D);
+    return ActOnTypeName(D);
   };
 
   // Local function that updates the declaration specifiers with
@@ -5211,7 +5211,7 @@ Decl *Sema::ActOnObjCExceptionDecl(Scope *S, Declarator &D) {
   if (getLangOpts().CPlusPlus)
     CheckExtraCXXDefaultArguments(D);
 
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
   QualType ExceptionType = TInfo->getType();
 
   VarDecl *New = BuildObjCExceptionDecl(TInfo, ExceptionType,
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 1dbb11ec53f4..e050feb9210e 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -2472,7 +2472,8 @@ bool Sema::DiagnoseDependentMemberLookup(const LookupResult &R) {
 bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
                                CorrectionCandidateCallback &CCC,
                                TemplateArgumentListInfo *ExplicitTemplateArgs,
-                               ArrayRef Args, TypoExpr **Out) {
+                               ArrayRef Args, DeclContext *LookupCtx,
+                               TypoExpr **Out) {
   DeclarationName Name = R.getLookupName();
 
   unsigned diagnostic = diag::err_undeclared_var_use;
@@ -2488,7 +2489,8 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
   // unqualified lookup.  This is useful when (for example) the
   // original lookup would not have found something because it was a
   // dependent name.
-  DeclContext *DC = SS.isEmpty() ? CurContext : nullptr;
+  DeclContext *DC =
+      LookupCtx ? LookupCtx : (SS.isEmpty() ? CurContext : nullptr);
   while (DC) {
     if (isa(DC)) {
       LookupQualifiedName(R, DC);
@@ -2531,12 +2533,12 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
           emitEmptyLookupTypoDiagnostic(TC, *this, SS, Name, TypoLoc, Args,
                                         diagnostic, diagnostic_suggest);
         },
-        nullptr, CTK_ErrorRecovery);
+        nullptr, CTK_ErrorRecovery, LookupCtx);
     if (*Out)
       return true;
-  } else if (S &&
-             (Corrected = CorrectTypo(R.getLookupNameInfo(), R.getLookupKind(),
-                                      S, &SS, CCC, CTK_ErrorRecovery))) {
+  } else if (S && (Corrected =
+                       CorrectTypo(R.getLookupNameInfo(), R.getLookupKind(), S,
+                                   &SS, CCC, CTK_ErrorRecovery, LookupCtx))) {
     std::string CorrectedStr(Corrected.getAsString(getLangOpts()));
     bool DroppedSpecifier =
         Corrected.WillReplaceSpecifier() && Name.getAsString() == CorrectedStr;
@@ -2826,7 +2828,7 @@ Sema::ActOnIdExpression(Scope *S, CXXScopeSpec &SS,
     // a template name, but we happen to have always already looked up the name
     // before we get here if it must be a template name.
     if (DiagnoseEmptyLookup(S, SS, R, CCC ? *CCC : DefaultValidator, nullptr,
-                            std::nullopt, &TE)) {
+                            std::nullopt, nullptr, &TE)) {
       if (TE && KeywordReplacement) {
         auto &State = getTypoExprState(TE);
         auto BestTC = State.Consumer->getNextCorrection();
@@ -5059,8 +5061,6 @@ static QualType getDependentArraySubscriptType(Expr *LHS, Expr *RHS,
   return Result->isDependentType() ? Result : Ctx.DependentTy;
 }
 
-static bool checkArgsForPlaceholders(Sema &S, MultiExprArg args);
-
 ExprResult Sema::ActOnArraySubscriptExpr(Scope *S, Expr *base,
                                          SourceLocation lbLoc,
                                          MultiExprArg ArgExprs,
@@ -5164,7 +5164,7 @@ ExprResult Sema::ActOnArraySubscriptExpr(Scope *S, Expr *base,
       return ExprError();
     ArgExprs[0] = result.get();
   } else {
-    if (checkArgsForPlaceholders(*this, ArgExprs))
+    if (CheckArgsForPlaceholders(ArgExprs))
       return ExprError();
   }
 
@@ -6916,15 +6916,13 @@ static bool isPlaceholderToRemoveAsArg(QualType type) {
   llvm_unreachable("bad builtin type kind");
 }
 
-/// Check an argument list for placeholders that we won't try to
-/// handle later.
-static bool checkArgsForPlaceholders(Sema &S, MultiExprArg args) {
+bool Sema::CheckArgsForPlaceholders(MultiExprArg args) {
   // Apply this processing to all the arguments at once instead of
   // dying at the first failure.
   bool hasInvalid = false;
   for (size_t i = 0, e = args.size(); i != e; i++) {
     if (isPlaceholderToRemoveAsArg(args[i]->getType())) {
-      ExprResult result = S.CheckPlaceholderExpr(args[i]);
+      ExprResult result = CheckPlaceholderExpr(args[i]);
       if (result.isInvalid()) hasInvalid = true;
       else args[i] = result.get();
     }
@@ -7198,7 +7196,7 @@ ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
   if (Result.isInvalid()) return ExprError();
   Fn = Result.get();
 
-  if (checkArgsForPlaceholders(*this, ArgExprs))
+  if (CheckArgsForPlaceholders(ArgExprs))
     return ExprError();
 
   if (getLangOpts().CPlusPlus) {
@@ -8697,10 +8695,10 @@ ExprResult Sema::ActOnParenListExpr(SourceLocation L,
 /// Emit a specialized diagnostic when one expression is a null pointer
 /// constant and the other is not a pointer.  Returns true if a diagnostic is
 /// emitted.
-bool Sema::DiagnoseConditionalForNull(Expr *LHSExpr, Expr *RHSExpr,
+bool Sema::DiagnoseConditionalForNull(const Expr *LHSExpr, const Expr *RHSExpr,
                                       SourceLocation QuestionLoc) {
-  Expr *NullExpr = LHSExpr;
-  Expr *NonPointerExpr = RHSExpr;
+  const Expr *NullExpr = LHSExpr;
+  const Expr *NonPointerExpr = RHSExpr;
   Expr::NullPointerConstantKind NullKind =
       NullExpr->isNullPointerConstant(Context,
                                       Expr::NPC_ValueDependentIsNotNull);
@@ -8736,7 +8734,8 @@ bool Sema::DiagnoseConditionalForNull(Expr *LHSExpr, Expr *RHSExpr,
 }
 
 /// Return false if the condition expression is valid, true otherwise.
-static bool checkCondition(Sema &S, Expr *Cond, SourceLocation QuestionLoc) {
+static bool checkCondition(Sema &S, const Expr *Cond,
+                           SourceLocation QuestionLoc) {
   QualType CondTy = Cond->getType();
 
   // OpenCL v1.1 s6.3.i says the condition cannot be a floating point type.
@@ -9548,28 +9547,27 @@ static bool IsArithmeticOp(BinaryOperatorKind Opc) {
 /// expression, either using a built-in or overloaded operator,
 /// and sets *OpCode to the opcode and *RHSExprs to the right-hand side
 /// expression.
-static bool IsArithmeticBinaryExpr(Expr *E, BinaryOperatorKind *Opcode,
-                                   Expr **RHSExprs) {
+static bool IsArithmeticBinaryExpr(const Expr *E, BinaryOperatorKind *Opcode,
+                                   const Expr **RHSExprs) {
   // Don't strip parenthesis: we should not warn if E is in parenthesis.
   E = E->IgnoreImpCasts();
   E = E->IgnoreConversionOperatorSingleStep();
   E = E->IgnoreImpCasts();
-  if (auto *MTE = dyn_cast(E)) {
+  if (const auto *MTE = dyn_cast(E)) {
     E = MTE->getSubExpr();
     E = E->IgnoreImpCasts();
   }
 
   // Built-in binary operator.
-  if (BinaryOperator *OP = dyn_cast(E)) {
-    if (IsArithmeticOp(OP->getOpcode())) {
-      *Opcode = OP->getOpcode();
-      *RHSExprs = OP->getRHS();
-      return true;
-    }
+  if (const auto *OP = dyn_cast(E);
+      OP && IsArithmeticOp(OP->getOpcode())) {
+    *Opcode = OP->getOpcode();
+    *RHSExprs = OP->getRHS();
+    return true;
   }
 
   // Overloaded operator.
-  if (CXXOperatorCallExpr *Call = dyn_cast(E)) {
+  if (const auto *Call = dyn_cast(E)) {
     if (Call->getNumArgs() != 2)
       return false;
 
@@ -9594,14 +9592,14 @@ static bool IsArithmeticBinaryExpr(Expr *E, BinaryOperatorKind *Opcode,
 /// ExprLooksBoolean - Returns true if E looks boolean, i.e. it has boolean type
 /// or is a logical expression such as (x==y) which has int type, but is
 /// commonly interpreted as boolean.
-static bool ExprLooksBoolean(Expr *E) {
+static bool ExprLooksBoolean(const Expr *E) {
   E = E->IgnoreParenImpCasts();
 
   if (E->getType()->isBooleanType())
     return true;
-  if (BinaryOperator *OP = dyn_cast(E))
+  if (const auto *OP = dyn_cast(E))
     return OP->isComparisonOp() || OP->isLogicalOp();
-  if (UnaryOperator *OP = dyn_cast(E))
+  if (const auto *OP = dyn_cast(E))
     return OP->getOpcode() == UO_LNot;
   if (E->getType()->isPointerType())
     return true;
@@ -9615,13 +9613,11 @@ static bool ExprLooksBoolean(Expr *E) {
 /// and binary operator are mixed in a way that suggests the programmer assumed
 /// the conditional operator has higher precedence, for example:
 /// "int x = a + someBinaryCondition ? 1 : 2".
-static void DiagnoseConditionalPrecedence(Sema &Self,
-                                          SourceLocation OpLoc,
-                                          Expr *Condition,
-                                          Expr *LHSExpr,
-                                          Expr *RHSExpr) {
+static void DiagnoseConditionalPrecedence(Sema &Self, SourceLocation OpLoc,
+                                          Expr *Condition, const Expr *LHSExpr,
+                                          const Expr *RHSExpr) {
   BinaryOperatorKind CondOpcode;
-  Expr *CondRHS;
+  const Expr *CondRHS;
 
   if (!IsArithmeticBinaryExpr(Condition, &CondOpcode, &CondRHS))
     return;
@@ -9809,8 +9805,7 @@ ExprResult Sema::ActOnConditionalOp(SourceLocation QuestionLoc,
 }
 
 // Check that the SME attributes for PSTATE.ZA and PSTATE.SM are compatible.
-bool Sema::IsInvalidSMECallConversion(QualType FromType, QualType ToType,
-                                      AArch64SMECallConversionKind C) {
+bool Sema::IsInvalidSMECallConversion(QualType FromType, QualType ToType) {
   unsigned FromAttributes = 0, ToAttributes = 0;
   if (const auto *FromFn =
           dyn_cast(Context.getCanonicalType(FromType)))
@@ -9821,25 +9816,7 @@ bool Sema::IsInvalidSMECallConversion(QualType FromType, QualType ToType,
     ToAttributes =
         ToFn->getAArch64SMEAttributes() & FunctionType::SME_AttributeMask;
 
-  if (FromAttributes == ToAttributes)
-    return false;
-
-  // If the '__arm_preserves_za' is the only difference between the types,
-  // check whether we're allowed to add or remove it.
-  if ((FromAttributes ^ ToAttributes) ==
-      FunctionType::SME_PStateZAPreservedMask) {
-    switch (C) {
-    case AArch64SMECallConversionKind::MatchExactly:
-      return true;
-    case AArch64SMECallConversionKind::MayAddPreservesZA:
-      return !(ToAttributes & FunctionType::SME_PStateZAPreservedMask);
-    case AArch64SMECallConversionKind::MayDropPreservesZA:
-      return !(FromAttributes & FunctionType::SME_PStateZAPreservedMask);
-    }
-  }
-
-  // There has been a mismatch of attributes
-  return true;
+  return FromAttributes != ToAttributes;
 }
 
 // Check if we have a conversion between incompatible cmse function pointer
@@ -10008,9 +9985,7 @@ checkPointerTypesForAssignment(Sema &S, QualType LHSType, QualType RHSType,
     return Sema::IncompatibleFunctionPointer;
   if (IsInvalidCmseNSCallConversion(S, ltrans, rtrans))
     return Sema::IncompatibleFunctionPointer;
-  if (S.IsInvalidSMECallConversion(
-          rtrans, ltrans,
-          Sema::AArch64SMECallConversionKind::MayDropPreservesZA))
+  if (S.IsInvalidSMECallConversion(rtrans, ltrans))
     return Sema::IncompatibleFunctionPointer;
   return ConvTy;
 }
@@ -17001,7 +16976,7 @@ void Sema::ActOnBlockArguments(SourceLocation CaretLoc, Declarator &ParamInfo,
   assert(ParamInfo.getContext() == DeclaratorContext::BlockLiteral);
   BlockScopeInfo *CurBlock = getCurBlock();
 
-  TypeSourceInfo *Sig = GetTypeForDeclarator(ParamInfo, CurScope);
+  TypeSourceInfo *Sig = GetTypeForDeclarator(ParamInfo);
   QualType T = Sig->getType();
 
   // FIXME: We should allow unexpanded parameter packs here, but that would,
@@ -20800,7 +20775,7 @@ void Sema::MarkMemberReferenced(MemberExpr *E) {
   bool MightBeOdrUse = true;
   if (E->performsVirtualDispatch(getLangOpts())) {
     if (CXXMethodDecl *Method = dyn_cast(E->getMemberDecl()))
-      if (Method->isPure())
+      if (Method->isPureVirtual())
         MightBeOdrUse = false;
   }
   SourceLocation Loc =
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 4ae04358d5df..953bfe484a52 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -1931,7 +1931,7 @@ Sema::ActOnCXXNew(SourceLocation StartLoc, bool UseGlobal,
     }
   }
 
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, /*Scope=*/nullptr);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
   QualType AllocType = TInfo->getType();
   if (D.isInvalidType())
     return ExprError();
@@ -1947,17 +1947,17 @@ Sema::ActOnCXXNew(SourceLocation StartLoc, bool UseGlobal,
 }
 
 static bool isLegalArrayNewInitializer(CXXNewInitializationStyle Style,
-                                       Expr *Init) {
+                                       Expr *Init, bool IsCPlusPlus20) {
   if (!Init)
     return true;
   if (ParenListExpr *PLE = dyn_cast(Init))
-    return PLE->getNumExprs() == 0;
+    return IsCPlusPlus20 || PLE->getNumExprs() == 0;
   if (isa(Init))
     return true;
   else if (CXXConstructExpr *CCE = dyn_cast(Init))
     return !CCE->isListInitialization() &&
            CCE->getConstructor()->isDefaultConstructor();
-  else if (Style == CXXNewInitializationStyle::List) {
+  else if (Style == CXXNewInitializationStyle::Braces) {
     assert(isa(Init) &&
            "Shouldn't create list CXXConstructExprs for arrays.");
     return true;
@@ -2011,9 +2011,9 @@ ExprResult Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
   CXXNewInitializationStyle InitStyle;
   if (DirectInitRange.isValid()) {
     assert(Initializer && "Have parens but no initializer.");
-    InitStyle = CXXNewInitializationStyle::Call;
+    InitStyle = CXXNewInitializationStyle::Parens;
   } else if (Initializer && isa(Initializer))
-    InitStyle = CXXNewInitializationStyle::List;
+    InitStyle = CXXNewInitializationStyle::Braces;
   else {
     assert((!Initializer || isa(Initializer) ||
             isa(Initializer)) &&
@@ -2023,7 +2023,7 @@ ExprResult Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
 
   MultiExprArg Exprs(&Initializer, Initializer ? 1 : 0);
   if (ParenListExpr *List = dyn_cast_or_null(Initializer)) {
-    assert(InitStyle == CXXNewInitializationStyle::Call &&
+    assert(InitStyle == CXXNewInitializationStyle::Parens &&
            "paren init for non-call init");
     Exprs = MultiExprArg(List->getExprs(), List->getNumExprs());
   }
@@ -2037,15 +2037,14 @@ ExprResult Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
     //       initialized (8.5); if no initialization is performed,
     //       the object has indeterminate value
     case CXXNewInitializationStyle::None:
-    case CXXNewInitializationStyle::Implicit:
       return InitializationKind::CreateDefault(TypeRange.getBegin());
     //     - Otherwise, the new-initializer is interpreted according to the
     //       initialization rules of 8.5 for direct-initialization.
-    case CXXNewInitializationStyle::Call:
+    case CXXNewInitializationStyle::Parens:
       return InitializationKind::CreateDirect(TypeRange.getBegin(),
                                               DirectInitRange.getBegin(),
                                               DirectInitRange.getEnd());
-    case CXXNewInitializationStyle::List:
+    case CXXNewInitializationStyle::Braces:
       return InitializationKind::CreateDirectList(TypeRange.getBegin(),
                                                   Initializer->getBeginLoc(),
                                                   Initializer->getEndLoc());
@@ -2072,14 +2071,13 @@ ExprResult Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
       return ExprError();
   } else if (Deduced && !Deduced->isDeduced()) {
     MultiExprArg Inits = Exprs;
-    bool Braced = (InitStyle == CXXNewInitializationStyle::List);
+    bool Braced = (InitStyle == CXXNewInitializationStyle::Braces);
     if (Braced) {
       auto *ILE = cast(Exprs[0]);
       Inits = MultiExprArg(ILE->getInits(), ILE->getNumInits());
     }
 
-    if (InitStyle == CXXNewInitializationStyle::None ||
-        InitStyle == CXXNewInitializationStyle::Implicit || Inits.empty())
+    if (InitStyle == CXXNewInitializationStyle::None || Inits.empty())
       return ExprError(Diag(StartLoc, diag::err_auto_new_requires_ctor_arg)
                        << AllocType << TypeRange);
     if (Inits.size() > 1) {
@@ -2286,6 +2284,9 @@ ExprResult Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
   bool PassAlignment = getLangOpts().AlignedAllocation &&
                        Alignment > NewAlignment;
 
+  if (CheckArgsForPlaceholders(PlacementArgs))
+    return ExprError();
+
   AllocationFunctionScope Scope = UseGlobal ? AFS_Global : AFS_Both;
   if (!AllocType->isDependentType() &&
       !Expr::hasAnyTypeDependentArguments(PlacementArgs) &&
@@ -2403,7 +2404,8 @@ ExprResult Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
   // Array 'new' can't have any initializers except empty parentheses.
   // Initializer lists are also allowed, in C++11. Rely on the parser for the
   // dialect distinction.
-  if (ArraySize && !isLegalArrayNewInitializer(InitStyle, Initializer)) {
+  if (ArraySize && !isLegalArrayNewInitializer(InitStyle, Initializer,
+                                               getLangOpts().CPlusPlus20)) {
     SourceRange InitRange(Exprs.front()->getBeginLoc(),
                           Exprs.back()->getEndLoc());
     Diag(StartLoc, diag::err_new_array_init_args) << InitRange;
@@ -2443,14 +2445,6 @@ ExprResult Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
       FullInit = Binder->getSubExpr();
 
     Initializer = FullInit.get();
-    // We don't know that we're generating an implicit initializer until now, so
-    // we have to update the initialization style as well.
-    //
-    // FIXME: it would be nice to determine the correct initialization style
-    // earlier so InitStyle doesn't need adjusting.
-    if (InitStyle == CXXNewInitializationStyle::None && Initializer) {
-      InitStyle = CXXNewInitializationStyle::Implicit;
-    }
 
     // FIXME: If we have a KnownArraySize, check that the array bound of the
     // initializer is no greater than that constant value.
@@ -3222,10 +3216,13 @@ void Sema::DeclareGlobalAllocationFunction(DeclarationName Name,
       Alloc->setLocalOwningModule(TheGlobalModuleFragment);
     }
 
-    Alloc->addAttr(VisibilityAttr::CreateImplicit(
-        Context, LangOpts.GlobalAllocationFunctionVisibilityHidden
-                     ? VisibilityAttr::Hidden
-                     : VisibilityAttr::Default));
+    if (LangOpts.hasGlobalAllocationFunctionVisibility())
+      Alloc->addAttr(VisibilityAttr::CreateImplicit(
+          Context, LangOpts.hasHiddenGlobalAllocationFunctionVisibility()
+                       ? VisibilityAttr::Hidden
+                   : LangOpts.hasProtectedGlobalAllocationFunctionVisibility()
+                       ? VisibilityAttr::Protected
+                       : VisibilityAttr::Default));
 
     llvm::SmallVector ParamDecls;
     for (QualType T : Params) {
diff --git a/clang/lib/Sema/SemaExprMember.cpp b/clang/lib/Sema/SemaExprMember.cpp
index 2abec3d86a27..32998ae60eaf 100644
--- a/clang/lib/Sema/SemaExprMember.cpp
+++ b/clang/lib/Sema/SemaExprMember.cpp
@@ -782,7 +782,8 @@ Sema::BuildMemberReferenceExpr(Expr *Base, QualType BaseType,
                                const Scope *S,
                                ActOnMemberAccessExtraArgs *ExtraArgs) {
   if (BaseType->isDependentType() ||
-      (SS.isSet() && isDependentScopeSpecifier(SS)))
+      (SS.isSet() && isDependentScopeSpecifier(SS)) ||
+      NameInfo.getName().isDependentName())
     return ActOnDependentMemberExpr(Base, BaseType,
                                     IsArrow, OpLoc,
                                     SS, TemplateKWLoc, FirstQualifierInScope,
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 61d244f3bb97..91e4cb7b68a2 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -12,6 +12,7 @@
 
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclObjC.h"
+#include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/ExprOpenMP.h"
@@ -4199,7 +4200,7 @@ static OverloadingResult ResolveConstructorOverload(
 /// \param IsListInit     Is this list-initialization?
 /// \param IsInitListCopy Is this non-list-initialization resulting from a
 ///                       list-initialization from {x} where x is the same
-///                       type as the entity?
+///                       aggregate type as the entity?
 static void TryConstructorInitialization(Sema &S,
                                          const InitializedEntity &Entity,
                                          const InitializationKind &Kind,
@@ -4229,6 +4230,14 @@ static void TryConstructorInitialization(Sema &S,
         Entity.getKind() !=
             InitializedEntity::EK_LambdaToBlockConversionBlockElement);
 
+  bool CopyElisionPossible = false;
+  auto ElideConstructor = [&] {
+    // Convert qualifications if necessary.
+    Sequence.AddQualificationConversionStep(DestType, VK_PRValue);
+    if (ILE)
+      Sequence.RewrapReferenceInitList(DestType, ILE);
+  };
+
   // C++17 [dcl.init]p17:
   //     - If the initializer expression is a prvalue and the cv-unqualified
   //       version of the source type is the same class as the class of the
@@ -4241,11 +4250,17 @@ static void TryConstructorInitialization(Sema &S,
   if (S.getLangOpts().CPlusPlus17 && !RequireActualConstructor &&
       UnwrappedArgs.size() == 1 && UnwrappedArgs[0]->isPRValue() &&
       S.Context.hasSameUnqualifiedType(UnwrappedArgs[0]->getType(), DestType)) {
-    // Convert qualifications if necessary.
-    Sequence.AddQualificationConversionStep(DestType, VK_PRValue);
-    if (ILE)
-      Sequence.RewrapReferenceInitList(DestType, ILE);
-    return;
+    if (ILE && !DestType->isAggregateType()) {
+      // CWG2311: T{ prvalue_of_type_T } is not eligible for copy elision
+      // Make this an elision if this won't call an initializer-list
+      // constructor. (Always on an aggregate type or check constructors first.)
+      assert(!IsInitListCopy &&
+             "IsInitListCopy only possible with aggregate types");
+      CopyElisionPossible = true;
+    } else {
+      ElideConstructor();
+      return;
+    }
   }
 
   const RecordType *DestRecordType = DestType->getAs();
@@ -4290,6 +4305,12 @@ static void TryConstructorInitialization(Sema &S,
           S, Kind.getLocation(), Args, CandidateSet, DestType, Ctors, Best,
           CopyInitialization, AllowExplicit,
           /*OnlyListConstructors=*/true, IsListInit, RequireActualConstructor);
+
+    if (CopyElisionPossible && Result == OR_No_Viable_Function) {
+      // No initializer list candidate
+      ElideConstructor();
+      return;
+    }
   }
 
   // C++11 [over.match.list]p1:
@@ -4571,9 +4592,9 @@ static void TryListInitialization(Sema &S,
     return;
   }
 
-  // C++11 [dcl.init.list]p3, per DR1467:
-  // - If T is a class type and the initializer list has a single element of
-  //   type cv U, where U is T or a class derived from T, the object is
+  // C++11 [dcl.init.list]p3, per DR1467 and DR2137:
+  // - If T is an aggregate class and the initializer list has a single element
+  //   of type cv U, where U is T or a class derived from T, the object is
   //   initialized from that element (by copy-initialization for
   //   copy-list-initialization, or by direct-initialization for
   //   direct-list-initialization).
@@ -4584,7 +4605,7 @@ static void TryListInitialization(Sema &S,
   // - Otherwise, if T is an aggregate, [...] (continue below).
   if (S.getLangOpts().CPlusPlus11 && InitList->getNumInits() == 1 &&
       !IsDesignatedInit) {
-    if (DestType->isRecordType()) {
+    if (DestType->isRecordType() && DestType->isAggregateType()) {
       QualType InitType = InitList->getInit(0)->getType();
       if (S.Context.hasSameUnqualifiedType(InitType, DestType) ||
           S.IsDerivedFrom(InitList->getBeginLoc(), InitType, DestType)) {
@@ -5495,7 +5516,7 @@ static void TryOrBuildParenListInitialization(
         return;
     }
     //   ...and value-initialized for each k < i <= n;
-    if (ArrayLength > Args.size()) {
+    if (ArrayLength > Args.size() || Entity.isVariableLengthArrayNew()) {
       InitializedEntity SubEntity = InitializedEntity::InitializeElement(
           S.getASTContext(), Args.size(), Entity);
       InitializationKind SubKind = InitializationKind::CreateValue(
@@ -5512,6 +5533,14 @@ static void TryOrBuildParenListInitialization(
   } else if (auto *RT = Entity.getType()->getAs()) {
     bool IsUnion = RT->isUnionType();
     const CXXRecordDecl *RD = cast(RT->getDecl());
+    if (RD->isInvalidDecl()) {
+      // Exit early to avoid confusion when processing members.
+      // We do the same for braced list initialization in
+      // `CheckStructUnionTypes`.
+      Sequence.SetFailed(
+          clang::InitializationSequence::FK_ParenthesizedListInitFailed);
+      return;
+    }
 
     if (!IsUnion) {
       for (const CXXBaseSpecifier &Base : RD->bases()) {
@@ -7575,14 +7604,27 @@ static void visitLifetimeBoundArguments(IndirectLocalPath &Path, Expr *Call,
     Path.pop_back();
   };
 
-  if (ObjectArg && implicitObjectParamIsLifetimeBound(Callee))
-    VisitLifetimeBoundArg(Callee, ObjectArg);
-
   bool CheckCoroCall = false;
   if (const auto *RD = Callee->getReturnType()->getAsRecordDecl()) {
     CheckCoroCall = RD->hasAttr() &&
-                    RD->hasAttr();
+                    RD->hasAttr() &&
+                    !Callee->hasAttr();
   }
+
+  if (ObjectArg) {
+    bool CheckCoroObjArg = CheckCoroCall;
+    // Coroutine lambda objects with empty capture list are not lifetimebound.
+    if (auto *LE = dyn_cast(ObjectArg->IgnoreImplicit());
+        LE && LE->captures().empty())
+      CheckCoroObjArg = false;
+    // Allow `get_return_object()` as the object param (__promise) is not
+    // lifetimebound.
+    if (Sema::CanBeGetReturnObject(Callee))
+      CheckCoroObjArg = false;
+    if (implicitObjectParamIsLifetimeBound(Callee) || CheckCoroObjArg)
+      VisitLifetimeBoundArg(Callee, ObjectArg);
+  }
+
   for (unsigned I = 0,
                 N = std::min(Callee->getNumParams(), Args.size());
        I != N; ++I) {
@@ -10368,11 +10410,6 @@ void InitializationSequence::dump() const {
   dump(llvm::errs());
 }
 
-static bool NarrowingErrs(const LangOptions &L) {
-  return L.CPlusPlus11 &&
-         (!L.MicrosoftExt || L.isCompatibleWithMSVC(LangOptions::MSVC2015));
-}
-
 static void DiagnoseNarrowingInInitList(Sema &S,
                                         const ImplicitConversionSequence &ICS,
                                         QualType PreNarrowingType,
@@ -10393,6 +10430,19 @@ static void DiagnoseNarrowingInInitList(Sema &S,
     return;
   }
 
+  auto MakeDiag = [&](bool IsConstRef, unsigned DefaultDiagID,
+                      unsigned ConstRefDiagID, unsigned WarnDiagID) {
+    unsigned DiagID;
+    auto &L = S.getLangOpts();
+    if (L.CPlusPlus11 &&
+        (!L.MicrosoftExt || L.isCompatibleWithMSVC(LangOptions::MSVC2015)))
+      DiagID = IsConstRef ? ConstRefDiagID : DefaultDiagID;
+    else
+      DiagID = WarnDiagID;
+    return S.Diag(PostInit->getBeginLoc(), DiagID)
+           << PostInit->getSourceRange();
+  };
+
   // C++11 [dcl.init.list]p7: Check whether this is a narrowing conversion.
   APValue ConstantValue;
   QualType ConstantType;
@@ -10408,13 +10458,9 @@ static void DiagnoseNarrowingInInitList(Sema &S,
     // narrowing conversion even if the value is a constant and can be
     // represented exactly as an integer.
     QualType T = EntityType.getNonReferenceType();
-    S.Diag(PostInit->getBeginLoc(),
-           NarrowingErrs(S.getLangOpts())
-               ? (T == EntityType
-                      ? diag::ext_init_list_type_narrowing
-                      : diag::ext_init_list_type_narrowing_const_reference)
-               : diag::warn_init_list_type_narrowing)
-        << PostInit->getSourceRange()
+    MakeDiag(T != EntityType, diag::ext_init_list_type_narrowing,
+             diag::ext_init_list_type_narrowing_const_reference,
+             diag::warn_init_list_type_narrowing)
         << PreNarrowingType.getLocalUnqualifiedType()
         << T.getLocalUnqualifiedType();
     break;
@@ -10422,14 +10468,10 @@ static void DiagnoseNarrowingInInitList(Sema &S,
 
   case NK_Constant_Narrowing: {
     // A constant value was narrowed.
-    QualType T = EntityType.getNonReferenceType();
-    S.Diag(PostInit->getBeginLoc(),
-           NarrowingErrs(S.getLangOpts())
-               ? (T == EntityType
-                      ? diag::ext_init_list_constant_narrowing
-                      : diag::ext_init_list_constant_narrowing_const_reference)
-               : diag::warn_init_list_constant_narrowing)
-        << PostInit->getSourceRange()
+    MakeDiag(EntityType.getNonReferenceType() != EntityType,
+             diag::ext_init_list_constant_narrowing,
+             diag::ext_init_list_constant_narrowing_const_reference,
+             diag::warn_init_list_constant_narrowing)
         << ConstantValue.getAsString(S.getASTContext(), ConstantType)
         << EntityType.getNonReferenceType().getLocalUnqualifiedType();
     break;
@@ -10437,14 +10479,10 @@ static void DiagnoseNarrowingInInitList(Sema &S,
 
   case NK_Variable_Narrowing: {
     // A variable's value may have been narrowed.
-    QualType T = EntityType.getNonReferenceType();
-    S.Diag(PostInit->getBeginLoc(),
-           NarrowingErrs(S.getLangOpts())
-               ? (T == EntityType
-                      ? diag::ext_init_list_variable_narrowing
-                      : diag::ext_init_list_variable_narrowing_const_reference)
-               : diag::warn_init_list_variable_narrowing)
-        << PostInit->getSourceRange()
+    MakeDiag(EntityType.getNonReferenceType() != EntityType,
+             diag::ext_init_list_variable_narrowing,
+             diag::ext_init_list_variable_narrowing_const_reference,
+             diag::warn_init_list_variable_narrowing)
         << PreNarrowingType.getLocalUnqualifiedType()
         << EntityType.getNonReferenceType().getLocalUnqualifiedType();
     break;
@@ -10713,7 +10751,14 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
     bool HasAnyDeductionGuide = false;
 
     auto SynthesizeAggrGuide = [&](InitListExpr *ListInit) {
-      auto *RD = cast(Template->getTemplatedDecl());
+      auto *Pattern = Template;
+      while (Pattern->getInstantiatedFromMemberTemplate()) {
+        if (Pattern->isMemberSpecialization())
+          break;
+        Pattern = Pattern->getInstantiatedFromMemberTemplate();
+      }
+
+      auto *RD = cast(Pattern->getTemplatedDecl());
       if (!(RD->getDefinition() && RD->isAggregate()))
         return;
       QualType Ty = Context.getRecordType(RD);
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index e7b6443c984c..5b95bae567b7 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -916,7 +916,7 @@ static TypeSourceInfo *getLambdaType(Sema &S, LambdaIntroducer &Intro,
       }
     }
 
-    MethodTyInfo = S.GetTypeForDeclarator(ParamInfo, CurScope);
+    MethodTyInfo = S.GetTypeForDeclarator(ParamInfo);
     assert(MethodTyInfo && "no type from lambda-declarator");
 
     // Check for unexpanded parameter packs in the method type.
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index 996f8b57233b..02b1a045df44 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -2982,6 +2982,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result,
     case TemplateArgument::Integral:
     case TemplateArgument::Expression:
     case TemplateArgument::NullPtr:
+    case TemplateArgument::StructuralValue:
       // [Note: non-type template arguments do not contribute to the set of
       //  associated namespaces. ]
       break;
diff --git a/clang/lib/Sema/SemaObjCProperty.cpp b/clang/lib/Sema/SemaObjCProperty.cpp
index 22540af1cda8..349c7fc9c91b 100644
--- a/clang/lib/Sema/SemaObjCProperty.cpp
+++ b/clang/lib/Sema/SemaObjCProperty.cpp
@@ -180,7 +180,7 @@ Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
   unsigned Attributes = ODS.getPropertyAttributes();
   FD.D.setObjCWeakProperty((Attributes & ObjCPropertyAttribute::kind_weak) !=
                            0);
-  TypeSourceInfo *TSI = GetTypeForDeclarator(FD.D, S);
+  TypeSourceInfo *TSI = GetTypeForDeclarator(FD.D);
   QualType T = TSI->getType();
   if (!getOwnershipRule(Attributes)) {
     Attributes |= deducePropertyOwnershipFromType(*this, T);
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 3826994ef212..217fcb979dee 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -5072,6 +5072,18 @@ static bool checkNestingOfRegions(Sema &SemaRef, const DSAStackTy *Stack,
         CurrentRegion != OMPD_cancellation_point &&
         CurrentRegion != OMPD_cancel && CurrentRegion != OMPD_scan)
       return false;
+    // Checks needed for mapping "loop" construct. Please check mapLoopConstruct
+    // for a detailed explanation
+    if (SemaRef.LangOpts.OpenMP >= 50 && CurrentRegion == OMPD_loop &&
+        (BindKind == OMPC_BIND_parallel || BindKind == OMPC_BIND_teams) &&
+        (isOpenMPWorksharingDirective(ParentRegion) ||
+         ParentRegion == OMPD_loop)) {
+      int ErrorMsgNumber = (BindKind == OMPC_BIND_parallel) ? 1 : 4;
+      SemaRef.Diag(StartLoc, diag::err_omp_prohibited_region)
+          << true << getOpenMPDirectiveName(ParentRegion) << ErrorMsgNumber
+          << getOpenMPDirectiveName(CurrentRegion);
+      return true;
+    }
     if (CurrentRegion == OMPD_cancellation_point ||
         CurrentRegion == OMPD_cancel) {
       // OpenMP [2.16, Nesting of Regions]
@@ -6124,21 +6136,25 @@ processImplicitMapsWithDefaultMappers(Sema &S, DSAStackTy *Stack,
 
 bool Sema::mapLoopConstruct(llvm::SmallVector &ClausesWithoutBind,
                             ArrayRef Clauses,
-                            OpenMPBindClauseKind BindKind,
+                            OpenMPBindClauseKind &BindKind,
                             OpenMPDirectiveKind &Kind,
-                            OpenMPDirectiveKind &PrevMappedDirective) {
+                            OpenMPDirectiveKind &PrevMappedDirective,
+                            SourceLocation StartLoc, SourceLocation EndLoc,
+                            const DeclarationNameInfo &DirName,
+                            OpenMPDirectiveKind CancelRegion) {
 
   bool UseClausesWithoutBind = false;
 
   // Restricting to "#pragma omp loop bind"
   if (getLangOpts().OpenMP >= 50 && Kind == OMPD_loop) {
+
+    const OpenMPDirectiveKind ParentDirective = DSAStack->getParentDirective();
+
     if (BindKind == OMPC_BIND_unknown) {
       // Setting the enclosing teams or parallel construct for the loop
       // directive without bind clause.
       BindKind = OMPC_BIND_thread; // Default bind(thread) if binding is unknown
 
-      const OpenMPDirectiveKind ParentDirective =
-          DSAStack->getParentDirective();
       if (ParentDirective == OMPD_unknown) {
         Diag(DSAStack->getDefaultDSALocation(),
              diag::err_omp_bind_required_on_loop);
@@ -6150,9 +6166,10 @@ bool Sema::mapLoopConstruct(llvm::SmallVector &ClausesWithoutBind,
         BindKind = OMPC_BIND_teams;
       }
     } else {
-      // bind clause is present, so we should set flag indicating to only
-      // use the clauses that aren't the bind clause for the new directive that
-      // loop is lowered to.
+      // bind clause is present in loop directive. When the loop directive is
+      // changed to a new directive the bind clause is not used. So, we should
+      // set flag indicating to only use the clauses that aren't the
+      // bind clause.
       UseClausesWithoutBind = true;
     }
 
@@ -6213,26 +6230,35 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
     OpenMPDirectiveKind PrevMappedDirective) {
   StmtResult Res = StmtError();
   OpenMPBindClauseKind BindKind = OMPC_BIND_unknown;
+  llvm::SmallVector ClausesWithoutBind;
+  bool UseClausesWithoutBind = false;
+
   if (const OMPBindClause *BC =
           OMPExecutableDirective::getSingleClause(Clauses))
     BindKind = BC->getBindKind();
+
+  // Variable used to note down the DirectiveKind because mapLoopConstruct may
+  // change "Kind" variable, due to mapping of "omp loop" to other directives.
+  OpenMPDirectiveKind DK = Kind;
+  if (Kind == OMPD_loop || PrevMappedDirective == OMPD_loop) {
+    UseClausesWithoutBind = mapLoopConstruct(
+        ClausesWithoutBind, Clauses, BindKind, Kind, PrevMappedDirective,
+        StartLoc, EndLoc, DirName, CancelRegion);
+    DK = OMPD_loop;
+  }
+
   // First check CancelRegion which is then used in checkNestingOfRegions.
   if (checkCancelRegion(*this, Kind, CancelRegion, StartLoc) ||
-      checkNestingOfRegions(*this, DSAStack, Kind, DirName, CancelRegion,
-                            BindKind, StartLoc))
+      checkNestingOfRegions(*this, DSAStack, DK, DirName, CancelRegion,
+                            BindKind, StartLoc)) {
     return StmtError();
+  }
 
   // Report affected OpenMP target offloading behavior when in HIP lang-mode.
   if (getLangOpts().HIP && (isOpenMPTargetExecutionDirective(Kind) ||
                             isOpenMPTargetDataManagementDirective(Kind)))
     Diag(StartLoc, diag::warn_hip_omp_target_directives);
 
-  llvm::SmallVector ClausesWithoutBind;
-  bool UseClausesWithoutBind = false;
-
-  UseClausesWithoutBind = mapLoopConstruct(ClausesWithoutBind, Clauses,
-                                           BindKind, Kind, PrevMappedDirective);
-
   llvm::SmallVector ClausesWithImplicit;
   VarsWithInheritedDSAType VarsWithInheritedDSA;
   bool ErrorFound = false;
@@ -7278,7 +7304,7 @@ void Sema::ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
                       LookupOrdinaryName);
   LookupParsedName(Lookup, S, &D.getCXXScopeSpec());
 
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
   QualType FType = TInfo->getType();
 
   bool IsConstexpr =
@@ -12683,7 +12709,7 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef Clauses,
       break;
     }
     case OMPC_fail: {
-      if (AtomicKind != OMPC_compare) {
+      if (!EncounteredAtomicKinds.contains(OMPC_compare)) {
         Diag(C->getBeginLoc(), diag::err_omp_atomic_fail_no_compare)
             << SourceRange(C->getBeginLoc(), C->getEndLoc());
         return StmtError();
@@ -22693,7 +22719,7 @@ Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareReductionDirectiveEnd(
 }
 
 TypeResult Sema::ActOnOpenMPDeclareMapperVarDecl(Scope *S, Declarator &D) {
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
   QualType T = TInfo->getType();
   if (D.isInvalidType())
     return true;
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 5026e1d603e5..030878899b81 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -1273,8 +1273,8 @@ static bool IsOverloadOrOverrideImpl(Sema &SemaRef, FunctionDecl *New,
       isa(NewQType.getTypePtr()))
     return false;
 
-  const FunctionProtoType *OldType = cast(OldQType);
-  const FunctionProtoType *NewType = cast(NewQType);
+  const auto *OldType = cast(OldQType);
+  const auto *NewType = cast(NewQType);
 
   // The signature of a function includes the types of its
   // parameters (C++ 1.3.10), which includes the presence or absence
@@ -1287,6 +1287,49 @@ static bool IsOverloadOrOverrideImpl(Sema &SemaRef, FunctionDecl *New,
        Old->isMemberLikeConstrainedFriend()) &&
       !New->getLexicalDeclContext()->Equals(Old->getLexicalDeclContext()))
     return true;
+
+  // Compare the parameter lists.
+  // This can only be done once we have establish that friend functions
+  // inhabit the same context, otherwise we might tried to instantiate
+  // references to non-instantiated entities during constraint substitution.
+  // GH78101.
+  if (NewTemplate) {
+    // C++ [temp.over.link]p4:
+    //   The signature of a function template consists of its function
+    //   signature, its return type and its template parameter list. The names
+    //   of the template parameters are significant only for establishing the
+    //   relationship between the template parameters and the rest of the
+    //   signature.
+    //
+    // We check the return type and template parameter lists for function
+    // templates first; the remaining checks follow.
+    bool SameTemplateParameterList = SemaRef.TemplateParameterListsAreEqual(
+        NewTemplate, NewTemplate->getTemplateParameters(), OldTemplate,
+        OldTemplate->getTemplateParameters(), false, Sema::TPL_TemplateMatch);
+    bool SameReturnType = SemaRef.Context.hasSameType(
+        Old->getDeclaredReturnType(), New->getDeclaredReturnType());
+    // FIXME(GH58571): Match template parameter list even for non-constrained
+    // template heads. This currently ensures that the code prior to C++20 is
+    // not newly broken.
+    bool ConstraintsInTemplateHead =
+        NewTemplate->getTemplateParameters()->hasAssociatedConstraints() ||
+        OldTemplate->getTemplateParameters()->hasAssociatedConstraints();
+    // C++ [namespace.udecl]p11:
+    //   The set of declarations named by a using-declarator that inhabits a
+    //   class C does not include member functions and member function
+    //   templates of a base class that "correspond" to (and thus would
+    //   conflict with) a declaration of a function or function template in
+    //   C.
+    // Comparing return types is not required for the "correspond" check to
+    // decide whether a member introduced by a shadow declaration is hidden.
+    if (UseMemberUsingDeclRules && ConstraintsInTemplateHead &&
+        !SameTemplateParameterList)
+      return true;
+    if (!UseMemberUsingDeclRules &&
+        (!SameTemplateParameterList || !SameReturnType))
+      return true;
+  }
+
   const auto *OldMethod = dyn_cast(Old);
   const auto *NewMethod = dyn_cast(New);
 
@@ -1410,43 +1453,6 @@ static bool IsOverloadOrOverrideImpl(Sema &SemaRef, FunctionDecl *New,
     }
   }
 
-  if (NewTemplate) {
-    // C++ [temp.over.link]p4:
-    //   The signature of a function template consists of its function
-    //   signature, its return type and its template parameter list. The names
-    //   of the template parameters are significant only for establishing the
-    //   relationship between the template parameters and the rest of the
-    //   signature.
-    //
-    // We check the return type and template parameter lists for function
-    // templates first; the remaining checks follow.
-    bool SameTemplateParameterList = SemaRef.TemplateParameterListsAreEqual(
-        NewTemplate, NewTemplate->getTemplateParameters(), OldTemplate,
-        OldTemplate->getTemplateParameters(), false, Sema::TPL_TemplateMatch);
-    bool SameReturnType = SemaRef.Context.hasSameType(
-        Old->getDeclaredReturnType(), New->getDeclaredReturnType());
-    // FIXME(GH58571): Match template parameter list even for non-constrained
-    // template heads. This currently ensures that the code prior to C++20 is
-    // not newly broken.
-    bool ConstraintsInTemplateHead =
-        NewTemplate->getTemplateParameters()->hasAssociatedConstraints() ||
-        OldTemplate->getTemplateParameters()->hasAssociatedConstraints();
-    // C++ [namespace.udecl]p11:
-    //   The set of declarations named by a using-declarator that inhabits a
-    //   class C does not include member functions and member function
-    //   templates of a base class that "correspond" to (and thus would
-    //   conflict with) a declaration of a function or function template in
-    //   C.
-    // Comparing return types is not required for the "correspond" check to
-    // decide whether a member introduced by a shadow declaration is hidden.
-    if (UseMemberUsingDeclRules && ConstraintsInTemplateHead &&
-        !SameTemplateParameterList)
-      return true;
-    if (!UseMemberUsingDeclRules &&
-        (!SameTemplateParameterList || !SameReturnType))
-      return true;
-  }
-
   if (!UseOverrideRules) {
     Expr *NewRC = New->getTrailingRequiresClause(),
          *OldRC = Old->getTrailingRequiresClause();
@@ -1562,19 +1568,37 @@ TryUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
     //   called for those cases.
     if (CXXConstructorDecl *Constructor
           = dyn_cast(ICS.UserDefined.ConversionFunction)) {
-      QualType FromCanon
-        = S.Context.getCanonicalType(From->getType().getUnqualifiedType());
+      QualType FromType;
+      SourceLocation FromLoc;
+      // C++11 [over.ics.list]p6, per DR2137:
+      // C++17 [over.ics.list]p6:
+      //   If C is not an initializer-list constructor and the initializer list
+      //   has a single element of type cv U, where U is X or a class derived
+      //   from X, the implicit conversion sequence has Exact Match rank if U is
+      //   X, or Conversion rank if U is derived from X.
+      if (const auto *InitList = dyn_cast(From);
+          InitList && InitList->getNumInits() == 1 &&
+          !S.isInitListConstructor(Constructor)) {
+        const Expr *SingleInit = InitList->getInit(0);
+        FromType = SingleInit->getType();
+        FromLoc = SingleInit->getBeginLoc();
+      } else {
+        FromType = From->getType();
+        FromLoc = From->getBeginLoc();
+      }
+      QualType FromCanon =
+          S.Context.getCanonicalType(FromType.getUnqualifiedType());
       QualType ToCanon
         = S.Context.getCanonicalType(ToType).getUnqualifiedType();
       if (Constructor->isCopyConstructor() &&
           (FromCanon == ToCanon ||
-           S.IsDerivedFrom(From->getBeginLoc(), FromCanon, ToCanon))) {
+           S.IsDerivedFrom(FromLoc, FromCanon, ToCanon))) {
         // Turn this into a "standard" conversion sequence, so that it
         // gets ranked with standard conversion sequences.
         DeclAccessPair Found = ICS.UserDefined.FoundConversionFunction;
         ICS.setStandard();
         ICS.Standard.setAsIdentityConversion();
-        ICS.Standard.setFromType(From->getType());
+        ICS.Standard.setFromType(FromType);
         ICS.Standard.setAllToTypes(ToType);
         ICS.Standard.CopyConstructor = Constructor;
         ICS.Standard.FoundCopyConstructor = Found;
@@ -1784,26 +1808,6 @@ bool Sema::IsFunctionConversion(QualType FromType, QualType ToType,
     Changed = true;
   }
 
-  // Drop the 'arm_preserves_za' if not present in the target type (we can do
-  // that because it is merely a hint).
-  if (const auto *FromFPT = dyn_cast(FromFn)) {
-    FunctionProtoType::ExtProtoInfo ExtInfo = FromFPT->getExtProtoInfo();
-    if (ExtInfo.AArch64SMEAttributes &
-        FunctionType::SME_PStateZAPreservedMask) {
-      unsigned ToFlags = 0;
-      if (const auto *ToFPT = dyn_cast(ToFn))
-        ToFlags = ToFPT->getExtProtoInfo().AArch64SMEAttributes;
-      if (!(ToFlags & FunctionType::SME_PStateZAPreservedMask)) {
-        ExtInfo.setArmSMEAttribute(FunctionType::SME_PStateZAPreservedMask,
-                                   false);
-        QualType QT = Context.getFunctionType(
-            FromFPT->getReturnType(), FromFPT->getParamTypes(), ExtInfo);
-        FromFn = QT->getAs();
-        Changed = true;
-      }
-    }
-  }
-
   // Drop 'noexcept' if not present in target type.
   if (const auto *FromFPT = dyn_cast(FromFn)) {
     const auto *ToFPT = cast(ToFn);
@@ -5320,18 +5324,18 @@ TryListConversion(Sema &S, InitListExpr *From, QualType ToType,
       IsDesignatedInit)
     return Result;
 
-  // Per DR1467:
-  //   If the parameter type is a class X and the initializer list has a single
-  //   element of type cv U, where U is X or a class derived from X, the
-  //   implicit conversion sequence is the one required to convert the element
-  //   to the parameter type.
+  // Per DR1467 and DR2137:
+  //   If the parameter type is an aggregate class X and the initializer list
+  //   has a single element of type cv U, where U is X or a class derived from
+  //   X, the implicit conversion sequence is the one required to convert the
+  //   element to the parameter type.
   //
   //   Otherwise, if the parameter type is a character array [... ]
   //   and the initializer list has a single element that is an
   //   appropriately-typed string literal (8.5.2 [dcl.init.string]), the
   //   implicit conversion sequence is the identity conversion.
   if (From->getNumInits() == 1 && !IsDesignatedInit) {
-    if (ToType->isRecordType()) {
+    if (ToType->isRecordType() && ToType->isAggregateType()) {
       QualType InitType = From->getInit(0)->getType();
       if (S.Context.hasSameUnqualifiedType(InitType, ToType) ||
           S.IsDerivedFrom(From->getBeginLoc(), InitType, ToType))
@@ -6056,6 +6060,16 @@ static ExprResult BuildConvertedConstantExpression(Sema &S, Expr *From,
                   diag::err_typecheck_converted_constant_expression_indirect)
            << From->getType() << From->getSourceRange() << T;
   }
+  // 'TryCopyInitialization' returns incorrect info for attempts to bind
+  // a reference to a bit-field due to C++ [over.ics.ref]p4. Namely,
+  // 'SCS->DirectBinding' occurs to be set to 'true' despite it is not
+  // the direct binding according to C++ [dcl.init.ref]p5. Hence, check this
+  // case explicitly.
+  if (From->refersToBitField() && T.getTypePtr()->isReferenceType()) {
+    return S.Diag(From->getBeginLoc(),
+                  diag::err_reference_bind_to_bitfield_in_cce)
+           << From->getSourceRange();
+  }
 
   // Usually we can simply apply the ImplicitConversionSequence we formed
   // earlier, but that's not guaranteed to work when initializing an object of
@@ -6203,7 +6217,15 @@ Sema::EvaluateConvertedConstantExpression(Expr *E, QualType T, APValue &Value,
 
     if (Notes.empty()) {
       // It's a constant expression.
-      Expr *E = ConstantExpr::Create(Context, Result.get(), Value);
+      Expr *E = Result.get();
+      if (const auto *CE = dyn_cast(E)) {
+        // We expect a ConstantExpr to have a value associated with it
+        // by this point.
+        assert(CE->getResultStorageKind() != ConstantResultStorageKind::None &&
+               "ConstantExpr has no value associated with it");
+      } else {
+        E = ConstantExpr::Create(Context, Result.get(), Value);
+      }
       if (!PreNarrowingValue.isAbsent())
         Value = std::move(PreNarrowingValue);
       return E;
@@ -7713,9 +7735,19 @@ bool Sema::CheckNonDependentConversions(
        ++I) {
     QualType ParamType = ParamTypes[I + Offset];
     if (!ParamType->isDependentType()) {
-      unsigned ConvIdx = PO == OverloadCandidateParamOrder::Reversed
-                             ? 0
-                             : (ThisConversions + I);
+      unsigned ConvIdx;
+      if (PO == OverloadCandidateParamOrder::Reversed) {
+        ConvIdx = Args.size() - 1 - I;
+        assert(Args.size() + ThisConversions == 2 &&
+               "number of args (including 'this') must be exactly 2 for "
+               "reversed order");
+        // For members, there would be only one arg 'Args[0]' whose ConvIdx
+        // would also be 0. 'this' got ConvIdx = 1 previously.
+        assert(!HasThisConversion || (ConvIdx == 0 && I == 0));
+      } else {
+        // For members, 'this' got ConvIdx = 0 previously.
+        ConvIdx = ThisConversions + I;
+      }
       Conversions[ConvIdx]
         = TryCopyInitialization(*this, Args[I], ParamType,
                                 SuppressUserConversions,
@@ -10111,11 +10143,23 @@ getImplicitObjectParamType(ASTContext &Context, const FunctionDecl *F) {
   return M->getFunctionObjectParameterReferenceType();
 }
 
-static bool haveSameParameterTypes(ASTContext &Context, const FunctionDecl *F1,
-                                   const FunctionDecl *F2) {
+// As a Clang extension, allow ambiguity among F1 and F2 if they represent
+// represent the same entity.
+static bool allowAmbiguity(ASTContext &Context, const FunctionDecl *F1,
+                           const FunctionDecl *F2) {
   if (declaresSameEntity(F1, F2))
     return true;
-
+  auto PT1 = F1->getPrimaryTemplate();
+  auto PT2 = F2->getPrimaryTemplate();
+  if (PT1 && PT2) {
+    if (declaresSameEntity(PT1, PT2) ||
+        declaresSameEntity(PT1->getInstantiatedFromMemberTemplate(),
+                           PT2->getInstantiatedFromMemberTemplate()))
+      return true;
+  }
+  // TODO: It is not clear whether comparing parameters is necessary (i.e.
+  // different functions with same params). Consider removing this (as no test
+  // fail w/o it).
   auto NextParam = [&](const FunctionDecl *F, unsigned &I, bool First) {
     if (First) {
       if (std::optional T = getImplicitObjectParamType(Context, F))
@@ -10319,14 +10363,14 @@ bool clang::isBetterOverloadCandidate(
     case ImplicitConversionSequence::Worse:
       if (Cand1.Function && Cand2.Function &&
           Cand1.isReversed() != Cand2.isReversed() &&
-          haveSameParameterTypes(S.Context, Cand1.Function, Cand2.Function)) {
+          allowAmbiguity(S.Context, Cand1.Function, Cand2.Function)) {
         // Work around large-scale breakage caused by considering reversed
         // forms of operator== in C++20:
         //
-        // When comparing a function against a reversed function with the same
-        // parameter types, if we have a better conversion for one argument and
-        // a worse conversion for the other, the implicit conversion sequences
-        // are treated as being equally good.
+        // When comparing a function against a reversed function, if we have a
+        // better conversion for one argument and a worse conversion for the
+        // other, the implicit conversion sequences are treated as being equally
+        // good.
         //
         // This prevents a comparison function from being considered ambiguous
         // with a reversed form that is written in the same way.
@@ -13995,6 +14039,22 @@ ExprResult Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn,
   OverloadingResult OverloadResult =
       CandidateSet.BestViableFunction(*this, Fn->getBeginLoc(), Best);
 
+  // Model the case with a call to a templated function whose definition
+  // encloses the call and whose return type contains a placeholder type as if
+  // the UnresolvedLookupExpr was type-dependent.
+  if (OverloadResult == OR_Success) {
+    const FunctionDecl *FDecl = Best->Function;
+    if (FDecl && FDecl->isTemplateInstantiation() &&
+        FDecl->getReturnType()->isUndeducedType()) {
+      if (const auto *TP =
+              FDecl->getTemplateInstantiationPattern(/*ForDefinition=*/false);
+          TP && TP->willHaveBody()) {
+        return CallExpr::Create(Context, Fn, Args, Context.DependentTy,
+                                VK_PRValue, RParenLoc, CurFPFeatureOverrides());
+      }
+    }
+  }
+
   return FinishOverloadedCallExpr(*this, S, Fn, ULE, LParenLoc, Args, RParenLoc,
                                   ExecConfig, &CandidateSet, &Best,
                                   OverloadResult, AllowTypoCorrection);
@@ -14272,12 +14332,17 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc,
     return ExprError();
 
   case OR_Deleted:
+    // CreateOverloadedUnaryOp fills the first element of ArgsArray with the
+    // object whose method was called. Later in NoteCandidates size of ArgsArray
+    // is passed further and it eventually ends up compared to number of
+    // function candidate parameters which never includes the object parameter,
+    // so slice ArgsArray to make sure apples are compared to apples.
     CandidateSet.NoteCandidates(
         PartialDiagnosticAt(OpLoc, PDiag(diag::err_ovl_deleted_oper)
                                        << UnaryOperator::getOpcodeStr(Opc)
                                        << Input->getSourceRange()),
-        *this, OCD_AllCandidates, ArgsArray, UnaryOperator::getOpcodeStr(Opc),
-        OpLoc);
+        *this, OCD_AllCandidates, ArgsArray.drop_front(),
+        UnaryOperator::getOpcodeStr(Opc), OpLoc);
     return ExprError();
   }
 
@@ -14500,7 +14565,7 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
           llvm::SmallVector AmbiguousWith;
           for (OverloadCandidate &Cand : CandidateSet) {
             if (Cand.Viable && Cand.Function && Cand.isReversed() &&
-                haveSameParameterTypes(Context, Cand.Function, FnDecl)) {
+                allowAmbiguity(Context, Cand.Function, FnDecl)) {
               for (unsigned ArgIdx = 0; ArgIdx < 2; ++ArgIdx) {
                 if (CompareImplicitConversionSequences(
                         *this, OpLoc, Cand.Conversions[ArgIdx],
@@ -15438,7 +15503,7 @@ ExprResult Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
   }
 
   if (isa(CurContext) &&
-      TheCall->getDirectCallee()->isPure()) {
+      TheCall->getDirectCallee()->isPureVirtual()) {
     const FunctionDecl *MD = TheCall->getDirectCallee();
 
     if (isa(MemExpr->getBase()->IgnoreParenCasts()) &&
diff --git a/clang/lib/Sema/SemaRISCVVectorLookup.cpp b/clang/lib/Sema/SemaRISCVVectorLookup.cpp
index 3ed3e6195441..00a5ea65f3f4 100644
--- a/clang/lib/Sema/SemaRISCVVectorLookup.cpp
+++ b/clang/lib/Sema/SemaRISCVVectorLookup.cpp
@@ -274,9 +274,8 @@ void RISCVIntrinsicManagerImpl::ConstructRVVIntrinsics(
         continue;
 
       if (BaseType == BasicType::Float16) {
-        if ((Record.RequiredExtensions & RVV_REQ_ZvfhminOrZvfh) ==
-            RVV_REQ_ZvfhminOrZvfh) {
-          if (!TI.hasFeature("zvfh") && !TI.hasFeature("zvfhmin"))
+        if ((Record.RequiredExtensions & RVV_REQ_Zvfhmin) == RVV_REQ_Zvfhmin) {
+          if (!TI.hasFeature("zvfhmin"))
             continue;
         } else if (!TI.hasFeature("zvfh")) {
           continue;
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index f0b03db69084..9e7c8c7e4e8c 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -786,6 +786,12 @@ bool Sema::checkMustTailAttr(const Stmt *St, const Attr &MTA) {
     return false;
   }
 
+  const auto *CalleeDecl = CE->getCalleeDecl();
+  if (CalleeDecl && CalleeDecl->hasAttr()) {
+    Diag(St->getBeginLoc(), diag::err_musttail_no_return) << &MTA;
+    return false;
+  }
+
   // Caller and callee must match in whether they have a "this" parameter.
   if (CallerType.This.isNull() != CalleeType.This.isNull()) {
     if (const auto *ND = dyn_cast_or_null(CE->getCalleeDecl())) {
@@ -3200,7 +3206,7 @@ static void DiagnoseForRangeConstVariableCopies(Sema &SemaRef,
   // (The function `getTypeSize` returns the size in bits.)
   ASTContext &Ctx = SemaRef.Context;
   if (Ctx.getTypeSize(VariableType) <= 64 * 8 &&
-      (VariableType.isTriviallyCopyableType(Ctx) ||
+      (VariableType.isTriviallyCopyConstructibleType(Ctx) ||
        hasTrivialABIAttr(VariableType)))
     return;
 
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 5fcc39ec7005..9bfa71dc8bcf 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -1498,7 +1498,7 @@ NamedDecl *Sema::ActOnNonTypeTemplateParameter(Scope *S, Declarator &D,
                                           unsigned Position,
                                           SourceLocation EqualLoc,
                                           Expr *Default) {
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
+  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
 
   // Check that we have valid decl-specifiers specified.
   auto CheckValidDeclSpecifiers = [this, &D] {
@@ -2418,6 +2418,9 @@ struct ConvertConstructorToDeductionGuideTransform {
     QualType Result = SemaRef.BuildFunctionType(DeducedType, ParamTypes, Loc,
                                                 DeductionGuideName, EPI);
     TypeSourceInfo *TSI = SemaRef.Context.getTrivialTypeSourceInfo(Result, Loc);
+    if (NestedPattern)
+      TSI = SemaRef.SubstType(TSI, OuterInstantiationArgs, Loc,
+                              DeductionGuideName);
 
     FunctionProtoTypeLoc FPTL =
         TSI->getTypeLoc().castAs();
@@ -2425,9 +2428,13 @@ struct ConvertConstructorToDeductionGuideTransform {
     // Build the parameters, needed during deduction / substitution.
     SmallVector Params;
     for (auto T : ParamTypes) {
-      ParmVarDecl *NewParam = ParmVarDecl::Create(
-          SemaRef.Context, DC, Loc, Loc, nullptr, T,
-          SemaRef.Context.getTrivialTypeSourceInfo(T, Loc), SC_None, nullptr);
+      auto *TSI = SemaRef.Context.getTrivialTypeSourceInfo(T, Loc);
+      if (NestedPattern)
+        TSI = SemaRef.SubstType(TSI, OuterInstantiationArgs, Loc,
+                                DeclarationName());
+      ParmVarDecl *NewParam =
+          ParmVarDecl::Create(SemaRef.Context, DC, Loc, Loc, nullptr,
+                              TSI->getType(), TSI, SC_None, nullptr);
       NewParam->setScopeInfo(0, Params.size());
       FPTL.setParam(Params.size(), NewParam);
       Params.push_back(NewParam);
@@ -2587,15 +2594,15 @@ struct ConvertConstructorToDeductionGuideTransform {
                           : ParamTy->isRValueReferenceType() ? VK_XValue
                                                              : VK_PRValue);
     }
+    // Handle arrays and functions decay.
+    auto NewType = NewDI->getType();
+    if (NewType->isArrayType() || NewType->isFunctionType())
+      NewType = SemaRef.Context.getDecayedType(NewType);
 
-    ParmVarDecl *NewParam = ParmVarDecl::Create(SemaRef.Context, DC,
-                                                OldParam->getInnerLocStart(),
-                                                OldParam->getLocation(),
-                                                OldParam->getIdentifier(),
-                                                NewDI->getType(),
-                                                NewDI,
-                                                OldParam->getStorageClass(),
-                                                NewDefArg.get());
+    ParmVarDecl *NewParam = ParmVarDecl::Create(
+        SemaRef.Context, DC, OldParam->getInnerLocStart(),
+        OldParam->getLocation(), OldParam->getIdentifier(), NewType, NewDI,
+        OldParam->getStorageClass(), NewDefArg.get());
     NewParam->setScopeInfo(OldParam->getFunctionScopeDepth(),
                            OldParam->getFunctionScopeIndex());
     SemaRef.CurrentInstantiationScope->InstantiatedLocal(OldParam, NewParam);
@@ -2670,8 +2677,14 @@ FunctionTemplateDecl *Sema::DeclareImplicitDeductionGuideFromInitList(
   if (BuildingDeductionGuides.isInvalid())
     return nullptr;
 
-  return cast(
+  ClassTemplateDecl *Pattern =
+      Transform.NestedPattern ? Transform.NestedPattern : Transform.Template;
+  ContextRAII SavedContext(*this, Pattern->getTemplatedDecl());
+
+  auto *DG = cast(
       Transform.buildSimpleDeductionGuide(ParamTypes));
+  SavedContext.pop();
+  return DG;
 }
 
 void Sema::DeclareImplicitDeductionGuides(TemplateDecl *Template,
@@ -4414,6 +4427,7 @@ static bool isTemplateArgumentTemplateParameter(
   case TemplateArgument::NullPtr:
   case TemplateArgument::Integral:
   case TemplateArgument::Declaration:
+  case TemplateArgument::StructuralValue:
   case TemplateArgument::Pack:
   case TemplateArgument::TemplateExpansion:
     return false;
@@ -5745,6 +5759,7 @@ bool Sema::CheckTemplateArgument(
 
     case TemplateArgument::Declaration:
     case TemplateArgument::Integral:
+    case TemplateArgument::StructuralValue:
     case TemplateArgument::NullPtr:
       // We've already checked this template argument, so just copy
       // it to the list of converted arguments.
@@ -5899,11 +5914,10 @@ bool Sema::CheckTemplateArgument(
     return true;
 
   case TemplateArgument::Declaration:
-    llvm_unreachable("Declaration argument with template template parameter");
   case TemplateArgument::Integral:
-    llvm_unreachable("Integral argument with template template parameter");
+  case TemplateArgument::StructuralValue:
   case TemplateArgument::NullPtr:
-    llvm_unreachable("Null pointer argument with template template parameter");
+    llvm_unreachable("non-type argument with template template parameter");
 
   case TemplateArgument::Pack:
     llvm_unreachable("Caller must expand template argument packs");
@@ -7398,44 +7412,9 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
     if (ArgResult.isInvalid())
       return ExprError();
 
-    // Convert the APValue to a TemplateArgument.
-    switch (Value.getKind()) {
-    case APValue::None:
-      assert(ParamType->isNullPtrType());
-      SugaredConverted = TemplateArgument(ParamType, /*isNullPtr=*/true);
-      CanonicalConverted = TemplateArgument(CanonParamType, /*isNullPtr=*/true);
-      break;
-    case APValue::Indeterminate:
-      llvm_unreachable("result of constant evaluation should be initialized");
-      break;
-    case APValue::Int:
-      assert(ParamType->isIntegralOrEnumerationType());
-      SugaredConverted = TemplateArgument(Context, Value.getInt(), ParamType);
-      CanonicalConverted =
-          TemplateArgument(Context, Value.getInt(), CanonParamType);
-      break;
-    case APValue::MemberPointer: {
-      assert(ParamType->isMemberPointerType());
-
-      // FIXME: We need TemplateArgument representation and mangling for these.
-      if (!Value.getMemberPointerPath().empty()) {
-        Diag(Arg->getBeginLoc(),
-             diag::err_template_arg_member_ptr_base_derived_not_supported)
-            << Value.getMemberPointerDecl() << ParamType
-            << Arg->getSourceRange();
-        return ExprError();
-      }
-
-      auto *VD = const_cast(Value.getMemberPointerDecl());
-      SugaredConverted = VD ? TemplateArgument(VD, ParamType)
-                            : TemplateArgument(ParamType, /*isNullPtr=*/true);
-      CanonicalConverted =
-          VD ? TemplateArgument(cast(VD->getCanonicalDecl()),
-                                CanonParamType)
-             : TemplateArgument(CanonParamType, /*isNullPtr=*/true);
-      break;
-    }
-    case APValue::LValue: {
+    // Prior to C++20, enforce restrictions on possible template argument
+    // values.
+    if (!getLangOpts().CPlusPlus20 && Value.isLValue()) {
       //   For a non-type template-parameter of pointer or reference type,
       //   the value of the constant expression shall not refer to
       assert(ParamType->isPointerType() || ParamType->isReferenceType() ||
@@ -7453,8 +7432,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
             << Arg->getSourceRange();
         return ExprError();
       }
-      // -- a subobject
-      // FIXME: Until C++20
+      // -- a subobject [until C++20]
       if (Value.hasLValuePath() && Value.getLValuePath().size() == 1 &&
           VD && VD->getType()->isArrayType() &&
           Value.getLValuePath()[0].getAsArrayIndex() == 0 &&
@@ -7472,37 +7450,13 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
              "null reference should not be a constant expression");
       assert((!VD || !ParamType->isNullPtrType()) &&
              "non-null value of type nullptr_t?");
-
-      SugaredConverted = VD ? TemplateArgument(VD, ParamType)
-                            : TemplateArgument(ParamType, /*isNullPtr=*/true);
-      CanonicalConverted =
-          VD ? TemplateArgument(cast(VD->getCanonicalDecl()),
-                                CanonParamType)
-             : TemplateArgument(CanonParamType, /*isNullPtr=*/true);
-      break;
     }
-    case APValue::Struct:
-    case APValue::Union: {
-      // Get or create the corresponding template parameter object.
-      TemplateParamObjectDecl *D =
-          Context.getTemplateParamObjectDecl(ParamType, Value);
-      SugaredConverted = TemplateArgument(D, ParamType);
-      CanonicalConverted =
-          TemplateArgument(D->getCanonicalDecl(), CanonParamType);
-      break;
-    }
-    case APValue::AddrLabelDiff:
+
+    if (Value.isAddrLabelDiff())
       return Diag(StartLoc, diag::err_non_type_template_arg_addr_label_diff);
-    case APValue::FixedPoint:
-    case APValue::Float:
-    case APValue::ComplexInt:
-    case APValue::ComplexFloat:
-    case APValue::Vector:
-    case APValue::Array:
-      return Diag(StartLoc, diag::err_non_type_template_arg_unsupported)
-             << ParamType;
-    }
 
+    SugaredConverted = TemplateArgument(Context, ParamType, Value);
+    CanonicalConverted = TemplateArgument(Context, CanonParamType, Value);
     return ArgResult.get();
   }
 
@@ -8086,12 +8040,9 @@ Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
 /// This routine takes care of the mapping from an integral template
 /// argument (which may have any integral type) to the appropriate
 /// literal value.
-ExprResult
-Sema::BuildExpressionFromIntegralTemplateArgument(const TemplateArgument &Arg,
-                                                  SourceLocation Loc) {
-  assert(Arg.getKind() == TemplateArgument::Integral &&
-         "Operation is only valid for integral template arguments");
-  QualType OrigT = Arg.getIntegralType();
+static Expr *BuildExpressionFromIntegralTemplateArgumentValue(
+    Sema &S, QualType OrigT, const llvm::APSInt &Int, SourceLocation Loc) {
+  assert(OrigT->isIntegralOrEnumerationType());
 
   // If this is an enum type that we're instantiating, we need to use an integer
   // type the same size as the enumerator.  We don't want to build an
@@ -8107,7 +8058,7 @@ Sema::BuildExpressionFromIntegralTemplateArgument(const TemplateArgument &Arg,
     CharacterLiteralKind Kind;
     if (T->isWideCharType())
       Kind = CharacterLiteralKind::Wide;
-    else if (T->isChar8Type() && getLangOpts().Char8)
+    else if (T->isChar8Type() && S.getLangOpts().Char8)
       Kind = CharacterLiteralKind::UTF8;
     else if (T->isChar16Type())
       Kind = CharacterLiteralKind::UTF16;
@@ -8116,29 +8067,133 @@ Sema::BuildExpressionFromIntegralTemplateArgument(const TemplateArgument &Arg,
     else
       Kind = CharacterLiteralKind::Ascii;
 
-    E = new (Context) CharacterLiteral(Arg.getAsIntegral().getZExtValue(),
-                                       Kind, T, Loc);
+    E = new (S.Context) CharacterLiteral(Int.getZExtValue(), Kind, T, Loc);
   } else if (T->isBooleanType()) {
-    E = CXXBoolLiteralExpr::Create(Context, Arg.getAsIntegral().getBoolValue(),
-                                   T, Loc);
-  } else if (T->isNullPtrType()) {
-    E = new (Context) CXXNullPtrLiteralExpr(Context.NullPtrTy, Loc);
+    E = CXXBoolLiteralExpr::Create(S.Context, Int.getBoolValue(), T, Loc);
   } else {
-    E = IntegerLiteral::Create(Context, Arg.getAsIntegral(), T, Loc);
+    E = IntegerLiteral::Create(S.Context, Int, T, Loc);
   }
 
   if (OrigT->isEnumeralType()) {
     // FIXME: This is a hack. We need a better way to handle substituted
     // non-type template parameters.
-    E = CStyleCastExpr::Create(Context, OrigT, VK_PRValue, CK_IntegralCast, E,
-                               nullptr, CurFPFeatureOverrides(),
-                               Context.getTrivialTypeSourceInfo(OrigT, Loc),
+    E = CStyleCastExpr::Create(S.Context, OrigT, VK_PRValue, CK_IntegralCast, E,
+                               nullptr, S.CurFPFeatureOverrides(),
+                               S.Context.getTrivialTypeSourceInfo(OrigT, Loc),
                                Loc, Loc);
   }
 
   return E;
 }
 
+static Expr *BuildExpressionFromNonTypeTemplateArgumentValue(
+    Sema &S, QualType T, const APValue &Val, SourceLocation Loc) {
+  auto MakeInitList = [&](ArrayRef Elts) -> Expr * {
+    auto *ILE = new (S.Context) InitListExpr(S.Context, Loc, Elts, Loc);
+    ILE->setType(T);
+    return ILE;
+  };
+
+  switch (Val.getKind()) {
+  case APValue::AddrLabelDiff:
+    // This cannot occur in a template argument at all.
+  case APValue::Array:
+  case APValue::Struct:
+  case APValue::Union:
+    // These can only occur within a template parameter object, which is
+    // represented as a TemplateArgument::Declaration.
+    llvm_unreachable("unexpected template argument value");
+
+  case APValue::Int:
+    return BuildExpressionFromIntegralTemplateArgumentValue(S, T, Val.getInt(),
+                                                            Loc);
+
+  case APValue::Float:
+    return FloatingLiteral::Create(S.Context, Val.getFloat(), /*IsExact=*/true,
+                                   T, Loc);
+
+  case APValue::FixedPoint:
+    return FixedPointLiteral::CreateFromRawInt(
+        S.Context, Val.getFixedPoint().getValue(), T, Loc,
+        Val.getFixedPoint().getScale());
+
+  case APValue::ComplexInt: {
+    QualType ElemT = T->castAs()->getElementType();
+    return MakeInitList({BuildExpressionFromIntegralTemplateArgumentValue(
+                             S, ElemT, Val.getComplexIntReal(), Loc),
+                         BuildExpressionFromIntegralTemplateArgumentValue(
+                             S, ElemT, Val.getComplexIntImag(), Loc)});
+  }
+
+  case APValue::ComplexFloat: {
+    QualType ElemT = T->castAs()->getElementType();
+    return MakeInitList(
+        {FloatingLiteral::Create(S.Context, Val.getComplexFloatReal(), true,
+                                 ElemT, Loc),
+         FloatingLiteral::Create(S.Context, Val.getComplexFloatImag(), true,
+                                 ElemT, Loc)});
+  }
+
+  case APValue::Vector: {
+    QualType ElemT = T->castAs()->getElementType();
+    llvm::SmallVector Elts;
+    for (unsigned I = 0, N = Val.getVectorLength(); I != N; ++I)
+      Elts.push_back(BuildExpressionFromNonTypeTemplateArgumentValue(
+          S, ElemT, Val.getVectorElt(I), Loc));
+    return MakeInitList(Elts);
+  }
+
+  case APValue::None:
+  case APValue::Indeterminate:
+    llvm_unreachable("Unexpected APValue kind.");
+  case APValue::LValue:
+  case APValue::MemberPointer:
+    // There isn't necessarily a valid equivalent source-level syntax for
+    // these; in particular, a naive lowering might violate access control.
+    // So for now we lower to a ConstantExpr holding the value, wrapped around
+    // an OpaqueValueExpr.
+    // FIXME: We should have a better representation for this.
+    ExprValueKind VK = VK_PRValue;
+    if (T->isReferenceType()) {
+      T = T->getPointeeType();
+      VK = VK_LValue;
+    }
+    auto *OVE = new (S.Context) OpaqueValueExpr(Loc, T, VK);
+    return ConstantExpr::Create(S.Context, OVE, Val);
+  }
+  llvm_unreachable("Unhandled APValue::ValueKind enum");
+}
+
+ExprResult
+Sema::BuildExpressionFromNonTypeTemplateArgument(const TemplateArgument &Arg,
+                                                 SourceLocation Loc) {
+  switch (Arg.getKind()) {
+  case TemplateArgument::Null:
+  case TemplateArgument::Type:
+  case TemplateArgument::Template:
+  case TemplateArgument::TemplateExpansion:
+  case TemplateArgument::Pack:
+    llvm_unreachable("not a non-type template argument");
+
+  case TemplateArgument::Expression:
+    return Arg.getAsExpr();
+
+  case TemplateArgument::NullPtr:
+  case TemplateArgument::Declaration:
+    return BuildExpressionFromDeclTemplateArgument(
+        Arg, Arg.getNonTypeTemplateArgumentType(), Loc);
+
+  case TemplateArgument::Integral:
+    return BuildExpressionFromIntegralTemplateArgumentValue(
+        *this, Arg.getIntegralType(), Arg.getAsIntegral(), Loc);
+
+  case TemplateArgument::StructuralValue:
+    return BuildExpressionFromNonTypeTemplateArgumentValue(
+        *this, Arg.getStructuralValueType(), Arg.getAsStructuralValue(), Loc);
+  }
+  llvm_unreachable("Unhandled TemplateArgument::ArgKind enum");
+}
+
 /// Match two template parameters within template parameter lists.
 static bool MatchTemplateParameterKind(
     Sema &S, NamedDecl *New,
@@ -9229,10 +9284,8 @@ void Sema::CheckConceptRedefinition(ConceptDecl *NewDecl,
 /// that has just been explicitly specialized.
 static void StripImplicitInstantiation(NamedDecl *D, bool MinGW) {
   if (MinGW || (isa(D) &&
-                cast(D)->isFunctionTemplateSpecialization())) {
-    D->dropAttr();
-    D->dropAttr();
-  }
+                cast(D)->isFunctionTemplateSpecialization()))
+    D->dropAttrs();
 
   if (FunctionDecl *FD = dyn_cast(D))
     FD->setInlineSpecified(false);
@@ -10521,7 +10574,7 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     S = S->getParent();
 
   // Determine the type of the declaration.
-  TypeSourceInfo *T = GetTypeForDeclarator(D, S);
+  TypeSourceInfo *T = GetTypeForDeclarator(D);
   QualType R = T->getType();
   if (R.isNull())
     return true;
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 699e0985e595..e9e7ab5bb669 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -268,6 +268,16 @@ checkDeducedTemplateArguments(ASTContext &Context,
     // All other combinations are incompatible.
     return DeducedTemplateArgument();
 
+  case TemplateArgument::StructuralValue:
+    // If we deduced a value and a dependent expression, keep the value.
+    if (Y.getKind() == TemplateArgument::Expression ||
+        (Y.getKind() == TemplateArgument::StructuralValue &&
+         X.structurallyEquals(Y)))
+      return X;
+
+    // All other combinations are incompatible.
+    return DeducedTemplateArgument();
+
   case TemplateArgument::Template:
     if (Y.getKind() == TemplateArgument::Template &&
         Context.hasSameTemplateName(X.getAsTemplate(), Y.getAsTemplate()))
@@ -2300,27 +2310,45 @@ DeduceTemplateArguments(Sema &S, TemplateParameterList *TemplateParams,
     Info.SecondArg = A;
     return Sema::TDK_NonDeducedMismatch;
 
+  case TemplateArgument::StructuralValue:
+    if (A.getKind() == TemplateArgument::StructuralValue &&
+        A.structurallyEquals(P))
+      return Sema::TDK_Success;
+
+    Info.FirstArg = P;
+    Info.SecondArg = A;
+    return Sema::TDK_NonDeducedMismatch;
+
   case TemplateArgument::Expression:
     if (const NonTypeTemplateParmDecl *NTTP =
             getDeducedParameterFromExpr(Info, P.getAsExpr())) {
-      if (A.getKind() == TemplateArgument::Integral)
+      switch (A.getKind()) {
+      case TemplateArgument::Integral:
+      case TemplateArgument::Expression:
+      case TemplateArgument::StructuralValue:
         return DeduceNonTypeTemplateArgument(
-            S, TemplateParams, NTTP, A.getAsIntegral(), A.getIntegralType(),
-            /*ArrayBound=*/false, Info, Deduced);
-      if (A.getKind() == TemplateArgument::NullPtr)
+            S, TemplateParams, NTTP, DeducedTemplateArgument(A),
+            A.getNonTypeTemplateArgumentType(), Info, Deduced);
+
+      case TemplateArgument::NullPtr:
         return DeduceNullPtrTemplateArgument(S, TemplateParams, NTTP,
                                              A.getNullPtrType(), Info, Deduced);
-      if (A.getKind() == TemplateArgument::Expression)
-        return DeduceNonTypeTemplateArgument(S, TemplateParams, NTTP,
-                                             A.getAsExpr(), Info, Deduced);
-      if (A.getKind() == TemplateArgument::Declaration)
+
+      case TemplateArgument::Declaration:
         return DeduceNonTypeTemplateArgument(
             S, TemplateParams, NTTP, A.getAsDecl(), A.getParamTypeForDecl(),
             Info, Deduced);
 
-      Info.FirstArg = P;
-      Info.SecondArg = A;
-      return Sema::TDK_NonDeducedMismatch;
+      case TemplateArgument::Null:
+      case TemplateArgument::Type:
+      case TemplateArgument::Template:
+      case TemplateArgument::TemplateExpansion:
+      case TemplateArgument::Pack:
+        Info.FirstArg = P;
+        Info.SecondArg = A;
+        return Sema::TDK_NonDeducedMismatch;
+      }
+      llvm_unreachable("Unknown template argument kind");
     }
 
     // Can't deduce anything, but that's okay.
@@ -2505,6 +2533,9 @@ static bool isSameTemplateArg(ASTContext &Context,
     case TemplateArgument::Integral:
       return hasSameExtendedValue(X.getAsIntegral(), Y.getAsIntegral());
 
+    case TemplateArgument::StructuralValue:
+      return X.structurallyEquals(Y);
+
     case TemplateArgument::Expression: {
       llvm::FoldingSetNodeID XID, YID;
       X.getAsExpr()->Profile(XID, Context, true);
@@ -2585,9 +2616,9 @@ Sema::getTrivialTemplateArgumentLoc(const TemplateArgument &Arg,
                                E);
   }
 
-  case TemplateArgument::Integral: {
-    Expr *E =
-        BuildExpressionFromIntegralTemplateArgument(Arg, Loc).getAs();
+  case TemplateArgument::Integral:
+  case TemplateArgument::StructuralValue: {
+    Expr *E = BuildExpressionFromNonTypeTemplateArgument(Arg, Loc).get();
     return TemplateArgumentLoc(TemplateArgument(E), E);
   }
 
@@ -4737,6 +4768,7 @@ namespace {
     QualType Replacement;
     bool ReplacementIsPack;
     bool UseTypeSugar;
+    using inherited = TreeTransform;
 
   public:
     SubstituteDeducedTypeTransform(Sema &SemaRef, DependentAuto DA)
@@ -4797,6 +4829,16 @@ namespace {
       // Lambdas never need to be transformed.
       return E;
     }
+    bool TransformExceptionSpec(SourceLocation Loc,
+                                FunctionProtoType::ExceptionSpecInfo &ESI,
+                                SmallVectorImpl &Exceptions,
+                                bool &Changed) {
+      if (ESI.Type == EST_Uninstantiated) {
+        ESI.instantiate();
+        Changed = true;
+      }
+      return inherited::TransformExceptionSpec(Loc, ESI, Exceptions, Changed);
+    }
 
     QualType Apply(TypeLoc TL) {
       // Create some scratch storage for the transformed type locations.
@@ -6438,11 +6480,8 @@ MarkUsedTemplateParameters(ASTContext &Ctx,
   case TemplateArgument::Null:
   case TemplateArgument::Integral:
   case TemplateArgument::Declaration:
-    break;
-
   case TemplateArgument::NullPtr:
-    MarkUsedTemplateParameters(Ctx, TemplateArg.getNullPtrType(), OnlyDeduced,
-                               Depth, Used);
+  case TemplateArgument::StructuralValue:
     break;
 
   case TemplateArgument::Type:
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index df6b40999e64..e12186d7d82f 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -35,7 +35,6 @@
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateDeduction.h"
 #include "clang/Sema/TemplateInstCallback.h"
-#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/TimeProfiler.h"
@@ -224,6 +223,9 @@ Response HandleFunction(const FunctionDecl *Function,
       (!Pattern || !Pattern->getLexicalDeclContext()->isFileContext())) {
     return Response::ChangeDecl(Function->getLexicalDeclContext());
   }
+
+  if (ForConstraintInstantiation && Function->getFriendObjectKind())
+    return Response::ChangeDecl(Function->getLexicalDeclContext());
   return Response::UseNextDecl(Function);
 }
 
@@ -345,15 +347,26 @@ MultiLevelTemplateArgumentList Sema::getTemplateInstantiationArgs(
 
   using namespace TemplateInstArgsHelpers;
   const Decl *CurDecl = ND;
+
+  if (!CurDecl)
+    CurDecl = Decl::castFromDeclContext(DC);
+
   if (Innermost) {
     Result.addOuterTemplateArguments(const_cast(ND),
                                      Innermost->asArray(), Final);
-    CurDecl = Response::UseNextDecl(ND).NextDecl;
+    // Populate placeholder template arguments for TemplateTemplateParmDecls.
+    // This is essential for the case e.g.
+    //
+    // template  concept Concept = false;
+    // template