Merge branch 'main' into 4.20-vr-fix-binary-userdata

.asf.yaml

@@ -58,6 +58,7 @@ github:
     - gpordeus
     - hsato03
     - bernardodemarco
+    - abh1sar
 
   protected_branches: ~
 

.github/linters/.flake8

@@ -22,8 +22,11 @@
 # E224 Tab after operator
 # E227 Missing whitespace around bitwise or shift operator
 # E242 Tab after ','
+# E271 Multiple spaces after keyword
+# E272 Multiple spaces before keyword
 # E273 Tab after keyword
 # E274 Tab before keyword
+# E713 Test for membership should be 'not in'
 # E742 Do not define classes named 'I', 'O', or 'l'
 # E743 Do not define functions named 'I', 'O', or 'l'
 # E901 SyntaxError or IndentationError
@@ -37,4 +40,4 @@
 exclude =
     .git,
     venv
-select = E112,E113,E133,E223,E224,E227,E242,E273,E274,E742,E743,E901,E902,W291,W292,W293,W391
+select = E112,E113,E133,E223,E224,E227,E242,E271,E272,E273,E274,E713,E742,E743,E901,E902,W291,W292,W293,W391

.github/linters/.markdown-lint.yml

@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# MD001/heading-increment Heading levels should only increment by one level at a time
+MD001: false
+
+# MD003/heading-style Heading style
+MD003: false
+
+# MD004/ul-style Unordered list style
+MD004: false
+
+# MD007/ul-indent Unordered list indentation
+MD007: false
+
+# MD009/no-trailing-spaces Trailing spaces
+MD009: false
+
+# MD010/no-hard-tabs Hard tabs
+MD010: false
+
+# MD012/no-multiple-blanks Multiple consecutive blank lines
+MD012: false
+
+# MD013/line-length Line length
+MD013: false
+
+# MD014/commands-show-output Dollar signs used before commands without showing output
+MD014: false
+
+# MD018/no-missing-space-atx No space after hash on atx style heading
+MD018: false
+
+# MD019/no-multiple-space-atx Multiple spaces after hash on atx style heading
+MD019: false
+
+# MD022/blanks-around-headings Headings should be surrounded by blank lines
+MD022: false
+
+# MD023/heading-start-left Headings must start at the beginning of the line
+MD023: false
+
+# MD024/no-duplicate-heading Multiple headings with the same content
+MD024: false
+
+# MD025/single-title/single-h1 Multiple top-level headings in the same document
+MD025: false
+
+# MD026/no-trailing-punctuation Trailing punctuation in heading
+MD026: false
+
+# MD028/no-blanks-blockquote Blank line inside blockquote
+MD028: false
+
+# MD029/ol-prefix Ordered list item prefix
+MD029: false
+
+# MD031/blanks-around-fences Fenced code blocks should be surrounded by blank lines
+MD031: false
+
+# MD032/blanks-around-lists Lists should be surrounded by blank lines
+MD032: false
+
+# MD033/no-inline-html Inline HTML
+MD033: false
+
+# MD034/no-bare-urls Bare URL used
+MD034: false
+
+# MD036/no-emphasis-as-heading Emphasis used instead of a heading
+MD036: false
+
+# MD037/no-space-in-emphasis Spaces inside emphasis markers
+MD037: false
+
+# MD040/fenced-code-language Fenced code blocks should have a language specified
+MD040: false
+
+# MD041/first-line-heading/first-line-h1 First line in a file should be a top-level heading
+MD041: false
+
+# MD046/code-block-style Code block style
+MD046: false
+
+# MD052/reference-links-images Reference links and images should use a label that is defined
+MD052: false

.github/workflows/ci.yml

@@ -86,7 +86,9 @@ jobs:
                   smoke/test_migration
                   smoke/test_multipleips_per_nic
                   smoke/test_nested_virtualization
-                  smoke/test_set_sourcenat",
+                  smoke/test_set_sourcenat
+                  smoke/test_webhook_lifecycle
+                  smoke/test_purge_expunged_vms",
                 "smoke/test_network
                   smoke/test_network_acl
                   smoke/test_network_ipv6
@@ -132,6 +134,7 @@ jobs:
                   smoke/test_usage
                   smoke/test_usage_events
                   smoke/test_vm_deployment_planner
+                  smoke/test_vm_strict_host_tags
                   smoke/test_vm_schedule
                   smoke/test_vm_life_cycle
                   smoke/test_vm_lifecycle_unmanage_import

.github/workflows/linter.yml

@@ -39,7 +39,7 @@ jobs:
           pip install pre-commit
       - name: Set PY
         run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pre-commit
           key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}

.github/workflows/main-sonar-check.yml

@@ -44,14 +44,14 @@ jobs:
           cache: 'maven'
 
       - name: Cache SonarCloud packages
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.sonar/cache
           key: ${{ runner.os }}-sonar
           restore-keys: ${{ runner.os }}-sonar
 
       - name: Cache local Maven repository
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}

.github/workflows/sonar-check.yml

@@ -46,14 +46,14 @@ jobs:
           cache: 'maven'
 
       - name: Cache SonarCloud packages
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.sonar/cache
           key: ${{ runner.os }}-sonar
           restore-keys: ${{ runner.os }}-sonar
 
       - name: Cache local Maven repository
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}

.pre-commit-config.yaml

@@ -36,6 +36,8 @@ repos:
       - id: check-vcs-permalinks
       #- id: check-yaml
       - id: destroyed-symlinks
+      - id: detect-aws-credentials
+        args: [--allow-missing-credentials]
       - id: detect-private-key
         exclude: >
           (?x)
@@ -53,9 +55,13 @@ repos:
       - id: end-of-file-fixer
         exclude: \.vhd$
       #- id: fix-byte-order-marker
+      - id: forbid-submodules
       - id: mixed-line-ending
         exclude: \.(cs|xml)$
-      # - id: trailing-whitespace
+      - id: trailing-whitespace
+        files: \.(header|in|java|md|properties|py|rb|sh|sql|txt|vue|yaml|yml)$
+        args: [--markdown-linebreak-ext=md]
+        exclude: ^services/console-proxy/rdpconsole/src/test/doc/freerdp-debug-log\.txt$
   - repo: https://github.com/pycqa/flake8
     rev: 7.0.0
     hooks:
@@ -72,3 +78,12 @@ repos:
         ^scripts/vm/hypervisor/xenserver/vmopspremium$|
         ^setup/bindir/cloud-setup-encryption\.in$|
         ^venv/.*$
+  - repo: https://github.com/igorshubovych/markdownlint-cli
+    rev: v0.40.0
+    hooks:
+      - id: markdownlint
+        name: run markdownlint
+        description: check Markdown files with markdownlint
+        args: [--config=.github/linters/.markdown-lint.yml]
+        types: [markdown]
+        files: \.(md|mdown|markdown)$

CHANGES.md

@@ -430,11 +430,11 @@ Bug ID | Description
 [CLOUDSTACK-6099](https://issues.apache.org/jira/browse/CLOUDSTACK-6099) | live migration is failing for vm deployed using dynamic compute offerings with NPE
 [CLOUDSTACK-7528](https://issues.apache.org/jira/browse/CLOUDSTACK-7528) | More verbose logging when sending alert fails
 [CLOUDSTACK-6624](https://issues.apache.org/jira/browse/CLOUDSTACK-6624) | set specifyIpRanges to true if specifyVlan is set to true
-[CLOUDSTACK-7404](https://issues.apache.org/jira/browse/CLOUDSTACK-7404) | Failed to start an instance when originating template has been deleted 
+[CLOUDSTACK-7404](https://issues.apache.org/jira/browse/CLOUDSTACK-7404) | Failed to start an instance when originating template has been deleted
 [CLOUDSTACK-6531](https://issues.apache.org/jira/browse/CLOUDSTACK-6531) | Stopping the router in case of command failures
 [CLOUDSTACK-6115](https://issues.apache.org/jira/browse/CLOUDSTACK-6115) | TravisCI configuration
 [CLOUDSTACK-7405](https://issues.apache.org/jira/browse/CLOUDSTACK-7405) | allowing VR meta-data to be accessed without trailing slash
-[CLOUDSTACK-7260](https://issues.apache.org/jira/browse/CLOUDSTACK-7260) | Management server not responding after some time for Vmware due to Oom 
+[CLOUDSTACK-7260](https://issues.apache.org/jira/browse/CLOUDSTACK-7260) | Management server not responding after some time for Vmware due to Oom
 [CLOUDSTACK-7038](https://issues.apache.org/jira/browse/CLOUDSTACK-7038) | Add mysql client dependency for mgmt server pkg for debian
 [CLOUDSTACK-6892](https://issues.apache.org/jira/browse/CLOUDSTACK-6892) | Create separate package for the mysql HA component
 [CLOUDSTACK-7038](https://issues.apache.org/jira/browse/CLOUDSTACK-7038) | Add mysql client dependency for mgmt server/rpms
@@ -449,12 +449,12 @@ Bug ID | Description
 [CLOUDSTACK-7006](https://issues.apache.org/jira/browse/CLOUDSTACK-7006) | Restore template ID in ROOT volume usages
 [CLOUDSTACK-6747](https://issues.apache.org/jira/browse/CLOUDSTACK-6747) | test to allow all cidrs on other end of vpc
 [CLOUDSTACK-6272](https://issues.apache.org/jira/browse/CLOUDSTACK-6272) | Fix recover/restore VM actions
-[CLOUDSTACK-6927](https://issues.apache.org/jira/browse/CLOUDSTACK-6927) | store virsh list in list instead of querying libvirt 
+[CLOUDSTACK-6927](https://issues.apache.org/jira/browse/CLOUDSTACK-6927) | store virsh list in list instead of querying libvirt
 [CLOUDSTACK-6317](https://issues.apache.org/jira/browse/CLOUDSTACK-6317) | [VMware] Tagged VLAN support broken for Management/Control/Storage traffic
 [CLOUDSTACK-5891](https://issues.apache.org/jira/browse/CLOUDSTACK-5891) | [VMware] If a template has been registered and "cpu.corespersocket=X" ,
 [CLOUDSTACK-6478](https://issues.apache.org/jira/browse/CLOUDSTACK-6478) | Failed to download Template when having 3 SSVM's in one
 [CLOUDSTACK-6464](https://issues.apache.org/jira/browse/CLOUDSTACK-6464) | if guest network type is vlan://untagged, and traffic label is used
-[CLOUDSTACK-6816](https://issues.apache.org/jira/browse/CLOUDSTACK-6816) | bugfix: cloudstack-setup-management make /root directory's permission 0777 
+[CLOUDSTACK-6816](https://issues.apache.org/jira/browse/CLOUDSTACK-6816) | bugfix: cloudstack-setup-management make /root directory's permission 0777
 [CLOUDSTACK-6204](https://issues.apache.org/jira/browse/CLOUDSTACK-6204) | applying missed patch
 [CLOUDSTACK-6472](https://issues.apache.org/jira/browse/CLOUDSTACK-6472) | (4.3 specific) listUsageRecords: Pull information from removed items as well
 [CLOUDSTACK-5976](https://issues.apache.org/jira/browse/CLOUDSTACK-5976) | Typo in "ssh_keypairs" table's foreign key constraints on the Upgraded Setup
@@ -657,11 +657,11 @@ Version 4.1.0
 -------------
 
 This is the second major release of CloudStack from within the Apache Software Foundation, and the
-first major release as a Top-Level Project (TLP). 
+first major release as a Top-Level Project (TLP).
 
 Build Tool Changes:
 
- * The project now uses Maven 3 exclusively to build. 
+ * The project now uses Maven 3 exclusively to build.
 
 New Features:
 * CLOUDSTACK-101: OVS support in KVM
@@ -976,14 +976,14 @@ Issues fixed in this release:
 * CLOUDSTACK-1845: KVM - storage migration often fails
 * CLOUDSTACK-1846: KVM - storage pools can silently fail to be unregistered, leading to failure to register later
 * CLOUDSTACK-2003: Deleting domain while deleted account is cleaning up leaves VMs expunging forever due to 'Failed to update resource count'
-* CLOUDSTACK-2090: Upgrade from version 4.0.1 to version 4.0.2 triggers the 4.0.0 to 4.0.1. 
+* CLOUDSTACK-2090: Upgrade from version 4.0.1 to version 4.0.2 triggers the 4.0.0 to 4.0.1.
 * CLOUDSTACK-2091: Error in API documentation for 4.0.x.
 
 
 Version 4.0.1-incubating
 ------------------------
 
-This is a bugfix release for Apache CloudStack 4.0.0-incubating, with no new features. 
+This is a bugfix release for Apache CloudStack 4.0.0-incubating, with no new features.
 
 Security Fixes:
 

CONTRIBUTING.md

@@ -3,25 +3,25 @@ Contributing to Apache CloudStack (ACS)
 
 Summary
 -------
-This document covers how to contribute to the ACS project. ACS uses GitHub PRs to manage code contributions. 
+This document covers how to contribute to the ACS project. ACS uses GitHub PRs to manage code contributions.
 These instructions assume you have a GitHub.com account, so if you don't have one you will have to create one. Your proposed code changes will be published to your own fork of the ACS project and you will submit a Pull Request for your changes to be added.
 
 _Lets get started!!!_
 
 Bug fixes
 ---------
 
-It's very important that we can easily track bug fix commits, so their hashes should remain the same in all branches. 
-Therefore, a pull request (PR) that fixes a bug, should be sent against a release branch. 
-This can be either the "current release" or the "previous release", depending on which ones are maintained. 
+It's very important that we can easily track bug fix commits, so their hashes should remain the same in all branches.
+Therefore, a pull request (PR) that fixes a bug, should be sent against a release branch.
+This can be either the "current release" or the "previous release", depending on which ones are maintained.
 Since the goal is a stable main, bug fixes should be "merged forward" to the next branch in order: "previous release" -> "current release" -> main (in other words: old to new)
 
 Developing new features
 -----------------------
 
-Development should be done in a feature branch, branched off of main. 
-Send a PR(steps below) to get it into main (2x LGTM applies). 
-PR will only be merged when main is open, will be held otherwise until main is open again. 
+Development should be done in a feature branch, branched off of main.
+Send a PR(steps below) to get it into main (2x LGTM applies).
+PR will only be merged when main is open, will be held otherwise until main is open again.
 No back porting / cherry-picking features to existing branches!
 
 PendingReleaseNotes file
@@ -33,7 +33,7 @@ When adding information to the PendingReleaseNotes file make sure that you write
 
 Updating the PendingReleaseNotes file is preferably a part of the original Pull Request, but that is up to the developers' discretion.
 
-Fork the code 
+Fork the code
 -------------
 
 In your browser, navigate to: [https://github.com/apache/cloudstack](https://github.com/apache/cloudstack)
@@ -136,4 +136,4 @@ $ git push origin :feature_x
 
 Release Principles
 ------------------
-Detailed information about ACS release principles is available at https://cwiki.apache.org/confluence/display/CLOUDSTACK/Release+principles+for+Apache+CloudStack+4.6+and+up 
+Detailed information about ACS release principles is available at https://cwiki.apache.org/confluence/display/CLOUDSTACK/Release+principles+for+Apache+CloudStack+4.6+and+up

PULL_REQUEST_TEMPLATE.md

@@ -23,6 +23,7 @@ This PR...
 - [ ] Enhancement (improves an existing feature and functionality)
 - [ ] Cleanup (Code refactoring and cleanup, that may add test cases)
 - [ ] build/CI
+- [ ] test (unit or integration test code)
 
 ### Feature/Enhancement Scale or Bug Severity
 

README.md

@@ -142,7 +142,7 @@ This distribution includes cryptographic software. The country in which you curr
 reside may have restrictions on the import, possession, use, and/or re-export to another
 country, of encryption software. BEFORE using any encryption software, please check your
 country's laws, regulations and policies concerning the import, possession, or use, and
-re-export of encryption software, to see if this is permitted. See [The Wassenaar Arrangement](http://www.wassenaar.org/) 
+re-export of encryption software, to see if this is permitted. See [The Wassenaar Arrangement](http://www.wassenaar.org/)
 for more information.
 
 The U.S. Government Department of Commerce, Bureau of Industry and Security (BIS), has

agent/bindir/cloud-setup-agent.in

@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

agent/bindir/cloud-ssh.in

@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

agent/conf/agent.properties

@@ -430,3 +430,6 @@ iscsi.session.cleanup.enabled=false
 # If set to "true", the agent will register for libvirt domain events, allowing for immediate updates on crashed or
 # unexpectedly stopped. Experimental, requires agent restart.
 # libvirt.events.enabled=false
+
+# Implicit host tags managed by agent.properties
+# host.tags=

agent/conf/cloudstack-agent.logrotate.in

@@ -15,11 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
-/var/log/cloudstack/agent/security_group.log /var/log/cloudstack/agent/resizevolume.log /var/log/cloudstack/agent/rolling-maintenance.log {
+/var/log/cloudstack/agent/security_group.log /var/log/cloudstack/agent/resizevolume.log /var/log/cloudstack/agent/rolling-maintenance.log /var/log/cloudstack/agent/agent.out /var/log/cloudstack/agent/agent.err {
     copytruncate
     daily
     rotate 5
     compress
     missingok
     size 10M
+    dateext
+    dateformat -%Y-%m-%d
 }