diff --git a/.asf.yaml b/.asf.yaml index 507a0ca6dd5e..8c1a5d51fdf1 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -58,6 +58,7 @@ github: - gpordeus - hsato03 - bernardodemarco + - abh1sar protected_branches: ~ diff --git a/.github/linters/.flake8 b/.github/linters/.flake8 index f250719ca198..3364ad14f290 100644 --- a/.github/linters/.flake8 +++ b/.github/linters/.flake8 @@ -22,8 +22,11 @@ # E224 Tab after operator # E227 Missing whitespace around bitwise or shift operator # E242 Tab after ',' +# E271 Multiple spaces after keyword +# E272 Multiple spaces before keyword # E273 Tab after keyword # E274 Tab before keyword +# E713 Test for membership should be 'not in' # E742 Do not define classes named 'I', 'O', or 'l' # E743 Do not define functions named 'I', 'O', or 'l' # E901 SyntaxError or IndentationError @@ -37,4 +40,4 @@ exclude = .git, venv -select = E112,E113,E133,E223,E224,E227,E242,E273,E274,E742,E743,E901,E902,W291,W292,W293,W391 +select = E112,E113,E133,E223,E224,E227,E242,E271,E272,E273,E274,E713,E742,E743,E901,E902,W291,W292,W293,W391 diff --git a/.github/linters/.markdown-lint.yml b/.github/linters/.markdown-lint.yml new file mode 100644 index 000000000000..df1b1a2825e3 --- /dev/null +++ b/.github/linters/.markdown-lint.yml @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# MD001/heading-increment Heading levels should only increment by one level at a time +MD001: false + +# MD003/heading-style Heading style +MD003: false + +# MD004/ul-style Unordered list style +MD004: false + +# MD007/ul-indent Unordered list indentation +MD007: false + +# MD009/no-trailing-spaces Trailing spaces +MD009: false + +# MD010/no-hard-tabs Hard tabs +MD010: false + +# MD012/no-multiple-blanks Multiple consecutive blank lines +MD012: false + +# MD013/line-length Line length +MD013: false + +# MD014/commands-show-output Dollar signs used before commands without showing output +MD014: false + +# MD018/no-missing-space-atx No space after hash on atx style heading +MD018: false + +# MD019/no-multiple-space-atx Multiple spaces after hash on atx style heading +MD019: false + +# MD022/blanks-around-headings Headings should be surrounded by blank lines +MD022: false + +# MD023/heading-start-left Headings must start at the beginning of the line +MD023: false + +# MD024/no-duplicate-heading Multiple headings with the same content +MD024: false + +# MD025/single-title/single-h1 Multiple top-level headings in the same document +MD025: false + +# MD026/no-trailing-punctuation Trailing punctuation in heading +MD026: false + +# MD028/no-blanks-blockquote Blank line inside blockquote +MD028: false + +# MD029/ol-prefix Ordered list item prefix +MD029: false + +# MD031/blanks-around-fences Fenced code blocks should be surrounded by blank lines +MD031: false + +# MD032/blanks-around-lists Lists should be surrounded by blank lines +MD032: false + +# MD033/no-inline-html Inline HTML +MD033: false + +# MD034/no-bare-urls Bare URL used +MD034: false + +# MD036/no-emphasis-as-heading Emphasis used instead of a heading +MD036: false + +# MD037/no-space-in-emphasis Spaces inside emphasis markers +MD037: false + +# MD040/fenced-code-language Fenced code blocks should have a language specified +MD040: false + +# MD041/first-line-heading/first-line-h1 First line in a file should be a top-level heading +MD041: false + +# MD046/code-block-style Code block style +MD046: false + +# MD052/reference-links-images Reference links and images should use a label that is defined +MD052: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fac2d6266fa5..133e2c35b4ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,9 @@ jobs: smoke/test_migration smoke/test_multipleips_per_nic smoke/test_nested_virtualization - smoke/test_set_sourcenat", + smoke/test_set_sourcenat + smoke/test_webhook_lifecycle + smoke/test_purge_expunged_vms", "smoke/test_network smoke/test_network_acl smoke/test_network_ipv6 @@ -132,6 +134,7 @@ jobs: smoke/test_usage smoke/test_usage_events smoke/test_vm_deployment_planner + smoke/test_vm_strict_host_tags smoke/test_vm_schedule smoke/test_vm_life_cycle smoke/test_vm_lifecycle_unmanage_import diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 784df0cf03ca..b6c814a36f4c 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -39,7 +39,7 @@ jobs: pip install pre-commit - name: Set PY run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: ~/.cache/pre-commit key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/main-sonar-check.yml b/.github/workflows/main-sonar-check.yml index 66bb1093e040..07d15583c825 100644 --- a/.github/workflows/main-sonar-check.yml +++ b/.github/workflows/main-sonar-check.yml @@ -44,14 +44,14 @@ jobs: cache: 'maven' - name: Cache SonarCloud packages - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} diff --git a/.github/workflows/sonar-check.yml b/.github/workflows/sonar-check.yml index 2ebcf1fb2db7..5d1bcc5dc223 100644 --- a/.github/workflows/sonar-check.yml +++ b/.github/workflows/sonar-check.yml @@ -46,14 +46,14 @@ jobs: cache: 'maven' - name: Cache SonarCloud packages - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9004ed9daeea..8736e5bac941 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,6 +36,8 @@ repos: - id: check-vcs-permalinks #- id: check-yaml - id: destroyed-symlinks + - id: detect-aws-credentials + args: [--allow-missing-credentials] - id: detect-private-key exclude: > (?x) @@ -53,9 +55,13 @@ repos: - id: end-of-file-fixer exclude: \.vhd$ #- id: fix-byte-order-marker + - id: forbid-submodules - id: mixed-line-ending exclude: \.(cs|xml)$ - # - id: trailing-whitespace + - id: trailing-whitespace + files: \.(header|in|java|md|properties|py|rb|sh|sql|txt|vue|yaml|yml)$ + args: [--markdown-linebreak-ext=md] + exclude: ^services/console-proxy/rdpconsole/src/test/doc/freerdp-debug-log\.txt$ - repo: https://github.com/pycqa/flake8 rev: 7.0.0 hooks: @@ -72,3 +78,12 @@ repos: ^scripts/vm/hypervisor/xenserver/vmopspremium$| ^setup/bindir/cloud-setup-encryption\.in$| ^venv/.*$ + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: v0.40.0 + hooks: + - id: markdownlint + name: run markdownlint + description: check Markdown files with markdownlint + args: [--config=.github/linters/.markdown-lint.yml] + types: [markdown] + files: \.(md|mdown|markdown)$ diff --git a/CHANGES.md b/CHANGES.md index ef498f8edf0b..9544fc014c75 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -430,11 +430,11 @@ Bug ID | Description [CLOUDSTACK-6099](https://issues.apache.org/jira/browse/CLOUDSTACK-6099) | live migration is failing for vm deployed using dynamic compute offerings with NPE [CLOUDSTACK-7528](https://issues.apache.org/jira/browse/CLOUDSTACK-7528) | More verbose logging when sending alert fails [CLOUDSTACK-6624](https://issues.apache.org/jira/browse/CLOUDSTACK-6624) | set specifyIpRanges to true if specifyVlan is set to true -[CLOUDSTACK-7404](https://issues.apache.org/jira/browse/CLOUDSTACK-7404) | Failed to start an instance when originating template has been deleted +[CLOUDSTACK-7404](https://issues.apache.org/jira/browse/CLOUDSTACK-7404) | Failed to start an instance when originating template has been deleted [CLOUDSTACK-6531](https://issues.apache.org/jira/browse/CLOUDSTACK-6531) | Stopping the router in case of command failures [CLOUDSTACK-6115](https://issues.apache.org/jira/browse/CLOUDSTACK-6115) | TravisCI configuration [CLOUDSTACK-7405](https://issues.apache.org/jira/browse/CLOUDSTACK-7405) | allowing VR meta-data to be accessed without trailing slash -[CLOUDSTACK-7260](https://issues.apache.org/jira/browse/CLOUDSTACK-7260) | Management server not responding after some time for Vmware due to Oom +[CLOUDSTACK-7260](https://issues.apache.org/jira/browse/CLOUDSTACK-7260) | Management server not responding after some time for Vmware due to Oom [CLOUDSTACK-7038](https://issues.apache.org/jira/browse/CLOUDSTACK-7038) | Add mysql client dependency for mgmt server pkg for debian [CLOUDSTACK-6892](https://issues.apache.org/jira/browse/CLOUDSTACK-6892) | Create separate package for the mysql HA component [CLOUDSTACK-7038](https://issues.apache.org/jira/browse/CLOUDSTACK-7038) | Add mysql client dependency for mgmt server/rpms @@ -449,12 +449,12 @@ Bug ID | Description [CLOUDSTACK-7006](https://issues.apache.org/jira/browse/CLOUDSTACK-7006) | Restore template ID in ROOT volume usages [CLOUDSTACK-6747](https://issues.apache.org/jira/browse/CLOUDSTACK-6747) | test to allow all cidrs on other end of vpc [CLOUDSTACK-6272](https://issues.apache.org/jira/browse/CLOUDSTACK-6272) | Fix recover/restore VM actions -[CLOUDSTACK-6927](https://issues.apache.org/jira/browse/CLOUDSTACK-6927) | store virsh list in list instead of querying libvirt +[CLOUDSTACK-6927](https://issues.apache.org/jira/browse/CLOUDSTACK-6927) | store virsh list in list instead of querying libvirt [CLOUDSTACK-6317](https://issues.apache.org/jira/browse/CLOUDSTACK-6317) | [VMware] Tagged VLAN support broken for Management/Control/Storage traffic [CLOUDSTACK-5891](https://issues.apache.org/jira/browse/CLOUDSTACK-5891) | [VMware] If a template has been registered and "cpu.corespersocket=X" , [CLOUDSTACK-6478](https://issues.apache.org/jira/browse/CLOUDSTACK-6478) | Failed to download Template when having 3 SSVM's in one [CLOUDSTACK-6464](https://issues.apache.org/jira/browse/CLOUDSTACK-6464) | if guest network type is vlan://untagged, and traffic label is used -[CLOUDSTACK-6816](https://issues.apache.org/jira/browse/CLOUDSTACK-6816) | bugfix: cloudstack-setup-management make /root directory's permission 0777 +[CLOUDSTACK-6816](https://issues.apache.org/jira/browse/CLOUDSTACK-6816) | bugfix: cloudstack-setup-management make /root directory's permission 0777 [CLOUDSTACK-6204](https://issues.apache.org/jira/browse/CLOUDSTACK-6204) | applying missed patch [CLOUDSTACK-6472](https://issues.apache.org/jira/browse/CLOUDSTACK-6472) | (4.3 specific) listUsageRecords: Pull information from removed items as well [CLOUDSTACK-5976](https://issues.apache.org/jira/browse/CLOUDSTACK-5976) | Typo in "ssh_keypairs" table's foreign key constraints on the Upgraded Setup @@ -657,11 +657,11 @@ Version 4.1.0 ------------- This is the second major release of CloudStack from within the Apache Software Foundation, and the -first major release as a Top-Level Project (TLP). +first major release as a Top-Level Project (TLP). Build Tool Changes: - * The project now uses Maven 3 exclusively to build. + * The project now uses Maven 3 exclusively to build. New Features: * CLOUDSTACK-101: OVS support in KVM @@ -976,14 +976,14 @@ Issues fixed in this release: * CLOUDSTACK-1845: KVM - storage migration often fails * CLOUDSTACK-1846: KVM - storage pools can silently fail to be unregistered, leading to failure to register later * CLOUDSTACK-2003: Deleting domain while deleted account is cleaning up leaves VMs expunging forever due to 'Failed to update resource count' -* CLOUDSTACK-2090: Upgrade from version 4.0.1 to version 4.0.2 triggers the 4.0.0 to 4.0.1. +* CLOUDSTACK-2090: Upgrade from version 4.0.1 to version 4.0.2 triggers the 4.0.0 to 4.0.1. * CLOUDSTACK-2091: Error in API documentation for 4.0.x. Version 4.0.1-incubating ------------------------ -This is a bugfix release for Apache CloudStack 4.0.0-incubating, with no new features. +This is a bugfix release for Apache CloudStack 4.0.0-incubating, with no new features. Security Fixes: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cdfbfe77b7ed..bb84e4e91fb3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ Contributing to Apache CloudStack (ACS) Summary ------- -This document covers how to contribute to the ACS project. ACS uses GitHub PRs to manage code contributions. +This document covers how to contribute to the ACS project. ACS uses GitHub PRs to manage code contributions. These instructions assume you have a GitHub.com account, so if you don't have one you will have to create one. Your proposed code changes will be published to your own fork of the ACS project and you will submit a Pull Request for your changes to be added. _Lets get started!!!_ @@ -11,17 +11,17 @@ _Lets get started!!!_ Bug fixes --------- -It's very important that we can easily track bug fix commits, so their hashes should remain the same in all branches. -Therefore, a pull request (PR) that fixes a bug, should be sent against a release branch. -This can be either the "current release" or the "previous release", depending on which ones are maintained. +It's very important that we can easily track bug fix commits, so their hashes should remain the same in all branches. +Therefore, a pull request (PR) that fixes a bug, should be sent against a release branch. +This can be either the "current release" or the "previous release", depending on which ones are maintained. Since the goal is a stable main, bug fixes should be "merged forward" to the next branch in order: "previous release" -> "current release" -> main (in other words: old to new) Developing new features ----------------------- -Development should be done in a feature branch, branched off of main. -Send a PR(steps below) to get it into main (2x LGTM applies). -PR will only be merged when main is open, will be held otherwise until main is open again. +Development should be done in a feature branch, branched off of main. +Send a PR(steps below) to get it into main (2x LGTM applies). +PR will only be merged when main is open, will be held otherwise until main is open again. No back porting / cherry-picking features to existing branches! PendingReleaseNotes file @@ -33,7 +33,7 @@ When adding information to the PendingReleaseNotes file make sure that you write Updating the PendingReleaseNotes file is preferably a part of the original Pull Request, but that is up to the developers' discretion. -Fork the code +Fork the code ------------- In your browser, navigate to: [https://github.com/apache/cloudstack](https://github.com/apache/cloudstack) @@ -136,4 +136,4 @@ $ git push origin :feature_x Release Principles ------------------ -Detailed information about ACS release principles is available at https://cwiki.apache.org/confluence/display/CLOUDSTACK/Release+principles+for+Apache+CloudStack+4.6+and+up +Detailed information about ACS release principles is available at https://cwiki.apache.org/confluence/display/CLOUDSTACK/Release+principles+for+Apache+CloudStack+4.6+and+up diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 8293a22973a2..e02cc6518535 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -23,6 +23,7 @@ This PR... - [ ] Enhancement (improves an existing feature and functionality) - [ ] Cleanup (Code refactoring and cleanup, that may add test cases) - [ ] build/CI +- [ ] test (unit or integration test code) ### Feature/Enhancement Scale or Bug Severity diff --git a/README.md b/README.md index e193913612f1..f66a4dc6f975 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ This distribution includes cryptographic software. The country in which you curr reside may have restrictions on the import, possession, use, and/or re-export to another country, of encryption software. BEFORE using any encryption software, please check your country's laws, regulations and policies concerning the import, possession, or use, and -re-export of encryption software, to see if this is permitted. See [The Wassenaar Arrangement](http://www.wassenaar.org/) +re-export of encryption software, to see if this is permitted. See [The Wassenaar Arrangement](http://www.wassenaar.org/) for more information. The U.S. Government Department of Commerce, Bureau of Industry and Security (BIS), has diff --git a/agent/bindir/cloud-setup-agent.in b/agent/bindir/cloud-setup-agent.in index 53c6c2f56aa4..18de64089ed0 100755 --- a/agent/bindir/cloud-setup-agent.in +++ b/agent/bindir/cloud-setup-agent.in @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/agent/bindir/cloud-ssh.in b/agent/bindir/cloud-ssh.in index e4b3c141a975..a5ea975c2f3a 100644 --- a/agent/bindir/cloud-ssh.in +++ b/agent/bindir/cloud-ssh.in @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index e600e8f8f201..3b6a7b7de292 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -430,3 +430,6 @@ iscsi.session.cleanup.enabled=false # If set to "true", the agent will register for libvirt domain events, allowing for immediate updates on crashed or # unexpectedly stopped. Experimental, requires agent restart. # libvirt.events.enabled=false + +# Implicit host tags managed by agent.properties +# host.tags= diff --git a/agent/conf/cloudstack-agent.logrotate.in b/agent/conf/cloudstack-agent.logrotate.in index 2b3dc87f2532..9f22b4bab868 100644 --- a/agent/conf/cloudstack-agent.logrotate.in +++ b/agent/conf/cloudstack-agent.logrotate.in @@ -15,11 +15,13 @@ # specific language governing permissions and limitations # under the License. -/var/log/cloudstack/agent/security_group.log /var/log/cloudstack/agent/resizevolume.log /var/log/cloudstack/agent/rolling-maintenance.log { +/var/log/cloudstack/agent/security_group.log /var/log/cloudstack/agent/resizevolume.log /var/log/cloudstack/agent/rolling-maintenance.log /var/log/cloudstack/agent/agent.out /var/log/cloudstack/agent/agent.err { copytruncate daily rotate 5 compress missingok size 10M + dateext + dateformat -%Y-%m-%d } diff --git a/agent/conf/environment.properties.in b/agent/conf/environment.properties.in index 514161a13fc5..b6cc5bbd9879 100644 --- a/agent/conf/environment.properties.in +++ b/agent/conf/environment.properties.in @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/agent/conf/log4j-cloud.xml.in b/agent/conf/log4j-cloud.xml.in index 44ebd1358af6..29c1d5ee6415 100644 --- a/agent/conf/log4j-cloud.xml.in +++ b/agent/conf/log4j-cloud.xml.in @@ -38,7 +38,7 @@ under the License. - + diff --git a/agent/src/main/java/com/cloud/agent/Agent.java b/agent/src/main/java/com/cloud/agent/Agent.java index 56732dad9936..d2d4f165979d 100644 --- a/agent/src/main/java/com/cloud/agent/Agent.java +++ b/agent/src/main/java/com/cloud/agent/Agent.java @@ -1127,6 +1127,12 @@ public void doTask(final Task task) throws TaskExecutionException { logger.error("Error parsing task", e); } } else if (task.getType() == Task.Type.DISCONNECT) { + try { + // an issue has been found if reconnect immediately after disconnecting. please refer to https://github.com/apache/cloudstack/issues/8517 + // wait 5 seconds before reconnecting + Thread.sleep(5000); + } catch (InterruptedException e) { + } reconnect(task.getLink()); return; } else if (task.getType() == Task.Type.OTHER) { diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 24a09ae2ac11..8f97edc39357 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -751,7 +751,7 @@ public Property getWorkers() { public static final Property IOTHREADS = new Property<>("iothreads", 1); /** - * Enable verbose mode for virt-v2v Instance Conversion from Vmware to KVM + * Enable verbose mode for virt-v2v Instance Conversion from VMware to KVM * Data type: Boolean.
* Default value: false */ @@ -803,6 +803,13 @@ public Property getWorkers() { */ public static final Property KEYSTORE_PASSPHRASE = new Property<>(KeyStoreUtils.KS_PASSPHRASE_PROPERTY, null, String.class); + /** + * Implicit host tags + * Data type: String.
+ * Default value: null + */ + public static final Property HOST_TAGS = new Property<>("host.tags", null, String.class); + public static class Property { private String name; private T defaultValue; diff --git a/agent/src/test/java/com/cloud/agent/AgentShellTest.java b/agent/src/test/java/com/cloud/agent/AgentShellTest.java index f7151779f585..4126692546f2 100644 --- a/agent/src/test/java/com/cloud/agent/AgentShellTest.java +++ b/agent/src/test/java/com/cloud/agent/AgentShellTest.java @@ -350,4 +350,16 @@ public void setHostTestValueIsNullPropertyDoesNotStartAndEndWithAtSignSetHosts() Mockito.verify(agentShellSpy).setHosts(expected); } + + @Test + public void updateAndGetConnectedHost() { + String expected = "test"; + + AgentShell shell = new AgentShell(); + shell.setHosts("test"); + shell.getNextHost(); + shell.updateConnectedHost(); + + Assert.assertEquals(expected, shell.getConnectedHost()); + } } diff --git a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java index 6e7aa8b21e28..d86eb2a3a7f7 100644 --- a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java @@ -18,40 +18,39 @@ */ package com.cloud.agent.api.to; +import java.io.Serializable; + import com.cloud.agent.api.LogLevel; import com.cloud.hypervisor.Hypervisor; -import java.io.Serializable; - public class RemoteInstanceTO implements Serializable { private Hypervisor.HypervisorType hypervisorType; - private String hostName; private String instanceName; - // Vmware Remote Instances parameters + // VMware Remote Instances parameters (required for exporting OVA through ovftool) // TODO: cloud.agent.transport.Request#getCommands() cannot handle gsoc decode for polymorphic classes private String vcenterUsername; @LogLevel(LogLevel.Log4jLevel.Off) private String vcenterPassword; private String vcenterHost; private String datacenterName; - private String clusterName; public RemoteInstanceTO() { } - public RemoteInstanceTO(String hostName, String instanceName, String vcenterHost, - String datacenterName, String clusterName, - String vcenterUsername, String vcenterPassword) { + public RemoteInstanceTO(String instanceName) { + this.hypervisorType = Hypervisor.HypervisorType.VMware; + this.instanceName = instanceName; + } + + public RemoteInstanceTO(String instanceName, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) { this.hypervisorType = Hypervisor.HypervisorType.VMware; - this.hostName = hostName; this.instanceName = instanceName; this.vcenterHost = vcenterHost; - this.datacenterName = datacenterName; - this.clusterName = clusterName; this.vcenterUsername = vcenterUsername; this.vcenterPassword = vcenterPassword; + this.datacenterName = datacenterName; } public Hypervisor.HypervisorType getHypervisorType() { @@ -62,10 +61,6 @@ public String getInstanceName() { return this.instanceName; } - public String getHostName() { - return this.hostName; - } - public String getVcenterUsername() { return vcenterUsername; } @@ -81,8 +76,4 @@ public String getVcenterHost() { public String getDatacenterName() { return datacenterName; } - - public String getClusterName() { - return clusterName; - } } diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 01ad12a71e08..d4235cbc9bcb 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -29,9 +29,9 @@ import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.ha.HAConfig; +import org.apache.cloudstack.quota.QuotaTariff; import org.apache.cloudstack.storage.object.Bucket; import org.apache.cloudstack.storage.object.ObjectStore; -import org.apache.cloudstack.quota.QuotaTariff; import org.apache.cloudstack.usage.Usage; import org.apache.cloudstack.vm.schedule.VMSchedule; @@ -451,6 +451,7 @@ public class EventTypes { public static final String EVENT_ENABLE_PRIMARY_STORAGE = "ENABLE.PS"; public static final String EVENT_DISABLE_PRIMARY_STORAGE = "DISABLE.PS"; public static final String EVENT_SYNC_STORAGE_POOL = "SYNC.STORAGE.POOL"; + public static final String EVENT_CHANGE_STORAGE_POOL_SCOPE = "CHANGE.STORAGE.POOL.SCOPE"; // VPN public static final String EVENT_REMOTE_ACCESS_VPN_CREATE = "VPN.REMOTE.ACCESS.CREATE"; @@ -721,6 +722,8 @@ public class EventTypes { // SystemVM public static final String EVENT_LIVE_PATCH_SYSTEMVM = "LIVE.PATCH.SYSTEM.VM"; + //Purge resources + public static final String EVENT_PURGE_EXPUNGED_RESOURCES = "PURGE.EXPUNGED.RESOURCES"; // OBJECT STORE public static final String EVENT_OBJECT_STORE_CREATE = "OBJECT.STORE.CREATE"; @@ -1000,6 +1003,7 @@ public class EventTypes { // Primary storage pool entityEventDetails.put(EVENT_ENABLE_PRIMARY_STORAGE, StoragePool.class); entityEventDetails.put(EVENT_DISABLE_PRIMARY_STORAGE, StoragePool.class); + entityEventDetails.put(EVENT_CHANGE_STORAGE_POOL_SCOPE, StoragePool.class); // VPN entityEventDetails.put(EVENT_REMOTE_ACCESS_VPN_CREATE, RemoteAccessVpn.class); @@ -1229,4 +1233,8 @@ public static Class getEntityClassForEvent(String eventName) { public static boolean isVpcEvent(String eventType) { return EventTypes.EVENT_VPC_CREATE.equals(eventType) || EventTypes.EVENT_VPC_DELETE.equals(eventType); } + + public static void addEntityEventDetail(String event, Class clazz) { + entityEventDetails.put(event, clazz); + } } diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index 7563bc3b7426..4a3b914364f8 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -54,6 +54,7 @@ public static String[] toStrings(Host.Type... types) { } public static final String HOST_UEFI_ENABLE = "host.uefi.enable"; public static final String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; + public static final String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; /** * @return name of the machine. diff --git a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java index 3c7dbac6442c..0c821b4e36c0 100644 --- a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java +++ b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.agent.api.Command; +import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.hypervisor.Hypervisor.HypervisorType; @@ -101,21 +102,20 @@ boolean attachRestoredVolumeToVirtualMachine(long zoneId, String location, Backu * Will generate commands to migrate a vm to a pool. For now this will only work for stopped VMs on Vmware. * * @param vm the stopped vm to migrate - * @param destination the primary storage pool to migrate to + * @param volumeToPool the primary storage pools to migrate to * @return a list of commands to perform for a successful migration */ List finalizeMigrate(VirtualMachine vm, Map volumeToPool); /** - * Will perform a clone of a VM on an external host (if the guru can handle) + * Will return the hypervisor VM (clone VM for PowerOn VMs), performs a clone of a VM if required on an external host (if the guru can handle) * @param hostIp VM's source host IP - * @param vmName name of the source VM to clone from + * @param vmName name of the source VM (clone VM name if cloned) * @param params hypervisor specific additional parameters - * @return a reference to the cloned VM + * @return a reference to the hypervisor or cloned VM, and cloned flag */ - UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmName, - Map params); + Pair getHypervisorVMOutOfBandAndCloneIfRequired(String hostIp, String vmName, Map params); /** * Removes a VM created as a clone of a VM on an external host @@ -124,6 +124,23 @@ UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmName, * @param params hypervisor specific additional parameters * @return true if the operation succeeds, false if not */ - boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName, - Map params); + boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName, Map params); + + /** + * Create an OVA/OVF template of a VM on an external host (if the guru can handle) + * @param hostIp VM's source host IP + * @param vmName name of the source VM to create template from + * @param params hypervisor specific additional parameters + * @param templateLocation datastore to create the template file + * @return the created template dir/name + */ + String createVMTemplateOutOfBand(String hostIp, String vmName, Map params, DataStoreTO templateLocation, int threadsCountToExportOvf); + + /** + * Removes the template on the location + * @param templateLocation datastore to remove the template file + * @param templateDir the template dir to remove from datastore + * @return true if the operation succeeds, false if not + */ + boolean removeVMTemplateOutOfBand(DataStoreTO templateLocation, String templateDir); } diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java similarity index 88% rename from api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java rename to api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java index e160227749db..a13c1b3a6a89 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java @@ -16,11 +16,14 @@ // under the License. package com.cloud.kubernetes.cluster; -import com.cloud.utils.component.Adapter; import org.apache.cloudstack.acl.ControlledEntity; -public interface KubernetesClusterHelper extends Adapter { +import com.cloud.uservm.UserVm; +import com.cloud.utils.component.Adapter; + +public interface KubernetesServiceHelper extends Adapter { ControlledEntity findByUuid(String uuid); ControlledEntity findByVmId(long vmId); + void checkVmCanBeDestroyed(UserVm userVm); } diff --git a/api/src/main/java/com/cloud/network/NetworkModel.java b/api/src/main/java/com/cloud/network/NetworkModel.java index 53ac735cf050..699dcbf6c508 100644 --- a/api/src/main/java/com/cloud/network/NetworkModel.java +++ b/api/src/main/java/com/cloud/network/NetworkModel.java @@ -317,6 +317,8 @@ public interface NetworkModel { void checkIp6Parameters(String startIPv6, String endIPv6, String ip6Gateway, String ip6Cidr) throws InvalidParameterValueException; + void checkIp6CidrSizeEqualTo64(String ip6Cidr) throws InvalidParameterValueException; + void checkRequestedIpAddresses(long networkId, IpAddresses ips) throws InvalidParameterValueException; String getStartIpv6Address(long id); diff --git a/api/src/main/java/com/cloud/network/NetworkService.java b/api/src/main/java/com/cloud/network/NetworkService.java index 51799e25cda6..b8dd464b3655 100644 --- a/api/src/main/java/com/cloud/network/NetworkService.java +++ b/api/src/main/java/com/cloud/network/NetworkService.java @@ -20,6 +20,7 @@ import java.util.Map; import com.cloud.dc.DataCenter; +import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.command.admin.address.ReleasePodIpCmdByAdmin; import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd; import org.apache.cloudstack.api.command.admin.network.ListDedicatedGuestVlanRangesCmd; @@ -102,6 +103,10 @@ IpAddress allocatePortableIP(Account ipOwner, int regionId, Long zoneId, Long ne Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException; + Network createGuestNetwork(long networkOfferingId, String name, String displayText, Account owner, + PhysicalNetwork physicalNetwork, long zoneId, ControlledEntity.ACLType aclType) throws + InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException; + Pair, Integer> searchForNetworks(ListNetworksCmd cmd); boolean deleteNetwork(long networkId, boolean forced); diff --git a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java index c47500c78495..cb92739d2837 100644 --- a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java +++ b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java @@ -17,17 +17,22 @@ package com.cloud.network; import java.util.List; +import java.util.Map; import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd; import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd; +import com.cloud.deploy.DeploymentPlanner; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.network.router.VirtualRouter; import com.cloud.user.Account; import com.cloud.utils.Pair; import com.cloud.vm.Nic; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; public interface VirtualNetworkApplianceService { /** @@ -62,6 +67,10 @@ public interface VirtualNetworkApplianceService { VirtualRouter startRouter(long id) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException; + void startRouterForHA(VirtualMachine vm, Map params, DeploymentPlanner planner) + throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, + OperationTimedoutException; + VirtualRouter destroyRouter(long routerId, Account caller, Long callerUserId) throws ResourceUnavailableException, ConcurrentOperationException; VirtualRouter findRouter(long routerId); diff --git a/api/src/main/java/com/cloud/network/VpcVirtualNetworkApplianceService.java b/api/src/main/java/com/cloud/network/VpcVirtualNetworkApplianceService.java index 5c3ee3f1032a..cd04db802cac 100644 --- a/api/src/main/java/com/cloud/network/VpcVirtualNetworkApplianceService.java +++ b/api/src/main/java/com/cloud/network/VpcVirtualNetworkApplianceService.java @@ -29,7 +29,6 @@ public interface VpcVirtualNetworkApplianceService extends VirtualNetworkApplian /** * @param router * @param network - * @param isRedundant * @param params TODO * @return * @throws ConcurrentOperationException @@ -42,11 +41,30 @@ boolean addVpcRouterToGuestNetwork(VirtualRouter router, Network network, Map updateHealthChecks(Network network, List lbrules); boolean handlesOnlyRulesInTransitionState(); + + default void expungeLbVmRefs(List vmIds, Long batchSize) { + } } diff --git a/api/src/main/java/com/cloud/network/guru/NetworkGuru.java b/api/src/main/java/com/cloud/network/guru/NetworkGuru.java index cbadbb18a8f1..7b81c75ed845 100644 --- a/api/src/main/java/com/cloud/network/guru/NetworkGuru.java +++ b/api/src/main/java/com/cloud/network/guru/NetworkGuru.java @@ -212,4 +212,7 @@ void reserve(NicProfile nic, Network network, VirtualMachineProfile vm, DeployDe boolean isMyTrafficType(TrafficType type); + default boolean isSlaacV6Only() { + return true; + } } diff --git a/api/src/main/java/com/cloud/network/vpc/VpcService.java b/api/src/main/java/com/cloud/network/vpc/VpcService.java index 2cdc034a16e1..0f0d29f4082c 100644 --- a/api/src/main/java/com/cloud/network/vpc/VpcService.java +++ b/api/src/main/java/com/cloud/network/vpc/VpcService.java @@ -132,6 +132,8 @@ Pair, Integer> listVpcs(Long id, String vpcName, String disp */ boolean startVpc(long vpcId, boolean destroyOnFailure) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException; + void startVpc(CreateVPCCmd cmd) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException; + /** * Shuts down the VPC which includes shutting down all VPC provider and rules cleanup on the backend * diff --git a/api/src/main/java/com/cloud/offering/ServiceOffering.java b/api/src/main/java/com/cloud/offering/ServiceOffering.java index 58c7b0dbaf96..acb7a9f1cf91 100644 --- a/api/src/main/java/com/cloud/offering/ServiceOffering.java +++ b/api/src/main/java/com/cloud/offering/ServiceOffering.java @@ -33,6 +33,9 @@ public interface ServiceOffering extends InfrastructureEntity, InternalIdentity, static final String internalLbVmDefaultOffUniqueName = "Cloud.Com-InternalLBVm"; // leaving cloud.com references as these are identifyers and no real world addresses (check against DB) + + static final String PURGE_DB_ENTITIES_KEY = "purge.db.entities"; + enum State { Inactive, Active, } diff --git a/api/src/main/java/com/cloud/storage/StorageService.java b/api/src/main/java/com/cloud/storage/StorageService.java index c3609cfd8eea..1ce335b01153 100644 --- a/api/src/main/java/com/cloud/storage/StorageService.java +++ b/api/src/main/java/com/cloud/storage/StorageService.java @@ -21,6 +21,7 @@ import java.util.Map; import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd; +import org.apache.cloudstack.api.command.admin.storage.ChangeStoragePoolScopeCmd; import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd; @@ -29,11 +30,13 @@ import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.UpdateImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; import com.cloud.exception.DiscoveryException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceInUseException; import com.cloud.exception.ResourceUnavailableException; import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd; @@ -110,6 +113,8 @@ public interface StorageService { */ ImageStore migrateToObjectStore(String name, String url, String providerName, Map details) throws DiscoveryException; + ImageStore updateImageStore(UpdateImageStoreCmd cmd); + ImageStore updateImageStoreStatus(Long id, Boolean readonly); void updateStorageCapabilities(Long poolId, boolean failOnChecks); @@ -127,4 +132,6 @@ public interface StorageService { boolean deleteObjectStore(DeleteObjectStoragePoolCmd cmd); ObjectStore updateObjectStore(Long id, UpdateObjectStoragePoolCmd cmd); + + void changeStoragePoolScope(ChangeStoragePoolScopeCmd cmd) throws IllegalArgumentException, InvalidParameterValueException, PermissionDeniedException; } diff --git a/api/src/main/java/com/cloud/user/ResourceLimitService.java b/api/src/main/java/com/cloud/user/ResourceLimitService.java index d0aa9f69f840..3b30b8fc4a57 100644 --- a/api/src/main/java/com/cloud/user/ResourceLimitService.java +++ b/api/src/main/java/com/cloud/user/ResourceLimitService.java @@ -38,7 +38,10 @@ public interface ResourceLimitService { static final ConfigKey MaxProjectSecondaryStorage = new ConfigKey<>("Project Defaults", Long.class, "max.project.secondary.storage", "400", "The default maximum secondary storage space (in GiB) that can be used for a project", false); static final ConfigKey ResourceCountCheckInterval = new ConfigKey<>("Advanced", Long.class, "resourcecount.check.interval", "300", - "Time (in seconds) to wait before running resource recalculation and fixing task. Default is 300 seconds, Setting this to 0 disables execution of the task", true); + "Time (in seconds) to wait before running resource recalculation and fixing tasks like stale resource reservation cleanup" + + ". Default is 300 seconds, Setting this to 0 disables execution of the task", true); + static final ConfigKey ResourceReservationCleanupDelay = new ConfigKey<>("Advanced", Long.class, "resource.reservation.cleanup.delay", "3600", + "Time (in seconds) after which a resource reservation gets deleted. Default is 3600 seconds, Setting this to 0 disables execution of the task", true); static final ConfigKey ResourceLimitHostTags = new ConfigKey<>("Advanced", String.class, "resource.limit.host.tags", "", "A comma-separated list of tags for host resource limits", true); static final ConfigKey ResourceLimitStorageTags = new ConfigKey<>("Advanced", String.class, "resource.limit.storage.tags", "", @@ -243,6 +246,8 @@ public interface ResourceLimitService { void checkVolumeResourceLimitForDiskOfferingChange(Account owner, Boolean display, Long currentSize, Long newSize, DiskOffering currentOffering, DiskOffering newOffering) throws ResourceAllocationException; + void checkPrimaryStorageResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering) throws ResourceAllocationException; + void incrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); diff --git a/api/src/main/java/com/cloud/uservm/UserVm.java b/api/src/main/java/com/cloud/uservm/UserVm.java index e30f5e030544..9035d2903c9a 100644 --- a/api/src/main/java/com/cloud/uservm/UserVm.java +++ b/api/src/main/java/com/cloud/uservm/UserVm.java @@ -48,4 +48,6 @@ public interface UserVm extends VirtualMachine, ControlledEntity { void setAccountId(long accountId); public boolean isDisplayVm(); + + String getUserVmType(); } diff --git a/api/src/main/java/com/cloud/vm/NicProfile.java b/api/src/main/java/com/cloud/vm/NicProfile.java index d3c1daa1f5da..183c8dcb2d59 100644 --- a/api/src/main/java/com/cloud/vm/NicProfile.java +++ b/api/src/main/java/com/cloud/vm/NicProfile.java @@ -62,6 +62,7 @@ public class NicProfile implements InternalIdentity, Serializable { String iPv4Dns1; String iPv4Dns2; String requestedIPv4; + boolean ipv4AllocationRaceCheck; // IPv6 String iPv6Address; @@ -405,6 +406,13 @@ public void setMtu(Integer mtu) { this.mtu = mtu; } + public boolean getIpv4AllocationRaceCheck() { + return this.ipv4AllocationRaceCheck; + } + + public void setIpv4AllocationRaceCheck(boolean ipv4AllocationRaceCheck) { + this.ipv4AllocationRaceCheck = ipv4AllocationRaceCheck; + } // // OTHER METHODS diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 9d8b196a4ff5..5a9301eef7fa 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -42,9 +42,11 @@ import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd; import com.cloud.dc.DataCenter; +import com.cloud.deploy.DeploymentPlanner; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.ManagementServerException; +import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.StorageUnavailableException; @@ -66,10 +68,7 @@ public interface UserVmService { /** * Destroys one virtual machine * - * @param userId - * the id of the user performing the action - * @param vmId - * the id of the virtual machine. + * @param cmd the API Command Object containg the parameters to use for this service action * @throws ConcurrentOperationException * @throws ResourceUnavailableException */ @@ -112,6 +111,12 @@ UserVm startVirtualMachine(StartVMCmd cmd) throws StorageUnavailableException, E UserVm rebootVirtualMachine(RebootVMCmd cmd) throws InsufficientCapacityException, ResourceUnavailableException, ResourceAllocationException; + void startVirtualMachine(UserVm vm) throws OperationTimedoutException, ResourceUnavailableException, InsufficientCapacityException; + + void startVirtualMachineForHA(VirtualMachine vm, Map params, + DeploymentPlanner planner) throws InsufficientCapacityException, ResourceUnavailableException, + ConcurrentOperationException, OperationTimedoutException; + UserVm updateVirtualMachine(UpdateVMCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException; /** @@ -148,14 +153,6 @@ UserVm startVirtualMachine(StartVMCmd cmd) throws StorageUnavailableException, E * Creates a Basic Zone User VM in the database and returns the VM to the * caller. * - * - * - * @param sshKeyPair - * - name of the ssh key pair used to login to the virtual - * machine - * @param cpuSpeed - * @param memory - * @param cpuNumber * @param zone * - availability zone for the virtual machine * @param serviceOffering @@ -231,9 +228,6 @@ UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering s * Creates a User VM in Advanced Zone (Security Group feature is enabled) in * the database and returns the VM to the caller. * - * - * - * @param type * @param zone * - availability zone for the virtual machine * @param serviceOffering @@ -309,14 +303,6 @@ UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, ServiceOfferin * Creates a User VM in Advanced Zone (Security Group feature is disabled) * in the database and returns the VM to the caller. * - * - * - * @param sshKeyPair - * - name of the ssh key pair used to login to the virtual - * machine - * @param cpuSpeed - * @param memory - * @param cpuNumber * @param zone * - availability zone for the virtual machine * @param serviceOffering diff --git a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java index f2ff3da8449e..c67ee4eabc28 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java @@ -192,6 +192,10 @@ public boolean equals(Object obj) { Map getParameters(); + void setCpuOvercommitRatio(Float cpuOvercommitRatio); + + void setMemoryOvercommitRatio(Float memoryOvercommitRatio); + Float getCpuOvercommitRatio(); Float getMemoryOvercommitRatio(); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java index aafc039b36b5..938936765167 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java @@ -17,7 +17,9 @@ package org.apache.cloudstack.api; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.cloudstack.region.PortableIp; import org.apache.commons.collections.CollectionUtils; @@ -81,15 +83,22 @@ public enum ApiCommandResourceType { ManagementServer(org.apache.cloudstack.management.ManagementServerHost.class), ObjectStore(org.apache.cloudstack.storage.object.ObjectStore.class), Bucket(org.apache.cloudstack.storage.object.Bucket.class), - QuotaTariff(org.apache.cloudstack.quota.QuotaTariff.class); + QuotaTariff(org.apache.cloudstack.quota.QuotaTariff.class), + KubernetesCluster(null), + KubernetesSupportedVersion(null); private final Class clazz; + static final Map> additionalClassMappings = new HashMap<>(); + private ApiCommandResourceType(Class clazz) { this.clazz = clazz; } public Class getAssociatedClass() { + if (this.clazz == null && additionalClassMappings.containsKey(this)) { + return additionalClassMappings.get(this); + } return this.clazz; } @@ -119,4 +128,8 @@ public static ApiCommandResourceType fromString(String value) { } return null; } + + public static void setClassMapping(ApiCommandResourceType type, Class clazz) { + additionalClassMappings.put(type, clazz); + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 6dfcf6561244..6db1ed06eff5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -175,6 +175,8 @@ public class ApiConstants { public static final String END_IPV6 = "endipv6"; public static final String END_PORT = "endport"; public static final String ENTRY_TIME = "entrytime"; + public static final String EVENT_ID = "eventid"; + public static final String EVENT_TYPE = "eventtype"; public static final String EXPIRES = "expires"; public static final String EXTRA_CONFIG = "extraconfig"; public static final String EXTRA_DHCP_OPTION = "extradhcpoption"; @@ -189,6 +191,7 @@ public class ApiConstants { public static final String FORCED = "forced"; public static final String FORCED_DESTROY_LOCAL_STORAGE = "forcedestroylocalstorage"; public static final String FORCE_DELETE_HOST = "forcedeletehost"; + public static final String FORCE_MS_TO_IMPORT_VM_FILES = "forcemstoimportvmfiles"; public static final String FORMAT = "format"; public static final String FOR_VIRTUAL_NETWORK = "forvirtualnetwork"; public static final String FOR_SYSTEM_VMS = "forsystemvms"; @@ -209,6 +212,7 @@ public class ApiConstants { public static final String HA_PROVIDER = "haprovider"; public static final String HA_STATE = "hastate"; public static final String HEALTH = "health"; + public static final String HEADERS = "headers"; public static final String HIDE_IP_ADDRESS_USAGE = "hideipaddressusage"; public static final String HOST_ID = "hostid"; public static final String HOST_IDS = "hostids"; @@ -236,6 +240,7 @@ public class ApiConstants { public static final String NEXT_ACL_RULE_ID = "nextaclruleid"; public static final String MOVE_ACL_CONSISTENCY_HASH = "aclconsistencyhash"; public static final String IMAGE_PATH = "imagepath"; + public static final String INSTANCE_CONVERSION_SUPPORTED = "instanceconversionsupported"; public static final String INTERNAL_DNS1 = "internaldns1"; public static final String INTERNAL_DNS2 = "internaldns2"; public static final String INTERNET_PROTOCOL = "internetprotocol"; @@ -263,8 +268,10 @@ public class ApiConstants { public static final String IS_CLEANUP_REQUIRED = "iscleanuprequired"; public static final String IS_DYNAMIC = "isdynamic"; public static final String IS_EDGE = "isedge"; + public static final String IS_ENCRYPTED = "isencrypted"; public static final String IS_EXTRACTABLE = "isextractable"; public static final String IS_FEATURED = "isfeatured"; + public static final String IS_IMPLICIT = "isimplicit"; public static final String IS_PORTABLE = "isportable"; public static final String IS_PUBLIC = "ispublic"; public static final String IS_PERSISTENT = "ispersistent"; @@ -280,6 +287,7 @@ public class ApiConstants { public static final String JOB_STATUS = "jobstatus"; public static final String KEEPALIVE_ENABLED = "keepaliveenabled"; public static final String KERNEL_VERSION = "kernelversion"; + public static final String KEY = "key"; public static final String LABEL = "label"; public static final String LASTNAME = "lastname"; public static final String LAST_BOOT = "lastboottime"; @@ -354,6 +362,7 @@ public class ApiConstants { public static final String SSHKEY_ENABLED = "sshkeyenabled"; public static final String PATH = "path"; public static final String PAYLOAD = "payload"; + public static final String PAYLOAD_URL = "payloadurl"; public static final String POD_ID = "podid"; public static final String POD_NAME = "podname"; public static final String POD_IDS = "podids"; @@ -381,6 +390,7 @@ public class ApiConstants { public static final String PUBLIC_START_PORT = "publicport"; public static final String PUBLIC_END_PORT = "publicendport"; public static final String PUBLIC_ZONE = "publiczone"; + public static final String PURGE_RESOURCES = "purgeresources"; public static final String RECEIVED_BYTES = "receivedbytes"; public static final String RECONNECT = "reconnect"; public static final String RECOVER = "recover"; @@ -399,11 +409,9 @@ public class ApiConstants { public static final String QUERY_FILTER = "queryfilter"; public static final String SCHEDULE = "schedule"; public static final String SCOPE = "scope"; - public static final String SECRET_KEY = "usersecretkey"; - public static final String SECONDARY_IP = "secondaryip"; - public static final String SINCE = "since"; - public static final String KEY = "key"; public static final String SEARCH_BASE = "searchbase"; + public static final String SECONDARY_IP = "secondaryip"; + public static final String SECRET_KEY = "secretkey"; public static final String SECURITY_GROUP_IDS = "securitygroupids"; public static final String SECURITY_GROUP_NAMES = "securitygroupnames"; public static final String SECURITY_GROUP_NAME = "securitygroupname"; @@ -421,6 +429,7 @@ public class ApiConstants { public static final String SHOW_UNIQUE = "showunique"; public static final String SIGNATURE = "signature"; public static final String SIGNATURE_VERSION = "signatureversion"; + public static final String SINCE = "since"; public static final String SIZE = "size"; public static final String SNAPSHOT = "snapshot"; public static final String SNAPSHOT_ID = "snapshotid"; @@ -428,8 +437,7 @@ public class ApiConstants { public static final String SNAPSHOT_TYPE = "snapshottype"; public static final String SNAPSHOT_QUIESCEVM = "quiescevm"; public static final String SOURCE_ZONE_ID = "sourcezoneid"; - public static final String SUITABLE_FOR_VM = "suitableforvirtualmachine"; - public static final String SUPPORTS_STORAGE_SNAPSHOT = "supportsstoragesnapshot"; + public static final String SSL_VERIFICATION = "sslverification"; public static final String START_DATE = "startdate"; public static final String START_ID = "startid"; public static final String START_IP = "startip"; @@ -442,18 +450,23 @@ public class ApiConstants { public static final String STORAGE_POLICY = "storagepolicy"; public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled"; public static final String STORAGE_CAPABILITIES = "storagecapabilities"; + public static final String STORAGE_CUSTOM_STATS = "storagecustomstats"; public static final String SUBNET = "subnet"; public static final String OWNER = "owner"; public static final String SWAP_OWNER = "swapowner"; public static final String SYSTEM_VM_TYPE = "systemvmtype"; public static final String TAGS = "tags"; public static final String STORAGE_TAGS = "storagetags"; + public static final String SUCCESS = "success"; + public static final String SUITABLE_FOR_VM = "suitableforvirtualmachine"; + public static final String SUPPORTS_STORAGE_SNAPSHOT = "supportsstoragesnapshot"; public static final String TARGET_IQN = "targetiqn"; public static final String TEMPLATE_FILTER = "templatefilter"; public static final String TEMPLATE_ID = "templateid"; public static final String TEMPLATE_IDS = "templateids"; public static final String TEMPLATE_NAME = "templatename"; public static final String TEMPLATE_TYPE = "templatetype"; + public static final String TEMPLATE_FORMAT = "templateformat"; public static final String TIMEOUT = "timeout"; public static final String TIMEZONE = "timezone"; public static final String TIMEZONEOFFSET = "timezoneoffset"; @@ -480,6 +493,7 @@ public class ApiConstants { public static final String USERNAME = "username"; public static final String USER_CONFIGURABLE = "userconfigurable"; public static final String USER_SECURITY_GROUP_LIST = "usersecuritygrouplist"; + public static final String USER_SECRET_KEY = "usersecretkey"; public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork"; public static final String UPDATE_IN_SEQUENCE = "updateinsequence"; public static final String VALUE = "value"; @@ -559,6 +573,7 @@ public class ApiConstants { public static final String ALLOCATION_STATE = "allocationstate"; public static final String MANAGED_STATE = "managedstate"; public static final String MANAGEMENT_SERVER_ID = "managementserverid"; + public static final String MANAGEMENT_SERVER_NAME = "managementservername"; public static final String STORAGE = "storage"; public static final String STORAGE_ID = "storageid"; public static final String PING_STORAGE_SERVER_IP = "pingstorageserverip"; @@ -898,6 +913,7 @@ public class ApiConstants { public static final String AUTOSCALE_VMGROUP_NAME = "autoscalevmgroupname"; public static final String BAREMETAL_DISCOVER_NAME = "baremetaldiscovername"; public static final String BAREMETAL_RCT_URL = "baremetalrcturl"; + public static final String BATCH_SIZE = "batchsize"; public static final String UCS_DN = "ucsdn"; public static final String GSLB_PROVIDER = "gslbprovider"; public static final String EXCLUSIVE_GSLB_PROVIDER = "isexclusivegslbprovider"; @@ -1119,6 +1135,11 @@ public class ApiConstants { public static final String PARAMETER_DESCRIPTION_IS_TAG_A_RULE = "Whether the informed tag is a JS interpretable rule or not."; + public static final String WEBHOOK_ID = "webhookid"; + public static final String WEBHOOK_NAME = "webhookname"; + + public static final String NFS_MOUNT_OPTIONS = "nfsmountopts"; + /** * This enum specifies IO Drivers, each option controls specific policies on I/O. * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0). @@ -1141,6 +1162,14 @@ public String toString() { } } + public static final String PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS = "The recommended format is \"yyyy-MM-dd'T'HH:mm:ssZ\" (e.g.: \"2023-01-01T12:00:00+0100\"); " + + "however, the following formats are also accepted: \"yyyy-MM-dd HH:mm:ss\" (e.g.: \"2023-01-01 12:00:00\") and \"yyyy-MM-dd\" (e.g.: \"2023-01-01\" - if the time is not " + + "added, it will be interpreted as \"00:00:00\"). If the recommended format is not used, the date will be considered in the server timezone."; + + public static final String PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS = "The recommended format is \"yyyy-MM-dd'T'HH:mm:ssZ\" (e.g.: \"2023-01-01T12:00:00+0100\"); " + + "however, the following formats are also accepted: \"yyyy-MM-dd HH:mm:ss\" (e.g.: \"2023-01-01 12:00:00\") and \"yyyy-MM-dd\" (e.g.: \"2023-01-01\" - if the time is not " + + "added, it will be interpreted as \"23:59:59\"). If the recommended format is not used, the date will be considered in the server timezone."; + public enum BootType { UEFI, BIOS; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateHypervisorCapabilitiesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateHypervisorCapabilitiesCmd.java index 50984188bf56..01f7af108416 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateHypervisorCapabilitiesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/config/UpdateHypervisorCapabilitiesCmd.java @@ -43,6 +43,12 @@ public class UpdateHypervisorCapabilitiesCmd extends BaseCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = HypervisorCapabilitiesResponse.class, description = "ID of the hypervisor capability") private Long id; + @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, description = "the hypervisor for which the hypervisor capabilities are to be updated", since = "4.19.1") + private String hypervisor; + + @Parameter(name = ApiConstants.HYPERVISOR_VERSION, type = CommandType.STRING, description = "the hypervisor version for which the hypervisor capabilities are to be updated", since = "4.19.1") + private String hypervisorVersion; + @Parameter(name = ApiConstants.SECURITY_GROUP_EANBLED, type = CommandType.BOOLEAN, description = "set true to enable security group for this hypervisor.") private Boolean securityGroupEnabled; @@ -73,6 +79,14 @@ public Long getId() { return id; } + public String getHypervisor() { + return hypervisor; + } + + public String getHypervisorVersion() { + return hypervisorVersion; + } + public Long getMaxGuestsLimit() { return maxGuestsLimit; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java index 4562aa7da19e..8f6d5413d72d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java @@ -54,7 +54,11 @@ public class CreateServiceOfferingCmd extends BaseCmd { @Parameter(name = ApiConstants.CPU_NUMBER, type = CommandType.INTEGER, required = false, description = "the CPU number of the service offering") private Integer cpuNumber; - @Parameter(name = ApiConstants.CPU_SPEED, type = CommandType.INTEGER, required = false, description = "the CPU speed of the service offering in MHz.") + @Parameter(name = ApiConstants.CPU_SPEED, type = CommandType.INTEGER, required = false, description = "For VMware and Xen based hypervisors this is the CPU speed of the service offering in MHz.\n" + + "For the KVM hypervisor," + + " the values of the parameters cpuSpeed and cpuNumber will be used to calculate the `shares` value. This value is used by the KVM hypervisor to calculate how much time" + + " the VM will have access to the host's CPU. The `shares` value does not have a unit, and its purpose is being a weight value for the host to compare between its guest" + + " VMs. For more information, see https://libvirt.org/formatdomain.html#cpu-tuning.") private Integer cpuSpeed; @Parameter(name = ApiConstants.DISPLAY_TEXT, type = CommandType.STRING, description = "The display text of the service offering, defaults to 'name'.") @@ -242,6 +246,12 @@ public class CreateServiceOfferingCmd extends BaseCmd { @Parameter(name = ApiConstants.ENCRYPT_ROOT, type = CommandType.BOOLEAN, description = "VMs using this offering require root volume encryption", since="4.18") private Boolean encryptRoot; + @Parameter(name = ApiConstants.PURGE_RESOURCES, type = CommandType.BOOLEAN, + description = "Whether to cleanup instance and its associated resource from database upon expunge of the instance", + since="4.20") + private Boolean purgeResources; + + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -269,7 +279,7 @@ public Integer getMemory() { public String getServiceOfferingName() { if (StringUtils.isEmpty(serviceOfferingName)) { - throw new InvalidParameterValueException("Failed to create service offering because offering name has not been spified."); + throw new InvalidParameterValueException("Failed to create service offering because offering name has not been specified."); } return serviceOfferingName; } @@ -477,6 +487,10 @@ public boolean getEncryptRoot() { return false; } + public boolean isPurgeResources() { + return Boolean.TRUE.equals(purgeResources); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java index 7d6bae860834..e580f0d9f41a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java @@ -89,6 +89,11 @@ public class UpdateServiceOfferingCmd extends BaseCmd { description = "state of the service offering") private String serviceOfferingState; + @Parameter(name = ApiConstants.PURGE_RESOURCES, type = CommandType.BOOLEAN, + description = "Whether to cleanup VM and its associated resource upon expunge", + since="4.20") + private Boolean purgeResources; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -185,6 +190,10 @@ public State getState() { return state; } + public boolean isPurgeResources() { + return Boolean.TRUE.equals(purgeResources); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/PurgeExpungedResourcesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/PurgeExpungedResourcesCmd.java new file mode 100644 index 000000000000..b6833f097336 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/PurgeExpungedResourcesCmd.java @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.resource; + + +import java.util.Date; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.PurgeExpungedResourcesResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.resource.ResourceCleanupService; + +import com.cloud.event.EventTypes; + +@APICommand(name = "purgeExpungedResources", + description = "Purge expunged resources", + responseObject = SuccessResponse.class, + responseView = ResponseObject.ResponseView.Full, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + authorized = {RoleType.Admin}, + since = "4.20") +public class PurgeExpungedResourcesCmd extends BaseAsyncCmd { + + @Inject + ResourceCleanupService resourceCleanupService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.RESOURCE_TYPE, type = BaseCmd.CommandType.STRING, + description = "The type of the resource which need to be purged. Supported types: " + + "VirtualMachine") + private String resourceType; + + @Parameter(name = ApiConstants.BATCH_SIZE, type = CommandType.LONG, + description = "The size of batch used during purging") + private Long batchSize; + + @Parameter(name = ApiConstants.START_DATE, + type = CommandType.DATE, + description = "The start date range of the expunged resources used for purging " + + "(use format \"yyyy-MM-dd\" or \"yyyy-MM-dd HH:mm:ss\")") + private Date startDate; + + @Parameter(name = ApiConstants.END_DATE, + type = CommandType.DATE, + description = "The end date range of the expunged resources used for purging " + + "(use format \"yyyy-MM-dd\" or \"yyyy-MM-dd HH:mm:ss\")") + private Date endDate; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + + public String getResourceType() { + return resourceType; + } + + public Long getBatchSize() { + return batchSize; + } + + public Date getStartDate() { + return startDate; + } + + public Date getEndDate() { + return endDate; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public String getEventType() { + return EventTypes.EVENT_PURGE_EXPUNGED_RESOURCES; + } + + @Override + public String getEventDescription() { + return "Purging expunged resources"; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + try { + long result = resourceCleanupService.purgeExpungedResources(this); + PurgeExpungedResourcesResponse response = new PurgeExpungedResourcesResponse(); + response.setResourceCount(result); + response.setObjectName(getCommandName().toLowerCase()); + setResponseObject(response); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getLocalizedMessage()); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ChangeStoragePoolScopeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ChangeStoragePoolScopeCmd.java new file mode 100644 index 000000000000..d3b6a0746106 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ChangeStoragePoolScopeCmd.java @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ClusterResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; +import com.cloud.storage.StoragePool; + +@APICommand(name = "changeStoragePoolScope", description = "Changes the scope of a storage pool when the pool is in Disabled state." + + "This feature is officially tested and supported for Hypervisors: KVM and VMware, Protocols: NFS and Ceph, and Storage Provider: DefaultPrimary. " + + "There might be extra steps involved to make this work for other hypervisors and storage options.", + responseObject = SuccessResponse.class, since= "4.19.1", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class ChangeStoragePoolScopeCmd extends BaseAsyncCmd { + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, required = true, description = "the Id of the storage pool") + private Long id; + + @Parameter(name = ApiConstants.SCOPE, type = CommandType.STRING, required = true, description = "the scope of the storage: cluster or zone") + private String scope; + + @Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.UUID, entityType = ClusterResponse.class, description = "the Id of the cluster to use if scope is being set to Cluster") + private Long clusterId; + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.StoragePool; + } + + @Override + public Long getApiResourceId() { + return getId(); + } + + public String getEventType() { + return EventTypes.EVENT_CHANGE_STORAGE_POOL_SCOPE; + } + + @Override + public String getEventDescription() { + String description = "Change storage pool scope. Storage pool Id: "; + StoragePool pool = _entityMgr.findById(StoragePool.class, getId()); + if (pool != null) { + description += pool.getUuid(); + } else { + description += getId(); + } + description += " to " + getScope(); + return description; + } + + @Override + public void execute() { + _storageService.changeStoragePoolScope(this); + SuccessResponse response = new SuccessResponse(getCommandName()); + this.setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + public Long getId() { + return id; + } + + public String getScope() { + return scope; + } + + public Long getClusterId() { + return clusterId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java index 293ed3103cbc..57a87939b6bd 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java @@ -72,7 +72,8 @@ public class ListStoragePoolsCmd extends BaseListCmd { @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, description = "host ID of the storage pools") private Long hostId; - + @Parameter(name = ApiConstants.STORAGE_CUSTOM_STATS, type = CommandType.BOOLEAN, description = "If true, lists the custom stats of the storage pool", since = "4.18.1") + private Boolean customStats; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -129,6 +130,10 @@ public void setScope(String scope) { this.scope = scope; } + public Boolean getCustomStats() { + return customStats != null && customStats; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateImageStoreCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateImageStoreCmd.java index bcc438b957bf..0e1631a46ba2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateImageStoreCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateImageStoreCmd.java @@ -39,10 +39,17 @@ public class UpdateImageStoreCmd extends BaseCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ImageStoreResponse.class, required = true, description = "Image Store UUID") private Long id; - @Parameter(name = ApiConstants.READ_ONLY, type = CommandType.BOOLEAN, required = true, description = "If set to true, it designates the corresponding image store to read-only, " + - "hence not considering them during storage migration") + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = false, description = "The new name for the Image Store.") + private String name; + + @Parameter(name = ApiConstants.READ_ONLY, type = CommandType.BOOLEAN, required = false, + description = "If set to true, it designates the corresponding image store to read-only, hence not considering them during storage migration") private Boolean readonly; + @Parameter(name = ApiConstants.CAPACITY_BYTES, type = CommandType.LONG, required = false, + description = "The number of bytes CloudStack can use on this image storage.\n\tNOTE: this will be overwritten by the StatsCollector as soon as there is a SSVM to query the storage.") + private Long capacityBytes; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -51,17 +58,25 @@ public Long getId() { return id; } + public String getName() { + return name; + } + public Boolean getReadonly() { return readonly; } + public Long getCapacityBytes() { + return capacityBytes; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @Override public void execute() { - ImageStore result = _storageService.updateImageStoreStatus(getId(), getReadonly()); + ImageStore result = _storageService.updateImageStore(this); ImageStoreResponse storeResponse = null; if (result != null) { storeResponse = _responseGenerator.createImageStoreResponse(result); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/usage/ListUsageRecordsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/usage/ListUsageRecordsCmd.java index 3cb148c2af03..9ce1fcb2bc99 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/usage/ListUsageRecordsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/usage/ListUsageRecordsCmd.java @@ -53,16 +53,12 @@ public class ListUsageRecordsCmd extends BaseListCmd { @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "List usage records for the specified domain.") private Long domainId; - @Parameter(name = ApiConstants.END_DATE, - type = CommandType.DATE, - required = true, - description = "End date range for usage record query (use format \"yyyy-MM-dd\" or the new format \"yyyy-MM-dd HH:mm:ss\", e.g. startDate=2015-01-01 or startdate=2015-01-01 10:30:00).") + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, required = true, description = "End date range for usage record query. " + + ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) private Date endDate; - @Parameter(name = ApiConstants.START_DATE, - type = CommandType.DATE, - required = true, - description = "Start date range for usage record query (use format \"yyyy-MM-dd\" or the new format \"yyyy-MM-dd HH:mm:ss\", e.g. startDate=2015-01-01 or startdate=2015-01-01 11:00:00).") + @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, required = true, description = "Start date range for usage record query. " + + ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified account") @@ -137,11 +133,11 @@ public void setDomainId(Long domainId) { } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + this.endDate = endDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; } public void setAccountId(Long accountId) { @@ -167,8 +163,8 @@ public Boolean isRecursive() { @Override public void execute() { Pair, Integer> usageRecords = _usageService.getUsageRecords(this); - ListResponse response = new ListResponse(); - List usageResponses = new ArrayList(); + ListResponse response = new ListResponse<>(); + List usageResponses = new ArrayList<>(); Map> resourceTagResponseMap = null; if (usageRecords != null) { //read the resource tags details for all the resources in usage data and store in Map diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/UpdateUserCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/UpdateUserCmd.java index 3f8d386d2669..c9e1e934152d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/UpdateUserCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/UpdateUserCmd.java @@ -66,7 +66,7 @@ public class UpdateUserCmd extends BaseCmd { @Parameter(name = ApiConstants.CURRENT_PASSWORD, type = CommandType.STRING, description = "Current password that was being used by the user. You must inform the current password when updating the password.", acceptedOnAdminPort = false) private String currentPassword; - @Parameter(name = ApiConstants.SECRET_KEY, type = CommandType.STRING, description = "The secret key for the user. Must be specified with userApiKey") + @Parameter(name = ApiConstants.USER_SECRET_KEY, type = CommandType.STRING, description = "The secret key for the user. Must be specified with userApiKey") private String secretKey; @Parameter(name = ApiConstants.TIMEZONE, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java index dd897218a4d3..ae6ceff26c7d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java @@ -201,8 +201,8 @@ public Map getNicNetworkList() { for (Map entry : (Collection>)nicNetworkList.values()) { String nic = entry.get(VmDetailConstants.NIC); String networkUuid = entry.get(VmDetailConstants.NETWORK); - if (logger.isTraceEnabled()) { - logger.trace(String.format("nic, '%s', goes on net, '%s'", nic, networkUuid)); + if (logger.isDebugEnabled()) { + logger.debug(String.format("nic, '%s', goes on net, '%s'", nic, networkUuid)); } if (StringUtils.isAnyEmpty(nic, networkUuid) || _entityMgr.findByUuid(Network.class, networkUuid) == null) { throw new InvalidParameterValueException(String.format("Network ID: %s for NIC ID: %s is invalid", networkUuid, nic)); @@ -219,8 +219,8 @@ public Map getNicIpAddressList() { for (Map entry : (Collection>)nicIpAddressList.values()) { String nic = entry.get(VmDetailConstants.NIC); String ipAddress = StringUtils.defaultIfEmpty(entry.get(VmDetailConstants.IP4_ADDRESS), null); - if (logger.isTraceEnabled()) { - logger.trace(String.format("nic, '%s', gets ip, '%s'", nic, ipAddress)); + if (logger.isDebugEnabled()) { + logger.debug(String.format("nic, '%s', gets ip, '%s'", nic, ipAddress)); } if (StringUtils.isEmpty(nic)) { throw new InvalidParameterValueException(String.format("NIC ID: '%s' is invalid for IP address mapping", nic)); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java index 1a34b7ea6cce..6f148ff0ee41 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -37,6 +37,7 @@ import org.apache.cloudstack.api.response.VmwareDatacenterResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.vm.VmImportService; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -116,40 +117,44 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { description = "Temp Path on external host for disk image copy" ) private String tmpPath; - // Import from Vmware to KVM migration parameters + // Import from VMware to KVM migration parameters @Parameter(name = ApiConstants.EXISTING_VCENTER_ID, type = CommandType.UUID, entityType = VmwareDatacenterResponse.class, - description = "(only for importing migrated VMs from Vmware to KVM) UUID of a linked existing vCenter") + description = "(only for importing VMs from VMware to KVM) UUID of a linked existing vCenter") private Long existingVcenterId; @Parameter(name = ApiConstants.HOST_IP, type = BaseCmd.CommandType.STRING, - description = "(only for importing migrated VMs from Vmware to KVM) VMware ESXi host IP/Name.") + description = "(only for importing VMs from VMware to KVM) VMware ESXi host IP/Name.") private String hostip; @Parameter(name = ApiConstants.VCENTER, type = CommandType.STRING, - description = "(only for importing migrated VMs from Vmware to KVM) The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") + description = "(only for importing VMs from VMware to KVM) The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") private String vcenter; @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING, - description = "(only for importing migrated VMs from Vmware to KVM) Name of VMware datacenter.") + description = "(only for importing VMs from VMware to KVM) Name of VMware datacenter.") private String datacenterName; @Parameter(name = ApiConstants.CLUSTER_NAME, type = CommandType.STRING, - description = "(only for importing migrated VMs from Vmware to KVM) Name of VMware cluster.") + description = "(only for importing VMs from VMware to KVM) Name of VMware cluster.") private String clusterName; @Parameter(name = ApiConstants.CONVERT_INSTANCE_HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, - description = "(only for importing migrated VMs from Vmware to KVM) optional - the host to perform the virt-v2v migration from VMware to KVM.") + description = "(only for importing VMs from VMware to KVM) optional - the host to perform the virt-v2v migration from VMware to KVM.") private Long convertInstanceHostId; @Parameter(name = ApiConstants.CONVERT_INSTANCE_STORAGE_POOL_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, - description = "(only for importing migrated VMs from Vmware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.") + description = "(only for importing VMs from VMware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.") private Long convertStoragePoolId; + @Parameter(name = ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES, type = CommandType.BOOLEAN, + description = "(only for importing VMs from VMware to KVM) optional - if true, forces MS to import VM file(s) to temporary storage, else uses KVM Host if ovftool is available, falls back to MS if not.") + private Boolean forceMsToImportVmFiles; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -198,6 +203,10 @@ public Long getConvertStoragePoolId() { return convertStoragePoolId; } + public Boolean getForceMsToImportVmFiles() { + return BooleanUtils.toBooleanDefaultIfNull(forceMsToImportVmFiles, false); + } + public String getHypervisor() { return hypervisor; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ListAffectedVmsForStorageScopeChangeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ListAffectedVmsForStorageScopeChangeCmd.java new file mode 100644 index 000000000000..d586a81b6856 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ListAffectedVmsForStorageScopeChangeCmd.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.command.admin.vm; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ClusterResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.api.response.VirtualMachineResponse; + +import com.cloud.vm.VirtualMachine; + +@APICommand(name = "listAffectedVmsForStorageScopeChange", + description = "List user and system VMs that need to be stopped and destroyed respectively for changing the scope of the storage pool from Zone to Cluster.", + responseObject = VirtualMachineResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.1", + authorized = {RoleType.Admin}) +public class ListAffectedVmsForStorageScopeChangeCmd extends BaseListCmd { + + @Parameter(name = ApiConstants.CLUSTER_ID, + type = CommandType.UUID, + entityType = ClusterResponse.class, + required = true, + description = "the Id of the cluster the scope of the storage pool is being changed to") + private Long clusterIdForScopeChange; + + @Parameter(name = ApiConstants.STORAGE_ID, + type = CommandType.UUID, + entityType = StoragePoolResponse.class, + required = true, + description = "the Id of the storage pool on which change scope operation is being done") + private Long storageId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getClusterIdForScopeChange() { + return clusterIdForScopeChange; + } + + public Long getStorageId() { + return storageId; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + ListResponse response = _queryService.listAffectedVmsForStorageScopeChange(this); + response.setResponseName(getCommandName()); + response.setObjectName(VirtualMachine.class.getSimpleName().toLowerCase()); + setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ListVnfAppliancesCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ListVnfAppliancesCmdByAdmin.java new file mode 100644 index 000000000000..1a820ab9a483 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ListVnfAppliancesCmdByAdmin.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.vm; + +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.command.admin.AdminCmd; + +import org.apache.cloudstack.api.command.user.vm.ListVnfAppliancesCmd; +import org.apache.cloudstack.api.response.UserVmResponse; + +@APICommand(name = "listVnfAppliances", description = "List VNF appliance owned by the account.", + responseObject = UserVmResponse.class, + responseView = ResponseView.Full, + entityType = {VirtualMachine.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin}, + since = "4.19.1") +public class ListVnfAppliancesCmdByAdmin extends ListVnfAppliancesCmd implements AdminCmd { +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java index f5b8c3da8550..2d84b20c5828 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java @@ -100,7 +100,7 @@ public class CreateAutoScaleVmProfileCmd extends BaseAsyncCreateCmd { description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " + "This binary data must be base64 encoded before adding it to the request. " + "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " + - "Using HTTP POST(via POST body), you can send up to 1MB of data after base64 encoding." + + "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + "You also need to change vm.userdata.max.length value", length = 1048576, since = "4.18.0") diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/UpdateAutoScaleVmProfileCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/UpdateAutoScaleVmProfileCmd.java index e8ca502b5cdb..0b73fd91b525 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/UpdateAutoScaleVmProfileCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/UpdateAutoScaleVmProfileCmd.java @@ -95,7 +95,7 @@ public class UpdateAutoScaleVmProfileCmd extends BaseAsyncCustomIdCmd { description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " + "This binary data must be base64 encoded before adding it to the request. " + "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " + - "Using HTTP POST(via POST body), you can send up to 1MB of data after base64 encoding." + + "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + "You also need to change vm.userdata.max.length value", length = 1048576, since = "4.18.0") diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/AssignToLoadBalancerRuleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/AssignToLoadBalancerRuleCmd.java index 81a52ce2dfec..c4424b1d9374 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/AssignToLoadBalancerRuleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/AssignToLoadBalancerRuleCmd.java @@ -72,7 +72,7 @@ public class AssignToLoadBalancerRuleCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID_IP, type = CommandType.MAP, - description = "VM ID and IP map, vmidipmap[0].vmid=1 vmidipmap[0].ip=10.1.1.75", + description = "VM ID and IP map, vmidipmap[0].vmid=1 vmidipmap[0].vmip=10.1.1.75", since = "4.4") private Map vmIdIpMap; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceCountCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceCountCmd.java index 0ea22b38a374..49c6ee605c8c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceCountCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceCountCmd.java @@ -34,8 +34,11 @@ import com.cloud.configuration.ResourceCount; import com.cloud.user.Account; -@APICommand(name = "updateResourceCount", description = "Recalculate and update resource count for an account or domain.", responseObject = ResourceCountResponse.class, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +@APICommand(name = "updateResourceCount", + description = "Recalculate and update resource count for an account or domain. " + + "This also executes some cleanup tasks before calculating resource counts.", + responseObject = ResourceCountResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class UpdateResourceCountCmd extends BaseCmd { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java index 8df25541a197..41d865d678c8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java @@ -72,7 +72,14 @@ public class RegisterUserDataCmd extends BaseCmd { @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "an optional project for the userdata") private Long projectId; - @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING, required = true, description = "Userdata content", length = 1048576) + @Parameter(name = ApiConstants.USER_DATA, + type = CommandType.STRING, + required = true, + description = "Base64 encoded userdata content. " + + "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " + + "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + + "You also need to change vm.userdata.max.length value", + length = 1048576) private String userData; @Parameter(name = ApiConstants.PARAMS, type = CommandType.STRING, description = "comma separated list of variables declared in userdata content") diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMScheduleCmd.java index 935f39bf4ddb..5811eb1abfc5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMScheduleCmd.java @@ -68,7 +68,7 @@ public class CreateVMScheduleCmd extends BaseCmd { @Parameter(name = ApiConstants.ACTION, type = CommandType.STRING, required = true, - description = "Action to take on the VM (start/stop/restart/force_stop/force_reboot).") + description = "Action to take on the VM (start/stop/reboot/force_stop/force_reboot).") private String action; @Parameter(name = ApiConstants.START_DATE, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index 446bdf30f07a..68dd49d9ce55 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -154,7 +154,11 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG private String hypervisor; @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING, - description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. This binary data must be base64 encoded before adding it to the request. Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. Using HTTP POST(via POST body), you can send up to 1MB of data after base64 encoding.", + description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " + + "This binary data must be base64 encoded before adding it to the request. " + + "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " + + "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + + "You also need to change vm.userdata.max.length value", length = 1048576) private String userData; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java index 2d1160fb7a75..50e1798112d2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java @@ -16,9 +16,10 @@ // under the License. package org.apache.cloudstack.api.command.user.vm; -import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.affinity.AffinityGroupResponse; @@ -45,6 +46,7 @@ import org.apache.cloudstack.api.response.VpcResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.collections.CollectionUtils; import com.cloud.exception.InvalidParameterValueException; import com.cloud.server.ResourceIcon; @@ -56,7 +58,6 @@ requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements UserCmd { - private static final String s_name = "listvirtualmachinesresponse"; ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// @@ -96,7 +97,8 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements collectionType = CommandType.STRING, description = "comma separated list of vm details requested, " + "value can be a list of [all, group, nics, stats, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp]." - + " If no parameter is passed in, the details will be defaulted to all") + + " When no parameters are passed, all the details are returned if list.vm.default.details.stats is true (default)," + + " otherwise when list.vm.default.details.stats is false the API response will exclude the stats details.") private List viewDetails; @Parameter(name = ApiConstants.TEMPLATE_ID, type = CommandType.UUID, entityType = TemplateResponse.class, description = "list vms by template") @@ -237,22 +239,32 @@ public Long getAutoScaleVmGroupId() { return autoScaleVmGroupId; } + protected boolean isViewDetailsEmpty() { + return CollectionUtils.isEmpty(viewDetails); + } + public EnumSet getDetails() throws InvalidParameterValueException { - EnumSet dv; - if (viewDetails == null || viewDetails.size() <= 0) { - dv = EnumSet.of(VMDetails.all); - } else { - try { - ArrayList dc = new ArrayList(); - for (String detail : viewDetails) { - dc.add(VMDetails.valueOf(detail)); - } - dv = EnumSet.copyOf(dc); - } catch (IllegalArgumentException e) { - throw new InvalidParameterValueException("The details parameter contains a non permitted value. The allowed values are " + EnumSet.allOf(VMDetails.class)); + if (isViewDetailsEmpty()) { + if (_queryService.ReturnVmStatsOnVmList.value()) { + return EnumSet.of(VMDetails.all); + } + + Set allDetails = new HashSet<>(Set.of(VMDetails.values())); + allDetails.remove(VMDetails.stats); + allDetails.remove(VMDetails.all); + return EnumSet.copyOf(allDetails); + } + + try { + Set dc = new HashSet<>(); + for (String detail : viewDetails) { + dc.add(VMDetails.valueOf(detail)); } + + return EnumSet.copyOf(dc); + } catch (IllegalArgumentException e) { + throw new InvalidParameterValueException("The details parameter contains a non permitted value. The allowed values are " + EnumSet.allOf(VMDetails.class)); } - return dv; } @Override @@ -275,10 +287,6 @@ public Boolean getVnf() { ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// - @Override - public String getCommandName() { - return s_name; - } @Override public ApiCommandResourceType getApiResourceType() { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVnfAppliancesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVnfAppliancesCmd.java new file mode 100644 index 000000000000..e7bbb97d78c3 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVnfAppliancesCmd.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.vm; + +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.command.user.UserCmd; + +import org.apache.cloudstack.api.response.UserVmResponse; + +@APICommand(name = "listVnfAppliances", description = "List VNF appliance owned by the account.", + responseObject = UserVmResponse.class, + responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}, + since = "4.19.1") +public class ListVnfAppliancesCmd extends ListVMsCmd implements UserCmd { + + @Override + public Boolean getVnf() { + return true; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ResetVMUserDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ResetVMUserDataCmd.java index 089dfaecf946..0ecf4ff13845 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ResetVMUserDataCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ResetVMUserDataCmd.java @@ -60,7 +60,7 @@ public class ResetVMUserDataCmd extends BaseCmd implements UserCmd { description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " + "This binary data must be base64 encoded before adding it to the request. " + "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " + - "Using HTTP POST(via POST body), you can send up to 1MB of data after base64 encoding." + + "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + "You also need to change vm.userdata.max.length value", length = 1048576) private String userData; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java index 9f72ac17c8f4..024337356fd5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java @@ -84,7 +84,7 @@ public class UpdateVMCmd extends BaseCustomIdCmd implements SecurityGroupAction, description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " + "This binary data must be base64 encoded before adding it to the request. " + "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " + - "Using HTTP POST(via POST body), you can send up to 1MB of data after base64 encoding." + + "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + "You also need to change vm.userdata.max.length value", length = 1048576, since = "4.16.0") diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ListVolumesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ListVolumesCmd.java index a583675da76f..a1024a988981 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ListVolumesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ListVolumesCmd.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.PodResponse; +import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.VolumeResponse; @@ -80,6 +81,12 @@ public class ListVolumesCmd extends BaseListRetrieveOnlyResourceCountCmd impleme RoleType.Admin}) private String storageId; + @Parameter(name = ApiConstants.SERVICE_OFFERING_ID, type = CommandType.UUID, + entityType = ServiceOfferingResponse.class, + description = "list volumes by disk offering of a service offering. If both service offering and " + + "disk offering are passed, service offering is ignored", since = "4.19.1") + private Long serviceOfferingId; + @Parameter(name = ApiConstants.DISK_OFFERING_ID, type = CommandType.UUID, entityType = DiskOfferingResponse.class, description = "list volumes by disk offering", since = "4.4") private Long diskOfferingId; @@ -94,6 +101,9 @@ public class ListVolumesCmd extends BaseListRetrieveOnlyResourceCountCmd impleme @Parameter(name = ApiConstants.STATE, type = CommandType.STRING, description = "state of the volume. Possible values are: Ready, Allocated, Destroy, Expunging, Expunged.") private String state; + @Parameter(name = ApiConstants.IS_ENCRYPTED, type = CommandType.BOOLEAN, description = "list only volumes that are encrypted", since = "4.19.1", + authorized = { RoleType.Admin }) + private Boolean encrypted; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -118,6 +128,10 @@ public Long getPodId() { return podId; } + public Long getServiceOfferingId() { + return serviceOfferingId; + } + public Long getDiskOfferingId() { return diskOfferingId; } @@ -151,6 +165,10 @@ public String getState() { return state; } + public Boolean isEncrypted() { + return encrypted; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java index 94f05f707a0a..89a65f8c27cc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java @@ -208,11 +208,7 @@ public void create() throws ResourceAllocationException { public void execute() { Vpc vpc = null; try { - if (isStart()) { - _vpcService.startVpc(getEntityId(), true); - } else { - logger.debug("Not starting VPC as " + ApiConstants.START + "=false was passed to the API"); - } + _vpcService.startVpc(this); vpc = _entityMgr.findById(Vpc.class, getEntityId()); } catch (ResourceUnavailableException ex) { logger.warn("Exception: ", ex); diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AutoScaleVmProfileResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AutoScaleVmProfileResponse.java index 9f2383447301..678c4fcaaca4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AutoScaleVmProfileResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AutoScaleVmProfileResponse.java @@ -69,7 +69,7 @@ public class AutoScaleVmProfileResponse extends BaseResponse implements Controll private Map counterParams; @SerializedName(ApiConstants.USER_DATA) - @Param(description = "Base 64 encoded VM user data") + @Param(description = "Base64 encoded VM user data") private String userData; @SerializedName(ApiConstants.USER_DATA_ID) @Param(description="the id of userdata used for the VM", since = "4.18.1") diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java index b75f36043244..a1cce6e43d7a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java @@ -98,7 +98,7 @@ public class BucketResponse extends BaseResponseWithTagInformation implements Co @Param(description = "Bucket Access Key") private String accessKey; - @SerializedName(ApiConstants.SECRET_KEY) + @SerializedName(ApiConstants.USER_SECRET_KEY) @Param(description = "Bucket Secret Key") private String secretKey; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java index 41a0fdc4567e..24015e0b4597 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java @@ -208,6 +208,14 @@ public class HostForMigrationResponse extends BaseResponse { @Param(description = "comma-separated list of tags for the host") private String hostTags; + @SerializedName("explicithosttags") + @Param(description = "comma-separated list of explicit host tags for the host", since = "4.20.0") + private String explicitHostTags; + + @SerializedName("implicithosttags") + @Param(description = "comma-separated list of implicit host tags for the host", since = "4.20.0") + private String implicitHostTags; + @SerializedName("hasenoughcapacity") @Param(description = "true if this host has enough CPU and RAM capacity to migrate a VM to it, false otherwise") private Boolean hasEnoughCapacity; @@ -414,6 +422,14 @@ public void setHostTags(String hostTags) { this.hostTags = hostTags; } + public void setExplicitHostTags(String explicitHostTags) { + this.explicitHostTags = explicitHostTags; + } + + public void setImplicitHostTags(String implicitHostTags) { + this.implicitHostTags = implicitHostTags; + } + public void setHasEnoughCapacity(Boolean hasEnoughCapacity) { this.hasEnoughCapacity = hasEnoughCapacity; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java index d72d23b99c9c..8d98b6573f40 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java @@ -29,6 +29,7 @@ import com.cloud.host.Host; import com.cloud.host.Status; +import com.cloud.hypervisor.Hypervisor; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; @@ -221,6 +222,14 @@ public class HostResponse extends BaseResponseWithAnnotations { @Param(description = "comma-separated list of tags for the host") private String hostTags; + @SerializedName("explicithosttags") + @Param(description = "comma-separated list of explicit host tags for the host", since = "4.20.0") + private String explicitHostTags; + + @SerializedName("implicithosttags") + @Param(description = "comma-separated list of implicit host tags for the host", since = "4.20.0") + private String implicitHostTags; + @SerializedName(ApiConstants.IS_TAG_A_RULE) @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) private Boolean isTagARule; @@ -277,6 +286,10 @@ public class HostResponse extends BaseResponseWithAnnotations { @Param(description = "true if the host supports encryption", since = "4.18") private Boolean encryptionSupported; + @SerializedName(ApiConstants.INSTANCE_CONVERSION_SUPPORTED) + @Param(description = "true if the host supports instance conversion (using virt-v2v)", since = "4.19.1") + private Boolean instanceConversionSupported; + @Override public String getObjectId() { return this.getId(); @@ -458,6 +471,22 @@ public void setHostTags(String hostTags) { this.hostTags = hostTags; } + public String getExplicitHostTags() { + return explicitHostTags; + } + + public void setExplicitHostTags(String explicitHostTags) { + this.explicitHostTags = explicitHostTags; + } + + public String getImplicitHostTags() { + return implicitHostTags; + } + + public void setImplicitHostTags(String implicitHostTags) { + this.implicitHostTags = implicitHostTags; + } + public void setHasEnoughCapacity(Boolean hasEnoughCapacity) { this.hasEnoughCapacity = hasEnoughCapacity; } @@ -526,7 +555,7 @@ public void setUsername(String username) { this.username = username; } - public void setDetails(Map details) { + public void setDetails(Map details, Hypervisor.HypervisorType hypervisorType) { if (details == null) { return; @@ -547,6 +576,15 @@ public void setDetails(Map details) { this.setEncryptionSupported(new Boolean(false)); // default } + if (Hypervisor.HypervisorType.KVM.equals(hypervisorType)) { + if (detailsCopy.containsKey(Host.HOST_INSTANCE_CONVERSION)) { + this.setInstanceConversionSupported(Boolean.parseBoolean((String) detailsCopy.get(Host.HOST_INSTANCE_CONVERSION))); + detailsCopy.remove(Host.HOST_INSTANCE_CONVERSION); + } else { + this.setInstanceConversionSupported(new Boolean(false)); // default + } + } + this.details = detailsCopy; } @@ -737,6 +775,10 @@ public void setEncryptionSupported(Boolean encryptionSupported) { this.encryptionSupported = encryptionSupported; } + public void setInstanceConversionSupported(Boolean instanceConversionSupported) { + this.instanceConversionSupported = instanceConversionSupported; + } + public Boolean getIsTagARule() { return isTagARule; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostTagResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostTagResponse.java index 4a924ea78a08..f772da6dcb66 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostTagResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostTagResponse.java @@ -19,6 +19,7 @@ import com.google.gson.annotations.SerializedName; import com.cloud.serializer.Param; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; public class HostTagResponse extends BaseResponse { @@ -34,6 +35,10 @@ public class HostTagResponse extends BaseResponse { @Param(description = "the name of the host tag") private String name; + @SerializedName(ApiConstants.IS_IMPLICIT) + @Param(description = "true if the host tag is implicit", since = "4.20.0") + private boolean isImplicit; + public String getId() { return id; } @@ -57,4 +62,12 @@ public String getName() { public void setName(String name) { this.name = name; } + + public boolean isImplicit() { + return isImplicit; + } + + public void setImplicit(boolean implicit) { + isImplicit = implicit; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ImageStoreResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ImageStoreResponse.java index 532963dbddc0..ee44b6bc4745 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ImageStoreResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ImageStoreResponse.java @@ -27,11 +27,11 @@ @EntityReference(value = ImageStore.class) public class ImageStoreResponse extends BaseResponseWithAnnotations { - @SerializedName("id") + @SerializedName(ApiConstants.ID) @Param(description = "the ID of the image store") private String id; - @SerializedName("zoneid") + @SerializedName(ApiConstants.ZONE_ID) @Param(description = "the Zone ID of the image store") private String zoneId; @@ -39,15 +39,15 @@ public class ImageStoreResponse extends BaseResponseWithAnnotations { @Param(description = "the Zone name of the image store") private String zoneName; - @SerializedName("name") + @SerializedName(ApiConstants.NAME) @Param(description = "the name of the image store") private String name; - @SerializedName("url") + @SerializedName(ApiConstants.URL) @Param(description = "the url of the image store") private String url; - @SerializedName("protocol") + @SerializedName(ApiConstants.PROTOCOL) @Param(description = "the protocol of the image store") private String protocol; @@ -55,11 +55,11 @@ public class ImageStoreResponse extends BaseResponseWithAnnotations { @Param(description = "the provider name of the image store") private String providerName; - @SerializedName("scope") + @SerializedName(ApiConstants.SCOPE) @Param(description = "the scope of the image store") private ScopeType scope; - @SerializedName("readonly") + @SerializedName(ApiConstants.READ_ONLY) @Param(description = "defines if store is read-only") private Boolean readonly; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/PurgeExpungedResourcesResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/PurgeExpungedResourcesResponse.java new file mode 100644 index 000000000000..3807d0d5b168 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/PurgeExpungedResourcesResponse.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.response; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class PurgeExpungedResourcesResponse extends BaseResponse { + + @SerializedName(ApiConstants.RESOURCE_COUNT) + @Param(description = "The count of the purged expunged resources") + private Long resourceCount; + + public Long getResourceCount() { + return resourceCount; + } + + public void setResourceCount(Long resourceCount) { + this.resourceCount = resourceCount; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java index c7740c19214b..0622b936f6e0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java @@ -234,6 +234,10 @@ public class ServiceOfferingResponse extends BaseResponseWithAnnotations { @Param(description = "true if virtual machine root disk will be encrypted on storage", since = "4.18") private Boolean encryptRoot; + @SerializedName(ApiConstants.PURGE_RESOURCES) + @Param(description = "Whether to cleanup VM and its associated resource upon expunge", since = "4.20") + private Boolean purgeResources; + public ServiceOfferingResponse() { } @@ -555,4 +559,8 @@ public String getDiskOfferingDisplayText() { } public void setEncryptRoot(Boolean encrypt) { this.encryptRoot = encrypt; } + + public void setPurgeResources(Boolean purgeResources) { + this.purgeResources = purgeResources; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java index 183290ec9ebe..9e7f5159e0e6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java @@ -97,10 +97,18 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { @Param(description = "total min IOPS currently in use by volumes") private Long allocatedIops; + @SerializedName(ApiConstants.STORAGE_CUSTOM_STATS) + @Param(description = "the storage pool custom stats", since = "4.18.1") + private Map customStats; + @SerializedName("tags") @Param(description = "the tags for the storage pool") private String tags; + @SerializedName(ApiConstants.NFS_MOUNT_OPTIONS) + @Param(description = "the nfs mount options for the storage pool", since = "4.19.1") + private String nfsMountOpts; + @SerializedName(ApiConstants.IS_TAG_A_RULE) @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) private Boolean isTagARule; @@ -300,6 +308,14 @@ public void setAllocatedIops(Long allocatedIops) { this.allocatedIops = allocatedIops; } + public Map getCustomStats() { + return customStats; + } + + public void setCustomStats(Map customStats) { + this.customStats = customStats; + } + public String getTags() { return tags; } @@ -347,4 +363,12 @@ public String getProvider() { public void setProvider(String provider) { this.provider = provider; } + + public String getNfsMountOpts() { + return nfsMountOpts; + } + + public void setNfsMountOpts(String nfsMountOpts) { + this.nfsMountOpts = nfsMountOpts; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UsageRecordResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UsageRecordResponse.java index 7bcb1afd2d24..4522315b499a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UsageRecordResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UsageRecordResponse.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.api.response; +import java.util.Date; import java.util.LinkedHashSet; import java.util.Set; @@ -133,11 +134,11 @@ public class UsageRecordResponse extends BaseResponseWithTagInformation implemen @SerializedName(ApiConstants.START_DATE) @Param(description = "start date of the usage record") - private String startDate; + private Date startDate; @SerializedName(ApiConstants.END_DATE) @Param(description = "end date of the usage record") - private String endDate; + private Date endDate; @SerializedName("issourcenat") @Param(description = "True if the IPAddress is source NAT") @@ -160,7 +161,7 @@ public class UsageRecordResponse extends BaseResponseWithTagInformation implemen private String vpcId; public UsageRecordResponse() { - tags = new LinkedHashSet(); + tags = new LinkedHashSet<>(); } public void setTags(Set tags) { @@ -245,11 +246,11 @@ public void setSize(Long size) { this.size = size; } - public void setStartDate(String startDate) { + public void setStartDate(Date startDate) { this.startDate = startDate; } - public void setEndDate(String endDate) { + public void setEndDate(Date endDate) { this.endDate = endDate; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java index 763265e109d3..8deae7d80d3e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java @@ -37,6 +37,7 @@ import com.cloud.uservm.UserVm; import com.cloud.vm.VirtualMachine; import com.google.gson.annotations.SerializedName; +import org.apache.commons.collections.CollectionUtils; @SuppressWarnings("unused") @EntityReference(value = {VirtualMachine.class, UserVm.class, VirtualRouter.class}) @@ -137,6 +138,10 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co @Param(description = "the type of the template for the virtual machine", since = "4.19.0") private String templateType; + @SerializedName(ApiConstants.TEMPLATE_FORMAT) + @Param(description = "the format of the template for the virtual machine", since = "4.19.1") + private String templateFormat; + @SerializedName("templatedisplaytext") @Param(description = " an alternate display text of the template for the virtual machine") private String templateDisplayText; @@ -269,6 +274,10 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co @Param(description = "the hypervisor on which the template runs") private String hypervisor; + @SerializedName(ApiConstants.IP_ADDRESS) + @Param(description = "the VM's primary IP address") + private String ipAddress; + @SerializedName(ApiConstants.PUBLIC_IP_ID) @Param(description = "public IP address id associated with vm via Static nat rule") private String publicIpId; @@ -623,6 +632,10 @@ public String getHypervisor() { return hypervisor; } + public String getIpAddress() { + return ipAddress; + } + public String getPublicIpId() { return publicIpId; } @@ -859,6 +872,13 @@ public void setForVirtualNetwork(Boolean forVirtualNetwork) { public void setNics(Set nics) { this.nics = nics; + setIpAddress(nics); + } + + public void setIpAddress(final Set nics) { + if (CollectionUtils.isNotEmpty(nics)) { + this.ipAddress = nics.iterator().next().getIpaddress(); + } } public void addNic(NicResponse nic) { @@ -1076,6 +1096,14 @@ public void setTemplateType(String templateType) { this.templateType = templateType; } + public String getTemplateFormat() { + return templateFormat; + } + + public void setTemplateFormat(String templateFormat) { + this.templateFormat = templateFormat; + } + public List getVnfNics() { return vnfNics; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VMUserDataResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VMUserDataResponse.java index 1b739e564421..cf819491c2c0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VMUserDataResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VMUserDataResponse.java @@ -30,7 +30,7 @@ public class VMUserDataResponse extends BaseResponse { private String vmId; @SerializedName(ApiConstants.USER_DATA) - @Param(description = "Base 64 encoded VM user data") + @Param(description = "Base64 encoded VM user data") private String userData; public void setUserData(String userData) { diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VirtualMachineResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VirtualMachineResponse.java new file mode 100644 index 000000000000..7d676292b8a0 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/VirtualMachineResponse.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; + +import com.cloud.serializer.Param; +import com.cloud.vm.VirtualMachine; +import com.google.gson.annotations.SerializedName; + +@EntityReference(value = VirtualMachine.class) +public class VirtualMachineResponse extends BaseResponse { + @SerializedName("id") + @Param(description = "the ID of the VM") + private String id; + + @SerializedName("type") + @Param(description = "the type of VM") + private String type; + + @SerializedName("name") + @Param(description = "the name of the VM") + private String name; + + @SerializedName("clusterid") + @Param(description = "the cluster ID for the VM") + private String clusterId; + + @SerializedName("clustername") + @Param(description = "the cluster name for the VM") + private String clusterName; + + @SerializedName("hostid") + @Param(description = "the host ID for the VM") + private String hostId; + + @SerializedName("hostname") + @Param(description = "the hostname for the VM") + private String hostName; + + @Override + public String getObjectId() { + return this.getId(); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getVmType() { + return type; + } + + public void setVmType(String type) { + this.type = type; + } + + public String getVmName() { + return name; + } + + public void setVmName(String name) { + this.name = name; + } + + public String getClusterId() { + return clusterId; + } + + public void setClusterId(String clusterId) { + this.clusterId = clusterId; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getHostId() { + return hostId; + } + + public void setHostId(String hostId) { + this.hostId = hostId; + } + + public String getHostName() { + return hostName; + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java index 0d502a6d7a73..726c9adf8a3c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java @@ -290,13 +290,17 @@ public class VolumeResponse extends BaseResponseWithTagInformation implements Co private String externalUuid; @SerializedName(ApiConstants.VOLUME_CHECK_RESULT) - @Param(description = "details for the volume check result, they may vary for different hypervisors, since = 4.19.1") + @Param(description = "details for the volume check result, they may vary for different hypervisors", since = "4.19.1") private Map volumeCheckResult; @SerializedName(ApiConstants.VOLUME_REPAIR_RESULT) - @Param(description = "details for the volume repair result, they may vary for different hypervisors, since = 4.19.1") + @Param(description = "details for the volume repair result, they may vary for different hypervisors", since = "4.19.1") private Map volumeRepairResult; + @SerializedName(ApiConstants.ENCRYPT_FORMAT) + @Param(description = "the format of the disk encryption if applicable", since = "4.19.1") + private String encryptionFormat; + public String getPath() { return path; } @@ -842,4 +846,8 @@ public Map getVolumeRepairResult() { public void setVolumeRepairResult(Map volumeRepairResult) { this.volumeRepairResult = volumeRepairResult; } + + public void setEncryptionFormat(String encryptionFormat) { + this.encryptionFormat = encryptionFormat; + } } diff --git a/api/src/main/java/org/apache/cloudstack/ca/CAManager.java b/api/src/main/java/org/apache/cloudstack/ca/CAManager.java index 12a9d3d7b41c..b0fb1ac73c21 100644 --- a/api/src/main/java/org/apache/cloudstack/ca/CAManager.java +++ b/api/src/main/java/org/apache/cloudstack/ca/CAManager.java @@ -77,6 +77,14 @@ public interface CAManager extends CAService, Configurable, PluggableService { "15", "The number of days before expiry of a client certificate, the validations are checked. Admins are alerted when auto-renewal is not allowed, otherwise auto-renewal is attempted.", true, ConfigKey.Scope.Cluster); + + ConfigKey CertManagementCustomSubjectAlternativeName = new ConfigKey<>("Advanced", String.class, + "ca.framework.cert.management.custom.san", + "cloudstack.internal", + "The custom Subject Alternative Name that will be added to the management server certificate. " + + "The actual implementation will depend on the configured CA provider.", + false); + /** * Returns a list of available CA provider plugins * @return returns list of CAProvider diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 3299e7537a2e..c93e43d9f371 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -52,6 +52,7 @@ import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; +import org.apache.cloudstack.api.command.admin.vm.ListAffectedVmsForStorageScopeChangeCmd; import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; import org.apache.cloudstack.api.command.user.vmgroup.ListVMGroupsCmd; import org.apache.cloudstack.api.command.user.volume.ListResourceDetailsCmd; @@ -89,6 +90,7 @@ import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.UserResponse; import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.api.response.VirtualMachineResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.framework.config.ConfigKey; @@ -125,6 +127,9 @@ public interface QueryService { static final ConfigKey SharePublicTemplatesWithOtherDomains = new ConfigKey<>("Advanced", Boolean.class, "share.public.templates.with.other.domains", "true", "If false, templates of this domain will not show up in the list templates of other domains.", true, ConfigKey.Scope.Domain); + ConfigKey ReturnVmStatsOnVmList = new ConfigKey<>("Advanced", Boolean.class, "list.vm.default.details.stats", "true", + "Determines whether VM stats should be returned when details are not explicitly specified in listVirtualMachines API request. When false, details default to [group, nics, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp]. When true, all details are returned including 'stats'.", true, ConfigKey.Scope.Global); + ListResponse searchForUsers(ListUsersCmd cmd) throws PermissionDeniedException; ListResponse searchForUsers(Long domainId, boolean recursive) throws PermissionDeniedException; @@ -137,6 +142,8 @@ public interface QueryService { ListResponse searchForUserVMs(ListVMsCmd cmd); + ListResponse listAffectedVmsForStorageScopeChange(ListAffectedVmsForStorageScopeChangeCmd cmd); + ListResponse searchForSecurityGroups(ListSecurityGroupsCmd cmd); ListResponse searchForRouters(ListRoutersCmd cmd); diff --git a/api/src/main/java/org/apache/cloudstack/resource/ResourceCleanupService.java b/api/src/main/java/org/apache/cloudstack/resource/ResourceCleanupService.java new file mode 100644 index 000000000000..0d72edb07489 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/resource/ResourceCleanupService.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.resource; + +import org.apache.cloudstack.api.command.admin.resource.PurgeExpungedResourcesCmd; +import org.apache.cloudstack.framework.config.ConfigKey; + +import com.cloud.vm.VirtualMachine; + +public interface ResourceCleanupService { + int MINIMUM_EXPUNGED_RESOURCE_PURGE_JOB_DELAY_IN_SECONDS = 3 * 60; + ConfigKey ExpungedResourcePurgeEnabled = new ConfigKey<>("Advanced", Boolean.class, + "expunged.resources.purge.enabled", "false", + "Whether to run a background task to purge the DB records of the expunged resources", + false, ConfigKey.Scope.Global); + ConfigKey ExpungedResourcePurgeResources = new ConfigKey<>("Advanced", String.class, + "expunged.resources.purge.resources", "", + "A comma-separated list of resource types that will be considered by the background task " + + "to purge the DB records of the expunged resources. Currently only VirtualMachine is supported. " + + "An empty value will result in considering all resource types for purging", + false, ConfigKey.Scope.Global); + ConfigKey ExpungedResourcesPurgeInterval = new ConfigKey<>("Advanced", Integer.class, + "expunged.resources.purge.interval", "86400", + "Interval (in seconds) for the background task to purge the DB records of the expunged resources", + false); + ConfigKey ExpungedResourcesPurgeDelay = new ConfigKey<>("Advanced", Integer.class, + "expunged.resources.purge.delay", "300", + "Initial delay (in seconds) to start the background task to purge the DB records of the " + + "expunged resources task", false); + ConfigKey ExpungedResourcesPurgeBatchSize = new ConfigKey<>("Advanced", Integer.class, + "expunged.resources.purge.batch.size", "50", + "Batch size to be used during purging of the DB records of the expunged resources", + true); + ConfigKey ExpungedResourcesPurgeStartTime = new ConfigKey<>("Advanced", String.class, + "expunged.resources.purge.start.time", "", + "Start time to be used by the background task to purge the DB records of the expunged " + + "resources. Use format \"yyyy-MM-dd\" or \"yyyy-MM-dd HH:mm:ss\"", true); + ConfigKey ExpungedResourcesPurgeKeepPastDays = new ConfigKey<>("Advanced", Integer.class, + "expunged.resources.purge.keep.past.days", "30", + "The number of days in the past from the execution time of the background task to purge " + + "the DB records of the expunged resources for which the expunged resources must not be purged. " + + "To enable purging DB records of the expunged resource till the execution of the background " + + "task, set the value to zero.", true); + ConfigKey ExpungedResourcePurgeJobDelay = new ConfigKey<>("Advanced", Integer.class, + "expunged.resource.purge.job.delay", + String.valueOf(MINIMUM_EXPUNGED_RESOURCE_PURGE_JOB_DELAY_IN_SECONDS), + String.format("Delay (in seconds) to execute the purging of the DB records of an expunged resource " + + "initiated by the configuration in the offering. Minimum value should be %d seconds " + + "and if a lower value is set then the minimum value will be used", + MINIMUM_EXPUNGED_RESOURCE_PURGE_JOB_DELAY_IN_SECONDS), + true); + + enum ResourceType { + VirtualMachine + } + + long purgeExpungedResources(PurgeExpungedResourcesCmd cmd); + void purgeExpungedVmResourcesLaterIfNeeded(VirtualMachine vm); +} diff --git a/api/src/main/java/org/apache/cloudstack/user/ResourceReservation.java b/api/src/main/java/org/apache/cloudstack/user/ResourceReservation.java index fb4fe121cc70..d49120d44919 100644 --- a/api/src/main/java/org/apache/cloudstack/user/ResourceReservation.java +++ b/api/src/main/java/org/apache/cloudstack/user/ResourceReservation.java @@ -22,6 +22,8 @@ import com.cloud.configuration.Resource; +import java.util.Date; + /** * an interface defining an {code}AutoClosable{code} reservation object */ @@ -39,4 +41,6 @@ String getTag(); Long getReservedAmount(); + + Date getCreated(); } diff --git a/api/src/main/java/org/apache/cloudstack/userdata/UserDataManager.java b/api/src/main/java/org/apache/cloudstack/userdata/UserDataManager.java index 4dfcd0a7de1b..b4bede248907 100644 --- a/api/src/main/java/org/apache/cloudstack/userdata/UserDataManager.java +++ b/api/src/main/java/org/apache/cloudstack/userdata/UserDataManager.java @@ -17,11 +17,16 @@ package org.apache.cloudstack.userdata; import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import com.cloud.utils.component.Manager; public interface UserDataManager extends Manager, Configurable { + String VM_USERDATA_MAX_LENGTH_STRING = "vm.userdata.max.length"; + ConfigKey VM_USERDATA_MAX_LENGTH = new ConfigKey<>("Advanced", Integer.class, VM_USERDATA_MAX_LENGTH_STRING, "32768", + "Max length of vm userdata after base64 encoding. Default is 32768 and maximum is 1048576", true); + String concatenateUserData(String userdata1, String userdata2, String userdataProvider); String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod); } diff --git a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java index 23e0e371714b..5697a040b811 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java +++ b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.vm; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + import java.util.List; public class UnmanagedInstanceTO { @@ -317,6 +319,16 @@ public void setDatastorePort(int datastorePort) { public int getDatastorePort() { return datastorePort; } + + @Override + public String toString() { + return "Disk {" + + "diskId='" + diskId + '\'' + + ", capacity=" + toHumanReadableSize(capacity) + + ", controller='" + controller + '\'' + + ", controllerUnit=" + controllerUnit + + "}"; + } } public static class Nic { @@ -409,5 +421,14 @@ public String getPciSlot() { public void setPciSlot(String pciSlot) { this.pciSlot = pciSlot; } + + @Override + public String toString() { + return "Nic{" + + "nicId='" + nicId + '\'' + + ", adapterType='" + adapterType + '\'' + + ", macAddress='" + macAddress + '\'' + + "}"; + } } } diff --git a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java index 53aece949649..b6233b9c2704 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java +++ b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java @@ -30,6 +30,46 @@ public interface UnmanagedVMsManager extends VmImportService, UnmanageVMService, "If set to true, do not remove VM nics (and its MAC addresses) when unmanaging a VM, leaving them allocated but not reserved. " + "If set to false, nics are removed and MAC addresses can be reassigned", true, ConfigKey.Scope.Zone); + ConfigKey RemoteKvmInstanceDisksCopyTimeout = new ConfigKey<>(Integer.class, + "remote.kvm.instance.disks.copy.timeout", + "Advanced", + "30", + "Timeout (in mins) to prepare and copy the disks of remote KVM instance while importing the instance from an external host", + true, + ConfigKey.Scope.Global, + null); + + ConfigKey ConvertVmwareInstanceToKvmTimeout = new ConfigKey<>(Integer.class, + "convert.vmware.instance.to.kvm.timeout", + "Advanced", + "3", + "Timeout (in hours) for the instance conversion process from VMware through the virt-v2v binary on a KVM host", + true, + ConfigKey.Scope.Global, + null); + + ConfigKey ThreadsOnMSToImportVMwareVMFiles = new ConfigKey<>(Integer.class, + "threads.on.ms.to.import.vmware.vm.files", + "Advanced", + "0", + "Threads to use on the management server when importing VM files from VMWare." + + " -1 or 1 disables threads, 0 uses a thread per VM disk (disabled for single disk) and >1 for manual thread configuration." + + " Max number is 10, Default is 0.", + true, + ConfigKey.Scope.Global, + null); + + ConfigKey ThreadsOnKVMHostToImportVMwareVMFiles = new ConfigKey<>(Integer.class, + "threads.on.kvm.host.to.import.vmware.vm.files", + "Advanced", + "0", + "Threads to use on the KVM host (by the ovftool, if the version is 4.4.0+) when importing VM files from VMWare." + + " -1 or 1 disables threads, 0 uses a thread per VM disk (disabled for single disk) and >1 for manual thread configuration." + + " Max number is 10, Default is 0.", + true, + ConfigKey.Scope.Global, + null); + static boolean isSupported(Hypervisor.HypervisorType hypervisorType) { return hypervisorType == VMware || hypervisorType == KVM; } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmdTest.java index f69e8cea4f3d..6daa5de07cbf 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmdTest.java @@ -37,4 +37,22 @@ public void testGetDisplayTextWhenEmpty() { Assert.assertEquals(createServiceOfferingCmd.getDisplayText(), netName); } + @Test + public void testIsPurgeResourcesNoOrNullValue() { + Assert.assertFalse(createServiceOfferingCmd.isPurgeResources()); + ReflectionTestUtils.setField(createServiceOfferingCmd, "purgeResources", false); + Assert.assertFalse(createServiceOfferingCmd.isPurgeResources()); + } + + @Test + public void testIsPurgeResourcesFalse() { + ReflectionTestUtils.setField(createServiceOfferingCmd, "purgeResources", false); + Assert.assertFalse(createServiceOfferingCmd.isPurgeResources()); + } + + @Test + public void testIsPurgeResourcesTrue() { + ReflectionTestUtils.setField(createServiceOfferingCmd, "purgeResources", true); + Assert.assertTrue(createServiceOfferingCmd.isPurgeResources()); + } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmdTest.java new file mode 100644 index 000000000000..1bb2be041e18 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmdTest.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.offering; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class UpdateServiceOfferingCmdTest { + + @InjectMocks + private UpdateServiceOfferingCmd updateServiceOfferingCmd; + + @Test + public void testIsPurgeResourcesNoOrNullValue() { + Assert.assertFalse(updateServiceOfferingCmd.isPurgeResources()); + ReflectionTestUtils.setField(updateServiceOfferingCmd, "purgeResources", false); + Assert.assertFalse(updateServiceOfferingCmd.isPurgeResources()); + } + + @Test + public void testIsPurgeResourcesFalse() { + ReflectionTestUtils.setField(updateServiceOfferingCmd, "purgeResources", false); + Assert.assertFalse(updateServiceOfferingCmd.isPurgeResources()); + } + + @Test + public void testIsPurgeResourcesTrue() { + ReflectionTestUtils.setField(updateServiceOfferingCmd, "purgeResources", true); + Assert.assertTrue(updateServiceOfferingCmd.isPurgeResources()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/resource/PurgeExpungedResourcesCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/resource/PurgeExpungedResourcesCmdTest.java new file mode 100644 index 000000000000..a628f13275c5 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/resource/PurgeExpungedResourcesCmdTest.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.resource; + +import static org.junit.Assert.assertNull; + +import java.util.Date; + +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.PurgeExpungedResourcesResponse; +import org.apache.cloudstack.resource.ResourceCleanupService; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.utils.exception.CloudRuntimeException; + +@RunWith(MockitoJUnitRunner.class) +public class PurgeExpungedResourcesCmdTest { + @Mock + ResourceCleanupService resourceCleanupService; + + @Spy + @InjectMocks + PurgeExpungedResourcesCmd spyCmd = Mockito.spy(new PurgeExpungedResourcesCmd()); + + @Test + public void testGetResourceType() { + PurgeExpungedResourcesCmd cmd = new PurgeExpungedResourcesCmd(); + assertNull(cmd.getResourceType()); + ReflectionTestUtils.setField(cmd, "resourceType", ResourceCleanupService.ResourceType.VirtualMachine.toString()); + Assert.assertEquals(ResourceCleanupService.ResourceType.VirtualMachine.toString(), cmd.getResourceType()); + } + + @Test + public void testGetBatchSize() { + PurgeExpungedResourcesCmd cmd = new PurgeExpungedResourcesCmd(); + assertNull(cmd.getBatchSize()); + Long batchSize = 100L; + ReflectionTestUtils.setField(cmd, "batchSize", batchSize); + Assert.assertEquals(batchSize, cmd.getBatchSize()); + } + + @Test + public void testGetStartDate() { + PurgeExpungedResourcesCmd cmd = new PurgeExpungedResourcesCmd(); + assertNull(cmd.getStartDate()); + Date date = new Date(); + ReflectionTestUtils.setField(cmd, "startDate", date); + Assert.assertEquals(date, cmd.getStartDate()); + } + + @Test + public void testGetEndDate() { + PurgeExpungedResourcesCmd cmd = new PurgeExpungedResourcesCmd(); + assertNull(cmd.getEndDate()); + Date date = new Date(); + ReflectionTestUtils.setField(cmd, "endDate", date); + Assert.assertEquals(date, cmd.getEndDate()); + } + + @Test + public void testExecute() { + final PurgeExpungedResourcesResponse[] executeResponse = new PurgeExpungedResourcesResponse[1]; + Long result = 100L; + Mockito.when(resourceCleanupService.purgeExpungedResources(Mockito.any())).thenReturn(result); + Mockito.doAnswer((Answer) invocation -> { + executeResponse[0] = (PurgeExpungedResourcesResponse)invocation.getArguments()[0]; + return null; + }).when(spyCmd).setResponseObject(Mockito.any()); + spyCmd.execute(); + PurgeExpungedResourcesResponse response = executeResponse[0]; + Assert.assertNotNull(response); + Assert.assertEquals(result, response.getResourceCount()); + } + + @Test(expected = ServerApiException.class) + public void testExecuteException() { + Mockito.doThrow(CloudRuntimeException.class).when(resourceCleanupService).purgeExpungedResources(Mockito.any()); + spyCmd.execute(); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmdTest.java index 79f27fd6687e..a28e9e9fd04f 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmdTest.java @@ -167,17 +167,16 @@ public void testCreate() throws ResourceAllocationException { @Test public void testExecute() throws ResourceUnavailableException, InsufficientCapacityException { - ReflectionTestUtils.setField(cmd, "start", true); Vpc vpc = Mockito.mock(Vpc.class); VpcResponse response = Mockito.mock(VpcResponse.class); ReflectionTestUtils.setField(cmd, "id", 1L); responseGenerator = Mockito.mock(ResponseGenerator.class); - Mockito.when(_vpcService.startVpc(1L, true)).thenReturn(true); + Mockito.doNothing().when(_vpcService).startVpc(cmd); Mockito.when(_entityMgr.findById(Mockito.eq(Vpc.class), Mockito.any(Long.class))).thenReturn(vpc); cmd._responseGenerator = responseGenerator; Mockito.when(responseGenerator.createVpcResponse(ResponseObject.ResponseView.Restricted, vpc)).thenReturn(response); cmd.execute(); - Mockito.verify(_vpcService, Mockito.times(1)).startVpc(Mockito.anyLong(), Mockito.anyBoolean()); + Mockito.verify(_vpcService, Mockito.times(1)).startVpc(cmd); } } diff --git a/api/src/test/java/org/apache/cloudstack/api/response/HostResponseTest.java b/api/src/test/java/org/apache/cloudstack/api/response/HostResponseTest.java index 523b3de9e3cf..04e3ad7df603 100644 --- a/api/src/test/java/org/apache/cloudstack/api/response/HostResponseTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/response/HostResponseTest.java @@ -23,6 +23,8 @@ import java.util.HashMap; import java.util.Map; +import com.cloud.hypervisor.Hypervisor; + public final class HostResponseTest extends TestCase { private static final String VALID_KEY = "validkey"; @@ -32,7 +34,7 @@ public final class HostResponseTest extends TestCase { public void testSetDetailsNull() { final HostResponse hostResponse = new HostResponse(); - hostResponse.setDetails(null); + hostResponse.setDetails(null, null); assertEquals(null, hostResponse.getDetails()); @@ -51,7 +53,7 @@ public void testSetDetailsWithRootCredentials() { final Map expectedDetails = new HashedMap(); expectedDetails.put(VALID_KEY, VALID_VALUE); - hostResponse.setDetails(details); + hostResponse.setDetails(details, Hypervisor.HypervisorType.KVM); final Map actualDetails = hostResponse.getDetails(); assertTrue(details != actualDetails); @@ -70,7 +72,7 @@ public void testSetDetailsWithoutRootCredentials() { final Map expectedDetails = new HashedMap(); expectedDetails.put(VALID_KEY, VALID_VALUE); - hostResponse.setDetails(details); + hostResponse.setDetails(details, Hypervisor.HypervisorType.KVM); final Map actualDetails = hostResponse.getDetails(); assertTrue(details != actualDetails); diff --git a/build/replace.properties b/build/replace.properties index 3d9a45970600..ce38727b80a7 100644 --- a/build/replace.properties +++ b/build/replace.properties @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/client/bindir/cloud-update-xenserver-licenses.in b/client/bindir/cloud-update-xenserver-licenses.in index 9ce1898dc817..2be3a083f973 100755 --- a/client/bindir/cloud-update-xenserver-licenses.in +++ b/client/bindir/cloud-update-xenserver-licenses.in @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -46,7 +46,7 @@ cfg = "@MSCONF@/db.properties" #---------------------- option parsing and command line checks ------------------------ -usage = """%prog <-a | host names / IP addresses...> +usage = """%prog <-a | host names / IP addresses...> This command deploys the license file specified in the command line into a specific XenServer host or all XenServer hosts known to the management server.""" @@ -88,7 +88,7 @@ def parseuserpwfromhosts(hosts): return creds class XenServerConfigurator(Thread): - + def __init__(self,host,user,password,keyfiledata): Thread.__init__(self) self.host = host @@ -99,7 +99,7 @@ class XenServerConfigurator(Thread): self.stdout = "" self.stderr = "" self.state = 'initialized' - + def run(self): try: self.state = 'running' @@ -120,18 +120,18 @@ class XenServerConfigurator(Thread): c.close() if self.retval != 0: self.state = 'failed' else: self.state = 'finished' - + except Exception as e: self.state = 'failed' self.retval = e #raise - + def __str__(self): if self.state == 'failed': return "<%s XenServerConfigurator on %s@%s: %s>"%(self.state,self.user,self.host,str(self.retval)) else: return "<%s XenServerConfigurator on %s@%s>"%(self.state,self.user,self.host) - + #------------- actual code -------------------- (options, args) = parser.parse_args() @@ -162,7 +162,7 @@ for host,(user,password) in list(creds.items()): for c in configurators: c.start() - + for c in configurators: print(c.host + "...", end=' ') c.join() diff --git a/client/conf/log4j-cloud.xml.in b/client/conf/log4j-cloud.xml.in index dbcf8c6198bb..148ccbfbb41b 100755 --- a/client/conf/log4j-cloud.xml.in +++ b/client/conf/log4j-cloud.xml.in @@ -69,7 +69,7 @@ under the License. - + diff --git a/client/pom.xml b/client/pom.xml index 1d11fa74650a..23e0f1886bdb 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -438,6 +438,11 @@ cloud-mom-kafka ${project.version} + + org.apache.cloudstack + cloud-mom-webhook + ${project.version} + org.apache.cloudstack cloud-framework-agent-lb diff --git a/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceAnswer.java b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceAnswer.java new file mode 100644 index 000000000000..a02ac92927a9 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceAnswer.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +public class CheckConvertInstanceAnswer extends Answer { + + private boolean ovfExportSupported = false; + + public CheckConvertInstanceAnswer() { + super(); + } + + public CheckConvertInstanceAnswer(Command command, boolean success) { + super(command, success, ""); + } + + public CheckConvertInstanceAnswer(Command command, boolean success, String details) { + super(command, success, details); + } + + public CheckConvertInstanceAnswer(Command command, boolean success, boolean ovfExportSupported, String details) { + super(command, success, details); + this.ovfExportSupported = ovfExportSupported; + } + + public boolean isOvfExportSupported() { + return ovfExportSupported; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java new file mode 100644 index 000000000000..fc066e5c5899 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +public class CheckConvertInstanceCommand extends Command { + boolean checkWindowsGuestConversionSupport = false; + + public CheckConvertInstanceCommand() { + } + + public CheckConvertInstanceCommand(boolean checkWindowsGuestConversionSupport) { + this.checkWindowsGuestConversionSupport = checkWindowsGuestConversionSupport; + } + + @Override + public boolean executeInSequence() { + return false; + } + + public boolean getCheckWindowsGuestConversionSupport() { + return checkWindowsGuestConversionSupport; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/CheckVolumeAnswer.java b/core/src/main/java/com/cloud/agent/api/CheckVolumeAnswer.java index dd136d8642f6..5a32ab59a7a2 100644 --- a/core/src/main/java/com/cloud/agent/api/CheckVolumeAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/CheckVolumeAnswer.java @@ -17,7 +17,6 @@ package com.cloud.agent.api; -@LogLevel(LogLevel.Log4jLevel.Trace) public class CheckVolumeAnswer extends Answer { private long size; diff --git a/core/src/main/java/com/cloud/agent/api/CheckVolumeCommand.java b/core/src/main/java/com/cloud/agent/api/CheckVolumeCommand.java index b4036bebf3ac..bd44b35c8954 100644 --- a/core/src/main/java/com/cloud/agent/api/CheckVolumeCommand.java +++ b/core/src/main/java/com/cloud/agent/api/CheckVolumeCommand.java @@ -21,7 +21,6 @@ import com.cloud.agent.api.to.StorageFilerTO; -@LogLevel(LogLevel.Log4jLevel.Trace) public class CheckVolumeCommand extends Command { String srcFile; diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java index 63234b044807..b8250903f858 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java @@ -28,16 +28,24 @@ public class ConvertInstanceCommand extends Command { private Hypervisor.HypervisorType destinationHypervisorType; private List destinationStoragePools; private DataStoreTO conversionTemporaryLocation; + private String templateDirOnConversionLocation; + private boolean checkConversionSupport; + private boolean exportOvfToConversionLocation; + private int threadsCountToExportOvf = 0; public ConvertInstanceCommand() { } public ConvertInstanceCommand(RemoteInstanceTO sourceInstance, Hypervisor.HypervisorType destinationHypervisorType, - List destinationStoragePools, DataStoreTO conversionTemporaryLocation) { + List destinationStoragePools, DataStoreTO conversionTemporaryLocation, + String templateDirOnConversionLocation, boolean checkConversionSupport, boolean exportOvfToConversionLocation) { this.sourceInstance = sourceInstance; this.destinationHypervisorType = destinationHypervisorType; this.destinationStoragePools = destinationStoragePools; this.conversionTemporaryLocation = conversionTemporaryLocation; + this.templateDirOnConversionLocation = templateDirOnConversionLocation; + this.checkConversionSupport = checkConversionSupport; + this.exportOvfToConversionLocation = exportOvfToConversionLocation; } public RemoteInstanceTO getSourceInstance() { @@ -56,6 +64,26 @@ public DataStoreTO getConversionTemporaryLocation() { return conversionTemporaryLocation; } + public String getTemplateDirOnConversionLocation() { + return templateDirOnConversionLocation; + } + + public boolean getCheckConversionSupport() { + return checkConversionSupport; + } + + public boolean getExportOvfToConversionLocation() { + return exportOvfToConversionLocation; + } + + public int getThreadsCountToExportOvf() { + return threadsCountToExportOvf; + } + + public void setThreadsCountToExportOvf(int threadsCountToExportOvf) { + this.threadsCountToExportOvf = threadsCountToExportOvf; + } + @Override public boolean executeInSequence() { return false; diff --git a/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeAnswer.java b/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeAnswer.java index f6d7cab45964..e79005be71b9 100644 --- a/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeAnswer.java @@ -17,7 +17,6 @@ package com.cloud.agent.api; -@LogLevel(LogLevel.Log4jLevel.Trace) public class CopyRemoteVolumeAnswer extends Answer { private String remoteIp; diff --git a/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeCommand.java b/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeCommand.java index 82bc4d7cb459..798336b0e72c 100644 --- a/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeCommand.java +++ b/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeCommand.java @@ -21,16 +21,13 @@ import com.cloud.agent.api.to.StorageFilerTO; -@LogLevel(LogLevel.Log4jLevel.Trace) public class CopyRemoteVolumeCommand extends Command { - String remoteIp; String username; + @LogLevel(LogLevel.Log4jLevel.Off) String password; String srcFile; - String tmpPath; - StorageFilerTO storageFilerTO; public CopyRemoteVolumeCommand(String remoteIp, String username, String password) { diff --git a/core/src/main/java/com/cloud/agent/api/GetRemoteVmsAnswer.java b/core/src/main/java/com/cloud/agent/api/GetRemoteVmsAnswer.java index 8cd072f1da1d..c4e590591d0c 100644 --- a/core/src/main/java/com/cloud/agent/api/GetRemoteVmsAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/GetRemoteVmsAnswer.java @@ -22,10 +22,10 @@ import java.util.HashMap; import java.util.List; -@LogLevel(LogLevel.Log4jLevel.Trace) public class GetRemoteVmsAnswer extends Answer { private String remoteIp; + @LogLevel(LogLevel.Log4jLevel.Trace) private HashMap unmanagedInstances; List vmNames; diff --git a/core/src/main/java/com/cloud/agent/api/GetRemoteVmsCommand.java b/core/src/main/java/com/cloud/agent/api/GetRemoteVmsCommand.java index 5c71d12dbd08..5b6b9bdd3606 100644 --- a/core/src/main/java/com/cloud/agent/api/GetRemoteVmsCommand.java +++ b/core/src/main/java/com/cloud/agent/api/GetRemoteVmsCommand.java @@ -19,11 +19,11 @@ package com.cloud.agent.api; -@LogLevel(LogLevel.Log4jLevel.Trace) public class GetRemoteVmsCommand extends Command { String remoteIp; String username; + @LogLevel(LogLevel.Log4jLevel.Off) String password; public GetRemoteVmsCommand(String remoteIp, String username, String password) { diff --git a/core/src/main/java/com/cloud/agent/api/GetUnmanagedInstancesAnswer.java b/core/src/main/java/com/cloud/agent/api/GetUnmanagedInstancesAnswer.java index 771d472be2ae..950930ec6149 100644 --- a/core/src/main/java/com/cloud/agent/api/GetUnmanagedInstancesAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/GetUnmanagedInstancesAnswer.java @@ -21,10 +21,10 @@ import org.apache.cloudstack.vm.UnmanagedInstanceTO; -@LogLevel(LogLevel.Log4jLevel.Trace) public class GetUnmanagedInstancesAnswer extends Answer { private String instanceName; + @LogLevel(LogLevel.Log4jLevel.Trace) private HashMap unmanagedInstances; GetUnmanagedInstancesAnswer() { diff --git a/core/src/main/java/com/cloud/agent/api/GetUnmanagedInstancesCommand.java b/core/src/main/java/com/cloud/agent/api/GetUnmanagedInstancesCommand.java index 2cd80aebea1b..c0b8987e152d 100644 --- a/core/src/main/java/com/cloud/agent/api/GetUnmanagedInstancesCommand.java +++ b/core/src/main/java/com/cloud/agent/api/GetUnmanagedInstancesCommand.java @@ -28,10 +28,10 @@ * All managed instances will be filtered while trying to find unmanaged instances. */ -@LogLevel(LogLevel.Log4jLevel.Trace) public class GetUnmanagedInstancesCommand extends Command { String instanceName; + @LogLevel(LogLevel.Log4jLevel.Trace) List managedInstancesNames; public GetUnmanagedInstancesCommand() { diff --git a/core/src/main/java/com/cloud/agent/api/GetVolumeStatAnswer.java b/core/src/main/java/com/cloud/agent/api/GetVolumeStatAnswer.java new file mode 100644 index 000000000000..8352c97c108b --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/GetVolumeStatAnswer.java @@ -0,0 +1,85 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import com.cloud.agent.api.LogLevel.Log4jLevel; +import com.cloud.storage.Storage.StoragePoolType; + +@LogLevel(Log4jLevel.Trace) +public class GetVolumeStatAnswer extends Answer { + String poolUuid; + StoragePoolType poolType; + String volumePath; + long size = 0; + long virtualSize = 0; + + public GetVolumeStatAnswer(GetVolumeStatCommand cmd, long size, long virtualSize) { + super(cmd, true, ""); + this.poolUuid = cmd.getPoolUuid(); + this.poolType = cmd.getPoolType(); + this.volumePath = cmd.getVolumePath(); + this.size = size; + this.virtualSize = virtualSize; + } + + public GetVolumeStatAnswer(GetVolumeStatCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + + protected GetVolumeStatAnswer() { + //no-args constructor for json serialization-deserialization + } + + public String getPoolUuid() { + return poolUuid; + } + + public void setPoolUuid(String poolUuid) { + this.poolUuid = poolUuid; + } + + public StoragePoolType getPoolType() { + return poolType; + } + + public void setPoolType(StoragePoolType poolType) { + this.poolType = poolType; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + + public long getVirtualSize() { + return virtualSize; + } + + public void setVirtualSize(long virtualSize) { + this.virtualSize = virtualSize; + } + + public String getString() { + return "GetVolumeStatAnswer [poolUuid=" + poolUuid + ", poolType=" + poolType + ", volumePath=" + volumePath + ", size=" + size + ", virtualSize=" + virtualSize + "]"; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/GetVolumeStatCommand.java b/core/src/main/java/com/cloud/agent/api/GetVolumeStatCommand.java new file mode 100644 index 000000000000..1be3d6a04195 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/GetVolumeStatCommand.java @@ -0,0 +1,72 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import com.cloud.agent.api.LogLevel.Log4jLevel; +import com.cloud.storage.Storage.StoragePoolType; + +@LogLevel(Log4jLevel.Trace) +public class GetVolumeStatCommand extends Command { + String volumePath; + StoragePoolType poolType; + String poolUuid; + + protected GetVolumeStatCommand() { + } + + public GetVolumeStatCommand(String volumePath, StoragePoolType poolType, String poolUuid) { + this.volumePath = volumePath; + this.poolType = poolType; + this.poolUuid = poolUuid; + } + + public String getVolumePath() { + return volumePath; + } + + public void setVolumePath(String volumePath) { + this.volumePath = volumePath; + } + + public StoragePoolType getPoolType() { + return poolType; + } + + public void setPoolType(StoragePoolType poolType) { + this.poolType = poolType; + } + + public String getPoolUuid() { + return poolUuid; + } + + public void setPoolUuid(String storeUuid) { + this.poolUuid = storeUuid; + } + + @Override + public boolean executeInSequence() { + return false; + } + + public String getString() { + return "GetVolumeStatCommand [volumePath=" + volumePath + ", poolType=" + poolType + ", poolUuid=" + poolUuid + "]"; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolCommand.java b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolCommand.java index ad05fe1d615d..06940266b538 100644 --- a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolCommand.java @@ -46,6 +46,10 @@ public ModifyStoragePoolCommand(boolean add, StoragePool pool, String localPath, this.details = details; } + public ModifyStoragePoolCommand(boolean add, StoragePool pool, Map details) { + this(add, pool, LOCAL_PATH_PREFIX + File.separator + UUID.nameUUIDFromBytes((pool.getHostAddress() + pool.getPath()).getBytes()), details); + } + public ModifyStoragePoolCommand(boolean add, StoragePool pool) { this(add, pool, LOCAL_PATH_PREFIX + File.separator + UUID.nameUUIDFromBytes((pool.getHostAddress() + pool.getPath()).getBytes())); } diff --git a/core/src/main/java/com/cloud/agent/api/PrepareStorageClientAnswer.java b/core/src/main/java/com/cloud/agent/api/PrepareStorageClientAnswer.java new file mode 100644 index 000000000000..85afb9256464 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/PrepareStorageClientAnswer.java @@ -0,0 +1,43 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import java.util.Map; + +public class PrepareStorageClientAnswer extends Answer { + Map detailsMap; + + public PrepareStorageClientAnswer() { + super(); + } + + public PrepareStorageClientAnswer(Command command, boolean success, Map detailsMap) { + super(command, success, ""); + this.detailsMap = detailsMap; + } + + public PrepareStorageClientAnswer(Command command, boolean success, String details) { + super(command, success, details); + } + + public Map getDetailsMap() { + return detailsMap; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/PrepareStorageClientCommand.java b/core/src/main/java/com/cloud/agent/api/PrepareStorageClientCommand.java new file mode 100644 index 000000000000..8dea9c11c535 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/PrepareStorageClientCommand.java @@ -0,0 +1,56 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import java.util.Map; + +import com.cloud.storage.Storage.StoragePoolType; + +public class PrepareStorageClientCommand extends Command { + private StoragePoolType poolType; + private String poolUuid; + private Map details; + + public PrepareStorageClientCommand() { + } + + public PrepareStorageClientCommand(StoragePoolType poolType, String poolUuid, Map details) { + this.poolType = poolType; + this.poolUuid = poolUuid; + this.details = details; + } + + @Override + public boolean executeInSequence() { + return false; + } + + public StoragePoolType getPoolType() { + return poolType; + } + + public String getPoolUuid() { + return poolUuid; + } + + public Map getDetails() { + return details; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/StartupRoutingCommand.java b/core/src/main/java/com/cloud/agent/api/StartupRoutingCommand.java index b4f9d20df5ed..2d4ed8c9cc42 100644 --- a/core/src/main/java/com/cloud/agent/api/StartupRoutingCommand.java +++ b/core/src/main/java/com/cloud/agent/api/StartupRoutingCommand.java @@ -174,6 +174,10 @@ public void setHostTags(String hostTag) { this.hostTags.add(hostTag); } + public void setHostTags(List hostTags) { + this.hostTags = hostTags; + } + public HashMap> getGpuGroupDetails() { return groupDetails; } diff --git a/core/src/main/java/com/cloud/agent/api/UnprepareStorageClientAnswer.java b/core/src/main/java/com/cloud/agent/api/UnprepareStorageClientAnswer.java new file mode 100644 index 000000000000..1280293db0dc --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/UnprepareStorageClientAnswer.java @@ -0,0 +1,34 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +public class UnprepareStorageClientAnswer extends Answer { + public UnprepareStorageClientAnswer() { + super(); + } + + public UnprepareStorageClientAnswer(Command command, boolean success) { + super(command, success, ""); + } + + public UnprepareStorageClientAnswer(Command command, boolean success, String details) { + super(command, success, details); + } +} diff --git a/core/src/main/java/com/cloud/agent/api/UnprepareStorageClientCommand.java b/core/src/main/java/com/cloud/agent/api/UnprepareStorageClientCommand.java new file mode 100644 index 000000000000..bebd30ca519d --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/UnprepareStorageClientCommand.java @@ -0,0 +1,48 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import com.cloud.storage.Storage.StoragePoolType; + +public class UnprepareStorageClientCommand extends Command { + private StoragePoolType poolType; + private String poolUuid; + + public UnprepareStorageClientCommand() { + } + + public UnprepareStorageClientCommand(StoragePoolType poolType, String poolUuid) { + this.poolType = poolType; + this.poolUuid = poolUuid; + } + + @Override + public boolean executeInSequence() { + return false; + } + + public StoragePoolType getPoolType() { + return poolType; + } + + public String getPoolUuid() { + return poolUuid; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/storage/CreateEntityDownloadURLCommand.java b/core/src/main/java/com/cloud/agent/api/storage/CreateEntityDownloadURLCommand.java index c84eceb0ca2f..33403580b6fe 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/CreateEntityDownloadURLCommand.java +++ b/core/src/main/java/com/cloud/agent/api/storage/CreateEntityDownloadURLCommand.java @@ -23,18 +23,19 @@ public class CreateEntityDownloadURLCommand extends AbstractDownloadCommand { - public CreateEntityDownloadURLCommand(String parent, String installPath, String uuid, DataTO data) { // this constructor is for creating template download url + public CreateEntityDownloadURLCommand(String parent, String installPath, String fileName, String filePath, DataTO data) { // this constructor is for creating template download url super(); this.parent = parent; // parent is required as not the template can be child of one of many parents this.installPath = installPath; - this.extractLinkUUID = uuid; + this.filenameInExtractURL = fileName; + this.filepathInExtractURL = filePath; this.data = data; } - public CreateEntityDownloadURLCommand(String installPath, String uuid) { + public CreateEntityDownloadURLCommand(String installPath, String filename) { super(); this.installPath = installPath; - this.extractLinkUUID = uuid; + this.filenameInExtractURL = filename; } public CreateEntityDownloadURLCommand() { @@ -42,7 +43,8 @@ public CreateEntityDownloadURLCommand() { private String installPath; private String parent; - private String extractLinkUUID; + private String filenameInExtractURL; + private String filepathInExtractURL; public DataTO getData() { return data; @@ -75,12 +77,19 @@ public void setParent(String parent) { this.parent = parent; } - public String getExtractLinkUUID() { - return extractLinkUUID; + public String getFilenameInExtractURL() { + return filenameInExtractURL; } - public void setExtractLinkUUID(String extractLinkUUID) { - this.extractLinkUUID = extractLinkUUID; + public void setFilenameInExtractURL(String filenameInExtractURL) { + this.filenameInExtractURL = filenameInExtractURL; } + public String getFilepathInExtractURL() { + return filepathInExtractURL; + } + + public void setFilepathInExtractURL(String filepathInExtractURL) { + this.filepathInExtractURL = filepathInExtractURL; + } } diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java index ebe5e9a7ec94..e435c838b7de 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java @@ -81,4 +81,5 @@ public class VRScripts { public static final String VR_UPDATE_INTERFACE_CONFIG = "update_interface_config.sh"; public static final String ROUTER_FILESYSTEM_WRITABLE_CHECK = "filesystem_writable_check.py"; + public static final String MANAGE_SERVICE = "manage_service.sh"; } diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index 3c86b3a0dcc4..4afac9b43cb3 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -34,6 +34,7 @@ import javax.naming.ConfigurationException; +import org.apache.cloudstack.agent.routing.ManageServiceCommand; import com.cloud.agent.api.routing.UpdateNetworkCommand; import com.cloud.agent.api.to.IpAddressTO; import com.cloud.network.router.VirtualRouter; @@ -144,6 +145,10 @@ public Answer executeRequest(final NetworkElementCommand cmd) { return execute((UpdateNetworkCommand) cmd); } + if (cmd instanceof ManageServiceCommand) { + return execute((ManageServiceCommand) cmd); + } + if (_vrAggregateCommandsSet.containsKey(routerName)) { _vrAggregateCommandsSet.get(routerName).add(cmd); aggregated = true; @@ -271,6 +276,20 @@ private Answer execute(UpdateNetworkCommand cmd) { return new Answer(cmd, new CloudRuntimeException("Failed to update interface mtu")); } + private Answer execute(ManageServiceCommand cmd) { + String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + String args = cmd.getAction() + " " + cmd.getServiceName(); + ExecutionResult result = _vrDeployer.executeInVR(routerIp, VRScripts.MANAGE_SERVICE, args); + if (result.isSuccess()) { + return new Answer(cmd, true, + String.format("Successfully executed action: %s on service: %s. Details: %s", + cmd.getAction(), cmd.getServiceName(), result.getDetails())); + } else { + return new Answer(cmd, false, String.format("Failed to execute action: %s on service: %s. Details: %s", + cmd.getAction(), cmd.getServiceName(), result.getDetails())); + } + } + private ExecutionResult applyConfigToVR(String routerAccessIp, ConfigItem c) { return applyConfigToVR(routerAccessIp, c, VRScripts.VR_SCRIPT_EXEC_TIMEOUT); } diff --git a/core/src/main/java/com/cloud/resource/CommandWrapper.java b/core/src/main/java/com/cloud/resource/CommandWrapper.java index a839234117be..72d1348dfe70 100644 --- a/core/src/main/java/com/cloud/resource/CommandWrapper.java +++ b/core/src/main/java/com/cloud/resource/CommandWrapper.java @@ -19,10 +19,13 @@ package com.cloud.resource; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; public abstract class CommandWrapper { protected Logger logger = LogManager.getLogger(getClass()); @@ -33,4 +36,26 @@ public abstract class CommandWrapper*?![]{}~".indexOf(c) != -1) { + sanitized.append('\\'); + } + sanitized.append(c); + } + return sanitized.toString(); + } + + public void removeDpdkPort(String portToRemove) { + logger.debug("Removing DPDK port: " + portToRemove); + int port; + try { + port = Integer.valueOf(portToRemove); + } catch (NumberFormatException nfe) { + throw new CloudRuntimeException(String.format("Invalid DPDK port specified: '%s'", portToRemove)); + } + Script.executeCommand("ovs-vsctl", "del-port", String.valueOf(port)); + } } diff --git a/core/src/main/java/org/apache/cloudstack/agent/routing/ManageServiceCommand.java b/core/src/main/java/org/apache/cloudstack/agent/routing/ManageServiceCommand.java new file mode 100644 index 000000000000..c83a5b695740 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/agent/routing/ManageServiceCommand.java @@ -0,0 +1,49 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.agent.routing; + +import com.cloud.agent.api.routing.NetworkElementCommand; + +public class ManageServiceCommand extends NetworkElementCommand { + + String serviceName; + String action; + + @Override + public boolean executeInSequence() { + return true; + } + + protected ManageServiceCommand() { + } + + public ManageServiceCommand(String serviceName, String action) { + this.serviceName = serviceName; + this.action = action; + } + + public String getServiceName() { + return serviceName; + } + + public String getAction() { + return action; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java index 2bb67c80ce43..6514038ac623 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java @@ -39,6 +39,7 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO { private DataStoreTO dataStore; private String name; private Long size; + private Long usableSize; private String path; private Long volumeId; private String vmName; @@ -161,6 +162,10 @@ public Long getSize() { return size; } + public Long getUsableSize() { + return usableSize; + } + @Override public DataObjectType getObjectType() { return DataObjectType.VOLUME; @@ -178,6 +183,10 @@ public void setSize(long size) { this.size = size; } + public void setUsableSize(Long usableSize) { + this.usableSize = usableSize; + } + public void setPath(String path) { this.path = path; } diff --git a/core/src/main/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml b/core/src/main/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml index 49775fe41e1c..732f36534d2d 100644 --- a/core/src/main/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml +++ b/core/src/main/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml @@ -288,10 +288,10 @@ + class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry"> - + @@ -339,7 +339,7 @@ class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry"> - @@ -358,4 +358,9 @@ + + + + diff --git a/core/src/main/resources/META-INF/cloudstack/event/module.properties b/core/src/main/resources/META-INF/cloudstack/event/module.properties new file mode 100644 index 000000000000..ab1f88e98448 --- /dev/null +++ b/core/src/main/resources/META-INF/cloudstack/event/module.properties @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name=event +parent=core diff --git a/core/src/main/resources/META-INF/cloudstack/event/spring-core-lifecycle-event-context-inheritable.xml b/core/src/main/resources/META-INF/cloudstack/event/spring-core-lifecycle-event-context-inheritable.xml new file mode 100644 index 000000000000..63d11c65bacb --- /dev/null +++ b/core/src/main/resources/META-INF/cloudstack/event/spring-core-lifecycle-event-context-inheritable.xml @@ -0,0 +1,31 @@ + + + + + + + + diff --git a/core/src/main/resources/META-INF/cloudstack/kubernetes/spring-core-lifecycle-kubernetes-context-inheritable.xml b/core/src/main/resources/META-INF/cloudstack/kubernetes/spring-core-lifecycle-kubernetes-context-inheritable.xml index df1a4b5c2298..96a9a634bae8 100644 --- a/core/src/main/resources/META-INF/cloudstack/kubernetes/spring-core-lifecycle-kubernetes-context-inheritable.xml +++ b/core/src/main/resources/META-INF/cloudstack/kubernetes/spring-core-lifecycle-kubernetes-context-inheritable.xml @@ -25,8 +25,8 @@ > - - + + diff --git a/debian/control b/debian/control index 3508c7b5f754..dab7b254b88c 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, lsb-release, ufw, apparmor +Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, lsb-release, ufw, apparmor, cpu-checker Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/debian/rules b/debian/rules index f8228e61e464..b52803702724 100755 --- a/debian/rules +++ b/debian/rules @@ -103,6 +103,8 @@ override_dh_auto_install: install -m0644 packaging/systemd/$(PACKAGE)-management.service debian/$(PACKAGE)-management/lib/systemd/system/$(PACKAGE)-management.service install -m0644 packaging/systemd/$(PACKAGE)-management.default $(DESTDIR)/$(SYSCONFDIR)/default/$(PACKAGE)-management + install -D -m0644 server/target/conf/cloudstack-management.logrotate $(DESTDIR)/$(SYSCONFDIR)/logrotate.d/cloudstack-management + # cloudstack-ui mkdir $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/ui mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-ui @@ -159,6 +161,8 @@ override_dh_auto_install: install -m0644 packaging/systemd/$(PACKAGE)-usage.service debian/$(PACKAGE)-usage/lib/systemd/system/$(PACKAGE)-usage.service install -m0644 packaging/systemd/$(PACKAGE)-usage.default $(DESTDIR)/$(SYSCONFDIR)/default/$(PACKAGE)-usage + install -D -m0644 usage/target/transformed/cloudstack-usage.logrotate $(DESTDIR)/$(SYSCONFDIR)/logrotate.d/cloudstack-usage + # cloudstack-marvin mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-marvin cp tools/marvin/dist/Marvin-*.tar.gz $(DESTDIR)/usr/share/$(PACKAGE)-marvin/ diff --git a/deps/install-non-oss.sh b/deps/install-non-oss.sh index c6b91e07cec4..ea40e9a55634 100755 --- a/deps/install-non-oss.sh +++ b/deps/install-non-oss.sh @@ -8,7 +8,7 @@ # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index 110592161f96..41bd74f11924 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -348,4 +348,6 @@ void implementNetworkElementsAndResources(DeployDestination dest, ReservationCon Pair importNic(final String macAddress, int deviceId, final Network network, final Boolean isDefaultNic, final VirtualMachine vm, final Network.IpAddresses ipAddresses, final DataCenter datacenter, boolean forced) throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException; void unmanageNics(VirtualMachineProfile vm); + + void expungeLbVmRefs(List vmIds, Long batchSize); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index c3525466ce19..7950dda4d68e 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.Set; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; @@ -126,7 +127,7 @@ DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Lon void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest); - void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException, StorageAccessException; + void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException, StorageAccessException, ResourceAllocationException; boolean canVmRestartOnAnotherServer(long vmId); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataObject.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataObject.java index 091c09d7a4d0..fe052f016065 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataObject.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataObject.java @@ -50,4 +50,6 @@ public interface DataObject { void decRefCount(); Long getRefCount(); + + String getName(); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java index 2c7d3c602783..d52c656f6dbc 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java @@ -18,6 +18,8 @@ */ package org.apache.cloudstack.engine.subsystem.api.storage; +import java.util.Map; + import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.storage.command.CommandResult; @@ -86,6 +88,22 @@ default boolean requiresAccessForMigration(DataObject dataObject) { */ boolean canProvideStorageStats(); + /** + * intended for managed storage + * returns true if the storage can provide its custom stats + */ + default boolean poolProvidesCustomStorageStats() { + return false; + } + + /** + * intended for managed storage + * returns the custom stats if the storage can provide them + */ + default Map getCustomStorageStats(StoragePool pool) { + return null; + } + /** * intended for managed storage * returns the total capacity and used size in bytes @@ -110,6 +128,14 @@ default boolean requiresAccessForMigration(DataObject dataObject) { */ boolean canHostAccessStoragePool(Host host, StoragePool pool); + /** + * intended for managed storage + * returns true if the host can prepare storage client to provide access the storage pool + */ + default boolean canHostPrepareStoragePoolAccess(Host host, StoragePool pool) { + return false; + } + /** * Used by storage pools which want to keep VMs' information * @return true if additional VM info is needed (intended for storage pools). @@ -157,4 +183,20 @@ default boolean volumesRequireGrantAccessWhenUsed() { default boolean zoneWideVolumesAvailableWithoutClusterMotion() { return false; } + + /** + * This method returns the actual size required on the pool for a volume. + * + * @param volumeSize + * Size of volume to be created on the store + * @param templateSize + * Size of template, if any, which will be used to create the volume + * @param isEncryptionRequired + * true if volume is encrypted + * + * @return the size required on the pool for the volume + */ + default long getVolumeSizeRequiredOnPool(long volumeSize, Long templateSize, boolean isEncryptionRequired) { + return volumeSize; + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreLifeCycle.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreLifeCycle.java index fcbc19c28b7b..54f3c63f8d73 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreLifeCycle.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreLifeCycle.java @@ -20,6 +20,7 @@ import java.util.Map; +import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.StoragePool; public interface PrimaryDataStoreLifeCycle extends DataStoreLifeCycle { @@ -29,4 +30,6 @@ public interface PrimaryDataStoreLifeCycle extends DataStoreLifeCycle { void updateStoragePool(StoragePool storagePool, Map details); void enableStoragePool(DataStore store); void disableStoragePool(DataStore store); + void changeStoragePoolScopeToZone(DataStore store, ClusterScope clusterScope, Hypervisor.HypervisorType hypervisorType); + void changeStoragePoolScopeToCluster(DataStore store, ClusterScope clusterScope, Hypervisor.HypervisorType hypervisorType); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java index 7c4d56e12b92..682473ec94fc 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java @@ -121,4 +121,6 @@ boolean copyPoliciesBetweenVolumesAndDestroySourceVolumeAfterMigration(ObjectInD Pair checkAndRepairVolume(VolumeInfo volume); void checkAndRepairVolumeBasedOnConfig(DataObject dataObject, Host host); + + void validateChangeDiskOfferingEncryptionType(long existingDiskOfferingId, long newDiskOfferingId); } diff --git a/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java b/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java index ecdb59667c96..3d4e6579f7ca 100644 --- a/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java +++ b/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java @@ -38,8 +38,10 @@ public interface AlertManager extends Manager, AlertService { public static final ConfigKey AlertSmtpUseStartTLS = new ConfigKey("Advanced", Boolean.class, "alert.smtp.useStartTLS", "false", "If set to true and if we enable security via alert.smtp.useAuth, this will enable StartTLS to secure the connection.", true); - public static final ConfigKey AlertSmtpEnabledSecurityProtocols = new ConfigKey("Advanced", String.class, "alert.smtp.enabledSecurityProtocols", "", - "White-space separated security protocols; ex: \"TLSv1 TLSv1.1\". Supported protocols: SSLv2Hello, SSLv3, TLSv1, TLSv1.1 and TLSv1.2", true); + public static final ConfigKey AlertSmtpUseAuth = new ConfigKey<>(ConfigKey.CATEGORY_ALERT, Boolean.class, "alert.smtp.useAuth", "false", "If true, use SMTP authentication when sending emails.", false, ConfigKey.Scope.ManagementServer); + + public static final ConfigKey AlertSmtpEnabledSecurityProtocols = new ConfigKey(ConfigKey.CATEGORY_ADVANCED, String.class, "alert.smtp.enabledSecurityProtocols", "", + "White-space separated security protocols; ex: \"TLSv1 TLSv1.1\". Supported protocols: SSLv2Hello, SSLv3, TLSv1, TLSv1.1 and TLSv1.2", true, ConfigKey.Kind.WhitespaceSeparatedListWithOptions, "SSLv2Hello,SSLv3,TLSv1,TLSv1.1,TLSv1.2"); public static final ConfigKey Ipv6SubnetCapacityThreshold = new ConfigKey("Advanced", Double.class, "zone.virtualnetwork.ipv6subnet.capacity.notificationthreshold", diff --git a/engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java b/engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java index ebbae0b31c28..c877ebbe8d2b 100644 --- a/engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java +++ b/engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.Set; +import com.cloud.dc.VlanVO; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.impl.ConfigurationSubGroupVO; @@ -60,9 +61,6 @@ public interface ConfigurationManager { public static final String MESSAGE_CREATE_VLAN_IP_RANGE_EVENT = "Message.CreateVlanIpRange.Event"; public static final String MESSAGE_DELETE_VLAN_IP_RANGE_EVENT = "Message.DeleteVlanIpRange.Event"; - static final String VM_USERDATA_MAX_LENGTH_STRING = "vm.userdata.max.length"; - static final ConfigKey VM_USERDATA_MAX_LENGTH = new ConfigKey<>("Advanced", Integer.class, VM_USERDATA_MAX_LENGTH_STRING, "32768", - "Max length of vm userdata after base64 decoding. Default is 32768 and maximum is 1048576", true); public static final ConfigKey AllowNonRFC1918CompliantIPs = new ConfigKey<>(Boolean.class, "allow.non.rfc1918.compliant.ips", "Advanced", "false", "Allows non-compliant RFC 1918 IPs for Shared, Isolated networks and VPCs", true, null); @@ -189,7 +187,7 @@ DataCenterVO createZone(long userId, String zoneName, String dns1, String dns2, * @param caller * @return success/failure */ - boolean deleteVlanAndPublicIpRange(long userId, long vlanDbId, Account caller); + VlanVO deleteVlanAndPublicIpRange(long userId, long vlanDbId, Account caller); void checkZoneAccess(Account caller, DataCenter zone); diff --git a/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java b/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java index 27f63c8c64b2..51d0846fafbc 100644 --- a/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java +++ b/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java @@ -25,15 +25,14 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; -import org.apache.commons.collections.MapUtils; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; -import org.springframework.beans.factory.NoSuchBeanDefinitionException; - import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.events.Event; import org.apache.cloudstack.framework.events.EventBus; -import org.apache.cloudstack.framework.events.EventBusException; +import org.apache.cloudstack.framework.events.EventDistributor; +import org.apache.commons.collections.MapUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.NoSuchBeanDefinitionException; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; @@ -50,6 +49,7 @@ public class UsageEventUtils { protected static Logger LOGGER = LogManager.getLogger(UsageEventUtils.class); protected static EventBus s_eventBus = null; protected static ConfigurationDao s_configDao; + private static EventDistributor eventDistributor; @Inject UsageEventDao usageEventDao; @@ -207,9 +207,9 @@ private static void publishUsageEvent(String usageEventType, Long accountId, Lon if( !configValue) return; try { - s_eventBus = ComponentContext.getComponent(EventBus.class); + eventDistributor = ComponentContext.getComponent(EventDistributor.class); } catch (NoSuchBeanDefinitionException nbe) { - return; // no provider is configured to provide events bus, so just return + return; // no provider is configured to provide events distributor, so just return } Account account = s_accountDao.findById(accountId); @@ -238,11 +238,7 @@ private static void publishUsageEvent(String usageEventType, Long accountId, Lon event.setDescription(eventDescription); - try { - s_eventBus.publish(event); - } catch (EventBusException e) { - LOGGER.warn("Failed to publish usage event on the event bus."); - } + eventDistributor.publish(event); } static final String Name = "management-server"; diff --git a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java index 72737d0b04d2..ae47b1d76ed8 100644 --- a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java +++ b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java @@ -156,4 +156,5 @@ enum Step { String getHaTag(); DeploymentPlanner getHAPlanner(); + int expungeWorkItemsByVmList(List vmIds, Long batchSize); } diff --git a/engine/components-api/src/main/java/com/cloud/network/NetworkStateListener.java b/engine/components-api/src/main/java/com/cloud/network/NetworkStateListener.java index 24be76e4d3be..107e177ef579 100644 --- a/engine/components-api/src/main/java/com/cloud/network/NetworkStateListener.java +++ b/engine/components-api/src/main/java/com/cloud/network/NetworkStateListener.java @@ -25,11 +25,9 @@ import javax.inject.Inject; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.events.EventBus; -import org.apache.cloudstack.framework.events.EventBusException; -import org.apache.logging.log4j.Logger; +import org.apache.cloudstack.framework.events.EventDistributor; import org.apache.logging.log4j.LogManager; -import org.springframework.beans.factory.NoSuchBeanDefinitionException; +import org.apache.logging.log4j.Logger; import com.cloud.event.EventCategory; import com.cloud.network.Network.Event; @@ -43,7 +41,7 @@ public class NetworkStateListener implements StateListener t return true; } - private void pubishOnEventBus(String event, String status, Network vo, State oldState, State newState) { - + private void pubishOnEventBus(String event, String status, Network vo, State oldState, State newState) { String configKey = "publish.resource.state.events"; String value = _configDao.getValue(configKey); boolean configValue = Boolean.parseBoolean(value); if(!configValue) return; - try { - s_eventBus = ComponentContext.getComponent(EventBus.class); - } catch (NoSuchBeanDefinitionException nbe) { - return; // no provider is configured to provide events bus, so just return + if (eventDistributor == null) { + setEventDistributor(ComponentContext.getComponent(EventDistributor.class)); } String resourceName = getEntityFromClassName(Network.class.getName()); org.apache.cloudstack.framework.events.Event eventMsg = - new org.apache.cloudstack.framework.events.Event("management-server", EventCategory.RESOURCE_STATE_CHANGE_EVENT.getName(), event, resourceName, vo.getUuid()); - Map eventDescription = new HashMap(); + new org.apache.cloudstack.framework.events.Event("management-server", EventCategory.RESOURCE_STATE_CHANGE_EVENT.getName(), event, resourceName, vo.getUuid()); + Map eventDescription = new HashMap<>(); eventDescription.put("resource", resourceName); eventDescription.put("id", vo.getUuid()); eventDescription.put("old-state", oldState.name()); @@ -92,11 +91,8 @@ private void pubishOnEventBus(String event, String status, Network vo, State old eventDescription.put("eventDateTime", eventDate); eventMsg.setDescription(eventDescription); - try { - s_eventBus.publish(eventMsg); - } catch (EventBusException e) { - logger.warn("Failed to publish state change event on the event bus."); - } + + eventDistributor.publish(eventMsg); } private String getEntityFromClassName(String entityClassName) { diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index 9308be5fb320..b2ae8b898378 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -126,12 +126,18 @@ public interface ResourceManager extends ResourceService, Configurable { public List listAllUpAndEnabledHostsInOneZoneByHypervisor(HypervisorType type, long dcId); + public List listAllUpHostsInOneZoneByHypervisor(HypervisorType type, long dcId); + public List listAllUpAndEnabledHostsInOneZone(long dcId); public List listAllHostsInOneZoneByType(Host.Type type, long dcId); public List listAllHostsInAllZonesByType(Type type); + public List listAllHostsInOneZoneNotInClusterByHypervisor(final HypervisorType type, long dcId, long clusterId); + + public List listAllHostsInOneZoneNotInClusterByHypervisors(List types, long dcId, long clusterId); + public List listAvailHypervisorInZone(Long hostId, Long zoneId); public HostVO findHostByGuid(String guid); diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index daeb4b19a187..c3909bc56b0d 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -18,6 +18,7 @@ import java.math.BigDecimal; import java.util.List; +import java.util.Map; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; @@ -96,14 +97,6 @@ public interface StorageManager extends StorageService { true, ConfigKey.Scope.Global, null); - ConfigKey ConvertVmwareInstanceToKvmTimeout = new ConfigKey<>(Integer.class, - "convert.vmware.instance.to.kvm.timeout", - "Storage", - "8", - "Timeout (in hours) for the instance conversion process from VMware through the virt-v2v binary on a KVM host", - true, - ConfigKey.Scope.Global, - null); ConfigKey KvmAutoConvergence = new ConfigKey<>(Boolean.class, "kvm.auto.convergence", "Storage", @@ -125,7 +118,7 @@ public interface StorageManager extends StorageService { "storage.pool.disk.wait", "Storage", "60", - "Timeout (in secs) for the storage pool disk (of managed pool) to become available in the host. Currently only supported for PowerFlex.", + "Timeout (in secs) for the storage pool disk (of managed pool) to become available in the host. Currently supported for PowerFlex only.", true, ConfigKey.Scope.StoragePool, null); @@ -134,7 +127,7 @@ public interface StorageManager extends StorageService { "storage.pool.client.timeout", "Storage", "60", - "Timeout (in secs) for the storage pool client connection timeout (for managed pools). Currently only supported for PowerFlex.", + "Timeout (in secs) for the API client connection timeout of storage pool (for managed pools). Currently supported for PowerFlex only.", false, ConfigKey.Scope.StoragePool, null); @@ -143,11 +136,20 @@ public interface StorageManager extends StorageService { "storage.pool.client.max.connections", "Storage", "100", - "Maximum connections for the storage pool client (for managed pools). Currently only supported for PowerFlex.", + "Maximum connections for the API client of storage pool (for managed pools). Currently supported for PowerFlex only.", false, ConfigKey.Scope.StoragePool, null); + ConfigKey STORAGE_POOL_CONNECTED_CLIENTS_LIMIT = new ConfigKey<>(Integer.class, + "storage.pool.connected.clients.limit", + "Storage", + "-1", + "Maximum connected storage pool clients supported for the storage (for managed pools), <= 0 for unlimited (default: -1). Currently supported for PowerFlex only.", + true, + ConfigKey.Scope.StoragePool, + null); + ConfigKey STORAGE_POOL_IO_POLICY = new ConfigKey<>(String.class, "kvm.storage.pool.io.policy", "Storage", @@ -259,6 +261,10 @@ static Boolean getFullCloneConfiguration(Long storeId) { boolean canPoolProvideStorageStats(StoragePool pool); + boolean poolProvidesCustomStorageStats(StoragePool pool); + + Map getCustomStorageStats(StoragePool pool); + /** * Checks if a host has running VMs that are using its local storage pool. * @return true if local storage is active on the host @@ -295,6 +301,8 @@ static Boolean getFullCloneConfiguration(Long storeId) { boolean canHostAccessStoragePool(Host host, StoragePool pool); + boolean canHostPrepareStoragePoolAccess(Host host, StoragePool pool); + Host getHost(long hostId); Host updateSecondaryStorage(long secStorageId, String newUrl); @@ -348,6 +356,10 @@ static Boolean getFullCloneConfiguration(Long storeId) { boolean registerHostListener(String providerUuid, HypervisorHostListener listener); + Pair, Boolean> getStoragePoolNFSMountOpts(StoragePool pool, Map details); + + String getStoragePoolMountFailureReason(String error); + boolean connectHostToSharedPool(long hostId, long poolId) throws StorageUnavailableException, StorageConflictException; void disconnectHostFromSharedPool(long hostId, long poolId) throws StorageUnavailableException, StorageConflictException; @@ -368,6 +380,8 @@ static Boolean getFullCloneConfiguration(Long storeId) { Long getDiskIopsWriteRate(ServiceOffering offering, DiskOffering diskOffering); + ImageStore updateImageStoreStatus(Long id, String name, Boolean readonly, Long capacityBytes); + void cleanupDownloadUrls(); void setDiskProfileThrottling(DiskProfile dskCh, ServiceOffering offering, DiskOffering diskOffering); diff --git a/engine/components-api/src/main/java/com/cloud/vm/VirtualMachineProfileImpl.java b/engine/components-api/src/main/java/com/cloud/vm/VirtualMachineProfileImpl.java index 2d51c3c08703..a1c54b90328b 100644 --- a/engine/components-api/src/main/java/com/cloud/vm/VirtualMachineProfileImpl.java +++ b/engine/components-api/src/main/java/com/cloud/vm/VirtualMachineProfileImpl.java @@ -264,11 +264,13 @@ public void setServiceOffering(ServiceOffering offering) { _offering = offering; } + @Override public void setCpuOvercommitRatio(Float cpuOvercommitRatio) { this.cpuOvercommitRatio = cpuOvercommitRatio; } + @Override public void setMemoryOvercommitRatio(Float memoryOvercommitRatio) { this.memoryOvercommitRatio = memoryOvercommitRatio; diff --git a/engine/components-api/src/test/java/com/cloud/network/NetworkStateListenerTest.java b/engine/components-api/src/test/java/com/cloud/network/NetworkStateListenerTest.java new file mode 100644 index 000000000000..30960210bfb6 --- /dev/null +++ b/engine/components-api/src/test/java/com/cloud/network/NetworkStateListenerTest.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network; + +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.events.EventDistributor; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; + +public class NetworkStateListenerTest { + @InjectMocks + NetworkStateListener networkStateListener = new NetworkStateListener(Mockito.mock(ConfigurationDao.class)); + + @Test + public void testSetEventDistributor() { + EventDistributor eventDistributor = null; + networkStateListener.setEventDistributor(eventDistributor); + Assert.assertNull(ReflectionTestUtils.getField(networkStateListener, "eventDistributor")); + eventDistributor = Mockito.mock(EventDistributor.class); + networkStateListener.setEventDistributor(eventDistributor); + Assert.assertEquals(eventDistributor, ReflectionTestUtils.getField(networkStateListener, "eventDistributor")); + } +} diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java index 22c0b3fd71a9..173fd9fc704a 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java @@ -45,6 +45,8 @@ import com.cloud.agent.api.CheckVirtualMachineCommand; import com.cloud.agent.api.CleanupNetworkRulesCmd; import com.cloud.agent.api.Command; +import com.cloud.agent.api.CreateStoragePoolCommand; +import com.cloud.agent.api.DeleteStoragePoolCommand; import com.cloud.agent.api.MaintainCommand; import com.cloud.agent.api.MigrateCommand; import com.cloud.agent.api.ModifySshKeysCommand; @@ -122,8 +124,9 @@ public int compare(final Object o1, final Object o2) { StopCommand.class.toString(), CheckVirtualMachineCommand.class.toString(), PingTestCommand.class.toString(), CheckHealthCommand.class.toString(), ReadyCommand.class.toString(), ShutdownCommand.class.toString(), SetupCommand.class.toString(), CleanupNetworkRulesCmd.class.toString(), CheckNetworkCommand.class.toString(), PvlanSetupCommand.class.toString(), CheckOnHostCommand.class.toString(), - ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(), ModifyStoragePoolCommand.class.toString(), SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString(), - CleanupPersistentNetworkResourceCommand.class.toString()}; + ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(), + CreateStoragePoolCommand.class.toString(), DeleteStoragePoolCommand.class.toString(), ModifyStoragePoolCommand.class.toString(), + SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString(), CleanupPersistentNetworkResourceCommand.class.toString()}; protected final static String[] s_commandsNotAllowedInConnectingMode = new String[] { StartCommand.class.toString(), CreateCommand.class.toString() }; static { Arrays.sort(s_commandsAllowedInMaintenanceMode); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 824b9f5f45d6..d21e8b0fc7b2 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -52,6 +52,7 @@ import com.cloud.configuration.Resource; import com.cloud.domain.Domain; import com.cloud.domain.dao.DomainDao; +import com.cloud.exception.ResourceAllocationException; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcDao; import com.cloud.user.dao.AccountDao; @@ -89,6 +90,7 @@ import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.reservation.dao.ReservationDao; +import org.apache.cloudstack.resource.ResourceCleanupService; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.VolumeObjectTO; @@ -401,6 +403,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private VpcDao vpcDao; @Inject private DomainDao domainDao; + @Inject + ResourceCleanupService resourceCleanupService; VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this); @@ -690,6 +694,7 @@ protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableExcepti if (logger.isDebugEnabled()) { logger.debug("Expunged " + vm); } + resourceCleanupService.purgeExpungedVmResourcesLaterIfNeeded(vm); } private void handleUnsuccessfulExpungeOperation(List finalizeExpungeCommands, List nicExpungeCommands, @@ -1076,6 +1081,10 @@ protected void checkAndAttemptMigrateVmAcrossCluster(final VMInstanceVO vm, fina return; } Host lastHost = _hostDao.findById(vm.getLastHostId()); + if (lastHost == null) { + logger.warn("Could not find last host with id [{}], skipping migrate VM [{}] across cluster check." , vm.getLastHostId(), vm.getUuid()); + return; + } if (destinationClusterId.equals(lastHost.getClusterId())) { return; } @@ -1221,21 +1230,9 @@ public void orchestrateStart(final String vmUuid, final Map 1f || Float.parseFloat(cluster_detail_ram.getValue()) > 1f)) { - userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true); - userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true); - } else if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) != null) { - userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true); - userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true); - } + final Long clusterId = dest.getCluster().getId(); + updateOverCommitRatioForVmProfile(vmProfile, clusterId); - vmProfile.setCpuOvercommitRatio(Float.parseFloat(cluster_detail_cpu.getValue())); - vmProfile.setMemoryOvercommitRatio(Float.parseFloat(cluster_detail_ram.getValue())); StartAnswer startAnswer = null; try { @@ -1250,7 +1247,7 @@ public void orchestrateStart(final String vmUuid, final Map 1f) || + (vmDetailCpu != null && Float.parseFloat(vmDetailCpu.getValue()) != parsedClusterCpuDetailCpu)) { + userVmDetailsDao.addDetail(vmProfile.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, clusterDetailCpu.getValue(), true); + } + if ((vmDetailRam == null && parsedClusterDetailRam > 1f) || + (vmDetailRam != null && Float.parseFloat(vmDetailRam.getValue()) != parsedClusterDetailRam)) { + userVmDetailsDao.addDetail(vmProfile.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, clusterDetailRam.getValue(), true); + } + + vmProfile.setCpuOvercommitRatio(Float.parseFloat(clusterDetailCpu.getValue())); + vmProfile.setMemoryOvercommitRatio(Float.parseFloat(clusterDetailRam.getValue())); + } + /** * Setting pod id to null can result in migration of Volumes across pods. This is not desirable for VMs which * have a volume in Ready state (happens when a VM is shutdown and started again). @@ -1992,20 +2010,24 @@ private void orchestrateStop(final String vmUuid, final boolean cleanUpEvenIfUna } private void updatePersistenceMap(Map vlanToPersistenceMap, NetworkVO networkVO) { + if (networkVO == null) { + return; + } NetworkOfferingVO offeringVO = networkOfferingDao.findById(networkVO.getNetworkOfferingId()); - if (offeringVO != null) { - Pair data = getVMNetworkDetails(networkVO, offeringVO.isPersistent()); - Boolean shouldDeleteNwResource = (MapUtils.isNotEmpty(vlanToPersistenceMap) && data != null) ? vlanToPersistenceMap.get(data.first()) : null; - if (data != null && (shouldDeleteNwResource == null || shouldDeleteNwResource)) { - vlanToPersistenceMap.put(data.first(), data.second()); - } + if (offeringVO == null) { + return; + } + Pair data = getVMNetworkDetails(networkVO, offeringVO.isPersistent()); + Boolean shouldDeleteNwResource = (MapUtils.isNotEmpty(vlanToPersistenceMap) && data != null) ? vlanToPersistenceMap.get(data.first()) : null; + if (data != null && (shouldDeleteNwResource == null || shouldDeleteNwResource)) { + vlanToPersistenceMap.put(data.first(), data.second()); } } private Map getVlanToPersistenceMapForVM(long vmId) { List userVmJoinVOs = userVmJoinDao.searchByIds(vmId); Map vlanToPersistenceMap = new HashMap<>(); - if (userVmJoinVOs != null && !userVmJoinVOs.isEmpty()) { + if (CollectionUtils.isNotEmpty(userVmJoinVOs)) { for (UserVmJoinVO userVmJoinVO : userVmJoinVOs) { NetworkVO networkVO = _networkDao.findById(userVmJoinVO.getNetworkId()); updatePersistenceMap(vlanToPersistenceMap, networkVO); @@ -2719,6 +2741,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy _networkMgr.prepareNicForMigration(profile, dest); volumeMgr.prepareForMigration(profile, dest); profile.setConfigDriveLabel(VmConfigDriveLabel.value()); + updateOverCommitRatioForVmProfile(profile, dest.getHost().getClusterId()); final VirtualMachineTO to = toVmTO(profile); final PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(to); @@ -3006,7 +3029,7 @@ protected void createStoragePoolMappingsForVolumes(VirtualMachineProfile profile executeManagedStorageChecksWhenTargetStoragePoolNotProvided(targetHost, currentPool, volume); if (ScopeType.HOST.equals(currentPool.getScope()) || isStorageCrossClusterMigration(plan.getClusterId(), currentPool)) { createVolumeToStoragePoolMappingIfPossible(profile, plan, volumeToPoolObjectMap, volume, currentPool); - } else if (shouldMapVolume(profile, volume, currentPool)){ + } else if (shouldMapVolume(profile, currentPool)){ volumeToPoolObjectMap.put(volume, currentPool); } } @@ -3018,11 +3041,10 @@ protected void createStoragePoolMappingsForVolumes(VirtualMachineProfile profile * Some context: VMware migration workflow requires all volumes to be mapped (even if volume stays on its current pool); * however, this is not necessary/desirable for the KVM flow. */ - protected boolean shouldMapVolume(VirtualMachineProfile profile, Volume volume, StoragePoolVO currentPool) { + protected boolean shouldMapVolume(VirtualMachineProfile profile, StoragePoolVO currentPool) { boolean isManaged = currentPool.isManaged(); boolean isNotKvm = HypervisorType.KVM != profile.getHypervisorType(); - boolean isNotDatadisk = Type.DATADISK != volume.getVolumeType(); - return isNotKvm || isNotDatadisk || isManaged; + return isNotKvm || isManaged; } /** @@ -4769,6 +4791,18 @@ protected void HandlePowerStateReport(final String subject, final String senderA } } + private ApiCommandResourceType getApiCommandResourceTypeForVm(VirtualMachine vm) { + switch (vm.getType()) { + case DomainRouter: + return ApiCommandResourceType.DomainRouter; + case ConsoleProxy: + return ApiCommandResourceType.ConsoleProxy; + case SecondaryStorageVm: + return ApiCommandResourceType.SystemVm; + } + return ApiCommandResourceType.VirtualMachine; + } + private void handlePowerOnReportWithNoPendingJobsOnVM(final VMInstanceVO vm) { Host host = _hostDao.findById(vm.getHostId()); Host poweredHost = _hostDao.findById(vm.getPowerHostId()); @@ -4816,7 +4850,7 @@ private void handlePowerOnReportWithNoPendingJobsOnVM(final VMInstanceVO vm) { + " -> Running) from out-of-context transition. VM network environment may need to be reset"); ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, vm.getDomainId(), - EventTypes.EVENT_VM_START, "Out of band VM power on", vm.getId(), ApiCommandResourceType.VirtualMachine.toString()); + EventTypes.EVENT_VM_START, "Out of band VM power on", vm.getId(), getApiCommandResourceTypeForVm(vm).toString()); logger.info("VM {} is sync-ed to at Running state according to power-on report from hypervisor.", vm.getInstanceName()); break; @@ -4849,7 +4883,7 @@ private void handlePowerOffReportWithNoPendingJobsOnVM(final VMInstanceVO vm) { case Running: case Stopped: ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM,vm.getDomainId(), - EventTypes.EVENT_VM_STOP, "Out of band VM power off", vm.getId(), ApiCommandResourceType.VirtualMachine.toString()); + EventTypes.EVENT_VM_STOP, "Out of band VM power off", vm.getId(), getApiCommandResourceTypeForVm(vm).toString()); case Migrating: logger.info("VM {} is at {} and we received a {} report while there is no pending jobs on it" , vm.getInstanceName(), vm.getState(), vm.getPowerState()); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index d07fee322765..ea34f62ecd58 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -259,6 +259,8 @@ import com.googlecode.ipv6.IPv6Address; import org.jetbrains.annotations.NotNull; +import static com.cloud.configuration.ConfigurationManager.MESSAGE_DELETE_VLAN_IP_RANGE_EVENT; + /** * NetworkManagerImpl implements NetworkManager. */ @@ -763,6 +765,14 @@ public List setupNetwork(final Account owner, final NetworkOf continue; } + // Ensure cidr size is equal to 64 for + // - networks other than shared networks + // - shared networks with SLAAC V6 only + if (predefined != null && StringUtils.isNotBlank(predefined.getIp6Cidr()) && + (!GuestType.Shared.equals(offering.getGuestType()) || guru.isSlaacV6Only())) { + _networkModel.checkIp6CidrSizeEqualTo64(predefined.getIp6Cidr()); + } + if (network.getId() != -1) { if (network instanceof NetworkVO) { networks.add((NetworkVO) network); @@ -1031,48 +1041,84 @@ public void saveExtraDhcpOptions(final String networkUuid, final Long nicId, fin } } - @DB - @Override - public Pair allocateNic(final NicProfile requested, final Network network, final Boolean isDefaultNic, int deviceId, final VirtualMachineProfile vm) - throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException { + private NicVO persistNicAfterRaceCheck(final NicVO nic, final Long networkId, final NicProfile profile, int deviceId) { + return Transaction.execute(new TransactionCallback() { + @Override + public NicVO doInTransaction(TransactionStatus status) { + NicVO vo = _nicDao.findByIp4AddressAndNetworkId(profile.getIPv4Address(), networkId); + if (vo == null) { + applyProfileToNic(nic, profile, deviceId); + vo = _nicDao.persist(nic); + return vo; + } else { + return null; + } + } + }); + } + private NicVO checkForRaceAndAllocateNic(final NicProfile requested, final Network network, final Boolean isDefaultNic, int deviceId, final VirtualMachineProfile vm) + throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException { final NetworkVO ntwkVO = _networksDao.findById(network.getId()); logger.debug("Allocating nic for vm {} in network {} with requested profile {}", vm.getVirtualMachine(), network, requested); final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, ntwkVO.getGuruName()); - if (requested != null && requested.getMode() == null) { - requested.setMode(network.getMode()); - } - final NicProfile profile = guru.allocate(network, requested, vm); - if (profile == null) { - return null; - } + NicVO vo = null; + boolean retryIpAllocation; + do { + retryIpAllocation = false; + final NicProfile profile = guru.allocate(network, requested, vm); + if (profile == null) { + return null; + } - if (isNicAllocatedForNsxPublicNetworkOnVR(network, profile, vm)) { - String guruName = "NsxPublicNetworkGuru"; - NetworkGuru nsxGuru = AdapterBase.getAdapterByName(networkGurus, guruName); - nsxGuru.allocate(network, profile, vm); - } + if (isDefaultNic != null) { + profile.setDefaultNic(isDefaultNic); + } - if (isDefaultNic != null) { - profile.setDefaultNic(isDefaultNic); - } + if (requested != null && requested.getMode() == null) { + profile.setMode(requested.getMode()); + } else { + profile.setMode(network.getMode()); + } - if (requested != null && requested.getMode() == null) { - profile.setMode(requested.getMode()); - } else { - profile.setMode(network.getMode()); - } + vo = new NicVO(guru.getName(), vm.getId(), network.getId(), vm.getType()); + + DataCenterVO dcVo = _dcDao.findById(network.getDataCenterId()); + if (dcVo.getNetworkType() == NetworkType.Basic) { + configureNicProfileBasedOnRequestedIp(requested, profile, network); + } + + if (profile.getIpv4AllocationRaceCheck()) { + vo = persistNicAfterRaceCheck(vo, network.getId(), profile, deviceId); + } else { + applyProfileToNic(vo, profile, deviceId); + vo = _nicDao.persist(vo); + } + + if (vo == null) { + if (requested.getRequestedIPv4() != null) { + throw new InsufficientVirtualNetworkCapacityException("Unable to acquire requested Guest IP address " + requested.getRequestedIPv4() + " for network " + network, DataCenter.class, dcVo.getId()); + } else { + requested.setIPv4Address(null); + } + retryIpAllocation = true; + } + } while (retryIpAllocation); + + return vo; + } - NicVO vo = new NicVO(guru.getName(), vm.getId(), network.getId(), vm.getType()); + @DB + @Override + public Pair allocateNic(final NicProfile requested, final Network network, final Boolean isDefaultNic, int deviceId, final VirtualMachineProfile vm) + throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException { - DataCenterVO dcVo = _dcDao.findById(network.getDataCenterId()); - if (dcVo.getNetworkType() == NetworkType.Basic) { - configureNicProfileBasedOnRequestedIp(requested, profile, network); + if (requested != null && requested.getMode() == null) { + requested.setMode(network.getMode()); } - deviceId = applyProfileToNic(vo, profile, deviceId); - vo = _nicDao.persist(vo); + NicVO vo = checkForRaceAndAllocateNic(requested, network, isDefaultNic, deviceId, vm); final Integer networkRate = _networkModel.getNetworkRate(network.getId(), vm.getId()); final NicProfile vmNic = new NicProfile(vo, network, vo.getBroadcastUri(), vo.getIsolationUri(), networkRate, _networkModel.isSecurityGroupSupportedInNetwork(network), @@ -2721,8 +2767,8 @@ private Network createGuestNetwork(final long networkOfferingId, final String na } } - if (ipv6 && NetUtils.getIp6CidrSize(ip6Cidr) != 64) { - throw new InvalidParameterValueException("IPv6 subnet should be exactly 64-bits in size"); + if (ipv6 && !GuestType.Shared.equals(ntwkOff.getGuestType())) { + _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr); } //TODO(VXLAN): Support VNI specified @@ -3062,17 +3108,7 @@ protected void checkL2OfferingServices(NetworkOfferingVO ntwkOff) { @Override @DB public boolean shutdownNetwork(final long networkId, final ReservationContext context, final boolean cleanupElements) { - NetworkVO network = _networksDao.findById(networkId); - if (network.getState() == Network.State.Allocated) { - logger.debug("Network is already shutdown: {}", network); - return true; - } - - if (network.getState() != Network.State.Implemented && network.getState() != Network.State.Shutdown) { - logger.debug("Network is not implemented: {}", network); - return false; - } - + NetworkVO network = null; try { //do global lock for the network network = _networksDao.acquireInLockTable(networkId, NetworkLockTimeout.value()); @@ -3324,17 +3360,17 @@ public boolean destroyNetwork(final long networkId, final ReservationContext con final NetworkVO networkFinal = network; try { - Transaction.execute(new TransactionCallbackNoReturn() { + final List deletedVlanRangeToPublish = Transaction.execute(new TransactionCallback>() { @Override - public void doInTransactionWithoutResult(final TransactionStatus status) { + public List doInTransaction(TransactionStatus status) { final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, networkFinal.getGuruName()); if (!guru.trash(networkFinal, _networkOfferingDao.findById(networkFinal.getNetworkOfferingId()))) { throw new CloudRuntimeException("Failed to trash network."); } - - if (!deleteVlansInNetwork(networkFinal, context.getCaller().getId(), callerAccount)) { - logger.warn("Failed to delete network {}; was unable to cleanup corresponding ip ranges", networkFinal); + Pair> deletedVlans = deleteVlansInNetwork(networkFinal, context.getCaller().getId(), callerAccount); + if (!deletedVlans.first()) { + logger.warn("Failed to delete network " + networkFinal + "; was unable to cleanup corresponding ip ranges"); throw new CloudRuntimeException("Failed to delete network " + networkFinal + "; was unable to cleanup corresponding ip ranges"); } else { // commit transaction only when ips and vlans for the network are released successfully @@ -3367,8 +3403,10 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { _resourceLimitMgr.decrementResourceCount(networkFinal.getAccountId(), ResourceType.network, networkFinal.getDisplayNetwork()); } } + return deletedVlans.second(); } }); + publishDeletedVlanRanges(deletedVlanRangeToPublish); if (_networksDao.findById(network.getId()) == null) { // remove its related ACL permission final Pair, Long> networkMsg = new Pair, Long>(Network.class, networkFinal.getId()); @@ -3386,6 +3424,14 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { return success; } + private void publishDeletedVlanRanges(List deletedVlanRangeToPublish) { + if (CollectionUtils.isNotEmpty(deletedVlanRangeToPublish)) { + for (VlanVO vlan : deletedVlanRangeToPublish) { + _messageBus.publish(_name, MESSAGE_DELETE_VLAN_IP_RANGE_EVENT, PublishScope.LOCAL, vlan); + } + } + } + @Override public boolean resourceCountNeedsUpdate(final NetworkOffering ntwkOff, final ACLType aclType) { //Update resource count only for Isolated account specific non-system networks @@ -3393,15 +3439,19 @@ public boolean resourceCountNeedsUpdate(final NetworkOffering ntwkOff, final ACL return updateResourceCount; } - protected boolean deleteVlansInNetwork(final NetworkVO network, final long userId, final Account callerAccount) { + protected Pair> deleteVlansInNetwork(final NetworkVO network, final long userId, final Account callerAccount) { final long networkId = network.getId(); //cleanup Public vlans final List publicVlans = _vlanDao.listVlansByNetworkId(networkId); + List deletedPublicVlanRange = new ArrayList<>(); boolean result = true; for (final VlanVO vlan : publicVlans) { - if (!_configMgr.deleteVlanAndPublicIpRange(userId, vlan.getId(), callerAccount)) { - logger.warn("Failed to delete vlan {});", vlan.getId()); + VlanVO vlanRange = _configMgr.deleteVlanAndPublicIpRange(userId, vlan.getId(), callerAccount); + if (vlanRange == null) { + logger.warn("Failed to delete vlan " + vlan.getId() + ");"); result = false; + } else { + deletedPublicVlanRange.add(vlanRange); } } @@ -3421,7 +3471,7 @@ protected boolean deleteVlansInNetwork(final NetworkVO network, final long userI _dcDao.releaseVnet(BroadcastDomainType.getValue(network.getBroadcastUri()), network.getDataCenterId(), network.getPhysicalNetworkId(), network.getAccountId(), network.getReservationId()); } - return result; + return new Pair<>(result, deletedPublicVlanRange); } public class NetworkGarbageCollector extends ManagedContextRunnable { @@ -4599,10 +4649,16 @@ public Pair importNic(final String macAddress, int deviceId final NicVO vo = Transaction.execute(new TransactionCallback() { @Override public NicVO doInTransaction(TransactionStatus status) { - NicVO existingNic = _nicDao.findByNetworkIdAndMacAddress(network.getId(), macAddress); - String macAddressToPersist = macAddress; + if (StringUtils.isBlank(macAddress)) { + throw new CloudRuntimeException("Mac address not specified"); + } + String macAddressToPersist = macAddress.trim(); + if (!NetUtils.isValidMac(macAddressToPersist)) { + throw new CloudRuntimeException("Invalid mac address: " + macAddressToPersist); + } + NicVO existingNic = _nicDao.findByNetworkIdAndMacAddress(network.getId(), macAddressToPersist); if (existingNic != null) { - macAddressToPersist = generateNewMacAddressIfForced(network, macAddress, forced); + macAddressToPersist = generateNewMacAddressIfForced(network, macAddressToPersist, forced); } NicVO vo = new NicVO(network.getGuruName(), vm.getId(), network.getId(), vm.getType()); vo.setMacAddress(macAddressToPersist); @@ -4647,7 +4703,7 @@ public NicVO doInTransaction(TransactionStatus status) { final NicProfile vmNic = new NicProfile(vo, network, vo.getBroadcastUri(), vo.getIsolationUri(), networkRate, _networkModel.isSecurityGroupSupportedInNetwork(network), _networkModel.getNetworkTag(vm.getHypervisorType(), network)); - return new Pair(vmNic, Integer.valueOf(deviceId)); + return new Pair<>(vmNic, Integer.valueOf(deviceId)); } protected String getSelectedIpForNicImport(Network network, DataCenter dataCenter, Network.IpAddresses ipAddresses) { @@ -4691,7 +4747,7 @@ protected Pair getNetworkGatewayAndNetmaskForNicImport(Network n private String generateNewMacAddressIfForced(Network network, String macAddress, boolean forced) { if (!forced) { - throw new CloudRuntimeException("NIC with MAC address = " + macAddress + " exists on network with ID = " + network.getId() + + throw new CloudRuntimeException("NIC with MAC address " + macAddress + " exists on network with ID " + network.getUuid() + " and forced flag is disabled"); } try { @@ -4722,6 +4778,19 @@ public void unmanageNics(VirtualMachineProfile vm) { } } + @Override + public void expungeLbVmRefs(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(networkElements) || CollectionUtils.isEmpty(vmIds)) { + return; + } + for (NetworkElement element : networkElements) { + if (element instanceof LoadBalancingServiceProvider) { + LoadBalancingServiceProvider lbProvider = (LoadBalancingServiceProvider)element; + lbProvider.expungeLbVmRefs(vmIds, batchSize); + } + } + } + @Override public String getConfigComponentName() { return NetworkOrchestrationService.class.getSimpleName(); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index ff9da6bccc2a..36e281459492 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -38,6 +38,11 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.user.AccountManager; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd; @@ -180,6 +185,8 @@ public enum UserVmCloneType { } + @Inject + private AccountManager _accountMgr; @Inject EntityManager _entityMgr; @Inject @@ -195,6 +202,8 @@ public enum UserVmCloneType { @Inject protected VolumeDao _volumeDao; @Inject + protected VMTemplateDao _templateDao; + @Inject protected SnapshotDao _snapshotDao; @Inject protected SnapshotDataStoreDao _snapshotDataStoreDao; @@ -1176,8 +1185,9 @@ public VolumeVO doInTransaction(TransactionStatus status) { logger.error("Unable to destroy existing volume [{}] due to [{}].", volumeToString, e.getMessage()); } // In case of VMware VM will continue to use the old root disk until expunged, so force expunge old root disk - if (vm.getHypervisorType() == HypervisorType.VMware) { - logger.info("Trying to expunge volume [{}] from primary data storage.", volumeToString); + // For system VM we do not need volume entry in Destroy state + if (vm.getHypervisorType() == HypervisorType.VMware || vm.getType().isUsedBySystem()) { + logger.info(String.format("Trying to expunge volume [%s] from primary data storage.", volumeToString)); AsyncCallFuture future = volService.expungeVolumeAsync(volFactory.getVolume(existingVolume.getId())); try { future.get(); @@ -1483,18 +1493,17 @@ public void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest for (VolumeVO vol : vols) { VolumeInfo volumeInfo = volFactory.getVolume(vol.getId()); - DataTO volTO = volumeInfo.getTO(); - DiskTO disk = storageMgr.getDiskWithThrottling(volTO, vol.getVolumeType(), vol.getDeviceId(), vol.getPath(), vm.getServiceOfferingId(), vol.getDiskOfferingId()); DataStore dataStore = dataStoreMgr.getDataStore(vol.getPoolId(), DataStoreRole.Primary); - disk.setDetails(getDetails(volumeInfo, dataStore)); - PrimaryDataStore primaryDataStore = (PrimaryDataStore)dataStore; // This might impact other managed storages, enable requires access for migration in relevant datastore driver (currently enabled for PowerFlex storage pool only) if (primaryDataStore.isManaged() && volService.requiresAccessForMigration(volumeInfo, dataStore)) { volService.grantAccess(volFactory.getVolume(vol.getId()), dest.getHost(), dataStore); } - + // make sure this is done AFTER grantAccess, as grantAccess may change the volume's state + DataTO volTO = volumeInfo.getTO(); + DiskTO disk = storageMgr.getDiskWithThrottling(volTO, vol.getVolumeType(), vol.getDeviceId(), vol.getPath(), vm.getServiceOfferingId(), vol.getDiskOfferingId()); + disk.setDetails(getDetails(volumeInfo, dataStore)); vm.addDisk(disk); } @@ -1677,7 +1686,7 @@ protected void checkAndUpdateVolumeAccountResourceCount(VolumeVO originalEntry, } } - private Pair recreateVolume(VolumeVO vol, VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, StorageAccessException { + private Pair recreateVolume(VolumeVO vol, VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, StorageAccessException, ResourceAllocationException { String volToString = getReflectOnlySelectedFields(vol); VolumeVO newVol; @@ -1710,6 +1719,7 @@ private Pair recreateVolume(VolumeVO vol, VirtualMachinePro } logger.debug("Created new volume [{}] from old volume [{}].", newVolToString, volToString); } + updateVolumeSize(destPool, newVol); VolumeInfo volume = volFactory.getVolume(newVol.getId(), destPool); Long templateId = newVol.getTemplateId(); for (int i = 0; i < 2; i++) { @@ -1841,8 +1851,39 @@ protected void grantVolumeAccessToHostIfNeeded(PrimaryDataStore volumeStore, lon } } + /** + * This method checks if size of volume on the data store would be different. + * If it's different it verifies the resource limits and updates the volume's size + */ + protected void updateVolumeSize(DataStore store, VolumeVO vol) throws ResourceAllocationException { + if (store == null || !(store.getDriver() instanceof PrimaryDataStoreDriver)) { + return; + } + + VMTemplateVO template = vol.getTemplateId() != null ? _templateDao.findById(vol.getTemplateId()) : null; + PrimaryDataStoreDriver driver = (PrimaryDataStoreDriver) store.getDriver(); + long newSize = driver.getVolumeSizeRequiredOnPool(vol.getSize(), + template == null ? null : template.getSize(), + vol.getPassphraseId() != null); + + if (newSize != vol.getSize()) { + DiskOfferingVO diskOffering = diskOfferingDao.findByIdIncludingRemoved(vol.getDiskOfferingId()); + if (newSize > vol.getSize()) { + _resourceLimitMgr.checkPrimaryStorageResourceLimit(_accountMgr.getActiveAccountById(vol.getAccountId()), + vol.isDisplay(), newSize - vol.getSize(), diskOffering); + _resourceLimitMgr.incrementVolumePrimaryStorageResourceCount(vol.getAccountId(), vol.isDisplay(), + newSize - vol.getSize(), diskOffering); + } else { + _resourceLimitMgr.decrementVolumePrimaryStorageResourceCount(vol.getAccountId(), vol.isDisplay(), + vol.getSize() - newSize, diskOffering); + } + vol.setSize(newSize); + _volsDao.persist(vol); + } + } + @Override - public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException, StorageAccessException { + public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException, StorageAccessException, ResourceAllocationException { if (dest == null) { String msg = String.format("Unable to prepare volumes for the VM [%s] because DeployDestination is null.", vm.getVirtualMachine()); logger.error(msg); @@ -1865,7 +1906,7 @@ public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws Sto String volToString = getReflectOnlySelectedFields(vol); - store = (PrimaryDataStore)dataStoreMgr.getDataStore(task.pool.getId(), DataStoreRole.Primary); + store = (PrimaryDataStore) dataStoreMgr.getDataStore(task.pool.getId(), DataStoreRole.Primary); // For zone-wide managed storage, it is possible that the VM can be started in another // cluster. In that case, make sure that the volume is in the right access group. @@ -1876,6 +1917,8 @@ public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws Sto long lastClusterId = lastHost == null || lastHost.getClusterId() == null ? -1 : lastHost.getClusterId(); long clusterId = host == null || host.getClusterId() == null ? -1 : host.getClusterId(); + updateVolumeSize(store, (VolumeVO) vol); + if (lastClusterId != clusterId) { if (lastHost != null) { storageMgr.removeStoragePoolFromCluster(lastHost.getId(), vol.get_iScsiName(), store); @@ -1895,6 +1938,7 @@ public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws Sto } } else if (task.type == VolumeTaskType.MIGRATE) { store = (PrimaryDataStore) dataStoreMgr.getDataStore(task.pool.getId(), DataStoreRole.Primary); + updateVolumeSize(store, task.volume); vol = migrateVolume(task.volume, store); } else if (task.type == VolumeTaskType.RECREATE) { Pair result = recreateVolume(task.volume, vm, dest); diff --git a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java index d1532cdbef14..58746a9a6cf2 100644 --- a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java +++ b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.engine.orchestration; +import static org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService.NetworkLockTimeout; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -30,6 +32,7 @@ import java.util.Map; import com.cloud.dc.DataCenter; +import com.cloud.exception.InsufficientVirtualNetworkCapacityException; import com.cloud.network.IpAddressManager; import com.cloud.utils.Pair; import org.junit.Assert; @@ -38,6 +41,7 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import org.mockito.ArgumentMatchers; +import org.mockito.MockedStatic; import org.mockito.Mockito; import com.cloud.api.query.dao.DomainRouterJoinDao; @@ -69,6 +73,9 @@ import com.cloud.network.vpc.VpcManager; import com.cloud.network.vpc.VpcVO; import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.utils.db.EntityManager; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Ip; import com.cloud.vm.DomainRouterVO; @@ -93,7 +100,7 @@ @RunWith(JUnit4.class) public class NetworkOrchestratorTest extends TestCase { - NetworkOrchestrator testOrchastrator = Mockito.spy(new NetworkOrchestrator()); + NetworkOrchestrator testOrchestrator = Mockito.spy(new NetworkOrchestrator()); private String guruName = "GuestNetworkGuru"; private String dhcpProvider = "VirtualRouter"; @@ -112,21 +119,22 @@ public class NetworkOrchestratorTest extends TestCase { @Before public void setUp() { // make class-scope mocks - testOrchastrator._nicDao = mock(NicDao.class); - testOrchastrator._networksDao = mock(NetworkDao.class); - testOrchastrator._networkModel = mock(NetworkModel.class); - testOrchastrator._nicSecondaryIpDao = mock(NicSecondaryIpDao.class); - testOrchastrator._ntwkSrvcDao = mock(NetworkServiceMapDao.class); - testOrchastrator._nicIpAliasDao = mock(NicIpAliasDao.class); - testOrchastrator._ipAddressDao = mock(IPAddressDao.class); - testOrchastrator._vlanDao = mock(VlanDao.class); - testOrchastrator._networkModel = mock(NetworkModel.class); - testOrchastrator._nicExtraDhcpOptionDao = mock(NicExtraDhcpOptionDao.class); - testOrchastrator.routerDao = mock(DomainRouterDao.class); - testOrchastrator.routerNetworkDao = mock(RouterNetworkDao.class); - testOrchastrator._vpcMgr = mock(VpcManager.class); - testOrchastrator.routerJoinDao = mock(DomainRouterJoinDao.class); - testOrchastrator._ipAddrMgr = mock(IpAddressManager.class); + testOrchestrator._nicDao = mock(NicDao.class); + testOrchestrator._networksDao = mock(NetworkDao.class); + testOrchestrator._networkModel = mock(NetworkModel.class); + testOrchestrator._nicSecondaryIpDao = mock(NicSecondaryIpDao.class); + testOrchestrator._ntwkSrvcDao = mock(NetworkServiceMapDao.class); + testOrchestrator._nicIpAliasDao = mock(NicIpAliasDao.class); + testOrchestrator._ipAddressDao = mock(IPAddressDao.class); + testOrchestrator._vlanDao = mock(VlanDao.class); + testOrchestrator._networkModel = mock(NetworkModel.class); + testOrchestrator._nicExtraDhcpOptionDao = mock(NicExtraDhcpOptionDao.class); + testOrchestrator.routerDao = mock(DomainRouterDao.class); + testOrchestrator.routerNetworkDao = mock(RouterNetworkDao.class); + testOrchestrator._vpcMgr = mock(VpcManager.class); + testOrchestrator.routerJoinDao = mock(DomainRouterJoinDao.class); + testOrchestrator._ipAddrMgr = mock(IpAddressManager.class); + testOrchestrator._entityMgr = mock(EntityManager.class); DhcpServiceProvider provider = mock(DhcpServiceProvider.class); Map capabilities = new HashMap(); @@ -135,13 +143,13 @@ public void setUp() { when(provider.getCapabilities()).thenReturn(services); capabilities.put(Network.Capability.DhcpAccrossMultipleSubnets, "true"); - when(testOrchastrator._ntwkSrvcDao.getProviderForServiceInNetwork(ArgumentMatchers.anyLong(), ArgumentMatchers.eq(Service.Dhcp))).thenReturn(dhcpProvider); - when(testOrchastrator._networkModel.getElementImplementingProvider(dhcpProvider)).thenReturn(provider); + when(testOrchestrator._ntwkSrvcDao.getProviderForServiceInNetwork(ArgumentMatchers.anyLong(), ArgumentMatchers.eq(Service.Dhcp))).thenReturn(dhcpProvider); + when(testOrchestrator._networkModel.getElementImplementingProvider(dhcpProvider)).thenReturn(provider); when(guru.getName()).thenReturn(guruName); List networkGurus = new ArrayList(); networkGurus.add(guru); - testOrchastrator.networkGurus = networkGurus; + testOrchestrator.networkGurus = networkGurus; when(networkOffering.getGuestType()).thenReturn(GuestType.L2); when(networkOffering.getId()).thenReturn(networkOfferingId); @@ -156,21 +164,21 @@ public void testRemoveDhcpServiceWithNic() { // make sure that release dhcp will be called when(vm.getType()).thenReturn(Type.User); - when(testOrchastrator._networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)).thenReturn(true); + when(testOrchestrator._networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)).thenReturn(true); when(network.getTrafficType()).thenReturn(TrafficType.Guest); when(network.getGuestType()).thenReturn(GuestType.Shared); - when(testOrchastrator._nicDao.listByNetworkIdTypeAndGatewayAndBroadcastUri(nic.getNetworkId(), VirtualMachine.Type.User, nic.getIPv4Gateway(), nic.getBroadcastUri())) + when(testOrchestrator._nicDao.listByNetworkIdTypeAndGatewayAndBroadcastUri(nic.getNetworkId(), VirtualMachine.Type.User, nic.getIPv4Gateway(), nic.getBroadcastUri())) .thenReturn(new ArrayList()); when(network.getGuruName()).thenReturn(guruName); - when(testOrchastrator._networksDao.findById(nic.getNetworkId())).thenReturn(network); + when(testOrchestrator._networksDao.findById(nic.getNetworkId())).thenReturn(network); - testOrchastrator.removeNic(vm, nic); + testOrchestrator.removeNic(vm, nic); verify(nic, times(1)).setState(Nic.State.Deallocating); - verify(testOrchastrator._networkModel, times(2)).getElementImplementingProvider(dhcpProvider); - verify(testOrchastrator._ntwkSrvcDao, times(2)).getProviderForServiceInNetwork(network.getId(), Service.Dhcp); - verify(testOrchastrator._networksDao, times(2)).findById(nic.getNetworkId()); + verify(testOrchestrator._networkModel, times(2)).getElementImplementingProvider(dhcpProvider); + verify(testOrchestrator._ntwkSrvcDao, times(2)).getProviderForServiceInNetwork(network.getId(), Service.Dhcp); + verify(testOrchestrator._networksDao, times(2)).findById(nic.getNetworkId()); } @Test public void testDontRemoveDhcpServiceFromDomainRouter() { @@ -183,14 +191,14 @@ public void testDontRemoveDhcpServiceFromDomainRouter() { when(vm.getType()).thenReturn(Type.DomainRouter); when(network.getGuruName()).thenReturn(guruName); - when(testOrchastrator._networksDao.findById(nic.getNetworkId())).thenReturn(network); + when(testOrchestrator._networksDao.findById(nic.getNetworkId())).thenReturn(network); - testOrchastrator.removeNic(vm, nic); + testOrchestrator.removeNic(vm, nic); verify(nic, times(1)).setState(Nic.State.Deallocating); - verify(testOrchastrator._networkModel, never()).getElementImplementingProvider(dhcpProvider); - verify(testOrchastrator._ntwkSrvcDao, never()).getProviderForServiceInNetwork(network.getId(), Service.Dhcp); - verify(testOrchastrator._networksDao, times(1)).findById(nic.getNetworkId()); + verify(testOrchestrator._networkModel, never()).getElementImplementingProvider(dhcpProvider); + verify(testOrchestrator._ntwkSrvcDao, never()).getProviderForServiceInNetwork(network.getId(), Service.Dhcp); + verify(testOrchestrator._networksDao, times(1)).findById(nic.getNetworkId()); } @Test public void testDontRemoveDhcpServiceWhenNotProvided() { @@ -201,45 +209,45 @@ public void testDontRemoveDhcpServiceWhenNotProvided() { // make sure that release dhcp will *not* be called when(vm.getType()).thenReturn(Type.User); - when(testOrchastrator._networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)).thenReturn(false); + when(testOrchestrator._networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)).thenReturn(false); when(network.getGuruName()).thenReturn(guruName); - when(testOrchastrator._networksDao.findById(nic.getNetworkId())).thenReturn(network); + when(testOrchestrator._networksDao.findById(nic.getNetworkId())).thenReturn(network); - testOrchastrator.removeNic(vm, nic); + testOrchestrator.removeNic(vm, nic); verify(nic, times(1)).setState(Nic.State.Deallocating); - verify(testOrchastrator._networkModel, never()).getElementImplementingProvider(dhcpProvider); - verify(testOrchastrator._ntwkSrvcDao, never()).getProviderForServiceInNetwork(network.getId(), Service.Dhcp); - verify(testOrchastrator._networksDao, times(1)).findById(nic.getNetworkId()); + verify(testOrchestrator._networkModel, never()).getElementImplementingProvider(dhcpProvider); + verify(testOrchestrator._ntwkSrvcDao, never()).getProviderForServiceInNetwork(network.getId(), Service.Dhcp); + verify(testOrchestrator._networksDao, times(1)).findById(nic.getNetworkId()); } @Test public void testCheckL2OfferingServicesEmptyServices() { - when(testOrchastrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(new ArrayList<>()); - when(testOrchastrator._networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.UserData)).thenReturn(false); - testOrchastrator.checkL2OfferingServices(networkOffering); + when(testOrchestrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(new ArrayList<>()); + when(testOrchestrator._networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.UserData)).thenReturn(false); + testOrchestrator.checkL2OfferingServices(networkOffering); } @Test public void testCheckL2OfferingServicesUserDataOnly() { - when(testOrchastrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(Arrays.asList(Service.UserData)); - when(testOrchastrator._networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.UserData)).thenReturn(true); - testOrchastrator.checkL2OfferingServices(networkOffering); + when(testOrchestrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(Arrays.asList(Service.UserData)); + when(testOrchestrator._networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.UserData)).thenReturn(true); + testOrchestrator.checkL2OfferingServices(networkOffering); } @Test(expected = InvalidParameterValueException.class) public void testCheckL2OfferingServicesMultipleServicesIncludingUserData() { - when(testOrchastrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(Arrays.asList(Service.UserData, Service.Dhcp)); - when(testOrchastrator._networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.UserData)).thenReturn(true); - testOrchastrator.checkL2OfferingServices(networkOffering); + when(testOrchestrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(Arrays.asList(Service.UserData, Service.Dhcp)); + when(testOrchestrator._networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.UserData)).thenReturn(true); + testOrchestrator.checkL2OfferingServices(networkOffering); } @Test(expected = InvalidParameterValueException.class) public void testCheckL2OfferingServicesMultipleServicesNotIncludingUserData() { - when(testOrchastrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(Arrays.asList(Service.Dns, Service.Dhcp)); - when(testOrchastrator._networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.UserData)).thenReturn(false); - testOrchastrator.checkL2OfferingServices(networkOffering); + when(testOrchestrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(Arrays.asList(Service.Dns, Service.Dhcp)); + when(testOrchestrator._networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.UserData)).thenReturn(false); + testOrchestrator.checkL2OfferingServices(networkOffering); } @Test @@ -251,7 +259,7 @@ public void testConfigureNicProfileBasedOnRequestedIpTestMacNull() { configureTestConfigureNicProfileBasedOnRequestedIpTests(nicProfile, 0l, false, IPAddressVO.State.Free, "192.168.100.1", "255.255.255.0", "00-88-14-4D-4C-FB", requestedNicProfile, null, "192.168.100.150"); - testOrchastrator.configureNicProfileBasedOnRequestedIp(requestedNicProfile, nicProfile, network); + testOrchestrator.configureNicProfileBasedOnRequestedIp(requestedNicProfile, nicProfile, network); verifyAndAssert("192.168.100.150", "192.168.100.1", "255.255.255.0", nicProfile, 1, 1); } @@ -265,7 +273,7 @@ public void testConfigureNicProfileBasedOnRequestedIpTestNicProfileMacNotNull() configureTestConfigureNicProfileBasedOnRequestedIpTests(nicProfile, 0l, false, IPAddressVO.State.Free, "192.168.100.1", "255.255.255.0", "00-88-14-4D-4C-FB", requestedNicProfile, "00-88-14-4D-4C-FB", "192.168.100.150"); - testOrchastrator.configureNicProfileBasedOnRequestedIp(requestedNicProfile, nicProfile, network); + testOrchestrator.configureNicProfileBasedOnRequestedIp(requestedNicProfile, nicProfile, network); verifyAndAssert("192.168.100.150", "192.168.100.1", "255.255.255.0", nicProfile, 1, 0); } @@ -292,7 +300,7 @@ private void testConfigureNicProfileBasedOnRequestedIpTestRequestedIp(String req configureTestConfigureNicProfileBasedOnRequestedIpTests(nicProfile, 0l, false, IPAddressVO.State.Free, "192.168.100.1", "255.255.255.0", "00-88-14-4D-4C-FB", requestedNicProfile, null, requestedIpv4Address); - testOrchastrator.configureNicProfileBasedOnRequestedIp(requestedNicProfile, nicProfile, network); + testOrchestrator.configureNicProfileBasedOnRequestedIp(requestedNicProfile, nicProfile, network); verifyAndAssert(null, null, null, nicProfile, 0, 0); } @@ -319,7 +327,7 @@ private void testConfigureNicProfileBasedOnRequestedIpTestGateway(String ipv4Gat configureTestConfigureNicProfileBasedOnRequestedIpTests(nicProfile, 0l, false, IPAddressVO.State.Free, ipv4Gateway, "255.255.255.0", "00-88-14-4D-4C-FB", requestedNicProfile, "00-88-14-4D-4C-FB", "192.168.100.150"); - testOrchastrator.configureNicProfileBasedOnRequestedIp(requestedNicProfile, nicProfile, network); + testOrchestrator.configureNicProfileBasedOnRequestedIp(requestedNicProfile, nicProfile, network); verifyAndAssert(null, null, null, nicProfile, 1, 0); } @@ -345,7 +353,7 @@ private void testConfigureNicProfileBasedOnRequestedIpTestNetmask(String ipv4Net configureTestConfigureNicProfileBasedOnRequestedIpTests(nicProfile, 0l, false, IPAddressVO.State.Free, "192.168.100.1", ipv4Netmask, "00-88-14-4D-4C-FB", requestedNicProfile, "00-88-14-4D-4C-FB", "192.168.100.150"); - testOrchastrator.configureNicProfileBasedOnRequestedIp(requestedNicProfile, nicProfile, network); + testOrchestrator.configureNicProfileBasedOnRequestedIp(requestedNicProfile, nicProfile, network); verifyAndAssert(null, null, null, nicProfile, 1, 0); } @@ -357,9 +365,9 @@ public void testConfigureNicProfileBasedOnRequestedIpTestIPAddressVONull() { configureTestConfigureNicProfileBasedOnRequestedIpTests(nicProfile, 0l, false, IPAddressVO.State.Free, "192.168.100.1", "255.255.255.0", "00-88-14-4D-4C-FB", requestedNicProfile, "00-88-14-4D-4C-FB", "192.168.100.150"); - when(testOrchastrator._vlanDao.findByNetworkIdAndIpv4(Mockito.anyLong(), Mockito.anyString())).thenReturn(null); + when(testOrchestrator._vlanDao.findByNetworkIdAndIpv4(Mockito.anyLong(), Mockito.anyString())).thenReturn(null); - testOrchastrator.configureNicProfileBasedOnRequestedIp(requestedNicProfile, nicProfile, network); + testOrchestrator.configureNicProfileBasedOnRequestedIp(requestedNicProfile, nicProfile, network); verifyAndAssert(null, null, null, nicProfile, 0, 0); } @@ -375,21 +383,21 @@ private void configureTestConfigureNicProfileBasedOnRequestedIpTests(NicProfile when(ipVoSpy.getState()).thenReturn(state); if (ipVoIsNull) { - when(testOrchastrator._ipAddressDao.findByIpAndSourceNetworkId(Mockito.anyLong(), Mockito.anyString())).thenReturn(ipVoSpy); + when(testOrchestrator._ipAddressDao.findByIpAndSourceNetworkId(Mockito.anyLong(), Mockito.anyString())).thenReturn(ipVoSpy); } else { - when(testOrchastrator._ipAddressDao.findByIpAndSourceNetworkId(Mockito.anyLong(), Mockito.anyString())).thenReturn(ipVoSpy); + when(testOrchestrator._ipAddressDao.findByIpAndSourceNetworkId(Mockito.anyLong(), Mockito.anyString())).thenReturn(ipVoSpy); } VlanVO vlanSpy = Mockito.spy(new VlanVO(Vlan.VlanType.DirectAttached, "vlanTag", vlanGateway, vlanNetmask, 0l, "192.168.100.100 - 192.168.100.200", 0l, new Long(0l), "ip6Gateway", "ip6Cidr", "ip6Range")); Mockito.doReturn(0l).when(vlanSpy).getId(); - when(testOrchastrator._vlanDao.findByNetworkIdAndIpv4(Mockito.anyLong(), Mockito.anyString())).thenReturn(vlanSpy); - when(testOrchastrator._ipAddressDao.acquireInLockTable(Mockito.anyLong())).thenReturn(ipVoSpy); - when(testOrchastrator._ipAddressDao.update(Mockito.anyLong(), Mockito.any(IPAddressVO.class))).thenReturn(true); - when(testOrchastrator._ipAddressDao.releaseFromLockTable(Mockito.anyLong())).thenReturn(true); + when(testOrchestrator._vlanDao.findByNetworkIdAndIpv4(Mockito.anyLong(), Mockito.anyString())).thenReturn(vlanSpy); + when(testOrchestrator._ipAddressDao.acquireInLockTable(Mockito.anyLong())).thenReturn(ipVoSpy); + when(testOrchestrator._ipAddressDao.update(Mockito.anyLong(), Mockito.any(IPAddressVO.class))).thenReturn(true); + when(testOrchestrator._ipAddressDao.releaseFromLockTable(Mockito.anyLong())).thenReturn(true); try { - when(testOrchastrator._networkModel.getNextAvailableMacAddressInNetwork(Mockito.anyLong())).thenReturn(macAddress); + when(testOrchestrator._networkModel.getNextAvailableMacAddressInNetwork(Mockito.anyLong())).thenReturn(macAddress); } catch (InsufficientAddressCapacityException e) { e.printStackTrace(); } @@ -397,9 +405,9 @@ private void configureTestConfigureNicProfileBasedOnRequestedIpTests(NicProfile private void verifyAndAssert(String requestedIpv4Address, String ipv4Gateway, String ipv4Netmask, NicProfile nicProfile, int acquireLockAndCheckIfIpv4IsFreeTimes, int nextMacAddressTimes) { - verify(testOrchastrator, times(acquireLockAndCheckIfIpv4IsFreeTimes)).acquireLockAndCheckIfIpv4IsFree(Mockito.any(Network.class), Mockito.anyString()); + verify(testOrchestrator, times(acquireLockAndCheckIfIpv4IsFreeTimes)).acquireLockAndCheckIfIpv4IsFree(Mockito.any(Network.class), Mockito.anyString()); try { - verify(testOrchastrator._networkModel, times(nextMacAddressTimes)).getNextAvailableMacAddressInNetwork(Mockito.anyLong()); + verify(testOrchestrator._networkModel, times(nextMacAddressTimes)).getNextAvailableMacAddressInNetwork(Mockito.anyLong()); } catch (InsufficientAddressCapacityException e) { e.printStackTrace(); } @@ -441,27 +449,27 @@ private void executeTestAcquireLockAndCheckIfIpv4IsFree(IPAddressVO.State state, ipVoSpy.setState(state); ipVoSpy.setState(state); if (isIPAddressVONull) { - when(testOrchastrator._ipAddressDao.findByIpAndSourceNetworkId(Mockito.anyLong(), Mockito.anyString())).thenReturn(null); + when(testOrchestrator._ipAddressDao.findByIpAndSourceNetworkId(Mockito.anyLong(), Mockito.anyString())).thenReturn(null); } else { - when(testOrchastrator._ipAddressDao.findByIpAndSourceNetworkId(Mockito.anyLong(), Mockito.anyString())).thenReturn(ipVoSpy); + when(testOrchestrator._ipAddressDao.findByIpAndSourceNetworkId(Mockito.anyLong(), Mockito.anyString())).thenReturn(ipVoSpy); } - when(testOrchastrator._ipAddressDao.acquireInLockTable(Mockito.anyLong())).thenReturn(ipVoSpy); - when(testOrchastrator._ipAddressDao.releaseFromLockTable(Mockito.anyLong())).thenReturn(true); - when(testOrchastrator._ipAddressDao.update(Mockito.anyLong(), Mockito.any(IPAddressVO.class))).thenReturn(true); + when(testOrchestrator._ipAddressDao.acquireInLockTable(Mockito.anyLong())).thenReturn(ipVoSpy); + when(testOrchestrator._ipAddressDao.releaseFromLockTable(Mockito.anyLong())).thenReturn(true); + when(testOrchestrator._ipAddressDao.update(Mockito.anyLong(), Mockito.any(IPAddressVO.class))).thenReturn(true); - testOrchastrator.acquireLockAndCheckIfIpv4IsFree(network, "192.168.100.150"); + testOrchestrator.acquireLockAndCheckIfIpv4IsFree(network, "192.168.100.150"); - verify(testOrchastrator._ipAddressDao, Mockito.times(findByIpTimes)).findByIpAndSourceNetworkId(Mockito.anyLong(), Mockito.anyString()); - verify(testOrchastrator._ipAddressDao, Mockito.times(acquireLockTimes)).acquireInLockTable(Mockito.anyLong()); - verify(testOrchastrator._ipAddressDao, Mockito.times(releaseFromLockTimes)).releaseFromLockTable(Mockito.anyLong()); - verify(testOrchastrator._ipAddressDao, Mockito.times(updateTimes)).update(Mockito.anyLong(), Mockito.any(IPAddressVO.class)); - verify(testOrchastrator, Mockito.times(validateTimes)).validateLockedRequestedIp(Mockito.any(IPAddressVO.class), Mockito.any(IPAddressVO.class)); + verify(testOrchestrator._ipAddressDao, Mockito.times(findByIpTimes)).findByIpAndSourceNetworkId(Mockito.anyLong(), Mockito.anyString()); + verify(testOrchestrator._ipAddressDao, Mockito.times(acquireLockTimes)).acquireInLockTable(Mockito.anyLong()); + verify(testOrchestrator._ipAddressDao, Mockito.times(releaseFromLockTimes)).releaseFromLockTable(Mockito.anyLong()); + verify(testOrchestrator._ipAddressDao, Mockito.times(updateTimes)).update(Mockito.anyLong(), Mockito.any(IPAddressVO.class)); + verify(testOrchestrator, Mockito.times(validateTimes)).validateLockedRequestedIp(Mockito.any(IPAddressVO.class), Mockito.any(IPAddressVO.class)); } @Test(expected = InvalidParameterValueException.class) public void validateLockedRequestedIpTestNullLockedIp() { IPAddressVO ipVoSpy = Mockito.spy(new IPAddressVO(new Ip("192.168.100.100"), 0l, 0l, 0l, true)); - testOrchastrator.validateLockedRequestedIp(ipVoSpy, null); + testOrchestrator.validateLockedRequestedIp(ipVoSpy, null); } @Test @@ -476,7 +484,7 @@ public void validateLockedRequestedIpTestNotFreeLockedIp() { IPAddressVO lockedIp = ipVoSpy; lockedIp.setState(states[i]); try { - testOrchastrator.validateLockedRequestedIp(ipVoSpy, lockedIp); + testOrchestrator.validateLockedRequestedIp(ipVoSpy, lockedIp); } catch (InvalidParameterValueException e) { expectedException = true; } @@ -489,7 +497,7 @@ public void validateLockedRequestedIpTestFreeAndNotNullIp() { IPAddressVO ipVoSpy = Mockito.spy(new IPAddressVO(new Ip("192.168.100.100"), 0l, 0l, 0l, true)); IPAddressVO lockedIp = ipVoSpy; lockedIp.setState(State.Free); - testOrchastrator.validateLockedRequestedIp(ipVoSpy, lockedIp); + testOrchestrator.validateLockedRequestedIp(ipVoSpy, lockedIp); } @Test @@ -500,16 +508,16 @@ public void testDontReleaseNicWhenPreserveNicsSettingEnabled() { when(vm.getType()).thenReturn(Type.User); when(network.getGuruName()).thenReturn(guruName); - when(testOrchastrator._networksDao.findById(nic.getNetworkId())).thenReturn(network); + when(testOrchestrator._networksDao.findById(nic.getNetworkId())).thenReturn(network); Long nicId = 1L; when(nic.getId()).thenReturn(nicId); when(vm.getParameter(VirtualMachineProfile.Param.PreserveNics)).thenReturn(true); - testOrchastrator.removeNic(vm, nic); + testOrchestrator.removeNic(vm, nic); verify(nic, never()).setState(Nic.State.Deallocating); - verify(testOrchastrator._nicDao, never()).remove(nicId); + verify(testOrchestrator._nicDao, never()).remove(nicId); } public void encodeVlanIdIntoBroadcastUriTestVxlan() { @@ -568,7 +576,7 @@ public void encodeVlanIdIntoBroadcastUriTestNullVxlanIdWithSchemaIsolationVlan() @Test(expected = InvalidParameterValueException.class) public void encodeVlanIdIntoBroadcastUriTestNullNetwork() { - URI resultUri = testOrchastrator.encodeVlanIdIntoBroadcastUri("vxlan://123", null); + URI resultUri = testOrchestrator.encodeVlanIdIntoBroadcastUri("vxlan://123", null); } private void encodeVlanIdIntoBroadcastUriPrepareAndTest(String vlanId, String isolationMethod, String expectedIsolation, String expectedUri) { @@ -577,7 +585,7 @@ private void encodeVlanIdIntoBroadcastUriPrepareAndTest(String vlanId, String is isolationMethods.add(isolationMethod); physicalNetwork.setIsolationMethods(isolationMethods); - URI resultUri = testOrchastrator.encodeVlanIdIntoBroadcastUri(vlanId, physicalNetwork); + URI resultUri = testOrchestrator.encodeVlanIdIntoBroadcastUri(vlanId, physicalNetwork); Assert.assertEquals(expectedIsolation, resultUri.getScheme()); Assert.assertEquals(expectedUri, resultUri.toString()); @@ -595,17 +603,17 @@ private NicProfile prepareMocksAndRunPrepareNic(VirtualMachine.Type vmType, bool Mockito.when(network.getDns2()).thenReturn(ip4Dns[1]); Mockito.when(network.getIp6Dns1()).thenReturn(ip6Dns[0]); Mockito.when(network.getIp6Dns2()).thenReturn(ip6Dns[1]); - Mockito.when(testOrchastrator._networkModel.getNetworkRate(networkId, vmId)).thenReturn(networkRate); + Mockito.when(testOrchestrator._networkModel.getNetworkRate(networkId, vmId)).thenReturn(networkRate); NicVO nicVO = Mockito.mock(NicVO.class); Mockito.when(nicVO.isDefaultNic()).thenReturn(isDefaultNic); - Mockito.when(testOrchastrator._nicDao.findById(nicId)).thenReturn(nicVO); - Mockito.when(testOrchastrator._nicDao.update(nicId, nicVO)).thenReturn(true); - Mockito.when(testOrchastrator._networkModel.isSecurityGroupSupportedInNetwork(network)).thenReturn(false); - Mockito.when(testOrchastrator._networkModel.getNetworkTag(hypervisorType, network)).thenReturn(null); - Mockito.when(testOrchastrator._ntwkSrvcDao.getDistinctProviders(networkId)).thenReturn(new ArrayList<>()); - testOrchastrator.networkElements = new ArrayList<>(); - Mockito.when(testOrchastrator._nicExtraDhcpOptionDao.listByNicId(nicId)).thenReturn(new ArrayList<>()); - Mockito.when(testOrchastrator._ntwkSrvcDao.areServicesSupportedInNetwork(networkId, Service.Dhcp)).thenReturn(false); + Mockito.when(testOrchestrator._nicDao.findById(nicId)).thenReturn(nicVO); + Mockito.when(testOrchestrator._nicDao.update(nicId, nicVO)).thenReturn(true); + Mockito.when(testOrchestrator._networkModel.isSecurityGroupSupportedInNetwork(network)).thenReturn(false); + Mockito.when(testOrchestrator._networkModel.getNetworkTag(hypervisorType, network)).thenReturn(null); + Mockito.when(testOrchestrator._ntwkSrvcDao.getDistinctProviders(networkId)).thenReturn(new ArrayList<>()); + testOrchestrator.networkElements = new ArrayList<>(); + Mockito.when(testOrchestrator._nicExtraDhcpOptionDao.listByNicId(nicId)).thenReturn(new ArrayList<>()); + Mockito.when(testOrchestrator._ntwkSrvcDao.areServicesSupportedInNetwork(networkId, Service.Dhcp)).thenReturn(false); VirtualMachineProfile virtualMachineProfile = Mockito.mock(VirtualMachineProfile.class); Mockito.when(virtualMachineProfile.getType()).thenReturn(vmType); Mockito.when(virtualMachineProfile.getId()).thenReturn(vmId); @@ -634,7 +642,7 @@ private NicProfile prepareMocksAndRunPrepareNic(VirtualMachine.Type vmType, bool Mockito.when(vpcVO.getIp4Dns1()).thenReturn(null); Mockito.when(vpcVO.getIp6Dns1()).thenReturn(null); } - Mockito.when(testOrchastrator._vpcMgr.getActiveVpc(vpcId)).thenReturn(vpcVO); + Mockito.when(testOrchestrator._vpcMgr.getActiveVpc(vpcId)).thenReturn(vpcVO); } else { Mockito.when(routerVO.getVpcId()).thenReturn(null); Long routerNetworkId = 2L; @@ -648,13 +656,13 @@ private NicProfile prepareMocksAndRunPrepareNic(VirtualMachine.Type vmType, bool Mockito.when(routerNetworkVO.getDns1()).thenReturn(null); Mockito.when(routerNetworkVO.getIp6Dns1()).thenReturn(null); } - Mockito.when(testOrchastrator.routerNetworkDao.getRouterNetworks(vmId)).thenReturn(List.of(routerNetworkId)); - Mockito.when(testOrchastrator._networksDao.findById(routerNetworkId)).thenReturn(routerNetworkVO); + Mockito.when(testOrchestrator.routerNetworkDao.getRouterNetworks(vmId)).thenReturn(List.of(routerNetworkId)); + Mockito.when(testOrchestrator._networksDao.findById(routerNetworkId)).thenReturn(routerNetworkVO); } - Mockito.when(testOrchastrator.routerDao.findById(vmId)).thenReturn(routerVO); + Mockito.when(testOrchestrator.routerDao.findById(vmId)).thenReturn(routerVO); NicProfile profile = null; try { - profile = testOrchastrator.prepareNic(virtualMachineProfile, deployDestination, reservationContext, nicId, network); + profile = testOrchestrator.prepareNic(virtualMachineProfile, deployDestination, reservationContext, nicId, network); } catch (InsufficientCapacityException | ResourceUnavailableException e) { Assert.fail(String.format("Failure with exception %s", e.getMessage())); } @@ -723,7 +731,7 @@ public void testGetNetworkGatewayAndNetmaskForNicImportAdvancedZone() { Mockito.when(dataCenter.getNetworkType()).thenReturn(DataCenter.NetworkType.Advanced); Mockito.when(network.getGateway()).thenReturn(networkGateway); Mockito.when(network.getCidr()).thenReturn(networkCidr); - Pair pair = testOrchastrator.getNetworkGatewayAndNetmaskForNicImport(network, dataCenter, ipAddress); + Pair pair = testOrchestrator.getNetworkGatewayAndNetmaskForNicImport(network, dataCenter, ipAddress); Assert.assertNotNull(pair); Assert.assertEquals(networkGateway, pair.first()); Assert.assertEquals(networkNetmask, pair.second()); @@ -743,9 +751,9 @@ public void testGetNetworkGatewayAndNetmaskForNicImportBasicZone() { Mockito.when(vlan.getVlanNetmask()).thenReturn(defaultNetworkNetmask); Mockito.when(dataCenter.getNetworkType()).thenReturn(DataCenter.NetworkType.Basic); Mockito.when(ipAddressVO.getVlanId()).thenReturn(1L); - Mockito.when(testOrchastrator._vlanDao.findById(1L)).thenReturn(vlan); - Mockito.when(testOrchastrator._ipAddressDao.findByIp(ipAddress)).thenReturn(ipAddressVO); - Pair pair = testOrchastrator.getNetworkGatewayAndNetmaskForNicImport(network, dataCenter, ipAddress); + Mockito.when(testOrchestrator._vlanDao.findById(1L)).thenReturn(vlan); + Mockito.when(testOrchestrator._ipAddressDao.findByIp(ipAddress)).thenReturn(ipAddressVO); + Pair pair = testOrchestrator.getNetworkGatewayAndNetmaskForNicImport(network, dataCenter, ipAddress); Assert.assertNotNull(pair); Assert.assertEquals(defaultNetworkGateway, pair.first()); Assert.assertEquals(defaultNetworkNetmask, pair.second()); @@ -757,7 +765,7 @@ public void testGetGuestIpForNicImportL2Network() { DataCenter dataCenter = Mockito.mock(DataCenter.class); Network.IpAddresses ipAddresses = Mockito.mock(Network.IpAddresses.class); Mockito.when(network.getGuestType()).thenReturn(GuestType.L2); - Assert.assertNull(testOrchastrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses)); + Assert.assertNull(testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses)); } @Test @@ -769,8 +777,8 @@ public void testGetGuestIpForNicImportAdvancedZone() { Mockito.when(dataCenter.getNetworkType()).thenReturn(DataCenter.NetworkType.Advanced); String ipAddress = "10.1.10.10"; Mockito.when(ipAddresses.getIp4Address()).thenReturn(ipAddress); - Mockito.when(testOrchastrator._ipAddrMgr.acquireGuestIpAddress(network, ipAddress)).thenReturn(ipAddress); - String guestIp = testOrchastrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses); + Mockito.when(testOrchestrator._ipAddrMgr.acquireGuestIpAddress(network, ipAddress)).thenReturn(ipAddress); + String guestIp = testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses); Assert.assertEquals(ipAddress, guestIp); } @@ -791,8 +799,8 @@ public void testGetGuestIpForNicImportBasicZoneAutomaticIP() { Mockito.when(ipAddressVO.getState()).thenReturn(State.Free); Mockito.when(network.getId()).thenReturn(networkId); Mockito.when(dataCenter.getId()).thenReturn(dataCenterId); - Mockito.when(testOrchastrator._ipAddressDao.findBySourceNetworkIdAndDatacenterIdAndState(networkId, dataCenterId, State.Free)).thenReturn(ipAddressVO); - String ipAddress = testOrchastrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses); + Mockito.when(testOrchestrator._ipAddressDao.findBySourceNetworkIdAndDatacenterIdAndState(networkId, dataCenterId, State.Free)).thenReturn(ipAddressVO); + String ipAddress = testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses); Assert.assertEquals(freeIp, ipAddress); } @@ -814,8 +822,8 @@ public void testGetGuestIpForNicImportBasicZoneManualIP() { Mockito.when(network.getId()).thenReturn(networkId); Mockito.when(dataCenter.getId()).thenReturn(dataCenterId); Mockito.when(ipAddresses.getIp4Address()).thenReturn(requestedIp); - Mockito.when(testOrchastrator._ipAddressDao.findByIp(requestedIp)).thenReturn(ipAddressVO); - String ipAddress = testOrchastrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses); + Mockito.when(testOrchestrator._ipAddressDao.findByIp(requestedIp)).thenReturn(ipAddressVO); + String ipAddress = testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses); Assert.assertEquals(requestedIp, ipAddress); } @@ -837,7 +845,168 @@ public void testGetGuestIpForNicImportBasicUsedIP() { Mockito.when(network.getId()).thenReturn(networkId); Mockito.when(dataCenter.getId()).thenReturn(dataCenterId); Mockito.when(ipAddresses.getIp4Address()).thenReturn(requestedIp); - Mockito.when(testOrchastrator._ipAddressDao.findByIp(requestedIp)).thenReturn(ipAddressVO); - testOrchastrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses); + Mockito.when(testOrchestrator._ipAddressDao.findByIp(requestedIp)).thenReturn(ipAddressVO); + testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses); + } + + @Test + public void testShutdownNetworkAcquireLockFailed() { + ReservationContext reservationContext = Mockito.mock(ReservationContext.class); + NetworkVO network = mock(NetworkVO.class); + long networkId = 1; + when(testOrchestrator._networksDao.acquireInLockTable(Mockito.anyLong(), Mockito.anyInt())).thenReturn(null); + + boolean shutdownNetworkStatus = testOrchestrator.shutdownNetwork(networkId, reservationContext, false); + Assert.assertFalse(shutdownNetworkStatus); + + verify(testOrchestrator._networksDao, times(1)).acquireInLockTable(networkId, NetworkLockTimeout.value()); + } + + @Test + public void testShutdownNetworkInAllocatedState() { + ReservationContext reservationContext = Mockito.mock(ReservationContext.class); + NetworkVO network = mock(NetworkVO.class); + long networkId = 1; + when(testOrchestrator._networksDao.acquireInLockTable(Mockito.anyLong(), Mockito.anyInt())).thenReturn(network); + when(network.getId()).thenReturn(networkId); + when(network.getState()).thenReturn(Network.State.Allocated); + + boolean shutdownNetworkStatus = testOrchestrator.shutdownNetwork(networkId, reservationContext, false); + Assert.assertTrue(shutdownNetworkStatus); + + verify(network, times(1)).getState(); + verify(testOrchestrator._networksDao, times(1)).acquireInLockTable(networkId, NetworkLockTimeout.value()); + verify(testOrchestrator._networksDao, times(1)).releaseFromLockTable(networkId); + } + + @Test + public void testShutdownNetworkInImplementingState() { + ReservationContext reservationContext = Mockito.mock(ReservationContext.class); + NetworkVO network = mock(NetworkVO.class); + long networkId = 1; + when(testOrchestrator._networksDao.acquireInLockTable(Mockito.anyLong(), Mockito.anyInt())).thenReturn(network); + when(network.getId()).thenReturn(networkId); + when(network.getState()).thenReturn(Network.State.Implementing); + + boolean shutdownNetworkStatus = testOrchestrator.shutdownNetwork(networkId, reservationContext, false); + Assert.assertFalse(shutdownNetworkStatus); + + verify(network, times(3)).getState(); + verify(testOrchestrator._networksDao, times(1)).acquireInLockTable(networkId, NetworkLockTimeout.value()); + verify(testOrchestrator._networksDao, times(1)).releaseFromLockTable(networkId); + } + + @Test(expected = InsufficientVirtualNetworkCapacityException.class) + public void testImportNicAcquireGuestIPFailed() throws Exception { + DataCenter dataCenter = Mockito.mock(DataCenter.class); + VirtualMachine vm = mock(VirtualMachine.class); + Network network = Mockito.mock(Network.class); + Mockito.when(network.getGuestType()).thenReturn(GuestType.Isolated); + Mockito.when(network.getNetworkOfferingId()).thenReturn(networkOfferingId); + long dataCenterId = 1L; + Mockito.when(network.getDataCenterId()).thenReturn(dataCenterId); + Network.IpAddresses ipAddresses = Mockito.mock(Network.IpAddresses.class); + String ipAddress = "10.1.10.10"; + Mockito.when(ipAddresses.getIp4Address()).thenReturn(ipAddress); + Mockito.when(testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses)).thenReturn(null); + Mockito.when(testOrchestrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(Arrays.asList(Service.Dns, Service.Dhcp)); + String macAddress = "02:01:01:82:00:01"; + int deviceId = 0; + testOrchestrator.importNic(macAddress, deviceId, network, true, vm, ipAddresses, dataCenter, false); + } + + @Test(expected = InsufficientVirtualNetworkCapacityException.class) + public void testImportNicAutoAcquireGuestIPFailed() throws Exception { + DataCenter dataCenter = Mockito.mock(DataCenter.class); + VirtualMachine vm = mock(VirtualMachine.class); + Network network = Mockito.mock(Network.class); + Mockito.when(network.getGuestType()).thenReturn(GuestType.Isolated); + Mockito.when(network.getNetworkOfferingId()).thenReturn(networkOfferingId); + long dataCenterId = 1L; + Mockito.when(network.getDataCenterId()).thenReturn(dataCenterId); + Network.IpAddresses ipAddresses = Mockito.mock(Network.IpAddresses.class); + String ipAddress = "auto"; + Mockito.when(ipAddresses.getIp4Address()).thenReturn(ipAddress); + Mockito.when(testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses)).thenReturn(null); + Mockito.when(testOrchestrator._networkModel.listNetworkOfferingServices(networkOfferingId)).thenReturn(Arrays.asList(Service.Dns, Service.Dhcp)); + String macAddress = "02:01:01:82:00:01"; + int deviceId = 0; + testOrchestrator.importNic(macAddress, deviceId, network, true, vm, ipAddresses, dataCenter, false); + } + + @Test + public void testImportNicNoIP4Address() throws Exception { + DataCenter dataCenter = Mockito.mock(DataCenter.class); + Long vmId = 1L; + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + VirtualMachine vm = mock(VirtualMachine.class); + Mockito.when(vm.getId()).thenReturn(vmId); + Mockito.when(vm.getHypervisorType()).thenReturn(hypervisorType); + Long networkId = 1L; + Network network = Mockito.mock(Network.class); + Mockito.when(network.getId()).thenReturn(networkId); + Network.IpAddresses ipAddresses = Mockito.mock(Network.IpAddresses.class); + Mockito.when(ipAddresses.getIp4Address()).thenReturn(null); + URI broadcastUri = URI.create("vlan://123"); + NicVO nic = mock(NicVO.class); + Mockito.when(nic.getBroadcastUri()).thenReturn(broadcastUri); + String macAddress = "02:01:01:82:00:01"; + int deviceId = 1; + Integer networkRate = 200; + Mockito.when(testOrchestrator._networkModel.getNetworkRate(networkId, vmId)).thenReturn(networkRate); + Mockito.when(testOrchestrator._networkModel.isSecurityGroupSupportedInNetwork(network)).thenReturn(false); + Mockito.when(testOrchestrator._networkModel.getNetworkTag(hypervisorType, network)).thenReturn("testtag"); + try (MockedStatic transactionMocked = Mockito.mockStatic(Transaction.class)) { + transactionMocked.when(() -> Transaction.execute(any(TransactionCallback.class))).thenReturn(nic); + Pair nicProfileIntegerPair = testOrchestrator.importNic(macAddress, deviceId, network, true, vm, ipAddresses, dataCenter, false); + verify(testOrchestrator._networkModel, times(1)).getNetworkRate(networkId, vmId); + verify(testOrchestrator._networkModel, times(1)).isSecurityGroupSupportedInNetwork(network); + verify(testOrchestrator._networkModel, times(1)).getNetworkTag(Hypervisor.HypervisorType.KVM, network); + assertEquals(deviceId, nicProfileIntegerPair.second().intValue()); + NicProfile nicProfile = nicProfileIntegerPair.first(); + assertEquals(broadcastUri, nicProfile.getBroadCastUri()); + assertEquals(networkRate, nicProfile.getNetworkRate()); + assertFalse(nicProfile.isSecurityGroupEnabled()); + assertEquals("testtag", nicProfile.getName()); + } + } + + @Test + public void testImportNicWithIP4Address() throws Exception { + DataCenter dataCenter = Mockito.mock(DataCenter.class); + Long vmId = 1L; + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + VirtualMachine vm = mock(VirtualMachine.class); + Mockito.when(vm.getId()).thenReturn(vmId); + Mockito.when(vm.getHypervisorType()).thenReturn(hypervisorType); + Long networkId = 1L; + Network network = Mockito.mock(Network.class); + Mockito.when(network.getId()).thenReturn(networkId); + String ipAddress = "10.1.10.10"; + Network.IpAddresses ipAddresses = Mockito.mock(Network.IpAddresses.class); + Mockito.when(ipAddresses.getIp4Address()).thenReturn(ipAddress); + URI broadcastUri = URI.create("vlan://123"); + NicVO nic = mock(NicVO.class); + Mockito.when(nic.getBroadcastUri()).thenReturn(broadcastUri); + String macAddress = "02:01:01:82:00:01"; + int deviceId = 1; + Integer networkRate = 200; + Mockito.when(testOrchestrator._networkModel.getNetworkRate(networkId, vmId)).thenReturn(networkRate); + Mockito.when(testOrchestrator._networkModel.isSecurityGroupSupportedInNetwork(network)).thenReturn(false); + Mockito.when(testOrchestrator._networkModel.getNetworkTag(hypervisorType, network)).thenReturn("testtag"); + try (MockedStatic transactionMocked = Mockito.mockStatic(Transaction.class)) { + transactionMocked.when(() -> Transaction.execute(any(TransactionCallback.class))).thenReturn(nic); + Pair nicProfileIntegerPair = testOrchestrator.importNic(macAddress, deviceId, network, true, vm, ipAddresses, dataCenter, false); + verify(testOrchestrator, times(1)).getSelectedIpForNicImport(network, dataCenter, ipAddresses); + verify(testOrchestrator._networkModel, times(1)).getNetworkRate(networkId, vmId); + verify(testOrchestrator._networkModel, times(1)).isSecurityGroupSupportedInNetwork(network); + verify(testOrchestrator._networkModel, times(1)).getNetworkTag(Hypervisor.HypervisorType.KVM, network); + assertEquals(deviceId, nicProfileIntegerPair.second().intValue()); + NicProfile nicProfile = nicProfileIntegerPair.first(); + assertEquals(broadcastUri, nicProfile.getBroadCastUri()); + assertEquals(networkRate, nicProfile.getNetworkRate()); + assertFalse(nicProfile.isSecurityGroupEnabled()); + assertEquals("testtag", nicProfile.getName()); + } } } diff --git a/engine/schema/src/main/java/com/cloud/capacity/CapacityVO.java b/engine/schema/src/main/java/com/cloud/capacity/CapacityVO.java index 132fd3fe5a23..cd62935f17ee 100644 --- a/engine/schema/src/main/java/com/cloud/capacity/CapacityVO.java +++ b/engine/schema/src/main/java/com/cloud/capacity/CapacityVO.java @@ -135,8 +135,8 @@ public Long getPodId() { return podId; } - public void setPodId(long podId) { - this.podId = new Long(podId); + public void setPodId(Long podId) { + this.podId = podId; } @Override @@ -144,8 +144,8 @@ public Long getClusterId() { return clusterId; } - public void setClusterId(long clusterId) { - this.clusterId = new Long(clusterId); + public void setClusterId(Long clusterId) { + this.clusterId = clusterId; } @Override diff --git a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDao.java b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDao.java index 51362cf885e0..6b53e49764e4 100644 --- a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDao.java @@ -31,4 +31,6 @@ public interface DomainDetailsDao extends GenericDao { void deleteDetails(long domainId); void update(long domainId, Map details); + + String getActualValue(DomainDetailVO domainDetailVO); } diff --git a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java index dad3fe9ad1eb..50097d154f5f 100644 --- a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java @@ -24,6 +24,7 @@ import com.cloud.domain.DomainDetailVO; import com.cloud.domain.DomainVO; +import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchBuilder; @@ -34,6 +35,7 @@ import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; public class DomainDetailsDaoImpl extends GenericDaoBase implements DomainDetailsDao, ScopedConfigStorage { protected final SearchBuilder domainSearch; @@ -111,7 +113,7 @@ public String getConfigValue(long id, ConfigKey key) { String enableDomainSettingsForChildDomain = _configDao.getValue("enable.domain.settings.for.child.domain"); if (!Boolean.parseBoolean(enableDomainSettingsForChildDomain)) { vo = findDetail(id, key.key()); - return vo == null ? null : vo.getValue(); + return vo == null ? null : getActualValue(vo); } DomainVO domain = _domainDao.findById(id); // if value is not configured in domain then check its parent domain till ROOT @@ -125,6 +127,15 @@ public String getConfigValue(long id, ConfigKey key) { break; } } - return vo == null ? null : vo.getValue(); + return vo == null ? null : getActualValue(vo); + } + + @Override + public String getActualValue(DomainDetailVO domainDetailVO) { + ConfigurationVO configurationVO = _configDao.findByName(domainDetailVO.getName()); + if (configurationVO != null && configurationVO.isEncrypted()) { + return DBEncryptionUtil.decrypt(domainDetailVO.getValue()); + } + return domainDetailVO.getValue(); } } diff --git a/engine/schema/src/main/java/com/cloud/host/HostTagVO.java b/engine/schema/src/main/java/com/cloud/host/HostTagVO.java index cd4ac29738d5..98071a2c0732 100644 --- a/engine/schema/src/main/java/com/cloud/host/HostTagVO.java +++ b/engine/schema/src/main/java/com/cloud/host/HostTagVO.java @@ -40,6 +40,9 @@ public class HostTagVO implements InternalIdentity { @Column(name = "tag") private String tag; + @Column(name = "is_implicit") + private boolean isImplicit = false; + @Column(name = "is_tag_a_rule") private boolean isTagARule; @@ -74,6 +77,13 @@ public boolean getIsTagARule() { return isTagARule; } + public void setIsImplicit(boolean isImplicit) { + this.isImplicit = isImplicit; + } + + public boolean getIsImplicit() { + return isImplicit; + } @Override public long getId() { diff --git a/engine/schema/src/main/java/com/cloud/host/HostVO.java b/engine/schema/src/main/java/com/cloud/host/HostVO.java index 3e64d20d0e2d..1a507da79570 100644 --- a/engine/schema/src/main/java/com/cloud/host/HostVO.java +++ b/engine/schema/src/main/java/com/cloud/host/HostVO.java @@ -16,13 +16,13 @@ // under the License. package com.cloud.host; -import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import javax.persistence.Column; @@ -45,6 +45,7 @@ import org.apache.cloudstack.util.HypervisorTypeConverter; import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang3.StringUtils; @@ -768,27 +769,48 @@ public void setUuid(String uuid) { this.uuid = uuid; } - public boolean checkHostServiceOfferingAndTemplateTags(ServiceOffering serviceOffering, VirtualMachineTemplate template) { - if (serviceOffering == null || template == null) { - return false; - } + private Set getHostServiceOfferingAndTemplateStrictTags(ServiceOffering serviceOffering, VirtualMachineTemplate template, Set strictHostTags) { if (StringUtils.isEmpty(serviceOffering.getHostTag()) && StringUtils.isEmpty(template.getTemplateTag())) { - return true; + return new HashSet<>(); } - if (getHostTags() == null) { - return false; - } - HashSet hostTagsSet = new HashSet<>(getHostTags()); - List tags = new ArrayList<>(); + List hostTagsList = getHostTags(); + HashSet hostTagsSet = CollectionUtils.isNotEmpty(hostTagsList) ? new HashSet<>(hostTagsList) : new HashSet<>(); + HashSet tags = new HashSet<>(); if (StringUtils.isNotEmpty(serviceOffering.getHostTag())) { tags.addAll(Arrays.asList(serviceOffering.getHostTag().split(","))); } - if (StringUtils.isNotEmpty(template.getTemplateTag()) && !tags.contains(template.getTemplateTag())) { + if (StringUtils.isNotEmpty(template.getTemplateTag())) { tags.add(template.getTemplateTag()); } + tags.removeIf(tag -> !strictHostTags.contains(tag)); + tags.removeAll(hostTagsSet); + return tags; + } + + public boolean checkHostServiceOfferingAndTemplateTags(ServiceOffering serviceOffering, VirtualMachineTemplate template, Set strictHostTags) { + if (serviceOffering == null || template == null) { + return false; + } + Set tags = getHostServiceOfferingAndTemplateStrictTags(serviceOffering, template, strictHostTags); + if (tags.isEmpty()) { + return true; + } + List hostTagsList = getHostTags(); + HashSet hostTagsSet = CollectionUtils.isNotEmpty(hostTagsList) ? new HashSet<>(hostTagsList) : new HashSet<>(); return hostTagsSet.containsAll(tags); } + public Set getHostServiceOfferingAndTemplateMissingTags(ServiceOffering serviceOffering, VirtualMachineTemplate template, Set strictHostTags) { + Set tags = getHostServiceOfferingAndTemplateStrictTags(serviceOffering, template, strictHostTags); + if (tags.isEmpty()) { + return new HashSet<>(); + } + List hostTagsList = getHostTags(); + HashSet hostTagsSet = CollectionUtils.isNotEmpty(hostTagsList) ? new HashSet<>(hostTagsList) : new HashSet<>(); + tags.removeAll(hostTagsSet); + return tags; + } + public boolean checkHostServiceOfferingTags(ServiceOffering serviceOffering) { if (serviceOffering == null) { return false; diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index ca180e2323fd..08380ed8b405 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -141,6 +141,8 @@ public interface HostDao extends GenericDao, StateDao listByHostCapability(Host.Type type, Long clusterId, Long podId, long dcId, String hostCapabilty); + List listByClusterHypervisorTypeAndHostCapability(Long clusterId, HypervisorType hypervisorType, String hostCapabilty); + List listByClusterAndHypervisorType(long clusterId, HypervisorType hypervisorType); HostVO findByName(String name); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 5faa877b458f..170c6a45fc3a 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -341,6 +341,7 @@ public void init() { ClusterHypervisorSearch.and("hypervisor", ClusterHypervisorSearch.entity().getHypervisorType(), SearchCriteria.Op.EQ); ClusterHypervisorSearch.and("type", ClusterHypervisorSearch.entity().getType(), SearchCriteria.Op.EQ); ClusterHypervisorSearch.and("status", ClusterHypervisorSearch.entity().getStatus(), SearchCriteria.Op.EQ); + ClusterHypervisorSearch.and("resourceState", ClusterHypervisorSearch.entity().getResourceState(), SearchCriteria.Op.EQ); ClusterHypervisorSearch.done(); UnmanagedDirectConnectSearch = createSearchBuilder(); @@ -1506,12 +1507,42 @@ public List listByHostCapability(Type type, Long clusterId, Long podId, return listBy(sc); } + @Override + public List listByClusterHypervisorTypeAndHostCapability(Long clusterId, HypervisorType hypervisorType, String hostCapabilty) { + SearchBuilder hostCapabilitySearch = _detailsDao.createSearchBuilder(); + DetailVO tagEntity = hostCapabilitySearch.entity(); + hostCapabilitySearch.and("capability", tagEntity.getName(), SearchCriteria.Op.EQ); + hostCapabilitySearch.and("value", tagEntity.getValue(), SearchCriteria.Op.EQ); + + SearchBuilder hostSearch = createSearchBuilder(); + HostVO entity = hostSearch.entity(); + hostSearch.and("clusterId", entity.getClusterId(), SearchCriteria.Op.EQ); + hostSearch.and("hypervisor", entity.getHypervisorType(), SearchCriteria.Op.EQ); + hostSearch.and("type", entity.getType(), SearchCriteria.Op.EQ); + hostSearch.and("status", entity.getStatus(), SearchCriteria.Op.EQ); + hostSearch.and("resourceState", entity.getResourceState(), SearchCriteria.Op.EQ); + hostSearch.join("hostCapabilitySearch", hostCapabilitySearch, entity.getId(), tagEntity.getHostId(), JoinBuilder.JoinType.INNER); + + SearchCriteria sc = hostSearch.create(); + sc.setJoinParameters("hostCapabilitySearch", "value", Boolean.toString(true)); + sc.setJoinParameters("hostCapabilitySearch", "capability", hostCapabilty); + + sc.setParameters("clusterId", clusterId); + sc.setParameters("hypervisor", hypervisorType); + sc.setParameters("type", Type.Routing); + sc.setParameters("status", Status.Up); + sc.setParameters("resourceState", ResourceState.Enabled); + return listBy(sc); + } + + @Override public List listByClusterAndHypervisorType(long clusterId, HypervisorType hypervisorType) { SearchCriteria sc = ClusterHypervisorSearch.create(); sc.setParameters("clusterId", clusterId); sc.setParameters("hypervisor", hypervisorType); sc.setParameters("type", Type.Routing); sc.setParameters("status", Status.Up); + sc.setParameters("resourceState", ResourceState.Enabled); return listBy(sc); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java index d134db334035..7a00829fd44e 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java @@ -20,6 +20,7 @@ import com.cloud.host.HostTagVO; import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.api.response.HostTagResponse; import org.apache.cloudstack.framework.config.ConfigKey; public interface HostTagsDao extends GenericDao { @@ -35,6 +36,13 @@ public interface HostTagsDao extends GenericDao { void deleteTags(long hostId); + boolean updateImplicitTags(long hostId, List hostTags); + + List getExplicitHostTags(long hostId); + List findHostRuleTags(); + HostTagResponse newHostTagResponse(HostTagVO hostTag); + + List searchByIds(Long... hostTagIds); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java index 65deb1d1c9b0..4aa14a31cfcf 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java @@ -16,10 +16,14 @@ // under the License. package com.cloud.host.dao; +import java.util.ArrayList; import java.util.List; +import org.apache.cloudstack.api.response.HostTagResponse; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import com.cloud.host.HostTagVO; @@ -30,14 +34,23 @@ import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.SearchCriteria.Func; +import javax.inject.Inject; + @Component public class HostTagsDaoImpl extends GenericDaoBase implements HostTagsDao, Configurable { protected final SearchBuilder HostSearch; protected final GenericSearchBuilder DistinctImplictTagsSearch; + private final SearchBuilder stSearch; + private final SearchBuilder tagIdsearch; + private final SearchBuilder ImplicitTagsSearch; + + @Inject + private ConfigurationDao _configDao; public HostTagsDaoImpl() { HostSearch = createSearchBuilder(); HostSearch.and("hostId", HostSearch.entity().getHostId(), SearchCriteria.Op.EQ); + HostSearch.and("isImplicit", HostSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ); HostSearch.and("isTagARule", HostSearch.entity().getIsTagARule(), SearchCriteria.Op.EQ); HostSearch.done(); @@ -46,6 +59,19 @@ public HostTagsDaoImpl() { DistinctImplictTagsSearch.and("hostIds", DistinctImplictTagsSearch.entity().getHostId(), SearchCriteria.Op.IN); DistinctImplictTagsSearch.and("implicitTags", DistinctImplictTagsSearch.entity().getTag(), SearchCriteria.Op.IN); DistinctImplictTagsSearch.done(); + + stSearch = createSearchBuilder(); + stSearch.and("idIN", stSearch.entity().getId(), SearchCriteria.Op.IN); + stSearch.done(); + + tagIdsearch = createSearchBuilder(); + tagIdsearch.and("id", tagIdsearch.entity().getId(), SearchCriteria.Op.EQ); + tagIdsearch.done(); + + ImplicitTagsSearch = createSearchBuilder(); + ImplicitTagsSearch.and("hostId", ImplicitTagsSearch.entity().getHostId(), SearchCriteria.Op.EQ); + ImplicitTagsSearch.and("isImplicit", ImplicitTagsSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ); + ImplicitTagsSearch.done(); } @Override @@ -74,6 +100,36 @@ public void deleteTags(long hostId) { txn.commit(); } + @Override + public boolean updateImplicitTags(long hostId, List hostTags) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + txn.start(); + SearchCriteria sc = ImplicitTagsSearch.create(); + sc.setParameters("hostId", hostId); + sc.setParameters("isImplicit", true); + boolean expunged = expunge(sc) > 0; + boolean persisted = false; + for (String tag : hostTags) { + if (StringUtils.isNotBlank(tag)) { + HostTagVO vo = new HostTagVO(hostId, tag.trim()); + vo.setIsImplicit(true); + persist(vo); + persisted = true; + } + } + txn.commit(); + return expunged || persisted; + } + + @Override + public List getExplicitHostTags(long hostId) { + SearchCriteria sc = ImplicitTagsSearch.create(); + sc.setParameters("hostId", hostId); + sc.setParameters("isImplicit", false); + + return search(sc, null); + } + @Override public List findHostRuleTags() { SearchCriteria sc = HostSearch.create(); @@ -89,6 +145,7 @@ public void persist(long hostId, List hostTags, Boolean isTagARule) { txn.start(); SearchCriteria sc = HostSearch.create(); sc.setParameters("hostId", hostId); + sc.setParameters("isImplicit", false); expunge(sc); for (String tag : hostTags) { @@ -110,4 +167,72 @@ public ConfigKey[] getConfigKeys() { public String getConfigComponentName() { return HostTagsDaoImpl.class.getSimpleName(); } + + @Override + public HostTagResponse newHostTagResponse(HostTagVO tag) { + HostTagResponse tagResponse = new HostTagResponse(); + + tagResponse.setName(tag.getTag()); + tagResponse.setHostId(tag.getHostId()); + tagResponse.setImplicit(tag.getIsImplicit()); + + tagResponse.setObjectName("hosttag"); + + return tagResponse; + } + + @Override + public List searchByIds(Long... tagIds) { + String batchCfg = _configDao.getValue("detail.batch.query.size"); + + final int detailsBatchSize = batchCfg != null ? Integer.parseInt(batchCfg) : 2000; + + // query details by batches + List tagList = new ArrayList<>(); + int curr_index = 0; + + if (tagIds.length > detailsBatchSize) { + while ((curr_index + detailsBatchSize) <= tagIds.length) { + Long[] ids = new Long[detailsBatchSize]; + + for (int k = 0, j = curr_index; j < curr_index + detailsBatchSize; j++, k++) { + ids[k] = tagIds[j]; + } + + SearchCriteria sc = stSearch.create(); + + sc.setParameters("idIN", (Object[])ids); + + List vms = searchIncludingRemoved(sc, null, null, false); + + if (vms != null) { + tagList.addAll(vms); + } + + curr_index += detailsBatchSize; + } + } + + if (curr_index < tagIds.length) { + int batch_size = (tagIds.length - curr_index); + // set the ids value + Long[] ids = new Long[batch_size]; + + for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) { + ids[k] = tagIds[j]; + } + + SearchCriteria sc = stSearch.create(); + + sc.setParameters("idIN", (Object[])ids); + + List tags = searchIncludingRemoved(sc, null, null, false); + + if (tags != null) { + tagList.addAll(tags); + } + } + + return tagList; + } } diff --git a/engine/schema/src/main/java/com/cloud/hypervisor/HypervisorCapabilitiesVO.java b/engine/schema/src/main/java/com/cloud/hypervisor/HypervisorCapabilitiesVO.java index 4455c7491ddb..a3b03280fdf6 100644 --- a/engine/schema/src/main/java/com/cloud/hypervisor/HypervisorCapabilitiesVO.java +++ b/engine/schema/src/main/java/com/cloud/hypervisor/HypervisorCapabilitiesVO.java @@ -80,6 +80,18 @@ public HypervisorCapabilitiesVO(HypervisorType hypervisorType, String hypervisor this.uuid = UUID.randomUUID().toString(); } + public HypervisorCapabilitiesVO(HypervisorCapabilitiesVO source) { + this.hypervisorType = source.getHypervisorType(); + this.hypervisorVersion = source.getHypervisorVersion(); + this.maxGuestsLimit = source.getMaxGuestsLimit(); + this.maxDataVolumesLimit = source.getMaxDataVolumesLimit(); + this.maxHostsPerCluster = source.getMaxHostsPerCluster(); + this.securityGroupEnabled = source.isSecurityGroupEnabled(); + this.storageMotionSupported = source.isStorageMotionSupported(); + this.vmSnapshotEnabled = source.isVmSnapshotEnabled(); + this.uuid = UUID.randomUUID().toString(); + } + /** * @param hypervisorType the hypervisorType to set */ diff --git a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java index 4b25c63403e2..718511746c2f 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java +++ b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java @@ -35,4 +35,6 @@ public interface AutoScaleVmGroupVmMapDao extends GenericDao vmIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java index 8fca4c26f9a7..1ae55d97da2c 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java @@ -18,7 +18,10 @@ import java.util.List; +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.network.as.AutoScaleVmGroupVmMapVO; @@ -31,9 +34,6 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.VMInstanceDao; -import javax.annotation.PostConstruct; -import javax.inject.Inject; - @Component public class AutoScaleVmGroupVmMapDaoImpl extends GenericDaoBase implements AutoScaleVmGroupVmMapDao { @@ -115,4 +115,16 @@ public boolean removeByGroup(long vmGroupId) { sc.setParameters("vmGroupId", vmGroupId); return remove(sc) >= 0; } + + @Override + public int expungeByVmList(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(vmIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getInstanceId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDao.java b/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDao.java index b1b1e1cf7571..3f8c36ac94ed 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDao.java @@ -105,4 +105,6 @@ public interface IPAddressDao extends GenericDao { void buildQuarantineSearchCriteria(SearchCriteria sc); IPAddressVO findBySourceNetworkIdAndDatacenterIdAndState(long sourceNetworkId, long dataCenterId, State state); + + int expungeByVmList(List vmIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDaoImpl.java index ca779f7e9cee..aa143838c343 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDaoImpl.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.resourcedetail.dao.UserIpAddressDetailsDao; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.dc.Vlan.VlanType; @@ -561,4 +562,16 @@ public IPAddressVO findBySourceNetworkIdAndDatacenterIdAndState(long sourceNetwo sc.setParameters("state", State.Free); return findOneBy(sc); } + + @Override + public int expungeByVmList(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(vmIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getAssociatedWithVmId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/InlineLoadBalancerNicMapDao.java b/engine/schema/src/main/java/com/cloud/network/dao/InlineLoadBalancerNicMapDao.java index ac3845beffe4..b1831b407a41 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/InlineLoadBalancerNicMapDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/InlineLoadBalancerNicMapDao.java @@ -16,10 +16,14 @@ // under the License. package com.cloud.network.dao; +import java.util.List; + import com.cloud.utils.db.GenericDao; public interface InlineLoadBalancerNicMapDao extends GenericDao { InlineLoadBalancerNicMapVO findByPublicIpAddress(String publicIpAddress); InlineLoadBalancerNicMapVO findByNicId(long nicId); + int expungeByNicList(List nicIds, Long batchSize); + } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/InlineLoadBalancerNicMapDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/InlineLoadBalancerNicMapDaoImpl.java index 1c3f231f9c1d..d64ba8b4155f 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/InlineLoadBalancerNicMapDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/InlineLoadBalancerNicMapDaoImpl.java @@ -17,9 +17,13 @@ package com.cloud.network.dao; +import java.util.List; + +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @Component @@ -41,4 +45,15 @@ public InlineLoadBalancerNicMapVO findByNicId(long nicId) { return findOneBy(sc); } + @Override + public int expungeByNicList(List nicIds, Long batchSize) { + if (CollectionUtils.isEmpty(nicIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("nicIds", sb.entity().getNicId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("nicIds", nicIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/LoadBalancerVMMapDao.java b/engine/schema/src/main/java/com/cloud/network/dao/LoadBalancerVMMapDao.java index a25534b7010f..be2941d5cb2f 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/LoadBalancerVMMapDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/LoadBalancerVMMapDao.java @@ -42,4 +42,5 @@ public interface LoadBalancerVMMapDao extends GenericDao vmIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/LoadBalancerVMMapDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/LoadBalancerVMMapDaoImpl.java index b32320a84cb0..dc37cdeefe3d 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/LoadBalancerVMMapDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/LoadBalancerVMMapDaoImpl.java @@ -18,11 +18,12 @@ import java.util.List; - +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Func; @@ -135,4 +136,16 @@ public List listByLoadBalancerIdAndVmId(long loadBalancerId sc.addAnd("instanceId", SearchCriteria.Op.EQ, instanceId); return listBy(sc); } + + @Override + public int expungeByVmList(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(vmIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getInstanceId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/OpRouterMonitorServiceDao.java b/engine/schema/src/main/java/com/cloud/network/dao/OpRouterMonitorServiceDao.java index ebc0f1af2271..0516e26e13a7 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/OpRouterMonitorServiceDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/OpRouterMonitorServiceDao.java @@ -18,8 +18,12 @@ package com.cloud.network.dao; +import java.util.List; + import com.cloud.utils.db.GenericDao; public interface OpRouterMonitorServiceDao extends GenericDao { + int expungeByVmList(List vmIds, Long batchSize); + } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/OpRouterMonitorServiceDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/OpRouterMonitorServiceDaoImpl.java index 451320ac9b6c..a8e818cfb189 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/OpRouterMonitorServiceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/OpRouterMonitorServiceDaoImpl.java @@ -17,10 +17,27 @@ package com.cloud.network.dao; -import com.cloud.utils.db.GenericDaoBase; +import java.util.List; + +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + @Component public class OpRouterMonitorServiceDaoImpl extends GenericDaoBase implements OpRouterMonitorServiceDao { + @Override + public int expungeByVmList(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(vmIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDao.java b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDao.java index b89d04ad15a0..8cd114b7fc4f 100644 --- a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDao.java +++ b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDao.java @@ -47,4 +47,5 @@ public interface PortForwardingRulesDao extends GenericDao listByNetworkAndDestIpAddr(String ip4Address, long networkId); + int expungeByVmList(List vmIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java index 29cba516d720..3a404b3f2df3 100644 --- a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java @@ -20,6 +20,7 @@ import javax.inject.Inject; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.network.dao.FirewallRulesCidrsDao; @@ -170,4 +171,16 @@ public PortForwardingRuleVO findByIdAndIp(long id, String secondaryIp) { sc.setParameters("dstIp", secondaryIp); return findOneBy(sc); } + + @Override + public int expungeByVmList(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(vmIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getVirtualMachineId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDao.java b/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDao.java index 98fc8c8687b8..5023aaa3794c 100644 --- a/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDao.java +++ b/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDao.java @@ -17,10 +17,12 @@ package com.cloud.secstorage; import java.util.Date; +import java.util.List; import com.cloud.utils.db.GenericDao; public interface CommandExecLogDao extends GenericDao { public void expungeExpiredRecords(Date cutTime); public Integer getCopyCmdCountForSSVM(Long id); + int expungeByVmList(List vmIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDaoImpl.java b/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDaoImpl.java index f89a1bbf4ccb..a37acdf60298 100644 --- a/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDaoImpl.java @@ -19,6 +19,7 @@ import java.util.Date; import java.util.List; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.utils.db.GenericDaoBase; @@ -57,4 +58,16 @@ public Integer getCopyCmdCountForSSVM(Long id) { List copyCmds = customSearch(sc, null); return copyCmds.size(); } + + @Override + public int expungeByVmList(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(vmIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getInstanceId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDao.java b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDao.java index d086ad1dac1f..48e63d8e2b55 100644 --- a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDao.java +++ b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDao.java @@ -54,7 +54,7 @@ List createSystemServiceOfferings(String name, String uniqueN List listPublicByCpuAndMemory(Integer cpus, Integer memory); - ServiceOfferingVO findServiceOfferingByComputeOnlyDiskOffering(long diskOfferingId); - List listByHostTag(String tag); + + ServiceOfferingVO findServiceOfferingByComputeOnlyDiskOffering(long diskOfferingId, boolean includingRemoved); } diff --git a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDaoImpl.java b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDaoImpl.java index 34ac7c47521d..706dcdc1b7b5 100644 --- a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDaoImpl.java @@ -282,10 +282,10 @@ public List listPublicByCpuAndMemory(Integer cpus, Integer me } @Override - public ServiceOfferingVO findServiceOfferingByComputeOnlyDiskOffering(long diskOfferingId) { + public ServiceOfferingVO findServiceOfferingByComputeOnlyDiskOffering(long diskOfferingId, boolean includingRemoved) { SearchCriteria sc = SearchComputeOfferingByComputeOnlyDiskOffering.create(); sc.setParameters("disk_offering_id", diskOfferingId); - List vos = listBy(sc); + List vos = includingRemoved ? listIncludingRemovedBy(sc) : listBy(sc); if (vos.size() == 0) { return null; } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDao.java index 998d0bbd724c..171634fb1044 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDao.java @@ -57,4 +57,5 @@ public interface SnapshotDao extends GenericDao, StateDao listByIds(Object... ids); + List searchByVolumes(List volumeIds); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java index 030d10d66827..f5fc9c47d036 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java @@ -18,11 +18,13 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.util.ArrayList; import java.util.List; import javax.annotation.PostConstruct; import javax.inject.Inject; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.server.ResourceTag.ResourceObjectType; @@ -285,4 +287,16 @@ public List listByStatusNotIn(long volumeId, Snapshot.State... statu sc.setParameters("status", (Object[]) status); return listBy(sc, null); } + + @Override + public List searchByVolumes(List volumeIds) { + if (CollectionUtils.isEmpty(volumeIds)) { + return new ArrayList<>(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("volumeIds", sb.entity().getVolumeId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("volumeIds", volumeIds.toArray()); + return search(sc, null); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java index 43bb5b3d4d53..02a0355d92dd 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java @@ -18,9 +18,12 @@ */ package com.cloud.storage.dao; +import java.util.List; + import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; import com.cloud.utils.db.GenericDao; public interface SnapshotDetailsDao extends GenericDao, ResourceDetailsDao { + public List findDetailsByZoneAndKey(long dcId, String key); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java index e4ae22cd021a..584a24817264 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java @@ -18,11 +18,44 @@ */ package com.cloud.storage.dao; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.exception.CloudRuntimeException; + public class SnapshotDetailsDaoImpl extends ResourceDetailsDaoBase implements SnapshotDetailsDao { + private static final String GET_SNAPSHOT_DETAILS_ON_ZONE = "SELECT s.* FROM snapshot_details s LEFT JOIN snapshots ss ON ss.id=s.snapshot_id WHERE ss.data_center_id = ? AND s.name = ?"; + @Override public void addDetail(long resourceId, String key, String value, boolean display) { super.addDetail(new SnapshotDetailsVO(resourceId, key, value, display)); } + + public List findDetailsByZoneAndKey(long dcId, String key) { + StringBuilder sql = new StringBuilder(GET_SNAPSHOT_DETAILS_ON_ZONE); + TransactionLegacy txn = TransactionLegacy.currentTxn(); + List snapshotDetailsOnZone = new ArrayList(); + try (PreparedStatement pstmt = txn.prepareStatement(sql.toString());) { + if (pstmt != null) { + pstmt.setLong(1, dcId); + pstmt.setString(2, key); + try (ResultSet rs = pstmt.executeQuery();) { + while (rs.next()) { + snapshotDetailsOnZone.add(toEntityBean(rs, false)); + } + } catch (SQLException e) { + throw new CloudRuntimeException("Could not find details by given zone and key due to:" + e.getMessage(), e); + } + } + return snapshotDetailsOnZone; + } catch (SQLException e) { + throw new CloudRuntimeException("Could not find details by given zone and key due to:" + e.getMessage(), e); + } + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDao.java index b099a6d6bdbb..62ef5b7570d1 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDao.java @@ -41,4 +41,6 @@ public interface StoragePoolHostDao extends GenericDao public void deleteStoragePoolHostDetails(long hostId, long poolId); List listByHostId(long hostId); + + Pair, Integer> listByPoolIdNotInCluster(long clusterId, long poolId); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDaoImpl.java index 9e7bdca11817..987a42f410e7 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDaoImpl.java @@ -23,12 +23,18 @@ import java.util.List; import java.util.stream.Collectors; +import javax.annotation.PostConstruct; +import javax.inject.Inject; + import org.springframework.stereotype.Component; +import com.cloud.host.HostVO; import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; import com.cloud.storage.StoragePoolHostVO; import com.cloud.utils.Pair; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; @@ -40,6 +46,11 @@ public class StoragePoolHostDaoImpl extends GenericDaoBase HostSearch; protected final SearchBuilder PoolHostSearch; + protected SearchBuilder poolNotInClusterSearch; + + @Inject + HostDao hostDao; + protected static final String HOST_FOR_POOL_SEARCH = "SELECT * FROM storage_pool_host_ref ph, host h where ph.host_id = h.id and ph.pool_id=? and h.status=? "; protected static final String HOSTS_FOR_POOLS_SEARCH = "SELECT DISTINCT(ph.host_id) FROM storage_pool_host_ref ph, host h WHERE ph.host_id = h.id AND h.status = 'Up' AND resource_state = 'Enabled' AND ph.pool_id IN (?)"; @@ -68,6 +79,15 @@ public StoragePoolHostDaoImpl() { } + @PostConstruct + public void init(){ + poolNotInClusterSearch = createSearchBuilder(); + poolNotInClusterSearch.and("poolId", poolNotInClusterSearch.entity().getPoolId(), SearchCriteria.Op.EQ); + SearchBuilder hostSearch = hostDao.createSearchBuilder(); + poolNotInClusterSearch.join("hostSearch", hostSearch, hostSearch.entity().getId(), poolNotInClusterSearch.entity().getHostId(), JoinBuilder.JoinType.INNER); + hostSearch.and("clusterId", hostSearch.entity().getClusterId(), SearchCriteria.Op.NEQ); + } + @Override public List listByPoolId(long id) { SearchCriteria sc = PoolSearch.create(); @@ -194,4 +214,12 @@ public void deleteStoragePoolHostDetails(long hostId, long poolId) { remove(sc); txn.commit(); } + + @Override + public Pair, Integer> listByPoolIdNotInCluster(long clusterId, long poolId) { + SearchCriteria sc = poolNotInClusterSearch.create(); + sc.setParameters("poolId", poolId); + sc.setJoinParameters("hostSearch", "clusterId", clusterId); + return searchAndCount(sc, null); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index 4e9c63699ca9..e6ffca06f9e0 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -112,7 +112,8 @@ public interface VolumeDao extends GenericDao, StateDao virtualRouters); @@ -158,4 +159,7 @@ public interface VolumeDao extends GenericDao, StateDao listAllocatedVolumesForAccountDiskOfferingIdsAndNotForVms(long accountId, List diskOfferingIds, List vmIds); + List searchRemovedByVms(List vmIds, Long batchSize); + + VolumeVO findOneByIScsiName(String iScsiName); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index 31d64daf147c..0c4d707635aa 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -27,14 +27,12 @@ import javax.inject.Inject; -import com.cloud.configuration.Resource; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallback; import org.apache.cloudstack.reservation.ReservationVO; import org.apache.cloudstack.reservation.dao.ReservationDao; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; +import com.cloud.configuration.Resource; import com.cloud.exception.InvalidParameterValueException; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.server.ResourceTag.ResourceObjectType; @@ -48,12 +46,15 @@ import com.cloud.tags.dao.ResourceTagDao; import com.cloud.utils.Pair; import com.cloud.utils.db.DB; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Func; import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.UpdateBuilder; import com.cloud.utils.exception.CloudRuntimeException; @@ -76,12 +77,12 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol protected GenericSearchBuilder primaryStorageSearch2; protected GenericSearchBuilder secondaryStorageSearch; private final SearchBuilder poolAndPathSearch; + @Inject ReservationDao reservationDao; @Inject - ResourceTagDao _tagsDao; + ResourceTagDao tagsDao; - protected static final String SELECT_VM_SQL = "SELECT DISTINCT instance_id from volumes v where v.host_id = ? and v.mirror_state = ?"; // need to account for zone-wide primary storage where storage_pool has // null-value pod and cluster, where hypervisor information is stored in // storage_pool @@ -395,6 +396,7 @@ public VolumeDaoImpl() { AllFieldsSearch.and("updatedCount", AllFieldsSearch.entity().getUpdatedCount(), Op.EQ); AllFieldsSearch.and("name", AllFieldsSearch.entity().getName(), Op.EQ); AllFieldsSearch.and("passphraseId", AllFieldsSearch.entity().getPassphraseId(), Op.EQ); + AllFieldsSearch.and("iScsiName", AllFieldsSearch.entity().get_iScsiName(), Op.EQ); AllFieldsSearch.done(); RootDiskStateSearch = createSearchBuilder(); @@ -502,7 +504,6 @@ public VolumeDaoImpl() { poolAndPathSearch.and("poolId", poolAndPathSearch.entity().getPoolId(), Op.EQ); poolAndPathSearch.and("path", poolAndPathSearch.entity().getPath(), Op.EQ); poolAndPathSearch.done(); - } @Override @@ -740,7 +741,7 @@ public boolean remove(Long id) { logger.debug(String.format("Removing volume %s from DB", id)); VolumeVO entry = findById(id); if (entry != null) { - _tagsDao.removeByIdAndType(id, ResourceObjectType.Volume); + tagsDao.removeByIdAndType(id, ResourceObjectType.Volume); } boolean result = super.remove(id); @@ -763,7 +764,7 @@ public boolean updateUuid(long srcVolId, long destVolId) { destVol.setInstanceId(instanceId); update(srcVolId, srcVol); update(destVolId, destVol); - _tagsDao.updateResourceId(srcVolId, destVolId, ResourceObjectType.Volume); + tagsDao.updateResourceId(srcVolId, destVolId, ResourceObjectType.Volume); } catch (Exception e) { throw new CloudRuntimeException("Unable to persist the sequence number for this host"); } @@ -896,4 +897,24 @@ public VolumeVO persist(VolumeVO entity) { return volume; }); } + + @Override + public List searchRemovedByVms(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(vmIds)) { + return new ArrayList<>(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getInstanceId(), SearchCriteria.Op.IN); + sb.and("removed", sb.entity().getRemoved(), SearchCriteria.Op.NNULL); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + Filter filter = new Filter(VolumeVO.class, "id", true, 0L, batchSize); + return searchIncludingRemoved(sc, filter, null, false); + } + + public VolumeVO findOneByIScsiName(String iScsiName) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("iScsiName", iScsiName); + return findOneIncludingRemovedBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDao.java index ff6af56c9c30..b4a596dfc8d5 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDao.java @@ -75,8 +75,10 @@ public interface VolumeStatsDao extends GenericDao { /** * Removes (expunges) all Volume stats with {@code timestamp} less than * a given Date. - * @param limit the maximum date to keep stored. Records that exceed this limit will be removed. + * @param limitDate the maximum date to keep stored. Records that exceed this limit will be removed. + * @param limitPerQuery the maximum amount of rows to be removed in a single query. We loop if there are still rows to be removed after a given query. + * If 0 or negative, no limit is used. */ - void removeAllByTimestampLessThan(Date limit); + void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java index 5d0d3c8921cf..d1149e474085 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java @@ -21,6 +21,8 @@ import javax.annotation.PostConstruct; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; import com.cloud.utils.db.Filter; @@ -33,6 +35,8 @@ @Component public class VolumeStatsDaoImpl extends GenericDaoBase implements VolumeStatsDao { + protected Logger logger = LogManager.getLogger(getClass()); + protected SearchBuilder volumeIdSearch; protected SearchBuilder volumeIdTimestampGreaterThanEqualSearch; protected SearchBuilder volumeIdTimestampLessThanEqualSearch; @@ -116,9 +120,21 @@ public void removeAllByVolumeId(long volumeId) { } @Override - public void removeAllByTimestampLessThan(Date limit) { + public void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery) { SearchCriteria sc = timestampSearch.create(); - sc.setParameters(TIMESTAMP, limit); - expunge(sc); + sc.setParameters(TIMESTAMP, limitDate); + + logger.debug(String.format("Starting to remove all volume_stats rows older than [%s].", limitDate)); + + long totalRemoved = 0; + long removed; + + do { + removed = expunge(sc, limitPerQuery); + totalRemoved += removed; + logger.trace(String.format("Removed [%s] volume_stats rows on the last update and a sum of [%s] volume_stats rows older than [%s] until now.", removed, totalRemoved, limitDate)); + } while (limitPerQuery > 0 && removed >= limitPerQuery); + + logger.info(String.format("Removed a total of [%s] volume_stats rows older than [%s].", totalRemoved, limitDate)); } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index ea8ce47611a2..d390a480e414 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -33,7 +33,6 @@ import javax.inject.Inject; -import com.cloud.upgrade.dao.Upgrade41900to42000; import com.cloud.utils.FileUtil; import org.apache.cloudstack.utils.CloudStackVersion; import org.apache.commons.lang3.StringUtils; @@ -87,6 +86,8 @@ import com.cloud.upgrade.dao.Upgrade41720to41800; import com.cloud.upgrade.dao.Upgrade41800to41810; import com.cloud.upgrade.dao.Upgrade41810to41900; +import com.cloud.upgrade.dao.Upgrade41900to41910; +import com.cloud.upgrade.dao.Upgrade41910to42000; import com.cloud.upgrade.dao.Upgrade420to421; import com.cloud.upgrade.dao.Upgrade421to430; import com.cloud.upgrade.dao.Upgrade430to440; @@ -226,7 +227,8 @@ public DatabaseUpgradeChecker() { .next("4.17.2.0", new Upgrade41720to41800()) .next("4.18.0.0", new Upgrade41800to41810()) .next("4.18.1.0", new Upgrade41810to41900()) - .next("4.19.0.0", new Upgrade41900to42000()) + .next("4.19.0.0", new Upgrade41900to41910()) + .next("4.19.1.0", new Upgrade41910to42000()) .build(); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41900to42000.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to42000.java similarity index 91% rename from engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41900to42000.java rename to engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to42000.java index 200c5fda2326..d8c7684f8e3a 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41900to42000.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to42000.java @@ -22,12 +22,12 @@ import com.cloud.upgrade.SystemVmTemplateRegistration; import com.cloud.utils.exception.CloudRuntimeException; -public class Upgrade41900to42000 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { +public class Upgrade41910to42000 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { private SystemVmTemplateRegistration systemVmTemplateRegistration; @Override public String[] getUpgradableVersionRange() { - return new String[] {"4.19.0.0", "4.20.0.0"}; + return new String[] {"4.19.1.0", "4.20.0.0"}; } @Override @@ -42,7 +42,7 @@ public boolean supportsRollingUpgrade() { @Override public InputStream[] getPrepareScripts() { - final String scriptFile = "META-INF/db/schema-41900to42000.sql"; + final String scriptFile = "META-INF/db/schema-41910to42000.sql"; final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile); if (script == null) { throw new CloudRuntimeException("Unable to find " + scriptFile); @@ -57,7 +57,7 @@ public void performDataMigration(Connection conn) { @Override public InputStream[] getCleanupScripts() { - final String scriptFile = "META-INF/db/schema-41900to42000-cleanup.sql"; + final String scriptFile = "META-INF/db/schema-41910to42000-cleanup.sql"; final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile); if (script == null) { throw new CloudRuntimeException("Unable to find " + scriptFile); diff --git a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDao.java b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDao.java index f4534ee41ee2..514433e80689 100644 --- a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDao.java @@ -34,4 +34,6 @@ public interface AccountDetailsDao extends GenericDao { * they will get created */ void update(long accountId, Map details); + + String getActualValue(AccountDetailVO accountDetailVO); } diff --git a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java index 5451192fc6d7..de562e27f9e5 100644 --- a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java @@ -23,6 +23,7 @@ import javax.inject.Inject; +import com.cloud.utils.crypt.DBEncryptionUtil; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; @@ -40,6 +41,7 @@ import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; public class AccountDetailsDaoImpl extends GenericDaoBase implements AccountDetailsDao, ScopedConfigStorage { protected final SearchBuilder accountSearch; @@ -119,7 +121,7 @@ public Scope getScope() { public String getConfigValue(long id, ConfigKey key) { // check if account level setting is configured AccountDetailVO vo = findDetail(id, key.key()); - String value = vo == null ? null : vo.getValue(); + String value = vo == null ? null : getActualValue(vo); if (value != null) { return value; } @@ -140,7 +142,7 @@ public String getConfigValue(long id, ConfigKey key) { while (domain != null) { DomainDetailVO domainVO = _domainDetailsDao.findDetail(domain.getId(), key.key()); if (domainVO != null) { - value = domainVO.getValue(); + value = _domainDetailsDao.getActualValue(domainVO); break; } else if (domain.getParent() != null) { domain = _domainDao.findById(domain.getParent()); @@ -152,4 +154,13 @@ public String getConfigValue(long id, ConfigKey key) { } return value; } + + @Override + public String getActualValue(AccountDetailVO accountDetailVO) { + ConfigurationVO configurationVO = _configDao.findByName(accountDetailVO.getName()); + if (configurationVO != null && configurationVO.isEncrypted()) { + return DBEncryptionUtil.decrypt(accountDetailVO.getValue()); + } + return accountDetailVO.getValue(); + } } diff --git a/engine/schema/src/main/java/com/cloud/user/UserDataVO.java b/engine/schema/src/main/java/com/cloud/user/UserDataVO.java index f54b1a8872ef..a8e48ad22b1a 100644 --- a/engine/schema/src/main/java/com/cloud/user/UserDataVO.java +++ b/engine/schema/src/main/java/com/cloud/user/UserDataVO.java @@ -24,8 +24,12 @@ import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; + +import java.util.Date; import java.util.UUID; +import com.cloud.utils.db.GenericDao; + @Entity @Table(name = "user_data") public class UserDataVO implements UserData { @@ -58,6 +62,9 @@ public UserDataVO() { @Column(name = "params", length = 4096) private String params; + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + @Override public long getDomainId() { return domainId; @@ -117,4 +124,12 @@ public void setUserData(String userData) { public void setParams(String params) { this.params = params; } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public Date getRemoved() { + return removed; + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/ItWorkDao.java b/engine/schema/src/main/java/com/cloud/vm/ItWorkDao.java index 2d4a5e138fea..ab07d6989fae 100644 --- a/engine/schema/src/main/java/com/cloud/vm/ItWorkDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/ItWorkDao.java @@ -41,5 +41,6 @@ public interface ItWorkDao extends GenericDao { boolean updateStep(ItWorkVO work, Step step); List listWorkInProgressFor(long nodeId); + int expungeByVmList(List vmIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/vm/ItWorkDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/ItWorkDaoImpl.java index ff727904dcb7..0cc0a0844431 100644 --- a/engine/schema/src/main/java/com/cloud/vm/ItWorkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/ItWorkDaoImpl.java @@ -18,7 +18,7 @@ import java.util.List; - +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.utils.db.GenericDaoBase; @@ -103,4 +103,16 @@ public List listWorkInProgressFor(long nodeId) { return search(sc, null); } + + @Override + public int expungeByVmList(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(vmIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getInstanceId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/UserVmVO.java b/engine/schema/src/main/java/com/cloud/vm/UserVmVO.java index ce0bd2d57173..ce3a9a84a34f 100644 --- a/engine/schema/src/main/java/com/cloud/vm/UserVmVO.java +++ b/engine/schema/src/main/java/com/cloud/vm/UserVmVO.java @@ -148,6 +148,7 @@ public boolean isUpdateParameters() { return updateParameters; } + @Override public String getUserVmType() { return userVmType; } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDao.java index 71b1aed1938d..79158dd13b26 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDao.java @@ -23,6 +23,7 @@ import com.cloud.utils.db.GenericDao; import java.util.Date; +import java.util.List; public interface ConsoleSessionDao extends GenericDao { @@ -33,4 +34,6 @@ public interface ConsoleSessionDao extends GenericDao { int expungeSessionsOlderThanDate(Date date); void acquireSession(String sessionUuid); + + int expungeByVmList(List vmIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDaoImpl.java index 8e7e229622e8..487096744512 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDaoImpl.java @@ -20,6 +20,9 @@ package com.cloud.vm.dao; import java.util.Date; +import java.util.List; + +import org.apache.commons.collections.CollectionUtils; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; @@ -65,5 +68,15 @@ public void acquireSession(String sessionUuid) { update(consoleSessionVO.getId(), consoleSessionVO); } - + @Override + public int expungeByVmList(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(vmIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getInstanceId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java index 23c26ea07180..d34b03c4cb08 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java @@ -100,4 +100,5 @@ public interface NicDao extends GenericDao { NicVO findByIpAddressAndVmType(String ip, VirtualMachine.Type vmType); List listByNetworkIdAndType(long networkId, VirtualMachine.Type vmType); + List searchRemovedByVms(List vmIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java index 3eee1d4e749d..7d1af1982ae1 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java @@ -17,11 +17,13 @@ package com.cloud.vm.dao; import java.net.URI; +import java.util.ArrayList; import java.util.List; import javax.annotation.PostConstruct; import javax.inject.Inject; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.utils.db.Filter; @@ -428,4 +430,18 @@ public List listByNetworkIdAndType(long networkId, VirtualMachine.Type vm sc.setParameters("vmType", vmType); return listBy(sc); } + + @Override + public List searchRemovedByVms(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(vmIds)) { + return new ArrayList<>(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getInstanceId(), SearchCriteria.Op.IN); + sb.and("removed", sb.entity().getRemoved(), SearchCriteria.Op.NNULL); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + Filter filter = new Filter(NicVO.class, "id", true, 0L, batchSize); + return searchIncludingRemoved(sc, filter, null, false); + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicExtraDhcpOptionDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicExtraDhcpOptionDao.java index 69d9c00e1e0b..7bae64a6acbd 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicExtraDhcpOptionDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicExtraDhcpOptionDao.java @@ -29,4 +29,5 @@ public interface NicExtraDhcpOptionDao extends GenericDao extraDhcpOptions); + int expungeByNicList(List nicIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicExtraDhcpOptionDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicExtraDhcpOptionDaoImpl.java index 3056c73938e7..0f3679d66a37 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicExtraDhcpOptionDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicExtraDhcpOptionDaoImpl.java @@ -16,13 +16,13 @@ // under the License. package com.cloud.vm.dao; -import org.springframework.stereotype.Component; - import java.util.List; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.stereotype.Component; + import com.cloud.utils.db.DB; import com.cloud.utils.db.GenericDaoBase; - import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.vm.NicExtraDhcpOption; @@ -74,4 +74,15 @@ public void removeByNicId(long nicId) { expunge(sc); } + @Override + public int expungeByNicList(List nicIds, Long batchSize) { + if (CollectionUtils.isEmpty(nicIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("nicIds", sb.entity().getNicId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("nicIds", nicIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicSecondaryIpDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicSecondaryIpDao.java index cbb52e57282b..ff7089ca4276 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicSecondaryIpDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicSecondaryIpDao.java @@ -55,4 +55,5 @@ public interface NicSecondaryIpDao extends GenericDao { List listSecondaryIpUsingKeyword(long nicId, String keyword); int moveSecondaryIps(long fromNicId, long toNicId); + int expungeByVmList(List vmIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicSecondaryIpDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicSecondaryIpDaoImpl.java index a56d35d5a63d..563b3279520c 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicSecondaryIpDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicSecondaryIpDaoImpl.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; @@ -192,4 +193,16 @@ public int moveSecondaryIps(long fromNicId, long toNicId) { return update(update, sc); } + + @Override + public int expungeByVmList(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(vmIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getVmId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 42c00231aac1..52bc5aac7e25 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -165,4 +165,9 @@ public interface VMInstanceDao extends GenericDao, StateDao< void updateSystemVmTemplateId(long templateId, Hypervisor.HypervisorType hypervisorType); List listByHostOrLastHostOrHostPod(List hostIds, long podId); + + List searchRemovedByRemoveDate(final Date startDate, final Date endDate, final Long batchSize, + List skippedVmIds); + + Pair, Integer> listByVmsNotInClusterUsingPool(long clusterId, long poolId); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index b7b787b00451..8ab3bad69d5a 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -24,21 +24,26 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.annotation.PostConstruct; import javax.inject.Inject; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.server.ResourceTag.ResourceObjectType; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; import com.cloud.tags.dao.ResourceTagDao; import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; import com.cloud.utils.db.Attribute; import com.cloud.utils.db.DB; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.JoinBuilder; @@ -95,11 +100,16 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem protected SearchBuilder NotMigratingSearch; protected SearchBuilder BackupSearch; protected SearchBuilder LastHostAndStatesSearch; + protected SearchBuilder VmsNotInClusterUsingPool; @Inject - ResourceTagDao _tagsDao; + ResourceTagDao tagsDao; @Inject - NicDao _nicDao; + NicDao nicDao; + @Inject + VolumeDao volumeDao; + @Inject + HostDao hostDao; protected Attribute _updateTimeAttr; @@ -276,7 +286,7 @@ protected void init() { _updateTimeAttr = _allAttributes.get("updateTime"); assert _updateTimeAttr != null : "Couldn't get this updateTime attribute"; - SearchBuilder nicSearch = _nicDao.createSearchBuilder(); + SearchBuilder nicSearch = nicDao.createSearchBuilder(); nicSearch.and("networkId", nicSearch.entity().getNetworkId(), SearchCriteria.Op.EQ); nicSearch.and("removedNic", nicSearch.entity().getRemoved(), SearchCriteria.Op.NULL); @@ -305,6 +315,16 @@ protected void init() { LastHostAndStatesSearch.and("states", LastHostAndStatesSearch.entity().getState(), Op.IN); LastHostAndStatesSearch.done(); + VmsNotInClusterUsingPool = createSearchBuilder(); + SearchBuilder volumeSearch = volumeDao.createSearchBuilder(); + volumeSearch.and("poolId", volumeSearch.entity().getPoolId(), Op.EQ); + volumeSearch.and("removed", volumeSearch.entity().getRemoved(), Op.NULL); + VmsNotInClusterUsingPool.join("volumeSearch", volumeSearch, volumeSearch.entity().getInstanceId(), VmsNotInClusterUsingPool.entity().getId(), JoinType.INNER); + SearchBuilder hostSearch2 = hostDao.createSearchBuilder(); + hostSearch2.and("clusterId", hostSearch2.entity().getClusterId(), SearchCriteria.Op.NEQ); + VmsNotInClusterUsingPool.join("hostSearch2", hostSearch2, hostSearch2.entity().getId(), VmsNotInClusterUsingPool.entity().getHostId(), JoinType.INNER); + VmsNotInClusterUsingPool.and("vmStates", VmsNotInClusterUsingPool.entity().getState(), Op.IN); + VmsNotInClusterUsingPool.done(); } @Override @@ -834,7 +854,7 @@ public Long countByZoneAndStateAndHostTag(long dcId, State state, String hostTag public List listNonRemovedVmsByTypeAndNetwork(long networkId, VirtualMachine.Type... types) { if (NetworkTypeSearch == null) { - SearchBuilder nicSearch = _nicDao.createSearchBuilder(); + SearchBuilder nicSearch = nicDao.createSearchBuilder(); nicSearch.and("networkId", nicSearch.entity().getNetworkId(), SearchCriteria.Op.EQ); NetworkTypeSearch = createSearchBuilder(); @@ -871,7 +891,7 @@ public boolean remove(Long id) { txn.start(); VMInstanceVO vm = findById(id); if (vm != null && vm.getType() == Type.User) { - _tagsDao.removeByIdAndType(id, ResourceObjectType.UserVm); + tagsDao.removeByIdAndType(id, ResourceObjectType.UserVm); } boolean result = super.remove(id); txn.commit(); @@ -1016,4 +1036,36 @@ public List listByHostOrLastHostOrHostPod(List hostIds, long sc.setParameters("podId", String.valueOf(podId)); return listBy(sc); } + + @Override + public List searchRemovedByRemoveDate(Date startDate, Date endDate, Long batchSize, + List skippedVmIds) { + SearchBuilder sb = createSearchBuilder(); + sb.and("removed", sb.entity().getRemoved(), SearchCriteria.Op.NNULL); + sb.and("startDate", sb.entity().getRemoved(), SearchCriteria.Op.GTEQ); + sb.and("endDate", sb.entity().getRemoved(), SearchCriteria.Op.LTEQ); + sb.and("skippedVmIds", sb.entity().getId(), Op.NOTIN); + SearchCriteria sc = sb.create(); + if (startDate != null) { + sc.setParameters("startDate", startDate); + } + if (endDate != null) { + sc.setParameters("endDate", endDate); + } + if (CollectionUtils.isNotEmpty(skippedVmIds)) { + sc.setParameters("skippedVmIds", skippedVmIds.toArray()); + } + Filter filter = new Filter(VMInstanceVO.class, "id", true, 0L, batchSize); + return searchIncludingRemoved(sc, filter, null, false); + } + + public Pair, Integer> listByVmsNotInClusterUsingPool(long clusterId, long poolId) { + SearchCriteria sc = VmsNotInClusterUsingPool.create(); + sc.setParameters("vmStates", State.Starting, State.Running, State.Stopping, State.Migrating, State.Restoring); + sc.setJoinParameters("volumeSearch", "poolId", poolId); + sc.setJoinParameters("hostSearch2", "clusterId", clusterId); + List vms = search(sc, null); + List uniqueVms = vms.stream().distinct().collect(Collectors.toList()); + return new Pair<>(uniqueVms, uniqueVms.size()); + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDao.java index 879faaf5c901..0d7aa703a8cb 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDao.java @@ -75,8 +75,10 @@ public interface VmStatsDao extends GenericDao { /** * Removes (expunges) all VM stats with {@code timestamp} less than * a given Date. - * @param limit the maximum date to keep stored. Records that exceed this limit will be removed. + * @param limitDate the maximum date to keep stored. Records that exceed this limit will be removed. + * @param limitPerQuery the maximum amount of rows to be removed in a single query. We loop if there are still rows to be removed after a given query. + * If 0 or negative, no limit is used. */ - void removeAllByTimestampLessThan(Date limit); + void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java index 1bef8f0626c3..aa58e489364a 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java @@ -21,6 +21,8 @@ import javax.annotation.PostConstruct; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; import com.cloud.utils.db.Filter; @@ -33,6 +35,8 @@ @Component public class VmStatsDaoImpl extends GenericDaoBase implements VmStatsDao { + protected Logger logger = LogManager.getLogger(getClass()); + protected SearchBuilder vmIdSearch; protected SearchBuilder vmIdTimestampGreaterThanEqualSearch; protected SearchBuilder vmIdTimestampLessThanEqualSearch; @@ -113,10 +117,22 @@ public void removeAllByVmId(long vmId) { } @Override - public void removeAllByTimestampLessThan(Date limit) { + public void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery) { SearchCriteria sc = timestampSearch.create(); - sc.setParameters("timestamp", limit); - expunge(sc); + sc.setParameters("timestamp", limitDate); + + logger.debug(String.format("Starting to remove all vm_stats rows older than [%s].", limitDate)); + + long totalRemoved = 0; + long removed; + + do { + removed = expunge(sc, limitPerQuery); + totalRemoved += removed; + logger.trace(String.format("Removed [%s] vm_stats rows on the last update and a sum of [%s] vm_stats rows older than [%s] until now.", removed, totalRemoved, limitDate)); + } while (limitPerQuery > 0 && removed >= limitPerQuery); + + logger.info(String.format("Removed a total of [%s] vm_stats rows older than [%s].", totalRemoved, limitDate)); } } diff --git a/engine/schema/src/main/java/com/cloud/vm/snapshot/dao/VMSnapshotDao.java b/engine/schema/src/main/java/com/cloud/vm/snapshot/dao/VMSnapshotDao.java index 31999ef15d66..0143aaa1e735 100644 --- a/engine/schema/src/main/java/com/cloud/vm/snapshot/dao/VMSnapshotDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/snapshot/dao/VMSnapshotDao.java @@ -38,4 +38,6 @@ public interface VMSnapshotDao extends GenericDao, StateDao< VMSnapshotVO findByName(Long vmId, String name); List listByAccountId(Long accountId); + List searchByVms(List vmIds); + List searchRemovedByVms(List vmIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/vm/snapshot/dao/VMSnapshotDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/snapshot/dao/VMSnapshotDaoImpl.java index 062960130aca..ab8f5f2cd849 100644 --- a/engine/schema/src/main/java/com/cloud/vm/snapshot/dao/VMSnapshotDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/snapshot/dao/VMSnapshotDaoImpl.java @@ -17,12 +17,14 @@ package com.cloud.vm.snapshot.dao; +import java.util.ArrayList; import java.util.Date; import java.util.List; - +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -180,4 +182,29 @@ public boolean updateState(State currentState, Event event, State nextState, VMS return rows > 0; } + @Override + public List searchByVms(List vmIds) { + if (CollectionUtils.isEmpty(vmIds)) { + return new ArrayList<>(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getVmId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + return search(sc, null); + } + + @Override + public List searchRemovedByVms(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(vmIds)) { + return new ArrayList<>(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getVmId(), SearchCriteria.Op.IN); + sb.and("removed", sb.entity().getRemoved(), SearchCriteria.Op.NNULL); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + Filter filter = new Filter(VMSnapshotVO.class, "id", true, 0L, batchSize); + return searchIncludingRemoved(sc, filter, null, false); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/reservation/ReservationVO.java b/engine/schema/src/main/java/org/apache/cloudstack/reservation/ReservationVO.java index df888312a92e..df0ede6821ad 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/reservation/ReservationVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/reservation/ReservationVO.java @@ -25,10 +25,14 @@ import javax.persistence.Id; import javax.persistence.Table; +import com.cloud.utils.db.GenericDao; import org.apache.cloudstack.user.ResourceReservation; import com.cloud.configuration.Resource; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.utils.identity.ManagementServerNode; + +import java.util.Date; @Entity @Table(name = "resource_reservation") @@ -57,6 +61,12 @@ public class ReservationVO implements ResourceReservation { @Column(name = "amount") long amount; + @Column(name = "mgmt_server_id") + Long managementServerId; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + protected ReservationVO() { } @@ -69,6 +79,7 @@ public ReservationVO(Long accountId, Long domainId, Resource.ResourceType resour this.resourceType = resourceType; this.tag = tag; this.amount = delta; + this.managementServerId = ManagementServerNode.getManagementServerId(); } public ReservationVO(Long accountId, Long domainId, Resource.ResourceType resourceType, Long delta) { @@ -114,4 +125,16 @@ public void setResourceId(long resourceId) { this.resourceId = resourceId; } + @Override + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public Long getManagementServerId() { + return managementServerId; + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDao.java b/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDao.java index 0433dc8c57d9..d6d494f61f92 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDao.java @@ -23,13 +23,17 @@ import com.cloud.configuration.Resource; import com.cloud.utils.db.GenericDao; +import java.util.Date; import java.util.List; public interface ReservationDao extends GenericDao { long getAccountReservation(Long account, Resource.ResourceType resourceType, String tag); long getDomainReservation(Long domain, Resource.ResourceType resourceType, String tag); void setResourceId(Resource.ResourceType type, Long resourceId); - List getResourceIds(long accountId, Resource.ResourceType type); List getReservationsForAccount(long accountId, Resource.ResourceType type, String tag); void removeByIds(List reservationIds); + + int removeByMsId(long managementServerId); + + int removeStaleReservations(Long accountId, Resource.ResourceType resourceType, String tag, Date createdBefore); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDaoImpl.java index 8d6e0b6eee0e..3b17f4e4294e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDaoImpl.java @@ -18,8 +18,8 @@ // package org.apache.cloudstack.reservation.dao; +import java.util.Date; import java.util.List; -import java.util.stream.Collectors; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.reservation.ReservationVO; @@ -42,6 +42,8 @@ public class ReservationDaoImpl extends GenericDaoBase impl private static final String ACCOUNT_ID = "accountId"; private static final String DOMAIN_ID = "domainId"; private static final String IDS = "ids"; + private static final String MS_ID = "managementServerId"; + private static final String CREATED = "created"; private final SearchBuilder listResourceByAccountAndTypeSearch; private final SearchBuilder listAccountAndTypeSearch; private final SearchBuilder listAccountAndTypeAndNoTagSearch; @@ -50,6 +52,7 @@ public class ReservationDaoImpl extends GenericDaoBase impl private final SearchBuilder listDomainAndTypeAndNoTagSearch; private final SearchBuilder listResourceByAccountAndTypeAndNoTagSearch; private final SearchBuilder listIdsSearch; + private final SearchBuilder listMsIdSearch; public ReservationDaoImpl() { @@ -71,12 +74,14 @@ public ReservationDaoImpl() { listAccountAndTypeSearch.and(ACCOUNT_ID, listAccountAndTypeSearch.entity().getAccountId(), SearchCriteria.Op.EQ); listAccountAndTypeSearch.and(RESOURCE_TYPE, listAccountAndTypeSearch.entity().getResourceType(), SearchCriteria.Op.EQ); listAccountAndTypeSearch.and(RESOURCE_TAG, listAccountAndTypeSearch.entity().getTag(), SearchCriteria.Op.EQ); + listAccountAndTypeSearch.and(CREATED, listAccountAndTypeSearch.entity().getCreated(), SearchCriteria.Op.LT); listAccountAndTypeSearch.done(); listAccountAndTypeAndNoTagSearch = createSearchBuilder(); listAccountAndTypeAndNoTagSearch.and(ACCOUNT_ID, listAccountAndTypeAndNoTagSearch.entity().getAccountId(), SearchCriteria.Op.EQ); listAccountAndTypeAndNoTagSearch.and(RESOURCE_TYPE, listAccountAndTypeAndNoTagSearch.entity().getResourceType(), SearchCriteria.Op.EQ); listAccountAndTypeAndNoTagSearch.and(RESOURCE_TAG, listAccountAndTypeAndNoTagSearch.entity().getTag(), SearchCriteria.Op.NULL); + listAccountAndTypeAndNoTagSearch.and(CREATED, listAccountAndTypeAndNoTagSearch.entity().getCreated(), SearchCriteria.Op.LT); listAccountAndTypeAndNoTagSearch.done(); listDomainAndTypeSearch = createSearchBuilder(); @@ -94,18 +99,24 @@ public ReservationDaoImpl() { listIdsSearch = createSearchBuilder(); listIdsSearch.and(IDS, listIdsSearch.entity().getId(), SearchCriteria.Op.IN); listIdsSearch.done(); + + listMsIdSearch = createSearchBuilder(); + listMsIdSearch.and(MS_ID, listMsIdSearch.entity().getManagementServerId(), SearchCriteria.Op.EQ); + listMsIdSearch.done(); } @Override public long getAccountReservation(Long accountId, Resource.ResourceType resourceType, String tag) { long total = 0; - SearchCriteria sc = tag == null ? - listAccountAndTypeAndNoTagSearch.create() : listAccountAndTypeSearch.create(); - sc.setParameters(ACCOUNT_ID, accountId); - sc.setParameters(RESOURCE_TYPE, resourceType); - if (tag != null) { + SearchCriteria sc; + if (tag == null) { + sc = listAccountAndTypeAndNoTagSearch.create(); + } else { + sc = listAccountAndTypeSearch.create(); sc.setParameters(RESOURCE_TAG, tag); } + sc.setParameters(ACCOUNT_ID, accountId); + sc.setParameters(RESOURCE_TYPE, resourceType); List reservations = listBy(sc); for (ReservationVO reservation : reservations) { total += reservation.getReservedAmount(); @@ -116,13 +127,15 @@ public long getAccountReservation(Long accountId, Resource.ResourceType resource @Override public long getDomainReservation(Long domainId, Resource.ResourceType resourceType, String tag) { long total = 0; - SearchCriteria sc = tag == null ? - listDomainAndTypeAndNoTagSearch.create() : listDomainAndTypeSearch.create(); - sc.setParameters(DOMAIN_ID, domainId); - sc.setParameters(RESOURCE_TYPE, resourceType); - if (tag != null) { + SearchCriteria sc; + if (tag == null) { + sc = listDomainAndTypeAndNoTagSearch.create(); + } else { + sc = listDomainAndTypeSearch.create(); sc.setParameters(RESOURCE_TAG, tag); } + sc.setParameters(DOMAIN_ID, domainId); + sc.setParameters(RESOURCE_TYPE, resourceType); List reservations = listBy(sc); for (ReservationVO reservation : reservations) { total += reservation.getReservedAmount(); @@ -149,23 +162,17 @@ public void setResourceId(Resource.ResourceType type, Long resourceId) { } } - @Override - public List getResourceIds(long accountId, Resource.ResourceType type) { - SearchCriteria sc = listResourceByAccountAndTypeSearch.create(); - sc.setParameters(ACCOUNT_ID, accountId); - sc.setParameters(RESOURCE_TYPE, type); - return listBy(sc).stream().map(ReservationVO::getResourceId).collect(Collectors.toList()); - } - @Override public List getReservationsForAccount(long accountId, Resource.ResourceType type, String tag) { - SearchCriteria sc = tag == null ? - listResourceByAccountAndTypeAndNoTagSearch.create() : listResourceByAccountAndTypeSearch.create(); - sc.setParameters(ACCOUNT_ID, accountId); - sc.setParameters(RESOURCE_TYPE, type); - if (tag != null) { + SearchCriteria sc; + if (tag == null) { + sc = listResourceByAccountAndTypeAndNoTagSearch.create(); + } else { + sc = listResourceByAccountAndTypeSearch.create(); sc.setParameters(RESOURCE_TAG, tag); } + sc.setParameters(ACCOUNT_ID, accountId); + sc.setParameters(RESOURCE_TYPE, type); return listBy(sc); } @@ -177,4 +184,28 @@ public void removeByIds(List reservationIds) { remove(sc); } } + + @Override + public int removeByMsId(long managementServerId) { + SearchCriteria sc = listMsIdSearch.create(); + sc.setParameters(MS_ID, managementServerId); + return remove(sc); + } + + @Override + public int removeStaleReservations(Long accountId, Resource.ResourceType resourceType, String tag, + Date createdBefore) { + SearchCriteria sc; + if (tag == null) { + sc = listAccountAndTypeAndNoTagSearch.create(); + } else { + sc = listAccountAndTypeSearch.create(); + sc.setParameters(RESOURCE_TAG, tag); + } + sc.setParameters(ACCOUNT_ID, accountId); + sc.setParameters(RESOURCE_TYPE, resourceType); + sc.setParameters(CREATED, createdBefore); + return remove(sc); + } + } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDao.java b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDao.java index 5a173191be1f..8f3d264da981 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDao.java @@ -97,4 +97,6 @@ public interface ResourceDetailsDao extends GenericDao public void addDetail(long resourceId, String key, String value, boolean display); public List findResourceIdsByNameAndValueIn(String name, Object[] values); + + public long batchExpungeForResources(List ids, Long batchSize); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java index 37ebfebf5ddd..4205a7823e44 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java @@ -21,13 +21,14 @@ import java.util.Map; import org.apache.cloudstack.api.ResourceDetail; +import org.apache.commons.collections.CollectionUtils; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.TransactionLegacy; public abstract class ResourceDetailsDaoBase extends GenericDaoBase implements ResourceDetailsDao { private SearchBuilder AllFieldsSearch; @@ -201,4 +202,17 @@ public List findResourceIdsByNameAndValueIn(String name, Object[] values) return customSearch(sc, null); } + + @Override + public long batchExpungeForResources(final List ids, final Long batchSize) { + if (CollectionUtils.isEmpty(ids)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("ids", sb.entity().getResourceId(), Op.IN); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("ids", ids.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index d42e863cbed9..f0c235e842c1 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -127,6 +127,10 @@ public interface PrimaryDataStoreDao extends GenericDao { List findZoneWideStoragePoolsByHypervisor(long dataCenterId, HypervisorType hypervisorType, String keyword); + List findZoneWideStoragePoolsByHypervisorAndPoolType(long dataCenterId, HypervisorType hypervisorType, Storage.StoragePoolType poolType); + + List findClusterWideStoragePoolsByHypervisorAndPoolType(long clusterId, HypervisorType hypervisorType, Storage.StoragePoolType poolType); + List findLocalStoragePoolsByHostAndTags(long hostId, String[] tags); List listLocalStoragePoolByPath(long datacenterId, String path); @@ -141,6 +145,8 @@ public interface PrimaryDataStoreDao extends GenericDao { List findPoolsByStorageType(Storage.StoragePoolType storageType); + StoragePoolVO findPoolByZoneAndPath(long zoneId, String datastorePath); + List listStoragePoolsWithActiveVolumesByOfferingId(long offeringid); Pair, Integer> searchForIdsAndCount(Long storagePoolId, String storagePoolName, Long zoneId, diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index e4dd66a86b81..1658fe0a537e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -28,6 +28,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.Storage; import com.cloud.utils.Pair; import com.cloud.utils.db.Filter; import org.apache.commons.collections.CollectionUtils; @@ -35,7 +36,6 @@ import com.cloud.host.Status; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.ScopeType; -import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.StoragePoolTagVO; @@ -622,6 +622,28 @@ public List findZoneWideStoragePoolsByHypervisor(long dataCenterI return sc.list(); } + @Override + public List findZoneWideStoragePoolsByHypervisorAndPoolType(long dataCenterId, HypervisorType hypervisorType, Storage.StoragePoolType poolType) { + QueryBuilder sc = QueryBuilder.create(StoragePoolVO.class); + sc.and(sc.entity().getDataCenterId(), Op.EQ, dataCenterId); + sc.and(sc.entity().getStatus(), Op.EQ, StoragePoolStatus.Up); + sc.and(sc.entity().getScope(), Op.EQ, ScopeType.ZONE); + sc.and(sc.entity().getHypervisor(), Op.EQ, hypervisorType); + sc.and(sc.entity().getPoolType(), Op.EQ, poolType); + return sc.list(); + } + + @Override + public List findClusterWideStoragePoolsByHypervisorAndPoolType(long clusterId, HypervisorType hypervisorType, Storage.StoragePoolType poolType) { + QueryBuilder sc = QueryBuilder.create(StoragePoolVO.class); + sc.and(sc.entity().getClusterId(), Op.EQ, clusterId); + sc.and(sc.entity().getStatus(), Op.EQ, StoragePoolStatus.Up); + sc.and(sc.entity().getScope(), Op.EQ, ScopeType.CLUSTER); + sc.and(sc.entity().getHypervisor(), Op.EQ, hypervisorType); + sc.and(sc.entity().getPoolType(), Op.EQ, poolType); + return sc.list(); + } + @Override public void deletePoolTags(long poolId) { _tagsDao.deleteTags(poolId); @@ -660,6 +682,16 @@ public List findPoolsByStorageType(Storage.StoragePoolType storag return listBy(sc); } + @Override + public StoragePoolVO findPoolByZoneAndPath(long zoneId, String datastorePath) { + SearchCriteria sc = AllFieldSearch.create(); + sc.setParameters("datacenterId", zoneId); + if (datastorePath != null) { + sc.addAnd("path", Op.LIKE, "%/" + datastorePath); + } + return findOneBy(sc); + } + @Override public List listStoragePoolsWithActiveVolumesByOfferingId(long offeringId) { TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index 344ff8b2a699..4cd29b465eeb 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -106,4 +106,6 @@ public interface SnapshotDataStoreDao extends GenericDao snapshotIds, Long batchSize); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index c095f4222e76..5bf67eb38819 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -559,4 +559,16 @@ public void updateDisplayForSnapshotStoreRole(long snapshotId, long storeId, Dat ref.setDisplay(display); update(ref.getId(), ref); } + + @Override + public int expungeBySnapshotList(final List snapshotIds, final Long batchSize) { + if (CollectionUtils.isEmpty(snapshotIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("snapshotIds", sb.entity().getSnapshotId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("snapshotIds", snapshotIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java index c3a4b58fbd50..3381391e70e4 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java @@ -59,4 +59,6 @@ public interface VolumeDataStoreDao extends GenericDao, List listByVolume(long volumeId, long storeId); List listByStoreIdAndInstallPaths(Long storeId, List paths); + + int expungeByVolumeList(List volumeIds, Long batchSize); } diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index c70c6d4334e6..8ab60a766246 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -187,7 +187,6 @@ - diff --git a/engine/schema/src/main/resources/META-INF/db/data-217to218.sql b/engine/schema/src/main/resources/META-INF/db/data-217to218.sql index 5c1253143f45..1a03e9b79981 100755 --- a/engine/schema/src/main/resources/META-INF/db/data-217to218.sql +++ b/engine/schema/src/main/resources/META-INF/db/data-217to218.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-20to21.sql b/engine/schema/src/main/resources/META-INF/db/schema-20to21.sql index 7013046ca439..000d0f077cc3 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-20to21.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-20to21.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -141,14 +141,14 @@ ALTER TABLE `cloud`.`host` ADD COLUMN `cluster_id` bigint unsigned; -- -- enforced in postporcess-20to21.sql -ALTER TABLE `cloud`.`host_pod_ref` ADD COLUMN `gateway` varchar(255); -- need to migrage data with user input +ALTER TABLE `cloud`.`host_pod_ref` ADD COLUMN `gateway` varchar(255); -- need to migrage data with user input -ALTER TABLE `cloud`.`service_offering` ADD COLUMN `recreatable` tinyint(1) unsigned NOT NULL DEFAULT 0; +ALTER TABLE `cloud`.`service_offering` ADD COLUMN `recreatable` tinyint(1) unsigned NOT NULL DEFAULT 0; ALTER TABLE `cloud`.`service_offering` ADD COLUMN `tags` varchar(255); -ALTER TABLE `cloud`.`user_vm` MODIFY COLUMN `domain_router_id` bigint unsigned; -- change from NOT NULL to NULL +ALTER TABLE `cloud`.`user_vm` MODIFY COLUMN `domain_router_id` bigint unsigned; -- change from NOT NULL to NULL -ALTER TABLE `cloud`.`event` ADD COLUMN `state` varchar(32) NOT NULL DEFAULT 'Completed'; +ALTER TABLE `cloud`.`event` ADD COLUMN `state` varchar(32) NOT NULL DEFAULT 'Completed'; ALTER TABLE `cloud`.`event` ADD COLUMN `start_id` bigint unsigned NOT NULL DEFAULT 0; ALTER TABLE `cloud`.`disk_offering` ADD COLUMN `tags` varchar(4096); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-217to218.sql b/engine/schema/src/main/resources/META-INF/db/schema-217to218.sql index f2b6b291f782..006a3f1cd7b3 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-217to218.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-217to218.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-21to22-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-21to22-cleanup.sql index c8757833fabf..8a3ca39d5e7a 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-21to22-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-21to22-cleanup.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-21to22-premium.sql b/engine/schema/src/main/resources/META-INF/db/schema-21to22-premium.sql index 45202840565f..a34b65dc8ab5 100755 --- a/engine/schema/src/main/resources/META-INF/db/schema-21to22-premium.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-21to22-premium.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -50,7 +50,7 @@ CREATE TABLE `cloud_usage`.`usage_event` ( `resource_name` varchar(255), `offering_id` bigint unsigned, `template_id` bigint unsigned, - `size` bigint unsigned, + `size` bigint unsigned, `processed` tinyint NOT NULL default '0', PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-21to22.sql b/engine/schema/src/main/resources/META-INF/db/schema-21to22.sql index eb473cfc7f6b..8da29caae53b 100755 --- a/engine/schema/src/main/resources/META-INF/db/schema-21to22.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-21to22.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -22,7 +22,7 @@ ALTER TABLE `cloud`.`cluster` ADD COLUMN `guid` varchar(255) UNIQUE DEFAULT NULL ALTER TABLE `cloud`.`cluster` ADD COLUMN `cluster_type` varchar(64) DEFAULT 'CloudManaged'; ALTER TABLE `cloud`.`vm_template` ADD COLUMN `hypervisor_type` varchar(32) COMMENT 'hypervisor that the template is belonged to'; ALTER TABLE `cloud`.`vm_template` ADD COLUMN `extractable` int(1) unsigned NOT NULL default 0 COMMENT 'Is this template extractable'; -ALTER TABLE `cloud`.`template_spool_ref` ADD CONSTRAINT `fk_template_spool_ref__template_id` FOREIGN KEY (`template_id`) REFERENCES `vm_template`(`id`); +ALTER TABLE `cloud`.`template_spool_ref` ADD CONSTRAINT `fk_template_spool_ref__template_id` FOREIGN KEY (`template_id`) REFERENCES `vm_template`(`id`); ALTER TABLE `cloud`.`guest_os` modify `name` varchar(255) ; @@ -104,7 +104,7 @@ CREATE TABLE `cloud`.`networks` ( `broadcast_domain_type` varchar(32) NOT NULL COMMENT 'type of broadcast domain used', `broadcast_uri` varchar(255) COMMENT 'broadcast domain specifier', `gateway` varchar(15) COMMENT 'gateway for this network configuration', - `cidr` varchar(18) COMMENT 'network cidr', + `cidr` varchar(18) COMMENT 'network cidr', `mode` varchar(32) COMMENT 'How to retrieve ip address in this network', `network_offering_id` bigint unsigned NOT NULL COMMENT 'network offering id that this configuration is created from', `data_center_id` bigint unsigned NOT NULL COMMENT 'data center id that this configuration is used in', @@ -167,7 +167,7 @@ CREATE TABLE `cloud`.`nics` ( `ip_type` varchar(32) COMMENT 'type of ip', `broadcast_uri` varchar(255) COMMENT 'broadcast uri', `network_id` bigint unsigned NOT NULL COMMENT 'network configuration id', - `mode` varchar(32) COMMENT 'mode of getting ip address', + `mode` varchar(32) COMMENT 'mode of getting ip address', `state` varchar(32) NOT NULL COMMENT 'state of the creation', `strategy` varchar(32) NOT NULL COMMENT 'reservation strategy', `reserver_name` varchar(255) COMMENT 'Name of the component that reserved the ip address', @@ -176,7 +176,7 @@ CREATE TABLE `cloud`.`nics` ( `update_time` timestamp NOT NULL COMMENT 'time the state was changed', `isolation_uri` varchar(255) COMMENT 'id for isolation', `ip6_address` char(40) COMMENT 'ip6 address', - `default_nic` tinyint NOT NULL COMMENT "None", + `default_nic` tinyint NOT NULL COMMENT "None", `created` datetime NOT NULL COMMENT 'date created', `removed` datetime COMMENT 'date removed if not null', PRIMARY KEY (`id`), @@ -253,7 +253,7 @@ CREATE TABLE `cloud`.`op_host` ( `id` bigint unsigned NOT NULL UNIQUE COMMENT 'host id', `sequence` bigint unsigned DEFAULT 1 NOT NULL COMMENT 'sequence for the host communication', PRIMARY KEY (`id`), - CONSTRAINT `fk_op_host__id` FOREIGN KEY (`id`) REFERENCES `host`(`id`) ON DELETE CASCADE + CONSTRAINT `fk_op_host__id` FOREIGN KEY (`id`) REFERENCES `host`(`id`) ON DELETE CASCADE ) ENGINE = InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `cloud`.`guest_os_hypervisor` ( @@ -261,7 +261,7 @@ CREATE TABLE `cloud`.`guest_os_hypervisor` ( `hypervisor_type` varchar(32) NOT NULL, `guest_os_name` varchar(255) NOT NULL, `guest_os_id` bigint unsigned NOT NULL, - PRIMARY KEY (`id`) + PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; INSERT INTO op_host(id, sequence) select id, sequence from host; @@ -269,7 +269,7 @@ INSERT INTO op_host(id, sequence) select id, sequence from host; -- Alter Tables to add Columns; ALTER TABLE `cloud`.`cluster` ADD COLUMN `hypervisor_type` varchar(32); -UPDATE `cloud`.`cluster` SET hypervisor_type=(SELECT DISTINCT host.hypervisor_type from host where host.cluster_id = cluster.id GROUP BY host.hypervisor_type); +UPDATE `cloud`.`cluster` SET hypervisor_type=(SELECT DISTINCT host.hypervisor_type from host where host.cluster_id = cluster.id GROUP BY host.hypervisor_type); ALTER TABLE `cloud`.`volumes` ADD COLUMN `attached` datetime; UPDATE `cloud`.`volumes` SET attached=now() WHERE removed IS NULL AND instance_id IS NOT NULL; @@ -286,7 +286,7 @@ ALTER TABLE `cloud`.`vlan` ADD COLUMN `network_id` bigint unsigned NOT NULL; ALTER TABLE `cloud`.`data_center` ADD COLUMN `domain` varchar(100); ALTER TABLE `cloud`.`data_center` ADD COLUMN `domain_id` bigint unsigned; -ALTER TABLE `cloud`.`data_center` ADD COLUMN `networktype` varchar(255) NOT NULL DEFAULT 'Basic'; +ALTER TABLE `cloud`.`data_center` ADD COLUMN `networktype` varchar(255) NOT NULL DEFAULT 'Basic'; ALTER TABLE `cloud`.`data_center` ADD COLUMN `dns_provider` char(64) DEFAULT 'VirtualRouter'; ALTER TABLE `cloud`.`data_center` ADD COLUMN `gateway_provider` char(64) DEFAULT 'VirtualRouter'; ALTER TABLE `cloud`.`data_center` ADD COLUMN `firewall_provider` char(64) DEFAULT 'VirtualRouter'; @@ -306,7 +306,7 @@ UPDATE `cloud`.`op_dc_link_local_ip_address_alloc` SET reservation_id=concat(cas ALTER TABLE `cloud`.`host_pod_ref` ADD COLUMN `enabled` tinyint NOT NULL DEFAULT 1; ALTER TABLE `cloud`.`op_dc_vnet_alloc` ADD COLUMN `reservation_id` char(40) NULL; -UPDATE op_dc_vnet_alloc set reservation_id=concat(cast(data_center_id as CHAR), concat("-", vnet)) WHERE taken is NOT NULL; +UPDATE op_dc_vnet_alloc set reservation_id=concat(cast(data_center_id as CHAR), concat("-", vnet)) WHERE taken is NOT NULL; ALTER TABLE `cloud`.`vm_instance` ADD COLUMN `service_offering_id` bigint unsigned NOT NULL; ALTER TABLE `cloud`.`vm_instance` ADD COLUMN `reservation_id` char(40); @@ -326,7 +326,7 @@ ALTER TABLE `cloud`.`user_vm` ADD COLUMN `display_name` varchar(255); UPDATE user_vm inner join vm_instance on user_vm.id=vm_instance.id set user_vm.iso_id=vm_instance.iso_id, user_vm.display_name=vm_instance.display_name where vm_instance.type='User'; ALTER TABLE `cloud`.`template_host_ref` ADD COLUMN `physical_size` bigint unsigned DEFAULT 0; -UPDATE template_host_ref INNER JOIN template_spool_ref ON template_host_ref.template_id=template_spool_ref.template_id SET template_host_ref.physical_size=template_spool_ref.template_size; +UPDATE template_host_ref INNER JOIN template_spool_ref ON template_host_ref.template_id=template_spool_ref.template_id SET template_host_ref.physical_size=template_spool_ref.template_size; CREATE TABLE `cloud`.`user_vm_details` ( @@ -412,7 +412,7 @@ CREATE TABLE `cloud`.`vpn_users` ( CONSTRAINT `fk_vpn_users__owner_id` FOREIGN KEY (`owner_id`) REFERENCES `account`(`id`) ON DELETE CASCADE, CONSTRAINT `fk_vpn_users__domain_id` FOREIGN KEY (`domain_id`) REFERENCES `domain`(`id`) ON DELETE CASCADE, INDEX `i_vpn_users_username`(`username`), - UNIQUE `i_vpn_users__account_id__username`(`owner_id`, `username`) + UNIQUE `i_vpn_users__account_id__username`(`owner_id`, `username`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; ALTER TABLE `cloud`.`storage_pool` ADD COLUMN `status` varchar(32); @@ -490,7 +490,7 @@ CREATE TABLE `cloud`.`usage_event` ( `resource_name` varchar(255), `offering_id` bigint unsigned, `template_id` bigint unsigned, - `size` bigint unsigned, + `size` bigint unsigned, `processed` tinyint NOT NULL default '0', PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-2210to2211.sql b/engine/schema/src/main/resources/META-INF/db/schema-2210to2211.sql index 01bec020d63d..f352f5ef44d6 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-2210to2211.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-2210to2211.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-2211to2212-premium.sql b/engine/schema/src/main/resources/META-INF/db/schema-2211to2212-premium.sql index 0cb187e2b9d4..eae13acc78cb 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-2211to2212-premium.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-2211to2212-premium.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-2211to2212.sql b/engine/schema/src/main/resources/META-INF/db/schema-2211to2212.sql index 94c3d75a29ff..00d0fcaac76e 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-2211to2212.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-2211to2212.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-2212to2213.sql b/engine/schema/src/main/resources/META-INF/db/schema-2212to2213.sql index 2e86599f7924..c69809e8205b 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-2212to2213.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-2212to2213.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-2213to2214.sql b/engine/schema/src/main/resources/META-INF/db/schema-2213to2214.sql index 6c0cc4b6c708..41e3944dc180 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-2213to2214.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-2213to2214.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -29,7 +29,7 @@ CREATE TABLE `cloud`.`mshost_peer` ( `peer_runid` bigint NOT NULL, `peer_state` varchar(10) NOT NULL DEFAULT 'Down', `last_update` DATETIME NULL COMMENT 'Last record update time', - + PRIMARY KEY (`id`), CONSTRAINT `fk_mshost_peer__owner_mshost` FOREIGN KEY (`owner_mshost`) REFERENCES `mshost`(`id`) ON DELETE CASCADE, CONSTRAINT `fk_mshost_peer__peer_mshost` FOREIGN KEY (`peer_mshost`) REFERENCES `mshost`(`id`), diff --git a/engine/schema/src/main/resources/META-INF/db/schema-2214to30-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-2214to30-cleanup.sql index c90707c75b71..844280d29b64 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-2214to30-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-2214to30-cleanup.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -22,7 +22,7 @@ ALTER TABLE `cloud_usage`.`usage_network` DROP COLUMN `current_bytes_sent`; ALTER TABLE `cloud`.`template_host_ref` DROP COLUMN `pool_id`; DELETE from `cloud`.`op_host_capacity` where capacity_type in (2,4,6); -ALTER TABLE `cloud`.`vm_instance` DROP COLUMN `private_netmask`; +ALTER TABLE `cloud`.`vm_instance` DROP COLUMN `private_netmask`; ALTER TABLE `cloud`.`security_group_rule` drop foreign key `fk_security_ingress_rule___security_group_id`; ALTER TABLE `cloud`.`security_group_rule` drop foreign key `fk_security_ingress_rule___allowed_network_id`; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-2214to30.sql b/engine/schema/src/main/resources/META-INF/db/schema-2214to30.sql index 22fda616649b..8be481e47368 100755 --- a/engine/schema/src/main/resources/META-INF/db/schema-2214to30.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-2214to30.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -54,7 +54,7 @@ CREATE TABLE `cloud`.`projects` ( PRIMARY KEY (`id`), CONSTRAINT `fk_projects__project_account_id` FOREIGN KEY(`project_account_id`) REFERENCES `account`(`id`) ON DELETE CASCADE, CONSTRAINT `fk_projects__domain_id` FOREIGN KEY(`domain_id`) REFERENCES `domain`(`id`) ON DELETE CASCADE, - INDEX `i_projects__removed`(`removed`) + INDEX `i_projects__removed`(`removed`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; @@ -177,97 +177,97 @@ ALTER TABLE `cloud`.`alert` ADD `cluster_id` bigint unsigned; ALTER TABLE `cloud`.`user_statistics` ADD COLUMN `agg_bytes_received` bigint unsigned NOT NULL default '0'; ALTER TABLE `cloud`.`user_statistics` ADD COLUMN `agg_bytes_sent` bigint unsigned NOT NULL default '0'; -ALTER TABLE `cloud`.`vm_instance` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`vm_instance` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`vm_instance` ADD CONSTRAINT `uc_vm_instance_uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`async_job` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`async_job` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`async_job` ADD CONSTRAINT `uc_async__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`domain` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`domain` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`domain` ADD CONSTRAINT `uc_domain__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`account` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`account` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`account` ADD CONSTRAINT `uc_account__uuid` UNIQUE (`uuid`); ALTER TABLE `cloud_usage`.`account` ADD COLUMN `uuid` varchar(40); -ALTER TABLE `cloud`.`user` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`user` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`user` ADD CONSTRAINT `uc_user__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`projects` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`projects` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`projects` ADD CONSTRAINT `uc_projects__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`data_center` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`data_center` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`data_center` ADD CONSTRAINT `uc_data_center__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`host` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`host` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`host` ADD CONSTRAINT `uc_host__uuid` UNIQUE (`uuid`); ALTER TABLE `cloud`.`host` ADD COLUMN `update_count` bigint unsigned NOT NULL DEFAULT 0 COMMENT 'atomic increase count making status update operation atomical'; -ALTER TABLE `cloud`.`vm_template` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`vm_template` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`vm_template` ADD CONSTRAINT `uc_vm_template__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`disk_offering` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`disk_offering` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`disk_offering` ADD CONSTRAINT `uc_disk_offering__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`networks` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`networks` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`networks` ADD CONSTRAINT `uc_networks__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`security_group` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`security_group` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`security_group` ADD CONSTRAINT `uc_security_group__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`instance_group` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`instance_group` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`instance_group` ADD CONSTRAINT `uc_instance_group__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`host_pod_ref` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`host_pod_ref` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`host_pod_ref` ADD CONSTRAINT `uc_host_pod_ref__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`snapshots` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`snapshots` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`snapshots` ADD CONSTRAINT `uc_snapshots__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`snapshot_policy` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`snapshot_policy` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`snapshot_policy` ADD CONSTRAINT `uc_snapshot_policy__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`snapshot_schedule` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`snapshot_schedule` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`snapshot_schedule` ADD CONSTRAINT `uc_snapshot_schedule__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`volumes` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`volumes` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`volumes` ADD CONSTRAINT `uc_volumes__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`vlan` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`vlan` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`vlan` ADD CONSTRAINT `uc_vlan__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`user_ip_address` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`user_ip_address` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`user_ip_address` ADD CONSTRAINT `uc_user_ip_address__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`firewall_rules` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`firewall_rules` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`firewall_rules` ADD CONSTRAINT `uc_firewall_rules__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`cluster` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`cluster` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`cluster` ADD CONSTRAINT `uc_cluster__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`network_offerings` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`network_offerings` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`network_offerings` ADD CONSTRAINT `uc_network_offerings__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`hypervisor_capabilities` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`hypervisor_capabilities` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`hypervisor_capabilities` ADD CONSTRAINT `uc_hypervisor_capabilities__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`vpn_users` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`vpn_users` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`vpn_users` ADD CONSTRAINT `uc_vpn_users__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`event` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`event` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`event` ADD CONSTRAINT `uc_event__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`alert` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`alert` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`alert` ADD CONSTRAINT `uc_alert__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`guest_os` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`guest_os` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`guest_os` ADD CONSTRAINT `uc_guest_os__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`guest_os_category` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`guest_os_category` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`guest_os_category` ADD CONSTRAINT `uc_guest_os_category__uuid` UNIQUE (`uuid`); -ALTER TABLE `cloud`.`nics` ADD COLUMN `uuid` varchar(40); +ALTER TABLE `cloud`.`nics` ADD COLUMN `uuid` varchar(40); ALTER TABLE `cloud`.`nics` ADD CONSTRAINT `uc_nics__uuid` UNIQUE (`uuid`); ALTER TABLE `cloud`.`op_host_capacity` ADD COLUMN `created` datetime; @@ -304,7 +304,7 @@ ALTER TABLE `cloud_usage`.`usage_ip_address` ADD COLUMN `is_system` smallint(1) INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Premium', 'DEFAULT', 'management-server', 'usage.sanity.check.interval', null, 'Interval (in days) to check sanity of usage data'); DELETE FROM `cloud`.`configuration` WHERE name='host.capacity.checker.wait'; -DELETE FROM `cloud`.`configuration` WHERE name='host.capacity.checker.interval'; +DELETE FROM `cloud`.`configuration` WHERE name='host.capacity.checker.interval'; INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'disable.extraction' , 'false', 'Flag for disabling extraction of template, isos and volumes'); INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'NetworkManager', 'router.check.interval' , '30', 'Interval (in seconds) to report redundant router status.'); @@ -362,9 +362,9 @@ CREATE TABLE `cloud`.`physical_network` ( `name` varchar(255) NOT NULL, `data_center_id` bigint unsigned NOT NULL COMMENT 'data center id that this physical network belongs to', `vnet` varchar(255), - `speed` varchar(32), + `speed` varchar(32), `domain_id` bigint unsigned COMMENT 'foreign key to domain id', - `broadcast_domain_range` varchar(32) NOT NULL DEFAULT 'POD' COMMENT 'range of broadcast domain : POD/ZONE', + `broadcast_domain_range` varchar(32) NOT NULL DEFAULT 'POD' COMMENT 'range of broadcast domain : POD/ZONE', `state` varchar(32) NOT NULL DEFAULT 'Disabled' COMMENT 'what state is this configuration in', `created` datetime COMMENT 'date created', `removed` datetime COMMENT 'date removed if not null', @@ -372,7 +372,7 @@ CREATE TABLE `cloud`.`physical_network` ( CONSTRAINT `fk_physical_network__data_center_id` FOREIGN KEY (`data_center_id`) REFERENCES `data_center`(`id`) ON DELETE CASCADE, CONSTRAINT `fk_physical_network__domain_id` FOREIGN KEY(`domain_id`) REFERENCES `domain`(`id`), CONSTRAINT `uc_physical_networks__uuid` UNIQUE (`uuid`), - INDEX `i_physical_network__removed`(`removed`) + INDEX `i_physical_network__removed`(`removed`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `cloud`.`physical_network_tags` ( @@ -690,7 +690,7 @@ CREATE TABLE `cloud_usage`.`usage_security_group` ( `vm_instance_id` bigint unsigned NOT NULL, `security_group_id` bigint unsigned NOT NULL, `created` DATETIME NOT NULL, - `deleted` DATETIME NULL + `deleted` DATETIME NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8; ALTER TABLE `cloud_usage`.`usage_security_group` ADD INDEX `i_usage_security_group__account_id`(`account_id`); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-221to222-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-221to222-cleanup.sql index d999b9391031..5ed5f834f682 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-221to222-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-221to222-cleanup.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-221to222-premium.sql b/engine/schema/src/main/resources/META-INF/db/schema-221to222-premium.sql index 5477fd8a348b..01603a406588 100755 --- a/engine/schema/src/main/resources/META-INF/db/schema-221to222-premium.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-221to222-premium.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-221to222.sql b/engine/schema/src/main/resources/META-INF/db/schema-221to222.sql index 0c663b1ca033..d07b71fb6cbe 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-221to222.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-221to222.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -23,7 +23,7 @@ update network_offerings set firewall_service=1, lb_service=1,vpn_service=1,gate alter table domain add column `state` char(32) NOT NULL default 'Active' COMMENT 'state of the domain'; alter table nics add column `vm_type` char(32); update nics set vm_type=(select type from vm_instance where vm_instance.id=nics.instance_id); -INSERT INTO configuration (`category`, `instance`, `component`, `name`, `value`, `description`) VALUES ('Network','DEFAULT','none','network.guest.cidr.limit','22','size limit for guest cidr; cant be less than this value'); +INSERT INTO configuration (`category`, `instance`, `component`, `name`, `value`, `description`) VALUES ('Network','DEFAULT','none','network.guest.cidr.limit','22','size limit for guest cidr; cant be less than this value'); alter table user_statistics add column `network_id` bigint unsigned; update op_networks set nics_count=(nics_count-1) where id in (select d.network_id from domain_router d, vm_instance i where i.state='Running' and i.id=d.id); update network_offerings set traffic_type='Guest' where system_only=0; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-222to224-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-222to224-cleanup.sql index 1bcd5d4c5f0a..31f6eefb5fa5 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-222to224-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-222to224-cleanup.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-222to224-premium.sql b/engine/schema/src/main/resources/META-INF/db/schema-222to224-premium.sql index 9a5f62794c48..33d954d3a884 100755 --- a/engine/schema/src/main/resources/META-INF/db/schema-222to224-premium.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-222to224-premium.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-222to224.sql b/engine/schema/src/main/resources/META-INF/db/schema-222to224.sql index 8be64169b448..abbaf447bba6 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-222to224.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-222to224.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -99,7 +99,7 @@ ALTER TABLE `cloud`.`op_host_capacity` MODIFY `used_capacity` bigint signed NOT ALTER TABLE `cloud`.`op_host_capacity` MODIFY `reserved_capacity` bigint signed NOT NULL; ALTER TABLE `cloud`.`op_host_capacity` MODIFY `total_capacity` bigint signed NOT NULL; -INSERT IGNORE INTO `cloud`.`configuration` VALUES +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced','DEFAULT','management-server','control.cidr','169.254.0.0/16','Changes the cidr for the control network traffic. Defaults to using link local. Must be unique within pods'), ('Advanced','DEFAULT','management-server','control.gateway','169.254.0.1','gateway for the control network traffic'), ('Advanced','DEFAULT','AgentManager','cmd.wait','7200','Time (in seconds) to wait for some heavy time-consuming commands'), @@ -110,7 +110,7 @@ INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Console Proxy','DEFAULT','AgentManager','consoleproxy.url.domain','realhostip.com','Console proxy url domain'), ('Advanced','DEFAULT','management-server','extract.url.cleanup.interval','120','The interval (in seconds) to wait before cleaning up the extract URL\'s '), ('Network','DEFAULT','AgentManager','guest.ip.network','10.1.1.1','The network address of the guest virtual network. Virtual machines will be assigned an IP in this subnet.'), -('Network','DEFAULT','AgentManager','guest.netmask','255.255.255.0','The netmask of the guest virtual network.'), +('Network','DEFAULT','AgentManager','guest.netmask','255.255.255.0','The netmask of the guest virtual network.'), ('Network','DEFAULT','management-server','guest.vlan.bits','12','The number of bits to reserve for the VLAN identifier in the guest subnet.'), ('Advanced','DEFAULT','management-server','host.capacity.checker.interval','3600','Time (in seconds) to wait before recalculating host\'s capacity'), ('Advanced','DEFAULT','management-server','host.capacity.checker.wait','3600','Time (in seconds) to wait before starting host capacity background checker'), @@ -175,7 +175,7 @@ ALTER TABLE `cloud`.`snapshot_schedule` ADD UNIQUE KEY `volume_id` (`volume_id` ALTER TABLE `cloud`.`storage_pool` MODIFY COLUMN `uuid` varchar(255) UNIQUE; ALTER TABLE `cloud`.`user_statistics` DROP KEY `account_id`; -ALTER TABLE `cloud`.`user_statistics` ADD UNIQUE KEY `account_id` (`account_id`,`data_center_id`, `public_ip_address`, `device_id`,`device_type`); +ALTER TABLE `cloud`.`user_statistics` ADD UNIQUE KEY `account_id` (`account_id`,`data_center_id`, `public_ip_address`, `device_id`,`device_type`); ALTER TABLE `cloud`.`usage_event` ADD INDEX `i_usage_event__created`(`created`); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-224to225-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-224to225-cleanup.sql index b018d7f280e7..5f24290a1fd2 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-224to225-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-224to225-cleanup.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-224to225.sql b/engine/schema/src/main/resources/META-INF/db/schema-224to225.sql index 65334af306fb..735c30a943c0 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-224to225.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-224to225.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-225to226.sql b/engine/schema/src/main/resources/META-INF/db/schema-225to226.sql index ec1baae2e69e..3527921af840 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-225to226.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-225to226.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-227to228-premium.sql b/engine/schema/src/main/resources/META-INF/db/schema-227to228-premium.sql index 40fcbfa68380..2e21f0c249ef 100755 --- a/engine/schema/src/main/resources/META-INF/db/schema-227to228-premium.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-227to228-premium.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-227to228.sql b/engine/schema/src/main/resources/META-INF/db/schema-227to228.sql index 343c7663fd29..ac3c014894dd 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-227to228.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-227to228.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -136,9 +136,9 @@ ALTER TABLE `cloud`.`vm_instance` ADD COLUMN `vm_type` varchar(32) NOT NULL; UPDATE vm_instance set vm_type=type; ALTER TABLE `cloud`.`networks` ADD COLUMN `is_domain_specific` int(1) unsigned NOT NULL DEFAULT 0 COMMENT '1 if network is domain specific, 0 false otherwise'; -INSERT INTO configuration (`category`, `instance`, `component`, `name`, `value`, `description`) VALUES ('Advanced', 'DEFAULT', 'NetworkManager', 'allow.subdomain.network.access', 'true', 'Allow subdomains to use networks dedicated to their parent domain(s)'); +INSERT INTO configuration (`category`, `instance`, `component`, `name`, `value`, `description`) VALUES ('Advanced', 'DEFAULT', 'NetworkManager', 'allow.subdomain.network.access', 'true', 'Allow subdomains to use networks dedicated to their parent domain(s)'); -INSERT INTO configuration (`category`, `instance`, `component`, `name`, `value`, `description`) VALUES ('Advanced', 'DEFAULT', 'management-server', 'encode.api.response', 'false', 'Do UTF-8 encoding for the api response, false by default'); +INSERT INTO configuration (`category`, `instance`, `component`, `name`, `value`, `description`) VALUES ('Advanced', 'DEFAULT', 'management-server', 'encode.api.response', 'false', 'Do UTF-8 encoding for the api response, false by default'); DELETE FROM load_balancer_vm_map WHERE instance_id IN (SELECT id FROM vm_instance WHERE removed IS NOT NULL); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-228to229.sql b/engine/schema/src/main/resources/META-INF/db/schema-228to229.sql index 9d5baa4c4032..2496dd4c472f 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-228to229.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-228to229.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-229to2210.sql b/engine/schema/src/main/resources/META-INF/db/schema-229to2210.sql index 9c5c46242af8..1d2980f65640 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-229to2210.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-229to2210.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-22beta1to22beta2.sql b/engine/schema/src/main/resources/META-INF/db/schema-22beta1to22beta2.sql index 1b7c6a64eb0b..dd03e61e17e5 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-22beta1to22beta2.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-22beta1to22beta2.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-22beta3to22beta4.sql b/engine/schema/src/main/resources/META-INF/db/schema-22beta3to22beta4.sql index c73d16537acb..a93d6d5d7e53 100755 --- a/engine/schema/src/main/resources/META-INF/db/schema-22beta3to22beta4.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-22beta3to22beta4.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -83,7 +83,7 @@ CREATE TABLE `cloud`.`user_vm_details` ( `value` varchar(1024) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; - + CREATE TABLE `cloud`.`cluster_details` ( `id` bigint unsigned NOT NULL auto_increment, `cluster_id` bigint unsigned NOT NULL COMMENT 'cluster id', @@ -99,9 +99,9 @@ ALTER TABLE `cloud`.`service_offering` ADD COLUMN `host_tag` varchar(255); ALTER TABLE `cloud`.`op_it_work` change created created_at bigint unsigned NOT NULL COMMENT 'when was this work detail created'; ALTER TABLE `cloud`.`op_it_work` change state step char(32) NOT NULL COMMENT 'state'; ALTER TABLE `cloud`.`op_it_work` change cancel_taken updated_at bigint unsigned NOT NULL COMMENT 'time it was taken over'; -ALTER TABLE `cloud`.`op_it_work` ADD COLUMN `instance_id` bigint unsigned NOT NULL COMMENT 'vm instance'; -ALTER TABLE `cloud`.`op_it_work` ADD COLUMN `resource_id` bigint unsigned COMMENT 'resource id being worked on'; -ALTER TABLE `cloud`.`op_it_work` ADD COLUMN `resource_type` char(32) COMMENT 'type of resource being worked on'; +ALTER TABLE `cloud`.`op_it_work` ADD COLUMN `instance_id` bigint unsigned NOT NULL COMMENT 'vm instance'; +ALTER TABLE `cloud`.`op_it_work` ADD COLUMN `resource_id` bigint unsigned COMMENT 'resource id being worked on'; +ALTER TABLE `cloud`.`op_it_work` ADD COLUMN `resource_type` char(32) COMMENT 'type of resource being worked on'; ALTER TABLE `cloud`.`hypervsior_properties` ADD COLUMN `is_default` int(1) unsigned NOT NULL DEFAULT 0 COMMENT '1 if network is default'; ALTER TABLE `cloud`.`network_offerings` drop column TYPE; ALTER TABLE `cloud`.`domain_router` ADD COLUMN `host_tag` varchar(255) COMMENT 'host tag specified by the service_offering'; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-301to302-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-301to302-cleanup.sql index 7922d98ea99a..d32644f471c9 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-301to302-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-301to302-cleanup.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-301to302.sql b/engine/schema/src/main/resources/META-INF/db/schema-301to302.sql index 4532757d052b..99a555dedcab 100755 --- a/engine/schema/src/main/resources/META-INF/db/schema-301to302.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-301to302.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -41,7 +41,7 @@ INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Hidden', 'DEFAULT', 'managem INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'network.disable.rpfilter', 'true', 'disable rp_filter on Domain Router VM public interfaces.'); INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'network.securitygroups.work.cleanup.interval', '120', 'Time interval (seconds) in which finished work is cleaned up from the work table'); INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'network.securitygroups.work.lock.timeout', '300', 'Lock wait timeout (seconds) while updating the security group work queues'); -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'network.securitygroups.work.per.agent.queue.size', '100', +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'network.securitygroups.work.per.agent.queue.size', '100', 'The number of outstanding security group work items that can be queued to a host. If exceeded, work items will get dropped to conserve memory. Security Group Sync will take care of ensuring that the host gets updated eventually'); INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'network.securitygroups.workers.pool.size', '50', 'Number of worker threads processing the security group update work queue'); INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Hidden', 'DEFAULT', 'management-server', 'ovm.guest.network.device', null, 'Specify the private bridge on host for private network'); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-302to303.sql b/engine/schema/src/main/resources/META-INF/db/schema-302to303.sql index b475a8e99580..07faf98cf743 100755 --- a/engine/schema/src/main/resources/META-INF/db/schema-302to303.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-302to303.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -55,7 +55,7 @@ CREATE TABLE `cloud`.`volume_host_ref` ( `local_path` varchar(255), `install_path` varchar(255), `url` varchar(255), - `format` varchar(32) NOT NULL COMMENT 'format for the volume', + `format` varchar(32) NOT NULL COMMENT 'format for the volume', `destroyed` tinyint(1) COMMENT 'indicates whether the volume_host entry was destroyed by the user or not', PRIMARY KEY (`id`), CONSTRAINT `fk_volume_host_ref__host_id` FOREIGN KEY `fk_volume_host_ref__host_id` (`host_id`) REFERENCES `host` (`id`) ON DELETE CASCADE, diff --git a/engine/schema/src/main/resources/META-INF/db/schema-302to40-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-302to40-cleanup.sql index 4d89a078b2d5..540c77161893 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-302to40-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-302to40-cleanup.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-302to40.sql b/engine/schema/src/main/resources/META-INF/db/schema-302to40.sql index ca99f0106d2c..47e4c3fd6d5d 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-302to40.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-302to40.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -59,7 +59,7 @@ CREATE TABLE `cloud`.`volume_host_ref` ( `local_path` varchar(255), `install_path` varchar(255), `url` varchar(255), - `format` varchar(32) NOT NULL COMMENT 'format for the volume', + `format` varchar(32) NOT NULL COMMENT 'format for the volume', `destroyed` tinyint(1) COMMENT 'indicates whether the volume_host entry was destroyed by the user or not', PRIMARY KEY (`id`), CONSTRAINT `fk_volume_host_ref__host_id` FOREIGN KEY `fk_volume_host_ref__host_id` (`host_id`) REFERENCES `host` (`id`) ON DELETE CASCADE, @@ -236,9 +236,9 @@ from information_schema.key_column_usage A JOIN information_schema.key_column_usage B ON B.table_name = 'physical_network_service_providers' AND B.COLUMN_NAME = 'provider_name' AND A.COLUMN_NAME ='physical_network_id' AND B.CONSTRAINT_NAME=A.CONSTRAINT_NAME where A.table_name = 'physical_network_service_providers' LIMIT 1); -PREPARE stmt1 FROM @constraintname; -EXECUTE stmt1; -DEALLOCATE PREPARE stmt1; +PREPARE stmt1 FROM @constraintname; +EXECUTE stmt1; +DEALLOCATE PREPARE stmt1; AlTER TABLE `cloud`.`physical_network_service_providers` ADD CONSTRAINT `fk_pnetwork_service_providers__physical_network_id` FOREIGN KEY (`physical_network_id`) REFERENCES `physical_network`(`id`) ON DELETE CASCADE; UPDATE `cloud`.`configuration` SET description='In second, timeout for creating volume from snapshot' WHERE name='create.volume.from.snapshot.wait'; @@ -299,7 +299,7 @@ CREATE TABLE `cloud`.`vpc` ( PRIMARY KEY (`id`), INDEX `i_vpc__removed`(`removed`), CONSTRAINT `fk_vpc__zone_id` FOREIGN KEY `fk_vpc__zone_id` (`zone_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE, - CONSTRAINT `fk_vpc__vpc_offering_id` FOREIGN KEY (`vpc_offering_id`) REFERENCES `vpc_offerings`(`id`), + CONSTRAINT `fk_vpc__vpc_offering_id` FOREIGN KEY (`vpc_offering_id`) REFERENCES `vpc_offerings`(`id`), CONSTRAINT `fk_vpc__account_id` FOREIGN KEY `fk_vpc__account_id` (`account_id`) REFERENCES `account`(`id`) ON DELETE CASCADE, CONSTRAINT `fk_vpc__domain_id` FOREIGN KEY `fk_vpc__domain_id` (`domain_id`) REFERENCES `domain`(`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8; @@ -360,7 +360,7 @@ CREATE TABLE `cloud`.`static_routes` ( `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', `uuid` varchar(40), `vpc_gateway_id` bigint unsigned COMMENT 'id of the corresponding ip address', - `cidr` varchar(18) COMMENT 'cidr for the static route', + `cidr` varchar(18) COMMENT 'cidr for the static route', `state` char(32) NOT NULL COMMENT 'current state of this rule', `vpc_id` bigint unsigned COMMENT 'vpc the firewall rule is associated with', `account_id` bigint unsigned NOT NULL COMMENT 'owner id', @@ -468,7 +468,7 @@ UPDATE `cloud`.`configuration` SET description='Comma separated list of cidrs in INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'site2site.vpn.vpngateway.connection.limit', '4', 'The maximum number of VPN connection per VPN gateway'); INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'site2site.vpn.customergateway.subnets.limit', '10', 'The maximum number of subnets per customer gateway'); -INSERT IGNORE INTO `cloud`.`guest_os_category` VALUES ('11','None',NULL); +INSERT IGNORE INTO `cloud`.`guest_os_category` VALUES ('11','None',NULL); ALTER TABLE `cloud`.`user` ADD COLUMN `incorrect_login_attempts` integer unsigned NOT NULL DEFAULT '0'; INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'incorrect.login.attempts.allowed', '5', 'Incorrect login attempts allowed before the user is disabled'); UPDATE `cloud`.`configuration` set description ='Uuid of the service offering used by console proxy; if NULL - system offering will be used' where name ='consoleproxy.service.offering'; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-304to305-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-304to305-cleanup.sql index 3b5c8f5a3565..1184c98e1824 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-304to305-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-304to305-cleanup.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-304to305.sql b/engine/schema/src/main/resources/META-INF/db/schema-304to305.sql index dfeff3f683bd..cb2efb3edd02 100755 --- a/engine/schema/src/main/resources/META-INF/db/schema-304to305.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-304to305.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -80,7 +80,7 @@ CREATE TABLE `cloud`.`vpc` ( PRIMARY KEY (`id`), INDEX `i_vpc__removed`(`removed`), CONSTRAINT `fk_vpc__zone_id` FOREIGN KEY `fk_vpc__zone_id` (`zone_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE, - CONSTRAINT `fk_vpc__vpc_offering_id` FOREIGN KEY (`vpc_offering_id`) REFERENCES `vpc_offerings`(`id`), + CONSTRAINT `fk_vpc__vpc_offering_id` FOREIGN KEY (`vpc_offering_id`) REFERENCES `vpc_offerings`(`id`), CONSTRAINT `fk_vpc__account_id` FOREIGN KEY `fk_vpc__account_id` (`account_id`) REFERENCES `account`(`id`) ON DELETE CASCADE, CONSTRAINT `fk_vpc__domain_id` FOREIGN KEY `fk_vpc__domain_id` (`domain_id`) REFERENCES `domain`(`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8; @@ -141,7 +141,7 @@ CREATE TABLE `cloud`.`static_routes` ( `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', `uuid` varchar(40), `vpc_gateway_id` bigint unsigned COMMENT 'id of the corresponding ip address', - `cidr` varchar(18) COMMENT 'cidr for the static route', + `cidr` varchar(18) COMMENT 'cidr for the static route', `state` char(32) NOT NULL COMMENT 'current state of this rule', `vpc_id` bigint unsigned COMMENT 'vpc the firewall rule is associated with', `account_id` bigint unsigned NOT NULL COMMENT 'owner id', diff --git a/engine/schema/src/main/resources/META-INF/db/schema-305to306-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-305to306-cleanup.sql index f15ad4fbc5cb..850d48b6526d 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-305to306-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-305to306-cleanup.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-305to306.sql b/engine/schema/src/main/resources/META-INF/db/schema-305to306.sql index b1294a21054f..e9a620bb4ccd 100755 --- a/engine/schema/src/main/resources/META-INF/db/schema-305to306.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-305to306.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-306to307.sql b/engine/schema/src/main/resources/META-INF/db/schema-306to307.sql index a43833efa437..0ddee9ec5db4 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-306to307.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-306to307.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-307to410.sql b/engine/schema/src/main/resources/META-INF/db/schema-307to410.sql index 944d910fec40..55d78b594377 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-307to410.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-307to410.sql @@ -22,7 +22,7 @@ SET foreign_key_checks = 0; --- DB upgrade steps from 302-40 +-- DB upgrade steps from 302-40 CREATE TABLE `cloud`.`external_nicira_nvp_devices` ( `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', `uuid` varchar(255) UNIQUE, @@ -59,9 +59,9 @@ from information_schema.key_column_usage A JOIN information_schema.key_column_usage B ON B.table_name = 'physical_network_service_providers' AND B.COLUMN_NAME = 'provider_name' AND A.COLUMN_NAME ='physical_network_id' AND B.CONSTRAINT_NAME=A.CONSTRAINT_NAME where A.table_name = 'physical_network_service_providers' LIMIT 1); -PREPARE stmt1 FROM @constraintname; -EXECUTE stmt1; -DEALLOCATE PREPARE stmt1; +PREPARE stmt1 FROM @constraintname; +EXECUTE stmt1; +DEALLOCATE PREPARE stmt1; AlTER TABLE physical_network_service_providers ADD CONSTRAINT `fk_pnetwork_service_providers__physical_network_id` FOREIGN KEY (`physical_network_id`) REFERENCES `physical_network`(`id`) ON DELETE CASCADE; UPDATE `cloud`.`configuration` SET description='Do URL encoding for the api response, false by default' WHERE name='encode.api.response'; @@ -351,8 +351,8 @@ ALTER TABLE `cloud`.`vlan` ADD COLUMN `ip6_range` varchar(255); ALTER TABLE `cloud`.`data_center` ADD COLUMN `ip6_dns1` varchar(255); ALTER TABLE `cloud`.`data_center` ADD COLUMN `ip6_dns2` varchar(255); -UPDATE `cloud`.`networks` INNER JOIN `cloud`.`vlan` ON networks.id = vlan.network_id -SET networks.gateway = vlan.vlan_gateway, networks.ip6_gateway = vlan.ip6_gateway, networks.ip6_cidr = vlan.ip6_cidr +UPDATE `cloud`.`networks` INNER JOIN `cloud`.`vlan` ON networks.id = vlan.network_id +SET networks.gateway = vlan.vlan_gateway, networks.ip6_gateway = vlan.ip6_gateway, networks.ip6_cidr = vlan.ip6_cidr WHERE networks.data_center_id = vlan.data_center_id AND networks.physical_network_id = vlan.physical_network_id; -- DB views for list api diff --git a/engine/schema/src/main/resources/META-INF/db/schema-30to301.sql b/engine/schema/src/main/resources/META-INF/db/schema-30to301.sql index 0cc51e7d08f7..81339bf2a53d 100755 --- a/engine/schema/src/main/resources/META-INF/db/schema-30to301.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-30to301.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-40to410.sql b/engine/schema/src/main/resources/META-INF/db/schema-40to410.sql index 1b3a29b27a45..845b31ca04ac 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-40to410.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-40to410.sql @@ -457,15 +457,15 @@ ALTER TABLE `cloud`.`vlan` ADD COLUMN `ip6_range` varchar(255); ALTER TABLE `cloud`.`data_center` ADD COLUMN `ip6_dns1` varchar(255); ALTER TABLE `cloud`.`data_center` ADD COLUMN `ip6_dns2` varchar(255); -UPDATE `cloud`.`networks` INNER JOIN `cloud`.`vlan` ON networks.id = vlan.network_id -SET networks.gateway = vlan.vlan_gateway, networks.ip6_gateway = vlan.ip6_gateway, networks.ip6_cidr = vlan.ip6_cidr +UPDATE `cloud`.`networks` INNER JOIN `cloud`.`vlan` ON networks.id = vlan.network_id +SET networks.gateway = vlan.vlan_gateway, networks.ip6_gateway = vlan.ip6_gateway, networks.ip6_cidr = vlan.ip6_cidr WHERE networks.data_center_id = vlan.data_center_id AND networks.physical_network_id = vlan.physical_network_id; -- DB views for list api DROP VIEW IF EXISTS `cloud`.`user_vm_view`; CREATE VIEW `cloud`.`user_vm_view` AS - select + select vm_instance.id id, vm_instance.name name, user_vm.display_name display_name, @@ -504,7 +504,7 @@ CREATE VIEW `cloud`.`user_vm_view` AS vm_instance.vm_type vm_type, data_center.id data_center_id, data_center.uuid data_center_uuid, - data_center.name data_center_name, + data_center.name data_center_name, data_center.is_security_group_enabled security_group_enabled, host.id host_id, host.uuid host_uuid, @@ -634,7 +634,7 @@ CREATE VIEW `cloud`.`user_vm_view` AS DROP VIEW IF EXISTS `cloud`.`domain_router_view`; CREATE VIEW `cloud`.`domain_router_view` AS - select + select vm_instance.id id, vm_instance.name name, account.id account_id, @@ -740,7 +740,7 @@ CREATE VIEW `cloud`.`domain_router_view` AS DROP VIEW IF EXISTS `cloud`.`security_group_view`; CREATE VIEW `cloud`.`security_group_view` AS - select + select security_group.id id, security_group.name name, security_group.description description, @@ -799,7 +799,7 @@ CREATE VIEW `cloud`.`security_group_view` AS DROP VIEW IF EXISTS `cloud`.`resource_tag_view`; CREATE VIEW `cloud`.`resource_tag_view` AS - select + select resource_tags.id, resource_tags.uuid, resource_tags.key, @@ -831,7 +831,7 @@ CREATE VIEW `cloud`.`resource_tag_view` AS DROP VIEW IF EXISTS `cloud`.`event_view`; CREATE VIEW `cloud`.`event_view` AS - select + select event.id, event.uuid, event.type, @@ -870,7 +870,7 @@ CREATE VIEW `cloud`.`event_view` AS DROP VIEW IF EXISTS `cloud`.`instance_group_view`; CREATE VIEW `cloud`.`instance_group_view` AS - select + select instance_group.id, instance_group.uuid, instance_group.name, @@ -898,7 +898,7 @@ CREATE VIEW `cloud`.`instance_group_view` AS DROP VIEW IF EXISTS `cloud`.`user_view`; CREATE VIEW `cloud`.`user_view` AS - select + select user.id, user.uuid, user.username, @@ -941,7 +941,7 @@ CREATE VIEW `cloud`.`user_view` AS DROP VIEW IF EXISTS `cloud`.`project_view`; CREATE VIEW `cloud`.`project_view` AS - select + select projects.id, projects.uuid, projects.name, @@ -982,7 +982,7 @@ CREATE VIEW `cloud`.`project_view` AS DROP VIEW IF EXISTS `cloud`.`project_account_view`; CREATE VIEW `cloud`.`project_account_view` AS - select + select project_account.id, account.id account_id, account.uuid account_uuid, @@ -1007,7 +1007,7 @@ CREATE VIEW `cloud`.`project_account_view` AS DROP VIEW IF EXISTS `cloud`.`project_invitation_view`; CREATE VIEW `cloud`.`project_invitation_view` AS - select + select project_invitations.id, project_invitations.uuid, project_invitations.email, @@ -1035,7 +1035,7 @@ CREATE VIEW `cloud`.`project_invitation_view` AS DROP VIEW IF EXISTS `cloud`.`host_view`; CREATE VIEW `cloud`.`host_view` AS - select + select host.id, host.uuid, host.name, @@ -1105,7 +1105,7 @@ CREATE VIEW `cloud`.`host_view` AS DROP VIEW IF EXISTS `cloud`.`volume_view`; CREATE VIEW `cloud`.`volume_view` AS - select + select volumes.id, volumes.uuid, volumes.name, @@ -1206,7 +1206,7 @@ CREATE VIEW `cloud`.`volume_view` AS DROP VIEW IF EXISTS `cloud`.`account_netstats_view`; CREATE VIEW `cloud`.`account_netstats_view` AS - SELECT + SELECT account_id, sum(net_bytes_received) + sum(current_bytes_received) as bytesReceived, sum(net_bytes_sent) + sum(current_bytes_sent) as bytesSent @@ -1217,7 +1217,7 @@ CREATE VIEW `cloud`.`account_netstats_view` AS DROP VIEW IF EXISTS `cloud`.`account_vmstats_view`; CREATE VIEW `cloud`.`account_vmstats_view` AS - SELECT + SELECT account_id, state, count(*) as vmcount from `cloud`.`vm_instance` @@ -1225,7 +1225,7 @@ CREATE VIEW `cloud`.`account_vmstats_view` AS DROP VIEW IF EXISTS `cloud`.`free_ip_view`; CREATE VIEW `cloud`.`free_ip_view` AS - select + select count(user_ip_address.id) free_ip from `cloud`.`user_ip_address` @@ -1237,7 +1237,7 @@ CREATE VIEW `cloud`.`free_ip_view` AS DROP VIEW IF EXISTS `cloud`.`account_view`; CREATE VIEW `cloud`.`account_view` AS - select + select account.id, account.uuid, account.account_name, @@ -1348,7 +1348,7 @@ CREATE VIEW `cloud`.`account_view` AS DROP VIEW IF EXISTS `cloud`.`async_job_view`; CREATE VIEW `cloud`.`async_job_view` AS - select + select account.id account_id, account.uuid account_uuid, account.account_name account_name, @@ -1457,7 +1457,7 @@ CREATE VIEW `cloud`.`async_job_view` AS DROP VIEW IF EXISTS `cloud`.`storage_pool_view`; CREATE VIEW `cloud`.`storage_pool_view` AS - select + select storage_pool.id, storage_pool.uuid, storage_pool.name, @@ -1475,7 +1475,7 @@ CREATE VIEW `cloud`.`storage_pool_view` AS cluster.cluster_type, data_center.id data_center_id, data_center.uuid data_center_uuid, - data_center.name data_center_name, + data_center.name data_center_name, host_pod_ref.id pod_id, host_pod_ref.uuid pod_uuid, host_pod_ref.name pod_name, @@ -1507,7 +1507,7 @@ CREATE VIEW `cloud`.`storage_pool_view` AS DROP VIEW IF EXISTS `cloud`.`disk_offering_view`; CREATE VIEW `cloud`.`disk_offering_view` AS - select + select disk_offering.id, disk_offering.uuid, disk_offering.name, @@ -1532,7 +1532,7 @@ CREATE VIEW `cloud`.`disk_offering_view` AS DROP VIEW IF EXISTS `cloud`.`service_offering_view`; CREATE VIEW `cloud`.`service_offering_view` AS - select + select service_offering.id, disk_offering.uuid, disk_offering.name, @@ -1563,10 +1563,10 @@ CREATE VIEW `cloud`.`service_offering_view` AS `cloud`.`disk_offering` ON service_offering.id = disk_offering.id left join `cloud`.`domain` ON disk_offering.domain_id = domain.id; - + DROP VIEW IF EXISTS `cloud`.`data_center_view`; CREATE VIEW `cloud`.`data_center_view` AS - select + select data_center.id, data_center.uuid, data_center.name, @@ -1593,8 +1593,8 @@ CREATE VIEW `cloud`.`data_center_view` AS from `cloud`.`data_center` left join - `cloud`.`domain` ON data_center.domain_id = domain.id; - + `cloud`.`domain` ON data_center.domain_id = domain.id; + CREATE TABLE `cloud`.`baremetal_dhcp_devices` ( `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', diff --git a/engine/schema/src/main/resources/META-INF/db/schema-410to420.sql b/engine/schema/src/main/resources/META-INF/db/schema-410to420.sql index 3556e7e1b4a2..35f73b35d3ca 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-410to420.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-410to420.sql @@ -113,7 +113,7 @@ CREATE TABLE `cloud`.`image_store` ( `uuid` varchar(255) COMMENT 'uuid of data store', `parent` varchar(255) COMMENT 'parent path for the storage server', `created` datetime COMMENT 'date the image store first signed on', - `removed` datetime COMMENT 'date removed if not null', + `removed` datetime COMMENT 'date removed if not null', `total_size` bigint unsigned COMMENT 'storage total size statistics', `used_bytes` bigint unsigned COMMENT 'storage available bytes statistics', PRIMARY KEY(`id`) @@ -131,7 +131,7 @@ CREATE TABLE `cloud`.`image_store_details` ( DROP VIEW IF EXISTS `cloud`.`image_store_view`; CREATE VIEW `cloud`.`image_store_view` AS - select + select image_store.id, image_store.uuid, image_store.name, @@ -153,9 +153,9 @@ CREATE VIEW `cloud`.`image_store_view` AS left join `cloud`.`image_store_details` ON image_store_details.store_id = image_store.id; - + -- here we have to allow null for store_id to accommodate baremetal case to search for ready templates since template state is only stored in this table --- FK also commented out due to this +-- FK also commented out due to this CREATE TABLE `cloud`.`template_store_ref` ( `id` bigint unsigned NOT NULL auto_increment, `store_id` bigint unsigned, @@ -165,7 +165,7 @@ CREATE TABLE `cloud`.`template_store_ref` ( `job_id` varchar(255), `download_pct` int(10) unsigned, `size` bigint unsigned, - `store_role` varchar(255), + `store_role` varchar(255), `physical_size` bigint unsigned DEFAULT 0, `download_state` varchar(255), `error_str` varchar(255), @@ -177,7 +177,7 @@ CREATE TABLE `cloud`.`template_store_ref` ( `is_copy` tinyint(1) NOT NULL DEFAULT 0 COMMENT 'indicates whether this was copied ', `update_count` bigint unsigned, `ref_cnt` bigint unsigned DEFAULT 0, - `updated` datetime, + `updated` datetime, PRIMARY KEY (`id`), -- CONSTRAINT `fk_template_store_ref__store_id` FOREIGN KEY `fk_template_store_ref__store_id` (`store_id`) REFERENCES `image_store` (`id`) ON DELETE CASCADE, INDEX `i_template_store_ref__store_id`(`store_id`), @@ -193,7 +193,7 @@ CREATE TABLE `cloud`.`template_store_ref` ( -- ALTER TABLE `cloud`.`snapshots` DROP COLUMN `sechost_id`; -- change upload host_id FK to point to image_store table -ALTER TABLE `cloud`.`upload` DROP FOREIGN KEY `fk_upload__host_id`; +ALTER TABLE `cloud`.`upload` DROP FOREIGN KEY `fk_upload__host_id`; ALTER TABLE `cloud`.`upload` ADD CONSTRAINT `fk_upload__store_id` FOREIGN KEY(`host_id`) REFERENCES `image_store` (`id`) ON DELETE CASCADE; CREATE TABLE `cloud`.`snapshot_store_ref` ( @@ -208,11 +208,11 @@ CREATE TABLE `cloud`.`snapshot_store_ref` ( `physical_size` bigint unsigned DEFAULT 0, `parent_snapshot_id` bigint unsigned DEFAULT 0, `install_path` varchar(255), - `state` varchar(255) NOT NULL, - -- `removed` datetime COMMENT 'date removed if not null', + `state` varchar(255) NOT NULL, + -- `removed` datetime COMMENT 'date removed if not null', `update_count` bigint unsigned, `ref_cnt` bigint unsigned, - `updated` datetime, + `updated` datetime, `volume_id` bigint unsigned, PRIMARY KEY (`id`), INDEX `i_snapshot_store_ref__store_id`(`store_id`), @@ -238,11 +238,11 @@ CREATE TABLE `cloud`.`volume_store_ref` ( `install_path` varchar(255), `url` varchar(255), `download_url` varchar(255), - `state` varchar(255) NOT NULL, + `state` varchar(255) NOT NULL, `destroyed` tinyint(1) COMMENT 'indicates whether the volume_host entry was destroyed by the user or not', `update_count` bigint unsigned, `ref_cnt` bigint unsigned, - `updated` datetime, + `updated` datetime, PRIMARY KEY (`id`), CONSTRAINT `fk_volume_store_ref__store_id` FOREIGN KEY `fk_volume_store_ref__store_id` (`store_id`) REFERENCES `image_store` (`id`) ON DELETE CASCADE, INDEX `i_volume_store_ref__store_id`(`store_id`), @@ -662,12 +662,12 @@ ALTER TABLE `cloud`.`remote_access_vpn` ADD COLUMN `id` bigint unsigned NOT NULL ALTER TABLE `cloud`.`remote_access_vpn` ADD COLUMN `uuid` varchar(40) UNIQUE; -- START: support for LXC - + INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled) VALUES (UUID(), 'LXC', 'default', 50, 1); ALTER TABLE `cloud`.`physical_network_traffic_types` ADD COLUMN `lxc_network_label` varchar(255) DEFAULT 'cloudbr0' COMMENT 'The network name label of the physical device dedicated to this traffic on a LXC host'; - + UPDATE configuration SET value='KVM,XenServer,VMware,BareMetal,Ovm,LXC' WHERE name='hypervisor.list'; - + INSERT INTO `cloud`.`vm_template` (id, uuid, unique_name, name, public, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, featured, cross_zones, hypervisor_type) VALUES (10, UUID(), 'routing-10', 'SystemVM Template (LXC)', 0, now(), 'SYSTEM', 0, 64, 1, 'http://download.cloudstack.org/templates/acton/acton-systemvm-02062012.qcow2.bz2', '2755de1f9ef2ce4d6f2bee2efbb4da92', 0, 'SystemVM Template (LXC)', 'QCOW2', 15, 0, 1, 'LXC'); @@ -717,10 +717,10 @@ CREATE TABLE `cloud`.`service_offering_details` ( CONSTRAINT `fk_service_offering_details__service_offering_id` FOREIGN KEY (`service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE, CONSTRAINT UNIQUE KEY `uk_service_offering_id_name` (`service_offering_id`, `name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; - + DROP VIEW IF EXISTS `cloud`.`user_vm_view`; CREATE VIEW `cloud`.`user_vm_view` AS - select + select vm_instance.id id, vm_instance.name name, user_vm.display_name display_name, @@ -898,7 +898,7 @@ CREATE VIEW `cloud`.`user_vm_view` AS DROP VIEW IF EXISTS `cloud`.`affinity_group_view`; CREATE VIEW `cloud`.`affinity_group_view` AS - select + select affinity_group.id id, affinity_group.name name, affinity_group.type type, @@ -933,7 +933,7 @@ CREATE VIEW `cloud`.`affinity_group_view` AS DROP VIEW IF EXISTS `cloud`.`host_view`; CREATE VIEW `cloud`.`host_view` AS - select + select host.id, host.uuid, host.name, @@ -1001,10 +1001,10 @@ CREATE VIEW `cloud`.`host_view` AS `cloud`.`async_job` ON async_job.instance_id = host.id and async_job.instance_type = 'Host' and async_job.job_status = 0; - + DROP VIEW IF EXISTS `cloud`.`storage_pool_view`; CREATE VIEW `cloud`.`storage_pool_view` AS - select + select storage_pool.id, storage_pool.uuid, storage_pool.name, @@ -1024,7 +1024,7 @@ CREATE VIEW `cloud`.`storage_pool_view` AS cluster.cluster_type, data_center.id data_center_id, data_center.uuid data_center_uuid, - data_center.name data_center_name, + data_center.name data_center_name, data_center.networktype data_center_type, host_pod_ref.id pod_id, host_pod_ref.uuid pod_uuid, @@ -1054,11 +1054,11 @@ CREATE VIEW `cloud`.`storage_pool_view` AS `cloud`.`async_job` ON async_job.instance_id = storage_pool.id and async_job.instance_type = 'StoragePool' and async_job.job_status = 0; - + DROP VIEW IF EXISTS `cloud`.`domain_router_view`; CREATE VIEW `cloud`.`domain_router_view` AS - select + select vm_instance.id id, vm_instance.name name, account.id account_id, @@ -1157,7 +1157,7 @@ CREATE VIEW `cloud`.`domain_router_view` AS `cloud`.`async_job` ON async_job.instance_id = vm_instance.id and async_job.instance_type = 'DomainRouter' and async_job.job_status = 0; - + CREATE TABLE `cloud`.`external_cisco_vnmc_devices` ( `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', `uuid` varchar(255) UNIQUE, @@ -1242,7 +1242,7 @@ INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'manag DROP VIEW IF EXISTS `cloud`.`service_offering_view`; CREATE VIEW `cloud`.`service_offering_view` AS - select + select service_offering.id, disk_offering.uuid, disk_offering.name, @@ -1289,7 +1289,7 @@ UPDATE `cloud_usage`.`account` SET `default`=1 WHERE id IN (1,2); UPDATE `cloud`.`user` SET `cloud`.`user`.`default`=1 WHERE id IN (1,2); CREATE OR REPLACE VIEW `cloud`.`user_view` AS - select + select user.id, user.uuid, user.username, @@ -1329,7 +1329,7 @@ CREATE OR REPLACE VIEW `cloud`.`user_view` AS `cloud`.`async_job` ON async_job.instance_id = user.id and async_job.instance_type = 'User' and async_job.job_status = 0; - + DROP VIEW IF EXISTS `cloud`.`account_view`; CREATE VIEW `cloud`.`account_view` AS @@ -1879,7 +1879,7 @@ ALTER TABLE `cloud`.`account_details` MODIFY value varchar(255); DROP VIEW IF EXISTS `cloud`.`template_view`; CREATE VIEW `cloud`.`template_view` AS - select + select vm_template.id, vm_template.uuid, vm_template.unique_name, @@ -1920,7 +1920,7 @@ CREATE VIEW `cloud`.`template_view` AS domain.path domain_path, projects.id project_id, projects.uuid project_uuid, - projects.name project_name, + projects.name project_name, data_center.id data_center_id, data_center.uuid data_center_uuid, data_center.name data_center_name, @@ -1950,23 +1950,23 @@ CREATE VIEW `cloud`.`template_view` AS from `cloud`.`vm_template` inner join - `cloud`.`guest_os` ON guest_os.id = vm_template.guest_os_id + `cloud`.`guest_os` ON guest_os.id = vm_template.guest_os_id inner join `cloud`.`account` ON account.id = vm_template.account_id inner join `cloud`.`domain` ON domain.id = account.domain_id left join - `cloud`.`projects` ON projects.project_account_id = account.id + `cloud`.`projects` ON projects.project_account_id = account.id left join - `cloud`.`vm_template_details` ON vm_template_details.template_id = vm_template.id + `cloud`.`vm_template_details` ON vm_template_details.template_id = vm_template.id left join - `cloud`.`vm_template` source_template ON source_template.id = vm_template.source_template_id + `cloud`.`vm_template` source_template ON source_template.id = vm_template.source_template_id left join `cloud`.`template_store_ref` ON template_store_ref.template_id = vm_template.id and template_store_ref.store_role = 'Image' left join - `cloud`.`image_store` ON image_store.removed is NULL AND template_store_ref.store_id is not NULL AND image_store.id = template_store_ref.store_id + `cloud`.`image_store` ON image_store.removed is NULL AND template_store_ref.store_id is not NULL AND image_store.id = template_store_ref.store_id left join - `cloud`.`template_zone_ref` ON template_zone_ref.template_id = vm_template.id AND template_store_ref.store_id is NULL AND template_zone_ref.removed is null + `cloud`.`template_zone_ref` ON template_zone_ref.template_id = vm_template.id AND template_store_ref.store_id is NULL AND template_zone_ref.removed is null left join `cloud`.`data_center` ON (image_store.data_center_id = data_center.id OR template_zone_ref.zone_id = data_center.id) left join @@ -1974,7 +1974,7 @@ CREATE VIEW `cloud`.`template_view` AS left join `cloud`.`resource_tags` ON resource_tags.resource_id = vm_template.id and (resource_tags.resource_type = 'Template' or resource_tags.resource_type='ISO'); - + INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'midonet.apiserver.address', 'http://localhost:8081', 'Specify the address at which the Midonet API server can be contacted (if using Midonet)'); INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'management-server', 'midonet.providerrouter.id', 'd7c5e6a3-e2f4-426b-b728-b7ce6a0448e5', 'Specifies the UUID of the Midonet provider router (if using Midonet)'); @@ -1996,7 +1996,7 @@ CREATE TABLE `cloud`.`account_vnet_map` ( ALTER TABLE `cloud`.`op_dc_vnet_alloc` ADD COLUMN account_vnet_map_id bigint unsigned; ALTER TABLE `cloud`.`op_dc_vnet_alloc` ADD CONSTRAINT `fk_op_dc_vnet_alloc__account_vnet_map_id` FOREIGN KEY `fk_op_dc_vnet_alloc__account_vnet_map_id` (`account_vnet_map_id`) REFERENCES `account_vnet_map` (`id`); - + update `cloud`.`vm_template` set state='Allocated' where state is NULL; update `cloud`.`vm_template` set update_count=0 where update_count is NULL; @@ -2100,7 +2100,7 @@ CREATE TABLE `cloud`.`vm_disk_statistics` ( CONSTRAINT `fk_vm_disk_statistics__account_id` FOREIGN KEY (`account_id`) REFERENCES `account` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8; -insert into `cloud`.`vm_disk_statistics`(data_center_id,account_id,vm_id,volume_id) +insert into `cloud`.`vm_disk_statistics`(data_center_id,account_id,vm_id,volume_id) select volumes.data_center_id, volumes.account_id, vm_instance.id, volumes.id from volumes,vm_instance where vm_instance.vm_type="User" and vm_instance.state<>"Expunging" and volumes.instance_id=vm_instance.id order by vm_instance.id; DROP TABLE IF EXISTS `cloud`.`ovs_providers`; @@ -2166,7 +2166,7 @@ INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'manag INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'vm.disk.throttling.bytes_write_rate', 0, 'Default disk I/O write rate in bytes per second allowed in User vm\'s disk. '); -- Re-enable foreign key checking, at the end of the upgrade path -SET foreign_key_checks = 1; +SET foreign_key_checks = 1; UPDATE `cloud`.`snapshot_policy` set uuid=id WHERE uuid is NULL; #update shared sg enabled network with not null name in Advance Security Group enabled network @@ -2220,7 +2220,7 @@ CREATE TABLE `cloud`.`external_stratosphere_ssp_credentials` ( DROP VIEW IF EXISTS `cloud`.`project_view`; CREATE VIEW `cloud`.`project_view` AS - select + select projects.id, projects.uuid, projects.name, @@ -2264,7 +2264,7 @@ INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Network', 'DEFAULT', 'manage ALTER TABLE `cloud`.`network_offerings` ADD COLUMN `concurrent_connections` int(10) unsigned COMMENT 'Load Balancer(haproxy) maximum number of concurrent connections(global max)'; - + ALTER TABLE `cloud`.`sync_queue` MODIFY `queue_size` smallint(6) NOT NULL DEFAULT '0' COMMENT 'number of items being processed by the queue'; ALTER TABLE `cloud`.`sync_queue` MODIFY `queue_size_limit` smallint(6) NOT NULL DEFAULT '1' COMMENT 'max number of items the queue can process concurrently'; @@ -2280,7 +2280,7 @@ INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'manag #update the account_vmstats_view - count only user vms DROP VIEW IF EXISTS `cloud`.`account_vmstats_view`; CREATE VIEW `cloud`.`account_vmstats_view` AS - SELECT + SELECT account_id, state, count(*) as vmcount from `cloud`.`vm_instance` @@ -2329,7 +2329,7 @@ INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Advanced", 'DEFAULT', 'manag DROP VIEW IF EXISTS `cloud`.`data_center_view`; CREATE VIEW `cloud`.`data_center_view` AS - select + select data_center.id, data_center.uuid, data_center.name, diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41310to41400.sql b/engine/schema/src/main/resources/META-INF/db/schema-41310to41400.sql index fbbf0a2aef8a..603e7712ebc0 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41310to41400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41310to41400.sql @@ -19,7 +19,7 @@ -- Schema upgrade from 4.13.1.0 to 4.14.0.0 --; --- Update the description to indicate this only works with KVM + Ceph +-- Update the description to indicate this only works with KVM + Ceph -- (not implemented properly atm for KVM+NFS/local, and it accidentally works with XS + NFS. Not applicable for VMware) UPDATE `cloud`.`configuration` SET `description`='Indicates whether to always backup primary storage snapshot to secondary storage. Keeping snapshots only on Primary storage is applicable for KVM + Ceph only.' WHERE `name`='snapshot.backup.to.secondary'; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41900to41910-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-41900to41910-cleanup.sql index b580d42686f0..2d57db2b778e 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41900to41910-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41900to41910-cleanup.sql @@ -18,3 +18,7 @@ --; -- Schema upgrade cleanup from 4.19.0.0 to 4.19.1.0 --; + +-- List VMs response optimisation, don't sum during API handling +UPDATE cloud.configuration set value='false' where name='vm.stats.increment.metrics'; +DELETE from cloud.configuration where name='vm.stats.increment.metrics.in.memory'; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql b/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql index c05f5503c3e2..0cb10f4a0ef9 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql @@ -63,3 +63,10 @@ CREATE TABLE IF NOT EXISTS `cloud_usage`.`usage_vpc` ( ) ENGINE=InnoDB CHARSET=utf8; CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.cloud_usage', 'state', 'VARCHAR(100) DEFAULT NULL'); + +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user_data', 'removed', 'datetime COMMENT "date removed or null, if still present"'); + +-- Update options for config - host.allocators.order +UPDATE `cloud`.`configuration` SET + `options` = 'FirstFitRouting,RandomAllocator,TestingAllocator,FirstFitAllocator,RecreateHostAllocator' +WHERE `name` = 'host.allocators.order'; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41900to42000.sql b/engine/schema/src/main/resources/META-INF/db/schema-41900to42000.sql deleted file mode 100644 index 1bb1905443a9..000000000000 --- a/engine/schema/src/main/resources/META-INF/db/schema-41900to42000.sql +++ /dev/null @@ -1,81 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, --- software distributed under the License is distributed on an --- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --- KIND, either express or implied. See the License for the --- specific language governing permissions and limitations --- under the License. - ---; --- Schema upgrade from 4.19.0.0 to 4.20.0.0 ---; - --- Add tag column to tables -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.resource_limit', 'tag', 'varchar(64) DEFAULT NULL COMMENT "tag for the limit" '); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.resource_count', 'tag', 'varchar(64) DEFAULT NULL COMMENT "tag for the resource count" '); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.resource_reservation', 'tag', 'varchar(64) DEFAULT NULL COMMENT "tag for the resource reservation" '); -ALTER TABLE `resource_count` -DROP INDEX `i_resource_count__type_accountId`, -DROP INDEX `i_resource_count__type_domaintId`, -ADD UNIQUE INDEX `i_resource_count__type_tag_accountId` (`type`,`tag`,`account_id`), -ADD UNIQUE INDEX `i_resource_count__type_tag_domaintId` (`type`,`tag`,`domain_id`); - - -ALTER TABLE `cloud`.`resource_reservation` - ADD COLUMN `resource_id` bigint unsigned NULL; - -ALTER TABLE `cloud`.`resource_reservation` - MODIFY COLUMN `amount` bigint NOT NULL; - - --- Update Default System offering for Router to 512MiB -UPDATE `cloud`.`service_offering` SET ram_size = 512 WHERE unique_name IN ("Cloud.Com-SoftwareRouter", "Cloud.Com-SoftwareRouter-Local", - "Cloud.Com-InternalLBVm", "Cloud.Com-InternalLBVm-Local", - "Cloud.Com-ElasticLBVm", "Cloud.Com-ElasticLBVm-Local") - AND system_use = 1 AND ram_size < 512; - --- NSX Plugin -- -CREATE TABLE `cloud`.`nsx_providers` ( - `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', - `uuid` varchar(40), - `zone_id` bigint unsigned NOT NULL COMMENT 'Zone ID', - `host_id` bigint unsigned NOT NULL COMMENT 'Host ID', - `provider_name` varchar(40), - `hostname` varchar(255) NOT NULL, - `port` varchar(255), - `username` varchar(255) NOT NULL, - `password` varchar(255) NOT NULL, - `tier0_gateway` varchar(255), - `edge_cluster` varchar(255), - `transport_zone` varchar(255), - `created` datetime NOT NULL COMMENT 'date created', - `removed` datetime COMMENT 'date removed if not null', - PRIMARY KEY (`id`), - CONSTRAINT `fk_nsx_providers__zone_id` FOREIGN KEY `fk_nsx_providers__zone_id` (`zone_id`) REFERENCES `data_center`(`id`) ON DELETE CASCADE, - INDEX `i_nsx_providers__zone_id`(`zone_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- NSX Plugin -- -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.network_offerings','for_nsx', 'int(1) unsigned DEFAULT "0" COMMENT "is nsx enabled for the resource"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.network_offerings','nsx_mode', 'varchar(32) COMMENT "mode in which the network would route traffic"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc_offerings','for_nsx', 'int(1) unsigned DEFAULT "0" COMMENT "is nsx enabled for the resource"'); -CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc_offerings','nsx_mode', 'varchar(32) COMMENT "mode in which the network would route traffic"'); - - --- Create table to persist quota email template configurations -CREATE TABLE IF NOT EXISTS `cloud_usage`.`quota_email_configuration`( - `account_id` int(11) NOT NULL, - `email_template_id` bigint(20) NOT NULL, - `enabled` int(1) UNSIGNED NOT NULL, - PRIMARY KEY (`account_id`, `email_template_id`), - CONSTRAINT `FK_quota_email_configuration_account_id` FOREIGN KEY (`account_id`) REFERENCES `cloud_usage`.`quota_account`(`account_id`), - CONSTRAINT `FK_quota_email_configuration_email_template_id` FOREIGN KEY (`email_template_id`) REFERENCES `cloud_usage`.`quota_email_templates`(`id`)); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41900to42000-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-41910to42000-cleanup.sql similarity index 100% rename from engine/schema/src/main/resources/META-INF/db/schema-41900to42000-cleanup.sql rename to engine/schema/src/main/resources/META-INF/db/schema-41910to42000-cleanup.sql diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql b/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql new file mode 100644 index 000000000000..295ad147a993 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql @@ -0,0 +1,152 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade from 4.19.0.0 to 4.20.0.0 +--; + +-- Add tag column to tables +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.resource_limit', 'tag', 'varchar(64) DEFAULT NULL COMMENT "tag for the limit" '); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.resource_count', 'tag', 'varchar(64) DEFAULT NULL COMMENT "tag for the resource count" '); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.resource_reservation', 'tag', 'varchar(64) DEFAULT NULL COMMENT "tag for the resource reservation" '); +CALL `cloud`.`IDEMPOTENT_DROP_INDEX`('i_resource_count__type_accountId', 'cloud.resource_count'); +CALL `cloud`.`IDEMPOTENT_DROP_INDEX`('i_resource_count__type_domaintId', 'cloud.resource_count'); + +DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_UNIQUE_INDEX`; +CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_UNIQUE_INDEX` ( + IN in_table_name VARCHAR(200), + IN in_index_name VARCHAR(200), + IN in_index_definition VARCHAR(1000) +) +BEGIN + DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name, ' ', 'ADD UNIQUE INDEX ', in_index_name, ' ', in_index_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END; + +CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_INDEX`('cloud.resource_count', 'i_resource_count__type_tag_accountId', '(type, tag, account_id)'); +CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_INDEX`('cloud.resource_count', 'i_resource_count__type_tag_domainId', '(type, tag, domain_id)'); + +ALTER TABLE `cloud`.`resource_reservation` + MODIFY COLUMN `amount` bigint NOT NULL; + +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.resource_reservation', 'resource_id', 'bigint unsigned NULL COMMENT "id of the resource" '); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.resource_reservation', 'mgmt_server_id', 'bigint unsigned NULL COMMENT "management server id" '); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.resource_reservation', 'created', 'datetime DEFAULT NULL COMMENT "date when the reservation was created" '); + +UPDATE `cloud`.`resource_reservation` SET `created` = now() WHERE created IS NULL; + + +-- Update Default System offering for Router to 512MiB +UPDATE `cloud`.`service_offering` SET ram_size = 512 WHERE unique_name IN ("Cloud.Com-SoftwareRouter", "Cloud.Com-SoftwareRouter-Local", + "Cloud.Com-InternalLBVm", "Cloud.Com-InternalLBVm-Local", + "Cloud.Com-ElasticLBVm", "Cloud.Com-ElasticLBVm-Local") + AND system_use = 1 AND ram_size < 512; + +-- NSX Plugin -- +CREATE TABLE `cloud`.`nsx_providers` ( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `uuid` varchar(40), + `zone_id` bigint unsigned NOT NULL COMMENT 'Zone ID', + `host_id` bigint unsigned NOT NULL COMMENT 'Host ID', + `provider_name` varchar(40), + `hostname` varchar(255) NOT NULL, + `port` varchar(255), + `username` varchar(255) NOT NULL, + `password` varchar(255) NOT NULL, + `tier0_gateway` varchar(255), + `edge_cluster` varchar(255), + `transport_zone` varchar(255), + `created` datetime NOT NULL COMMENT 'date created', + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY (`id`), + CONSTRAINT `fk_nsx_providers__zone_id` FOREIGN KEY `fk_nsx_providers__zone_id` (`zone_id`) REFERENCES `data_center`(`id`) ON DELETE CASCADE, + INDEX `i_nsx_providers__zone_id`(`zone_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- NSX Plugin -- +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.network_offerings','for_nsx', 'int(1) unsigned DEFAULT "0" COMMENT "is nsx enabled for the resource"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.network_offerings','nsx_mode', 'varchar(32) COMMENT "mode in which the network would route traffic"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc_offerings','for_nsx', 'int(1) unsigned DEFAULT "0" COMMENT "is nsx enabled for the resource"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc_offerings','nsx_mode', 'varchar(32) COMMENT "mode in which the network would route traffic"'); + +-- Create table to persist quota email template configurations +CREATE TABLE IF NOT EXISTS `cloud_usage`.`quota_email_configuration`( + `account_id` int(11) NOT NULL, + `email_template_id` bigint(20) NOT NULL, + `enabled` int(1) UNSIGNED NOT NULL, + PRIMARY KEY (`account_id`, `email_template_id`), + CONSTRAINT `FK_quota_email_configuration_account_id` FOREIGN KEY (`account_id`) REFERENCES `cloud_usage`.`quota_account`(`account_id`), + CONSTRAINT `FK_quota_email_configuration_email_template_id` FOREIGN KEY (`email_template_id`) REFERENCES `cloud_usage`.`quota_email_templates`(`id`)); + +-- Add `is_implicit` column to `host_tags` table +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.host_tags', 'is_implicit', 'int(1) UNSIGNED NOT NULL DEFAULT 0 COMMENT "If host tag is implicit or explicit" '); + +-- Webhooks feature +DROP TABLE IF EXISTS `cloud`.`webhook`; +CREATE TABLE `cloud`.`webhook` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id of the webhook', + `uuid` varchar(255) COMMENT 'uuid of the webhook', + `name` varchar(255) NOT NULL COMMENT 'name of the webhook', + `description` varchar(4096) COMMENT 'description for the webhook', + `state` char(32) NOT NULL COMMENT 'state of the webhook - Enabled or Disabled', + `domain_id` bigint unsigned NOT NULL COMMENT 'id of the owner domain of the webhook', + `account_id` bigint unsigned NOT NULL COMMENT 'id of the owner account of the webhook', + `payload_url` varchar(255) COMMENT 'payload URL for the webhook', + `secret_key` varchar(255) COMMENT 'secret key for the webhook', + `ssl_verification` boolean COMMENT 'for https payload url, if true then strict ssl verification', + `scope` char(32) NOT NULL COMMENT 'scope for the webhook - Local, Domain, Global', + `created` datetime COMMENT 'date the webhook was created', + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY(`id`), + INDEX `i_webhook__account_id`(`account_id`), + CONSTRAINT `fk_webhook__account_id` FOREIGN KEY (`account_id`) REFERENCES `account`(`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `cloud`.`webhook_delivery`; +CREATE TABLE `cloud`.`webhook_delivery` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id of the webhook delivery', + `uuid` varchar(255) COMMENT 'uuid of the webhook', + `event_id` bigint unsigned NOT NULL COMMENT 'id of the event', + `webhook_id` bigint unsigned NOT NULL COMMENT 'id of the webhook', + `mshost_msid` bigint unsigned NOT NULL COMMENT 'msid of the management server', + `headers` TEXT COMMENT 'headers for the webhook delivery', + `payload` TEXT COMMENT 'payload for the webhook delivery', + `success` boolean COMMENT 'webhook delivery succeeded or not', + `response` TEXT COMMENT 'response of the webhook delivery', + `start_time` datetime COMMENT 'start timestamp of the webhook delivery', + `end_time` datetime COMMENT 'end timestamp of the webhook delivery', + PRIMARY KEY(`id`), + INDEX `i_webhook__event_id`(`event_id`), + INDEX `i_webhook__webhook_id`(`webhook_id`), + CONSTRAINT `fk_webhook__event_id` FOREIGN KEY (`event_id`) REFERENCES `event`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_webhook__webhook_id` FOREIGN KEY (`webhook_id`) REFERENCES `webhook`(`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Normalize quota.usage.smtp.useStartTLS, quota.usage.smtp.useAuth, alert.smtp.useAuth and project.smtp.useAuth values +UPDATE + `cloud`.`configuration` +SET + value = "true" +WHERE + name IN ("quota.usage.smtp.useStartTLS", "quota.usage.smtp.useAuth", "alert.smtp.useAuth", "project.smtp.useAuth") + AND value IN ("true", "y", "t", "1", "on", "yes"); + +UPDATE + `cloud`.`configuration` +SET + value = "false" +WHERE + name IN ("quota.usage.smtp.useStartTLS", "quota.usage.smtp.useAuth", "alert.smtp.useAuth", "project.smtp.useAuth") + AND value NOT IN ("true", "y", "t", "1", "on", "yes"); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-420to421.sql b/engine/schema/src/main/resources/META-INF/db/schema-420to421.sql index b99af287bc5e..25c025c56511 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-420to421.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-420to421.sql @@ -20,10 +20,10 @@ --; -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 's3.singleupload.max.size', '5', +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 's3.singleupload.max.size', '5', 'The maximum size limit for S3 single part upload API(in GB). If it is set to 0, then it means always use multi-part upload to upload object to S3. If it is set to -1, then it means always use single-part upload to upload object to S3.'); -INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Storage", 'DEFAULT', 'management-server', "enable.ha.storage.migration", "true", "Enable/disable storage migration across primary storage during HA"); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Storage", 'DEFAULT', 'management-server', "enable.ha.storage.migration", "true", "Enable/disable storage migration across primary storage during HA"); UPDATE `cloud`.`configuration` SET description="Specify whether or not to reserve CPU based on CPU overprovisioning factor" where name="vmware.reserve.cpu"; UPDATE `cloud`.`configuration` SET description="Specify whether or not to reserve memory based on memory overprovisioning factor" where name="vmware.reserve.mem"; -- Remove Windows Server 8 from guest_os_type dropdown to use Windows Server 2012 diff --git a/engine/schema/src/main/resources/META-INF/db/schema-442to450.sql b/engine/schema/src/main/resources/META-INF/db/schema-442to450.sql index 90a52bd42732..d2ba408241e0 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-442to450.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-442to450.sql @@ -671,7 +671,7 @@ CREATE VIEW `cloud`.`user_vm_view` AS resource_tags.resource_id tag_resource_id, resource_tags.resource_uuid tag_resource_uuid, resource_tags.resource_type tag_resource_type, - resource_tags.customer tag_customer, + resource_tags.customer tag_customer, async_job.id job_id, async_job.uuid job_uuid, async_job.job_status job_status, @@ -752,7 +752,7 @@ CREATE VIEW `cloud`.`user_vm_view` AS left join `cloud`.`user_vm_details` `custom_speed` ON (((`custom_speed`.`vm_id` = `cloud`.`vm_instance`.`id`) and (`custom_speed`.`name` = 'CpuSpeed'))) left join - `cloud`.`user_vm_details` `custom_ram_size` ON (((`custom_ram_size`.`vm_id` = `cloud`.`vm_instance`.`id`) and (`custom_ram_size`.`name` = 'memory'))); + `cloud`.`user_vm_details` `custom_ram_size` ON (((`custom_ram_size`.`vm_id` = `cloud`.`vm_instance`.`id`) and (`custom_ram_size`.`name` = 'memory'))); INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (231, UUID(), 1, 'CentOS 5 (32-bit)', utc_timestamp()); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-481to490-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-481to490-cleanup.sql index 1868a0908006..b8dd0477db96 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-481to490-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-481to490-cleanup.sql @@ -22,7 +22,7 @@ -- Added in CLOUDSTACK-9340: General DB optimization, 4 cases: ----- 1) Incorrect PRIMARY key -ALTER TABLE `cloud`.`ovs_tunnel_network` +ALTER TABLE `cloud`.`ovs_tunnel_network` DROP PRIMARY KEY, ADD PRIMARY KEY (`id`), DROP INDEX `id` , diff --git a/engine/schema/src/main/resources/META-INF/db/schema-481to490.sql b/engine/schema/src/main/resources/META-INF/db/schema-481to490.sql index 49cfc8346c54..bac3b1e6fabd 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-481to490.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-481to490.sql @@ -23,9 +23,9 @@ ALTER TABLE `event` ADD INDEX `archived` (`archived`); ALTER TABLE `event` ADD INDEX `state` (`state`); DROP VIEW IF EXISTS `cloud`.`template_view`; -CREATE +CREATE VIEW `template_view` AS - SELECT + SELECT `vm_template`.`id` AS `id`, `vm_template`.`uuid` AS `uuid`, `vm_template`.`unique_name` AS `unique_name`, @@ -124,9 +124,9 @@ VIEW `template_view` AS OR (`resource_tags`.`resource_type` = 'ISO'))))); DROP VIEW IF EXISTS `cloud`.`volume_view`; -CREATE +CREATE VIEW `volume_view` AS - SELECT + SELECT `volumes`.`id` AS `id`, `volumes`.`uuid` AS `uuid`, `volumes`.`name` AS `name`, @@ -234,9 +234,9 @@ VIEW `volume_view` AS AND (`async_job`.`job_status` = 0)))); DROP VIEW IF EXISTS `cloud`.`user_vm_view`; -CREATE +CREATE VIEW `user_vm_view` AS - SELECT + SELECT `vm_instance`.`id` AS `id`, `vm_instance`.`name` AS `name`, `user_vm`.`display_name` AS `display_name`, @@ -423,10 +423,10 @@ ALTER TABLE `cloud`.`ssh_keypairs` ADD INDEX `i_public_key` (`public_key` (64) A ALTER TABLE `cloud`.`user_vm_details` ADD INDEX `i_name_vm_id` (`vm_id` ASC, `name` ASC); ALTER TABLE `cloud`.`instance_group` ADD INDEX `i_name` (`name` ASC); ------ 4) Some views query (Change view to improve account retrieval speed) +----- 4) Some views query (Change view to improve account retrieval speed) CREATE OR REPLACE VIEW `account_vmstats_view` AS - SELECT + SELECT `vm_instance`.`account_id` AS `account_id`, `vm_instance`.`state` AS `state`, COUNT(0) AS `vmcount` diff --git a/engine/schema/src/main/resources/META-INF/db/schema-4930to41000.sql b/engine/schema/src/main/resources/META-INF/db/schema-4930to41000.sql index dc0cd6d4d75a..23670757247a 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-4930to41000.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-4930to41000.sql @@ -147,7 +147,7 @@ CREATE TABLE IF NOT EXISTS `cloud`.`storage_pool_tags` ( ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; -- Insert storage tags from storage_pool_details -INSERT INTO `cloud`.`storage_pool_tags` (pool_id, tag) SELECT pool_id, +INSERT INTO `cloud`.`storage_pool_tags` (pool_id, tag) SELECT pool_id, name FROM `cloud`.`storage_pool_details` WHERE value = 'true'; -- Alter view storage_pool_view @@ -227,7 +227,7 @@ ALTER TABLE `cloud`.`vm_snapshots` ADD CONSTRAINT `fk_vm_snapshots_service_offer INSERT INTO `cloud`.`vm_snapshot_details` (vm_snapshot_id, name, value) SELECT s.id, d.name, d.value FROM `cloud`.`user_vm_details` d JOIN `cloud`.`vm_instance` v ON (d.vm_id = v.id) -JOIN `cloud`.`service_offering` o ON (v.service_offering_id = o.id) +JOIN `cloud`.`service_offering` o ON (v.service_offering_id = o.id) JOIN `cloud`.`vm_snapshots` s ON (s.service_offering_id = o.id AND s.vm_id = v.id) WHERE (o.cpu is null AND o.speed IS NULL AND o.ram_size IS NULL) AND (d.name = 'cpuNumber' OR d.name = 'cpuSpeed' OR d.name = 'memory'); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-level.sql b/engine/schema/src/main/resources/META-INF/db/schema-level.sql index 72aade4e5016..fef961502fa5 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-level.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-level.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-snapshot-217to224.sql b/engine/schema/src/main/resources/META-INF/db/schema-snapshot-217to224.sql index 7320bda59063..5e29435855df 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-snapshot-217to224.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-snapshot-217to224.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/schema-snapshot-223to224.sql b/engine/schema/src/main/resources/META-INF/db/schema-snapshot-223to224.sql index 668cbb692b99..5c27eed68f00 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-snapshot-223to224.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-snapshot-223to224.sql @@ -5,9 +5,9 @@ -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at --- +-- -- http://www.apache.org/licenses/LICENSE-2.0 --- +-- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql index 5c6d4fd772b3..7bd4b3cc4a91 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql @@ -53,7 +53,9 @@ SELECT host_pod_ref.uuid pod_uuid, host_pod_ref.name pod_name, GROUP_CONCAT(DISTINCT(host_tags.tag)) AS tag, - `host_tags`.`is_tag_a_rule` AS `is_tag_a_rule`, + GROUP_CONCAT(DISTINCT(explicit_host_tags.tag)) AS explicit_tag, + GROUP_CONCAT(DISTINCT(implicit_host_tags.tag)) AS implicit_tag, + `explicit_host_tags`.`is_tag_a_rule` AS `is_tag_a_rule`, guest_os_category.id guest_os_category_id, guest_os_category.uuid guest_os_category_uuid, guest_os_category.name guest_os_category_name, @@ -89,6 +91,10 @@ FROM LEFT JOIN `cloud`.`host_tags` ON host_tags.host_id = host.id LEFT JOIN + `cloud`.`host_tags` AS explicit_host_tags ON explicit_host_tags.host_id = host.id AND explicit_host_tags.is_implicit = 0 + LEFT JOIN + `cloud`.`host_tags` AS implicit_host_tags ON implicit_host_tags.host_id = host.id AND implicit_host_tags.is_implicit = 1 + LEFT JOIN `cloud`.`op_host_capacity` mem_caps ON host.id = mem_caps.host_id AND mem_caps.capacity_type = 0 LEFT JOIN diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql index 7a057dc0330b..62294ed5d890 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql @@ -74,6 +74,7 @@ SELECT `vm_template`.`uuid` AS `template_uuid`, `vm_template`.`name` AS `template_name`, `vm_template`.`type` AS `template_type`, + `vm_template`.`format` AS `template_format`, `vm_template`.`display_text` AS `template_display_text`, `vm_template`.`enable_password` AS `password_enabled`, `iso`.`id` AS `iso_id`, @@ -195,7 +196,7 @@ FROM LEFT JOIN `networks` ON ((`nics`.`network_id` = `networks`.`id`))) LEFT JOIN `vpc` ON (((`networks`.`vpc_id` = `vpc`.`id`) AND ISNULL(`vpc`.`removed`)))) - LEFT JOIN `user_ip_address` ON ((`user_ip_address`.`vm_id` = `vm_instance`.`id`))) + LEFT JOIN `user_ip_address` FORCE INDEX(`fk_user_ip_address__vm_id`) ON ((`user_ip_address`.`vm_id` = `vm_instance`.`id`))) LEFT JOIN `user_vm_details` `ssh_details` ON (((`ssh_details`.`vm_id` = `vm_instance`.`id`) AND (`ssh_details`.`name` = 'SSH.KeyPairNames')))) LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_instance`.`id`) diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.volume_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.volume_view.sql index fd21fff14944..950dcddf4c71 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.volume_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.volume_view.sql @@ -39,6 +39,7 @@ SELECT `volumes`.`path` AS `path`, `volumes`.`chain_info` AS `chain_info`, `volumes`.`external_uuid` AS `external_uuid`, + `volumes`.`encrypt_format` AS `encrypt_format`, `account`.`id` AS `account_id`, `account`.`uuid` AS `account_uuid`, `account`.`account_name` AS `account_name`, diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.webhook_delivery_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.webhook_delivery_view.sql new file mode 100644 index 000000000000..54ba52fba4a6 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.webhook_delivery_view.sql @@ -0,0 +1,48 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`webhook_delivery_view`; + +DROP VIEW IF EXISTS `cloud`.`webhook_delivery_view`; +CREATE VIEW `cloud`.`webhook_delivery_view` AS + SELECT + webhook_delivery.id, + webhook_delivery.uuid, + webhook_delivery.headers, + webhook_delivery.payload, + webhook_delivery.success, + webhook_delivery.response, + webhook_delivery.start_time, + webhook_delivery.end_time, + event.id event_id, + event.uuid event_uuid, + event.type event_type, + webhook.id webhook_id, + webhook.uuid webhook_uuid, + webhook.name webhook_name, + mshost.id mshost_id, + mshost.uuid mshost_uuid, + mshost.msid mshost_msid, + mshost.name mshost_name + FROM + `cloud`.`webhook_delivery` + INNER JOIN + `cloud`.`event` ON webhook_delivery.event_id = event.id + INNER JOIN + `cloud`.`webhook` ON webhook_delivery.webhook_id = webhook.id + LEFT JOIN + `cloud`.`mshost` ON mshost.msid = webhook_delivery.mshost_msid; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.webhook_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.webhook_view.sql new file mode 100644 index 000000000000..443463eec4bd --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.webhook_view.sql @@ -0,0 +1,52 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`webhook_view`; + +DROP VIEW IF EXISTS `cloud`.`webhook_view`; +CREATE VIEW `cloud`.`webhook_view` AS + SELECT + webhook.id, + webhook.uuid, + webhook.name, + webhook.description, + webhook.state, + webhook.payload_url, + webhook.secret_key, + webhook.ssl_verification, + webhook.scope, + webhook.created, + webhook.removed, + account.id account_id, + account.uuid account_uuid, + account.account_name account_name, + account.type account_type, + domain.id domain_id, + domain.uuid domain_uuid, + domain.name domain_name, + domain.path domain_path, + projects.id project_id, + projects.uuid project_uuid, + projects.name project_name + FROM + `cloud`.`webhook` + INNER JOIN + `cloud`.`account` ON webhook.account_id = account.id + INNER JOIN + `cloud`.`domain` ON webhook.domain_id = domain.id + LEFT JOIN + `cloud`.`projects` ON projects.project_account_id = webhook.account_id; diff --git a/engine/schema/src/test/java/com/cloud/host/HostVOTest.java b/engine/schema/src/test/java/com/cloud/host/HostVOTest.java index cd9ac3cc1723..3262c4cc2918 100755 --- a/engine/schema/src/test/java/com/cloud/host/HostVOTest.java +++ b/engine/schema/src/test/java/com/cloud/host/HostVOTest.java @@ -20,14 +20,18 @@ import com.cloud.service.ServiceOfferingVO; import com.cloud.template.VirtualMachineTemplate; import com.cloud.vm.VirtualMachine; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Set; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import org.junit.Test; -import org.junit.Before; -import org.mockito.Mockito; public class HostVOTest { HostVO host; @@ -37,7 +41,7 @@ public class HostVOTest { public void setUp() throws Exception { host = new HostVO(); offering = new ServiceOfferingVO("TestSO", 0, 0, 0, 0, 0, - false, "TestSO", false,VirtualMachine.Type.User,false); + false, "TestSO", false, VirtualMachine.Type.User, false); } @Test @@ -52,14 +56,14 @@ public void testNoTag() { @Test public void testRightTag() { - host.setHostTags(Arrays.asList("tag1","tag2"), false); + host.setHostTags(Arrays.asList("tag1", "tag2"), false); offering.setHostTag("tag2,tag1"); assertTrue(host.checkHostServiceOfferingTags(offering)); } @Test public void testWrongTag() { - host.setHostTags(Arrays.asList("tag1","tag2"), false); + host.setHostTags(Arrays.asList("tag1", "tag2"), false); offering.setHostTag("tag2,tag4"); assertFalse(host.checkHostServiceOfferingTags(offering)); } @@ -87,40 +91,59 @@ public void checkHostServiceOfferingTagsTestRuleTagWithNullServiceTag() { @Test public void testEitherNoSOOrTemplate() { - assertFalse(host.checkHostServiceOfferingAndTemplateTags(null, Mockito.mock(VirtualMachineTemplate.class))); - assertFalse(host.checkHostServiceOfferingAndTemplateTags(Mockito.mock(ServiceOffering.class), null)); + assertFalse(host.checkHostServiceOfferingAndTemplateTags(null, Mockito.mock(VirtualMachineTemplate.class), null)); + assertFalse(host.checkHostServiceOfferingAndTemplateTags(Mockito.mock(ServiceOffering.class), null, null)); } @Test public void testNoTagOfferingTemplate() { - assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, Mockito.mock(VirtualMachineTemplate.class))); + assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, Mockito.mock(VirtualMachineTemplate.class), Collections.emptySet())); + assertTrue(host.getHostServiceOfferingAndTemplateMissingTags(offering, Mockito.mock(VirtualMachineTemplate.class), Collections.emptySet()).isEmpty()); + assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, Mockito.mock(VirtualMachineTemplate.class), Set.of("tag1", "tag2"))); + assertTrue(host.getHostServiceOfferingAndTemplateMissingTags(offering, Mockito.mock(VirtualMachineTemplate.class), Set.of("tag1", "tag2")).isEmpty()); } @Test public void testRightTagOfferingTemplate() { host.setHostTags(Arrays.asList("tag1", "tag2"), false); offering.setHostTag("tag2,tag1"); - assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, Mockito.mock(VirtualMachineTemplate.class))); + assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, Mockito.mock(VirtualMachineTemplate.class), Set.of("tag1"))); + Set actualMissingTags = host.getHostServiceOfferingAndTemplateMissingTags(offering, Mockito.mock(VirtualMachineTemplate.class), Set.of("tag1")); + assertTrue(actualMissingTags.isEmpty()); + host.setHostTags(Arrays.asList("tag1", "tag2", "tag3"), false); offering.setHostTag("tag2,tag1"); VirtualMachineTemplate template = Mockito.mock(VirtualMachineTemplate.class); Mockito.when(template.getTemplateTag()).thenReturn("tag3"); - assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, template)); + assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, template, Set.of("tag2", "tag3"))); + actualMissingTags = host.getHostServiceOfferingAndTemplateMissingTags(offering, template, Set.of("tag2", "tag3")); + assertTrue(actualMissingTags.isEmpty()); host.setHostTags(List.of("tag3"), false); offering.setHostTag(null); - assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, template)); + assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, template, Set.of("tag3"))); + actualMissingTags = host.getHostServiceOfferingAndTemplateMissingTags(offering, template, Set.of("tag3")); + assertTrue(actualMissingTags.isEmpty()); + + assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, template, Set.of("tag2", "tag1"))); + actualMissingTags = host.getHostServiceOfferingAndTemplateMissingTags(offering, template, Set.of("tag2", "tag1")); + assertTrue(actualMissingTags.isEmpty()); } @Test public void testWrongOfferingTag() { - host.setHostTags(Arrays.asList("tag1","tag2"), false); + host.setHostTags(Arrays.asList("tag1", "tag2"), false); offering.setHostTag("tag2,tag4"); VirtualMachineTemplate template = Mockito.mock(VirtualMachineTemplate.class); Mockito.when(template.getTemplateTag()).thenReturn("tag1"); - assertFalse(host.checkHostServiceOfferingAndTemplateTags(offering, template)); + assertFalse(host.checkHostServiceOfferingAndTemplateTags(offering, template, Set.of("tag1", "tag2", "tag3", "tag4"))); + Set actualMissingTags = host.getHostServiceOfferingAndTemplateMissingTags(offering, template, Set.of("tag1", "tag2", "tag3", "tag4")); + assertEquals(Set.of("tag4"), actualMissingTags); + offering.setHostTag("tag1,tag2"); template = Mockito.mock(VirtualMachineTemplate.class); Mockito.when(template.getTemplateTag()).thenReturn("tag3"); - assertFalse(host.checkHostServiceOfferingAndTemplateTags(offering, template)); + actualMissingTags = host.getHostServiceOfferingAndTemplateMissingTags(offering, template, Set.of("tag1", "tag2", "tag3", "tag4")); + assertFalse(host.checkHostServiceOfferingAndTemplateTags(offering, template, Set.of("tag1", "tag2", "tag3", "tag4"))); + assertEquals(Set.of("tag3"), actualMissingTags); } } diff --git a/engine/schema/src/test/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImplTest.java b/engine/schema/src/test/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImplTest.java index e13ad42ec808..6de8960ae747 100644 --- a/engine/schema/src/test/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImplTest.java @@ -19,11 +19,9 @@ package com.cloud.network.as.dao; -import com.cloud.network.as.AutoScaleVmGroupVmMapVO; -import com.cloud.utils.db.GenericSearchBuilder; -import com.cloud.utils.db.SearchBuilder; -import com.cloud.utils.db.SearchCriteria; -import com.cloud.vm.VirtualMachine; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import org.junit.Assert; import org.junit.Before; @@ -33,9 +31,13 @@ import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; -import java.util.Arrays; -import java.util.List; +import com.cloud.network.as.AutoScaleVmGroupVmMapVO; +import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.vm.VirtualMachine; @RunWith(MockitoJUnitRunner.class) public class AutoScaleVmGroupVmMapDaoImplTest { @@ -198,4 +200,33 @@ public void testRemoveByGroupFailed() { Mockito.verify(searchCriteriaAutoScaleVmGroupVmMapVOMock).setParameters("vmGroupId", groupId); Mockito.verify(AutoScaleVmGroupVmMapDaoImplSpy).remove(searchCriteriaAutoScaleVmGroupVmMapVOMock); } + + @Test + public void testExpungeByVmListNoVms() { + Assert.assertEquals(0, AutoScaleVmGroupVmMapDaoImplSpy.expungeByVmList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, AutoScaleVmGroupVmMapDaoImplSpy.expungeByVmList( + null, 100L)); + } + + @Test + public void testExpungeByVmList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(AutoScaleVmGroupVmMapDaoImplSpy).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(AutoScaleVmGroupVmMapDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final AutoScaleVmGroupVmMapVO mockedVO = Mockito.mock(AutoScaleVmGroupVmMapVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), AutoScaleVmGroupVmMapDaoImplSpy.expungeByVmList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("vmIds", array); + Mockito.verify(AutoScaleVmGroupVmMapDaoImplSpy, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } } diff --git a/engine/schema/src/test/java/com/cloud/network/dao/IPAddressDaoImplTest.java b/engine/schema/src/test/java/com/cloud/network/dao/IPAddressDaoImplTest.java new file mode 100644 index 000000000000..d8f6a08d8d33 --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/network/dao/IPAddressDaoImplTest.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.dao; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class IPAddressDaoImplTest { + + @Spy + IPAddressDaoImpl ipAddressDaoImplSpy; + + @Test + public void testExpungeByVmListNoVms() { + Assert.assertEquals(0, ipAddressDaoImplSpy.expungeByVmList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, ipAddressDaoImplSpy.expungeByVmList( + null, 100L)); + } + + @Test + public void testExpungeByVmList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(ipAddressDaoImplSpy).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(ipAddressDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final IPAddressVO mockedVO = Mockito.mock(IPAddressVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), ipAddressDaoImplSpy.expungeByVmList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("vmIds", array); + Mockito.verify(ipAddressDaoImplSpy, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } +} diff --git a/engine/schema/src/test/java/com/cloud/network/dao/InlineLoadBalancerNicMapDaoImplTest.java b/engine/schema/src/test/java/com/cloud/network/dao/InlineLoadBalancerNicMapDaoImplTest.java new file mode 100644 index 000000000000..8e06c7618f6a --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/network/dao/InlineLoadBalancerNicMapDaoImplTest.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.dao; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class InlineLoadBalancerNicMapDaoImplTest { + + @Spy + InlineLoadBalancerNicMapDaoImpl inlineLoadBalancerNicMapDaoImplSpy; + + @Test + public void testExpungeByNicListNoVms() { + Assert.assertEquals(0, inlineLoadBalancerNicMapDaoImplSpy.expungeByNicList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, inlineLoadBalancerNicMapDaoImplSpy.expungeByNicList( + null, 100L)); + } + + @Test + public void testExpungeByNicList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(inlineLoadBalancerNicMapDaoImplSpy).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(inlineLoadBalancerNicMapDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final InlineLoadBalancerNicMapVO mockedVO = Mockito.mock(InlineLoadBalancerNicMapVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), inlineLoadBalancerNicMapDaoImplSpy.expungeByNicList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("nicIds", array); + Mockito.verify(inlineLoadBalancerNicMapDaoImplSpy, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } +} diff --git a/engine/schema/src/test/java/com/cloud/network/dao/LoadBalancerVMMapDaoImplTest.java b/engine/schema/src/test/java/com/cloud/network/dao/LoadBalancerVMMapDaoImplTest.java new file mode 100644 index 000000000000..fa9571949031 --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/network/dao/LoadBalancerVMMapDaoImplTest.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.dao; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class LoadBalancerVMMapDaoImplTest { + + @Spy + LoadBalancerVMMapDaoImpl loadBalancerVMMapDaoImplSpy; + + @Test + public void testExpungeByVmListNoVms() { + Assert.assertEquals(0, loadBalancerVMMapDaoImplSpy.expungeByVmList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, loadBalancerVMMapDaoImplSpy.expungeByVmList( + null, 100L)); + } + + @Test + public void testExpungeByVmList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(loadBalancerVMMapDaoImplSpy).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(loadBalancerVMMapDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final LoadBalancerVMMapVO mockedVO = Mockito.mock(LoadBalancerVMMapVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), loadBalancerVMMapDaoImplSpy.expungeByVmList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("vmIds", array); + Mockito.verify(loadBalancerVMMapDaoImplSpy, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } +} diff --git a/engine/schema/src/test/java/com/cloud/network/dao/OpRouterMonitorServiceDaoImplTest.java b/engine/schema/src/test/java/com/cloud/network/dao/OpRouterMonitorServiceDaoImplTest.java new file mode 100644 index 000000000000..7d0b1b069baa --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/network/dao/OpRouterMonitorServiceDaoImplTest.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.dao; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class OpRouterMonitorServiceDaoImplTest { + + @Spy + OpRouterMonitorServiceDaoImpl opRouterMonitorServiceDaoImplSpy; + + @Test + public void testExpungeByVmListNoVms() { + Assert.assertEquals(0, opRouterMonitorServiceDaoImplSpy.expungeByVmList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, opRouterMonitorServiceDaoImplSpy.expungeByVmList( + null, 100L)); + } + + @Test + public void testExpungeByVmList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(opRouterMonitorServiceDaoImplSpy).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(opRouterMonitorServiceDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final OpRouterMonitorServiceVO mockedVO = Mockito.mock(OpRouterMonitorServiceVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), opRouterMonitorServiceDaoImplSpy.expungeByVmList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("vmIds", array); + Mockito.verify(opRouterMonitorServiceDaoImplSpy, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } +} diff --git a/engine/schema/src/test/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImplTest.java b/engine/schema/src/test/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImplTest.java new file mode 100644 index 000000000000..c60e9b1f1bfa --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImplTest.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.rules.dao; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class PortForwardingRulesDaoImplTest { + + @Spy + PortForwardingRulesDaoImpl portForwardingRulesDaoImplSpy; + + @Test + public void testExpungeByVmListNoVms() { + Assert.assertEquals(0, portForwardingRulesDaoImplSpy.expungeByVmList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, portForwardingRulesDaoImplSpy.expungeByVmList( + null, 100L)); + } + + @Test + public void testExpungeByVmList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(portForwardingRulesDaoImplSpy).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(portForwardingRulesDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final PortForwardingRuleVO mockedVO = Mockito.mock(PortForwardingRuleVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), portForwardingRulesDaoImplSpy.expungeByVmList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("vmIds", array); + Mockito.verify(portForwardingRulesDaoImplSpy, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } +} diff --git a/engine/schema/src/test/java/com/cloud/secstorage/CommandExecLogDaoImplTest.java b/engine/schema/src/test/java/com/cloud/secstorage/CommandExecLogDaoImplTest.java new file mode 100644 index 000000000000..f86df6bdd36f --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/secstorage/CommandExecLogDaoImplTest.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.secstorage; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class CommandExecLogDaoImplTest { + + @Spy + CommandExecLogDaoImpl commandExecLogDaoImplSpy; + + @Test + public void testExpungeByVmListNoVms() { + Assert.assertEquals(0, commandExecLogDaoImplSpy.expungeByVmList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, commandExecLogDaoImplSpy.expungeByVmList( + null, 100L)); + } + + @Test + public void testExpungeByVmList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(commandExecLogDaoImplSpy).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(commandExecLogDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final CommandExecLogVO mockedVO = Mockito.mock(CommandExecLogVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), commandExecLogDaoImplSpy.expungeByVmList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("vmIds", array); + Mockito.verify(commandExecLogDaoImplSpy, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } +} diff --git a/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java b/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java index 7968ee4a375e..9445efeb089c 100644 --- a/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/storage/dao/VolumeDaoImplTest.java @@ -26,16 +26,25 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.collections.CollectionUtils; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import com.cloud.storage.VolumeVO; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; @RunWith(MockitoJUnitRunner.class) @@ -48,6 +57,7 @@ public class VolumeDaoImplTest { private static MockedStatic mockedTransactionLegacy; + @Spy private final VolumeDaoImpl volumeDao = new VolumeDaoImpl(); @BeforeClass @@ -102,4 +112,34 @@ public void testListPoolIdsByVolumeCount_without_cluster_details() throws SQLExc verify(preparedStatementMock, times(2)).setLong(anyInt(), anyLong()); verify(preparedStatementMock, times(1)).executeQuery(); } + + @Test + public void testSearchRemovedByVmsNoVms() { + Assert.assertTrue(CollectionUtils.isEmpty(volumeDao.searchRemovedByVms( + new ArrayList<>(), 100L))); + Assert.assertTrue(CollectionUtils.isEmpty(volumeDao.searchRemovedByVms( + null, 100L))); + } + + @Test + public void testSearchRemovedByVms() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doReturn(new ArrayList<>()).when(volumeDao).searchIncludingRemoved( + Mockito.any(SearchCriteria.class), Mockito.any(Filter.class), Mockito.eq(null), + Mockito.eq(false)); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(sb); + final VolumeVO mockedVO = Mockito.mock(VolumeVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + volumeDao.searchRemovedByVms(List.of(1L, 2L), batchSize); + Mockito.verify(sc).setParameters("vmIds", array); + Mockito.verify(volumeDao, Mockito.times(1)).searchIncludingRemoved( + Mockito.any(SearchCriteria.class), Mockito.any(Filter.class), Mockito.eq(null), + Mockito.eq(false)); + } + } diff --git a/engine/schema/src/test/java/com/cloud/vm/ItWorkDaoImplTest.java b/engine/schema/src/test/java/com/cloud/vm/ItWorkDaoImplTest.java new file mode 100644 index 000000000000..04bc125e05f8 --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/vm/ItWorkDaoImplTest.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.vm; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class ItWorkDaoImplTest { + + @Spy + ItWorkDaoImpl itWorkDaoImplSpy; + + @Test + public void testExpungeByVmListNoVms() { + Assert.assertEquals(0, itWorkDaoImplSpy.expungeByVmList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, itWorkDaoImplSpy.expungeByVmList( + null, 100L)); + } + + @Test + public void testExpungeByVmList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(itWorkDaoImplSpy).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(itWorkDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final ItWorkVO mockedVO = Mockito.mock(ItWorkVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), itWorkDaoImplSpy.expungeByVmList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("vmIds", array); + Mockito.verify(itWorkDaoImplSpy, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } +} diff --git a/engine/schema/src/test/java/com/cloud/vm/dao/ConsoleSessionDaoImplTest.java b/engine/schema/src/test/java/com/cloud/vm/dao/ConsoleSessionDaoImplTest.java new file mode 100644 index 000000000000..c9919e26af6c --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/vm/dao/ConsoleSessionDaoImplTest.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.vm.dao; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.vm.ConsoleSessionVO; + +@RunWith(MockitoJUnitRunner.class) +public class ConsoleSessionDaoImplTest { + + @Spy + ConsoleSessionDaoImpl consoleSessionDaoImplSpy; + + @Test + public void testExpungeByVmListNoVms() { + Assert.assertEquals(0, consoleSessionDaoImplSpy.expungeByVmList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, consoleSessionDaoImplSpy.expungeByVmList( + null, 100L)); + } + + @Test + public void testExpungeByVmList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(consoleSessionDaoImplSpy).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(consoleSessionDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final ConsoleSessionVO mockedVO = Mockito.mock(ConsoleSessionVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), consoleSessionDaoImplSpy.expungeByVmList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("vmIds", array); + Mockito.verify(consoleSessionDaoImplSpy, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } +} diff --git a/engine/schema/src/test/java/com/cloud/vm/dao/NicDaoImplTest.java b/engine/schema/src/test/java/com/cloud/vm/dao/NicDaoImplTest.java new file mode 100644 index 000000000000..506fdb7fc92b --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/vm/dao/NicDaoImplTest.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.vm.dao; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.collections.CollectionUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.vm.NicVO; + +@RunWith(MockitoJUnitRunner.class) +public class NicDaoImplTest { + + @Spy + NicDaoImpl nicDaoImplSpy; + + @Test + public void testSearchRemovedByVmsNoVms() { + Assert.assertTrue(CollectionUtils.isEmpty(nicDaoImplSpy.searchRemovedByVms( + new ArrayList<>(), 100L))); + Assert.assertTrue(CollectionUtils.isEmpty(nicDaoImplSpy.searchRemovedByVms( + null, 100L))); + } + + @Test + public void testSearchRemovedByVms() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doReturn(new ArrayList<>()).when(nicDaoImplSpy).searchIncludingRemoved( + Mockito.any(SearchCriteria.class), Mockito.any(Filter.class), Mockito.eq(null), + Mockito.eq(false)); + Mockito.when(nicDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final NicVO mockedVO = Mockito.mock(NicVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + nicDaoImplSpy.searchRemovedByVms(List.of(1L, 2L), batchSize); + Mockito.verify(sc).setParameters("vmIds", array); + Mockito.verify(nicDaoImplSpy, Mockito.times(1)).searchIncludingRemoved( + Mockito.any(SearchCriteria.class), Mockito.any(Filter.class), Mockito.eq(null), + Mockito.eq(false)); + } +} diff --git a/engine/schema/src/test/java/com/cloud/vm/dao/NicExtraDhcpOptionDaoImplTest.java b/engine/schema/src/test/java/com/cloud/vm/dao/NicExtraDhcpOptionDaoImplTest.java new file mode 100644 index 000000000000..7a1e32e95caf --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/vm/dao/NicExtraDhcpOptionDaoImplTest.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.vm.dao; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.vm.NicExtraDhcpOptionVO; + +@RunWith(MockitoJUnitRunner.class) +public class NicExtraDhcpOptionDaoImplTest { + + @Spy + NicExtraDhcpOptionDaoImpl nicExtraDhcpOptionDaoImplSpy; + + @Test + public void testExpungeByNicListNoVms() { + Assert.assertEquals(0, nicExtraDhcpOptionDaoImplSpy.expungeByNicList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, nicExtraDhcpOptionDaoImplSpy.expungeByNicList( + null, 100L)); + } + + @Test + public void testExpungeByNicList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(nicExtraDhcpOptionDaoImplSpy).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(nicExtraDhcpOptionDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final NicExtraDhcpOptionVO mockedVO = Mockito.mock(NicExtraDhcpOptionVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), nicExtraDhcpOptionDaoImplSpy.expungeByNicList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("nicIds", array); + Mockito.verify(nicExtraDhcpOptionDaoImplSpy, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } +} diff --git a/engine/schema/src/test/java/com/cloud/vm/dao/NicSecondaryIpDaoImplTest.java b/engine/schema/src/test/java/com/cloud/vm/dao/NicSecondaryIpDaoImplTest.java new file mode 100644 index 000000000000..a9f798dbc017 --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/vm/dao/NicSecondaryIpDaoImplTest.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.vm.dao; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class NicSecondaryIpDaoImplTest { + + @Spy + NicSecondaryIpDaoImpl nicSecondaryIpDaoImplSpy; + + @Test + public void testExpungeByVmListNoVms() { + Assert.assertEquals(0, nicSecondaryIpDaoImplSpy.expungeByVmList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, nicSecondaryIpDaoImplSpy.expungeByVmList( + null, 100L)); + } + + @Test + public void testExpungeByVmList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(nicSecondaryIpDaoImplSpy).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(nicSecondaryIpDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final NicSecondaryIpVO mockedVO = Mockito.mock(NicSecondaryIpVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), nicSecondaryIpDaoImplSpy.expungeByVmList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("vmIds", array); + Mockito.verify(nicSecondaryIpDaoImplSpy, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } +} diff --git a/engine/schema/src/test/java/com/cloud/vm/dao/VMInstanceDaoImplTest.java b/engine/schema/src/test/java/com/cloud/vm/dao/VMInstanceDaoImplTest.java index 4a32dc083590..43679081550b 100644 --- a/engine/schema/src/test/java/com/cloud/vm/dao/VMInstanceDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/vm/dao/VMInstanceDaoImplTest.java @@ -30,6 +30,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.util.ArrayList; +import java.util.Calendar; import java.util.Date; import org.joda.time.DateTime; @@ -37,10 +39,14 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.Spy; import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; @@ -199,4 +205,29 @@ public void testUpdatePowerStateNoChangeMaxUpdatesInvalidStateVmRunning() { assertTrue(result); } + + @Test + public void testSearchRemovedByRemoveDate() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.when(vmInstanceDao.createSearchBuilder()).thenReturn(sb); + final VMInstanceVO mockedVO = Mockito.mock(VMInstanceVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + Mockito.doReturn(new ArrayList<>()).when(vmInstanceDao).searchIncludingRemoved( + Mockito.any(SearchCriteria.class), Mockito.any(Filter.class), Mockito.eq(null), + Mockito.eq(false)); + Calendar cal = Calendar.getInstance(); + Date endDate = new Date(); + cal.setTime(endDate); + cal.add(Calendar.DATE, -1 * 10); + Date startDate = cal.getTime(); + vmInstanceDao.searchRemovedByRemoveDate(startDate, endDate, 50L, new ArrayList<>()); + Mockito.verify(sc).setParameters("startDate", startDate); + Mockito.verify(sc).setParameters("endDate", endDate); + Mockito.verify(sc, Mockito.never()).setParameters(Mockito.eq("skippedVmIds"), Mockito.any()); + Mockito.verify(vmInstanceDao, Mockito.times(1)).searchIncludingRemoved( + Mockito.any(SearchCriteria.class), Mockito.any(Filter.class), Mockito.eq(null), + Mockito.eq(false)); + } } diff --git a/engine/schema/src/test/java/com/cloud/vm/snapshot/dao/VMSnapshotDaoImplTest.java b/engine/schema/src/test/java/com/cloud/vm/snapshot/dao/VMSnapshotDaoImplTest.java new file mode 100644 index 000000000000..e71518080d24 --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/vm/snapshot/dao/VMSnapshotDaoImplTest.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.vm.snapshot.dao; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.collections.CollectionUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.vm.snapshot.VMSnapshotVO; + +@RunWith(MockitoJUnitRunner.class) +public class VMSnapshotDaoImplTest { + + @Spy + VMSnapshotDaoImpl vmSnapshotDaoImplSpy; + + @Test + public void testSearchRemovedByVmsNoVms() { + Assert.assertTrue(CollectionUtils.isEmpty(vmSnapshotDaoImplSpy.searchRemovedByVms( + new ArrayList<>(), 100L))); + Assert.assertTrue(CollectionUtils.isEmpty(vmSnapshotDaoImplSpy.searchRemovedByVms( + null, 100L))); + } + + @Test + public void testSearchRemovedByVms() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doReturn(new ArrayList<>()).when(vmSnapshotDaoImplSpy).searchIncludingRemoved( + Mockito.any(SearchCriteria.class), Mockito.any(Filter.class), Mockito.eq(null), + Mockito.eq(false)); + Mockito.when(vmSnapshotDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final VMSnapshotVO mockedVO = Mockito.mock(VMSnapshotVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + vmSnapshotDaoImplSpy.searchRemovedByVms(List.of(1L, 2L), batchSize); + Mockito.verify(sc).setParameters("vmIds", array); + Mockito.verify(vmSnapshotDaoImplSpy, Mockito.times(1)).searchIncludingRemoved( + Mockito.any(SearchCriteria.class), Mockito.any(Filter.class), Mockito.eq(null), + Mockito.eq(false)); + } +} diff --git a/engine/schema/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java b/engine/schema/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java new file mode 100644 index 000000000000..85240ab4a058 --- /dev/null +++ b/engine/schema/src/test/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImplTest.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.db; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class SnapshotDataStoreDaoImplTest { + + @Spy + SnapshotDataStoreDaoImpl snapshotDataStoreDaoImplSpy; + + @Test + public void testExpungeByVmListNoVms() { + Assert.assertEquals(0, snapshotDataStoreDaoImplSpy.expungeBySnapshotList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, snapshotDataStoreDaoImplSpy.expungeBySnapshotList( + null, 100L)); + } + + @Test + public void testExpungeByVmList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(snapshotDataStoreDaoImplSpy).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(snapshotDataStoreDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final SnapshotDataStoreVO mockedVO = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), snapshotDataStoreDaoImplSpy.expungeBySnapshotList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("snapshotIds", array); + Mockito.verify(snapshotDataStoreDaoImplSpy, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } +} diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 03aa5b509888..07b538426405 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -44,6 +45,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; @@ -69,7 +71,6 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -82,7 +83,6 @@ import com.cloud.agent.api.ModifyTargetsAnswer; import com.cloud.agent.api.ModifyTargetsCommand; import com.cloud.agent.api.PrepareForMigrationCommand; -import com.cloud.agent.api.storage.CheckStorageAvailabilityCommand; import com.cloud.agent.api.storage.CopyVolumeAnswer; import com.cloud.agent.api.storage.CopyVolumeCommand; import com.cloud.agent.api.storage.MigrateVolumeAnswer; @@ -104,6 +104,7 @@ import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.MigrationOptions; +import com.cloud.storage.ScopeType; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage; @@ -141,12 +142,16 @@ import java.util.stream.Collectors; import org.apache.commons.collections.CollectionUtils; +import static org.apache.cloudstack.vm.UnmanagedVMsManagerImpl.KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME; +import static org.apache.cloudstack.vm.UnmanagedVMsManagerImpl.VM_IMPORT_DEFAULT_TEMPLATE_NAME; + public class StorageSystemDataMotionStrategy implements DataMotionStrategy { protected Logger logger = LogManager.getLogger(getClass()); private static final Random RANDOM = new Random(System.nanoTime()); private static final int LOCK_TIME_IN_SECONDS = 300; private static final String OPERATION_NOT_SUPPORTED = "This operation is not supported."; + @Inject protected AgentManager agentManager; @Inject @@ -684,8 +689,6 @@ private Scope getZoneScope(Scope scope) { private void handleVolumeMigrationFromNonManagedStorageToManagedStorage(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, AsyncCompletionCallback callback) { - String errMsg = null; - try { HypervisorType hypervisorType = srcVolumeInfo.getHypervisorType(); @@ -696,37 +699,21 @@ private void handleVolumeMigrationFromNonManagedStorageToManagedStorage(VolumeIn if (HypervisorType.XenServer.equals(hypervisorType)) { handleVolumeMigrationForXenServer(srcVolumeInfo, destVolumeInfo); - } - else { + destVolumeInfo = _volumeDataFactory.getVolume(destVolumeInfo.getId(), destVolumeInfo.getDataStore()); + DataTO dataTO = destVolumeInfo.getTO(); + CopyCmdAnswer copyCmdAnswer = new CopyCmdAnswer(dataTO); + CopyCommandResult result = new CopyCommandResult(null, copyCmdAnswer); + callback.complete(result); + } else { handleVolumeMigrationForKVM(srcVolumeInfo, destVolumeInfo, callback); } } catch (Exception ex) { - errMsg = "Migration operation failed in 'StorageSystemDataMotionStrategy.handleVolumeMigrationFromNonManagedStorageToManagedStorage': " + + String errMsg = "Migration operation failed in 'StorageSystemDataMotionStrategy.handleVolumeMigrationFromNonManagedStorageToManagedStorage': " + ex.getMessage(); throw new CloudRuntimeException(errMsg, ex); } - finally { - CopyCmdAnswer copyCmdAnswer; - - if (errMsg != null) { - copyCmdAnswer = new CopyCmdAnswer(errMsg); - } - else { - destVolumeInfo = _volumeDataFactory.getVolume(destVolumeInfo.getId(), destVolumeInfo.getDataStore()); - - DataTO dataTO = destVolumeInfo.getTO(); - - copyCmdAnswer = new CopyCmdAnswer(dataTO); - } - - CopyCommandResult result = new CopyCommandResult(null, copyCmdAnswer); - - result.setResult(errMsg); - - callback.complete(result); - } } private void handleVolumeMigrationForXenServer(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo) { @@ -845,12 +832,25 @@ private void handleVolumeMigrationForKVM(VolumeInfo srcVolumeInfo, VolumeInfo de checkAvailableForMigration(vm); String errMsg = null; + HostVO hostVO = null; try { destVolumeInfo.getDataStore().getDriver().createAsync(destVolumeInfo.getDataStore(), destVolumeInfo, null); VolumeVO volumeVO = _volumeDao.findById(destVolumeInfo.getId()); updatePathFromScsiName(volumeVO); destVolumeInfo = _volumeDataFactory.getVolume(destVolumeInfo.getId(), destVolumeInfo.getDataStore()); - HostVO hostVO = getHostOnWhichToExecuteMigrationCommand(srcVolumeInfo, destVolumeInfo); + hostVO = getHostOnWhichToExecuteMigrationCommand(srcVolumeInfo, destVolumeInfo); + + // if managed we need to grant access + PrimaryDataStore pds = (PrimaryDataStore)this.dataStoreMgr.getPrimaryDataStore(destVolumeInfo.getDataStore().getUuid()); + if (pds == null) { + throw new CloudRuntimeException("Unable to find primary data store driver for this volume"); + } + + // grant access (for managed volumes) + _volumeService.grantAccess(destVolumeInfo, hostVO, destVolumeInfo.getDataStore()); + + // re-retrieve volume to get any updated information from grant + destVolumeInfo = _volumeDataFactory.getVolume(destVolumeInfo.getId(), destVolumeInfo.getDataStore()); // migrate the volume via the hypervisor String path = migrateVolumeForKVM(srcVolumeInfo, destVolumeInfo, hostVO, "Unable to migrate the volume from non-managed storage to managed storage"); @@ -871,6 +871,18 @@ private void handleVolumeMigrationForKVM(VolumeInfo srcVolumeInfo, VolumeInfo de throw new CloudRuntimeException(errMsg, ex); } } finally { + // revoke access (for managed volumes) + if (hostVO != null) { + try { + _volumeService.revokeAccess(destVolumeInfo, hostVO, destVolumeInfo.getDataStore()); + } catch (Exception e) { + logger.warn(String.format("Failed to revoke access for volume 'name=%s,uuid=%s' after a migration attempt", destVolumeInfo.getVolume(), destVolumeInfo.getUuid()), e); + } + } + + // re-retrieve volume to get any updated information from grant + destVolumeInfo = _volumeDataFactory.getVolume(destVolumeInfo.getId(), destVolumeInfo.getDataStore()); + CopyCmdAnswer copyCmdAnswer; if (errMsg != null) { copyCmdAnswer = new CopyCmdAnswer(errMsg); @@ -911,16 +923,141 @@ private HostVO getHostOnWhichToExecuteMigrationCommand(VolumeInfo srcVolumeInfo, HostVO hostVO; - if (srcStoragePoolVO.getClusterId() != null) { - hostVO = getHostInCluster(srcStoragePoolVO.getClusterId()); - } - else { - hostVO = getHost(destVolumeInfo.getDataCenterId(), HypervisorType.KVM, false); + // if either source or destination is a HOST-scoped storage pool, the migration MUST be performed on that host + if (ScopeType.HOST.equals(srcVolumeInfo.getDataStore().getScope().getScopeType())) { + hostVO = _hostDao.findById(srcVolumeInfo.getDataStore().getScope().getScopeId()); + } else if (ScopeType.HOST.equals(destVolumeInfo.getDataStore().getScope().getScopeType())) { + hostVO = _hostDao.findById(destVolumeInfo.getDataStore().getScope().getScopeId()); + } else { + if (srcStoragePoolVO.getClusterId() != null) { + hostVO = getHostInCluster(srcStoragePoolVO.getClusterId()); + } else { + hostVO = getHost(destVolumeInfo.getDataCenterId(), HypervisorType.KVM, false); + } } return hostVO; } + private VolumeInfo createTemporaryVolumeCopyOfSnapshotAdaptive(SnapshotInfo snapshotInfo) { + VolumeInfo tempVolumeInfo = null; + VolumeVO tempVolumeVO = null; + try { + tempVolumeVO = new VolumeVO(Volume.Type.DATADISK, snapshotInfo.getName() + "_" + System.currentTimeMillis() + ".TMP", + snapshotInfo.getDataCenterId(), snapshotInfo.getDomainId(), snapshotInfo.getAccountId(), 0, ProvisioningType.THIN, snapshotInfo.getSize(), 0L, 0L, ""); + tempVolumeVO.setPoolId(snapshotInfo.getDataStore().getId()); + _volumeDao.persist(tempVolumeVO); + tempVolumeInfo = this._volFactory.getVolume(tempVolumeVO.getId()); + + if (snapshotInfo.getDataStore().getDriver().canCopy(snapshotInfo, tempVolumeInfo)) { + snapshotInfo.getDataStore().getDriver().copyAsync(snapshotInfo, tempVolumeInfo, null, null); + // refresh volume info as data could have changed + tempVolumeInfo = this._volFactory.getVolume(tempVolumeVO.getId()); + } else { + throw new CloudRuntimeException("Storage driver indicated it could create a volume from the snapshot but rejected the subsequent request to do so"); + } + return tempVolumeInfo; + } catch (Throwable e) { + try { + if (tempVolumeInfo != null) { + tempVolumeInfo.getDataStore().getDriver().deleteAsync(tempVolumeInfo.getDataStore(), tempVolumeInfo, null); + } + + // cleanup temporary volume + if (tempVolumeVO != null) { + _volumeDao.remove(tempVolumeVO.getId()); + } + } catch (Throwable e2) { + logger.warn("Failed to delete temporary volume created for copy", e2); + } + + throw e; + } + } + + /** + * Simplier logic for copy from snapshot for adaptive driver only. + * @param snapshotInfo + * @param destData + * @param callback + */ + private void handleCopyAsyncToSecondaryStorageAdaptive(SnapshotInfo snapshotInfo, DataObject destData, AsyncCompletionCallback callback) { + CopyCmdAnswer copyCmdAnswer = null; + DataObject srcFinal = null; + HostVO hostVO = null; + DataStore srcDataStore = null; + boolean tempRequired = false; + + try { + snapshotInfo.processEvent(Event.CopyingRequested); + hostVO = getHost(snapshotInfo); + DataObject destOnStore = destData; + srcDataStore = snapshotInfo.getDataStore(); + int primaryStorageDownloadWait = StorageManager.PRIMARY_STORAGE_DOWNLOAD_WAIT.value(); + CopyCommand copyCommand = null; + if (!Boolean.parseBoolean(srcDataStore.getDriver().getCapabilities().get("CAN_DIRECT_ATTACH_SNAPSHOT"))) { + srcFinal = createTemporaryVolumeCopyOfSnapshotAdaptive(snapshotInfo); + tempRequired = true; + } else { + srcFinal = snapshotInfo; + } + + _volumeService.grantAccess(srcFinal, hostVO, srcDataStore); + + DataTO srcTo = srcFinal.getTO(); + + // have to set PATH as extraOptions due to logic in KVM hypervisor processor + HashMap extraDetails = new HashMap<>(); + extraDetails.put(DiskTO.PATH, srcTo.getPath()); + + copyCommand = new CopyCommand(srcFinal.getTO(), destOnStore.getTO(), primaryStorageDownloadWait, + VirtualMachineManager.ExecuteInSequence.value()); + copyCommand.setOptions(extraDetails); + copyCmdAnswer = (CopyCmdAnswer)agentManager.send(hostVO.getId(), copyCommand); + } catch (Exception ex) { + String msg = "Failed to create template from snapshot (Snapshot ID = " + snapshotInfo.getId() + ") : "; + logger.warn(msg, ex); + throw new CloudRuntimeException(msg + ex.getMessage(), ex); + } + finally { + // remove access tot he volume that was used + if (srcFinal != null && hostVO != null && srcDataStore != null) { + _volumeService.revokeAccess(srcFinal, hostVO, srcDataStore); + } + + // delete the temporary volume if it was needed + if (srcFinal != null && tempRequired) { + try { + srcFinal.getDataStore().getDriver().deleteAsync(srcFinal.getDataStore(), srcFinal, null); + } catch (Throwable e) { + logger.warn("Failed to delete temporary volume created for copy", e); + } + } + + // check we have a reasonable result + String errMsg = null; + if (copyCmdAnswer == null || (!copyCmdAnswer.getResult() && copyCmdAnswer.getDetails() == null)) { + errMsg = "Unable to create template from snapshot"; + copyCmdAnswer = new CopyCmdAnswer(errMsg); + } else if (!copyCmdAnswer.getResult() && StringUtils.isEmpty(copyCmdAnswer.getDetails())) { + errMsg = "Unable to create template from snapshot"; + } else if (!copyCmdAnswer.getResult()) { + errMsg = copyCmdAnswer.getDetails(); + } + + //submit processEvent + if (StringUtils.isEmpty(errMsg)) { + snapshotInfo.processEvent(Event.OperationSuccessed); + } else { + snapshotInfo.processEvent(Event.OperationFailed); + } + + CopyCommandResult result = new CopyCommandResult(null, copyCmdAnswer); + result.setResult(copyCmdAnswer.getDetails()); + callback.complete(result); + } + } + /** * This function is responsible for copying a snapshot from managed storage to secondary storage. This is used in the following two cases: * 1) When creating a template from a snapshot @@ -931,6 +1068,13 @@ private HostVO getHostOnWhichToExecuteMigrationCommand(VolumeInfo srcVolumeInfo, * @param callback callback for async */ private void handleCopyAsyncToSecondaryStorage(SnapshotInfo snapshotInfo, DataObject destData, AsyncCompletionCallback callback) { + + // if this flag is set (true or false), we will fall out to use simplier logic for the Adaptive handler + if (snapshotInfo.getDataStore().getDriver().getCapabilities().get("CAN_DIRECT_ATTACH_SNAPSHOT") != null) { + handleCopyAsyncToSecondaryStorageAdaptive(snapshotInfo, destData, callback); + return; + } + String errMsg = null; CopyCmdAnswer copyCmdAnswer = null; boolean usingBackendSnapshot = false; @@ -1697,14 +1841,13 @@ private void handleCreateVolumeFromVolumeOnSecondaryStorage(VolumeInfo srcVolume private CopyCmdAnswer copyImageToVolume(DataObject srcDataObject, VolumeInfo destVolumeInfo, HostVO hostVO) { int primaryStorageDownloadWait = StorageManager.PRIMARY_STORAGE_DOWNLOAD_WAIT.value(); - CopyCommand copyCommand = new CopyCommand(srcDataObject.getTO(), destVolumeInfo.getTO(), primaryStorageDownloadWait, - VirtualMachineManager.ExecuteInSequence.value()); - CopyCmdAnswer copyCmdAnswer; try { _volumeService.grantAccess(destVolumeInfo, hostVO, destVolumeInfo.getDataStore()); + CopyCommand copyCommand = new CopyCommand(srcDataObject.getTO(), destVolumeInfo.getTO(), primaryStorageDownloadWait, + VirtualMachineManager.ExecuteInSequence.value()); Map destDetails = getVolumeDetails(destVolumeInfo); copyCommand.setOptions2(destDetails); @@ -1729,42 +1872,6 @@ private CopyCmdAnswer copyImageToVolume(DataObject srcDataObject, VolumeInfo des return copyCmdAnswer; } - /** - * Use normal volume semantics (create a volume known to cloudstack, ask the storage driver to create it as a copy of the snapshot) - - * @param volumeVO - * @param snapshotInfo - */ - public void prepTempVolumeForCopyFromSnapshot(SnapshotInfo snapshotInfo) { - VolumeVO volumeVO = null; - try { - volumeVO = new VolumeVO(Volume.Type.DATADISK, snapshotInfo.getName() + "_" + System.currentTimeMillis() + ".TMP", - snapshotInfo.getDataCenterId(), snapshotInfo.getDomainId(), snapshotInfo.getAccountId(), 0, ProvisioningType.THIN, snapshotInfo.getSize(), 0L, 0L, ""); - volumeVO.setPoolId(snapshotInfo.getDataStore().getId()); - _volumeDao.persist(volumeVO); - VolumeInfo tempVolumeInfo = this._volFactory.getVolume(volumeVO.getId()); - - if (snapshotInfo.getDataStore().getDriver().canCopy(snapshotInfo, tempVolumeInfo)) { - snapshotInfo.getDataStore().getDriver().copyAsync(snapshotInfo, tempVolumeInfo, null, null); - // refresh volume info as data could have changed - tempVolumeInfo = this._volFactory.getVolume(volumeVO.getId()); - // save the "temp" volume info into the snapshot details (we need this to clean up at the end) - _snapshotDetailsDao.addDetail(snapshotInfo.getId(), "TemporaryVolumeCopyUUID", tempVolumeInfo.getUuid(), true); - _snapshotDetailsDao.addDetail(snapshotInfo.getId(), "TemporaryVolumeCopyPath", tempVolumeInfo.getPath(), true); - // NOTE: for this to work, the Driver must return a custom SnapshotObjectTO object from getTO() - // whenever the TemporaryVolumeCopyPath is set. - } else { - throw new CloudRuntimeException("Storage driver indicated it could create a volume from the snapshot but rejected the subsequent request to do so"); - } - } catch (Throwable e) { - // cleanup temporary volume - if (volumeVO != null) { - _volumeDao.remove(volumeVO.getId()); - } - throw e; - } - } - /** * If the underlying storage system is making use of read-only snapshots, this gives the storage system the opportunity to * create a volume from the snapshot so that we can copy the VHD file that should be inside of the snapshot to secondary storage. @@ -1776,13 +1883,8 @@ public void prepTempVolumeForCopyFromSnapshot(SnapshotInfo snapshotInfo) { * resign the SR and the VDI that should be inside of the snapshot before copying the VHD file to secondary storage. */ private void createVolumeFromSnapshot(SnapshotInfo snapshotInfo) { - if ("true".equalsIgnoreCase(snapshotInfo.getDataStore().getDriver().getCapabilities().get("CAN_CREATE_TEMP_VOLUME_FROM_SNAPSHOT"))) { - prepTempVolumeForCopyFromSnapshot(snapshotInfo); - return; - - } - SnapshotDetailsVO snapshotDetails = handleSnapshotDetails(snapshotInfo.getId(), "create"); + try { snapshotInfo.getDataStore().getDriver().createAsync(snapshotInfo.getDataStore(), snapshotInfo, null); } @@ -1797,31 +1899,20 @@ private void createVolumeFromSnapshot(SnapshotInfo snapshotInfo) { * invocation of createVolumeFromSnapshot(SnapshotInfo). */ private void deleteVolumeFromSnapshot(SnapshotInfo snapshotInfo) { - VolumeVO volumeVO = null; - // cleanup any temporary volume previously created for copy from a snapshot - if ("true".equalsIgnoreCase(snapshotInfo.getDataStore().getDriver().getCapabilities().get("CAN_CREATE_TEMP_VOLUME_FROM_SNAPSHOT"))) { - SnapshotDetailsVO tempUuid = null; - tempUuid = _snapshotDetailsDao.findDetail(snapshotInfo.getId(), "TemporaryVolumeCopyUUID"); - if (tempUuid == null || tempUuid.getValue() == null) { - return; - } + try { + logger.debug("Cleaning up temporary volume created for copy from a snapshot"); - volumeVO = _volumeDao.findByUuid(tempUuid.getValue()); - if (volumeVO != null) { - _volumeDao.remove(volumeVO.getId()); - } - _snapshotDetailsDao.remove(tempUuid.getId()); - _snapshotDetailsDao.removeDetail(snapshotInfo.getId(), "TemporaryVolumeCopyUUID"); - return; - } + SnapshotDetailsVO snapshotDetails = handleSnapshotDetails(snapshotInfo.getId(), "delete"); - SnapshotDetailsVO snapshotDetails = handleSnapshotDetails(snapshotInfo.getId(), "delete"); + try { + snapshotInfo.getDataStore().getDriver().createAsync(snapshotInfo.getDataStore(), snapshotInfo, null); + } + finally { + _snapshotDetailsDao.remove(snapshotDetails.getId()); + } - try { - snapshotInfo.getDataStore().getDriver().createAsync(snapshotInfo.getDataStore(), snapshotInfo, null); - } - finally { - _snapshotDetailsDao.remove(snapshotDetails.getId()); + } catch (Throwable e) { + logger.warn("Failed to clean up temporary volume created for copy from a snapshot, transction will not be failed but an adminstrator should clean this up: " + snapshotInfo.getUuid() + " - " + snapshotInfo.getPath(), e); } } @@ -1906,7 +1997,7 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach throw new CloudRuntimeException("Invalid hypervisor type (only KVM supported for this operation at the time being)"); } - verifyLiveMigrationForKVM(volumeDataStoreMap, destHost); + verifyLiveMigrationForKVM(volumeDataStoreMap); VMInstanceVO vmInstance = _vmDao.findById(vmTO.getId()); vmTO.setState(vmInstance.getState()); @@ -1933,7 +2024,10 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach continue; } - if (srcVolumeInfo.getTemplateId() != null) { + VMTemplateVO vmTemplate = _vmTemplateDao.findById(vmInstance.getTemplateId()); + if (srcVolumeInfo.getTemplateId() != null && + Objects.nonNull(vmTemplate) && + !Arrays.asList(KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME, VM_IMPORT_DEFAULT_TEMPLATE_NAME).contains(vmTemplate.getName())) { logger.debug(String.format("Copying template [%s] of volume [%s] from source storage pool [%s] to target storage pool [%s].", srcVolumeInfo.getTemplateId(), srcVolumeInfo.getId(), sourceStoragePool.getId(), destStoragePool.getId())); copyTemplateToTargetFilesystemStorageIfNeeded(srcVolumeInfo, sourceStoragePool, destDataStore, destStoragePool, destHost); } else { @@ -1977,8 +2071,8 @@ public void copyAsync(Map volumeDataStoreMap, VirtualMach MigrateCommand.MigrateDiskInfo migrateDiskInfo; - boolean isNonManagedNfsToNfsOrSharedMountPointToNfs = supportStoragePoolType(sourceStoragePool.getPoolType()) && destStoragePool.getPoolType() == StoragePoolType.NetworkFilesystem && !managedStorageDestination; - if (isNonManagedNfsToNfsOrSharedMountPointToNfs) { + boolean isNonManagedToNfs = supportStoragePoolType(sourceStoragePool.getPoolType(), StoragePoolType.Filesystem) && destStoragePool.getPoolType() == StoragePoolType.NetworkFilesystem && !managedStorageDestination; + if (isNonManagedToNfs) { migrateDiskInfo = new MigrateCommand.MigrateDiskInfo(srcVolumeInfo.getPath(), MigrateCommand.MigrateDiskInfo.DiskType.FILE, MigrateCommand.MigrateDiskInfo.DriverType.QCOW2, @@ -2152,7 +2246,7 @@ String getVolumeBackingFile(VolumeInfo srcVolumeInfo) { if (srcVolumeInfo.getHypervisorType() == HypervisorType.KVM && srcVolumeInfo.getTemplateId() != null && srcVolumeInfo.getPoolId() != null) { VMTemplateVO template = _vmTemplateDao.findById(srcVolumeInfo.getTemplateId()); - if (template.getFormat() != null && template.getFormat() != Storage.ImageFormat.ISO) { + if (Objects.nonNull(template) && template.getFormat() != null && template.getFormat() != Storage.ImageFormat.ISO) { VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(srcVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId(), null); return ref != null ? ref.getInstallPath() : null; } @@ -2357,9 +2451,8 @@ protected void prepareDiskWithSecretConsumerDetail(VirtualMachineTO vmTO, Volume * At a high level: The source storage cannot be managed and * the destination storages can be all managed or all not managed, not mixed. */ - protected void verifyLiveMigrationForKVM(Map volumeDataStoreMap, Host destHost) { + protected void verifyLiveMigrationForKVM(Map volumeDataStoreMap) { Boolean storageTypeConsistency = null; - Map sourcePools = new HashMap<>(); for (Map.Entry entry : volumeDataStoreMap.entrySet()) { VolumeInfo volumeInfo = entry.getKey(); @@ -2386,47 +2479,6 @@ protected void verifyLiveMigrationForKVM(Map volumeDataSt } else if (storageTypeConsistency != destStoragePoolVO.isManaged()) { throw new CloudRuntimeException("Destination storage pools must be either all managed or all not managed"); } - - addSourcePoolToPoolsMap(sourcePools, srcStoragePoolVO, destStoragePoolVO); - } - verifyDestinationStorage(sourcePools, destHost); - } - - /** - * Adds source storage pool to the migration map if the destination pool is not managed and it is NFS. - */ - protected void addSourcePoolToPoolsMap(Map sourcePools, StoragePoolVO srcStoragePoolVO, StoragePoolVO destStoragePoolVO) { - if (destStoragePoolVO.isManaged() || !StoragePoolType.NetworkFilesystem.equals(destStoragePoolVO.getPoolType())) { - logger.trace(String.format("Skipping adding source pool [%s] to map due to destination pool [%s] is managed or not NFS.", srcStoragePoolVO, destStoragePoolVO)); - return; - } - - String sourceStoragePoolUuid = srcStoragePoolVO.getUuid(); - if (!sourcePools.containsKey(sourceStoragePoolUuid)) { - sourcePools.put(sourceStoragePoolUuid, srcStoragePoolVO.getPoolType()); - } - } - - /** - * Perform storage validation on destination host for KVM live storage migrations. - * Validate that volume source storage pools are mounted on the destination host prior the migration - * @throws CloudRuntimeException if any source storage pool is not mounted on the destination host - */ - private void verifyDestinationStorage(Map sourcePools, Host destHost) { - if (MapUtils.isNotEmpty(sourcePools)) { - logger.debug("Verifying source pools are already available on destination host " + destHost.getUuid()); - CheckStorageAvailabilityCommand cmd = new CheckStorageAvailabilityCommand(sourcePools); - try { - Answer answer = agentManager.send(destHost.getId(), cmd); - if (answer == null || !answer.getResult()) { - throw new CloudRuntimeException("Storage verification failed on host " - + destHost.getUuid() +": " + answer.getDetails()); - } - } catch (AgentUnavailableException | OperationTimedoutException e) { - e.printStackTrace(); - throw new CloudRuntimeException("Cannot perform storage verification on host " + destHost.getUuid() + - "due to: " + e.getMessage()); - } } } @@ -2497,15 +2549,15 @@ private void handleCreateTemplateFromManagedVolume(VolumeInfo volumeInfo, Templa int primaryStorageDownloadWait = StorageManager.PRIMARY_STORAGE_DOWNLOAD_WAIT.value(); - CopyCommand copyCommand = new CopyCommand(volumeInfo.getTO(), templateInfo.getTO(), primaryStorageDownloadWait, VirtualMachineManager.ExecuteInSequence.value()); - try { handleQualityOfServiceForVolumeMigration(volumeInfo, PrimaryDataStoreDriver.QualityOfServiceState.MIGRATION); - if (srcVolumeDetached || StoragePoolType.PowerFlex == storagePoolVO.getPoolType()) { + if (srcVolumeDetached || StoragePoolType.PowerFlex == storagePoolVO.getPoolType() || StoragePoolType.FiberChannel == storagePoolVO.getPoolType()) { _volumeService.grantAccess(volumeInfo, hostVO, srcDataStore); } + CopyCommand copyCommand = new CopyCommand(volumeInfo.getTO(), templateInfo.getTO(), primaryStorageDownloadWait, VirtualMachineManager.ExecuteInSequence.value()); + Map srcDetails = getVolumeDetails(volumeInfo); copyCommand.setOptions(srcDetails); @@ -2534,7 +2586,7 @@ private void handleCreateTemplateFromManagedVolume(VolumeInfo volumeInfo, Templa throw new CloudRuntimeException(msg + ex.getMessage(), ex); } finally { - if (srcVolumeDetached || StoragePoolType.PowerFlex == storagePoolVO.getPoolType()) { + if (srcVolumeDetached || StoragePoolType.PowerFlex == storagePoolVO.getPoolType() || StoragePoolType.FiberChannel == storagePoolVO.getPoolType()) { try { _volumeService.revokeAccess(volumeInfo, hostVO, srcDataStore); } @@ -2629,13 +2681,7 @@ private Map getSnapshotDetails(SnapshotInfo snapshotInfo) { long snapshotId = snapshotInfo.getId(); - // if the snapshot required a temporary volume be created check if the UUID is set so we can - // retrieve the temporary volume's path to use during remote copy - List storedDetails = _snapshotDetailsDao.findDetails(snapshotInfo.getId(), "TemporaryVolumeCopyPath"); - if (storedDetails != null && storedDetails.size() > 0) { - String value = storedDetails.get(0).getValue(); - snapshotDetails.put(DiskTO.PATH, value); - } else if (storagePoolVO.getPoolType() == StoragePoolType.PowerFlex || storagePoolVO.getPoolType() == StoragePoolType.FiberChannel) { + if (storagePoolVO.getPoolType() == StoragePoolType.PowerFlex || storagePoolVO.getPoolType() == StoragePoolType.FiberChannel) { snapshotDetails.put(DiskTO.IQN, snapshotInfo.getPath()); } else { snapshotDetails.put(DiskTO.IQN, getSnapshotProperty(snapshotId, DiskTO.IQN)); @@ -2851,6 +2897,8 @@ private String migrateVolumeForKVM(VolumeInfo srcVolumeInfo, VolumeInfo destVolu Map srcDetails = getVolumeDetails(srcVolumeInfo); Map destDetails = getVolumeDetails(destVolumeInfo); + _volumeService.grantAccess(srcVolumeInfo, hostVO, srcVolumeInfo.getDataStore()); + MigrateVolumeCommand migrateVolumeCommand = new MigrateVolumeCommand(srcVolumeInfo.getTO(), destVolumeInfo.getTO(), srcDetails, destDetails, StorageManager.KvmStorageOfflineMigrationWait.value()); @@ -2893,18 +2941,18 @@ private String copyManagedVolumeToSecondaryStorage(VolumeInfo srcVolumeInfo, Vol StoragePoolVO storagePoolVO = _storagePoolDao.findById(srcVolumeInfo.getPoolId()); Map srcDetails = getVolumeDetails(srcVolumeInfo); - CopyVolumeCommand copyVolumeCommand = new CopyVolumeCommand(srcVolumeInfo.getId(), destVolumeInfo.getPath(), storagePoolVO, - destVolumeInfo.getDataStore().getUri(), true, StorageManager.KvmStorageOfflineMigrationWait.value(), true); - - copyVolumeCommand.setSrcData(srcVolumeInfo.getTO()); - copyVolumeCommand.setSrcDetails(srcDetails); - handleQualityOfServiceForVolumeMigration(srcVolumeInfo, PrimaryDataStoreDriver.QualityOfServiceState.MIGRATION); if (srcVolumeDetached) { _volumeService.grantAccess(srcVolumeInfo, hostVO, srcVolumeInfo.getDataStore()); } + CopyVolumeCommand copyVolumeCommand = new CopyVolumeCommand(srcVolumeInfo.getId(), destVolumeInfo.getPath(), storagePoolVO, + destVolumeInfo.getDataStore().getUri(), true, StorageManager.KvmStorageOfflineMigrationWait.value(), true); + + copyVolumeCommand.setSrcData(srcVolumeInfo.getTO()); + copyVolumeCommand.setSrcDetails(srcDetails); + CopyVolumeAnswer copyVolumeAnswer = (CopyVolumeAnswer)agentManager.send(hostVO.getId(), copyVolumeCommand); if (copyVolumeAnswer == null || !copyVolumeAnswer.getResult()) { @@ -2976,19 +3024,20 @@ private CopyCmdAnswer performCopyOfVdi(VolumeInfo volumeInfo, SnapshotInfo snaps srcData = cacheData; } - CopyCommand copyCommand = new CopyCommand(srcData.getTO(), volumeInfo.getTO(), primaryStorageDownloadWait, VirtualMachineManager.ExecuteInSequence.value()); - try { + CopyCommand copyCommand = null; if (Snapshot.LocationType.PRIMARY.equals(locationType)) { _volumeService.grantAccess(snapshotInfo, hostVO, snapshotInfo.getDataStore()); Map srcDetails = getSnapshotDetails(snapshotInfo); + copyCommand = new CopyCommand(srcData.getTO(), volumeInfo.getTO(), primaryStorageDownloadWait, VirtualMachineManager.ExecuteInSequence.value()); copyCommand.setOptions(srcDetails); + } else { + _volumeService.grantAccess(volumeInfo, hostVO, volumeInfo.getDataStore()); + copyCommand = new CopyCommand(srcData.getTO(), volumeInfo.getTO(), primaryStorageDownloadWait, VirtualMachineManager.ExecuteInSequence.value()); } - _volumeService.grantAccess(volumeInfo, hostVO, volumeInfo.getDataStore()); - Map destDetails = getVolumeDetails(volumeInfo); copyCommand.setOptions2(destDetails); diff --git a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java index 87a2288cfdc7..b7468195f5da 100644 --- a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java +++ b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java @@ -476,19 +476,19 @@ public void testCanHandleLiveMigrationUnmanagedStorage() { @Test public void testVerifyLiveMigrationMapForKVM() { - kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap, host2); + kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap); } @Test(expected = CloudRuntimeException.class) public void testVerifyLiveMigrationMapForKVMNotExistingSource() { when(primaryDataStoreDao.findById(POOL_1_ID)).thenReturn(null); - kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap, host2); + kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap); } @Test(expected = CloudRuntimeException.class) public void testVerifyLiveMigrationMapForKVMNotExistingDest() { when(primaryDataStoreDao.findById(POOL_2_ID)).thenReturn(null); - kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap, host2); + kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap); } @Test(expected = CloudRuntimeException.class) @@ -497,7 +497,7 @@ public void testVerifyLiveMigrationMapForKVMMixedManagedUnmagedStorage() { when(pool1.getId()).thenReturn(POOL_1_ID); when(pool2.getId()).thenReturn(POOL_2_ID); lenient().when(pool2.isManaged()).thenReturn(false); - kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap, host2); + kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap); } @Test diff --git a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategyTest.java b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategyTest.java index cea9de3f1b47..45357fa64b2a 100644 --- a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategyTest.java +++ b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategyTest.java @@ -23,7 +23,6 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.MockitoAnnotations.initMocks; import java.util.HashMap; @@ -48,7 +47,6 @@ import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; -import org.mockito.verification.VerificationMode; import com.cloud.agent.api.MigrateCommand; import com.cloud.host.HostVO; @@ -62,7 +60,6 @@ import java.util.AbstractMap; import java.util.Arrays; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -372,72 +369,4 @@ public void validateIsStoragePoolTypeInListReturnsFalse() { assertFalse(strategy.isStoragePoolTypeInList(StoragePoolType.SharedMountPoint, listTypes)); } - - @Test - public void validateAddSourcePoolToPoolsMapDestinationPoolIsManaged() { - Mockito.doReturn(true).when(destinationStoragePoolVoMock).isManaged(); - strategy.addSourcePoolToPoolsMap(mapStringStoragePoolTypeMock, sourceStoragePoolVoMock, destinationStoragePoolVoMock); - - Mockito.verify(destinationStoragePoolVoMock).isManaged(); - Mockito.verifyNoMoreInteractions(mapStringStoragePoolTypeMock, sourceStoragePoolVoMock, destinationStoragePoolVoMock); - } - - @Test - public void validateAddSourcePoolToPoolsMapDestinationPoolIsNotNFS() { - List storagePoolTypes = new LinkedList<>(Arrays.asList(StoragePoolType.values())); - storagePoolTypes.remove(StoragePoolType.NetworkFilesystem); - - Mockito.doReturn(false).when(destinationStoragePoolVoMock).isManaged(); - storagePoolTypes.forEach(poolType -> { - Mockito.doReturn(poolType).when(destinationStoragePoolVoMock).getPoolType(); - strategy.addSourcePoolToPoolsMap(mapStringStoragePoolTypeMock, sourceStoragePoolVoMock, destinationStoragePoolVoMock); - }); - - VerificationMode times = Mockito.times(storagePoolTypes.size()); - Mockito.verify(destinationStoragePoolVoMock, times).isManaged(); - Mockito.verify(destinationStoragePoolVoMock, times).getPoolType(); - Mockito.verifyNoMoreInteractions(mapStringStoragePoolTypeMock, sourceStoragePoolVoMock, destinationStoragePoolVoMock); - } - - @Test - public void validateAddSourcePoolToPoolsMapMapContainsKey() { - Mockito.doReturn(false).when(destinationStoragePoolVoMock).isManaged(); - Mockito.doReturn(StoragePoolType.NetworkFilesystem).when(destinationStoragePoolVoMock).getPoolType(); - Mockito.doReturn("").when(sourceStoragePoolVoMock).getUuid(); - Mockito.doReturn(true).when(mapStringStoragePoolTypeMock).containsKey(Mockito.anyString()); - strategy.addSourcePoolToPoolsMap(mapStringStoragePoolTypeMock, sourceStoragePoolVoMock, destinationStoragePoolVoMock); - - Mockito.verify(destinationStoragePoolVoMock, never()).getScope(); - Mockito.verify(destinationStoragePoolVoMock).isManaged(); - Mockito.verify(destinationStoragePoolVoMock).getPoolType(); - Mockito.verify(sourceStoragePoolVoMock).getUuid(); - Mockito.verify(mapStringStoragePoolTypeMock).containsKey(Mockito.anyString()); - Mockito.verifyNoMoreInteractions(mapStringStoragePoolTypeMock, sourceStoragePoolVoMock, destinationStoragePoolVoMock); - } - - @Test - public void validateAddSourcePoolToPoolsMapMapDoesNotContainsKey() { - List storagePoolTypes = new LinkedList<>(Arrays.asList(StoragePoolType.values())); - - Mockito.doReturn(false).when(destinationStoragePoolVoMock).isManaged(); - Mockito.doReturn(StoragePoolType.NetworkFilesystem).when(destinationStoragePoolVoMock).getPoolType(); - Mockito.doReturn("").when(sourceStoragePoolVoMock).getUuid(); - Mockito.doReturn(false).when(mapStringStoragePoolTypeMock).containsKey(Mockito.anyString()); - Mockito.doReturn(null).when(mapStringStoragePoolTypeMock).put(Mockito.anyString(), Mockito.any()); - - storagePoolTypes.forEach(poolType -> { - Mockito.doReturn(poolType).when(sourceStoragePoolVoMock).getPoolType(); - strategy.addSourcePoolToPoolsMap(mapStringStoragePoolTypeMock, sourceStoragePoolVoMock, destinationStoragePoolVoMock); - }); - - VerificationMode times = Mockito.times(storagePoolTypes.size()); - Mockito.verify(destinationStoragePoolVoMock, never()).getScope(); - Mockito.verify(destinationStoragePoolVoMock, times).isManaged(); - Mockito.verify(destinationStoragePoolVoMock, times).getPoolType(); - Mockito.verify(sourceStoragePoolVoMock, times).getUuid(); - Mockito.verify(mapStringStoragePoolTypeMock, times).containsKey(Mockito.anyString()); - Mockito.verify(sourceStoragePoolVoMock, times).getPoolType(); - Mockito.verify(mapStringStoragePoolTypeMock, times).put(Mockito.anyString(), Mockito.any()); - Mockito.verifyNoMoreInteractions(mapStringStoragePoolTypeMock, sourceStoragePoolVoMock, destinationStoragePoolVoMock); - } } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java index ba783e81586a..5109118fb54e 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java @@ -100,6 +100,9 @@ public TemplateInfo getTemplate(long templateId) { @Override public TemplateInfo getTemplate(long templateId, DataStore store) { VMTemplateVO templ = imageDataDao.findById(templateId); + if (templ == null) { + return null; + } if (store == null && !templ.isDirectDownload()) { TemplateObject tmpl = TemplateObject.getTemplate(templ, null, null); return tmpl; diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java index 11a13e7ccb44..d2f08260aa32 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java @@ -202,7 +202,7 @@ public int compare(DataStore store1, DataStore store2) { // No store with space found logger.error(String.format("Can't find an image storage in zone with less than %d usage", - Math.round(_statsCollector.getImageStoreCapacityThreshold()*100))); + Math.round(_statsCollector.getImageStoreCapacityThreshold() * 100))); return null; } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java index fdb4fe6753a2..37ddc3573c11 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java @@ -82,6 +82,10 @@ public TemplateObject() { } protected void configure(VMTemplateVO template, DataStore dataStore) { + if (template == null) { + String msg = String.format("Template Object is not properly initialised %s", this.toString()); + logger.warn(msg); + } imageVO = template; this.dataStore = dataStore; } @@ -98,6 +102,10 @@ public void setSize(Long size) { } public VMTemplateVO getImage() { + if (imageVO == null) { + String msg = String.format("Template Object is not properly initialised %s", this.toString()); + logger.error(msg); + } // somehow the nullpointer is needed : refacter needed!?! return imageVO; } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java index 2c3d5ccfdde8..9eae1fc0711c 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java @@ -407,4 +407,16 @@ public boolean updateVolumeId(long srcVolId, long destVolId) { } return true; } + + @Override + public int expungeByVolumeList(List volumeIds, Long batchSize) { + if (CollectionUtils.isEmpty(volumeIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("volumeIds", sb.entity().getVolumeId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("volumeIds", volumeIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java index 8044a2dfa5e2..e4c269326198 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java @@ -32,6 +32,7 @@ import org.apache.logging.log4j.LogManager; import org.springframework.stereotype.Component; +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; @@ -43,17 +44,20 @@ import com.cloud.capacity.Capacity; import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDao; -import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.DataStoreRole; import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StoragePoolStatus; -import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.utils.crypt.DBEncryptionUtil; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; @Component @@ -266,4 +270,48 @@ public boolean deletePrimaryDataStore(DataStore store) { return true; } + public void switchToZone(DataStore store, HypervisorType hypervisorType) { + StoragePoolVO pool = dataStoreDao.findById(store.getId()); + CapacityVO capacity = _capacityDao.findByHostIdType(store.getId(), Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED); + Transaction.execute(new TransactionCallbackNoReturn() { + public void doInTransactionWithoutResult(TransactionStatus status) { + pool.setScope(ScopeType.ZONE); + pool.setPodId(null); + pool.setClusterId(null); + pool.setHypervisor(hypervisorType); + dataStoreDao.update(pool.getId(), pool); + + capacity.setPodId(null); + capacity.setClusterId(null); + _capacityDao.update(capacity.getId(), capacity); + } + }); + logger.debug("Scope of storage pool id=" + pool.getId() + " is changed to zone"); + } + + public void switchToCluster(DataStore store, ClusterScope clusterScope) { + List hostPoolRecords = storagePoolHostDao.listByPoolIdNotInCluster(clusterScope.getScopeId(), store.getId()).first(); + StoragePoolVO pool = dataStoreDao.findById(store.getId()); + CapacityVO capacity = _capacityDao.findByHostIdType(store.getId(), Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED); + + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + if (hostPoolRecords != null) { + for (StoragePoolHostVO host : hostPoolRecords) { + storagePoolHostDao.deleteStoragePoolHostDetails(host.getHostId(), host.getPoolId()); + } + } + pool.setScope(ScopeType.CLUSTER); + pool.setPodId(clusterScope.getPodId()); + pool.setClusterId(clusterScope.getScopeId()); + dataStoreDao.update(pool.getId(), pool); + + capacity.setPodId(clusterScope.getPodId()); + capacity.setClusterId(clusterScope.getScopeId()); + _capacityDao.update(capacity.getId(), capacity); + } + }); + logger.debug("Scope of storage pool id=" + pool.getId() + " is changed to cluster id=" + clusterScope.getScopeId()); + } } diff --git a/engine/storage/src/test/java/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImplTest.java b/engine/storage/src/test/java/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImplTest.java new file mode 100644 index 000000000000..0cd886999564 --- /dev/null +++ b/engine/storage/src/test/java/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImplTest.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.image.db; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class VolumeDataStoreDaoImplTest { + + @Spy + VolumeDataStoreDaoImpl volumeDataStoreDaoImplSpy; + + @Test + public void testExpungeByVmListNoVms() { + Assert.assertEquals(0, volumeDataStoreDaoImplSpy.expungeByVolumeList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, volumeDataStoreDaoImplSpy.expungeByVolumeList( + null, 100L)); + } + + @Test + public void testExpungeByVmList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(volumeDataStoreDaoImplSpy).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(volumeDataStoreDaoImplSpy.createSearchBuilder()).thenReturn(sb); + final VolumeDataStoreVO mockedVO = Mockito.mock(VolumeDataStoreVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), volumeDataStoreDaoImplSpy.expungeByVolumeList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("volumeIds", array); + Mockito.verify(volumeDataStoreDaoImplSpy, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } +} diff --git a/engine/storage/src/test/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelperTest.java b/engine/storage/src/test/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelperTest.java new file mode 100644 index 000000000000..3927b43f3939 --- /dev/null +++ b/engine/storage/src/test/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelperTest.java @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.volume.datastore; + +import java.util.List; + +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityVO; +import com.cloud.capacity.dao.CapacityDao; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; +import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.utils.Pair; + +@RunWith(MockitoJUnitRunner.class) +public class PrimaryDataStoreHelperTest { + + @Mock + private PrimaryDataStoreDao dataStoreDao; + + @Mock + private CapacityDao capacityDao; + + @Mock + private StoragePoolHostDao storagePoolHostDao; + + @Spy + @InjectMocks + PrimaryDataStoreHelper dataStoreHelper; + + private static final Long ZONE_ID = 1L; + private static final Long CLUSTER_ID = 2L; + private static final Long POD_ID = 3L; + private static final Long POOL_ID = 4L; + private static final Short capacityType = 0; + private static final Float usedPercentage = 0.0f; + + @Test + public void testSwitchToZone() { + StoragePoolVO pool = new StoragePoolVO(POOL_ID, null, null, Storage.StoragePoolType.NetworkFilesystem, ZONE_ID, POD_ID, 0L, 0L, null, 0, null); + pool.setClusterId(CLUSTER_ID); + pool.setScope(ScopeType.CLUSTER); + CapacityVO capacity = new CapacityVO(ZONE_ID, POD_ID, CLUSTER_ID, capacityType, usedPercentage); + + Mockito.when(dataStoreDao.findById(pool.getId())).thenReturn(pool); + Mockito.when(capacityDao.findByHostIdType(pool.getId(), Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED)).thenReturn(capacity); + DataStore storeMock = Mockito.mock(DataStore.class); + Mockito.when(storeMock.getId()).thenReturn(POOL_ID); + + dataStoreHelper.switchToZone(storeMock, HypervisorType.KVM); + + Assert.assertEquals(pool.getScope(), ScopeType.ZONE); + Assert.assertEquals(pool.getPodId(), null); + Assert.assertEquals(pool.getClusterId(), null); + Assert.assertEquals(pool.getHypervisor(), HypervisorType.KVM); + Assert.assertEquals(capacity.getPodId(), null); + Assert.assertEquals(capacity.getClusterId(), null); + } + + @Test + public void testSwitchToCluster() { + StoragePoolVO pool = new StoragePoolVO(POOL_ID, null, null, Storage.StoragePoolType.NetworkFilesystem, ZONE_ID, null, 0L, 0L, null, 0, null); + pool.setScope(ScopeType.ZONE); + CapacityVO capacity = new CapacityVO(ZONE_ID, null, null, capacityType, usedPercentage); + ClusterScope clusterScope = new ClusterScope(CLUSTER_ID, POD_ID, ZONE_ID); + + Pair, Integer> hostPoolRecords = new Pair<>(null, 0); + Mockito.when(storagePoolHostDao.listByPoolIdNotInCluster(CLUSTER_ID, POOL_ID)).thenReturn(hostPoolRecords); + Mockito.when(dataStoreDao.findById(pool.getId())).thenReturn(pool); + Mockito.when(capacityDao.findByHostIdType(pool.getId(), Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED)).thenReturn(capacity); + DataStore storeMock = Mockito.mock(DataStore.class); + Mockito.when(storeMock.getId()).thenReturn(POOL_ID); + + dataStoreHelper.switchToCluster(storeMock, clusterScope); + + Mockito.verify(storagePoolHostDao, Mockito.never()).deleteStoragePoolHostDetails(Mockito.anyLong(), Mockito.anyLong()); + + Assert.assertEquals(pool.getScope(), ScopeType.CLUSTER); + Assert.assertEquals(pool.getPodId(), POD_ID); + Assert.assertEquals(pool.getClusterId(), CLUSTER_ID); + Assert.assertEquals(capacity.getPodId(), POD_ID); + Assert.assertEquals(capacity.getClusterId(), CLUSTER_ID); + } +} diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImpl.java new file mode 100644 index 000000000000..1ee4d40a5678 --- /dev/null +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImpl.java @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.datastore.lifecycle; + +import java.util.Arrays; +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.DeleteStoragePoolCommand; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.resource.ResourceManager; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.utils.Pair; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class BasePrimaryDataStoreLifeCycleImpl { + protected Logger logger = LogManager.getLogger(getClass()); + + @Inject + AgentManager agentMgr; + @Inject + protected ResourceManager resourceMgr; + @Inject + StorageManager storageMgr; + @Inject + PrimaryDataStoreHelper dataStoreHelper; + @Inject + protected HostDao hostDao; + @Inject + protected StoragePoolHostDao storagePoolHostDao; + + private List getPoolHostsList(ClusterScope clusterScope, HypervisorType hypervisorType) { + List hosts; + if (hypervisorType != null) { + hosts = resourceMgr + .listAllHostsInOneZoneNotInClusterByHypervisor(hypervisorType, clusterScope.getZoneId(), clusterScope.getScopeId()); + } else { + List hypervisorTypes = Arrays.asList(HypervisorType.KVM, HypervisorType.VMware); + hosts = resourceMgr + .listAllHostsInOneZoneNotInClusterByHypervisors(hypervisorTypes, clusterScope.getZoneId(), clusterScope.getScopeId()); + } + return hosts; + } + + public void changeStoragePoolScopeToZone(DataStore store, ClusterScope clusterScope, HypervisorType hypervisorType) { + List hosts = getPoolHostsList(clusterScope, hypervisorType); + logger.debug("Changing scope of the storage pool to Zone"); + if (hosts != null) { + for (HostVO host : hosts) { + try { + storageMgr.connectHostToSharedPool(host.getId(), store.getId()); + } catch (Exception e) { + logger.warn("Unable to establish a connection between " + host + " and " + store, e); + } + } + } + dataStoreHelper.switchToZone(store, hypervisorType); + } + + public void changeStoragePoolScopeToCluster(DataStore store, ClusterScope clusterScope, HypervisorType hypervisorType) { + Pair, Integer> hostPoolRecords = storagePoolHostDao.listByPoolIdNotInCluster(clusterScope.getScopeId(), store.getId()); + logger.debug("Changing scope of the storage pool to Cluster"); + if (hostPoolRecords.second() > 0) { + StoragePool pool = (StoragePool) store; + for (StoragePoolHostVO host : hostPoolRecords.first()) { + DeleteStoragePoolCommand deleteCmd = new DeleteStoragePoolCommand(pool); + final Answer answer = agentMgr.easySend(host.getHostId(), deleteCmd); + + if (answer != null) { + if (!answer.getResult()) { + logger.debug("Failed to delete storage pool: " + answer.getResult()); + } else if (HypervisorType.KVM != hypervisorType) { + break; + } + } + } + } + dataStoreHelper.switchToCluster(store, clusterScope); + } +} diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java index b8f90e465380..c6d9fab5f17a 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java @@ -42,7 +42,9 @@ import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StorageService; import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; + import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -54,7 +56,9 @@ import org.apache.logging.log4j.LogManager; import javax.inject.Inject; + import java.util.List; +import java.util.Map; public class DefaultHostListener implements HypervisorHostListener { protected Logger logger = LogManager.getLogger(getClass()); @@ -126,7 +130,9 @@ private NicTO createNicTOFromNetworkAndOffering(NetworkVO networkVO, NetworkOffe @Override public boolean hostConnect(long hostId, long poolId) throws StorageConflictException { StoragePool pool = (StoragePool) this.dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary); - ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool); + Pair, Boolean> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null); + + ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool, nfsMountOpts.first()); cmd.setWait(modifyStoragePoolCommandWait); logger.debug(String.format("Sending modify storage pool command to agent: %d for storage pool: %d with timeout %d seconds", hostId, poolId, cmd.getWait())); @@ -139,7 +145,7 @@ public boolean hostConnect(long hostId, long poolId) throws StorageConflictExcep if (!answer.getResult()) { String msg = "Unable to attach storage pool" + poolId + " to the host" + hostId; alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, pool.getDataCenterId(), pool.getPodId(), msg, msg); - throw new CloudRuntimeException("Unable establish connection from storage head to storage pool " + pool.getId() + " due to " + answer.getDetails() + + throw new CloudRuntimeException("Unable to establish connection from storage head to storage pool " + pool.getId() + " due to " + answer.getDetails() + pool.getId()); } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 24a0db0b74a7..9a3319f79a33 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -32,7 +32,6 @@ import javax.inject.Inject; -import com.cloud.storage.VolumeApiServiceImpl; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.command.user.volume.CheckAndRepairVolumeCmd; @@ -105,6 +104,7 @@ import com.cloud.dc.dao.ClusterDao; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.StorageAccessException; import com.cloud.host.Host; @@ -118,6 +118,7 @@ import com.cloud.resource.ResourceState; import com.cloud.server.ManagementService; import com.cloud.storage.CheckAndRepairVolumePayload; +import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.DataStoreRole; import com.cloud.storage.RegisterVolumePayload; import com.cloud.storage.ScopeType; @@ -130,6 +131,7 @@ import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiServiceImpl; import com.cloud.storage.Volume.State; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; @@ -215,7 +217,7 @@ public class VolumeServiceImpl implements VolumeService { @Inject private PassphraseDao passphraseDao; @Inject - private DiskOfferingDao diskOfferingDao; + protected DiskOfferingDao diskOfferingDao; public VolumeServiceImpl() { } @@ -290,12 +292,12 @@ public boolean requiresAccessForMigration(DataObject dataObject, DataStore dataS @Override public AsyncCallFuture createVolumeAsync(VolumeInfo volume, DataStore dataStore) { - AsyncCallFuture future = new AsyncCallFuture(); + AsyncCallFuture future = new AsyncCallFuture<>(); DataObject volumeOnStore = dataStore.create(volume); volumeOnStore.processEvent(Event.CreateOnlyRequested); try { - CreateVolumeContext context = new CreateVolumeContext(null, volumeOnStore, future); + CreateVolumeContext context = new CreateVolumeContext<>(null, volumeOnStore, future); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createVolumeCallback(null, null)).setContext(context); @@ -371,7 +373,7 @@ private boolean canVolumeBeRemoved(long volumeId) { @DB @Override public AsyncCallFuture expungeVolumeAsync(VolumeInfo volume) { - AsyncCallFuture future = new AsyncCallFuture(); + AsyncCallFuture future = new AsyncCallFuture<>(); VolumeApiResult result = new VolumeApiResult(volume); if (volume.getDataStore() == null) { logger.info("Expunge volume with no data store specified"); @@ -427,7 +429,7 @@ public AsyncCallFuture expungeVolumeAsync(VolumeInfo volume) { volume.processEvent(Event.ExpungeRequested); } - DeleteVolumeContext context = new DeleteVolumeContext(null, vo, future); + DeleteVolumeContext context = new DeleteVolumeContext<>(null, vo, future); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().deleteVolumeCallback(null, null)).setContext(context); @@ -503,7 +505,9 @@ public Void deleteVolumeCallback(AsyncCallbackDispatcher context = new CreateBaseImageContext(null, volume, dataStore, template, future, templateOnPrimaryStoreObj, templatePoolRefId); + CreateBaseImageContext context = new CreateBaseImageContext<>(null, volume, dataStore, template, future, templateOnPrimaryStoreObj, templatePoolRefId); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().copyBaseImageCallback(null, null)).setContext(context); @@ -806,7 +810,7 @@ protected void createVolumeFromBaseImageAsync(VolumeInfo volume, DataObject temp DataObject volumeOnPrimaryStorage = pd.create(volume, volume.getDeployAsIsConfiguration()); volumeOnPrimaryStorage.processEvent(Event.CreateOnlyRequested); - CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, volumeOnPrimaryStorage, pd, templateOnPrimaryStore, future, null, volume.getDeployAsIsConfiguration()); + CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext<>(null, volumeOnPrimaryStorage, pd, templateOnPrimaryStore, future, null, volume.getDeployAsIsConfiguration()); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createVolumeFromBaseImageCallBack(null, null)); caller.setContext(context); @@ -1174,7 +1178,7 @@ private void createManagedVolumeCopyManagedTemplateAsync(VolumeInfo volumeInfo, // Refresh the volume info from the DB. volumeInfo = volFactory.getVolume(volumeInfo.getId(), destPrimaryDataStore); - Map details = new HashMap(); + Map details = new HashMap<>(); details.put(PrimaryDataStore.MANAGED, Boolean.TRUE.toString()); details.put(PrimaryDataStore.STORAGE_HOST, destPrimaryDataStore.getHostAddress()); details.put(PrimaryDataStore.STORAGE_PORT, String.valueOf(destPrimaryDataStore.getPort())); @@ -1278,12 +1282,12 @@ private void createManagedVolumeCopyTemplateAsync(VolumeInfo volumeInfo, Primary // Refresh the volume info from the DB. volumeInfo = volFactory.getVolume(volumeInfo.getId(), primaryDataStore); - ManagedCreateBaseImageContext context = new ManagedCreateBaseImageContext(null, volumeInfo, primaryDataStore, srcTemplateInfo, future); + ManagedCreateBaseImageContext context = new ManagedCreateBaseImageContext<>(null, volumeInfo, primaryDataStore, srcTemplateInfo, future); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().managedCopyBaseImageCallback(null, null)).setContext(context); - Map details = new HashMap(); + Map details = new HashMap<>(); details.put(PrimaryDataStore.MANAGED, Boolean.TRUE.toString()); details.put(PrimaryDataStore.STORAGE_HOST, primaryDataStore.getHostAddress()); @@ -1639,14 +1643,14 @@ public void destroyVolume(long volumeId) { @Override public AsyncCallFuture createVolumeFromSnapshot(VolumeInfo volume, DataStore store, SnapshotInfo snapshot) { - AsyncCallFuture future = new AsyncCallFuture(); + AsyncCallFuture future = new AsyncCallFuture<>(); try { DataObject volumeOnStore = store.create(volume); volumeOnStore.processEvent(Event.CreateOnlyRequested); _volumeDetailsDao.addDetail(volume.getId(), SNAPSHOT_ID, Long.toString(snapshot.getId()), false); - CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, volume, store, volumeOnStore, future, snapshot, null); + CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext<>(null, volume, store, volumeOnStore, future, snapshot, null); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createVolumeFromSnapshotCallback(null, null)).setContext(context); motionSrv.copyAsync(snapshot, volumeOnStore, caller); @@ -1733,7 +1737,7 @@ public CopyVolumeContext(AsyncCompletionCallback callback, AsyncCallFuture copyVolumeFromImageToPrimary(VolumeInfo srcVolume, DataStore destStore) { - AsyncCallFuture future = new AsyncCallFuture(); + AsyncCallFuture future = new AsyncCallFuture<>(); VolumeApiResult res = new VolumeApiResult(srcVolume); VolumeInfo destVolume = null; try { @@ -1741,7 +1745,7 @@ protected AsyncCallFuture copyVolumeFromImageToPrimary(VolumeIn destVolume.processEvent(Event.CopyingRequested); srcVolume.processEvent(Event.CopyingRequested); - CopyVolumeContext context = new CopyVolumeContext(null, future, srcVolume, destVolume, destStore); + CopyVolumeContext context = new CopyVolumeContext<>(null, future, srcVolume, destVolume, destStore); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().copyVolumeFromImageToPrimaryCallback(null, null)).setContext(context); @@ -1787,7 +1791,7 @@ protected Void copyVolumeFromImageToPrimaryCallback(AsyncCallbackDispatcher copyVolumeFromPrimaryToImage(VolumeInfo srcVolume, DataStore destStore) { - AsyncCallFuture future = new AsyncCallFuture(); + AsyncCallFuture future = new AsyncCallFuture<>(); VolumeApiResult res = new VolumeApiResult(srcVolume); VolumeInfo destVolume = null; try { @@ -1795,7 +1799,7 @@ protected AsyncCallFuture copyVolumeFromPrimaryToImage(VolumeIn srcVolume.processEvent(Event.MigrationRequested); // this is just used for locking that src volume record in DB to avoid using lock destVolume.processEventOnly(Event.CreateOnlyRequested); - CopyVolumeContext context = new CopyVolumeContext(null, future, srcVolume, destVolume, destStore); + CopyVolumeContext context = new CopyVolumeContext<>(null, future, srcVolume, destVolume, destStore); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().copyVolumeFromPrimaryToImageCallback(null, null)).setContext(context); @@ -1868,7 +1872,7 @@ public AsyncCallFuture copyVolume(VolumeInfo srcVolume, DataSto // OfflineVmwareMigration: aren't we missing secondary to secondary in this logic? - AsyncCallFuture future = new AsyncCallFuture(); + AsyncCallFuture future = new AsyncCallFuture<>(); VolumeApiResult res = new VolumeApiResult(srcVolume); try { if (!snapshotMgr.canOperateOnVolume(srcVolume)) { @@ -1884,7 +1888,7 @@ public AsyncCallFuture copyVolume(VolumeInfo srcVolume, DataSto destVolume.processEvent(Event.MigrationCopyRequested); srcVolume.processEvent(Event.MigrationRequested); - CopyVolumeContext context = new CopyVolumeContext(null, future, srcVolume, destVolume, destStore); + CopyVolumeContext context = new CopyVolumeContext<>(null, future, srcVolume, destVolume, destStore); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().copyVolumeCallBack(null, null)).setContext(context); motionSrv.copyAsync(srcVolume, destVolume, caller); @@ -2018,7 +2022,7 @@ public CopyManagedVolumeContext(AsyncCompletionCallback callback, AsyncCallFu } private AsyncCallFuture copyManagedVolume(VolumeInfo srcVolume, DataStore destStore) { - AsyncCallFuture future = new AsyncCallFuture(); + AsyncCallFuture future = new AsyncCallFuture<>(); VolumeApiResult res = new VolumeApiResult(srcVolume); try { if (!snapshotMgr.canOperateOnVolume(srcVolume)) { @@ -2035,7 +2039,7 @@ private AsyncCallFuture copyManagedVolume(VolumeInfo srcVolume, return future; } - List poolIds = new ArrayList(); + List poolIds = new ArrayList<>(); poolIds.add(srcVolume.getPoolId()); poolIds.add(destStore.getId()); @@ -2067,7 +2071,7 @@ private AsyncCallFuture copyManagedVolume(VolumeInfo srcVolume, PrimaryDataStore srcPrimaryDataStore = (PrimaryDataStore) srcVolume.getDataStore(); if (srcPrimaryDataStore.isManaged()) { - Map srcPrimaryDataStoreDetails = new HashMap(); + Map srcPrimaryDataStoreDetails = new HashMap<>(); srcPrimaryDataStoreDetails.put(PrimaryDataStore.MANAGED, Boolean.TRUE.toString()); srcPrimaryDataStoreDetails.put(PrimaryDataStore.STORAGE_HOST, srcPrimaryDataStore.getHostAddress()); srcPrimaryDataStoreDetails.put(PrimaryDataStore.STORAGE_PORT, String.valueOf(srcPrimaryDataStore.getPort())); @@ -2080,7 +2084,7 @@ private AsyncCallFuture copyManagedVolume(VolumeInfo srcVolume, } PrimaryDataStore destPrimaryDataStore = (PrimaryDataStore) destStore; - Map destPrimaryDataStoreDetails = new HashMap(); + Map destPrimaryDataStoreDetails = new HashMap<>(); destPrimaryDataStoreDetails.put(PrimaryDataStore.MANAGED, Boolean.TRUE.toString()); destPrimaryDataStoreDetails.put(PrimaryDataStore.STORAGE_HOST, destPrimaryDataStore.getHostAddress()); destPrimaryDataStoreDetails.put(PrimaryDataStore.STORAGE_PORT, String.valueOf(destPrimaryDataStore.getPort())); @@ -2095,7 +2099,7 @@ private AsyncCallFuture copyManagedVolume(VolumeInfo srcVolume, destVolume.processEvent(Event.CreateRequested); srcVolume.processEvent(Event.MigrationRequested); - CopyManagedVolumeContext context = new CopyManagedVolumeContext(null, future, srcVolume, destVolume, hostWithPoolsAccess); + CopyManagedVolumeContext context = new CopyManagedVolumeContext<>(null, future, srcVolume, destVolume, hostWithPoolsAccess); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().copyManagedVolumeCallBack(null, null)).setContext(context); @@ -2233,7 +2237,7 @@ public MigrateVolumeContext(AsyncCompletionCallback callback, AsyncCallFuture @Override public AsyncCallFuture migrateVolume(VolumeInfo srcVolume, DataStore destStore) { - AsyncCallFuture future = new AsyncCallFuture(); + AsyncCallFuture future = new AsyncCallFuture<>(); VolumeApiResult res = new VolumeApiResult(srcVolume); try { if (!snapshotMgr.canOperateOnVolume(srcVolume)) { @@ -2245,7 +2249,7 @@ public AsyncCallFuture migrateVolume(VolumeInfo srcVolume, Data VolumeInfo destVolume = volFactory.getVolume(srcVolume.getId(), destStore); srcVolume.processEvent(Event.MigrationRequested); - MigrateVolumeContext context = new MigrateVolumeContext(null, future, srcVolume, destVolume, destStore); + MigrateVolumeContext context = new MigrateVolumeContext<>(null, future, srcVolume, destVolume, destStore); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().migrateVolumeCallBack(null, null)).setContext(context); motionSrv.copyAsync(srcVolume, destVolume, caller); @@ -2298,13 +2302,13 @@ public MigrateVmWithVolumesContext(AsyncCompletionCallback callback, AsyncCal @Override public AsyncCallFuture migrateVolumes(Map volumeMap, VirtualMachineTO vmTo, Host srcHost, Host destHost) { - AsyncCallFuture future = new AsyncCallFuture(); + AsyncCallFuture future = new AsyncCallFuture<>(); CommandResult res = new CommandResult(); try { // Check to make sure there are no snapshot operations on a volume // and // put it in the migrating state. - List volumesMigrating = new ArrayList(); + List volumesMigrating = new ArrayList<>(); for (Map.Entry entry : volumeMap.entrySet()) { VolumeInfo volume = entry.getKey(); if (!snapshotMgr.canOperateOnVolume(volume)) { @@ -2324,7 +2328,7 @@ public AsyncCallFuture migrateVolumes(Map } } - MigrateVmWithVolumesContext context = new MigrateVmWithVolumesContext(null, future, volumeMap); + MigrateVmWithVolumesContext context = new MigrateVmWithVolumesContext<>(null, future, volumeMap); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().migrateVmWithVolumesCallBack(null, null)).setContext(context); motionSrv.copyAsync(volumeMap, vmTo, srcHost, destHost, caller); @@ -2371,13 +2375,13 @@ protected Void migrateVmWithVolumesCallBack(AsyncCallbackDispatcher registerVolume(VolumeInfo volume, DataStore store) { - AsyncCallFuture future = new AsyncCallFuture(); + AsyncCallFuture future = new AsyncCallFuture<>(); DataObject volumeOnStore = store.create(volume); volumeOnStore.processEvent(Event.CreateOnlyRequested); try { - CreateVolumeContext context = new CreateVolumeContext(null, volumeOnStore, future); + CreateVolumeContext context = new CreateVolumeContext<>(null, volumeOnStore, future); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().registerVolumeCallback(null, null)); caller.setContext(context); @@ -2472,7 +2476,7 @@ protected Void registerVolumeCallback(AsyncCallbackDispatcher resize(VolumeInfo volume) { - AsyncCallFuture future = new AsyncCallFuture(); + AsyncCallFuture future = new AsyncCallFuture<>(); VolumeApiResult result = new VolumeApiResult(volume); try { volume.processEvent(Event.ResizeRequested); @@ -2482,7 +2486,7 @@ public AsyncCallFuture resize(VolumeInfo volume) { future.complete(result); return future; } - CreateVolumeContext context = new CreateVolumeContext(null, volume, future); + CreateVolumeContext context = new CreateVolumeContext<>(null, volume, future); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().resizeVolumeCallback(caller, context)).setContext(context); @@ -2581,7 +2585,7 @@ public void handleVolumeSync(DataStore store) { // find all the db volumes including those with NULL url column to avoid accidentally deleting volumes on image store later. List dbVolumes = _volumeStoreDao.listByStoreId(storeId); - List toBeDownloaded = new ArrayList(dbVolumes); + List toBeDownloaded = new ArrayList<>(dbVolumes); for (VolumeDataStoreVO volumeStore : dbVolumes) { VolumeVO volume = volDao.findById(volumeStore.getVolumeId()); if (volume == null) { @@ -2797,6 +2801,16 @@ public void checkAndRepairVolumeBasedOnConfig(DataObject dataObject, Host host) } } + @Override + public void validateChangeDiskOfferingEncryptionType(long existingDiskOfferingId, long newDiskOfferingId) { + DiskOfferingVO existingDiskOffering = diskOfferingDao.findByIdIncludingRemoved(existingDiskOfferingId); + DiskOfferingVO newDiskOffering = diskOfferingDao.findById(newDiskOfferingId); + + if (existingDiskOffering.getEncrypt() != newDiskOffering.getEncrypt()) { + throw new InvalidParameterValueException("Cannot change the encryption type of a volume, please check the selected offering"); + } + } + @Override public Pair checkAndRepairVolume(VolumeInfo volume) { Long poolId = volume.getPoolId(); diff --git a/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImplTest.java b/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImplTest.java new file mode 100644 index 000000000000..355eb075129e --- /dev/null +++ b/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImplTest.java @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.datastore.lifecycle; + +import static org.mockito.ArgumentMatchers.eq; + +import java.util.Arrays; +import java.util.List; + +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.storage.datastore.PrimaryDataStoreImpl; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.DeleteStoragePoolCommand; +import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.resource.ResourceManager; +import com.cloud.storage.Storage; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.utils.Pair; + +@RunWith(MockitoJUnitRunner.class) +public class BasePrimaryDataStoreLifeCycleImplTest { + + @Mock + private StoragePoolHostDao storagePoolHostDao; + + @Mock + private PrimaryDataStoreHelper dataStoreHelper; + + @Mock + private AgentManager agentManager; + + @Mock + private ResourceManager resourceManager; + + @Mock + private StorageManager storageManager; + + @Spy + @InjectMocks + private BasePrimaryDataStoreLifeCycleImpl dataStoreLifeCycle; + + private static final Long POOL_ID = 1L; + private static final Long CLUSTER_ID = 2L; + private static final Long POD_ID = 3L; + private static final Long ZONE_ID = 4L; + private static final Long HOST_ID = 5L; + + private static ClusterScope clusterScope; + private static PrimaryDataStoreImpl store; + + + @BeforeClass + public static void init() { + clusterScope = new ClusterScope(CLUSTER_ID, POD_ID, ZONE_ID); + StoragePoolVO pool = new StoragePoolVO(POOL_ID, null, null, Storage.StoragePoolType.NetworkFilesystem, 0L, 0L, 0L, 0L, null, 0, null); + store = new PrimaryDataStoreImpl(); + store.configure(pool, null, null); + } + + @Test + public void testChangeStoragePoolScopeToZone() throws Exception { + Mockito.when(resourceManager.listAllHostsInOneZoneNotInClusterByHypervisor(HypervisorType.KVM, ZONE_ID, CLUSTER_ID)).thenReturn(null); + + dataStoreLifeCycle.changeStoragePoolScopeToZone(store, clusterScope, HypervisorType.KVM); + + Mockito.verify(dataStoreHelper, Mockito.times(1)).switchToZone(store, HypervisorType.KVM); + + HostVO host = new HostVO(null); + ReflectionTestUtils.setField(host, "id", HOST_ID); + List hypervisorTypes = Arrays.asList(HypervisorType.KVM, HypervisorType.VMware); + Mockito.when(resourceManager.listAllHostsInOneZoneNotInClusterByHypervisors(hypervisorTypes, ZONE_ID, CLUSTER_ID)).thenReturn(Arrays.asList(host)); + Mockito.when(storageManager.connectHostToSharedPool(HOST_ID, POOL_ID)).thenReturn(true); + + dataStoreLifeCycle.changeStoragePoolScopeToZone(store, clusterScope, null); + + Mockito.verify(dataStoreHelper, Mockito.times(1)).switchToZone(store, null); + } + + @Test + public void testChangeStoragePoolScopeToCluster() { + Pair, Integer> hostPoolRecords = new Pair<>(null, 0); + Mockito.when(storagePoolHostDao.listByPoolIdNotInCluster(CLUSTER_ID, POOL_ID)).thenReturn(hostPoolRecords); + Mockito.doNothing().when(dataStoreHelper).switchToCluster(store, clusterScope); + + dataStoreLifeCycle.changeStoragePoolScopeToCluster(store, clusterScope, HypervisorType.KVM); + + hostPoolRecords.set(Arrays.asList(new StoragePoolHostVO(POOL_ID, HOST_ID, null)), 1); + Answer answer = new Answer(null, false, null); + Mockito.when(storagePoolHostDao.listByPoolIdNotInCluster(CLUSTER_ID, POOL_ID)).thenReturn(hostPoolRecords); + Mockito.when(agentManager.easySend(eq(HOST_ID), Mockito.any(DeleteStoragePoolCommand.class))).thenReturn(answer); + + dataStoreLifeCycle.changeStoragePoolScopeToCluster(store, clusterScope, HypervisorType.KVM); + + Mockito.verify(dataStoreHelper, Mockito.times(2)).switchToCluster(store, clusterScope); + } +} diff --git a/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeServiceTest.java b/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeServiceTest.java index 3a7fcfb6338e..c4241dfbc3a1 100644 --- a/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeServiceTest.java +++ b/engine/storage/volume/src/test/java/org/apache/cloudstack/storage/volume/VolumeServiceTest.java @@ -19,6 +19,23 @@ package org.apache.cloudstack.storage.volume; +import com.cloud.agent.api.storage.CheckAndRepairVolumeAnswer; +import com.cloud.agent.api.storage.CheckAndRepairVolumeCommand; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.StorageUnavailableException; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.storage.CheckAndRepairVolumePayload; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.Storage; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StoragePool; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.snapshot.SnapshotManager; +import com.cloud.utils.Pair; import java.util.ArrayList; import java.util.Arrays; @@ -39,21 +56,6 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; -import com.cloud.agent.api.storage.CheckAndRepairVolumeAnswer; -import com.cloud.agent.api.storage.CheckAndRepairVolumeCommand; -import com.cloud.agent.api.to.StorageFilerTO; -import com.cloud.exception.StorageUnavailableException; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.storage.CheckAndRepairVolumePayload; -import com.cloud.storage.Storage; -import com.cloud.storage.StorageManager; -import com.cloud.storage.StoragePool; -import com.cloud.storage.VolumeVO; -import com.cloud.storage.dao.VolumeDao; -import com.cloud.storage.snapshot.SnapshotManager; -import com.cloud.utils.Pair; - import junit.framework.TestCase; @RunWith(MockitoJUnitRunner.class) @@ -92,6 +94,9 @@ public class VolumeServiceTest extends TestCase{ @Mock HostDao hostDaoMock; + @Mock + DiskOfferingDao diskOfferingDaoMock; + @Before public void setup(){ volumeServiceImplSpy = Mockito.spy(new VolumeServiceImpl()); @@ -100,6 +105,7 @@ public void setup(){ volumeServiceImplSpy.snapshotMgr = snapshotManagerMock; volumeServiceImplSpy._storageMgr = storageManagerMock; volumeServiceImplSpy._hostDao = hostDaoMock; + volumeServiceImplSpy.diskOfferingDao = diskOfferingDaoMock; } @Test(expected = InterruptedException.class) @@ -309,4 +315,40 @@ public void testCheckAndRepairVolumeWhenFailure() throws StorageUnavailableExcep Assert.assertEquals(null, result); } + + @Test + public void validateDiskOfferingCheckForEncryption1Test() { + prepareOfferingsForEncryptionValidation(1L, true); + prepareOfferingsForEncryptionValidation(2L, true); + volumeServiceImplSpy.validateChangeDiskOfferingEncryptionType(1L, 2L); + } + + @Test + public void validateDiskOfferingCheckForEncryption2Test() { + prepareOfferingsForEncryptionValidation(1L, false); + prepareOfferingsForEncryptionValidation(2L, false); + volumeServiceImplSpy.validateChangeDiskOfferingEncryptionType(1L, 2L); + } + + @Test (expected = InvalidParameterValueException.class) + public void validateDiskOfferingCheckForEncryptionFail1Test() { + prepareOfferingsForEncryptionValidation(1L, false); + prepareOfferingsForEncryptionValidation(2L, true); + volumeServiceImplSpy.validateChangeDiskOfferingEncryptionType(1L, 2L); + } + + @Test (expected = InvalidParameterValueException.class) + public void validateDiskOfferingCheckForEncryptionFail2Test() { + prepareOfferingsForEncryptionValidation(1L, true); + prepareOfferingsForEncryptionValidation(2L, false); + volumeServiceImplSpy.validateChangeDiskOfferingEncryptionType(1L, 2L); + } + + private void prepareOfferingsForEncryptionValidation(long diskOfferingId, boolean encryption) { + DiskOfferingVO diskOffering = Mockito.mock(DiskOfferingVO.class); + + Mockito.when(diskOffering.getEncrypt()).thenReturn(encryption); + Mockito.when(diskOfferingDaoMock.findByIdIncludingRemoved(diskOfferingId)).thenReturn(diskOffering); + Mockito.when(diskOfferingDaoMock.findById(diskOfferingId)).thenReturn(diskOffering); + } } diff --git a/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java index 91f24fe70458..664d308e28d1 100644 --- a/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java +++ b/engine/userdata/src/main/java/org/apache/cloudstack/userdata/UserDataManagerImpl.java @@ -27,18 +27,15 @@ import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang3.StringUtils; -import com.cloud.configuration.ConfigurationManager; import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; public class UserDataManagerImpl extends ManagerBase implements UserDataManager { - - private static final int MAX_USER_DATA_LENGTH_BYTES = 2048; - private static final int MAX_HTTP_GET_LENGTH = 2 * MAX_USER_DATA_LENGTH_BYTES; + private static final int MAX_HTTP_GET_LENGTH = 2 * MAX_USER_DATA_LENGTH_BYTES; // 4KB private static final int NUM_OF_2K_BLOCKS = 512; - private static final int MAX_HTTP_POST_LENGTH = NUM_OF_2K_BLOCKS * MAX_USER_DATA_LENGTH_BYTES; + private static final int MAX_HTTP_POST_LENGTH = NUM_OF_2K_BLOCKS * MAX_USER_DATA_LENGTH_BYTES; // 1MB private List userDataProviders; private static Map userDataProvidersMap = new HashMap<>(); @@ -67,7 +64,7 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {}; + return new ConfigKey[] {VM_USERDATA_MAX_LENGTH}; } protected UserDataProvider getUserdataProvider(String name) { @@ -90,49 +87,57 @@ public String concatenateUserData(String userdata1, String userdata2, String use @Override public String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod) { - byte[] decodedUserData = null; - if (userData != null) { - - if (userData.contains("%")) { - try { - userData = URLDecoder.decode(userData, "UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new InvalidParameterValueException("Url decoding of userdata failed."); - } - } + logger.trace(String.format("Validating base64 encoded user data: [%s].", userData)); + if (StringUtils.isBlank(userData)) { + logger.debug("Null/empty base64 encoded user data set"); + return null; + } - if (!Base64.isBase64(userData)) { - throw new InvalidParameterValueException("User data is not base64 encoded"); - } - // If GET, use 4K. If POST, support up to 1M. - if (httpmethod.equals(BaseCmd.HTTPMethod.GET)) { - decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_GET_LENGTH, BaseCmd.HTTPMethod.GET); - } else if (httpmethod.equals(BaseCmd.HTTPMethod.POST)) { - decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_POST_LENGTH, BaseCmd.HTTPMethod.POST); + if (userData.contains("%")) { + try { + userData = URLDecoder.decode(userData, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new InvalidParameterValueException("Url decoding of user data failed."); } + } - if (decodedUserData == null || decodedUserData.length < 1) { - throw new InvalidParameterValueException("User data is too short"); - } - // Re-encode so that the '=' paddings are added if necessary since 'isBase64' does not require it, but python does on the VR. - return Base64.encodeBase64String(decodedUserData); + if (!Base64.isBase64(userData)) { + throw new InvalidParameterValueException("User data is not base64 encoded."); } - return null; - } - private byte[] validateAndDecodeByHTTPMethod(String userData, int maxHTTPLength, BaseCmd.HTTPMethod httpMethod) { byte[] decodedUserData = null; - if (userData.length() >= maxHTTPLength) { - throw new InvalidParameterValueException(String.format("User data is too long for an http %s request", httpMethod.toString())); + // If GET, use 4K. If POST, support up to 1M. + if (httpmethod.equals(BaseCmd.HTTPMethod.GET)) { + decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_GET_LENGTH, BaseCmd.HTTPMethod.GET); + } else if (httpmethod.equals(BaseCmd.HTTPMethod.POST)) { + decodedUserData = validateAndDecodeByHTTPMethod(userData, MAX_HTTP_POST_LENGTH, BaseCmd.HTTPMethod.POST); } - if (userData.length() > ConfigurationManager.VM_USERDATA_MAX_LENGTH.value()) { - throw new InvalidParameterValueException("User data has exceeded configurable max length : " + ConfigurationManager.VM_USERDATA_MAX_LENGTH.value()); + + // Re-encode so that the '=' paddings are added if necessary since 'isBase64' does not require it, but python does on the VR. + return Base64.encodeBase64String(decodedUserData); + } + + private byte[] validateAndDecodeByHTTPMethod(String userData, int maxHTTPLength, BaseCmd.HTTPMethod httpMethod) { + byte[] decodedUserData = Base64.decodeBase64(userData.getBytes()); + if (decodedUserData == null || decodedUserData.length < 1) { + throw new InvalidParameterValueException("User data is too short."); } - decodedUserData = Base64.decodeBase64(userData.getBytes()); - if (decodedUserData.length > maxHTTPLength) { + + logger.trace(String.format("Decoded user data: [%s].", decodedUserData)); + int userDataLength = userData.length(); + int decodedUserDataLength = decodedUserData.length; + logger.info(String.format("Configured base64 encoded user data size: %d bytes, actual user data size: %d bytes", userDataLength, decodedUserDataLength)); + + if (userDataLength > maxHTTPLength) { + logger.warn(String.format("Base64 encoded user data (size: %d bytes) too long for http %s request (accepted size: %d bytes)", userDataLength, httpMethod.toString(), maxHTTPLength)); throw new InvalidParameterValueException(String.format("User data is too long for http %s request", httpMethod.toString())); } + if (userDataLength > VM_USERDATA_MAX_LENGTH.value()) { + logger.warn(String.format("Base64 encoded user data (size: %d bytes) has exceeded configurable max length of %d bytes", userDataLength, VM_USERDATA_MAX_LENGTH.value())); + throw new InvalidParameterValueException("User data has exceeded configurable max length: " + VM_USERDATA_MAX_LENGTH.value()); + } + return decodedUserData; } } diff --git a/framework/ca/src/main/java/org/apache/cloudstack/framework/ca/CAProvider.java b/framework/ca/src/main/java/org/apache/cloudstack/framework/ca/CAProvider.java index 388cae7e0074..77b3ee27783c 100644 --- a/framework/ca/src/main/java/org/apache/cloudstack/framework/ca/CAProvider.java +++ b/framework/ca/src/main/java/org/apache/cloudstack/framework/ca/CAProvider.java @@ -22,6 +22,7 @@ import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.KeyStoreException; +import java.security.cert.CertificateParsingException; import java.security.cert.X509Certificate; import java.util.List; import java.util.Map; @@ -45,6 +46,7 @@ public interface CAProvider { /** * Issues certificate with provided options + * * @param domainNames * @param ipAddresses * @param validityDays @@ -104,4 +106,6 @@ public interface CAProvider { * @return returns description */ String getDescription(); + + boolean isManagementCertificate(java.security.cert.Certificate certificate) throws CertificateParsingException; } diff --git a/framework/ca/src/main/java/org/apache/cloudstack/framework/ca/CAService.java b/framework/ca/src/main/java/org/apache/cloudstack/framework/ca/CAService.java index facf13a5cb68..721c88bee507 100644 --- a/framework/ca/src/main/java/org/apache/cloudstack/framework/ca/CAService.java +++ b/framework/ca/src/main/java/org/apache/cloudstack/framework/ca/CAService.java @@ -21,6 +21,7 @@ import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.KeyStoreException; +import java.security.cert.CertificateParsingException; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; @@ -47,4 +48,6 @@ public interface CAService { * @return returns char[] passphrase */ char[] getKeyStorePassphrase(); + + boolean isManagementCertificate(java.security.cert.Certificate certificate) throws CertificateParsingException; } diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManager.java b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManager.java index 1b1406c1cecc..54f575830e4a 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManager.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManager.java @@ -16,8 +16,8 @@ // under the License. package com.cloud.cluster; -import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.management.ManagementServerHost; import com.cloud.utils.component.Manager; @@ -77,6 +77,8 @@ public interface ClusterManager extends Manager { */ String getSelfPeerName(); + String getSelfNodeIP(); + long getManagementNodeId(); /** diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java index e4e55eb9348a..050c2a7a1aa8 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java @@ -40,16 +40,16 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.cluster.dao.ManagementServerStatusDao; -import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.utils.identity.ManagementServerNode; import com.cloud.cluster.dao.ManagementServerHostDao; import com.cloud.cluster.dao.ManagementServerHostPeerDao; +import com.cloud.cluster.dao.ManagementServerStatusDao; import com.cloud.utils.DateUtil; import com.cloud.utils.Profiler; import com.cloud.utils.component.ComponentLifecycle; @@ -128,7 +128,7 @@ public ClusterManagerImpl() { // recursive remote calls between nodes // _executor = Executors.newCachedThreadPool(new NamedThreadFactory("Cluster-Worker")); - setRunLevel(ComponentLifecycle.RUN_LEVEL_FRAMEWORK); + setRunLevel(ComponentLifecycle.RUN_LEVEL_COMPONENT); } private void registerRequestPdu(final ClusterServiceRequestPdu pdu) { @@ -473,6 +473,7 @@ public String getSelfPeerName() { return Long.toString(_msId); } + @Override public String getSelfNodeIP() { return _clusterNodeIP; } diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceAdapter.java b/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceAdapter.java index 735de5bdac9d..e073a28a6221 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceAdapter.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceAdapter.java @@ -28,7 +28,5 @@ public interface ClusterServiceAdapter extends Adapter { public ClusterService getPeerService(String strPeer) throws RemoteException; - public String getServiceEndpointName(String strPeer); - public int getServicePort(); } diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceServletAdapter.java b/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceServletAdapter.java index 937ef4a62491..3e498b1fbec3 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceServletAdapter.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceServletAdapter.java @@ -23,6 +23,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.framework.config.ConfigDepot; import com.cloud.cluster.dao.ManagementServerHostDao; @@ -42,6 +43,8 @@ public class ClusterServiceServletAdapter extends AdapterBase implements Cluster @Inject private ManagementServerHostDao _mshostDao; @Inject + private CAManager caService; + @Inject protected ConfigDepot _configDepot; private ClusterServiceServletContainer _servletContainer; @@ -49,7 +52,7 @@ public class ClusterServiceServletAdapter extends AdapterBase implements Cluster private int _clusterServicePort = DEFAULT_SERVICE_PORT; public ClusterServiceServletAdapter() { - setRunLevel(ComponentLifecycle.RUN_LEVEL_FRAMEWORK); + setRunLevel(ComponentLifecycle.RUN_LEVEL_COMPONENT); } @Override @@ -64,12 +67,10 @@ public ClusterService getPeerService(String strPeer) throws RemoteException { String serviceUrl = getServiceEndpointName(strPeer); if (serviceUrl == null) return null; - - return new ClusterServiceServletImpl(serviceUrl); + return new ClusterServiceServletImpl(serviceUrl, caService); } - @Override - public String getServiceEndpointName(String strPeer) { + protected String getServiceEndpointName(String strPeer) { try { init(); } catch (ConfigurationException e) { @@ -93,7 +94,7 @@ public int getServicePort() { private String composeEndpointName(String nodeIP, int port) { StringBuffer sb = new StringBuffer(); - sb.append("http://").append(nodeIP).append(":").append(port).append("/clusterservice"); + sb.append("https://").append(nodeIP).append(":").append(port).append("/clusterservice"); return sb.toString(); } @@ -106,7 +107,8 @@ public boolean configure(String name, Map params) throws Configu @Override public boolean start() { _servletContainer = new ClusterServiceServletContainer(); - _servletContainer.start(new ClusterServiceServletHttpHandler(_manager), _clusterServicePort); + _servletContainer.start(new ClusterServiceServletHttpHandler(_manager), _manager.getSelfNodeIP(), + _clusterServicePort, caService); return true; } diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceServletContainer.java b/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceServletContainer.java index ac468089f473..e8c3de980168 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceServletContainer.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceServletContainer.java @@ -17,11 +17,23 @@ package com.cloud.cluster; import java.io.IOException; -import java.net.ServerSocket; +import java.net.InetAddress; import java.net.Socket; +import java.security.GeneralSecurityException; +import java.security.cert.Certificate; +import java.security.cert.CertificateParsingException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLServerSocketFactory; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocket; + +import org.apache.cloudstack.framework.ca.CAService; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.http.ConnectionClosedException; import org.apache.http.HttpException; import org.apache.http.impl.DefaultConnectionReuseStrategy; @@ -41,12 +53,12 @@ import org.apache.http.protocol.ResponseContent; import org.apache.http.protocol.ResponseDate; import org.apache.http.protocol.ResponseServer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; - +import com.cloud.utils.StringUtils; import com.cloud.utils.concurrency.NamedThreadFactory; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; +import com.cloud.utils.nio.Link; public class ClusterServiceServletContainer { @@ -55,9 +67,9 @@ public class ClusterServiceServletContainer { public ClusterServiceServletContainer() { } - public boolean start(HttpRequestHandler requestHandler, int port) { + public boolean start(HttpRequestHandler requestHandler, String ip, int port, CAService caService) { - listenerThread = new ListenerThread(requestHandler, port); + listenerThread = new ListenerThread(requestHandler, ip, port, caService); listenerThread.start(); return true; @@ -69,26 +81,46 @@ public void stop() { } } + + protected static SSLServerSocket getSecuredServerSocket(SSLContext sslContext, String ip, int port) + throws IOException { + SSLServerSocketFactory sslFactory = sslContext.getServerSocketFactory(); + SSLServerSocket serverSocket = null; + if (StringUtils.isNotEmpty(ip)) { + serverSocket = (SSLServerSocket) sslFactory.createServerSocket(port, 0, + InetAddress.getByName(ip)); + } else { + serverSocket = (SSLServerSocket) sslFactory.createServerSocket(port); + } + serverSocket.setNeedClientAuth(true); + return serverSocket; + } + static class ListenerThread extends Thread { private static Logger LOGGER = LogManager.getLogger(ListenerThread.class); - private HttpService _httpService = null; - private volatile ServerSocket _serverSocket = null; - private HttpParams _params = null; - private ExecutorService _executor; + private HttpService httpService = null; + private volatile SSLServerSocket serverSocket = null; + private HttpParams params = null; + private ExecutorService executor; + private CAService caService = null; - public ListenerThread(HttpRequestHandler requestHandler, int port) { - _executor = Executors.newCachedThreadPool(new NamedThreadFactory("Cluster-Listener")); + public ListenerThread(HttpRequestHandler requestHandler, String ip, int port, + CAService caService) { + this.executor = Executors.newCachedThreadPool(new NamedThreadFactory("Cluster-Listener")); + this.caService = caService; try { - _serverSocket = new ServerSocket(port); - } catch (IOException ioex) { - LOGGER.error("error initializing cluster service servlet container", ioex); + SSLContext sslContext = Link.initManagementSSLContext(caService); + serverSocket = getSecuredServerSocket(sslContext, ip, port); + } catch (IOException | GeneralSecurityException e) { + LOGGER.error("Error initializing cluster service servlet container for secure connection", + e); return; } - _params = new BasicHttpParams(); - _params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 5000) + params = new BasicHttpParams(); + params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 5000) .setIntParameter(CoreConnectionPNames.SOCKET_BUFFER_SIZE, 8 * 1024) .setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, false) .setBooleanParameter(CoreConnectionPNames.TCP_NODELAY, true) @@ -106,35 +138,55 @@ public ListenerThread(HttpRequestHandler requestHandler, int port) { reqistry.register("/clusterservice", requestHandler); // Set up the HTTP service - _httpService = new HttpService(httpproc, new DefaultConnectionReuseStrategy(), new DefaultHttpResponseFactory()); - _httpService.setParams(_params); - _httpService.setHandlerResolver(reqistry); + httpService = new HttpService(httpproc, new DefaultConnectionReuseStrategy(), new DefaultHttpResponseFactory()); + httpService.setParams(params); + httpService.setHandlerResolver(reqistry); } public void stopRunning() { - if (_serverSocket != null) { + if (serverSocket != null) { try { - _serverSocket.close(); + serverSocket.close(); } catch (IOException e) { LOGGER.info("[ignored] error on closing server socket", e); } - _serverSocket = null; + serverSocket = null; } } + protected boolean isValidPeerConnection(Socket socket) throws SSLPeerUnverifiedException, + CertificateParsingException { + SSLSocket sslSocket = (SSLSocket) socket; + SSLSession session = sslSocket.getSession(); + if (session == null || !session.isValid()) { + return false; + } + Certificate[] certs = session.getPeerCertificates(); + if (certs == null || certs.length < 1) { + return false; + } + return caService.isManagementCertificate(certs[0]); + } + @Override public void run() { if (LOGGER.isInfoEnabled()) - LOGGER.info("Cluster service servlet container listening on port " + _serverSocket.getLocalPort()); + LOGGER.info(String.format("Cluster service servlet container listening on host: %s and port %d", + serverSocket.getInetAddress().getHostAddress(), serverSocket.getLocalPort())); - while (_serverSocket != null) { + while (serverSocket != null) { try { // Set up HTTP connection - Socket socket = _serverSocket.accept(); + Socket socket = serverSocket.accept(); final DefaultHttpServerConnection conn = new DefaultHttpServerConnection(); - conn.bind(socket, _params); - - _executor.execute(new ManagedContextRunnable() { + conn.bind(socket, params); + if (!isValidPeerConnection(socket)) { + LOGGER.warn(String.format("Failure during validating cluster request from %s", + socket.getInetAddress().getHostAddress())); + conn.shutdown(); + continue; + } + executor.execute(new ManagedContextRunnable() { @Override protected void runInContext() { HttpContext context = new BasicHttpContext(null); @@ -143,7 +195,7 @@ protected void runInContext() { if (LOGGER.isTraceEnabled()) LOGGER.trace("dispatching cluster request from " + conn.getRemoteAddress().toString()); - _httpService.handleRequest(conn, context); + httpService.handleRequest(conn, context); if (LOGGER.isTraceEnabled()) LOGGER.trace("Cluster request from " + conn.getRemoteAddress().toString() + " is processed"); @@ -178,7 +230,7 @@ protected void runInContext() { } } - _executor.shutdown(); + executor.shutdown(); if (LOGGER.isInfoEnabled()) LOGGER.info("Cluster service servlet container shutdown"); } diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceServletImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceServletImpl.java index b60012dbeef1..d582538c31e0 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceServletImpl.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceServletImpl.java @@ -17,99 +17,144 @@ package com.cloud.cluster; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.rmi.RemoteException; +import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.List; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpException; +import javax.net.ssl.SSLContext; + +import org.apache.cloudstack.framework.ca.CAService; import org.apache.commons.httpclient.HttpStatus; -import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; -import org.apache.commons.httpclient.methods.PostMethod; -import org.apache.commons.httpclient.params.HttpClientParams; -import org.apache.logging.log4j.Logger; +import org.apache.http.NameValuePair; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import com.cloud.utils.HttpUtils; import com.cloud.utils.Profiler; +import com.cloud.utils.nio.Link; +import com.google.gson.Gson; public class ClusterServiceServletImpl implements ClusterService { private static final long serialVersionUID = 4574025200012566153L; protected Logger logger = LogManager.getLogger(getClass()); - private String _serviceUrl; + private String serviceUrl; + + private CAService caService; + + private Gson gson = new Gson(); + + protected static CloseableHttpClient s_client = null; - protected static HttpClient s_client = null; + private void logPostParametersForFailedEncoding(List parameters) { + if (logger.isTraceEnabled()) { + logger.trace(String.format("%s encoding failed for POST parameters: %s", HttpUtils.UTF_8, + gson.toJson(parameters))); + } + } public ClusterServiceServletImpl() { } - public ClusterServiceServletImpl(final String serviceUrl) { - logger.info("Setup cluster service servlet. service url: " + serviceUrl + ", request timeout: " + ClusterServiceAdapter.ClusterMessageTimeOut.value() + - " seconds"); + public ClusterServiceServletImpl(final String serviceUrl, final CAService caService) { + logger.info(String.format("Setup cluster service servlet. service url: %s, request timeout: %d seconds", serviceUrl, + ClusterServiceAdapter.ClusterMessageTimeOut.value())); + this.serviceUrl = serviceUrl; + this.caService = caService; + } - _serviceUrl = serviceUrl; + protected List getClusterServicePduPostParameters(final ClusterServicePdu pdu) { + List postParameters = new ArrayList<>(); + postParameters.add(new BasicNameValuePair("method", Integer.toString(RemoteMethodConstants.METHOD_DELIVER_PDU))); + postParameters.add(new BasicNameValuePair("sourcePeer", pdu.getSourcePeer())); + postParameters.add(new BasicNameValuePair("destPeer", pdu.getDestPeer())); + postParameters.add(new BasicNameValuePair("pduSeq", Long.toString(pdu.getSequenceId()))); + postParameters.add(new BasicNameValuePair("pduAckSeq", Long.toString(pdu.getAckSequenceId()))); + postParameters.add(new BasicNameValuePair("agentId", Long.toString(pdu.getAgentId()))); + postParameters.add(new BasicNameValuePair("gsonPackage", pdu.getJsonPackage())); + postParameters.add(new BasicNameValuePair("stopOnError", pdu.isStopOnError() ? "1" : "0")); + postParameters.add(new BasicNameValuePair("pduType", Integer.toString(pdu.getPduType()))); + return postParameters; } @Override public String execute(final ClusterServicePdu pdu) throws RemoteException { - - final HttpClient client = getHttpClient(); - final PostMethod method = new PostMethod(_serviceUrl); - - method.addParameter("method", Integer.toString(RemoteMethodConstants.METHOD_DELIVER_PDU)); - method.addParameter("sourcePeer", pdu.getSourcePeer()); - method.addParameter("destPeer", pdu.getDestPeer()); - method.addParameter("pduSeq", Long.toString(pdu.getSequenceId())); - method.addParameter("pduAckSeq", Long.toString(pdu.getAckSequenceId())); - method.addParameter("agentId", Long.toString(pdu.getAgentId())); - method.addParameter("gsonPackage", pdu.getJsonPackage()); - method.addParameter("stopOnError", pdu.isStopOnError() ? "1" : "0"); - method.addParameter("pduType", Integer.toString(pdu.getPduType())); + if (logger.isDebugEnabled()) { + logger.debug(String.format("Executing ClusterServicePdu with service URL: %s", serviceUrl)); + } + final CloseableHttpClient client = getHttpClient(); + final HttpPost method = new HttpPost(serviceUrl); + final List postParameters = getClusterServicePduPostParameters(pdu); + try { + method.setEntity(new UrlEncodedFormEntity(postParameters, HttpUtils.UTF_8)); + } catch (UnsupportedEncodingException e) { + logger.error("Failed to encode request POST parameters", e); + logPostParametersForFailedEncoding(postParameters); + throw new RemoteException("Failed to encode request POST parameters", e); + } return executePostMethod(client, method); } + protected List getPingPostParameters(final String callingPeer) { + List postParameters = new ArrayList<>(); + postParameters.add(new BasicNameValuePair("method", Integer.toString(RemoteMethodConstants.METHOD_PING))); + postParameters.add(new BasicNameValuePair("callingPeer", callingPeer)); + return postParameters; + } + @Override public boolean ping(final String callingPeer) throws RemoteException { if (logger.isDebugEnabled()) { - logger.debug("Ping at " + _serviceUrl); + logger.debug("Ping at " + serviceUrl); } - final HttpClient client = getHttpClient(); - final PostMethod method = new PostMethod(_serviceUrl); + final CloseableHttpClient client = getHttpClient(); + final HttpPost method = new HttpPost(serviceUrl); - method.addParameter("method", Integer.toString(RemoteMethodConstants.METHOD_PING)); - method.addParameter("callingPeer", callingPeer); + List postParameters = getPingPostParameters(callingPeer); + try { + method.setEntity(new UrlEncodedFormEntity(postParameters, HttpUtils.UTF_8)); + } catch (UnsupportedEncodingException e) { + logger.error("Failed to encode ping request POST parameters", e); + logPostParametersForFailedEncoding(postParameters); + throw new RemoteException("Failed to encode ping request POST parameters", e); + } final String returnVal = executePostMethod(client, method); - if ("true".equalsIgnoreCase(returnVal)) { - return true; - } - return false; + return Boolean.TRUE.toString().equalsIgnoreCase(returnVal); } - private String executePostMethod(final HttpClient client, final PostMethod method) { - int response = 0; + private String executePostMethod(final CloseableHttpClient client, final HttpPost method) { String result = null; try { final Profiler profiler = new Profiler(); profiler.start(); - response = client.executeMethod(method); + CloseableHttpResponse httpResponse = client.execute(method); + int response = httpResponse.getStatusLine().getStatusCode(); if (response == HttpStatus.SC_OK) { - result = method.getResponseBodyAsString(); + result = EntityUtils.toString(httpResponse.getEntity()); profiler.stop(); if (logger.isDebugEnabled()) { - logger.debug("POST " + _serviceUrl + " response :" + result + ", responding time: " + profiler.getDurationInMillis() + " ms"); + logger.debug("POST " + serviceUrl + " response :" + result + ", responding time: " + profiler.getDurationInMillis() + " ms"); } } else { profiler.stop(); - logger.error("Invalid response code : " + response + ", from : " + _serviceUrl + ", method : " + method.getParameter("method") + " responding time: " + + logger.error("Invalid response code : " + response + ", from : " + serviceUrl + ", method : " + method.getParams().getParameter("method") + " responding time: " + profiler.getDurationInMillis()); } - } catch (final HttpException e) { - logger.error("HttpException from : " + _serviceUrl + ", method : " + method.getParameter("method")); - } catch (final IOException e) { - logger.error("IOException from : " + _serviceUrl + ", method : " + method.getParameter("method")); - } catch (final Throwable e) { - logger.error("Exception from : " + _serviceUrl + ", method : " + method.getParameter("method") + ", exception :", e); + } catch (IOException e) { + logger.error("Exception from : " + serviceUrl + ", method : " + method.getParams().getParameter("method") + ", exception :", e); } finally { method.releaseConnection(); } @@ -117,20 +162,25 @@ private String executePostMethod(final HttpClient client, final PostMethod metho return result; } - private HttpClient getHttpClient() { - + private CloseableHttpClient getHttpClient() { if (s_client == null) { - final MultiThreadedHttpConnectionManager mgr = new MultiThreadedHttpConnectionManager(); - mgr.getParams().setDefaultMaxConnectionsPerHost(4); - - // TODO make it configurable - mgr.getParams().setMaxTotalConnections(1000); + SSLContext sslContext = null; + try { + sslContext = Link.initManagementSSLContext(caService); + } catch (GeneralSecurityException | IOException e) { + throw new RuntimeException(e); + } - s_client = new HttpClient(mgr); - final HttpClientParams clientParams = new HttpClientParams(); - clientParams.setSoTimeout(ClusterServiceAdapter.ClusterMessageTimeOut.value() * 1000); + int timeout = ClusterServiceAdapter.ClusterMessageTimeOut.value() * 1000; + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout) + .setSocketTimeout(timeout).build(); - s_client.setParams(clientParams); + s_client = HttpClientBuilder.create() + .setDefaultRequestConfig(config) + .setSSLContext(sslContext) + .build(); } return s_client; } diff --git a/framework/cluster/src/test/java/com/cloud/cluster/ClusterManagerImplTest.java b/framework/cluster/src/test/java/com/cloud/cluster/ClusterManagerImplTest.java new file mode 100644 index 000000000000..9b1854f73487 --- /dev/null +++ b/framework/cluster/src/test/java/com/cloud/cluster/ClusterManagerImplTest.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class ClusterManagerImplTest { + @InjectMocks + ClusterManagerImpl clusterManager = new ClusterManagerImpl(); + + @Test + public void testGetSelfNodeIP() { + String ip = "1.2.3.4"; + ReflectionTestUtils.setField(clusterManager, "_clusterNodeIP", ip); + Assert.assertEquals(ip, clusterManager.getSelfNodeIP()); + } +} diff --git a/framework/cluster/src/test/java/com/cloud/cluster/ClusterServiceServletAdapterTest.java b/framework/cluster/src/test/java/com/cloud/cluster/ClusterServiceServletAdapterTest.java index 25266106f43f..6f4b7d6aa9ee 100644 --- a/framework/cluster/src/test/java/com/cloud/cluster/ClusterServiceServletAdapterTest.java +++ b/framework/cluster/src/test/java/com/cloud/cluster/ClusterServiceServletAdapterTest.java @@ -50,7 +50,7 @@ public void setup() throws IllegalArgumentException, IllegalAccessException, NoS @Test public void testRunLevel() { int runLevel = clusterServiceServletAdapter.getRunLevel(); - assertTrue(runLevel == ComponentLifecycle.RUN_LEVEL_FRAMEWORK); + assertTrue(runLevel == ComponentLifecycle.RUN_LEVEL_COMPONENT); assertTrue(runLevel == clusterManagerImpl.getRunLevel()); } } diff --git a/framework/cluster/src/test/java/com/cloud/cluster/ClusterServiceServletContainerTest.java b/framework/cluster/src/test/java/com/cloud/cluster/ClusterServiceServletContainerTest.java new file mode 100644 index 000000000000..baf4e5841bdb --- /dev/null +++ b/framework/cluster/src/test/java/com/cloud/cluster/ClusterServiceServletContainerTest.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.cluster; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.List; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLContextSpi; +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLServerSocketFactory; + +import org.apache.commons.collections.CollectionUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.utils.StringUtils; + +@RunWith(MockitoJUnitRunner.class) +public class ClusterServiceServletContainerTest { + + private void runGetSecuredServerSocket(String ip) { + SSLContext sslContext = Mockito.mock(SSLContext.class); + SSLContextSpi sslContextSpi = Mockito.mock(SSLContextSpi.class); + ReflectionTestUtils.setField(sslContext, "contextSpi", sslContextSpi); + SSLServerSocketFactory factory = Mockito.mock(SSLServerSocketFactory.class); + Mockito.when(sslContext.getServerSocketFactory()).thenReturn(factory); + int port = 9090; + final List socketNeedClientAuth = new ArrayList<>(); + try { + SSLServerSocket socketMock = Mockito.mock(SSLServerSocket.class); + if (StringUtils.isBlank(ip)) { + Mockito.when(factory.createServerSocket(port)).thenReturn(socketMock); + } else { + Mockito.when(factory.createServerSocket(Mockito.anyInt(), Mockito.anyInt(), + Mockito.any(InetAddress.class))).thenReturn(socketMock); + } + Mockito.doAnswer((Answer) invocationOnMock -> { + boolean needClientAuth = (boolean) invocationOnMock.getArguments()[0]; + socketNeedClientAuth.add(needClientAuth); + return null; + }).when(socketMock).setNeedClientAuth(Mockito.anyBoolean()); + SSLServerSocket socket = ClusterServiceServletContainer.getSecuredServerSocket(sslContext, ip, 9090); + if (StringUtils.isBlank(ip)) { + Mockito.verify(factory, Mockito.times(1)).createServerSocket(port); + } else { + Mockito.verify(factory, Mockito.times(1)).createServerSocket(port, 0, InetAddress.getByName(ip)); + } + Mockito.verify(socket, Mockito.times(1)).setNeedClientAuth(Mockito.anyBoolean()); + Assert.assertTrue(CollectionUtils.isNotEmpty(socketNeedClientAuth)); + Assert.assertTrue(socketNeedClientAuth.get(0)); + } catch (IOException e) { + Assert.fail("Exception occurred: " + e.getMessage()); + } + } + + @Test + public void testGetSecuredServerSocketNoIp() { + runGetSecuredServerSocket(""); + } + + @Test + public void testGetSecuredServerSocketIp() { + runGetSecuredServerSocket("1.2.3.4"); + } +} diff --git a/framework/cluster/src/test/java/com/cloud/cluster/ClusterServiceServletImplTest.java b/framework/cluster/src/test/java/com/cloud/cluster/ClusterServiceServletImplTest.java new file mode 100644 index 000000000000..361c77dbeff4 --- /dev/null +++ b/framework/cluster/src/test/java/com/cloud/cluster/ClusterServiceServletImplTest.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster; + +import java.util.List; +import java.util.Optional; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.http.NameValuePair; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class ClusterServiceServletImplTest { + + @InjectMocks + ClusterServiceServletImpl clusterServiceServlet = new ClusterServiceServletImpl(); + + @Test + public void testClusterServicePduPostParameters() { + List parameters = + clusterServiceServlet.getClusterServicePduPostParameters(Mockito.mock(ClusterServicePdu.class)); + Assert.assertTrue(CollectionUtils.isNotEmpty(parameters)); + Optional opt = parameters.stream().filter(x -> x.getName().equals("method")).findFirst(); + Assert.assertTrue(opt.isPresent()); + NameValuePair val = opt.get(); + Assert.assertEquals(Integer.toString(RemoteMethodConstants.METHOD_DELIVER_PDU), val.getValue()); + } + + @Test + public void testPingPostParameters() { + String peer = "1.2.3.4"; + List parameters = + clusterServiceServlet.getPingPostParameters(peer); + Assert.assertTrue(CollectionUtils.isNotEmpty(parameters)); + Optional opt = parameters.stream().filter(x -> x.getName().equals("method")).findFirst(); + Assert.assertTrue(opt.isPresent()); + NameValuePair val = opt.get(); + Assert.assertEquals(Integer.toString(RemoteMethodConstants.METHOD_PING), val.getValue()); + opt = parameters.stream().filter(x -> x.getName().equals("callingPeer")).findFirst(); + Assert.assertTrue(opt.isPresent()); + val = opt.get(); + Assert.assertEquals(peer, val.getValue()); + } +} diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index df93f78fa833..fa570e0e8fbe 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -41,7 +41,7 @@ public enum Scope { } public enum Kind { - CSV, Order, Select + CSV, Order, Select, WhitespaceSeparatedListWithOptions } private final String _category; @@ -136,6 +136,10 @@ public ConfigKey(String category, Class type, String name, String defaultValu this(type, name, category, defaultValue, description, isDynamic, Scope.Global, null); } + public ConfigKey(String category, Class type, String name, String defaultValue, String description, boolean isDynamic, Kind kind, String options) { + this(type, name, category, defaultValue, description, isDynamic, Scope.Global, null, null, null, null, null, kind, options); + } + public ConfigKey(String category, Class type, String name, String defaultValue, String description, boolean isDynamic, String parent) { this(type, name, category, defaultValue, description, isDynamic, Scope.Global, null, null, parent, null, null, null, null); } @@ -211,7 +215,7 @@ public boolean isSameKeyAs(Object obj) { public T value() { if (_value == null || isDynamic()) { - ConfigurationVO vo = s_depot != null ? s_depot.global().findById(key()) : null; + ConfigurationVO vo = (s_depot != null && s_depot.global() != null) ? s_depot.global().findById(key()) : null; final String value = (vo != null && vo.getValue() != null) ? vo.getValue() : defaultValue(); _value = ((value == null) ? (T)defaultValue() : valueOf(value)); } diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java index 2fc02301cb7a..84750c2068c8 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java @@ -229,6 +229,32 @@ public interface GenericDao { */ int expunge(final SearchCriteria sc); + /** + * remove the entity bean specified by the search criteria and filter + * @param sc + * @param filter + * @return number of rows deleted + */ + int expunge(final SearchCriteria sc, final Filter filter); + + /** + * remove the entity bean specified by the search criteria and batchSize + * @param sc + * @param batchSize + * @return number of rows deleted + */ + int batchExpunge(final SearchCriteria sc, final Long batchSize); + + int expungeList(List ids); + + /** + * Delete the entity beans specified by the search criteria with a given limit + * @param sc Search criteria + * @param limit Maximum number of rows that will be affected + * @return Number of rows deleted + */ + int expunge(SearchCriteria sc, long limit); + /** * expunge the removed rows. */ diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java index a09f323905e4..b7e4f44cf8cd 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java @@ -20,6 +20,8 @@ import java.io.UnsupportedEncodingException; import java.lang.reflect.Array; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.math.BigInteger; @@ -59,9 +61,12 @@ import javax.persistence.Table; import javax.persistence.TableGenerator; -import com.amazonaws.util.CollectionUtils; import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import com.amazonaws.util.CollectionUtils; import com.cloud.utils.DateUtil; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; @@ -74,8 +79,6 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Ip; import com.cloud.utils.net.NetUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; import net.sf.cglib.proxy.Callback; import net.sf.cglib.proxy.CallbackFilter; @@ -1234,7 +1237,13 @@ public boolean expunge(final ID id) { // FIXME: Does not work for joins. @Override - public int expunge(final SearchCriteria sc) { + public int expunge(final SearchCriteria sc, long limit) { + Filter filter = new Filter(limit); + return expunge(sc, filter); + } + + @Override + public int expunge(final SearchCriteria sc, final Filter filter) { if (sc == null) { throw new CloudRuntimeException("Call to throw new expunge with null search Criteria"); } @@ -1246,6 +1255,7 @@ public int expunge(final SearchCriteria sc) { if (sc != null && sc.getWhereClause().length() > 0) { str.append(sc.getWhereClause()); } + addFilter(str, filter); final String sql = str.toString(); @@ -1264,6 +1274,47 @@ public int expunge(final SearchCriteria sc) { throw new CloudRuntimeException("Caught: " + pstmt, e); } } + @Override + public int expunge(final SearchCriteria sc) { + return expunge(sc, null); + } + + @Override + public int batchExpunge(final SearchCriteria sc, final Long batchSize) { + Filter filter = null; + final long batchSizeFinal = ObjectUtils.defaultIfNull(batchSize, 0L); + if (batchSizeFinal > 0) { + filter = new Filter(batchSizeFinal); + } + int expunged = 0; + int currentExpunged = 0; + do { + currentExpunged = expunge(sc, filter); + expunged += currentExpunged; + } while (batchSizeFinal > 0 && currentExpunged >= batchSizeFinal); + return expunged; + } + + @Override + public int expungeList(final List ids) { + if (org.apache.commons.collections.CollectionUtils.isEmpty(ids)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + Object obj = null; + try { + Method m = sb.entity().getClass().getMethod("getId"); + obj = m.invoke(sb.entity()); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException ignored) {} + if (obj == null) { + logger.warn(String.format("Unable to get ID object for entity: %s", _entityBeanType.getSimpleName())); + return 0; + } + sb.and("id", obj, SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("id", ids.toArray()); + return expunge(sc); + } @DB() protected StringBuilder createPartialSelectSql(SearchCriteria sc, final boolean whereClause, final boolean enableQueryCache) { diff --git a/framework/events/src/main/java/org/apache/cloudstack/framework/events/Event.java b/framework/events/src/main/java/org/apache/cloudstack/framework/events/Event.java index 4a3eaf9e68c0..7a14f385fa15 100644 --- a/framework/events/src/main/java/org/apache/cloudstack/framework/events/Event.java +++ b/framework/events/src/main/java/org/apache/cloudstack/framework/events/Event.java @@ -20,22 +20,49 @@ package org.apache.cloudstack.framework.events; import com.google.gson.Gson; +import com.google.gson.annotations.Expose; public class Event { + @Expose(serialize = false, deserialize = false) + Long eventId; + @Expose(serialize = false, deserialize = false) + String eventUuid; String eventCategory; String eventType; String eventSource; String resourceType; String resourceUUID; String description; + @Expose(serialize = false, deserialize = false) + Long resourceAccountId; + @Expose(serialize = false, deserialize = false) + String resourceAccountUuid; + @Expose(serialize = false, deserialize = false) + Long resourceDomainId; public Event(String eventSource, String eventCategory, String eventType, String resourceType, String resourceUUID) { - this.eventCategory = eventCategory; - this.eventType = eventType; - this.eventSource = eventSource; - this.resourceType = resourceType; - this.resourceUUID = resourceUUID; + setEventCategory(eventCategory); + setEventType(eventType); + setEventSource(eventSource); + setResourceType(resourceType); + setResourceUUID(resourceUUID); + } + + public Long getEventId() { + return eventId; + } + + public void setEventId(Long eventId) { + this.eventId = eventId; + } + + public String getEventUuid() { + return eventUuid; + } + + public void setEventUuid(String eventUuid) { + this.eventUuid = eventUuid; } public String getEventCategory() { @@ -68,7 +95,7 @@ public String getDescription() { public void setDescription(Object message) { Gson gson = new Gson(); - this.description = gson.toJson(message).toString(); + this.description = gson.toJson(message); } public void setDescription(String description) { @@ -90,4 +117,28 @@ public void setResourceUUID(String uuid) { public String getResourceUUID() { return resourceUUID; } + + public Long getResourceAccountId() { + return resourceAccountId; + } + + public void setResourceAccountId(Long resourceAccountId) { + this.resourceAccountId = resourceAccountId; + } + + public String getResourceAccountUuid() { + return resourceAccountUuid; + } + + public void setResourceAccountUuid(String resourceAccountUuid) { + this.resourceAccountUuid = resourceAccountUuid; + } + + public Long getResourceDomainId() { + return resourceDomainId; + } + + public void setResourceDomainId(Long resourceDomainId) { + this.resourceDomainId = resourceDomainId; + } } diff --git a/framework/events/src/main/java/org/apache/cloudstack/framework/events/EventBus.java b/framework/events/src/main/java/org/apache/cloudstack/framework/events/EventBus.java index 8c8c08fcfdf2..6cd6256ce81d 100644 --- a/framework/events/src/main/java/org/apache/cloudstack/framework/events/EventBus.java +++ b/framework/events/src/main/java/org/apache/cloudstack/framework/events/EventBus.java @@ -27,6 +27,8 @@ */ public interface EventBus { + String getName(); + /** * publish an event on to the event bus * diff --git a/framework/events/src/main/java/org/apache/cloudstack/framework/events/EventDistributor.java b/framework/events/src/main/java/org/apache/cloudstack/framework/events/EventDistributor.java new file mode 100644 index 000000000000..01185359d6f9 --- /dev/null +++ b/framework/events/src/main/java/org/apache/cloudstack/framework/events/EventDistributor.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.framework.events; + +import java.util.Map; + +import com.cloud.utils.component.Manager; + +public interface EventDistributor extends Manager { + /** + * Publish an event on to all the available event buses + * + * @param event event that needs to be published on the event bus + * @return Map of bus names and EventBusException for buses that failed with + * exception + */ + Map publish(Event event); +} diff --git a/framework/events/src/main/java/org/apache/cloudstack/framework/events/EventDistributorImpl.java b/framework/events/src/main/java/org/apache/cloudstack/framework/events/EventDistributorImpl.java new file mode 100644 index 000000000000..a67ff5cc9266 --- /dev/null +++ b/framework/events/src/main/java/org/apache/cloudstack/framework/events/EventDistributorImpl.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.framework.events; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.PostConstruct; + +import org.apache.commons.lang3.StringUtils; + +import com.cloud.utils.component.ManagerBase; + +public class EventDistributorImpl extends ManagerBase implements EventDistributor { + + List eventBuses; + + public void setEventBuses(List eventBuses) { + this.eventBuses = eventBuses; + } + + @PostConstruct + public void init() { + logger.trace("Found {} event buses : {}", () -> eventBuses.size(), + () -> StringUtils.join(eventBuses.stream().map(x->x.getClass().getName()).toArray())); + } + + @Override + public Map publish(Event event) { + Map exceptions = new HashMap<>(); + if (event == null) { + return exceptions; + } + logger.trace("Publishing event [category: {}, type: {}]: {} to {} event buses", + event.getEventCategory(), event.getEventType(), + event.getDescription(), eventBuses.size()); + for (EventBus bus : eventBuses) { + try { + bus.publish(event); + } catch (EventBusException e) { + logger.warn("Failed to publish event [category: {}, type: {}] on bus {}", + event.getEventCategory(), event.getEventType(), bus.getName()); + logger.trace(event.getDescription()); + exceptions.put(bus.getName(), e); + } + } + return exceptions; + } + +} diff --git a/framework/events/src/test/java/org/apache/cloudstack/framework/events/EventDistributorImplTest.java b/framework/events/src/test/java/org/apache/cloudstack/framework/events/EventDistributorImplTest.java new file mode 100644 index 000000000000..8a8dd91b9d83 --- /dev/null +++ b/framework/events/src/test/java/org/apache/cloudstack/framework/events/EventDistributorImplTest.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.events; + +import java.util.List; +import java.util.Map; + +import org.apache.commons.collections.MapUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class EventDistributorImplTest { + + @InjectMocks + EventDistributorImpl eventDistributor = new EventDistributorImpl(); + + @Test + public void testSetEventBuses() { + Assert.assertNull(ReflectionTestUtils.getField(eventDistributor, "eventBuses")); + EventBus eventBus = Mockito.mock(EventBus.class); + EventBus eventBus1 = Mockito.mock(EventBus.class); + eventDistributor.setEventBuses(List.of(eventBus, eventBus1)); + Assert.assertNotNull(ReflectionTestUtils.getField(eventDistributor, "eventBuses")); + } + + @Test + public void testPublishNullEvent() { + Map exceptionMap = eventDistributor.publish(null); + Assert.assertTrue(MapUtils.isEmpty(exceptionMap)); + } + + @Test + public void testPublishOneReturnsException() throws EventBusException { + String busName = "Test"; + EventBus eventBus = Mockito.mock(EventBus.class); + Mockito.doReturn(busName).when(eventBus).getName(); + Mockito.doThrow(EventBusException.class).when(eventBus).publish(Mockito.any(Event.class)); + EventBus eventBus1 = Mockito.mock(EventBus.class); + Mockito.doNothing().when(eventBus1).publish(Mockito.any(Event.class)); + eventDistributor.eventBuses = List.of(eventBus, eventBus1); + Map exceptionMap = eventDistributor.publish(Mockito.mock(Event.class)); + Assert.assertTrue(MapUtils.isNotEmpty(exceptionMap)); + Assert.assertEquals(1, exceptionMap.size()); + Assert.assertTrue(exceptionMap.containsKey(busName)); + } +} diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDao.java index 89601e6b5d20..b3bfda0334cf 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDao.java @@ -39,4 +39,5 @@ public interface VmWorkJobDao extends GenericDao { void expungeCompletedWorkJobs(Date cutDate); void expungeLeftoverWorkJobs(long msid); + int expungeByVmList(List vmIds, Long batchSize); } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImpl.java index e66221cc8fe0..3b167498a377 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImpl.java @@ -24,10 +24,10 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; - import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO; import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO.Step; import org.apache.cloudstack.jobs.JobInfo; +import org.apache.commons.collections.CollectionUtils; import com.cloud.utils.DateUtil; import com.cloud.utils.db.Filter; @@ -212,4 +212,16 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } }); } + + @Override + public int expungeByVmList(List vmIds, Long batchSize) { + if (CollectionUtils.isEmpty(vmIds)) { + return 0; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("vmIds", sb.entity().getVmInstanceId(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + sc.setParameters("vmIds", vmIds.toArray()); + return batchExpunge(sc, batchSize); + } } diff --git a/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImplTest.java b/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImplTest.java new file mode 100644 index 000000000000..3e2bc15b1e0c --- /dev/null +++ b/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/dao/VmWorkJobDaoImplTest.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.dao; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class VmWorkJobDaoImplTest { + + @Spy + VmWorkJobDaoImpl vmWorkJobDaoImpl; + + @Test + public void testExpungeByVmListNoVms() { + Assert.assertEquals(0, vmWorkJobDaoImpl.expungeByVmList( + new ArrayList<>(), 100L)); + Assert.assertEquals(0, vmWorkJobDaoImpl.expungeByVmList( + null, 100L)); + } + + @Test + public void testExpungeByVmList() { + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(sb.create()).thenReturn(sc); + Mockito.doAnswer((Answer) invocationOnMock -> { + Long batchSize = (Long)invocationOnMock.getArguments()[1]; + return batchSize == null ? 0 : batchSize.intValue(); + }).when(vmWorkJobDaoImpl).batchExpunge(Mockito.any(SearchCriteria.class), Mockito.anyLong()); + Mockito.when(vmWorkJobDaoImpl.createSearchBuilder()).thenReturn(sb); + final VmWorkJobVO mockedVO = Mockito.mock(VmWorkJobVO.class); + Mockito.when(sb.entity()).thenReturn(mockedVO); + List vmIds = List.of(1L, 2L); + Object[] array = vmIds.toArray(); + Long batchSize = 50L; + Assert.assertEquals(batchSize.intValue(), vmWorkJobDaoImpl.expungeByVmList(List.of(1L, 2L), batchSize)); + Mockito.verify(sc).setParameters("vmIds", array); + Mockito.verify(vmWorkJobDaoImpl, Mockito.times(1)) + .batchExpunge(sc, batchSize); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java index 9c15a47444ae..ded35338aeaf 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.TimeZone; @@ -54,6 +55,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; +import org.apache.commons.lang3.time.DateUtils; import org.springframework.stereotype.Component; import com.cloud.usage.UsageVO; @@ -145,79 +147,81 @@ protected void processQuotaBalanceForAccount(AccountVO accountVo, List> periods = accountQuotaUsages.stream() + .map(quotaUsageVO -> new Pair<>(quotaUsageVO.getStartDate(), quotaUsageVO.getEndDate())) + .collect(Collectors.toCollection(LinkedHashSet::new)); + + logger.info(String.format("Processing quota balance for account[{}] between [{}] and [{}].", accountToString, startDate, lastQuotaUsageEndDate)); - BigDecimal aggregatedUsage = BigDecimal.ZERO; long accountId = accountVo.getAccountId(); long domainId = accountVo.getDomainId(); + BigDecimal accountBalance = retrieveBalanceForUsageCalculation(accountId, domainId, startDate, accountToString); - aggregatedUsage = getUsageValueAccordingToLastQuotaUsageEntryAndLastQuotaBalance(accountId, domainId, startDate, endDate, aggregatedUsage, accountToString); - - for (QuotaUsageVO quotaUsage : accountQuotaUsages) { - Date quotaUsageStartDate = quotaUsage.getStartDate(); - Date quotaUsageEndDate = quotaUsage.getEndDate(); - BigDecimal quotaUsed = quotaUsage.getQuotaUsed(); - - if (quotaUsed.equals(BigDecimal.ZERO)) { - aggregatedUsage = aggregatedUsage.add(aggregateCreditBetweenDates(accountId, domainId, quotaUsageStartDate, quotaUsageEndDate, accountToString)); - continue; - } + for (Pair period : periods) { + startDate = period.first(); + endDate = period.second(); - if (startDate.compareTo(quotaUsageStartDate) == 0) { - aggregatedUsage = aggregatedUsage.subtract(quotaUsed); - continue; - } - - _quotaBalanceDao.saveQuotaBalance(new QuotaBalanceVO(accountId, domainId, aggregatedUsage, endDate)); - - aggregatedUsage = BigDecimal.ZERO; - startDate = quotaUsageStartDate; - endDate = quotaUsageEndDate; + accountBalance = calculateBalanceConsideringCreditsAddedAndQuotaUsed(accountBalance, accountQuotaUsages, accountId, domainId, startDate, endDate, accountToString); + _quotaBalanceDao.saveQuotaBalance(new QuotaBalanceVO(accountId, domainId, accountBalance, endDate)); + } + saveQuotaAccount(accountId, accountBalance, endDate); + } - QuotaBalanceVO lastRealBalanceEntry = _quotaBalanceDao.findLastBalanceEntry(accountId, domainId, endDate); - Date lastBalanceDate = new Date(0); + /** + * Calculates the balance for the given account considering the specified period. The balance is calculated as follows: + *
    + *
  1. The credits added in this period are added to the balance.
    2. + *
  2. All quota consumed in this period are subtracted from the account balance.
    3. + *
+ */ + protected BigDecimal calculateBalanceConsideringCreditsAddedAndQuotaUsed(BigDecimal accountBalance, List accountQuotaUsages, long accountId, long domainId, + Date startDate, Date endDate, String accountToString) { + accountBalance = accountBalance.add(aggregateCreditBetweenDates(accountId, domainId, startDate, endDate, accountToString)); - if (lastRealBalanceEntry != null) { - lastBalanceDate = lastRealBalanceEntry.getUpdatedOn(); - aggregatedUsage = aggregatedUsage.add(lastRealBalanceEntry.getCreditBalance()); + for (QuotaUsageVO quotaUsageVO : accountQuotaUsages) { + if (DateUtils.isSameInstant(quotaUsageVO.getStartDate(), startDate)) { + accountBalance = accountBalance.subtract(quotaUsageVO.getQuotaUsed()); } - - aggregatedUsage = aggregatedUsage.add(aggregateCreditBetweenDates(accountId, domainId, lastBalanceDate, endDate, accountToString)); - aggregatedUsage = aggregatedUsage.subtract(quotaUsed); } - - _quotaBalanceDao.saveQuotaBalance(new QuotaBalanceVO(accountId, domainId, aggregatedUsage, endDate)); - saveQuotaAccount(accountId, aggregatedUsage, endDate); + return accountBalance; } - protected BigDecimal getUsageValueAccordingToLastQuotaUsageEntryAndLastQuotaBalance(long accountId, long domainId, Date startDate, Date endDate, BigDecimal aggregatedUsage, - String accountToString) { + /** + * Retrieves the initial balance prior to the period of the quota processing. + *
    + *
  • + * If it is the first time of processing for the account, the credits prior to the quota processing are added, and the first balance is persisted in the DB. + *
    • + *
  • + * Otherwise, the last real balance of the account is retrieved. + *
    • + *
+ */ + protected BigDecimal retrieveBalanceForUsageCalculation(long accountId, long domainId, Date startDate, String accountToString) { + BigDecimal accountBalance = BigDecimal.ZERO; QuotaUsageVO lastQuotaUsage = _quotaUsageDao.findLastQuotaUsageEntry(accountId, domainId, startDate); if (lastQuotaUsage == null) { - aggregatedUsage = aggregatedUsage.add(aggregateCreditBetweenDates(accountId, domainId, new Date(0), startDate, accountToString)); - QuotaBalanceVO firstBalance = new QuotaBalanceVO(accountId, domainId, aggregatedUsage, startDate); + accountBalance = accountBalance.add(aggregateCreditBetweenDates(accountId, domainId, new Date(0), startDate, accountToString)); + QuotaBalanceVO firstBalance = new QuotaBalanceVO(accountId, domainId, accountBalance, startDate); logger.debug(String.format("Persisting the first quota balance [%s] for account [%s].", firstBalance, accountToString)); _quotaBalanceDao.saveQuotaBalance(firstBalance); } else { - QuotaBalanceVO lastRealBalance = _quotaBalanceDao.findLastBalanceEntry(accountId, domainId, endDate); + QuotaBalanceVO lastRealBalance = _quotaBalanceDao.findLastBalanceEntry(accountId, domainId, startDate); - if (lastRealBalance != null) { - aggregatedUsage = aggregatedUsage.add(lastRealBalance.getCreditBalance()); - aggregatedUsage = aggregatedUsage.add(aggregateCreditBetweenDates(accountId, domainId, lastRealBalance.getUpdatedOn(), endDate, accountToString)); + if (lastRealBalance == null) { + logger.warn("Account [{}] has quota usage entries, however it does not have a quota balance.", accountToString); } else { - logger.warn(String.format("Account [%s] has quota usage entries, however it does not have a quota balance.", accountToString)); + accountBalance = accountBalance.add(lastRealBalance.getCreditBalance()); } } - return aggregatedUsage; + return accountBalance; } protected void saveQuotaAccount(long accountId, BigDecimal aggregatedUsage, Date endDate) { diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Account.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Account.java index c0b1f762f70e..37c90ab0bcd9 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Account.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Account.java @@ -17,7 +17,9 @@ package org.apache.cloudstack.quota.activationrule.presetvariables; -public class Account extends GenericPresetVariable{ +public class Account extends GenericPresetVariable { + @PresetVariableDefinition(description = "Role of the account. This field will not exist if the account is a project.") + private Role role; public Role getRole() { diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/BackupOffering.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/BackupOffering.java index 457e71a141f2..d8457d294ec3 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/BackupOffering.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/BackupOffering.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.quota.activationrule.presetvariables; public class BackupOffering extends GenericPresetVariable { + @PresetVariableDefinition(description = "External ID of the backup offering that generated the backup.") private String externalId; public String getExternalId() { diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ComputeOffering.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ComputeOffering.java index b42c32a584e1..1d294276d470 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ComputeOffering.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ComputeOffering.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.quota.activationrule.presetvariables; public class ComputeOffering extends GenericPresetVariable { + @PresetVariableDefinition(description = "A boolean informing if the compute offering is customized or not.") private boolean customized; public boolean isCustomized() { diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ComputingResources.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ComputingResources.java index d4f335b081c9..9c86d2d6e0c9 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ComputingResources.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ComputingResources.java @@ -21,8 +21,13 @@ import org.apache.commons.lang3.builder.ToStringStyle; public class ComputingResources { + @PresetVariableDefinition(description = "Current VM's memory (in MiB).") private Integer memory; + + @PresetVariableDefinition(description = "Current VM's vCPUs.") private Integer cpuNumber; + + @PresetVariableDefinition(description = "Current VM's CPU speed (in MHz).") private Integer cpuSpeed; public Integer getMemory() { diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Domain.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Domain.java index 01b702feb1a9..6d83da4cd8fb 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Domain.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Domain.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.quota.activationrule.presetvariables; public class Domain extends GenericPresetVariable { + @PresetVariableDefinition(description = "Path of the domain owner of the resource.") private String path; public String getPath() { diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/GenericPresetVariable.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/GenericPresetVariable.java index b081e57611f0..f59f23abdc15 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/GenericPresetVariable.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/GenericPresetVariable.java @@ -23,8 +23,12 @@ import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; public class GenericPresetVariable { + @PresetVariableDefinition(description = "ID of the resource.") private String id; + + @PresetVariableDefinition(description = "Name of the resource.") private String name; + protected transient Set fieldNamesToIncludeInToString = new HashSet<>(); public String getId() { diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java index fef3e4376dcb..4a0fd2f5a078 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java @@ -20,8 +20,10 @@ import java.util.List; public class Host extends GenericPresetVariable { + @PresetVariableDefinition(description = "List of tags of the host where the VM is running (i.e.: [\"a\", \"b\"]).") private List tags; + @PresetVariableDefinition(description = "Whether the tag is a rule interpreted in JavaScript.") private Boolean isTagARule; public List getTags() { diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableDefinition.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableDefinition.java new file mode 100644 index 000000000000..0e10a8af9d12 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableDefinition.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.quota.activationrule.presetvariables; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +/** + * Describes the preset variable and indicates to which Quota usage types it is loaded. + */ +@Target(FIELD) +@Retention(RUNTIME) +public @interface PresetVariableDefinition { + /** + * An array indicating for which Quota usage types the preset variable is loaded. + * @return an array with the usage types for which the preset variable is loaded. + */ + int[] supportedTypes() default 0; + + /** + * A {@link String} describing the preset variable. + * @return the description of the preset variable. + */ + String description() default ""; +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariables.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariables.java index 2fb6e1ac131d..b27bf589c167 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariables.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariables.java @@ -19,11 +19,22 @@ public class PresetVariables { + @PresetVariableDefinition(description = "Account owner of the resource.") private Account account; + + @PresetVariableDefinition(description = "Domain owner of the resource.") private Domain domain; + + @PresetVariableDefinition(description = "Project owner of the resource. This field will not exist if the resource belongs to an account.") private GenericPresetVariable project; + + @PresetVariableDefinition(description = "Type of the record used. Examples for this are: VirtualMachine, DomainRouter, SourceNat, KVM.") private String resourceType; + + @PresetVariableDefinition(description = "Data related to the resource being processed.") private Value value; + + @PresetVariableDefinition(description = "Zone where the resource is.") private GenericPresetVariable zone; public Account getAccount() { diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Role.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Role.java index fc4716fc309e..3f953b3a4ff8 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Role.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Role.java @@ -20,6 +20,7 @@ import org.apache.cloudstack.acl.RoleType; public class Role extends GenericPresetVariable { + @PresetVariableDefinition(description = "Role type of the resource's owner.") private RoleType type; public RoleType getType() { diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java index 6be1dfb025a2..9b6cfb310922 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java @@ -22,9 +22,13 @@ import com.cloud.storage.ScopeType; public class Storage extends GenericPresetVariable { + @PresetVariableDefinition(description = "List of string representing the tags of the storage where the volume is (i.e.: [\"a\", \"b\"]).") private List tags; + @PresetVariableDefinition(description = "Whether the tag is a rule interpreted in JavaScript. Applicable only for primary storages.") private Boolean isTagARule; + + @PresetVariableDefinition(description = "Scope of the storage where the volume is. Values can be: ZONE, CLUSTER or HOST. Applicable only for primary storages.") private ScopeType scope; public List getTags() { diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java index a1dc7b3c1bb0..d87146d8798e 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java @@ -23,25 +23,75 @@ import com.cloud.storage.Snapshot; import com.cloud.storage.Storage.ProvisioningType; import com.cloud.vm.snapshot.VMSnapshot; +import org.apache.cloudstack.quota.constant.QuotaTypes; public class Value extends GenericPresetVariable { + + @PresetVariableDefinition(description = "ID of the resource.", supportedTypes = {QuotaTypes.ALLOCATED_VM, QuotaTypes.RUNNING_VM, QuotaTypes.VOLUME, QuotaTypes.TEMPLATE, + QuotaTypes.ISO, QuotaTypes.SNAPSHOT, QuotaTypes.NETWORK_OFFERING, QuotaTypes.VM_SNAPSHOT}) + private String id; + + @PresetVariableDefinition(description = "Name of the resource.", supportedTypes = {QuotaTypes.ALLOCATED_VM, QuotaTypes.RUNNING_VM, QuotaTypes.VOLUME, QuotaTypes.TEMPLATE, + QuotaTypes.ISO, QuotaTypes.SNAPSHOT, QuotaTypes.NETWORK_OFFERING, QuotaTypes.VM_SNAPSHOT}) + private String name; + + @PresetVariableDefinition(description = "Host where the VM is running.", supportedTypes = {QuotaTypes.RUNNING_VM}) private Host host; + + @PresetVariableDefinition(description = "OS of the VM/template.", supportedTypes = {QuotaTypes.RUNNING_VM, QuotaTypes.ALLOCATED_VM, QuotaTypes.TEMPLATE, QuotaTypes.ISO}) private String osName; + + @PresetVariableDefinition(description = "A list of resources of the account between the start and end date of the usage record being calculated " + + "(i.e.: [{zoneId: ..., domainId:...}]).") private List accountResources; + + @PresetVariableDefinition(supportedTypes = {QuotaTypes.ALLOCATED_VM, QuotaTypes.RUNNING_VM, QuotaTypes.VOLUME, QuotaTypes.TEMPLATE, QuotaTypes.ISO, QuotaTypes.SNAPSHOT, + QuotaTypes.VM_SNAPSHOT}, description = "List of tags of the resource in the format key:value (i.e.: {\"a\":\"b\", \"c\":\"d\"}).") private Map tags; + + @PresetVariableDefinition(description = "Tag of the network offering.", supportedTypes = {QuotaTypes.NETWORK_OFFERING}) private String tag; + + @PresetVariableDefinition(description = "Size of the resource (in MiB).", supportedTypes = {QuotaTypes.TEMPLATE, QuotaTypes.ISO, QuotaTypes.VOLUME, QuotaTypes.SNAPSHOT, + QuotaTypes.BACKUP}) private Long size; + + @PresetVariableDefinition(description = "Virtual size of the backup.", supportedTypes = {QuotaTypes.BACKUP}) private Long virtualSize; + + @PresetVariableDefinition(description = "Provisioning type of the resource. Values can be: thin, sparse or fat.", supportedTypes = {QuotaTypes.VOLUME}) private ProvisioningType provisioningType; + + @PresetVariableDefinition(description = "Type of the snapshot. Values can be: MANUAL, RECURRING, HOURLY, DAILY, WEEKLY and MONTHLY.", supportedTypes = {QuotaTypes.SNAPSHOT}) private Snapshot.Type snapshotType; + + @PresetVariableDefinition(description = "Type of the VM snapshot. Values can be: Disk or DiskAndMemory.", supportedTypes = {QuotaTypes.VM_SNAPSHOT}) private VMSnapshot.Type vmSnapshotType; + + @PresetVariableDefinition(description = "Computing offering of the VM.", supportedTypes = {QuotaTypes.RUNNING_VM, QuotaTypes.ALLOCATED_VM}) private ComputeOffering computeOffering; + + @PresetVariableDefinition(description = "Template/ISO with which the VM was created.", supportedTypes = {QuotaTypes.RUNNING_VM, QuotaTypes.ALLOCATED_VM}) private GenericPresetVariable template; + + @PresetVariableDefinition(description = "Disk offering of the volume.", supportedTypes = {QuotaTypes.VOLUME}) private GenericPresetVariable diskOffering; + + @PresetVariableDefinition(description = "Storage where the volume or snapshot is. While handling with snapshots, this value can be from the primary storage if the global " + + "setting 'snapshot.backup.to.secondary' is false, otherwise it will be from secondary storage.", supportedTypes = {QuotaTypes.VOLUME, QuotaTypes.SNAPSHOT}) private Storage storage; + + @PresetVariableDefinition(description = "Computing resources consumed by the VM.", supportedTypes = {QuotaTypes.RUNNING_VM}) private ComputingResources computingResources; + + @PresetVariableDefinition(description = "Backup offering of the backup.", supportedTypes = {QuotaTypes.BACKUP}) private BackupOffering backupOffering; + + @PresetVariableDefinition(description = "The hypervisor where the resource was deployed. Values can be: XenServer, KVM, VMware, Hyperv, BareMetal, Ovm, Ovm3 and LXC.", + supportedTypes = {QuotaTypes.RUNNING_VM, QuotaTypes.ALLOCATED_VM, QuotaTypes.VM_SNAPSHOT, QuotaTypes.SNAPSHOT}) private String hypervisorType; + + @PresetVariableDefinition(description = "The volume format. Values can be: RAW, VHD, VHDX, OVA and QCOW2.", supportedTypes = {QuotaTypes.VOLUME, QuotaTypes.VOLUME_SECONDARY}) private String volumeFormat; private String state; diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaConfig.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaConfig.java index df7ffa5c3cdf..81b4643eb450 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaConfig.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaConfig.java @@ -48,16 +48,16 @@ public interface QuotaConfig { public static final ConfigKey QuotaSmtpPort = new ConfigKey("Advanced", String.class, "quota.usage.smtp.port", "", "Quota SMTP port.", true); - public static final ConfigKey QuotaSmtpAuthType = new ConfigKey("Advanced", String.class, "quota.usage.smtp.useAuth", "", + public static final ConfigKey QuotaSmtpAuthType = new ConfigKey("Advanced", Boolean.class, "quota.usage.smtp.useAuth", "false", "If true, use secure SMTP authentication when sending emails.", true); public static final ConfigKey QuotaSmtpSender = new ConfigKey("Advanced", String.class, "quota.usage.smtp.sender", "", "Sender of quota alert email (will be in the From header of the email).", true); public static final ConfigKey QuotaSmtpEnabledSecurityProtocols = new ConfigKey("Advanced", String.class, "quota.usage.smtp.enabledSecurityProtocols", "", - "White-space separated security protocols; ex: \"TLSv1 TLSv1.1\". Supported protocols: SSLv2Hello, SSLv3, TLSv1, TLSv1.1 and TLSv1.2.", true); + "White-space separated security protocols; ex: \"TLSv1 TLSv1.1\". Supported protocols: SSLv2Hello, SSLv3, TLSv1, TLSv1.1 and TLSv1.2.", true, ConfigKey.Kind.WhitespaceSeparatedListWithOptions, "SSLv2Hello,SSLv3,TLSv1,TLSv1.1,TLSv1.2"); - public static final ConfigKey QuotaSmtpUseStartTLS = new ConfigKey("Advanced", String.class, "quota.usage.smtp.useStartTLS", "false", + public static final ConfigKey QuotaSmtpUseStartTLS = new ConfigKey("Advanced", Boolean.class, "quota.usage.smtp.useStartTLS", "false", "If set to true and if we enable security via quota.usage.smtp.useAuth, this will enable StartTLS to secure the connection.", true); public static final ConfigKey QuotaActivationRuleTimeout = new ConfigKey<>("Advanced", Long.class, "quota.activationrule.timeout", "2000", "The maximum runtime," diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java index 3ed162b2ba1e..947183577a88 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java @@ -22,6 +22,7 @@ import org.apache.cloudstack.usage.UsageTypes; import org.apache.cloudstack.usage.UsageUnitTypes; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; public class QuotaTypes extends UsageTypes { private final Integer quotaType; @@ -100,4 +101,13 @@ static public String getDescription(int quotaType) { } return null; } + + static public QuotaTypes getQuotaType(int quotaType) { + return quotaTypeMap.get(quotaType); + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "quotaType", "quotaName"); + } } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/ValueTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/ValueTest.java index 4d0162b33c98..bad33da88367 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/ValueTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/ValueTest.java @@ -25,6 +25,20 @@ @RunWith(MockitoJUnitRunner.class) public class ValueTest { + @Test + public void setIdTestAddFieldIdToCollection() { + Value variable = new Value(); + variable.setId(null); + Assert.assertTrue(variable.fieldNamesToIncludeInToString.contains("id")); + } + + @Test + public void setNameTestAddFieldNameToCollection() { + Value variable = new Value(); + variable.setName(null); + Assert.assertTrue(variable.fieldNamesToIncludeInToString.contains("name")); + } + @Test public void setHostTestAddFieldHostToCollection() { Value variable = new Value(); diff --git a/framework/spring/module/src/main/java/org/apache/cloudstack/spring/module/model/impl/DefaultModuleDefinitionSet.java b/framework/spring/module/src/main/java/org/apache/cloudstack/spring/module/model/impl/DefaultModuleDefinitionSet.java index d61e26fc3a8c..2a6d0b63e5c2 100644 --- a/framework/spring/module/src/main/java/org/apache/cloudstack/spring/module/model/impl/DefaultModuleDefinitionSet.java +++ b/framework/spring/module/src/main/java/org/apache/cloudstack/spring/module/model/impl/DefaultModuleDefinitionSet.java @@ -102,7 +102,9 @@ public void with(ModuleDefinition def, Stack parents) { logger.debug(String.format("Trying to obtain module [%s] context.", moduleDefinitionName)); ApplicationContext context = getApplicationContext(moduleDefinitionName); try { - if (context.containsBean("moduleStartup")) { + if (context == null) { + logger.warn(String.format("Application context not found for module definition [%s]", moduleDefinitionName)); + } else if (context.containsBean("moduleStartup")) { Runnable runnable = context.getBean("moduleStartup", Runnable.class); logger.info(String.format("Starting module [%s].", moduleDefinitionName)); runnable.run(); diff --git a/packaging/centos7/cloud.spec b/packaging/centos7/cloud.spec index 99ecca784add..80b8443e09b9 100644 --- a/packaging/centos7/cloud.spec +++ b/packaging/centos7/cloud.spec @@ -314,6 +314,7 @@ install -D packaging/systemd/cloudstack-management.default ${RPM_BUILD_ROOT}%{_s install -D server/target/conf/cloudstack-sudoers ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid #install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina +install -D server/target/conf/cloudstack-management.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-management # SystemVM template mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm @@ -378,6 +379,7 @@ cp client/target/lib/mysql*jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib/ install -D packaging/systemd/cloudstack-usage.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-usage.service install -D packaging/systemd/cloudstack-usage.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-usage mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/usage/ +install -D usage/target/transformed/cloudstack-usage.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-usage # CLI cp -r cloud-cli/cloudtool ${RPM_BUILD_ROOT}%{python_sitearch}/ @@ -623,6 +625,7 @@ pip3 install --upgrade urllib3 %{_defaultdocdir}/%{name}-management-%{version}/LICENSE %{_defaultdocdir}/%{name}-management-%{version}/NOTICE #%attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-catalina +%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-management %{_datadir}/%{name}-management/setup/wheel/*.whl %files agent @@ -670,6 +673,7 @@ pip3 install --upgrade urllib3 %files usage %attr(0644,root,root) %{_unitdir}/%{name}-usage.service %config(noreplace) %{_sysconfdir}/default/%{name}-usage +%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-usage %attr(0644,root,root) %{_datadir}/%{name}-usage/*.jar %attr(0644,root,root) %{_datadir}/%{name}-usage/lib/*.jar %dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/usage diff --git a/packaging/centos7/replace.properties b/packaging/centos7/replace.properties index 8c3560dd4622..18b4a770a881 100644 --- a/packaging/centos7/replace.properties +++ b/packaging/centos7/replace.properties @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/packaging/centos8/cloud.spec b/packaging/centos8/cloud.spec index 37fe007e3fb2..ebb7af125830 100644 --- a/packaging/centos8/cloud.spec +++ b/packaging/centos8/cloud.spec @@ -296,6 +296,7 @@ install -D packaging/systemd/cloudstack-management.default ${RPM_BUILD_ROOT}%{_s install -D server/target/conf/cloudstack-sudoers ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid #install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina +install -D server/target/conf/cloudstack-management.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-management # SystemVM template mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm @@ -360,6 +361,7 @@ cp client/target/lib/mysql*jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib/ install -D packaging/systemd/cloudstack-usage.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-usage.service install -D packaging/systemd/cloudstack-usage.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-usage mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/usage/ +install -D usage/target/transformed/cloudstack-usage.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-usage # Marvin mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-marvin @@ -577,6 +579,7 @@ pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz %config(noreplace) %{_sysconfdir}/%{name}/management/log4j2.xml %config(noreplace) %{_sysconfdir}/%{name}/management/environment.properties %config(noreplace) %{_sysconfdir}/%{name}/management/java.security.ciphers +%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-management %attr(0644,root,root) %{_unitdir}/%{name}-management.service %attr(0755,cloud,cloud) %{_localstatedir}/run/%{name}-management.pid %attr(0755,root,root) %{_bindir}/%{name}-setup-management @@ -648,6 +651,7 @@ pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz %files usage %attr(0644,root,root) %{_unitdir}/%{name}-usage.service %config(noreplace) %{_sysconfdir}/default/%{name}-usage +%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-usage %attr(0644,root,root) %{_datadir}/%{name}-usage/*.jar %attr(0644,root,root) %{_datadir}/%{name}-usage/lib/*.jar %dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/usage diff --git a/packaging/centos8/replace.properties b/packaging/centos8/replace.properties index 8f1fb11f9995..efeab01166ea 100644 --- a/packaging/centos8/replace.properties +++ b/packaging/centos8/replace.properties @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/packaging/debian/replace.properties b/packaging/debian/replace.properties index 5007360a2b70..db88310d81cd 100644 --- a/packaging/debian/replace.properties +++ b/packaging/debian/replace.properties @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/packaging/systemd/cloudstack-agent.service b/packaging/systemd/cloudstack-agent.service index e04d672beaaa..5e2e5db0b210 100644 --- a/packaging/systemd/cloudstack-agent.service +++ b/packaging/systemd/cloudstack-agent.service @@ -31,6 +31,8 @@ EnvironmentFile=/etc/default/cloudstack-agent ExecStart=/usr/bin/java $JAVA_OPTS $JAVA_DEBUG -cp $CLASSPATH $JAVA_CLASS Restart=always RestartSec=10s +StandardOutput=append:/var/log/cloudstack/agent/agent.out +StandardError=append:/var/log/cloudstack/agent/agent.err [Install] WantedBy=multi-user.target diff --git a/packaging/systemd/cloudstack-management.default b/packaging/systemd/cloudstack-management.default index ca8ff628fc19..994a1ee86997 100644 --- a/packaging/systemd/cloudstack-management.default +++ b/packaging/systemd/cloudstack-management.default @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -JAVA_OPTS="-Djava.security.properties=/etc/cloudstack/management/java.security.ciphers -Djava.awt.headless=true -Dcom.sun.management.jmxremote=false -Xmx2G -XX:+UseParallelGC -XX:MaxGCPauseMillis=500 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/cloudstack/management/ -XX:ErrorFile=/var/log/cloudstack/management/cloudstack-management.err --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED" +JAVA_OPTS="-Djava.security.properties=/etc/cloudstack/management/java.security.ciphers -Djava.awt.headless=true -Xmx2G -XX:+UseParallelGC -XX:MaxGCPauseMillis=500 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/cloudstack/management/ -XX:ErrorFile=/var/log/cloudstack/management/cloudstack-management.err --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED" CLASSPATH="/usr/share/cloudstack-management/lib/*:/etc/cloudstack/management:/usr/share/cloudstack-common:/usr/share/cloudstack-management/setup:/usr/share/cloudstack-management:/usr/share/java/mysql-connector-java.jar:/usr/share/cloudstack-mysql-ha/lib/*" diff --git a/packaging/systemd/cloudstack-management.service b/packaging/systemd/cloudstack-management.service index b979f7f375a7..55780af7a5c6 100644 --- a/packaging/systemd/cloudstack-management.service +++ b/packaging/systemd/cloudstack-management.service @@ -35,6 +35,8 @@ EnvironmentFile=/etc/default/cloudstack-management WorkingDirectory=/var/log/cloudstack/management PIDFile=/var/run/cloudstack-management.pid ExecStart=/usr/bin/java $JAVA_DEBUG $JAVA_OPTS -cp $CLASSPATH $BOOTSTRAP_CLASS +StandardOutput=append:/var/log/cloudstack/management/management-server.out +StandardError=append:/var/log/cloudstack/management/management-server.err [Install] WantedBy=multi-user.target diff --git a/packaging/systemd/cloudstack-usage.service b/packaging/systemd/cloudstack-usage.service index c23814eaac66..bf5bd2a189b9 100644 --- a/packaging/systemd/cloudstack-usage.service +++ b/packaging/systemd/cloudstack-usage.service @@ -35,6 +35,8 @@ Environment=JAVA_PID=$$ ExecStart=/bin/sh -ec '/usr/bin/java -Dpid=${JAVA_PID} $JAVA_OPTS $JAVA_DEBUG -cp $CLASSPATH $JAVA_CLASS' Restart=always RestartSec=10s +StandardOutput=append:/var/log/cloudstack/usage/usage.out +StandardError=append:/var/log/cloudstack/usage/usage.err [Install] WantedBy=multi-user.target diff --git a/plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java b/plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java index 94b763d013f9..db40b6e68ddd 100644 --- a/plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java +++ b/plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java @@ -120,7 +120,9 @@ public boolean checkAccess(Account account, String commandName) { } if (accountRole.getRoleType() == RoleType.Admin && accountRole.getId() == RoleType.Admin.getId()) { - logger.info(String.format("Account [%s] is Root Admin or Domain Admin, all APIs are allowed.", account)); + if (logger.isTraceEnabled()) { + logger.trace(String.format("Account [%s] is Root Admin or Domain Admin, all APIs are allowed.", account)); + } return true; } diff --git a/plugins/acl/project-role-based/src/main/java/org/apache/cloudstack/acl/ProjectRoleBasedApiAccessChecker.java b/plugins/acl/project-role-based/src/main/java/org/apache/cloudstack/acl/ProjectRoleBasedApiAccessChecker.java index 1e766468ba8f..2e7ae23d6f1b 100644 --- a/plugins/acl/project-role-based/src/main/java/org/apache/cloudstack/acl/ProjectRoleBasedApiAccessChecker.java +++ b/plugins/acl/project-role-based/src/main/java/org/apache/cloudstack/acl/ProjectRoleBasedApiAccessChecker.java @@ -72,7 +72,9 @@ public List getApisAllowedToUser(Role role, User user, List apiN Project project = CallContext.current().getProject(); if (project == null) { - logger.warn(String.format("Project is null, ProjectRoleBasedApiAccessChecker only applies to projects, returning APIs [%s] for user [%s] as allowed.", apiNames, user)); + if (logger.isTraceEnabled()) { + logger.trace(String.format("Project is null, ProjectRoleBasedApiAccessChecker only applies to projects, returning APIs [%s] for user [%s] as allowed.", apiNames, user)); + } return apiNames; } @@ -110,8 +112,10 @@ public boolean checkAccess(User user, String apiCommandName) throws PermissionDe Project project = CallContext.current().getProject(); if (project == null) { - logger.warn(String.format("Project is null, ProjectRoleBasedApiAccessChecker only applies to projects, returning API [%s] for user [%s] as allowed.", apiCommandName, + if (logger.isTraceEnabled()) { + logger.trace(String.format("Project is null, ProjectRoleBasedApiAccessChecker only applies to projects, returning API [%s] for user [%s] as allowed.", apiCommandName, user)); + } return true; } diff --git a/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiDiscoveryResponse.java b/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiDiscoveryResponse.java index dccf5a68e118..81a9df750cbe 100644 --- a/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiDiscoveryResponse.java +++ b/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiDiscoveryResponse.java @@ -16,13 +16,14 @@ // under the License. package org.apache.cloudstack.api.response; -import com.cloud.serializer.Param; -import com.google.gson.annotations.SerializedName; +import java.util.HashSet; +import java.util.Set; + import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; -import java.util.HashSet; -import java.util.Set; +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; @SuppressWarnings("unused") public class ApiDiscoveryResponse extends BaseResponse { @@ -64,6 +65,18 @@ public ApiDiscoveryResponse() { isAsync = false; } + public ApiDiscoveryResponse(ApiDiscoveryResponse another) { + this.name = another.getName(); + this.description = another.getDescription(); + this.since = another.getSince(); + this.isAsync = another.getAsync(); + this.related = another.getRelated(); + this.params = new HashSet<>(another.getParams()); + this.apiResponse = new HashSet<>(another.getApiResponse()); + this.type = another.getType(); + this.setObjectName(another.getObjectName()); + } + public void setName(String name) { this.name = name; } @@ -123,4 +136,8 @@ public void addApiResponse(ApiResponseResponse apiResponse) { public Set getApiResponse() { return apiResponse; } + + public String getType() { + return type; + } } diff --git a/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiParameterResponse.java b/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiParameterResponse.java index 7713f6b5d693..75f0aacd5046 100644 --- a/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiParameterResponse.java +++ b/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiParameterResponse.java @@ -16,12 +16,14 @@ // under the License. package org.apache.cloudstack.api.response; -import com.google.gson.annotations.SerializedName; +import java.util.List; +import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; public class ApiParameterResponse extends BaseResponse { @SerializedName(ApiConstants.NAME) @@ -52,6 +54,8 @@ public class ApiParameterResponse extends BaseResponse { @Param(description = "comma separated related apis to get the parameter") private String related; + private transient List authorizedRoleTypes = null; + public ApiParameterResponse() { } @@ -87,4 +91,11 @@ public void setRelated(String related) { this.related = related; } + public void setAuthorizedRoleTypes(List authorizedRoleTypes) { + this.authorizedRoleTypes = authorizedRoleTypes; + } + + public List getAuthorizedRoleTypes() { + return authorizedRoleTypes; + } } diff --git a/plugins/api/discovery/src/main/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java b/plugins/api/discovery/src/main/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java index 239bc49a65a2..452b95cf2c05 100644 --- a/plugins/api/discovery/src/main/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java +++ b/plugins/api/discovery/src/main/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java @@ -18,8 +18,10 @@ import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -28,21 +30,22 @@ import javax.inject.Inject; import org.apache.cloudstack.acl.APIChecker; +import org.apache.cloudstack.acl.Role; +import org.apache.cloudstack.acl.RoleService; +import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.BaseAsyncCreateCmd; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.acl.Role; -import org.apache.cloudstack.acl.RoleService; -import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.command.user.discovery.ListApisCmd; import org.apache.cloudstack.api.response.ApiDiscoveryResponse; import org.apache.cloudstack.api.response.ApiParameterResponse; import org.apache.cloudstack.api.response.ApiResponseResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.reflections.ReflectionUtils; import org.springframework.stereotype.Component; @@ -215,6 +218,9 @@ private ApiDiscoveryResponse getCmdRequestMap(Class cmdClass, APICommand apiC paramResponse.setSince(parameterAnnotation.since()); } paramResponse.setRelated(parameterAnnotation.entityType()[0].getName()); + if (parameterAnnotation.authorized() != null) { + paramResponse.setAuthorizedRoleTypes(Arrays.asList(parameterAnnotation.authorized())); + } response.addParam(paramResponse); } } @@ -247,6 +253,7 @@ public ListResponse listApis(User user, String name) { if (user == null) return null; + Account account = accountService.getAccount(user.getAccountId()); if (name != null) { if (!s_apiNameDiscoveryResponseMap.containsKey(name)) @@ -260,10 +267,9 @@ public ListResponse listApis(User user, String name) { return null; } } - responseList.add(s_apiNameDiscoveryResponseMap.get(name)); + responseList.add(getApiDiscoveryResponseWithAccessibleParams(name, account)); } else { - Account account = accountService.getAccount(user.getAccountId()); if (account == null) { throw new PermissionDeniedException(String.format("The account with id [%s] for user [%s] is null.", user.getAccountId(), user)); } @@ -284,13 +290,33 @@ public ListResponse listApis(User user, String name) { } for (String apiName: apisAllowed) { - responseList.add(s_apiNameDiscoveryResponseMap.get(apiName)); + responseList.add(getApiDiscoveryResponseWithAccessibleParams(apiName, account)); } } response.setResponses(responseList); return response; } + private static ApiDiscoveryResponse getApiDiscoveryResponseWithAccessibleParams(String name, Account account) { + if (Account.Type.ADMIN.equals(account.getType())) { + return s_apiNameDiscoveryResponseMap.get(name); + } + ApiDiscoveryResponse apiDiscoveryResponse = + new ApiDiscoveryResponse(s_apiNameDiscoveryResponseMap.get(name)); + Iterator iterator = apiDiscoveryResponse.getParams().iterator(); + while (iterator.hasNext()) { + ApiParameterResponse parameterResponse = iterator.next(); + List authorizedRoleTypes = parameterResponse.getAuthorizedRoleTypes(); + RoleType accountRoleType = RoleType.getByAccountType(account.getType()); + if (CollectionUtils.isNotEmpty(parameterResponse.getAuthorizedRoleTypes()) && + accountRoleType != null && + !authorizedRoleTypes.contains(accountRoleType)) { + iterator.remove(); + } + } + return apiDiscoveryResponse; + } + @Override public List> getCommands() { List> cmdList = new ArrayList>(); diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index 8a193c1ce80f..d911736090cb 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -346,7 +346,7 @@ private boolean checkTaskStatus(final HttpResponse response) throws IOException String type = pair.second(); String path = url.replace(apiURI.toString(), ""); if (type.equals("RestoreSession")) { - return checkIfRestoreSessionFinished(type, path); + checkIfRestoreSessionFinished(type, path); } } return true; @@ -362,17 +362,29 @@ private boolean checkTaskStatus(final HttpResponse response) throws IOException return false; } - protected boolean checkIfRestoreSessionFinished(String type, String path) throws IOException { - for (int j = 0; j < this.restoreTimeout; j++) { + + /** + * Checks the status of the restore session. Checked states are "Success" and "Failure".
+ * There is also a timeout defined in the global configuration, backup.plugin.veeam.restore.timeout,
+ * that is used to wait for the restore to complete before throwing a {@link CloudRuntimeException}. + */ + protected void checkIfRestoreSessionFinished(String type, String path) throws IOException { + for (int j = 0; j < restoreTimeout; j++) { HttpResponse relatedResponse = get(path); RestoreSession session = parseRestoreSessionResponse(relatedResponse); if (session.getResult().equals("Success")) { - return true; + return; } + if (session.getResult().equalsIgnoreCase("Failed")) { String sessionUid = session.getUid(); + logger.error(String.format("Failed to restore backup [%s] of VM [%s] due to [%s].", + sessionUid, session.getVmDisplayName(), + getRestoreVmErrorDescription(StringUtils.substringAfterLast(sessionUid, ":")))); throw new CloudRuntimeException(String.format("Restore job [%s] failed.", sessionUid)); } + logger.debug(String.format("Waiting %s seconds, out of a total of %s seconds, for the restore backup process to finish.", j, restoreTimeout)); + try { Thread.sleep(1000); } catch (InterruptedException ignored) { @@ -931,6 +943,29 @@ public Pair restoreVMToDifferentLocation(String restorePointId, return new Pair<>(result.first(), restoreLocation); } + /** + * Tries to retrieve the error's description of the Veeam restore task that resulted in an error. + * @param uid Session uid in Veeam of the restore process; + * @return the description found in Veeam about the cause of error in the restore process. + */ + protected String getRestoreVmErrorDescription(String uid) { + logger.debug(String.format("Trying to find the cause of error in the restore process [%s].", uid)); + List cmds = Arrays.asList( + String.format("$restoreUid = '%s'", uid), + "$restore = Get-VBRRestoreSession -Id $restoreUid", + "if ($restore) {", + "Write-Output $restore.Description", + "} else {", + "Write-Output 'Cannot find restore session with provided uid $restoreUid'", + "}" + ); + Pair result = executePowerShellCommands(cmds); + if (result != null && result.first()) { + return result.second(); + } + return String.format("Failed to get the description of the failed restore session [%s]. Please contact an administrator.", uid); + } + private boolean isLegacyServer() { return this.veeamServerVersion != null && (this.veeamServerVersion > 0 && this.veeamServerVersion < 11); } diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java index b00455968c6e..63d6896bb85c 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java @@ -38,7 +38,7 @@ import org.apache.cloudstack.backup.BackupOffering; import org.apache.cloudstack.backup.veeam.api.RestoreSession; import org.apache.http.HttpResponse; -import org.apache.logging.log4j.core.Logger; +import org.apache.logging.log4j.Logger; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -163,7 +163,7 @@ public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOExcepti Mockito.when(mockClient.get(Mockito.anyString())).thenReturn(httpResponse); Mockito.when(mockClient.parseRestoreSessionResponse(httpResponse)).thenReturn(restoreSession); Mockito.when(restoreSession.getResult()).thenReturn("No Success"); - Mockito.when(mockClient.checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"), Mockito.eq("any"))).thenCallRealMethod(); + Mockito.doCallRealMethod().when(mockClient).checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"), Mockito.eq("any")); mockClient.checkIfRestoreSessionFinished("RestoreTest", "any"); fail(); } catch (Exception e) { @@ -172,6 +172,42 @@ public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOExcepti Mockito.verify(mockClient, times(10)).get(Mockito.anyString()); } + @Test + public void getRestoreVmErrorDescriptionTestFindErrorDescription() { + Pair response = new Pair<>(true, "Example of error description found in Veeam."); + Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); + Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(response); + String result = mockClient.getRestoreVmErrorDescription("uuid"); + Assert.assertEquals("Example of error description found in Veeam.", result); + } + + @Test + public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() { + Pair response = new Pair<>(true, "Cannot find restore session with provided uid uuid"); + Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); + Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(response); + String result = mockClient.getRestoreVmErrorDescription("uuid"); + Assert.assertEquals("Cannot find restore session with provided uid uuid", result); + } + + @Test + public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsNull() { + Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); + Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(null); + String result = mockClient.getRestoreVmErrorDescription("uuid"); + Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result); + } + + @Test + public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsFalse() { + Pair response = new Pair<>(false, null); + Mockito.when(mockClient.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); + Mockito.when(mockClient.executePowerShellCommands(Mockito.any())).thenReturn(response); + String result = mockClient.getRestoreVmErrorDescription("uuid"); + Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result); + } + + private void verifyBackupMetrics(Map metrics) { Assert.assertEquals(2, metrics.size()); diff --git a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java index d7001ce941aa..25c45ed2a102 100644 --- a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java +++ b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java @@ -35,9 +35,11 @@ import java.security.SignatureException; import java.security.UnrecoverableKeyException; import java.security.cert.CertificateException; +import java.security.cert.CertificateParsingException; import java.security.cert.X509Certificate; import java.security.spec.InvalidKeySpecException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Enumeration; import java.util.HashSet; @@ -53,7 +55,6 @@ import javax.net.ssl.TrustManagerFactory; import javax.xml.bind.DatatypeConverter; -import com.cloud.configuration.Config; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.framework.ca.CAProvider; import org.apache.cloudstack.framework.ca.Certificate; @@ -62,6 +63,8 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.utils.security.CertUtils; import org.apache.cloudstack.utils.security.KeyStoreUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.bouncycastle.asn1.pkcs.Attribute; import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers; import org.bouncycastle.asn1.x509.Extension; @@ -75,11 +78,11 @@ import org.bouncycastle.util.io.pem.PemReader; import com.cloud.certificate.dao.CrlDao; +import com.cloud.configuration.Config; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; -import org.apache.commons.lang3.StringUtils; public final class RootCAProvider extends AdapterBase implements CAProvider, Configurable { @@ -130,6 +133,8 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con "true", "When set to true, it will allow expired client certificate during SSL handshake.", true); + private static String managementCertificateCustomSAN; + /////////////////////////////////////////////////////////// /////////////// Root CA Private Methods /////////////////// @@ -371,8 +376,11 @@ private boolean loadManagementKeyStore() { List nicIps = NetUtils.getAllDefaultNicIps(); addConfiguredManagementIp(nicIps); nicIps = new ArrayList<>(new HashSet<>(nicIps)); + List domainNames = new ArrayList<>(); + domainNames.add(NetUtils.getHostName()); + domainNames.add(CAManager.CertManagementCustomSubjectAlternativeName.value()); - final Certificate serverCertificate = issueCertificate(Collections.singletonList(NetUtils.getHostName()), nicIps, getCaValidityDays()); + final Certificate serverCertificate = issueCertificate(domainNames, nicIps, getCaValidityDays()); if (serverCertificate == null || serverCertificate.getPrivateKey() == null) { throw new CloudRuntimeException("Failed to generate management server certificate and load management server keystore"); @@ -431,6 +439,7 @@ private boolean setupCA() { @Override public boolean start() { + managementCertificateCustomSAN = CAManager.CertManagementCustomSubjectAlternativeName.value(); return loadRootCAKeyPair() && loadRootCAKeyPair() && loadManagementKeyStore(); } @@ -485,4 +494,26 @@ public String getProviderName() { public String getDescription() { return "CloudStack's Root CA provider plugin"; } + + @Override + public boolean isManagementCertificate(java.security.cert.Certificate certificate) throws CertificateParsingException { + if (!(certificate instanceof X509Certificate)) { + return false; + } + X509Certificate x509Certificate = (X509Certificate) certificate; + + // Check for alternative names + Collection> altNames = x509Certificate.getSubjectAlternativeNames(); + if (CollectionUtils.isEmpty(altNames)) { + return false; + } + for (List altName : altNames) { + int type = (Integer) altName.get(0); + String name = (String) altName.get(1); + if (type == GeneralName.dNSName && managementCertificateCustomSAN.equals(name)) { + return true; + } + } + return false; + } } diff --git a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java index 15514b91c785..8311f4d45abc 100644 --- a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java +++ b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java @@ -26,8 +26,13 @@ import java.security.NoSuchProviderException; import java.security.SignatureException; import java.security.cert.CertificateException; +import java.security.cert.CertificateParsingException; import java.security.cert.X509Certificate; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.UUID; import javax.net.ssl.SSLEngine; @@ -35,15 +40,16 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.utils.security.CertUtils; import org.apache.cloudstack.utils.security.SSLUtils; +import org.bouncycastle.asn1.x509.GeneralName; import org.joda.time.DateTime; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; - -import org.mockito.junit.MockitoJUnitRunner; import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; @RunWith(MockitoJUnitRunner.class) @@ -150,4 +156,56 @@ public void testGetProviderName() throws Exception { Assert.assertEquals(provider.getProviderName(), "root"); } + @Test + public void testIsManagementCertificateNotX509() { + try { + Assert.assertFalse(provider.isManagementCertificate(Mockito.mock(java.security.cert.Certificate.class))); + } catch (CertificateParsingException e) { + Assert.fail(String.format("Exception occurred: %s", e.getMessage())); + } + } + + @Test + public void testIsManagementCertificateNoAltNames() { + try { + X509Certificate certificate = Mockito.mock(X509Certificate.class); + Mockito.when(certificate.getSubjectAlternativeNames()).thenReturn(new ArrayList<>()); + Assert.assertFalse(provider.isManagementCertificate(certificate)); + } catch (CertificateParsingException e) { + Assert.fail(String.format("Exception occurred: %s", e.getMessage())); + } + } + + @Test + public void testIsManagementCertificateNoMatch() { + ReflectionTestUtils.setField(provider, "managementCertificateCustomSAN", "cloudstack"); + try { + X509Certificate certificate = Mockito.mock(X509Certificate.class); + List> altNames = new ArrayList<>(); + altNames.add(List.of(GeneralName.dNSName, UUID.randomUUID().toString())); + altNames.add(List.of(GeneralName.dNSName, UUID.randomUUID().toString())); + Collection> collection = new ArrayList<>(altNames); + Mockito.when(certificate.getSubjectAlternativeNames()).thenReturn(collection); + Assert.assertFalse(provider.isManagementCertificate(certificate)); + } catch (CertificateParsingException e) { + Assert.fail(String.format("Exception occurred: %s", e.getMessage())); + } + } + + @Test + public void testIsManagementCertificateMatch() { + String customSAN = "cloudstack"; + ReflectionTestUtils.setField(provider, "managementCertificateCustomSAN", customSAN); + try { + X509Certificate certificate = Mockito.mock(X509Certificate.class); + List> altNames = new ArrayList<>(); + altNames.add(List.of(GeneralName.dNSName, customSAN)); + altNames.add(List.of(GeneralName.dNSName, UUID.randomUUID().toString())); + Collection> collection = new ArrayList<>(altNames); + Mockito.when(certificate.getSubjectAlternativeNames()).thenReturn(collection); + Assert.assertTrue(provider.isManagementCertificate(certificate)); + } catch (CertificateParsingException e) { + Assert.fail(String.format("Exception occurred: %s", e.getMessage())); + } + } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java index 53d82fae604c..218e3c2b2f91 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java @@ -43,10 +43,12 @@ public class QuotaBalanceCmd extends BaseCmd { @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "If domain Id is given and the caller is domain admin then the statement is generated for domain.") private Long domainId; - @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "End date range for quota query. Use yyyy-MM-dd as the date format, e.g. startDate=2009-06-03.") + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "End of the period of the Quota balance." + + ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) private Date endDate; - @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "Start date range quota query. Use yyyy-MM-dd as the date format, e.g. startDate=2009-06-01.") + @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "Start of the period of the Quota balance. " + + ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified account") diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaPresetVariablesListCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaPresetVariablesListCmd.java new file mode 100644 index 000000000000..8de16dd2741e --- /dev/null +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaPresetVariablesListCmd.java @@ -0,0 +1,66 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.command; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.user.Account; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.QuotaPresetVariablesItemResponse; +import org.apache.cloudstack.api.response.QuotaResponseBuilder; +import org.apache.cloudstack.quota.constant.QuotaTypes; + +import javax.inject.Inject; +import java.util.List; + +@APICommand(name = "quotaPresetVariablesList", responseObject = QuotaPresetVariablesItemResponse.class, description = "List the preset variables available for using in the " + + "Quota tariff activation rules given the usage type.", since = "4.20", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class QuotaPresetVariablesListCmd extends BaseCmd { + + @Inject + QuotaResponseBuilder quotaResponseBuilder; + + @Parameter(name = ApiConstants.USAGE_TYPE, type = CommandType.INTEGER, required = true, description = "The usage type for which the preset variables will be retrieved.") + private Integer quotaType; + + @Override + public void execute() { + List responses = quotaResponseBuilder.listQuotaPresetVariables(this); + ListResponse listResponse = new ListResponse<>(); + listResponse.setResponses(responses); + listResponse.setResponseName(getCommandName()); + setResponseObject(listResponse); + } + + public QuotaTypes getQuotaType() { + QuotaTypes quotaTypes = QuotaTypes.getQuotaType(quotaType); + + if (quotaTypes == null) { + throw new InvalidParameterValueException(String.format("Usage type not found for value [%s].", quotaType)); + } + + return quotaTypes; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java index cc02ed31d2de..4fb33f79672e 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java @@ -45,10 +45,12 @@ public class QuotaStatementCmd extends BaseCmd { @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "Optional, If domain Id is given and the caller is domain admin then the statement is generated for domain.") private Long domainId; - @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, required = true, description = "End date range for quota query. Use yyyy-MM-dd as the date format, e.g. startDate=2009-06-03.") + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, required = true, description = "End of the period of the Quota statement. " + + ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) private Date endDate; - @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, required = true, description = "Start date range quota query. Use yyyy-MM-dd as the date format, e.g. startDate=2009-06-01.") + @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, required = true, description = "Start of the period of the Quota statement. " + + ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; @Parameter(name = ApiConstants.TYPE, type = CommandType.INTEGER, description = "List quota usage records for the specified usage type") diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffCreateCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffCreateCmd.java index ef9ffc23d13d..b9406754b31f 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffCreateCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffCreateCmd.java @@ -60,12 +60,12 @@ public class QuotaTariffCreateCmd extends BaseCmd { "value will be applied.", length = 65535) private String activationRule; - @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "The effective start date on/after which the quota tariff is effective. Use yyyy-MM-dd as" - + " the date format, e.g. startDate=2009-06-03. Inform null to use the current date.") + @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "The effective start date on/after which the quota tariff is effective. Inform null to " + + "use the current date. " + ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; - @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "The end date of the quota tariff. Use yyyy-MM-dd as the date format, e.g." - + " endDate=2009-06-03.") + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "The end date of the quota tariff. If not informed, the tariff will be valid indefinitely. " + + ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) private Date endDate; @Override diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffListCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffListCmd.java index c47fdbfa1d87..b4e8c868e40d 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffListCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffListCmd.java @@ -44,18 +44,18 @@ public class QuotaTariffListCmd extends BaseListCmd { @Parameter(name = ApiConstants.USAGE_TYPE, type = CommandType.INTEGER, description = "Usage type of the resource") private Integer usageType; - @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "The start date of the quota tariff. Use yyyy-MM-dd as the date format, " - + "e.g. startDate=2009-06-03.") + @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "The start date of the quota tariff. " + + ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date effectiveDate; - @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "The end date of the quota tariff. Use yyyy-MM-dd as the date format, e.g. " - + "endDate=2021-11-03.", since = "4.18.0.0") + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "The end date of the quota tariff. " + + ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS, since = "4.18.0.0") private Date endDate; @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "The name of the quota tariff.", since = "4.18.0.0") private String name; - @Parameter(name = ApiConstants.LIST_ALL, type = CommandType.BOOLEAN, description = "False will list only not removed quota tariffs. If set to True, we will " + @Parameter(name = ApiConstants.LIST_ALL, type = CommandType.BOOLEAN, description = "False will list only not removed quota tariffs. If set to true, we will " + "list all, including the removed ones. The default is false.", since = "4.18.0.0") private boolean listAll = false; diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffUpdateCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffUpdateCmd.java index 175500604d69..4fc1f08da881 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffUpdateCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffUpdateCmd.java @@ -52,8 +52,8 @@ public class QuotaTariffUpdateCmd extends BaseCmd { "Use yyyy-MM-dd as the date format, e.g. startDate=2009-06-03.") private Date startDate; - @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "The end date of the quota tariff. Use yyyy-MM-dd as the date format, e.g." - + " endDate=2009-06-03.", since = "4.18.0.0") + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "The end date of the quota tariff. " + + ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS, since = "4.18.0.0") private Date endDate; @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "Quota tariff's name", length = 65535, since = "4.18.0.0") diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaPresetVariablesItemResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaPresetVariablesItemResponse.java new file mode 100644 index 000000000000..a1b80fd94ebf --- /dev/null +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaPresetVariablesItemResponse.java @@ -0,0 +1,47 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.BaseResponse; + +public class QuotaPresetVariablesItemResponse extends BaseResponse { + @SerializedName("variable") + @Param(description = "variable") + private String variable; + + @SerializedName("description") + @Param(description = "description") + private String description; + + public QuotaPresetVariablesItemResponse() { + super("variables"); + } + + public void setVariable(String variable) { + this.variable = variable; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java index 57aa04e00faa..ecbb809b60bd 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java @@ -20,6 +20,7 @@ import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; +import org.apache.cloudstack.api.command.QuotaPresetVariablesListCmd; import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; import org.apache.cloudstack.api.command.QuotaTariffListCmd; @@ -72,6 +73,13 @@ public interface QuotaResponseBuilder { boolean deleteQuotaTariff(String quotaTariffUuid); + /** + * Lists the preset variables for the usage type informed in the command. + * @param cmd used to retrieve the Quota usage type parameter. + * @return the response consisting of a {@link List} of the preset variables and their descriptions. + */ + List listQuotaPresetVariables(QuotaPresetVariablesListCmd cmd); + Pair configureQuotaEmail(QuotaConfigureEmailCmd cmd); QuotaConfigureEmailResponse createQuotaConfigureEmailResponse(QuotaEmailConfigurationVO quotaEmailConfigurationVO, Double minBalance, long accountId); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index 94897b410f42..7b5667ac13db 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -16,11 +16,15 @@ //under the License. package org.apache.cloudstack.api.response; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.lang.reflect.ParameterizedType; import java.math.BigDecimal; import java.math.RoundingMode; import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; +import java.util.Arrays; import java.util.Calendar; import java.util.Collection; import java.util.Collections; @@ -31,6 +35,7 @@ import java.util.List; import java.util.ListIterator; import java.util.function.Consumer; +import java.util.stream.Collectors; import javax.inject.Inject; @@ -41,6 +46,7 @@ import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; +import org.apache.cloudstack.api.command.QuotaPresetVariablesListCmd; import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; import org.apache.cloudstack.api.command.QuotaTariffListCmd; @@ -50,6 +56,11 @@ import org.apache.cloudstack.quota.QuotaManagerImpl; import org.apache.cloudstack.quota.QuotaService; import org.apache.cloudstack.quota.QuotaStatement; +import org.apache.cloudstack.quota.activationrule.presetvariables.ComputingResources; +import org.apache.cloudstack.quota.activationrule.presetvariables.GenericPresetVariable; +import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariableDefinition; +import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariables; +import org.apache.cloudstack.quota.activationrule.presetvariables.Value; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.constant.QuotaTypes; import org.apache.cloudstack.quota.dao.QuotaAccountDao; @@ -67,6 +78,8 @@ import org.apache.cloudstack.quota.vo.QuotaTariffVO; import org.apache.cloudstack.quota.vo.QuotaUsageVO; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.springframework.stereotype.Component; @@ -119,6 +132,8 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @Inject private QuotaEmailConfigurationDao quotaEmailConfigurationDao; + private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class}; + @Override public QuotaTariffResponse createQuotaTariffResponse(QuotaTariffVO tariff) { final QuotaTariffResponse response = new QuotaTariffResponse(); @@ -134,7 +149,7 @@ public QuotaTariffResponse createQuotaTariffResponse(QuotaTariffVO tariff) { response.setName(tariff.getName()); response.setEndDate(tariff.getEndDate()); response.setDescription(tariff.getDescription()); - response.setUuid(tariff.getUuid()); + response.setId(tariff.getUuid()); response.setRemoved(tariff.getRemoved()); return response; } @@ -376,8 +391,8 @@ public int compare(QuotaUsageVO o1, QuotaUsageVO o2) { @Override public Pair, Integer> listQuotaTariffPlans(final QuotaTariffListCmd cmd) { - Date startDate = _quotaService.computeAdjustedTime(cmd.getEffectiveDate()); - Date endDate = _quotaService.computeAdjustedTime(cmd.getEndDate()); + Date startDate = cmd.getEffectiveDate(); + Date endDate = cmd.getEndDate(); Integer usageType = cmd.getUsageType(); String name = cmd.getName(); boolean listAll = cmd.isListAll(); @@ -395,10 +410,10 @@ public Pair, Integer> listQuotaTariffPlans(final QuotaTariff public QuotaTariffVO updateQuotaTariffPlan(QuotaTariffUpdateCmd cmd) { String name = cmd.getName(); Double value = cmd.getValue(); - Date endDate = _quotaService.computeAdjustedTime(cmd.getEndDate()); + Date endDate = cmd.getEndDate(); String description = cmd.getDescription(); String activationRule = cmd.getActivationRule(); - Date now = _quotaService.computeAdjustedTime(new Date()); + Date now = new Date(); warnQuotaTariffUpdateDeprecatedFields(cmd); @@ -488,7 +503,7 @@ protected void validateEndDateOnCreatingNewQuotaTariff(QuotaTariffVO newQuotaTar endDate, startDate)); } - Date now = _quotaService.computeAdjustedTime(new Date()); + Date now = new Date(); if (endDate.compareTo(now) < 0) { throw new InvalidParameterValueException(String.format("The quota tariff's end date [%s] cannot be less than now [%s].", endDate, now)); @@ -499,7 +514,7 @@ protected void validateEndDateOnCreatingNewQuotaTariff(QuotaTariffVO newQuotaTar @Override public QuotaCreditsResponse addQuotaCredits(Long accountId, Long domainId, Double amount, Long updatedBy, Boolean enforce) { - Date despositedOn = _quotaService.computeAdjustedTime(new Date()); + Date despositedOn = new Date(); QuotaBalanceVO qb = _quotaBalanceDao.findLaterBalanceEntry(accountId, domainId, despositedOn); if (qb != null) { @@ -643,8 +658,8 @@ public QuotaTariffVO createQuotaTariff(QuotaTariffCreateCmd cmd) { int usageType = cmd.getUsageType(); Date startDate = cmd.getStartDate(); Date now = new Date(); - startDate = _quotaService.computeAdjustedTime(startDate == null ? now : startDate); - Date endDate = _quotaService.computeAdjustedTime(cmd.getEndDate()); + startDate = startDate == null ? now : startDate; + Date endDate = cmd.getEndDate(); Double value = cmd.getValue(); String description = cmd.getDescription(); String activationRule = cmd.getActivationRule(); @@ -675,13 +690,124 @@ public boolean deleteQuotaTariff(String quotaTariffUuid) { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Quota tariff with the provided UUID does not exist."); } - quotaTariff.setRemoved(_quotaService.computeAdjustedTime(new Date())); - + quotaTariff.setRemoved(new Date()); CallContext.current().setEventResourceId(quotaTariff.getId()); - return _quotaTariffDao.updateQuotaTariff(quotaTariff); } + @Override + public List listQuotaPresetVariables(QuotaPresetVariablesListCmd cmd) { + List response; + List> variables = new ArrayList<>(); + + QuotaTypes quotaType = cmd.getQuotaType(); + addAllPresetVariables(PresetVariables.class, quotaType, variables, null); + response = createQuotaPresetVariablesResponse(variables); + + return response; + } + + /** + * Adds all preset variables for the given quota type. It recursively finds all presets variables for the given {@link Class} and puts it in a {@link List}. Each item in the + * list is a {@link Pair} that consists of the variable name and its description. + * + * @param clazz used to find the non-transient fields. If it is equal to the {@link Value} class, then it only gets the declared fields, otherwise, it gets all fields, + * including its parent's fields. + * @param quotaType used to check if the field supports the quota resource type. It uses the annotation method {@link PresetVariableDefinition#supportedTypes()} for this + * verification. + * @param variables the {@link List} which contains the {@link Pair} of the preset variable and its description. + * @param recursiveVariableName {@link String} used for recursively building the preset variable string. + */ + public void addAllPresetVariables(Class clazz, QuotaTypes quotaType, List> variables, String recursiveVariableName) { + Field[] allFields = Value.class.equals(clazz) ? clazz.getDeclaredFields() : FieldUtils.getAllFields(clazz); + List fieldsNonTransients = Arrays.stream(allFields).filter(field -> !Modifier.isTransient(field.getModifiers())).collect(Collectors.toList()); + for (Field field : fieldsNonTransients) { + PresetVariableDefinition presetVariableDefinitionAnnotation = field.getAnnotation(PresetVariableDefinition.class); + Class fieldClass = getClassOfField(field); + String presetVariableName = field.getName(); + + if (presetVariableDefinitionAnnotation == null) { + continue; + } + + if (StringUtils.isNotEmpty(recursiveVariableName)) { + presetVariableName = String.format("%s.%s", recursiveVariableName, field.getName()); + } + filterSupportedTypes(variables, quotaType, presetVariableDefinitionAnnotation, fieldClass, presetVariableName); + } + } + + /** + * Returns the class of the {@link Field} depending on its type. This method is required for retrieving the Class of Generic Types, i.e. {@link List}. + */ + protected Class getClassOfField(Field field){ + if (field.getGenericType() instanceof ParameterizedType) { + ParameterizedType genericType = (ParameterizedType) field.getGenericType(); + return (Class) genericType.getActualTypeArguments()[0]; + } + + return field.getType(); + } + + /** + * Checks if the {@link PresetVariableDefinition} supports the given {@link QuotaTypes}. If it supports it, it adds the preset variable to the {@link List} recursively + * if it is from the one of the classes in the {@link QuotaResponseBuilderImpl#assignableClasses} array or directly if not. + * + * @param variables {@link List} of the {@link Pair} of the preset variable and its description. + * @param quotaType the given {@link QuotaTypes} to filter. + * @param presetVariableDefinitionAnnotation used to check if the quotaType is supported. + * @param fieldClass class of the field used to verify if it is from the {@link GenericPresetVariable} or {@link ComputingResources} classes. If it is, then it calls + * {@link QuotaResponseBuilderImpl#addAllPresetVariables(Class, QuotaTypes, List, String)} to add the preset variable. Otherwise, the {@link Pair} is + * added directly to the variables {@link List}. + * @param presetVariableName {@link String} that contains the recursive created preset variable name. + */ + public void filterSupportedTypes(List> variables, QuotaTypes quotaType, PresetVariableDefinition presetVariableDefinitionAnnotation, Class fieldClass, + String presetVariableName) { + if (Arrays.stream(presetVariableDefinitionAnnotation.supportedTypes()).noneMatch(supportedType -> + supportedType == quotaType.getQuotaType() || supportedType == 0)) { + return; + } + + String presetVariableDescription = presetVariableDefinitionAnnotation.description(); + + Pair pair = new Pair<>(presetVariableName, presetVariableDescription); + variables.add(pair); + + if (isRecursivePresetVariable(fieldClass)) { + addAllPresetVariables(fieldClass, quotaType, variables, presetVariableName); + } + } + + /** + * Returns true if the {@link Class} of the {@link Field} is from one of the classes in the array {@link QuotaResponseBuilderImpl#assignableClasses}, i.e., it is a recursive + * {@link PresetVariables}, returns false otherwise. + */ + private boolean isRecursivePresetVariable(Class fieldClass) { + for (Class clazz : assignableClasses) { + if (clazz.isAssignableFrom(fieldClass)) { + return true; + } + } + return false; + } + + public List createQuotaPresetVariablesResponse(List> variables) { + final List responses = new ArrayList<>(); + + for (Pair variable : variables) { + responses.add(createPresetVariablesItemResponse(variable)); + } + + return responses; + } + + public QuotaPresetVariablesItemResponse createPresetVariablesItemResponse(Pair variable) { + QuotaPresetVariablesItemResponse response = new QuotaPresetVariablesItemResponse(); + response.setVariable(variable.first()); + response.setDescription(variable.second()); + return response; + } + @Override public Pair configureQuotaEmail(QuotaConfigureEmailCmd cmd) { validateQuotaConfigureEmailCmdParameters(cmd); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaTariffResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaTariffResponse.java index ce4c5953641d..cec3634c76d8 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaTariffResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaTariffResponse.java @@ -19,6 +19,7 @@ import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import java.math.BigDecimal; @@ -74,9 +75,9 @@ public class QuotaTariffResponse extends BaseResponse { @Param(description = "description") private String description; - @SerializedName("uuid") - @Param(description = "uuid") - private String uuid; + @SerializedName(ApiConstants.ID) + @Param(description = "the ID of the tariff") + private String id; @SerializedName("removed") @Param(description = "when the quota tariff was removed") @@ -87,15 +88,6 @@ public QuotaTariffResponse() { this.setObjectName("quotatariff"); } - public QuotaTariffResponse(final int usageType) { - super(); - this.usageType = usageType; - } - - public String getUsageName() { - return usageName; - } - public void setUsageName(String usageName) { this.usageName = usageName; } @@ -108,18 +100,10 @@ public void setUsageType(int usageType) { this.usageType = usageType; } - public String getUsageUnit() { - return usageUnit; - } - public void setUsageUnit(String usageUnit) { this.usageUnit = usageUnit; } - public String getUsageDiscriminator() { - return usageDiscriminator; - } - public void setUsageDiscriminator(String usageDiscriminator) { this.usageDiscriminator = usageDiscriminator; } @@ -132,26 +116,14 @@ public void setTariffValue(BigDecimal tariffValue) { this.tariffValue = tariffValue; } - public String getUsageTypeDescription() { - return usageTypeDescription; - } - public void setUsageTypeDescription(String usageTypeDescription) { this.usageTypeDescription = usageTypeDescription; } - public Date getEffectiveOn() { - return effectiveOn; - } - public void setEffectiveOn(Date effectiveOn) { this.effectiveOn = effectiveOn; } - public String getCurrency() { - return currency; - } - public void setCurrency(String currency) { this.currency = currency; } @@ -188,16 +160,12 @@ public void setDescription(String description) { this.description = description; } - public String getUuid() { - return uuid; - } - - public void setUuid(String uuid) { - this.uuid = uuid; + public String getId() { + return id; } - public Date getRemoved() { - return removed; + public void setId(String id) { + this.id = id; } public void setRemoved(Date removed) { diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java index fe634715d89e..8f3c34982c0b 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java @@ -32,8 +32,6 @@ public interface QuotaService extends PluggableService { List findQuotaBalanceVO(Long accountId, String accountName, Long domainId, Date startDate, Date endDate); - Date computeAdjustedTime(Date date); - void setLockAccount(Long accountId, Boolean state); void setMinBalance(Long accountId, Double balance); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java index da3f50b165a5..17fa7bd8425e 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java @@ -18,7 +18,6 @@ import java.math.BigDecimal; import java.util.ArrayList; -import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.Map; @@ -34,6 +33,7 @@ import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; import org.apache.cloudstack.api.command.QuotaEnabledCmd; import org.apache.cloudstack.api.command.QuotaListEmailConfigurationCmd; +import org.apache.cloudstack.api.command.QuotaPresetVariablesListCmd; import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; @@ -120,6 +120,7 @@ public List> getCommands() { cmdList.add(QuotaTariffDeleteCmd.class); cmdList.add(QuotaConfigureEmailCmd.class); cmdList.add(QuotaListEmailConfigurationCmd.class); + cmdList.add(QuotaPresetVariablesListCmd.class); return cmdList; } @@ -165,36 +166,32 @@ public List findQuotaBalanceVO(Long accountId, String accountNam if (endDate == null) { // adjust start date to end of day as there is no end date - Date adjustedStartDate = computeAdjustedTime(_respBldr.startOfNextDay(startDate)); + startDate = _respBldr.startOfNextDay(startDate); if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance1: Getting quota balance records for account: " + accountId + ", domainId: " + domainId + ", on or before " + adjustedStartDate); + logger.debug("getQuotaBalance1: Getting quota balance records for account: " + accountId + ", domainId: " + domainId + ", on or before " + startDate); } - List qbrecords = _quotaBalanceDao.lastQuotaBalanceVO(accountId, domainId, adjustedStartDate); + List qbrecords = _quotaBalanceDao.lastQuotaBalanceVO(accountId, domainId, startDate); if (logger.isDebugEnabled()) { logger.debug("Found records size=" + qbrecords.size()); } if (qbrecords.isEmpty()) { - logger.info("Incorrect Date there are no quota records before this date " + adjustedStartDate); + logger.info("Incorrect Date there are no quota records before this date " + startDate); return qbrecords; } else { return qbrecords; } } else { - Date adjustedStartDate = computeAdjustedTime(startDate); - if (endDate.after(_respBldr.startOfNextDay())) { - throw new InvalidParameterValueException("Incorrect Date Range. End date:" + endDate + " should not be in future. "); - } else if (startDate.before(endDate)) { - Date adjustedEndDate = computeAdjustedTime(endDate); + if (startDate.before(endDate)) { if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance2: Getting quota balance records for account: " + accountId + ", domainId: " + domainId + ", between " + adjustedStartDate - + " and " + adjustedEndDate); + logger.debug("getQuotaBalance2: Getting quota balance records for account: " + accountId + ", domainId: " + domainId + ", between " + startDate + + " and " + endDate); } - List qbrecords = _quotaBalanceDao.findQuotaBalance(accountId, domainId, adjustedStartDate, adjustedEndDate); + List qbrecords = _quotaBalanceDao.findQuotaBalance(accountId, domainId, startDate, endDate); if (logger.isDebugEnabled()) { logger.debug("getQuotaBalance3: Found records size=" + qbrecords.size()); } if (qbrecords.isEmpty()) { - logger.info("There are no quota records between these dates start date " + adjustedStartDate + " and end date:" + endDate); + logger.info("There are no quota records between these dates start date " + startDate + " and end date:" + endDate); return qbrecords; } else { return qbrecords; @@ -230,44 +227,11 @@ public List getQuotaUsage(Long accountId, String accountName, Long if (startDate.after(endDate)) { throw new InvalidParameterValueException("Incorrect Date Range. Start date: " + startDate + " is after end date:" + endDate); } - if (endDate.after(_respBldr.startOfNextDay())) { - throw new InvalidParameterValueException("Incorrect Date Range. End date:" + endDate + " should not be in future. "); - } - Date adjustedEndDate = computeAdjustedTime(endDate); - Date adjustedStartDate = computeAdjustedTime(startDate); - if (logger.isDebugEnabled()) { - logger.debug("Getting quota records for account: " + accountId + ", domainId: " + domainId + ", between " + adjustedStartDate + " and " + adjustedEndDate); - } - return _quotaUsageDao.findQuotaUsage(accountId, domainId, usageType, adjustedStartDate, adjustedEndDate); - } - - @Override - public Date computeAdjustedTime(final Date date) { - if (date == null) { - return null; - } - - Calendar cal = Calendar.getInstance(); - cal.setTime(date); - TimeZone localTZ = cal.getTimeZone(); - int timezoneOffset = cal.get(Calendar.ZONE_OFFSET); - if (localTZ.inDaylightTime(date)) { - timezoneOffset += (60 * 60 * 1000); - } - cal.add(Calendar.MILLISECOND, timezoneOffset); - - Date newTime = cal.getTime(); - - Calendar calTS = Calendar.getInstance(_usageTimezone); - calTS.setTime(newTime); - timezoneOffset = calTS.get(Calendar.ZONE_OFFSET); - if (_usageTimezone.inDaylightTime(date)) { - timezoneOffset += (60 * 60 * 1000); - } - calTS.add(Calendar.MILLISECOND, -1 * timezoneOffset); + logger.debug("Getting quota records of type [{}] for account [{}] in domain [{}], between [{}] and [{}].", + usageType, accountId, domainId, startDate, endDate); - return calTS.getTime(); + return _quotaUsageDao.findQuotaUsage(accountId, domainId, usageType, startDate, endDate); } @Override diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java index 899ce649fce0..da02b6d37093 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java @@ -29,6 +29,7 @@ import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; +import com.cloud.utils.Pair; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; @@ -36,6 +37,9 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.quota.QuotaService; import org.apache.cloudstack.quota.QuotaStatement; +import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariableDefinition; +import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariables; +import org.apache.cloudstack.quota.activationrule.presetvariables.Value; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.constant.QuotaTypes; import org.apache.cloudstack.quota.dao.QuotaAccountDao; @@ -177,7 +181,6 @@ public void testAddQuotaCredits() { Mockito.when(quotaCreditsDaoMock.saveCredits(Mockito.any(QuotaCreditsVO.class))).thenReturn(credit); Mockito.when(quotaBalanceDaoMock.lastQuotaBalance(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(Date.class))).thenReturn(new BigDecimal(111)); - Mockito.when(quotaServiceMock.computeAdjustedTime(Mockito.any(Date.class))).thenReturn(new Date()); AccountVO account = new AccountVO(); account.setState(Account.State.LOCKED); @@ -245,7 +248,6 @@ public void testCreateQuotaLastBalanceResponse() { entry.setCreditBalance(new BigDecimal(100)); quotaBalance.add(entry); quotaBalance.add(entry); - Mockito.lenient().when(quotaServiceMock.computeAdjustedTime(Mockito.any(Date.class))).thenReturn(new Date()); QuotaBalanceResponse resp = quotaResponseBuilderSpy.createQuotaLastBalanceResponse(quotaBalance, null); assertTrue(resp.getStartQuota().compareTo(new BigDecimal(200)) == 0); } @@ -326,16 +328,14 @@ public void validateEndDateOnCreatingNewQuotaTariffTestEndDateLessThanNowThrowIn Date startDate = DateUtils.addDays(date, -100); Date endDate = DateUtils.addDays(new Date(), -1); - Mockito.doReturn(date).when(quotaServiceMock).computeAdjustedTime(Mockito.any(Date.class)); quotaResponseBuilderSpy.validateEndDateOnCreatingNewQuotaTariff(quotaTariffVoMock, startDate, endDate); } @Test public void validateEndDateOnCreatingNewQuotaTariffTestSetValidEndDate() { Date startDate = DateUtils.addDays(date, -100); - Date endDate = date; + Date endDate = DateUtils.addMinutes(new Date(), 1); - Mockito.doReturn(DateUtils.addDays(date, -10)).when(quotaServiceMock).computeAdjustedTime(Mockito.any(Date.class)); quotaResponseBuilderSpy.validateEndDateOnCreatingNewQuotaTariff(quotaTariffVoMock, startDate, endDate); Mockito.verify(quotaTariffVoMock).setEndDate(Mockito.any(Date.class)); } @@ -387,7 +387,6 @@ public void deleteQuotaTariffTestQuotaDoesNotExistThrowsServerApiException() { public void deleteQuotaTariffTestUpdateRemoved() { Mockito.doReturn(quotaTariffVoMock).when(quotaTariffDaoMock).findByUuid(Mockito.anyString()); Mockito.doReturn(true).when(quotaTariffDaoMock).updateQuotaTariff(Mockito.any(QuotaTariffVO.class)); - Mockito.doReturn(new Date()).when(quotaServiceMock).computeAdjustedTime(Mockito.any(Date.class)); Assert.assertTrue(quotaResponseBuilderSpy.deleteQuotaTariff("")); @@ -424,6 +423,46 @@ public void getQuotaSummaryResponseTestAccountIsNotNullQuotaIsEnabledShouldRetur assertTrue(quotaSummaryResponse.getQuotaEnabled()); } + @Test + public void filterSupportedTypesTestReturnWhenQuotaTypeDoesNotMatch() throws NoSuchFieldException { + List> variables = new ArrayList<>(); + Class clazz = Value.class; + PresetVariableDefinition presetVariableDefinitionAnnotation = clazz.getDeclaredField("host").getAnnotation(PresetVariableDefinition.class); + QuotaTypes quotaType = QuotaTypes.getQuotaType(QuotaTypes.NETWORK_OFFERING); + int expectedVariablesSize = 0; + + quotaResponseBuilderSpy.filterSupportedTypes(variables, quotaType, presetVariableDefinitionAnnotation, clazz, null); + + assertEquals(expectedVariablesSize, variables.size()); + } + + @Test + public void filterSupportedTypesTestAddPresetVariableWhenClassIsNotInstanceOfGenericPresetVariableAndComputingResource() throws NoSuchFieldException { + List> variables = new ArrayList<>(); + Class clazz = PresetVariables.class; + PresetVariableDefinition presetVariableDefinitionAnnotation = clazz.getDeclaredField("resourceType").getAnnotation(PresetVariableDefinition.class); + QuotaTypes quotaType = QuotaTypes.getQuotaType(QuotaTypes.NETWORK_OFFERING); + int expectedVariablesSize = 1; + String expectedVariableName = "variable.name"; + + quotaResponseBuilderSpy.filterSupportedTypes(variables, quotaType, presetVariableDefinitionAnnotation, clazz, "variable.name"); + + assertEquals(expectedVariablesSize, variables.size()); + assertEquals(expectedVariableName, variables.get(0).first()); + } + + @Test + public void filterSupportedTypesTestCallRecursiveMethodWhenIsGenericPresetVariableClassOrComputingResourceClass() throws NoSuchFieldException { + List> variables = new ArrayList<>(); + Class clazz = Value.class; + PresetVariableDefinition presetVariableDefinitionAnnotation = clazz.getDeclaredField("storage").getAnnotation(PresetVariableDefinition.class); + QuotaTypes quotaType = QuotaTypes.getQuotaType(QuotaTypes.VOLUME); + + quotaResponseBuilderSpy.filterSupportedTypes(variables, quotaType, presetVariableDefinitionAnnotation, clazz, "variable.name"); + + Mockito.verify(quotaResponseBuilderSpy, Mockito.atLeastOnce()).addAllPresetVariables(Mockito.any(), Mockito.any(QuotaTypes.class), Mockito.anyList(), + Mockito.anyString()); + } @Test (expected = InvalidParameterValueException.class) public void validateQuotaConfigureEmailCmdParametersTestNullQuotaAccount() { @@ -447,7 +486,6 @@ public void validateQuotaConfigureEmailCmdParametersTestEnableNullAndTemplateNam quotaResponseBuilderSpy.validateQuotaConfigureEmailCmdParameters(quotaConfigureEmailCmdMock); } - @Test public void validateQuotaConfigureEmailCmdParametersTestNullTemplateName() { Mockito.doReturn(quotaAccountVOMock).when(quotaAccountDaoMock).findByIdQuotaAccount(Mockito.any()); diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java index fa58c35ea5d5..19e756d1d973 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java @@ -30,7 +30,6 @@ import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -102,13 +101,6 @@ public void setup() throws IllegalAccessException, NoSuchFieldException, Configu quotaService.configure("randomName", null); } - @Test - public void testComputeAdjustedTime() { - DateTime now = new DateTime(DateTimeZone.UTC); - DateTime result = new DateTime(quotaService.computeAdjustedTime(now.toDate())); - // FIXME: fix this test - } - @Test public void testFindQuotaBalanceVO() { final long accountId = 2L; @@ -123,7 +115,6 @@ public void testFindQuotaBalanceVO() { qb.setAccountId(accountId); records.add(qb); - Mockito.when(respBldr.startOfNextDay()).thenReturn(endDate); Mockito.when(respBldr.startOfNextDay(Mockito.any(Date.class))).thenReturn(startDate); Mockito.when(quotaBalanceDao.findQuotaBalance(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.any(Date.class), Mockito.any(Date.class))).thenReturn(records); Mockito.when(quotaBalanceDao.lastQuotaBalanceVO(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.any(Date.class))).thenReturn(records); @@ -142,7 +133,6 @@ public void testGetQuotaUsage() { final Date startDate = new DateTime().minusDays(2).toDate(); final Date endDate = new Date(); - Mockito.when(respBldr.startOfNextDay()).thenReturn(endDate); quotaService.getQuotaUsage(accountId, accountName, domainId, QuotaTypes.IP_ADDRESS, startDate, endDate); Mockito.verify(quotaUsageDao, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(Date.class), Mockito.any(Date.class)); } diff --git a/plugins/event-bus/inmemory/src/main/java/org/apache/cloudstack/mom/inmemory/InMemoryEventBus.java b/plugins/event-bus/inmemory/src/main/java/org/apache/cloudstack/mom/inmemory/InMemoryEventBus.java index d5d362781922..0c00c0639fd9 100644 --- a/plugins/event-bus/inmemory/src/main/java/org/apache/cloudstack/mom/inmemory/InMemoryEventBus.java +++ b/plugins/event-bus/inmemory/src/main/java/org/apache/cloudstack/mom/inmemory/InMemoryEventBus.java @@ -60,6 +60,8 @@ public UUID subscribe(EventTopic topic, EventSubscriber subscriber) throws Event if (subscriber == null || topic == null) { throw new EventBusException("Invalid EventSubscriber/EventTopic object passed."); } + logger.debug("subscribing '{}' to events of type '{}' from '{}'", subscriber.toString(), topic.getEventType(), topic.getEventSource()); + UUID subscriberId = UUID.randomUUID(); subscribers.put(subscriberId, new Pair(topic, subscriber)); @@ -68,6 +70,7 @@ public UUID subscribe(EventTopic topic, EventSubscriber subscriber) throws Event @Override public void unsubscribe(UUID subscriberId, EventSubscriber subscriber) throws EventBusException { + logger.debug("unsubscribing '{}'", subscriberId); if (subscriberId == null) { throw new EventBusException("Cannot unregister a null subscriberId."); } @@ -85,7 +88,9 @@ public void unsubscribe(UUID subscriberId, EventSubscriber subscriber) throws Ev @Override public void publish(Event event) throws EventBusException { + logger.trace("publish '{}'", event.getDescription()); if (subscribers == null || subscribers.isEmpty()) { + logger.trace("no subscribers, no publish"); return; // no subscriber to publish to, so just return } diff --git a/plugins/event-bus/kafka/src/main/java/org/apache/cloudstack/mom/kafka/KafkaEventBus.java b/plugins/event-bus/kafka/src/main/java/org/apache/cloudstack/mom/kafka/KafkaEventBus.java index 01888779fc6a..f2589d2d7d09 100644 --- a/plugins/event-bus/kafka/src/main/java/org/apache/cloudstack/mom/kafka/KafkaEventBus.java +++ b/plugins/event-bus/kafka/src/main/java/org/apache/cloudstack/mom/kafka/KafkaEventBus.java @@ -87,19 +87,23 @@ public void setName(String name) { @Override public UUID subscribe(EventTopic topic, EventSubscriber subscriber) throws EventBusException { + logger.debug("subscribing '{}' to events of type '{}' from '{}'", subscriber.toString(), topic.getEventType(), topic.getEventSource()); + /* NOOP */ return UUID.randomUUID(); } @Override public void unsubscribe(UUID subscriberId, EventSubscriber subscriber) throws EventBusException { + logger.debug("unsubscribing '{}'", subscriberId); /* NOOP */ } @Override public void publish(Event event) throws EventBusException { - ProducerRecord record = new ProducerRecord(_topic, event.getResourceUUID(), event.getDescription()); - _producer.send(record); + logger.trace("publish '{}'", event.getDescription()); + ProducerRecord newRecord = new ProducerRecord<>(_topic, event.getResourceUUID(), event.getDescription()); + _producer.send(newRecord); } @Override diff --git a/plugins/event-bus/rabbitmq/src/main/java/org/apache/cloudstack/mom/rabbitmq/RabbitMQEventBus.java b/plugins/event-bus/rabbitmq/src/main/java/org/apache/cloudstack/mom/rabbitmq/RabbitMQEventBus.java index 8cd2289f9f31..e8067e75b406 100644 --- a/plugins/event-bus/rabbitmq/src/main/java/org/apache/cloudstack/mom/rabbitmq/RabbitMQEventBus.java +++ b/plugins/event-bus/rabbitmq/src/main/java/org/apache/cloudstack/mom/rabbitmq/RabbitMQEventBus.java @@ -185,11 +185,12 @@ public static void setRetryInterval(Integer retryInterval) { */ @Override public UUID subscribe(EventTopic topic, EventSubscriber subscriber) throws EventBusException { - if (subscriber == null || topic == null) { throw new EventBusException("Invalid EventSubscriber/EventTopic object passed."); } + logger.debug("subscribing '{}' to events of type '{}' from '{}'", subscriber.toString(), topic.getEventType(), topic.getEventSource()); + // create a UUID, that will be used for managing subscriptions and also used as queue name // for on the queue used for the subscriber on the AMQP broker UUID queueId = UUID.randomUUID(); @@ -250,6 +251,7 @@ public void handleDelivery(String queueName, Envelope envelope, AMQP.BasicProper @Override public void unsubscribe(UUID subscriberId, EventSubscriber subscriber) throws EventBusException { + logger.debug("unsubscribing '{}'", subscriberId); try { String classname = subscriber.getClass().getName(); String queueName = UUID.nameUUIDFromBytes(classname.getBytes()).toString(); @@ -265,6 +267,7 @@ public void unsubscribe(UUID subscriberId, EventSubscriber subscriber) throws Ev // publish event on to the exchange created on AMQP server @Override public void publish(Event event) throws EventBusException { + logger.trace("publish '{}'", event.getDescription()); String routingKey = createRoutingKey(event); String eventDescription = event.getDescription(); diff --git a/plugins/event-bus/webhook/pom.xml b/plugins/event-bus/webhook/pom.xml new file mode 100644 index 000000000000..278f4dc0ec59 --- /dev/null +++ b/plugins/event-bus/webhook/pom.xml @@ -0,0 +1,46 @@ + + + 4.0.0 + cloud-mom-webhook + Apache CloudStack Plugin - Webhook Event Bus + + org.apache.cloudstack + cloudstack-plugins + 4.20.0.0-SNAPSHOT + ../../pom.xml + + + + org.apache.cloudstack + cloud-framework-events + ${project.version} + + + org.apache.cloudstack + cloud-engine-api + ${project.version} + + + org.json + json + + + diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/Webhook.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/Webhook.java new file mode 100644 index 000000000000..1cc73ae31df3 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/Webhook.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook; + +import java.util.Date; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface Webhook extends ControlledEntity, Identity, InternalIdentity { + public static final long ID_DUMMY = 0L; + public static final String NAME_DUMMY = "Test"; + enum State { + Enabled, Disabled; + }; + + enum Scope { + Local, Domain, Global; + }; + + long getId(); + String getName(); + String getDescription(); + State getState(); + long getDomainId(); + long getAccountId(); + String getPayloadUrl(); + String getSecretKey(); + boolean isSslVerification(); + Scope getScope(); + Date getCreated(); +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookApiService.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookApiService.java new file mode 100644 index 000000000000..edd77e5b414c --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookApiService.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook; + +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.mom.webhook.api.command.user.CreateWebhookCmd; +import org.apache.cloudstack.mom.webhook.api.command.user.DeleteWebhookCmd; +import org.apache.cloudstack.mom.webhook.api.command.user.DeleteWebhookDeliveryCmd; +import org.apache.cloudstack.mom.webhook.api.command.user.ExecuteWebhookDeliveryCmd; +import org.apache.cloudstack.mom.webhook.api.command.user.ListWebhookDeliveriesCmd; +import org.apache.cloudstack.mom.webhook.api.command.user.ListWebhooksCmd; +import org.apache.cloudstack.mom.webhook.api.command.user.UpdateWebhookCmd; +import org.apache.cloudstack.mom.webhook.api.response.WebhookDeliveryResponse; +import org.apache.cloudstack.mom.webhook.api.response.WebhookResponse; + +import com.cloud.utils.component.PluggableService; +import com.cloud.utils.exception.CloudRuntimeException; + +public interface WebhookApiService extends PluggableService { + + ListResponse listWebhooks(ListWebhooksCmd cmd); + WebhookResponse createWebhook(CreateWebhookCmd cmd) throws CloudRuntimeException; + boolean deleteWebhook(DeleteWebhookCmd cmd) throws CloudRuntimeException; + WebhookResponse updateWebhook(UpdateWebhookCmd cmd) throws CloudRuntimeException; + WebhookResponse createWebhookResponse(long webhookId); + ListResponse listWebhookDeliveries(ListWebhookDeliveriesCmd cmd); + int deleteWebhookDelivery(DeleteWebhookDeliveryCmd cmd) throws CloudRuntimeException; + WebhookDeliveryResponse executeWebhookDelivery(ExecuteWebhookDeliveryCmd cmd) throws CloudRuntimeException; +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookApiServiceImpl.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookApiServiceImpl.java new file mode 100644 index 000000000000..187b140d5d84 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookApiServiceImpl.java @@ -0,0 +1,574 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.mom.webhook.api.command.user.CreateWebhookCmd; +import org.apache.cloudstack.mom.webhook.api.command.user.DeleteWebhookCmd; +import org.apache.cloudstack.mom.webhook.api.command.user.DeleteWebhookDeliveryCmd; +import org.apache.cloudstack.mom.webhook.api.command.user.ExecuteWebhookDeliveryCmd; +import org.apache.cloudstack.mom.webhook.api.command.user.ListWebhookDeliveriesCmd; +import org.apache.cloudstack.mom.webhook.api.command.user.ListWebhooksCmd; +import org.apache.cloudstack.mom.webhook.api.command.user.UpdateWebhookCmd; +import org.apache.cloudstack.mom.webhook.api.response.WebhookDeliveryResponse; +import org.apache.cloudstack.mom.webhook.api.response.WebhookResponse; +import org.apache.cloudstack.mom.webhook.dao.WebhookDao; +import org.apache.cloudstack.mom.webhook.dao.WebhookDeliveryDao; +import org.apache.cloudstack.mom.webhook.dao.WebhookDeliveryJoinDao; +import org.apache.cloudstack.mom.webhook.dao.WebhookJoinDao; +import org.apache.cloudstack.mom.webhook.vo.WebhookDeliveryJoinVO; +import org.apache.cloudstack.mom.webhook.vo.WebhookDeliveryVO; +import org.apache.cloudstack.mom.webhook.vo.WebhookJoinVO; +import org.apache.cloudstack.mom.webhook.vo.WebhookVO; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.api.ApiResponseHelper; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.domain.Domain; +import com.cloud.domain.dao.DomainDao; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.projects.Project; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; +import com.cloud.utils.UriUtils; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.rest.HttpConstants; + +public class WebhookApiServiceImpl extends ManagerBase implements WebhookApiService { + + @Inject + AccountManager accountManager; + @Inject + DomainDao domainDao; + @Inject + WebhookDao webhookDao; + @Inject + WebhookJoinDao webhookJoinDao; + @Inject + WebhookDeliveryDao webhookDeliveryDao; + @Inject + WebhookDeliveryJoinDao webhookDeliveryJoinDao; + @Inject + ManagementServerHostDao managementServerHostDao; + @Inject + WebhookService webhookService; + + protected WebhookResponse createWebhookResponse(WebhookJoinVO webhookVO) { + WebhookResponse response = new WebhookResponse(); + response.setObjectName("webhook"); + response.setId(webhookVO.getUuid()); + response.setName(webhookVO.getName()); + response.setDescription(webhookVO.getDescription()); + ApiResponseHelper.populateOwner(response, webhookVO); + response.setState(webhookVO.getState().toString()); + response.setPayloadUrl(webhookVO.getPayloadUrl()); + response.setSecretKey(webhookVO.getSecretKey()); + response.setSslVerification(webhookVO.isSslVerification()); + response.setScope(webhookVO.getScope().toString()); + response.setCreated(webhookVO.getCreated()); + return response; + } + + protected List getIdsOfAccessibleWebhooks(Account caller) { + if (Account.Type.ADMIN.equals(caller.getType())) { + return new ArrayList<>(); + } + String domainPath = null; + if (Account.Type.DOMAIN_ADMIN.equals(caller.getType())) { + Domain domain = domainDao.findById(caller.getDomainId()); + domainPath = domain.getPath(); + } + List webhooks = webhookJoinDao.listByAccountOrDomain(caller.getId(), domainPath); + return webhooks.stream().map(WebhookJoinVO::getId).collect(Collectors.toList()); + } + + protected ManagementServerHostVO basicWebhookDeliveryApiCheck(Account caller, final Long id, final Long webhookId, + final Long managementServerId, final Date startDate, final Date endDate) { + if (id != null) { + WebhookDeliveryVO webhookDeliveryVO = webhookDeliveryDao.findById(id); + if (webhookDeliveryVO == null) { + throw new InvalidParameterValueException("Invalid ID specified"); + } + WebhookVO webhookVO = webhookDao.findById(webhookDeliveryVO.getWebhookId()); + if (webhookVO != null) { + accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, webhookVO); + } + } + if (webhookId != null) { + WebhookVO webhookVO = webhookDao.findById(webhookId); + if (webhookVO == null) { + throw new InvalidParameterValueException("Invalid Webhook specified"); + } + accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, webhookVO); + } + if (endDate != null && startDate != null && endDate.before(startDate)) { + throw new InvalidParameterValueException(String.format("Invalid %s specified", ApiConstants.END_DATE)); + } + ManagementServerHostVO managementServerHostVO = null; + if (managementServerId != null) { + if (!Account.Type.ADMIN.equals(caller.getType())) { + throw new PermissionDeniedException("Invalid parameter specified"); + } + managementServerHostVO = managementServerHostDao.findById(managementServerId); + if (managementServerHostVO == null) { + throw new InvalidParameterValueException("Invalid management server specified"); + } + } + return managementServerHostVO; + } + + protected WebhookDeliveryResponse createWebhookDeliveryResponse(WebhookDeliveryJoinVO webhookDeliveryVO) { + WebhookDeliveryResponse response = new WebhookDeliveryResponse(); + response.setObjectName(WebhookDelivery.class.getSimpleName().toLowerCase()); + response.setId(webhookDeliveryVO.getUuid()); + response.setEventId(webhookDeliveryVO.getEventUuid()); + response.setEventType(webhookDeliveryVO.getEventType()); + response.setWebhookId(webhookDeliveryVO.getWebhookUuId()); + response.setWebhookName(webhookDeliveryVO.getWebhookName()); + response.setManagementServerId(webhookDeliveryVO.getManagementServerUuId()); + response.setManagementServerName(webhookDeliveryVO.getManagementServerName()); + response.setHeaders(webhookDeliveryVO.getHeaders()); + response.setPayload(webhookDeliveryVO.getPayload()); + response.setSuccess(webhookDeliveryVO.isSuccess()); + response.setResponse(webhookDeliveryVO.getResponse()); + response.setStartTime(webhookDeliveryVO.getStartTime()); + response.setEndTime(webhookDeliveryVO.getEndTime()); + return response; + } + + protected WebhookDeliveryResponse createTestWebhookDeliveryResponse(WebhookDelivery webhookDelivery, + Webhook webhook) { + WebhookDeliveryResponse response = new WebhookDeliveryResponse(); + response.setObjectName(WebhookDelivery.class.getSimpleName().toLowerCase()); + response.setId(webhookDelivery.getUuid()); + response.setEventType(WebhookDelivery.TEST_EVENT_TYPE); + if (webhook != null) { + response.setWebhookId(webhook.getUuid()); + response.setWebhookName(webhook.getName()); + } + ManagementServerHostVO msHost = + managementServerHostDao.findByMsid(webhookDelivery.getManagementServerId()); + if (msHost != null) { + response.setManagementServerId(msHost.getUuid()); + response.setManagementServerName(msHost.getName()); + } + response.setHeaders(webhookDelivery.getHeaders()); + response.setPayload(webhookDelivery.getPayload()); + response.setSuccess(webhookDelivery.isSuccess()); + response.setResponse(webhookDelivery.getResponse()); + response.setStartTime(webhookDelivery.getStartTime()); + response.setEndTime(webhookDelivery.getEndTime()); + return response; + } + + /** + * @param cmd + * @return Account + */ + protected Account getOwner(final CreateWebhookCmd cmd) { + final Account caller = CallContext.current().getCallingAccount(); + return accountManager.finalizeOwner(caller, cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId()); + } + + protected String getNormalizedPayloadUrl(String payloadUrl) { + if (StringUtils.isBlank(payloadUrl) || payloadUrl.startsWith("http://") || payloadUrl.startsWith("https://")) { + return payloadUrl; + } + return String.format("http://%s", payloadUrl); + } + + protected void validateWebhookOwnerPayloadUrl(Account owner, String payloadUrl, Webhook currentWebhook) { + WebhookVO webhookVO = webhookDao.findByAccountAndPayloadUrl(owner.getId(), payloadUrl); + if (webhookVO == null) { + return; + } + if (currentWebhook != null && webhookVO.getId() == currentWebhook.getId()) { + return; + } + String error = String.format("Payload URL: %s is already in use by another webhook", payloadUrl); + logger.error(String.format("%s: %s for Account [%s]", error, webhookVO, owner)); + throw new InvalidParameterValueException(error); + } + + @Override + public ListResponse listWebhooks(ListWebhooksCmd cmd) { + final CallContext ctx = CallContext.current(); + final Account caller = ctx.getCallingAccount(); + final Long clusterId = cmd.getId(); + final String stateStr = cmd.getState(); + final String name = cmd.getName(); + final String keyword = cmd.getKeyword(); + final String scopeStr = cmd.getScope(); + List responsesList = new ArrayList<>(); + List permittedAccounts = new ArrayList<>(); + Ternary domainIdRecursiveListProject = + new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); + accountManager.buildACLSearchParameters(caller, clusterId, cmd.getAccountName(), cmd.getProjectId(), + permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); + Long domainId = domainIdRecursiveListProject.first(); + Boolean isRecursive = domainIdRecursiveListProject.second(); + Project.ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); + + + Filter searchFilter = new Filter(WebhookJoinVO.class, "id", true, cmd.getStartIndex(), + cmd.getPageSizeVal()); + SearchBuilder sb = webhookJoinDao.createSearchBuilder(); + accountManager.buildACLSearchBuilder(sb, domainId, isRecursive, permittedAccounts, + listProjectResourcesCriteria); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("keyword", sb.entity().getName(), SearchCriteria.Op.LIKE); + sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ); + sb.and("scope", sb.entity().getScope(), SearchCriteria.Op.EQ); + SearchCriteria sc = sb.create(); + accountManager.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, + listProjectResourcesCriteria); + Webhook.Scope scope = null; + if (StringUtils.isNotEmpty(scopeStr)) { + try { + scope = Webhook.Scope.valueOf(scopeStr); + } catch (IllegalArgumentException iae) { + throw new InvalidParameterValueException("Invalid scope specified"); + } + } + if ((Webhook.Scope.Global.equals(scope) && !Account.Type.ADMIN.equals(caller.getType())) || + (Webhook.Scope.Domain.equals(scope) && + !List.of(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN).contains(caller.getType()))) { + throw new InvalidParameterValueException(String.format("Scope %s can not be specified", scope)); + } + Webhook.State state = null; + if (StringUtils.isNotEmpty(stateStr)) { + try { + state = Webhook.State.valueOf(stateStr); + } catch (IllegalArgumentException iae) { + throw new InvalidParameterValueException("Invalid state specified"); + } + } + if (scope != null) { + sc.setParameters("scope", scope.name()); + } + if (state != null) { + sc.setParameters("state", state.name()); + } + if(keyword != null){ + sc.setParameters("keyword", "%" + keyword + "%"); + } + if (clusterId != null) { + sc.setParameters("id", clusterId); + } + if (name != null) { + sc.setParameters("name", name); + } + Pair, Integer> webhooksAndCount = webhookJoinDao.searchAndCount(sc, searchFilter); + for (WebhookJoinVO webhook : webhooksAndCount.first()) { + WebhookResponse response = createWebhookResponse(webhook); + responsesList.add(response); + } + ListResponse response = new ListResponse<>(); + response.setResponses(responsesList, webhooksAndCount.second()); + return response; + } + + @Override + public WebhookResponse createWebhook(CreateWebhookCmd cmd) throws CloudRuntimeException { + final Account owner = getOwner(cmd); + final String name = cmd.getName(); + final String description = cmd.getDescription(); + final String payloadUrl = getNormalizedPayloadUrl(cmd.getPayloadUrl()); + final String secretKey = cmd.getSecretKey(); + final boolean sslVerification = cmd.isSslVerification(); + final String scopeStr = cmd.getScope(); + final String stateStr = cmd.getState(); + Webhook.Scope scope = Webhook.Scope.Local; + if (StringUtils.isNotEmpty(scopeStr)) { + try { + scope = Webhook.Scope.valueOf(scopeStr); + } catch (IllegalArgumentException iae) { + throw new InvalidParameterValueException("Invalid scope specified"); + } + } + if ((Webhook.Scope.Global.equals(scope) && !Account.Type.ADMIN.equals(owner.getType())) || + (Webhook.Scope.Domain.equals(scope) && + !List.of(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN).contains(owner.getType()))) { + throw new InvalidParameterValueException( + String.format("Scope %s can not be specified for owner %s", scope, owner.getName())); + } + Webhook.State state = Webhook.State.Enabled; + if (StringUtils.isNotEmpty(stateStr)) { + try { + state = Webhook.State.valueOf(stateStr); + } catch (IllegalArgumentException iae) { + throw new InvalidParameterValueException("Invalid state specified"); + } + } + UriUtils.validateUrl(payloadUrl); + validateWebhookOwnerPayloadUrl(owner, payloadUrl, null); + URI uri = URI.create(payloadUrl); + if (sslVerification && !HttpConstants.HTTPS.equalsIgnoreCase(uri.getScheme())) { + throw new InvalidParameterValueException( + String.format("SSL verification can be specified only for HTTPS URLs, %s", payloadUrl)); + } + long domainId = owner.getDomainId(); + Long cmdDomainId = cmd.getDomainId(); + if (cmdDomainId != null && + List.of(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN).contains(owner.getType()) && + Webhook.Scope.Domain.equals(scope)) { + domainId = cmdDomainId; + } + WebhookVO webhook = new WebhookVO(name, description, state, domainId, owner.getId(), payloadUrl, secretKey, + sslVerification, scope); + webhook = webhookDao.persist(webhook); + return createWebhookResponse(webhook.getId()); + } + + @Override + public boolean deleteWebhook(DeleteWebhookCmd cmd) throws CloudRuntimeException { + final Account caller = CallContext.current().getCallingAccount(); + final long id = cmd.getId(); + Webhook webhook = webhookDao.findById(id); + if (webhook == null) { + throw new InvalidParameterValueException("Unable to find the webhook with the specified ID"); + } + accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, webhook); + return webhookDao.remove(id); + } + + @Override + public WebhookResponse updateWebhook(UpdateWebhookCmd cmd) throws CloudRuntimeException { + final Account caller = CallContext.current().getCallingAccount(); + final long id = cmd.getId(); + final String name = cmd.getName(); + final String description = cmd.getDescription(); + final String payloadUrl = getNormalizedPayloadUrl(cmd.getPayloadUrl()); + String secretKey = cmd.getSecretKey(); + final Boolean sslVerification = cmd.isSslVerification(); + final String scopeStr = cmd.getScope(); + final String stateStr = cmd.getState(); + WebhookVO webhook = webhookDao.findById(id); + if (webhook == null) { + throw new InvalidParameterValueException("Unable to find the webhook with the specified ID"); + } + accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, webhook); + boolean updateNeeded = false; + if (StringUtils.isNotBlank(name)) { + webhook.setName(name); + updateNeeded = true; + } + if (description != null) { + webhook.setDescription(description); + updateNeeded = true; + } + if (StringUtils.isNotEmpty(stateStr)) { + try { + Webhook.State state = Webhook.State.valueOf(stateStr); + webhook.setState(state); + updateNeeded = true; + } catch (IllegalArgumentException iae) { + throw new InvalidParameterValueException("Invalid state specified"); + } + } + Account owner = accountManager.getAccount(webhook.getAccountId()); + if (StringUtils.isNotEmpty(scopeStr)) { + try { + Webhook.Scope scope = Webhook.Scope.valueOf(scopeStr); + if ((Webhook.Scope.Global.equals(scope) && !Account.Type.ADMIN.equals(owner.getType())) || + (Webhook.Scope.Domain.equals(scope) && + !List.of(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN).contains(owner.getType()))) { + throw new InvalidParameterValueException( + String.format("Scope %s can not be specified for owner %s", scope, owner.getName())); + } + webhook.setScope(scope); + updateNeeded = true; + } catch (IllegalArgumentException iae) { + throw new InvalidParameterValueException("Invalid scope specified"); + } + } + URI uri = URI.create(webhook.getPayloadUrl()); + if (StringUtils.isNotEmpty(payloadUrl)) { + UriUtils.validateUrl(payloadUrl); + validateWebhookOwnerPayloadUrl(owner, payloadUrl, webhook); + uri = URI.create(payloadUrl); + webhook.setPayloadUrl(payloadUrl); + updateNeeded = true; + } + if (sslVerification != null) { + if (Boolean.TRUE.equals(sslVerification) && !HttpConstants.HTTPS.equalsIgnoreCase(uri.getScheme())) { + throw new InvalidParameterValueException( + String.format("SSL verification can be specified only for HTTPS URLs, %s", payloadUrl)); + } + webhook.setSslVerification(sslVerification); + updateNeeded = true; + } + if (secretKey != null) { + if (StringUtils.isBlank(secretKey)) { + secretKey = null; + } + webhook.setSecretKey(secretKey); + updateNeeded = true; + } + if (updateNeeded && !webhookDao.update(id, webhook)) { + return null; + } + return createWebhookResponse(webhook.getId()); + } + + @Override + public WebhookResponse createWebhookResponse(long webhookId) { + WebhookJoinVO webhookVO = webhookJoinDao.findById(webhookId); + return createWebhookResponse(webhookVO); + } + + @Override + public ListResponse listWebhookDeliveries(ListWebhookDeliveriesCmd cmd) { + final CallContext ctx = CallContext.current(); + final Account caller = ctx.getCallingAccount(); + final Long id = cmd.getId(); + final Long webhookId = cmd.getWebhookId(); + final Long managementServerId = cmd.getManagementServerId(); + final String keyword = cmd.getKeyword(); + final Date startDate = cmd.getStartDate(); + final Date endDate = cmd.getEndDate(); + final String eventType = cmd.getEventType(); + List responsesList = new ArrayList<>(); + ManagementServerHostVO host = basicWebhookDeliveryApiCheck(caller, id, webhookId, managementServerId, + startDate, endDate); + + Filter searchFilter = new Filter(WebhookDeliveryJoinVO.class, "id", false, cmd.getStartIndex(), + cmd.getPageSizeVal()); + List webhookIds = new ArrayList<>(); + if (webhookId != null) { + webhookIds.add(webhookId); + } else { + webhookIds.addAll(getIdsOfAccessibleWebhooks(caller)); + } + Pair, Integer> deliveriesAndCount = + webhookDeliveryJoinDao.searchAndCountByListApiParameters(id, webhookIds, + (host != null ? host.getMsid() : null), keyword, startDate, endDate, eventType, searchFilter); + for (WebhookDeliveryJoinVO delivery : deliveriesAndCount.first()) { + WebhookDeliveryResponse response = createWebhookDeliveryResponse(delivery); + responsesList.add(response); + } + ListResponse response = new ListResponse<>(); + response.setResponses(responsesList, deliveriesAndCount.second()); + return response; + } + + @Override + public int deleteWebhookDelivery(DeleteWebhookDeliveryCmd cmd) throws CloudRuntimeException { + final CallContext ctx = CallContext.current(); + final Account caller = ctx.getCallingAccount(); + final Long id = cmd.getId(); + final Long webhookId = cmd.getWebhookId(); + final Long managementServerId = cmd.getManagementServerId(); + final Date startDate = cmd.getStartDate(); + final Date endDate = cmd.getEndDate(); + ManagementServerHostVO host = basicWebhookDeliveryApiCheck(caller, id, webhookId, managementServerId, + startDate, endDate); + int removed = webhookDeliveryDao.deleteByDeleteApiParams(id, webhookId, + (host != null ? host.getMsid() : null), startDate, endDate); + logger.info("{} webhook deliveries removed", removed); + return removed; + } + + @Override + public WebhookDeliveryResponse executeWebhookDelivery(ExecuteWebhookDeliveryCmd cmd) throws CloudRuntimeException { + final CallContext ctx = CallContext.current(); + final Account caller = ctx.getCallingAccount(); + final Long deliveryId = cmd.getId(); + final Long webhookId = cmd.getWebhookId(); + final String payloadUrl = getNormalizedPayloadUrl(cmd.getPayloadUrl()); + final String secretKey = cmd.getSecretKey(); + final Boolean sslVerification = cmd.isSslVerification(); + final String payload = cmd.getPayload(); + final Account owner = accountManager.finalizeOwner(caller, null, null, null); + + if (ObjectUtils.allNull(deliveryId, webhookId) && StringUtils.isBlank(payloadUrl)) { + throw new InvalidParameterValueException(String.format("One of the %s, %s or %s must be specified", + ApiConstants.ID, ApiConstants.WEBHOOK_ID, ApiConstants.PAYLOAD_URL)); + } + WebhookDeliveryVO existingDelivery = null; + WebhookVO webhook = null; + if (deliveryId != null) { + existingDelivery = webhookDeliveryDao.findById(deliveryId); + if (existingDelivery == null) { + throw new InvalidParameterValueException("Invalid webhook delivery specified"); + } + webhook = webhookDao.findById(existingDelivery.getWebhookId()); + } + if (StringUtils.isNotBlank(payloadUrl)) { + UriUtils.validateUrl(payloadUrl); + } + if (webhookId != null) { + webhook = webhookDao.findById(webhookId); + if (webhook == null) { + throw new InvalidParameterValueException("Invalid webhook specified"); + } + if (StringUtils.isNotBlank(payloadUrl)) { + webhook.setPayloadUrl(payloadUrl); + } + if (StringUtils.isNotBlank(secretKey)) { + webhook.setSecretKey(secretKey); + } + if (sslVerification != null) { + webhook.setSslVerification(Boolean.TRUE.equals(sslVerification)); + } + } + if (ObjectUtils.allNull(deliveryId, webhookId)) { + webhook = new WebhookVO(owner.getDomainId(), owner.getId(), payloadUrl, secretKey, + Boolean.TRUE.equals(sslVerification)); + } + WebhookDelivery webhookDelivery = webhookService.executeWebhookDelivery(existingDelivery, webhook, payload); + if (webhookDelivery.getId() != WebhookDelivery.ID_DUMMY) { + return createWebhookDeliveryResponse(webhookDeliveryJoinDao.findById(webhookDelivery.getId())); + } + return createTestWebhookDeliveryResponse(webhookDelivery, webhook); + } + + @Override + public List> getCommands() { + List> cmdList = new ArrayList<>(); + cmdList.add(CreateWebhookCmd.class); + cmdList.add(ListWebhooksCmd.class); + cmdList.add(UpdateWebhookCmd.class); + cmdList.add(DeleteWebhookCmd.class); + cmdList.add(ListWebhookDeliveriesCmd.class); + cmdList.add(DeleteWebhookDeliveryCmd.class); + cmdList.add(ExecuteWebhookDeliveryCmd.class); + return cmdList; + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDelivery.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDelivery.java new file mode 100644 index 000000000000..b24891539f9b --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDelivery.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook; + +import java.util.Date; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface WebhookDelivery extends Identity, InternalIdentity { + public static final long ID_DUMMY = 0L; + public static final String TEST_EVENT_TYPE = "TEST.WEBHOOK"; + + long getId(); + long getEventId(); + long getWebhookId(); + long getManagementServerId(); + String getHeaders(); + String getPayload(); + boolean isSuccess(); + String getResponse(); + Date getStartTime(); + Date getEndTime(); +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java new file mode 100644 index 000000000000..ac840c00be32 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java @@ -0,0 +1,287 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.security.InvalidKeyException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +import javax.crypto.Mac; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.framework.async.AsyncRpcContext; +import org.apache.cloudstack.framework.events.Event; +import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.TrustAllStrategy; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +public class WebhookDeliveryThread implements Runnable { + protected static Logger LOGGER = LogManager.getLogger(WebhookDeliveryThread.class); + + private static final String HEADER_X_CS_EVENT_ID = "X-CS-Event-ID"; + private static final String HEADER_X_CS_EVENT = "X-CS-Event"; + private static final String HEADER_X_CS_SIGNATURE = "X-CS-Signature"; + private static final String PREFIX_HEADER_USER_AGENT = "CS-Hookshot/"; + private final Webhook webhook; + private final Event event; + private CloseableHttpClient httpClient; + private String headers; + private String payload; + private String response; + private Date startTime; + private int deliveryTries = 3; + private int deliveryTimeout = 10; + + AsyncCompletionCallback callback; + + protected boolean isValidJson(String json) { + try { + new JSONObject(json); + } catch (JSONException ex) { + try { + new JSONArray(json); + } catch (JSONException ex1) { + return false; + } + } + return true; + } + + protected void setHttpClient() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { + if (webhook.isSslVerification()) { + httpClient = HttpClients.createDefault(); + return; + } + httpClient = HttpClients + .custom() + .setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, + TrustAllStrategy.INSTANCE).build()) + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .build(); + } + + protected HttpPost getBasicHttpPostRequest() throws URISyntaxException { + final URI uri = new URI(webhook.getPayloadUrl()); + HttpPost request = new HttpPost(); + RequestConfig.Builder requestConfig = RequestConfig.custom(); + requestConfig.setConnectTimeout(deliveryTimeout * 1000); + requestConfig.setConnectionRequestTimeout(deliveryTimeout * 1000); + requestConfig.setSocketTimeout(deliveryTimeout * 1000); + request.setConfig(requestConfig.build()); + request.setURI(uri); + return request; + } + + protected void updateRequestHeaders(HttpPost request) throws DecoderException, NoSuchAlgorithmException, + InvalidKeyException { + request.addHeader(HEADER_X_CS_EVENT_ID, event.getEventUuid()); + request.addHeader(HEADER_X_CS_EVENT, event.getEventType()); + request.setHeader(HttpHeaders.USER_AGENT, String.format("%s%s", PREFIX_HEADER_USER_AGENT, + event.getResourceAccountUuid())); + if (StringUtils.isNotBlank(webhook.getSecretKey())) { + request.addHeader(HEADER_X_CS_SIGNATURE, generateHMACSignature(payload, webhook.getSecretKey())); + } + List
headers = new ArrayList<>(Arrays.asList(request.getAllHeaders())); + HttpEntity entity = request.getEntity(); + if (entity.getContentLength() > 0 && !request.containsHeader(HttpHeaders.CONTENT_LENGTH)) { + headers.add(new BasicHeader(HttpHeaders.CONTENT_LENGTH, Long.toString(entity.getContentLength()))); + } + if (entity.getContentType() != null && !request.containsHeader(HttpHeaders.CONTENT_TYPE)) { + headers.add(entity.getContentType()); + } + if (entity.getContentEncoding() != null && !request.containsHeader(HttpHeaders.CONTENT_ENCODING)) { + headers.add(entity.getContentEncoding()); + } + this.headers = StringUtils.join(headers, "\n"); + } + + public WebhookDeliveryThread(Webhook webhook, Event event, + AsyncCompletionCallback callback) { + this.webhook = webhook; + this.event = event; + this.callback = callback; + } + + public void setDeliveryTries(int deliveryTries) { + this.deliveryTries = deliveryTries; + } + + public void setDeliveryTimeout(int deliveryTimeout) { + this.deliveryTimeout = deliveryTimeout; + } + + @Override + public void run() { + LOGGER.debug("Delivering event: {} for {}", event.getEventType(), webhook); + if (event == null) { + LOGGER.warn("Invalid event received for delivering to {}", webhook); + return; + } + payload = event.getDescription(); + LOGGER.trace("Payload: {}", payload); + int attempt = 0; + boolean success = false; + try { + setHttpClient(); + } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { + response = String.format("Failed to initiate delivery due to : %s", e.getMessage()); + callback.complete(new WebhookDeliveryResult(headers, payload, success, response, new Date())); + return; + } + while (attempt < deliveryTries) { + attempt++; + if (delivery(attempt)) { + success = true; + break; + } + } + callback.complete(new WebhookDeliveryResult(headers, payload, success, response, startTime)); + } + + protected void updateResponseFromRequest(HttpEntity entity) { + try { + this.response = EntityUtils.toString(entity, StandardCharsets.UTF_8); + } catch (IOException e) { + LOGGER.error("Failed to parse response for event: {} for {}", + event.getEventType(), webhook); + this.response = ""; + } + } + + protected boolean delivery(int attempt) { + startTime = new Date(); + try { + HttpPost request = getBasicHttpPostRequest(); + StringEntity input = new StringEntity(payload, + isValidJson(payload) ? ContentType.APPLICATION_JSON : ContentType.TEXT_PLAIN); + request.setEntity(input); + updateRequestHeaders(request); + LOGGER.trace("Delivering event: {} for {} with timeout: {}, " + + "attempt #{}", event.getEventType(), webhook, + deliveryTimeout, attempt); + final CloseableHttpResponse response = httpClient.execute(request); + updateResponseFromRequest(response.getEntity()); + if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + LOGGER.trace("Successfully delivered event: {} for {}", + event.getEventType(), webhook); + return true; + } + } catch (URISyntaxException | IOException | DecoderException | NoSuchAlgorithmException | + InvalidKeyException e) { + LOGGER.warn("Failed to deliver {}, in attempt #{} due to: {}", + webhook, attempt, e.getMessage()); + response = String.format("Failed due to : %s", e.getMessage()); + } + return false; + } + + public static String generateHMACSignature(String data, String key) + throws InvalidKeyException, NoSuchAlgorithmException, DecoderException { + Mac mac = Mac.getInstance("HMACSHA256"); + SecretKey secretKey = new SecretKeySpec(key.getBytes(StandardCharsets.UTF_8), mac.getAlgorithm()); + mac.init(secretKey); + byte[] dataAsBytes = data.getBytes(StandardCharsets.UTF_8); + byte[] encodedText = mac.doFinal(dataAsBytes); + return new String(Base64.encodeBase64(encodedText)).trim(); + } + + public static class WebhookDeliveryContext extends AsyncRpcContext { + private final Long eventId; + private final Long ruleId; + + public WebhookDeliveryContext(AsyncCompletionCallback callback, Long eventId, Long ruleId) { + super(callback); + this.eventId = eventId; + this.ruleId = ruleId; + } + + public Long getEventId() { + return eventId; + } + + public Long getRuleId() { + return ruleId; + } + } + + public static class WebhookDeliveryResult extends CommandResult { + private final String headers; + private final String payload; + private final Date starTime; + private final Date endTime; + + public WebhookDeliveryResult(String headers, String payload, boolean success, String response, Date starTime) { + super(); + this.headers = headers; + this.payload = payload; + this.setResult(response); + this.setSuccess(success); + this.starTime = starTime; + this.endTime = new Date(); + } + + public String getHeaders() { + return headers; + } + + public String getPayload() { + return payload; + } + + public Date getStarTime() { + return starTime; + } + + public Date getEndTime() { + return endTime; + } + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookEventBus.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookEventBus.java new file mode 100644 index 000000000000..c2dade843618 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookEventBus.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.mom.webhook; + +import java.util.Map; +import java.util.UUID; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.framework.events.Event; +import org.apache.cloudstack.framework.events.EventBus; +import org.apache.cloudstack.framework.events.EventBusException; +import org.apache.cloudstack.framework.events.EventSubscriber; +import org.apache.cloudstack.framework.events.EventTopic; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.utils.component.ManagerBase; +import com.google.gson.Gson; + +public class WebhookEventBus extends ManagerBase implements EventBus { + + protected static Logger LOGGER = LogManager.getLogger(WebhookEventBus.class); + private static Gson gson; + + @Inject + WebhookService webhookService; + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + _name = name; + return true; + } + + @Override + public void setName(String name) { + _name = name; + } + + @Override + public UUID subscribe(EventTopic topic, EventSubscriber subscriber) throws EventBusException { + /* NOOP */ + return UUID.randomUUID(); + } + + @Override + public void unsubscribe(UUID subscriberId, EventSubscriber subscriber) throws EventBusException { + /* NOOP */ + } + + @Override + public void publish(Event event) throws EventBusException { + webhookService.handleEvent(event); + } + + @Override + public String getName() { + return _name; + } + + @Override + public boolean start() { + return true; + } + + @Override + public boolean stop() { + return true; + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookService.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookService.java new file mode 100644 index 000000000000..5a5aced288d1 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookService.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.events.Event; +import org.apache.cloudstack.framework.events.EventBusException; + +import com.cloud.utils.component.PluggableService; +import com.cloud.utils.exception.CloudRuntimeException; + +public interface WebhookService extends PluggableService, Configurable { + + ConfigKey WebhookDeliveryTimeout = new ConfigKey<>("Advanced", Integer.class, + "webhook.delivery.timeout", "10", + "Wait timeout (in seconds) for a webhook delivery delivery", + true, ConfigKey.Scope.Domain); + + ConfigKey WebhookDeliveryTries = new ConfigKey<>("Advanced", Integer.class, + "webhook.delivery.tries", "3", + "Number of tries to be made for a webhook delivery", + true, ConfigKey.Scope.Domain); + + ConfigKey WebhookDeliveryThreadPoolSize = new ConfigKey<>("Advanced", Integer.class, + "webhook.delivery.thread.pool.size", "5", + "Size of the thread pool for webhook deliveries", + false, ConfigKey.Scope.Global); + + ConfigKey WebhookDeliveriesLimit = new ConfigKey<>("Advanced", Integer.class, + "webhook.deliveries.limit", "10", + "Limit for the number of deliveries to keep in DB per webhook", + true, ConfigKey.Scope.Global); + + ConfigKey WebhookDeliveriesCleanupInitialDelay = new ConfigKey<>("Advanced", Integer.class, + "webhook.deliveries.cleanup.initial.delay", "180", + "Initial delay (in seconds) for webhook deliveries cleanup task", + false, ConfigKey.Scope.Global); + + ConfigKey WebhookDeliveriesCleanupInterval = new ConfigKey<>("Advanced", Integer.class, + "webhook.deliveries.cleanup.interval", "3600", + "Interval (in seconds) for cleaning up webhook deliveries", + false, ConfigKey.Scope.Global); + + void handleEvent(Event event) throws EventBusException; + WebhookDelivery executeWebhookDelivery(WebhookDelivery delivery, Webhook webhook, String payload) + throws CloudRuntimeException; +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookServiceImpl.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookServiceImpl.java new file mode 100644 index 000000000000..58b265a99c06 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookServiceImpl.java @@ -0,0 +1,354 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; +import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.framework.async.AsyncRpcContext; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.events.Event; +import org.apache.cloudstack.framework.events.EventBusException; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.mom.webhook.dao.WebhookDao; +import org.apache.cloudstack.mom.webhook.dao.WebhookDeliveryDao; +import org.apache.cloudstack.mom.webhook.vo.WebhookDeliveryVO; +import org.apache.cloudstack.mom.webhook.vo.WebhookVO; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.cloudstack.webhook.WebhookHelper; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.api.query.vo.EventJoinVO; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.domain.dao.DomainDao; +import com.cloud.event.EventCategory; +import com.cloud.event.dao.EventJoinDao; +import com.cloud.server.ManagementService; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.exception.CloudRuntimeException; + +public class WebhookServiceImpl extends ManagerBase implements WebhookService, WebhookHelper { + public static final String WEBHOOK_JOB_POOL_THREAD_PREFIX = "Webhook-Job-Executor"; + private ExecutorService webhookJobExecutor; + private ScheduledExecutorService webhookDeliveriesCleanupExecutor; + + @Inject + EventJoinDao eventJoinDao; + @Inject + WebhookDao webhookDao; + @Inject + protected WebhookDeliveryDao webhookDeliveryDao; + @Inject + ManagementServerHostDao managementServerHostDao; + @Inject + DomainDao domainDao; + @Inject + AccountManager accountManager; + + protected WebhookDeliveryThread getDeliveryJob(Event event, Webhook webhook, Pair configs) { + WebhookDeliveryThread.WebhookDeliveryContext context = + new WebhookDeliveryThread.WebhookDeliveryContext<>(null, event.getEventId(), webhook.getId()); + AsyncCallbackDispatcher caller = + AsyncCallbackDispatcher.create(this); + caller.setCallback(caller.getTarget().deliveryCompleteCallback(null, null)) + .setContext(context); + WebhookDeliveryThread job = new WebhookDeliveryThread(webhook, event, caller); + job = ComponentContext.inject(job); + job.setDeliveryTries(configs.first()); + job.setDeliveryTimeout(configs.second()); + return job; + } + + protected List getDeliveryJobs(Event event) throws EventBusException { + List jobs = new ArrayList<>(); + if (!EventCategory.ACTION_EVENT.getName().equals(event.getEventCategory())) { + return jobs; + } + if (event.getResourceAccountId() == null) { + logger.warn("Skipping delivering event [ID: {}, description: {}] to any webhook as account ID is missing", + event.getEventId(), event.getDescription()); + throw new EventBusException(String.format("Account missing for the event ID: %s", event.getEventUuid())); + } + List domainIds = new ArrayList<>(); + if (event.getResourceDomainId() != null) { + domainIds.add(event.getResourceDomainId()); + domainIds.addAll(domainDao.getDomainParentIds(event.getResourceDomainId())); + } + List webhooks = + webhookDao.listByEnabledForDelivery(event.getResourceAccountId(), domainIds); + Map> domainConfigs = new HashMap<>(); + for (WebhookVO webhook : webhooks) { + if (!domainConfigs.containsKey(webhook.getDomainId())) { + domainConfigs.put(webhook.getDomainId(), + new Pair<>(WebhookDeliveryTries.valueIn(webhook.getDomainId()), + WebhookDeliveryTimeout.valueIn(webhook.getDomainId()))); + } + Pair configs = domainConfigs.get(webhook.getDomainId()); + WebhookDeliveryThread job = getDeliveryJob(event, webhook, configs); + jobs.add(job); + } + return jobs; + } + + protected Runnable getManualDeliveryJob(WebhookDelivery existingDelivery, Webhook webhook, String payload, + AsyncCallFuture future) { + if (StringUtils.isBlank(payload)) { + payload = "{ \"CloudStack\": \"works!\" }"; + } + long eventId = Webhook.ID_DUMMY; + String eventType = WebhookDelivery.TEST_EVENT_TYPE; + String eventUuid = UUID.randomUUID().toString(); + String description = payload; + String resourceAccountUuid = null; + if (existingDelivery != null) { + EventJoinVO eventJoinVO = eventJoinDao.findById(existingDelivery.getEventId()); + eventId = eventJoinVO.getId(); + eventType = eventJoinVO.getType(); + eventUuid = eventJoinVO.getUuid(); + description = existingDelivery.getPayload(); + resourceAccountUuid = eventJoinVO.getAccountUuid(); + } else { + Account account = accountManager.getAccount(webhook.getAccountId()); + resourceAccountUuid = account.getUuid(); + } + Event event = new Event(ManagementService.Name, EventCategory.ACTION_EVENT.toString(), + eventType, null, null); + event.setEventId(eventId); + event.setEventUuid(eventUuid); + event.setDescription(description); + event.setResourceAccountUuid(resourceAccountUuid); + ManualDeliveryContext context = + new ManualDeliveryContext<>(null, webhook, future); + AsyncCallbackDispatcher caller = + AsyncCallbackDispatcher.create(this); + caller.setCallback(caller.getTarget().manualDeliveryCompleteCallback(null, null)) + .setContext(context); + WebhookDeliveryThread job = new WebhookDeliveryThread(webhook, event, caller); + job.setDeliveryTries(WebhookDeliveryTries.valueIn(webhook.getDomainId())); + job.setDeliveryTimeout(WebhookDeliveryTimeout.valueIn(webhook.getDomainId())); + return job; + } + + protected Void deliveryCompleteCallback( + AsyncCallbackDispatcher callback, + WebhookDeliveryThread.WebhookDeliveryContext context) { + WebhookDeliveryThread.WebhookDeliveryResult result = callback.getResult(); + WebhookDeliveryVO deliveryVO = new WebhookDeliveryVO(context.getEventId(), context.getRuleId(), + ManagementServerNode.getManagementServerId(), result.getHeaders(), result.getPayload(), + result.isSuccess(), result.getResult(), result.getStarTime(), result.getEndTime()); + webhookDeliveryDao.persist(deliveryVO); + return null; + } + + protected Void manualDeliveryCompleteCallback( + AsyncCallbackDispatcher callback, + ManualDeliveryContext context) { + WebhookDeliveryThread.WebhookDeliveryResult result = callback.getResult(); + context.future.complete(result); + return null; + } + + protected long cleanupOldWebhookDeliveries(long deliveriesLimit) { + Filter filter = new Filter(WebhookVO.class, "id", true, 0L, 50L); + Pair, Integer> webhooksAndCount = + webhookDao.searchAndCount(webhookDao.createSearchCriteria(), filter); + List webhooks = webhooksAndCount.first(); + long count = webhooksAndCount.second(); + long processed = 0; + do { + for (WebhookVO webhook : webhooks) { + webhookDeliveryDao.removeOlderDeliveries(webhook.getId(), deliveriesLimit); + processed++; + } + if (processed < count) { + filter.setOffset(processed); + webhooks = webhookDao.listAll(filter); + } + } while (processed < count); + return processed; + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + try { + webhookJobExecutor = Executors.newFixedThreadPool(WebhookDeliveryThreadPoolSize.value(), + new NamedThreadFactory(WEBHOOK_JOB_POOL_THREAD_PREFIX)); + webhookDeliveriesCleanupExecutor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("Webhook-Deliveries-Cleanup-Worker")); + } catch (final Exception e) { + throw new ConfigurationException("Unable to to configure WebhookServiceImpl"); + } + return true; + } + + @Override + public boolean start() { + long webhookDeliveriesCleanupInitialDelay = WebhookDeliveriesCleanupInitialDelay.value(); + long webhookDeliveriesCleanupInterval = WebhookDeliveriesCleanupInterval.value(); + logger.debug("Scheduling webhook deliveries cleanup task with initial delay={}s and interval={}s", + webhookDeliveriesCleanupInitialDelay, webhookDeliveriesCleanupInterval); + webhookDeliveriesCleanupExecutor.scheduleWithFixedDelay(new WebhookDeliveryCleanupWorker(), + webhookDeliveriesCleanupInitialDelay, webhookDeliveriesCleanupInterval, TimeUnit.SECONDS); + return true; + } + + @Override + public boolean stop() { + webhookJobExecutor.shutdown(); + return true; + } + + @Override + public String getConfigComponentName() { + return WebhookService.class.getName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{ + WebhookDeliveryTimeout, + WebhookDeliveryTries, + WebhookDeliveryThreadPoolSize, + WebhookDeliveriesLimit, + WebhookDeliveriesCleanupInitialDelay, + WebhookDeliveriesCleanupInterval + }; + } + + @Override + public void deleteWebhooksForAccount(long accountId) { + webhookDao.deleteByAccount(accountId); + } + + @Override + public List listWebhooksByAccount(long accountId) { + return webhookDao.listByAccount(accountId); + } + + @Override + public void handleEvent(Event event) throws EventBusException { + List jobs = getDeliveryJobs(event); + for(Runnable job : jobs) { + webhookJobExecutor.submit(job); + } + } + + @Override + public WebhookDelivery executeWebhookDelivery(WebhookDelivery delivery, Webhook webhook, String payload) + throws CloudRuntimeException { + AsyncCallFuture future = new AsyncCallFuture<>(); + Runnable job = getManualDeliveryJob(delivery, webhook, payload, future); + webhookJobExecutor.submit(job); + WebhookDeliveryThread.WebhookDeliveryResult result = null; + WebhookDeliveryVO webhookDeliveryVO; + try { + result = future.get(); + if (delivery != null) { + webhookDeliveryVO = new WebhookDeliveryVO(delivery.getEventId(), delivery.getWebhookId(), + ManagementServerNode.getManagementServerId(), result.getHeaders(), result.getPayload(), + result.isSuccess(), result.getResult(), result.getStarTime(), result.getEndTime()); + webhookDeliveryVO = webhookDeliveryDao.persist(webhookDeliveryVO); + } else { + webhookDeliveryVO = new WebhookDeliveryVO(ManagementServerNode.getManagementServerId(), + result.getHeaders(), result.getPayload(), result.isSuccess(), result.getResult(), + result.getStarTime(), result.getEndTime()); + } + } catch (InterruptedException | ExecutionException e) { + logger.error(String.format("Failed to execute test webhook delivery due to: %s", e.getMessage()), e); + throw new CloudRuntimeException("Failed to execute test webhook delivery"); + } + return webhookDeliveryVO; + } + + @Override + public List> getCommands() { + return new ArrayList<>(); + } + + static public class ManualDeliveryContext extends AsyncRpcContext { + final Webhook webhook; + final AsyncCallFuture future; + + public ManualDeliveryContext(AsyncCompletionCallback callback, Webhook webhook, + AsyncCallFuture future) { + super(callback); + this.webhook = webhook; + this.future = future; + } + + } + + public class WebhookDeliveryCleanupWorker extends ManagedContextRunnable { + + protected void runCleanupForLongestRunningManagementServer() { + try { + ManagementServerHostVO msHost = managementServerHostDao.findOneByLongestRuntime(); + if (msHost == null || (msHost.getMsid() != ManagementServerNode.getManagementServerId())) { + logger.debug("Skipping the webhook delivery cleanup task on this management server"); + return; + } + long deliveriesLimit = WebhookDeliveriesLimit.value(); + logger.debug("Clearing old deliveries for webhooks with limit={} using management server {}", + deliveriesLimit, msHost.getMsid()); + long processed = cleanupOldWebhookDeliveries(deliveriesLimit); + logger.debug("Cleared old deliveries with limit={} for {} webhooks", deliveriesLimit, processed); + } catch (Exception e) { + logger.warn("Cleanup task failed to cleanup old webhook deliveries", e); + } + } + + @Override + protected void runInContext() { + GlobalLock gcLock = GlobalLock.getInternLock("WebhookDeliveriesCleanup"); + try { + if (gcLock.lock(3)) { + try { + runCleanupForLongestRunningManagementServer(); + } finally { + gcLock.unlock(); + } + } + } finally { + gcLock.releaseRef(); + } + } + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/CreateWebhookCmd.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/CreateWebhookCmd.java new file mode 100644 index 000000000000..d3d2cf18e1fd --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/CreateWebhookCmd.java @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.apache.cloudstack.mom.webhook.Webhook; +import org.apache.cloudstack.mom.webhook.api.response.WebhookResponse; + +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "createWebhook", + description = "Creates a Webhook", + responseObject = WebhookResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {Webhook.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}, + since = "4.20.0") +public class CreateWebhookCmd extends BaseCmd { + + @Inject + WebhookApiService webhookApiService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.NAME, type = BaseCmd.CommandType.STRING, required = true, description = "Name for the Webhook") + private String name; + + @Parameter(name = ApiConstants.DESCRIPTION, type = BaseCmd.CommandType.STRING, description = "Description for the Webhook") + private String description; + + @Parameter(name = ApiConstants.STATE, type = BaseCmd.CommandType.STRING, description = "State of the Webhook") + private String state; + + @ACL(accessType = SecurityChecker.AccessType.UseEntry) + @Parameter(name = ApiConstants.ACCOUNT, type = BaseCmd.CommandType.STRING, description = "An optional account for the" + + " Webhook. Must be used with domainId.") + private String accountName; + + @ACL(accessType = SecurityChecker.AccessType.UseEntry) + @Parameter(name = ApiConstants.DOMAIN_ID, type = BaseCmd.CommandType.UUID, entityType = DomainResponse.class, + description = "an optional domainId for the Webhook. If the account parameter is used, domainId must also be used.") + private Long domainId; + + @ACL(accessType = SecurityChecker.AccessType.UseEntry) + @Parameter(name = ApiConstants.PROJECT_ID, type = BaseCmd.CommandType.UUID, entityType = ProjectResponse.class, + description = "Project for the Webhook") + private Long projectId; + + @Parameter(name = ApiConstants.PAYLOAD_URL, + type = BaseCmd.CommandType.STRING, + required = true, + description = "Payload URL of the Webhook") + private String payloadUrl; + + @Parameter(name = ApiConstants.SECRET_KEY, type = BaseCmd.CommandType.STRING, description = "Secret key of the Webhook") + private String secretKey; + + @Parameter(name = ApiConstants.SSL_VERIFICATION, type = BaseCmd.CommandType.BOOLEAN, description = "If set to true then SSL verification will be done for the Webhook otherwise not") + private Boolean sslVerification; + + @Parameter(name = ApiConstants.SCOPE, type = BaseCmd.CommandType.STRING, description = "Scope of the Webhook", + authorized = {RoleType.Admin, RoleType.DomainAdmin}) + private String scope; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getState() { + return state; + } + + public String getAccountName() { + return accountName; + } + + public Long getDomainId() { + return domainId; + } + + public Long getProjectId() { + return projectId; + } + + public String getPayloadUrl() { + return payloadUrl; + } + + public String getSecretKey() { + return secretKey; + } + + public boolean isSslVerification() { + return Boolean.TRUE.equals(sslVerification); + } + + public String getScope() { + return scope; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException { + try { + WebhookResponse response = webhookApiService.createWebhook(this); + if (response == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create webhook"); + } + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/DeleteWebhookCmd.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/DeleteWebhookCmd.java new file mode 100644 index 000000000000..c9fb01580c2d --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/DeleteWebhookCmd.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.apache.cloudstack.mom.webhook.Webhook; +import org.apache.cloudstack.mom.webhook.api.response.WebhookResponse; + +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "deleteWebhook", + description = "Deletes a Webhook", + responseObject = SuccessResponse.class, + entityType = {Webhook.class}, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}, + since = "4.20.0") +public class DeleteWebhookCmd extends BaseCmd { + + @Inject + WebhookApiService webhookApiService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = WebhookResponse.class, + required = true, + description = "The ID of the Webhook") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + public Long getId() { + return id; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public void execute() throws ServerApiException { + try { + if (!webhookApiService.deleteWebhook(this)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to delete webhook ID: %d", getId())); + } + SuccessResponse response = new SuccessResponse(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/DeleteWebhookDeliveryCmd.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/DeleteWebhookDeliveryCmd.java new file mode 100644 index 000000000000..dcfe71bf1713 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/DeleteWebhookDeliveryCmd.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + +import java.util.Date; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.apache.cloudstack.mom.webhook.WebhookDelivery; +import org.apache.cloudstack.mom.webhook.api.response.WebhookDeliveryResponse; +import org.apache.cloudstack.mom.webhook.api.response.WebhookResponse; + +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "deleteWebhookDelivery", + description = "Deletes Webhook delivery", + responseObject = SuccessResponse.class, + entityType = {WebhookDelivery.class}, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}, + since = "4.20.0") +public class DeleteWebhookDeliveryCmd extends BaseCmd { + + @Inject + WebhookApiService webhookApiService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, + entityType = WebhookDeliveryResponse.class, + description = "The ID of the Webhook delivery") + private Long id; + + @Parameter(name = ApiConstants.WEBHOOK_ID, type = BaseCmd.CommandType.UUID, + entityType = WebhookResponse.class, + description = "The ID of the Webhook") + private Long webhookId; + + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = BaseCmd.CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "The ID of the management server", + authorized = {RoleType.Admin}) + private Long managementServerId; + + @Parameter(name = ApiConstants.START_DATE, + type = CommandType.DATE, + description = "The start date range for the Webhook delivery " + + "(use format \"yyyy-MM-dd\" or \"yyyy-MM-dd HH:mm:ss\"). " + + "All deliveries having start date equal to or after the specified date will be considered.") + private Date startDate; + + @Parameter(name = ApiConstants.END_DATE, + type = CommandType.DATE, + description = "The end date range for the Webhook delivery " + + "(use format \"yyyy-MM-dd\" or \"yyyy-MM-dd HH:mm:ss\"). " + + "All deliveries having end date equal to or before the specified date will be considered.") + private Date endDate; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + public Long getId() { + return id; + } + + public Long getWebhookId() { + return webhookId; + } + + public Long getManagementServerId() { + return managementServerId; + } + + public Date getStartDate() { + return startDate; + } + + public Date getEndDate() { + return endDate; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public void execute() throws ServerApiException { + try { + webhookApiService.deleteWebhookDelivery(this); + SuccessResponse response = new SuccessResponse(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/ExecuteWebhookDeliveryCmd.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/ExecuteWebhookDeliveryCmd.java new file mode 100644 index 000000000000..f31a5481376d --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/ExecuteWebhookDeliveryCmd.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.apache.cloudstack.mom.webhook.WebhookDelivery; +import org.apache.cloudstack.mom.webhook.api.response.WebhookDeliveryResponse; +import org.apache.cloudstack.mom.webhook.api.response.WebhookResponse; + +import com.cloud.utils.exception.CloudRuntimeException; + + +@APICommand(name = "executeWebhookDelivery", + description = "Executes a Webhook delivery", + responseObject = WebhookDeliveryResponse.class, + entityType = {WebhookDelivery.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}, + since = "4.20.0") +public class ExecuteWebhookDeliveryCmd extends BaseCmd { + + @Inject + WebhookApiService webhookApiService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = WebhookDeliveryResponse.class, + description = "The ID of the Webhook delivery for redelivery") + private Long id; + + @Parameter(name = ApiConstants.WEBHOOK_ID, type = CommandType.UUID, + entityType = WebhookResponse.class, + description = "The ID of the Webhook") + private Long webhookId; + + @Parameter(name = ApiConstants.PAYLOAD_URL, + type = BaseCmd.CommandType.STRING, + description = "Payload URL of the Webhook delivery") + private String payloadUrl; + + @Parameter(name = ApiConstants.SECRET_KEY, type = BaseCmd.CommandType.STRING, description = "Secret key of the Webhook delivery") + private String secretKey; + + @Parameter(name = ApiConstants.SSL_VERIFICATION, type = BaseCmd.CommandType.BOOLEAN, description = "If set to true then SSL verification will be done for the Webhook delivery otherwise not") + private Boolean sslVerification; + + @Parameter(name = ApiConstants.PAYLOAD, + type = BaseCmd.CommandType.STRING, + description = "Payload of the Webhook delivery") + private String payload; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + + public Long getId() { + return id; + } + + public Long getWebhookId() { + return webhookId; + } + + public String getPayloadUrl() { + return payloadUrl; + } + + public String getSecretKey() { + return secretKey; + } + + public Boolean isSslVerification() { + return sslVerification; + } + + public String getPayload() { + return payload; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException { + try { + WebhookDeliveryResponse response = webhookApiService.executeWebhookDelivery(this); + if (response == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to test Webhook delivery"); + } + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/ListWebhookDeliveriesCmd.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/ListWebhookDeliveriesCmd.java new file mode 100644 index 000000000000..466dad0d1224 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/ListWebhookDeliveriesCmd.java @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + +import java.util.Date; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.apache.cloudstack.mom.webhook.WebhookDelivery; +import org.apache.cloudstack.mom.webhook.api.response.WebhookDeliveryResponse; +import org.apache.cloudstack.mom.webhook.api.response.WebhookResponse; + +@APICommand(name = "listWebhookDeliveries", + description = "Lists Webhook deliveries", + responseObject = WebhookResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {WebhookDelivery.class}, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}, + since = "4.20.0") +public class ListWebhookDeliveriesCmd extends BaseListCmd { + + @Inject + WebhookApiService webhookApiService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, + entityType = WebhookDeliveryResponse.class, + description = "The ID of the Webhook delivery") + private Long id; + + @Parameter(name = ApiConstants.WEBHOOK_ID, type = BaseCmd.CommandType.UUID, + entityType = WebhookResponse.class, + description = "The ID of the Webhook") + private Long webhookId; + + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = BaseCmd.CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "The ID of the management server", + authorized = {RoleType.Admin}) + private Long managementServerId; + + @Parameter(name = ApiConstants.START_DATE, + type = CommandType.DATE, + description = "The start date range for the Webhook delivery " + + "(use format \"yyyy-MM-dd\" or \"yyyy-MM-dd HH:mm:ss\"). " + + "All deliveries having start date equal to or after the specified date will be listed.") + private Date startDate; + + @Parameter(name = ApiConstants.END_DATE, + type = CommandType.DATE, + description = "The end date range for the Webhook delivery " + + "(use format \"yyyy-MM-dd\" or \"yyyy-MM-dd HH:mm:ss\"). " + + "All deliveries having end date equal to or before the specified date will be listed.") + private Date endDate; + + @Parameter(name = ApiConstants.EVENT_TYPE, + type = CommandType.STRING, + description = "The event type of the Webhook delivery") + private String eventType; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + public Long getId() { + return id; + } + + public Long getWebhookId() { + return webhookId; + } + + public Long getManagementServerId() { + return managementServerId; + } + + public Date getStartDate() { + return startDate; + } + + public Date getEndDate() { + return endDate; + } + + public String getEventType() { + return eventType; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public void execute() throws ServerApiException { + ListResponse response = webhookApiService.listWebhookDeliveries(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/ListWebhooksCmd.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/ListWebhooksCmd.java new file mode 100644 index 000000000000..6510c308f6e7 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/ListWebhooksCmd.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.apache.cloudstack.mom.webhook.Webhook; +import org.apache.cloudstack.mom.webhook.api.response.WebhookResponse; + +@APICommand(name = "listWebhooks", + description = "Lists Webhooks", + responseObject = WebhookResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {Webhook.class}, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}, + since = "4.20.0") +public class ListWebhooksCmd extends BaseListProjectAndAccountResourcesCmd { + + @Inject + WebhookApiService webhookApiService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = WebhookResponse.class, + description = "The ID of the Webhook") + private Long id; + + @Parameter(name = ApiConstants.STATE, type = CommandType.STRING, description = "The state of the Webhook") + private String state; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "The name of the Webhook") + private String name; + + @Parameter(name = ApiConstants.SCOPE, + type = CommandType.STRING, + description = "The scope of the Webhook", + authorized = {RoleType.Admin, RoleType.DomainAdmin}) + private String scope; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + public Long getId() { + return id; + } + + public String getState() { + return state; + } + + public String getName() { + return name; + } + + public String getScope() { + return scope; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public void execute() throws ServerApiException { + ListResponse response = webhookApiService.listWebhooks(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/UpdateWebhookCmd.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/UpdateWebhookCmd.java new file mode 100644 index 000000000000..c2be1d3f4fa2 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/command/user/UpdateWebhookCmd.java @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.apache.cloudstack.mom.webhook.Webhook; +import org.apache.cloudstack.mom.webhook.api.response.WebhookResponse; + +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = "updateWebhook", + description = "Updates a Webhook", + responseObject = SuccessResponse.class, + entityType = {Webhook.class}, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}, + since = "4.20.0") +public class UpdateWebhookCmd extends BaseCmd { + + @Inject + WebhookApiService webhookApiService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = WebhookResponse.class, + required = true, + description = "The ID of the Webhook") + private Long id; + @Parameter(name = ApiConstants.NAME, type = BaseCmd.CommandType.STRING, description = "Name for the Webhook") + private String name; + + @Parameter(name = ApiConstants.DESCRIPTION, type = BaseCmd.CommandType.STRING, description = "Description for the Webhook") + private String description; + + @Parameter(name = ApiConstants.STATE, type = BaseCmd.CommandType.STRING, description = "State of the Webhook") + private String state; + + @Parameter(name = ApiConstants.PAYLOAD_URL, + type = BaseCmd.CommandType.STRING, + description = "Payload URL of the Webhook") + private String payloadUrl; + + @Parameter(name = ApiConstants.SECRET_KEY, type = BaseCmd.CommandType.STRING, description = "Secret key of the Webhook") + private String secretKey; + + @Parameter(name = ApiConstants.SSL_VERIFICATION, type = BaseCmd.CommandType.BOOLEAN, description = "If set to true then SSL verification will be done for the Webhook otherwise not") + private Boolean sslVerification; + + @Parameter(name = ApiConstants.SCOPE, type = BaseCmd.CommandType.STRING, description = "Scope of the Webhook", + authorized = {RoleType.Admin, RoleType.DomainAdmin}) + private String scope; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + public Long getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getState() { + return state; + } + + public String getPayloadUrl() { + return payloadUrl; + } + + public String getSecretKey() { + return secretKey; + } + + public Boolean isSslVerification() { + return sslVerification; + } + + public String getScope() { + return scope; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public void execute() throws ServerApiException { + try { + WebhookResponse response = webhookApiService.updateWebhook(this); + if (response == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update Webhook"); + } + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/response/WebhookDeliveryResponse.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/response/WebhookDeliveryResponse.java new file mode 100644 index 000000000000..6463fe9b48b7 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/response/WebhookDeliveryResponse.java @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.response; + + +import java.util.Date; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.mom.webhook.WebhookDelivery; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +@EntityReference(value = {WebhookDelivery.class}) +public class WebhookDeliveryResponse extends BaseResponse { + @SerializedName(ApiConstants.ID) + @Param(description = "The ID of the Webhook delivery") + private String id; + + @SerializedName(ApiConstants.EVENT_ID) + @Param(description = "The ID of the event") + private String eventId; + + @SerializedName(ApiConstants.EVENT_TYPE) + @Param(description = "The type of the event") + private String eventType; + + @SerializedName(ApiConstants.WEBHOOK_ID) + @Param(description = "The ID of the Webhook") + private String webhookId; + + @SerializedName(ApiConstants.WEBHOOK_NAME) + @Param(description = "The name of the Webhook") + private String webhookName; + + @SerializedName(ApiConstants.MANAGEMENT_SERVER_ID) + @Param(description = "The ID of the management server which executed delivery") + private String managementServerId; + + @SerializedName(ApiConstants.MANAGEMENT_SERVER_NAME) + @Param(description = "The name of the management server which executed delivery") + private String managementServerName; + + @SerializedName(ApiConstants.HEADERS) + @Param(description = "The headers of the webhook delivery") + private String headers; + + @SerializedName(ApiConstants.PAYLOAD) + @Param(description = "The payload of the webhook delivery") + private String payload; + + @SerializedName(ApiConstants.SUCCESS) + @Param(description = "Whether Webhook delivery succeeded or not") + private boolean success; + + @SerializedName(ApiConstants.RESPONSE) + @Param(description = "The response of the webhook delivery") + private String response; + + @SerializedName(ApiConstants.START_DATE) + @Param(description = "The start time of the Webhook delivery") + private Date startTime; + + @SerializedName(ApiConstants.END_DATE) + @Param(description = "The end time of the Webhook delivery") + private Date endTime; + + public void setId(String id) { + this.id = id; + } + + public void setEventId(String eventId) { + this.eventId = eventId; + } + + public void setEventType(String eventType) { + this.eventType = eventType; + } + + public void setWebhookId(String webhookId) { + this.webhookId = webhookId; + } + + public void setWebhookName(String webhookName) { + this.webhookName = webhookName; + } + + public void setManagementServerId(String managementServerId) { + this.managementServerId = managementServerId; + } + + public void setManagementServerName(String managementServerName) { + this.managementServerName = managementServerName; + } + + public void setHeaders(String headers) { + this.headers = headers; + } + + public void setPayload(String payload) { + this.payload = payload; + } + + public void setSuccess(boolean success) { + this.success = success; + } + + public void setResponse(String response) { + this.response = response; + } + + public void setStartTime(Date startTime) { + this.startTime = startTime; + } + + public void setEndTime(Date endTime) { + this.endTime = endTime; + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/response/WebhookResponse.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/response/WebhookResponse.java new file mode 100644 index 000000000000..161b8c5796af --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/api/response/WebhookResponse.java @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.response; + +import java.util.Date; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.api.response.ControlledViewEntityResponse; +import org.apache.cloudstack.mom.webhook.Webhook; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +@EntityReference(value = {Webhook.class}) +public class WebhookResponse extends BaseResponse implements ControlledViewEntityResponse { + @SerializedName(ApiConstants.ID) + @Param(description = "The ID of the Webhook") + private String id; + + @SerializedName(ApiConstants.NAME) + @Param(description = "The name of the Webhook") + private String name; + + @SerializedName(ApiConstants.DESCRIPTION) + @Param(description = "The description of the Webhook") + private String description; + + @SerializedName(ApiConstants.STATE) + @Param(description = "The state of the Webhook") + private String state; + + @SerializedName(ApiConstants.DOMAIN_ID) + @Param(description = "The ID of the domain in which the Webhook exists") + private String domainId; + + @SerializedName(ApiConstants.DOMAIN) + @Param(description = "The name of the domain in which the Webhook exists") + private String domainName; + + @SerializedName(ApiConstants.ACCOUNT) + @Param(description = "The account associated with the Webhook") + private String accountName; + + @SerializedName(ApiConstants.PROJECT_ID) + @Param(description = "the project id of the Kubernetes cluster") + private String projectId; + + @SerializedName(ApiConstants.PROJECT) + @Param(description = "the project name of the Kubernetes cluster") + private String projectName; + + @SerializedName(ApiConstants.PAYLOAD_URL) + @Param(description = "The payload URL end point for the Webhook") + private String payloadUrl; + + @SerializedName(ApiConstants.SECRET_KEY) + @Param(description = "The secret key for the Webhook") + private String secretKey; + + @SerializedName(ApiConstants.SSL_VERIFICATION) + @Param(description = "Whether SSL verification is enabled for the Webhook") + private boolean sslVerification; + + @SerializedName(ApiConstants.SCOPE) + @Param(description = "The scope of the Webhook") + private String scope; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "The date when this Webhook was created") + private Date created; + + public void setId(String id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setState(String state) { + this.state = state; + } + + @Override + public void setDomainId(String domainId) { + this.domainId = domainId; + } + + @Override + public void setDomainName(String domainName) { + this.domainName = domainName; + } + + @Override + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + @Override + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + @Override + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public void setPayloadUrl(String payloadUrl) { + this.payloadUrl = payloadUrl; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public void setSslVerification(boolean sslVerification) { + this.sslVerification = sslVerification; + } + + public void setScope(String scope) { + this.scope = scope; + } + + public void setCreated(Date created) { + this.created = created; + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDao.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDao.java new file mode 100644 index 000000000000..d26e5db7dbad --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.dao; + +import java.util.List; + +import org.apache.cloudstack.mom.webhook.vo.WebhookVO; + +import com.cloud.utils.db.GenericDao; + +public interface WebhookDao extends GenericDao { + List listByEnabledForDelivery(Long accountId, List domainIds); + void deleteByAccount(long accountId); + List listByAccount(long accountId); + WebhookVO findByAccountAndPayloadUrl(long accountId, String payloadUrl); +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDaoImpl.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDaoImpl.java new file mode 100644 index 000000000000..2ef2269a9b9c --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDaoImpl.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.dao; + +import java.util.List; +import java.util.Map; + +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.mom.webhook.Webhook; +import org.apache.cloudstack.mom.webhook.vo.WebhookVO; +import org.apache.commons.collections.CollectionUtils; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +public class WebhookDaoImpl extends GenericDaoBase implements WebhookDao { + SearchBuilder accountIdSearch; + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + + accountIdSearch = createSearchBuilder(); + accountIdSearch.and("accountId", accountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + + return true; + } + @Override + public List listByEnabledForDelivery(Long accountId, List domainIds) { + SearchBuilder sb = createSearchBuilder(); + sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ); + sb.and().op("scopeGlobal", sb.entity().getScope(), SearchCriteria.Op.EQ); + if (accountId != null) { + sb.or().op("scopeLocal", sb.entity().getScope(), SearchCriteria.Op.EQ); + sb.and("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ); + sb.cp(); + } + if (CollectionUtils.isNotEmpty(domainIds)) { + sb.or().op("scopeDomain", sb.entity().getScope(), SearchCriteria.Op.EQ); + sb.and("domainId", sb.entity().getDomainId(), SearchCriteria.Op.IN); + sb.cp(); + } + sb.cp(); + SearchCriteria sc = sb.create(); + sc.setParameters("state", Webhook.State.Enabled.name()); + sc.setParameters("scopeGlobal", Webhook.Scope.Global.name()); + if (accountId != null) { + sc.setParameters("scopeLocal", Webhook.Scope.Local.name()); + sc.setParameters("accountId", accountId); + } + if (CollectionUtils.isNotEmpty(domainIds)) { + sc.setParameters("scopeDomain", Webhook.Scope.Domain.name()); + sc.setParameters("domainId", domainIds.toArray()); + } + return listBy(sc); + } + + @Override + public void deleteByAccount(long accountId) { + SearchCriteria sc = accountIdSearch.create(); + sc.setParameters("accountId", accountId); + remove(sc); + } + + @Override + public List listByAccount(long accountId) { + SearchCriteria sc = accountIdSearch.create(); + sc.setParameters("accountId", accountId); + return listBy(sc); + } + + @Override + public WebhookVO findByAccountAndPayloadUrl(long accountId, String payloadUrl) { + SearchBuilder sb = createSearchBuilder(); + sb.and("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ); + sb.and("payloadUrl", sb.entity().getPayloadUrl(), SearchCriteria.Op.EQ); + SearchCriteria sc = sb.create(); + sc.setParameters("accountId", accountId); + sc.setParameters("payloadUrl", payloadUrl); + return findOneBy(sc); + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDeliveryDao.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDeliveryDao.java new file mode 100644 index 000000000000..0fe76d2904e2 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDeliveryDao.java @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.dao; + +import java.util.Date; + +import org.apache.cloudstack.mom.webhook.vo.WebhookDeliveryVO; + +import com.cloud.utils.db.GenericDao; + +public interface WebhookDeliveryDao extends GenericDao { + int deleteByDeleteApiParams(Long id, Long webhookId, Long managementServerId, Date startDate, Date endDate); + void removeOlderDeliveries(long webhookId, long limit); +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDeliveryDaoImpl.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDeliveryDaoImpl.java new file mode 100644 index 000000000000..088ed53772a3 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDeliveryDaoImpl.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.dao; + +import java.util.Date; +import java.util.List; + +import org.apache.cloudstack.mom.webhook.vo.WebhookDeliveryVO; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +public class WebhookDeliveryDaoImpl extends GenericDaoBase implements WebhookDeliveryDao { + @Override + public int deleteByDeleteApiParams(Long id, Long webhookId, Long managementServerId, Date startDate, + Date endDate) { + SearchBuilder sb = createSearchBuilder(); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("webhookId", sb.entity().getWebhookId(), SearchCriteria.Op.EQ); + sb.and("managementServerId", sb.entity().getManagementServerId(), SearchCriteria.Op.EQ); + sb.and("startDate", sb.entity().getStartTime(), SearchCriteria.Op.GTEQ); + sb.and("endDate", sb.entity().getEndTime(), SearchCriteria.Op.LTEQ); + SearchCriteria sc = sb.create(); + if (id != null) { + sc.setParameters("id", id); + } + if (webhookId != null) { + sc.setParameters("webhookId", webhookId); + } + if (managementServerId != null) { + sc.setParameters("managementServerId", managementServerId); + } + if (startDate != null) { + sc.setParameters("startDate", startDate); + } + if (endDate != null) { + sc.setParameters("endDate", endDate); + } + return remove(sc); + } + + @Override + public void removeOlderDeliveries(long webhookId, long limit) { + Filter searchFilter = new Filter(WebhookDeliveryVO.class, "id", false, 0L, limit); + SearchBuilder sb = createSearchBuilder(); + sb.and("webhookId", sb.entity().getWebhookId(), SearchCriteria.Op.EQ); + SearchCriteria sc = sb.create(); + sc.setParameters("webhookId", webhookId); + List keep = listBy(sc, searchFilter); + SearchBuilder sbDelete = createSearchBuilder(); + sbDelete.and("id", sbDelete.entity().getId(), SearchCriteria.Op.NOTIN); + SearchCriteria scDelete = sbDelete.create(); + scDelete.setParameters("id", keep.stream().map(WebhookDeliveryVO::getId).toArray()); + remove(scDelete); + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDeliveryJoinDao.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDeliveryJoinDao.java new file mode 100644 index 000000000000..70fec4d7cbfc --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDeliveryJoinDao.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.dao; + +import java.util.Date; +import java.util.List; + +import org.apache.cloudstack.mom.webhook.vo.WebhookDeliveryJoinVO; + +import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDao; + +public interface WebhookDeliveryJoinDao extends GenericDao { + Pair, Integer> searchAndCountByListApiParameters(Long id, + List webhookIds, Long managementServerId, final String keyword, final Date startDate, + final Date endDate, final String eventType, Filter searchFilter); +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDeliveryJoinDaoImpl.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDeliveryJoinDaoImpl.java new file mode 100644 index 000000000000..db84010fbc47 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookDeliveryJoinDaoImpl.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.dao; + +import java.util.Date; +import java.util.List; + +import org.apache.cloudstack.mom.webhook.vo.WebhookDeliveryJoinVO; +import org.apache.commons.collections.CollectionUtils; + +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +public class WebhookDeliveryJoinDaoImpl extends GenericDaoBase + implements WebhookDeliveryJoinDao { + @Override + public Pair, Integer> searchAndCountByListApiParameters(Long id, + List webhookIds, Long managementServerId, String keyword, final Date startDate, + final Date endDate, final String eventType, Filter searchFilter) { + SearchBuilder sb = createSearchBuilder(); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("webhookId", sb.entity().getWebhookId(), SearchCriteria.Op.IN); + sb.and("managementServerId", sb.entity().getManagementServerMsId(), SearchCriteria.Op.EQ); + sb.and("keyword", sb.entity().getPayload(), SearchCriteria.Op.LIKE); + sb.and("startDate", sb.entity().getStartTime(), SearchCriteria.Op.GTEQ); + sb.and("endDate", sb.entity().getEndTime(), SearchCriteria.Op.LTEQ); + sb.and("eventType", sb.entity().getEventType(), SearchCriteria.Op.EQ); + SearchCriteria sc = sb.create(); + if (id != null) { + sc.setParameters("id", id); + } + if (CollectionUtils.isNotEmpty(webhookIds)) { + sc.setParameters("webhookId", webhookIds.toArray()); + } + if (managementServerId != null) { + sc.setParameters("managementServerId", managementServerId); + } + if (keyword != null) { + sc.setParameters("keyword", "%" + keyword + "%"); + } + if (startDate != null) { + sc.setParameters("startDate", startDate); + } + if (endDate != null) { + sc.setParameters("endDate", endDate); + } + if (StringUtils.isNotBlank(eventType)) { + sc.setParameters("eventType", eventType); + } + return searchAndCount(sc, searchFilter); + } +} diff --git a/server/src/main/java/com/cloud/api/query/dao/HostTagDao.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookJoinDao.java similarity index 73% rename from server/src/main/java/com/cloud/api/query/dao/HostTagDao.java rename to plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookJoinDao.java index ab43e71221cf..87b4871a14b4 100644 --- a/server/src/main/java/com/cloud/api/query/dao/HostTagDao.java +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookJoinDao.java @@ -14,17 +14,15 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.api.query.dao; + +package org.apache.cloudstack.mom.webhook.dao; import java.util.List; -import org.apache.cloudstack.api.response.HostTagResponse; +import org.apache.cloudstack.mom.webhook.vo.WebhookJoinVO; -import com.cloud.api.query.vo.HostTagVO; import com.cloud.utils.db.GenericDao; -public interface HostTagDao extends GenericDao { - HostTagResponse newHostTagResponse(HostTagVO hostTag); - - List searchByIds(Long... hostTagIds); +public interface WebhookJoinDao extends GenericDao { + List listByAccountOrDomain(long accountId, String domainPath); } diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookJoinDaoImpl.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookJoinDaoImpl.java new file mode 100644 index 000000000000..986e8bc2f199 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/dao/WebhookJoinDaoImpl.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.dao; + +import java.util.List; + +import org.apache.cloudstack.mom.webhook.vo.WebhookJoinVO; +import org.apache.commons.lang3.StringUtils; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +public class WebhookJoinDaoImpl extends GenericDaoBase implements WebhookJoinDao { + @Override + public List listByAccountOrDomain(long accountId, String domainPath) { + SearchBuilder sb = createSearchBuilder(); + sb.and().op("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ); + if (StringUtils.isNotBlank(domainPath)) { + sb.or("domainPath", sb.entity().getDomainPath(), SearchCriteria.Op.LIKE); + } + sb.cp(); + SearchCriteria sc = sb.create(); + sc.setParameters("accountId", accountId); + if (StringUtils.isNotBlank(domainPath)) { + sc.setParameters("domainPath", domainPath); + } + return listBy(sc); + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/vo/WebhookDeliveryJoinVO.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/vo/WebhookDeliveryJoinVO.java new file mode 100644 index 000000000000..e36f870c8d95 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/vo/WebhookDeliveryJoinVO.java @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.vo; + + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import com.cloud.api.query.vo.BaseViewVO; + +@Entity +@Table(name = "webhook_delivery_view") +public class WebhookDeliveryJoinVO extends BaseViewVO implements InternalIdentity, Identity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "event_id") + private long eventId; + + @Column(name = "event_uuid") + private String eventUuid; + + @Column(name = "event_type") + private String eventType; + + @Column(name = "webhook_id") + private long webhookId; + + @Column(name = "webhook_uuid") + private String webhookUuId; + + @Column(name = "webhook_name") + private String webhookName; + + @Column(name = "mshost_id") + private long managementServerId; + + @Column(name = "mshost_uuid") + private String managementServerUuId; + + @Column(name = "mshost_msid") + private long managementServerMsId; + + @Column(name = "mshost_name") + private String managementServerName; + + @Column(name = "headers", length = 65535) + private String headers; + + @Column(name = "payload", length = 65535) + private String payload; + + @Column(name = "success") + private boolean success; + + @Column(name = "response", length = 65535) + private String response; + + @Column(name = "start_time") + @Temporal(value = TemporalType.TIMESTAMP) + private Date startTime; + + @Column(name = "end_time") + @Temporal(value = TemporalType.TIMESTAMP) + private Date endTime; + + @Override + public long getId() { + return 0; + } + + @Override + public String getUuid() { + return uuid; + } + + public long getEventId() { + return eventId; + } + + public String getEventUuid() { + return eventUuid; + } + + public String getEventType() { + return eventType; + } + + public long getWebhookId() { + return webhookId; + } + + public String getWebhookUuId() { + return webhookUuId; + } + + public String getWebhookName() { + return webhookName; + } + + public long getManagementServerId() { + return managementServerId; + } + + public String getManagementServerUuId() { + return managementServerUuId; + } + + public long getManagementServerMsId() { + return managementServerMsId; + } + + public String getManagementServerName() { + return managementServerName; + } + + public String getHeaders() { + return headers; + } + + public String getPayload() { + return payload; + } + + public boolean isSuccess() { + return success; + } + + public String getResponse() { + return response; + } + + public Date getStartTime() { + return startTime; + } + + public Date getEndTime() { + return endTime; + } + + @Override + public String toString() { + return String.format("WebhookDelivery [%s]", ReflectionToStringBuilderUtils.reflectOnlySelectedFields( + this, "id", "uuid", "webhookId", "startTime", "success")); + } + + public WebhookDeliveryJoinVO() { + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/vo/WebhookDeliveryVO.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/vo/WebhookDeliveryVO.java new file mode 100644 index 000000000000..e39f57a26637 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/vo/WebhookDeliveryVO.java @@ -0,0 +1,174 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.vo; + + +import java.util.Date; +import java.util.UUID; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import org.apache.cloudstack.mom.webhook.WebhookDelivery; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +@Entity +@Table(name = "webhook_delivery") +public class WebhookDeliveryVO implements WebhookDelivery { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "event_id") + private long eventId; + + @Column(name = "webhook_id") + private long webhookId; + + @Column(name = "mshost_msid") + private long mangementServerId; + + @Column(name = "headers", length = 65535) + private String headers; + + @Column(name = "payload", length = 65535) + private String payload; + + @Column(name = "success") + private boolean success; + + @Column(name = "response", length = 65535) + private String response; + + @Column(name = "start_time") + @Temporal(value = TemporalType.TIMESTAMP) + private Date startTime; + + @Column(name = "end_time") + @Temporal(value = TemporalType.TIMESTAMP) + private Date endTime; + + @Override + public long getId() { + return id; + } + + @Override + public String getUuid() { + return uuid; + } + + @Override + public long getEventId() { + return eventId; + } + + @Override + public long getWebhookId() { + return webhookId; + } + + @Override + public long getManagementServerId() { + return mangementServerId; + } + + public String getHeaders() { + return headers; + } + + @Override + public String getPayload() { + return payload; + } + + @Override + public boolean isSuccess() { + return success; + } + + @Override + public String getResponse() { + return response; + } + + @Override + public Date getStartTime() { + return startTime; + } + + @Override + public Date getEndTime() { + return endTime; + } + + @Override + public String toString() { + return String.format("WebhookDelivery [%s]", ReflectionToStringBuilderUtils.reflectOnlySelectedFields( + this, "id", "uuid", "webhookId", "startTime", "success")); + } + + public WebhookDeliveryVO() { + this.uuid = UUID.randomUUID().toString(); + } + + public WebhookDeliveryVO(long eventId, long webhookId, long managementServerId, String headers, String payload, + boolean success, String response, Date startTime, Date endTime) { + this.uuid = UUID.randomUUID().toString(); + this.eventId = eventId; + this.webhookId = webhookId; + this.mangementServerId = managementServerId; + this.headers = headers; + this.payload = payload; + this.success = success; + this.response = response; + this.startTime = startTime; + this.endTime = endTime; + } + + + + /* + * For creating a dummy object for testing delivery + */ + public WebhookDeliveryVO(long managementServerId, String headers, String payload, boolean success, + String response, Date startTime, Date endTime) { + this.id = WebhookDelivery.ID_DUMMY; + this.uuid = UUID.randomUUID().toString(); + this.eventId = WebhookDelivery.ID_DUMMY; + this.webhookId = WebhookDelivery.ID_DUMMY; + this.mangementServerId = managementServerId; + this.headers = headers; + this.payload = payload; + this.success = success; + this.response = response; + this.startTime = startTime; + this.endTime = endTime; + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/vo/WebhookJoinVO.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/vo/WebhookJoinVO.java new file mode 100644 index 000000000000..f17086095871 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/vo/WebhookJoinVO.java @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.vo; + + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.mom.webhook.Webhook; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import com.cloud.api.query.vo.ControlledViewEntity; +import com.cloud.user.Account; +import com.cloud.utils.db.Encrypt; +import com.cloud.utils.db.GenericDao; + +@Entity +@Table(name = "webhook_view") +public class WebhookJoinVO implements ControlledViewEntity { + + @Id + @Column(name = "id", updatable = false, nullable = false) + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "name") + private String name; + + @Column(name = "description", length = 4096) + private String description; + + @Column(name = "state") + @Enumerated(value = EnumType.STRING) + private Webhook.State state; + + @Column(name = "payload_url") + private String payloadUrl; + + @Column(name = "secret_key") + @Encrypt + private String secretKey; + + @Column(name = "ssl_verification") + private boolean sslVerification; + + @Column(name = "scope") + @Enumerated(value = EnumType.STRING) + private Webhook.Scope scope; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + + @Column(name = "account_id") + private long accountId; + + @Column(name = "account_uuid") + private String accountUuid; + + @Column(name = "account_name") + private String accountName; + + @Column(name = "account_type") + @Enumerated(value = EnumType.STRING) + private Account.Type accountType; + + @Column(name = "domain_id") + private long domainId; + + @Column(name = "domain_uuid") + private String domainUuid; + + @Column(name = "domain_name") + private String domainName; + + @Column(name = "domain_path") + private String domainPath; + + @Column(name = "project_id") + private long projectId; + + @Column(name = "project_uuid") + private String projectUuid; + + @Column(name = "project_name") + private String projectName; + + @Override + public long getId() { + return id; + } + + @Override + public String getUuid() { + return uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Webhook.State getState() { + return state; + } + + public String getPayloadUrl() { + return payloadUrl; + } + + public void setPayloadUrl(String payloadUrl) { + this.payloadUrl = payloadUrl; + } + + public String getSecretKey() { + return secretKey; + } + + public Webhook.Scope getScope() { + return scope; + } + + public boolean isSslVerification() { + return sslVerification; + } + + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + @Override + public long getDomainId() { + return domainId; + } + + @Override + public String getDomainPath() { + return domainPath; + } + + @Override + public String getDomainUuid() { + return domainUuid; + } + + @Override + public String getDomainName() { + return domainName; + } + + @Override + public Account.Type getAccountType() { + return accountType; + } + + @Override + public long getAccountId() { + return accountId; + } + + @Override + public String getAccountUuid() { + return accountUuid; + } + + @Override + public String getAccountName() { + return accountName; + } + + @Override + public String getProjectUuid() { + return projectUuid; + } + + @Override + public String getProjectName() { + return projectName; + } + + @Override + public Class getEntityType() { + return Webhook.class; + } + + @Override + public String toString() { + return String.format("Webhook [%s]", ReflectionToStringBuilderUtils.reflectOnlySelectedFields( + this, "id", "uuid", "name")); + } + + public WebhookJoinVO() { + } +} diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/vo/WebhookVO.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/vo/WebhookVO.java new file mode 100644 index 000000000000..93e3e801423e --- /dev/null +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/vo/WebhookVO.java @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.vo; + + +import java.util.Date; +import java.util.UUID; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.mom.webhook.Webhook; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import com.cloud.utils.db.Encrypt; +import com.cloud.utils.db.GenericDao; + +@Entity +@Table(name = "webhook") +public class WebhookVO implements Webhook { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "name") + private String name; + + @Column(name = "description", length = 4096) + private String description; + + @Column(name = "state") + @Enumerated(value = EnumType.STRING) + private State state; + + @Column(name = "domain_id") + private long domainId; + + @Column(name = "account_id") + private long accountId; + + @Column(name = "payload_url") + private String payloadUrl; + + @Column(name = "secret_key") + @Encrypt + private String secretKey; + + @Column(name = "ssl_verification") + private boolean sslVerification; + + @Column(name = "scope") + @Enumerated(value = EnumType.STRING) + private Scope scope; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + + @Override + public long getId() { + return id; + } + + @Override + public String getUuid() { + return uuid; + } + + @Override + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Override + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + @Override + public State getState() { + return state; + } + + public void setState(State state) { + this.state = state; + } + + @Override + public long getDomainId() { + return domainId; + } + + public void setDomainId(long domainId) { + this.domainId = domainId; + } + + @Override + public long getAccountId() { + return accountId; + } + + public void setAccountId(long accountId) { + this.accountId = accountId; + } + + @Override + public String getPayloadUrl() { + return payloadUrl; + } + + public void setPayloadUrl(String payloadUrl) { + this.payloadUrl = payloadUrl; + } + + @Override + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + @Override + public Scope getScope() { + return scope; + } + + public void setScope(Scope scope) { + this.scope = scope; + } + + @Override + public boolean isSslVerification() { + return sslVerification; + } + + public void setSslVerification(boolean sslVerification) { + this.sslVerification = sslVerification; + } + + @Override + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + @Override + public Class getEntityType() { + return Webhook.class; + } + + @Override + public String toString() { + return String.format("Webhook [%s]",ReflectionToStringBuilderUtils.reflectOnlySelectedFields( + this, "id", "uuid", "name", "payloadUrl")); + } + + public WebhookVO() { + this.uuid = UUID.randomUUID().toString(); + } + + public WebhookVO(String name, String description, State state, long domainId, long accountId, + String payloadUrl, String secretKey, boolean sslVerification, Scope scope) { + this.uuid = UUID.randomUUID().toString(); + this.name = name; + this.description = description; + this.state = state; + this.domainId = domainId; + this.accountId = accountId; + this.payloadUrl = payloadUrl; + this.secretKey = secretKey; + this.sslVerification = sslVerification; + this.scope = scope; + } + + /* + * For creating a dummy rule for testing delivery + */ + public WebhookVO(long domainId, long accountId, String payloadUrl, String secretKey, boolean sslVerification) { + this.uuid = UUID.randomUUID().toString(); + this.id = ID_DUMMY; + this.name = NAME_DUMMY; + this.description = NAME_DUMMY; + this.state = State.Enabled; + this.domainId = domainId; + this.accountId = accountId; + this.payloadUrl = payloadUrl; + this.secretKey = secretKey; + this.sslVerification = sslVerification; + this.scope = Scope.Local; + } +} diff --git a/plugins/event-bus/webhook/src/main/resources/META-INF/cloudstack/webhook/module.properties b/plugins/event-bus/webhook/src/main/resources/META-INF/cloudstack/webhook/module.properties new file mode 100644 index 000000000000..299144ff82a2 --- /dev/null +++ b/plugins/event-bus/webhook/src/main/resources/META-INF/cloudstack/webhook/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=webhook +parent=event diff --git a/plugins/event-bus/webhook/src/main/resources/META-INF/cloudstack/webhook/spring-event-webhook-context.xml b/plugins/event-bus/webhook/src/main/resources/META-INF/cloudstack/webhook/spring-event-webhook-context.xml new file mode 100644 index 000000000000..22f688c781fe --- /dev/null +++ b/plugins/event-bus/webhook/src/main/resources/META-INF/cloudstack/webhook/spring-event-webhook-context.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + diff --git a/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/WebhookApiServiceImplTest.java b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/WebhookApiServiceImplTest.java new file mode 100644 index 000000000000..dff358069840 --- /dev/null +++ b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/WebhookApiServiceImplTest.java @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.mom.webhook; + +import java.util.List; + +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.mom.webhook.api.command.user.DeleteWebhookCmd; +import org.apache.cloudstack.mom.webhook.api.response.WebhookResponse; +import org.apache.cloudstack.mom.webhook.dao.WebhookDao; +import org.apache.cloudstack.mom.webhook.dao.WebhookJoinDao; +import org.apache.cloudstack.mom.webhook.vo.WebhookJoinVO; +import org.apache.cloudstack.mom.webhook.vo.WebhookVO; +import org.apache.commons.collections.CollectionUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.api.ApiResponseHelper; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; + +@RunWith(MockitoJUnitRunner.class) +public class WebhookApiServiceImplTest { + + @Mock + WebhookDao webhookDao; + @Mock + WebhookJoinDao webhookJoinDao; + @Mock + AccountManager accountManager; + + @Mock + DomainDao domainDao; + + @InjectMocks + WebhookApiServiceImpl webhookApiServiceImpl = new WebhookApiServiceImpl(); + + private WebhookJoinVO prepareTestWebhookJoinVO() { + String name = "webhook"; + String description = "webhook-description"; + Webhook.State state = Webhook.State.Enabled; + String payloadUrl = "url"; + String secretKey = "key"; + boolean sslVerification = false; + Webhook.Scope scope = Webhook.Scope.Local; + WebhookJoinVO webhookJoinVO = new WebhookJoinVO(); + ReflectionTestUtils.setField(webhookJoinVO, "name", name); + ReflectionTestUtils.setField(webhookJoinVO, "description", description); + ReflectionTestUtils.setField(webhookJoinVO, "state", state); + ReflectionTestUtils.setField(webhookJoinVO, "payloadUrl", payloadUrl); + ReflectionTestUtils.setField(webhookJoinVO, "secretKey", secretKey); + ReflectionTestUtils.setField(webhookJoinVO, "sslVerification", sslVerification); + ReflectionTestUtils.setField(webhookJoinVO, "scope", scope); + return webhookJoinVO; + } + + private void validateWebhookResponseWithWebhookJoinVO(WebhookResponse response, WebhookJoinVO webhookJoinVO) { + Assert.assertEquals(webhookJoinVO.getName(), ReflectionTestUtils.getField(response, "name")); + Assert.assertEquals(webhookJoinVO.getDescription(), ReflectionTestUtils.getField(response, "description")); + Assert.assertEquals(webhookJoinVO.getState().toString(), ReflectionTestUtils.getField(response, "state")); + Assert.assertEquals(webhookJoinVO.getPayloadUrl(), ReflectionTestUtils.getField(response, "payloadUrl")); + Assert.assertEquals(webhookJoinVO.getSecretKey(), ReflectionTestUtils.getField(response, "secretKey")); + Assert.assertEquals(webhookJoinVO.isSslVerification(), ReflectionTestUtils.getField(response, "sslVerification")); + Assert.assertEquals(webhookJoinVO.getScope().toString(), ReflectionTestUtils.getField(response, "scope")); + } + + @Test + public void testCreateWebhookResponse() { + WebhookJoinVO webhookJoinVO = prepareTestWebhookJoinVO(); + try (MockedStatic mockedApiResponseHelper = Mockito.mockStatic(ApiResponseHelper.class)) { + WebhookResponse response = webhookApiServiceImpl.createWebhookResponse(webhookJoinVO); + validateWebhookResponseWithWebhookJoinVO(response, webhookJoinVO); + } + } + + @Test + public void testCreateWebhookResponseId() { + WebhookJoinVO webhookJoinVO = prepareTestWebhookJoinVO(); + long id = 1L; + Mockito.when(webhookJoinDao.findById(id)).thenReturn(webhookJoinVO); + try (MockedStatic mockedApiResponseHelper = Mockito.mockStatic(ApiResponseHelper.class)) { + WebhookResponse response = webhookApiServiceImpl.createWebhookResponse(id); + validateWebhookResponseWithWebhookJoinVO(response, webhookJoinVO); + } + } + + @Test + public void testGetIdsOfAccessibleWebhooksAdmin() { + Account account = Mockito.mock(Account.class); + Mockito.when(account.getType()).thenReturn(Account.Type.ADMIN); + Assert.assertTrue(CollectionUtils.isEmpty(webhookApiServiceImpl.getIdsOfAccessibleWebhooks(account))); + } + + @Test + public void testGetIdsOfAccessibleWebhooksDomainAdmin() { + Long accountId = 1L; + Account account = Mockito.mock(Account.class); + Mockito.when(account.getType()).thenReturn(Account.Type.DOMAIN_ADMIN); + Mockito.when(account.getDomainId()).thenReturn(1L); + Mockito.when(account.getId()).thenReturn(accountId); + String domainPath = "d1"; + DomainVO domain = Mockito.mock(DomainVO.class); + Mockito.when(domain.getPath()).thenReturn(domainPath); + Mockito.when(domainDao.findById(1L)).thenReturn(domain); + WebhookJoinVO webhookJoinVO = Mockito.mock(WebhookJoinVO.class); + Mockito.when(webhookJoinVO.getId()).thenReturn(1L); + Mockito.when(webhookJoinDao.listByAccountOrDomain(accountId, domainPath)).thenReturn(List.of(webhookJoinVO)); + List result = webhookApiServiceImpl.getIdsOfAccessibleWebhooks(account); + Assert.assertTrue(CollectionUtils.isNotEmpty(result)); + Assert.assertEquals(1, result.size()); + } + + @Test + public void testGetIdsOfAccessibleWebhooksNormalUser() { + Long accountId = 1L; + Account account = Mockito.mock(Account.class); + Mockito.when(account.getType()).thenReturn(Account.Type.NORMAL); + Mockito.when(account.getId()).thenReturn(accountId); + WebhookJoinVO webhookJoinVO = Mockito.mock(WebhookJoinVO.class); + Mockito.when(webhookJoinVO.getId()).thenReturn(1L); + Mockito.when(webhookJoinDao.listByAccountOrDomain(accountId, null)).thenReturn(List.of(webhookJoinVO)); + List result = webhookApiServiceImpl.getIdsOfAccessibleWebhooks(account); + Assert.assertTrue(CollectionUtils.isNotEmpty(result)); + Assert.assertEquals(1, result.size()); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDeleteWebhookInvalidWebhook() { + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + DeleteWebhookCmd cmd = Mockito.mock(DeleteWebhookCmd.class); + Mockito.when(cmd.getId()).thenReturn(1L); + CallContext callContextMock = Mockito.mock(CallContext.class); + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + webhookApiServiceImpl.deleteWebhook(cmd); + } + } + + @Test(expected = PermissionDeniedException.class) + public void testDeleteWebhookNoPermission() { + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + DeleteWebhookCmd cmd = Mockito.mock(DeleteWebhookCmd.class); + Mockito.when(cmd.getId()).thenReturn(1L); + WebhookVO webhookVO = Mockito.mock(WebhookVO.class); + Mockito.when(webhookDao.findById(1L)).thenReturn(webhookVO); + CallContext callContextMock = Mockito.mock(CallContext.class); + Account account = Mockito.mock(Account.class); + Mockito.when(callContextMock.getCallingAccount()).thenReturn(account); + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + Mockito.doThrow(PermissionDeniedException.class).when(accountManager).checkAccess(account, + SecurityChecker.AccessType.OperateEntry, false, webhookVO); + webhookApiServiceImpl.deleteWebhook(cmd); + } + } + + @Test + public void testDeleteWebhook() { + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + DeleteWebhookCmd cmd = Mockito.mock(DeleteWebhookCmd.class); + Mockito.when(cmd.getId()).thenReturn(1L); + WebhookVO webhookVO = Mockito.mock(WebhookVO.class); + Mockito.when(webhookDao.findById(1L)).thenReturn(webhookVO); + CallContext callContextMock = Mockito.mock(CallContext.class); + Account account = Mockito.mock(Account.class); + Mockito.when(callContextMock.getCallingAccount()).thenReturn(account); + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + Mockito.doNothing().when(accountManager).checkAccess(account, + SecurityChecker.AccessType.OperateEntry, false, webhookVO); + Mockito.doReturn(true).when(webhookDao).remove(Mockito.anyLong()); + Assert.assertTrue(webhookApiServiceImpl.deleteWebhook(cmd)); + } + } + + @Test + public void testValidateWebhookOwnerPayloadUrlNonExistent() { + Mockito.when(webhookDao.findByAccountAndPayloadUrl(Mockito.anyLong(), Mockito.anyString())).thenReturn(null); + Account account = Mockito.mock(Account.class); + String url = "url"; + webhookApiServiceImpl.validateWebhookOwnerPayloadUrl(account, url, null); + webhookApiServiceImpl.validateWebhookOwnerPayloadUrl(account, url, Mockito.mock(Webhook.class)); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateWebhookOwnerPayloadUrlCreateExist() { + Mockito.when(webhookDao.findByAccountAndPayloadUrl(Mockito.anyLong(), Mockito.anyString())) + .thenReturn(Mockito.mock(WebhookVO.class)); + webhookApiServiceImpl.validateWebhookOwnerPayloadUrl(Mockito.mock(Account.class), "url", + null); + } + + private Webhook mockWebhook(long id) { + Webhook webhook = Mockito.mock(Webhook.class); + Mockito.when(webhook.getId()).thenReturn(id); + return webhook; + } + + private WebhookVO mockWebhookVO(long id) { + WebhookVO webhook = Mockito.mock(WebhookVO.class); + Mockito.when(webhook.getId()).thenReturn(id); + return webhook; + } + + @Test + public void testValidateWebhookOwnerPayloadUrlUpdateSameExist() { + WebhookVO webhookVO = mockWebhookVO(1L); + Mockito.when(webhookDao.findByAccountAndPayloadUrl(Mockito.anyLong(), Mockito.anyString())) + .thenReturn(webhookVO); + webhookApiServiceImpl.validateWebhookOwnerPayloadUrl(Mockito.mock(Account.class), "url", + mockWebhook(1L)); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateWebhookOwnerPayloadUrlUpdateDifferentExist() { + WebhookVO webhookVO = mockWebhookVO(2L); + Mockito.when(webhookDao.findByAccountAndPayloadUrl(Mockito.anyLong(), Mockito.anyString())) + .thenReturn(webhookVO); + webhookApiServiceImpl.validateWebhookOwnerPayloadUrl(Mockito.mock(Account.class), "url", + mockWebhook(1L)); + } + + @Test + public void testGetNormalizedPayloadUrl() { + Assert.assertEquals("http://abc.com", webhookApiServiceImpl.getNormalizedPayloadUrl("abc.com")); + Assert.assertEquals("http://abc.com", webhookApiServiceImpl.getNormalizedPayloadUrl("http://abc.com")); + Assert.assertEquals("https://abc.com", + webhookApiServiceImpl.getNormalizedPayloadUrl("https://abc.com")); + } +} diff --git a/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThreadTest.java b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThreadTest.java new file mode 100644 index 000000000000..3be8dee5c2ec --- /dev/null +++ b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThreadTest.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook; + +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +import org.apache.commons.codec.DecoderException; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class WebhookDeliveryThreadTest { + @InjectMocks + WebhookDeliveryThread webhookDeliveryThread; + + @Test + public void testIsValidJson() { + Assert.assertFalse(webhookDeliveryThread.isValidJson("text")); + Assert.assertTrue(webhookDeliveryThread.isValidJson("{ \"CloudStack\": \"works!\" }")); + Assert.assertTrue(webhookDeliveryThread.isValidJson("[{ \"CloudStack\": \"works!\" }]")); + } + + @Test + public void testGenerateHMACSignature() { + String data = "CloudStack works!"; + String key = "Pj4pnwSUBZ4wQFXw2zWdVY1k5Ku9bIy70wCNG1DmS8keO7QapCLw2Axtgc2nEPYzfFCfB38ATNLt6caDqU2dSw"; + String result = "HYLWSII5Ap23WeSaykNsIo6mOhmV3d18s5p2cq2ebCA="; + try { + String sign = WebhookDeliveryThread.generateHMACSignature(data, key); + Assert.assertEquals(result, sign); + } catch (InvalidKeyException | NoSuchAlgorithmException | DecoderException e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void testSetDeliveryTries() { + int tries = 2; + webhookDeliveryThread.setDeliveryTries(tries); + Assert.assertEquals(tries, ReflectionTestUtils.getField(webhookDeliveryThread, "deliveryTries")); + } +} diff --git a/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/WebhookEventBusTest.java b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/WebhookEventBusTest.java new file mode 100644 index 000000000000..ebd3f9e828c2 --- /dev/null +++ b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/WebhookEventBusTest.java @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook; + +import java.util.HashMap; +import java.util.UUID; + +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.framework.events.Event; +import org.apache.cloudstack.framework.events.EventBusException; +import org.apache.cloudstack.framework.events.EventSubscriber; +import org.apache.cloudstack.framework.events.EventTopic; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class WebhookEventBusTest { + + @Mock + WebhookService webhookService; + @InjectMocks + WebhookEventBus eventBus = new WebhookEventBus(); + + @Test + public void testConfigure() { + String name = "name"; + try { + Assert.assertTrue(eventBus.configure(name, new HashMap<>())); + String result = (String)ReflectionTestUtils.getField(eventBus, "_name"); + Assert.assertEquals(name, result); + } catch (ConfigurationException e) { + Assert.fail("Error configuring"); + } + } + + @Test + public void testSetName() { + String name = "name"; + eventBus.setName(name); + String result = (String)ReflectionTestUtils.getField(eventBus, "_name"); + Assert.assertEquals(name, result); + } + + @Test + public void testGetName() { + String name = "name"; + ReflectionTestUtils.setField(eventBus, "_name", name); + Assert.assertEquals(name, eventBus.getName()); + } + + @Test + public void testStart() { + Assert.assertTrue(eventBus.start()); + } + + @Test + public void testStop() { + Assert.assertTrue(eventBus.stop()); + } + + @Test + public void testSubscribe() { + try { + Assert.assertNotNull(eventBus.subscribe(Mockito.mock(EventTopic.class), Mockito.mock(EventSubscriber.class))); + } catch (EventBusException e) { + Assert.fail("Error subscribing"); + } + } + + @Test + public void testUnsubscribe() { + try { + eventBus.unsubscribe(Mockito.mock(UUID.class), Mockito.mock(EventSubscriber.class)); + } catch (EventBusException e) { + Assert.fail("Error unsubscribing"); + } + } + + @Test(expected = EventBusException.class) + public void testPublishException() throws EventBusException { + Mockito.doThrow(EventBusException.class).when(webhookService).handleEvent(Mockito.any(Event.class)); + eventBus.publish(Mockito.mock(Event.class)); + } +} diff --git a/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/CreateWebhookCmdTest.java b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/CreateWebhookCmdTest.java new file mode 100644 index 000000000000..7736a42af04f --- /dev/null +++ b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/CreateWebhookCmdTest.java @@ -0,0 +1,173 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.UUID; + +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.apache.cloudstack.mom.webhook.api.response.WebhookResponse; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.user.Account; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserVO; +import com.cloud.utils.exception.CloudRuntimeException; + +@RunWith(MockitoJUnitRunner.class) +public class CreateWebhookCmdTest { + + @Mock + WebhookApiService webhookApiService; + + private Object getCommandMethodValue(Object obj, String methodName) { + Object result = null; + try { + Method method = obj.getClass().getMethod(methodName); + result = method.invoke(obj); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + Assert.fail(String.format("Failed to get method %s value", methodName)); + } + return result; + } + + private void runStringMemberTest(String memberName) { + String methodName = "get" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + CreateWebhookCmd cmd = new CreateWebhookCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + String value = UUID.randomUUID().toString(); + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + private void runLongMemberTest(String memberName) { + String methodName = "get" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + CreateWebhookCmd cmd = new CreateWebhookCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + Long value = 100L; + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + private void runBooleanMemberTest(String memberName) { + String methodName = "is" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + CreateWebhookCmd cmd = new CreateWebhookCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertFalse((boolean)getCommandMethodValue(cmd, methodName)); + Boolean value = true; + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + @Test + public void testGetName() { + runStringMemberTest("name"); + } + + @Test + public void testGetDescription() { + runStringMemberTest("description"); + } + + @Test + public void testGetPayloadUrl() { + runStringMemberTest("payloadUrl"); + } + + @Test + public void testGetSecretKey() { + runStringMemberTest("secretKey"); + } + + @Test + public void testGetScope() { + runStringMemberTest("scope"); + } + + @Test + public void testGetState() { + runStringMemberTest("state"); + } + + @Test + public void testGetAccount() { + runStringMemberTest("accountName"); + } + + @Test + public void testGetDomainId() { + runLongMemberTest("domainId"); + } + + @Test + public void testGetProjectId() { + runLongMemberTest("projectId"); + } + + @Test + public void testIsSslVerification() { + runBooleanMemberTest("sslVerification"); + } + + @Test + public void testGetEntityOwnerId() { + Account account = new AccountVO("testaccount", 1L, "networkdomain", Account.Type.NORMAL, "uuid"); + UserVO user = new UserVO(1, "testuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + CallContext.register(user, account); + CreateWebhookCmd cmd = new CreateWebhookCmd(); + Assert.assertEquals(account.getId(), cmd.getEntityOwnerId()); + } + + @Test(expected = ServerApiException.class) + public void testExecuteNullResponse() { + CreateWebhookCmd cmd = new CreateWebhookCmd(); + cmd.webhookApiService = webhookApiService; + Mockito.when(webhookApiService.createWebhook(cmd)).thenReturn(null); + cmd.execute(); + } + + @Test(expected = ServerApiException.class) + public void testExecuteCRE() { + CreateWebhookCmd cmd = new CreateWebhookCmd(); + cmd.webhookApiService = webhookApiService; + Mockito.when(webhookApiService.createWebhook(cmd)).thenThrow(CloudRuntimeException.class); + cmd.execute(); + } + + @Test + public void testExecute() { + CreateWebhookCmd cmd = new CreateWebhookCmd(); + cmd.webhookApiService = webhookApiService; + WebhookResponse response = new WebhookResponse(); + Mockito.when(webhookApiService.createWebhook(cmd)).thenReturn(response); + cmd.execute(); + Assert.assertEquals(cmd.getCommandName(), response.getResponseName()); + } +} diff --git a/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/DeleteWebhookCmdTest.java b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/DeleteWebhookCmdTest.java new file mode 100644 index 000000000000..e9aa61aabb83 --- /dev/null +++ b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/DeleteWebhookCmdTest.java @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.UUID; + +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.user.Account; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserVO; +import com.cloud.utils.exception.CloudRuntimeException; + +@RunWith(MockitoJUnitRunner.class) +public class DeleteWebhookCmdTest { + + @Mock + WebhookApiService webhookApiService; + + private Object getCommandMethodValue(Object obj, String methodName) { + Object result = null; + try { + Method method = obj.getClass().getMethod(methodName); + result = method.invoke(obj); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + Assert.fail(String.format("Failed to get method %s value", methodName)); + } + return result; + } + + private void runLongMemberTest(String memberName) { + String methodName = "get" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + DeleteWebhookCmd cmd = new DeleteWebhookCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + Long value = 100L; + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + @Test + public void testGetId() { + runLongMemberTest("id"); + } + + @Test + public void testGetEntityOwnerId() { + Account account = new AccountVO("testaccount", 1L, "networkdomain", Account.Type.NORMAL, "uuid"); + UserVO user = new UserVO(1, "testuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + CallContext.register(user, account); + DeleteWebhookCmd cmd = new DeleteWebhookCmd(); + Assert.assertEquals(account.getId(), cmd.getEntityOwnerId()); + } + + @Test(expected = ServerApiException.class) + public void testExecuteFalseResponse() { + DeleteWebhookCmd cmd = new DeleteWebhookCmd(); + cmd.webhookApiService = webhookApiService; + Mockito.when(webhookApiService.deleteWebhook(cmd)).thenReturn(false); + cmd.execute(); + } + + @Test(expected = ServerApiException.class) + public void testExecuteCRE() { + DeleteWebhookCmd cmd = new DeleteWebhookCmd(); + cmd.webhookApiService = webhookApiService; + Mockito.when(webhookApiService.deleteWebhook(cmd)).thenThrow(CloudRuntimeException.class); + cmd.execute(); + } + + @Test + public void testExecute() { + DeleteWebhookCmd cmd = new DeleteWebhookCmd(); + cmd.webhookApiService = webhookApiService; + Mockito.when(webhookApiService.deleteWebhook(cmd)).thenReturn(true); + cmd.execute(); + Assert.assertNotNull(cmd.getResponseObject()); + } +} diff --git a/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/DeleteWebhookDeliveryCmdTest.java b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/DeleteWebhookDeliveryCmdTest.java new file mode 100644 index 000000000000..2a090eb7fb12 --- /dev/null +++ b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/DeleteWebhookDeliveryCmdTest.java @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.UUID; + +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.user.Account; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserVO; +import com.cloud.utils.exception.CloudRuntimeException; + +@RunWith(MockitoJUnitRunner.class) +public class DeleteWebhookDeliveryCmdTest { + + @Mock + WebhookApiService webhookApiService; + + private Object getCommandMethodValue(Object obj, String methodName) { + Object result = null; + try { + Method method = obj.getClass().getMethod(methodName); + result = method.invoke(obj); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + Assert.fail(String.format("Failed to get method %s value", methodName)); + } + return result; + } + + private void runLongMemberTest(String memberName) { + String methodName = "get" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + DeleteWebhookDeliveryCmd cmd = new DeleteWebhookDeliveryCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + Long value = 100L; + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + @Test + public void testGetId() { + runLongMemberTest("id"); + } + + @Test + public void testGetWebhookId() { + runLongMemberTest("webhookId"); + } + + @Test + public void testGetManagementServerId() { + runLongMemberTest("managementServerId"); + } + + @Test + public void testGetEntityOwnerId() { + Account account = new AccountVO("testaccount", 1L, "networkdomain", Account.Type.NORMAL, "uuid"); + UserVO user = new UserVO(1, "testuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + CallContext.register(user, account); + DeleteWebhookDeliveryCmd cmd = new DeleteWebhookDeliveryCmd(); + Assert.assertEquals(account.getId(), cmd.getEntityOwnerId()); + } + + @Test(expected = ServerApiException.class) + public void testExecuteCRE() { + DeleteWebhookDeliveryCmd cmd = new DeleteWebhookDeliveryCmd(); + cmd.webhookApiService = webhookApiService; + Mockito.when(webhookApiService.deleteWebhookDelivery(cmd)).thenThrow(CloudRuntimeException.class); + cmd.execute(); + } + + @Test + public void testExecute() { + DeleteWebhookDeliveryCmd cmd = new DeleteWebhookDeliveryCmd(); + cmd.webhookApiService = webhookApiService; + Mockito.when(webhookApiService.deleteWebhookDelivery(cmd)).thenReturn(10); + cmd.execute(); + Assert.assertNotNull(cmd.getResponseObject()); + } +} diff --git a/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/ExecuteWebhookDeliveryCmdTest.java b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/ExecuteWebhookDeliveryCmdTest.java new file mode 100644 index 000000000000..84d51a1e18d5 --- /dev/null +++ b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/ExecuteWebhookDeliveryCmdTest.java @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.UUID; + +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.apache.cloudstack.mom.webhook.api.response.WebhookDeliveryResponse; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.user.Account; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserVO; +import com.cloud.utils.exception.CloudRuntimeException; + +@RunWith(MockitoJUnitRunner.class) +public class ExecuteWebhookDeliveryCmdTest { + + @Mock + WebhookApiService webhookApiService; + + private Object getCommandMethodValue(Object obj, String methodName) { + Object result = null; + try { + Method method = obj.getClass().getMethod(methodName); + result = method.invoke(obj); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + Assert.fail(String.format("Failed to get method %s value", methodName)); + } + return result; + } + + private void runStringMemberTest(String memberName) { + String methodName = "get" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + ExecuteWebhookDeliveryCmd cmd = new ExecuteWebhookDeliveryCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + String value = UUID.randomUUID().toString(); + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + private void runLongMemberTest(String memberName) { + String methodName = "get" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + ExecuteWebhookDeliveryCmd cmd = new ExecuteWebhookDeliveryCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + Long value = 100L; + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + private void runBooleanMemberTest(String memberName) { + String methodName = "is" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + ExecuteWebhookDeliveryCmd cmd = new ExecuteWebhookDeliveryCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + Boolean value = true; + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + @Test + public void testGetId() { + runLongMemberTest("id"); + } + + @Test + public void testGetWebhookId() { + runLongMemberTest("webhookId"); + } + + @Test + public void testGetPayloadUrl() { + runStringMemberTest("payloadUrl"); + } + + @Test + public void testGetSecretKey() { + runStringMemberTest("secretKey"); + } + + @Test + public void testIsSslVerification() { + runBooleanMemberTest("sslVerification"); + } + + @Test + public void testGetPayload() { + runStringMemberTest("payload"); + } + + @Test + public void testGetEntityOwnerId() { + Account account = new AccountVO("testaccount", 1L, "networkdomain", Account.Type.NORMAL, "uuid"); + UserVO user = new UserVO(1, "testuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + CallContext.register(user, account); + ExecuteWebhookDeliveryCmd cmd = new ExecuteWebhookDeliveryCmd(); + Assert.assertEquals(account.getId(), cmd.getEntityOwnerId()); + } + + @Test(expected = ServerApiException.class) + public void testExecuteNullResponse() { + ExecuteWebhookDeliveryCmd cmd = new ExecuteWebhookDeliveryCmd(); + cmd.webhookApiService = webhookApiService; + Mockito.when(webhookApiService.executeWebhookDelivery(cmd)).thenReturn(null); + cmd.execute(); + } + + @Test(expected = ServerApiException.class) + public void testExecuteCRE() { + ExecuteWebhookDeliveryCmd cmd = new ExecuteWebhookDeliveryCmd(); + cmd.webhookApiService = webhookApiService; + Mockito.when(webhookApiService.executeWebhookDelivery(cmd)).thenThrow(CloudRuntimeException.class); + cmd.execute(); + } + + @Test + public void testExecute() { + ExecuteWebhookDeliveryCmd cmd = new ExecuteWebhookDeliveryCmd(); + cmd.webhookApiService = webhookApiService; + WebhookDeliveryResponse response = new WebhookDeliveryResponse(); + Mockito.when(webhookApiService.executeWebhookDelivery(cmd)).thenReturn(response); + cmd.execute(); + Assert.assertEquals(cmd.getCommandName(), response.getResponseName()); + } +} diff --git a/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/ListWebhookDeliveriesCmdTest.java b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/ListWebhookDeliveriesCmdTest.java new file mode 100644 index 000000000000..6359b042c408 --- /dev/null +++ b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/ListWebhookDeliveriesCmdTest.java @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.apache.cloudstack.mom.webhook.api.response.WebhookDeliveryResponse; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.user.Account; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserVO; + +@RunWith(MockitoJUnitRunner.class) +public class ListWebhookDeliveriesCmdTest { + + @Mock + WebhookApiService webhookApiService; + + private Object getCommandMethodValue(Object obj, String methodName) { + Object result = null; + try { + Method method = obj.getClass().getMethod(methodName); + result = method.invoke(obj); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + Assert.fail(String.format("Failed to get method %s value", methodName)); + } + return result; + } + + private void runLongMemberTest(String memberName) { + String methodName = "get" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + ListWebhookDeliveriesCmd cmd = new ListWebhookDeliveriesCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + Long value = 100L; + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + private void runStringMemberTest(String memberName) { + String methodName = "get" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + ListWebhookDeliveriesCmd cmd = new ListWebhookDeliveriesCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + String value = UUID.randomUUID().toString(); + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + private void runDateMemberTest(String memberName) { + String methodName = "get" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + ListWebhookDeliveriesCmd cmd = new ListWebhookDeliveriesCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + Date value = new Date(); + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + @Test + public void testGetId() { + runLongMemberTest("id"); + } + + @Test + public void testGetWebhookId() { + runLongMemberTest("webhookId"); + } + + @Test + public void testGetManagementServerId() { + runLongMemberTest("managementServerId"); + } + + @Test + public void testStartDate() { + runDateMemberTest("startDate"); + } + + @Test + public void testEndDate() { + runDateMemberTest("endDate"); + } + + @Test + public void testEventType() { + runStringMemberTest("eventType"); + } + + @Test + public void testGetEntityOwnerId() { + Account account = new AccountVO("testaccount", 1L, "networkdomain", Account.Type.NORMAL, "uuid"); + UserVO user = new UserVO(1, "testuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + CallContext.register(user, account); + ListWebhookDeliveriesCmd cmd = new ListWebhookDeliveriesCmd(); + Assert.assertEquals(account.getId(), cmd.getEntityOwnerId()); + } + + @Test + public void testExecute() { + ListWebhookDeliveriesCmd cmd = new ListWebhookDeliveriesCmd(); + cmd.webhookApiService = webhookApiService; + List responseList = new ArrayList<>(); + ListResponse listResponse = new ListResponse<>(); + listResponse.setResponses(responseList); + Mockito.when(webhookApiService.listWebhookDeliveries(cmd)).thenReturn(listResponse); + cmd.execute(); + Assert.assertNotNull(cmd.getResponseObject()); + } +} diff --git a/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/ListWebhooksCmdTest.java b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/ListWebhooksCmdTest.java new file mode 100644 index 000000000000..1cbf9d1e8361 --- /dev/null +++ b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/ListWebhooksCmdTest.java @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.apache.cloudstack.mom.webhook.api.response.WebhookResponse; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class ListWebhooksCmdTest { + + @Mock + WebhookApiService webhookApiService; + + private Object getCommandMethodValue(Object obj, String methodName) { + Object result = null; + try { + Method method = obj.getClass().getMethod(methodName); + result = method.invoke(obj); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + Assert.fail(String.format("Failed to get method %s value", methodName)); + } + return result; + } + + private void runStringMemberTest(String memberName) { + String methodName = "get" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + ListWebhooksCmd cmd = new ListWebhooksCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + String value = UUID.randomUUID().toString(); + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + private void runLongMemberTest(String memberName) { + String methodName = "get" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + ListWebhooksCmd cmd = new ListWebhooksCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + Long value = 100L; + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + @Test + public void testGetId() { + runLongMemberTest("id"); + } + + @Test + public void testGetName() { + runStringMemberTest("name"); + } + + @Test + public void testGetState() { + runStringMemberTest("state"); + } + + @Test + public void testGetScope() { + runStringMemberTest("scope"); + } + + @Test + public void testExecute() { + ListWebhooksCmd cmd = new ListWebhooksCmd(); + cmd.webhookApiService = webhookApiService; + List responseList = new ArrayList<>(); + ListResponse listResponse = new ListResponse<>(); + listResponse.setResponses(responseList); + Mockito.when(webhookApiService.listWebhooks(cmd)).thenReturn(listResponse); + cmd.execute(); + Assert.assertNotNull(cmd.getResponseObject()); + } +} diff --git a/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/UpdateWebhookCmdTest.java b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/UpdateWebhookCmdTest.java new file mode 100644 index 000000000000..719e63cadf2c --- /dev/null +++ b/plugins/event-bus/webhook/src/test/java/org/apache/cloudstack/mom/webhook/api/command/user/UpdateWebhookCmdTest.java @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.mom.webhook.api.command.user; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.UUID; + +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.mom.webhook.WebhookApiService; +import org.apache.cloudstack.mom.webhook.api.response.WebhookResponse; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.user.Account; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserVO; +import com.cloud.utils.exception.CloudRuntimeException; + +@RunWith(MockitoJUnitRunner.class) +public class UpdateWebhookCmdTest { + + @Mock + WebhookApiService webhookApiService; + + private Object getCommandMethodValue(Object obj, String methodName) { + Object result = null; + try { + Method method = obj.getClass().getMethod(methodName); + result = method.invoke(obj); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + Assert.fail(String.format("Failed to get method %s value", methodName)); + } + return result; + } + + private void runStringMemberTest(String memberName) { + String methodName = "get" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + UpdateWebhookCmd cmd = new UpdateWebhookCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + String value = UUID.randomUUID().toString(); + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + private void runLongMemberTest(String memberName) { + String methodName = "get" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + UpdateWebhookCmd cmd = new UpdateWebhookCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + Long value = 100L; + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + private void runBooleanMemberTest(String memberName) { + String methodName = "is" + memberName.substring(0, 1).toUpperCase() + memberName.substring(1); + UpdateWebhookCmd cmd = new UpdateWebhookCmd(); + ReflectionTestUtils.setField(cmd, memberName, null); + Assert.assertNull(getCommandMethodValue(cmd, methodName)); + Boolean value = true; + ReflectionTestUtils.setField(cmd, memberName, value); + Assert.assertEquals(value, getCommandMethodValue(cmd, methodName)); + } + + @Test + public void testGetId() { + runLongMemberTest("id"); + } + + @Test + public void testGetName() { + runStringMemberTest("name"); + } + + @Test + public void testGetDescription() { + runStringMemberTest("description"); + } + + @Test + public void testGetPayloadUrl() { + runStringMemberTest("payloadUrl"); + } + + @Test + public void testGetSecretKey() { + runStringMemberTest("secretKey"); + } + + @Test + public void testGetScope() { + runStringMemberTest("scope"); + } + + @Test + public void testGetState() { + runStringMemberTest("state"); + } + + @Test + public void testIsSslVerification() { + runBooleanMemberTest("sslVerification"); + } + + @Test + public void testGetEntityOwnerId() { + Account account = new AccountVO("testaccount", 1L, "networkdomain", Account.Type.NORMAL, "uuid"); + UserVO user = new UserVO(1, "testuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + CallContext.register(user, account); + UpdateWebhookCmd cmd = new UpdateWebhookCmd(); + Assert.assertEquals(account.getId(), cmd.getEntityOwnerId()); + } + + @Test(expected = ServerApiException.class) + public void testExecuteNullResponse() { + UpdateWebhookCmd cmd = new UpdateWebhookCmd(); + cmd.webhookApiService = webhookApiService; + Mockito.when(webhookApiService.updateWebhook(cmd)).thenReturn(null); + cmd.execute(); + } + + @Test(expected = ServerApiException.class) + public void testExecuteCRE() { + UpdateWebhookCmd cmd = new UpdateWebhookCmd(); + cmd.webhookApiService = webhookApiService; + Mockito.when(webhookApiService.updateWebhook(cmd)).thenThrow(CloudRuntimeException.class); + cmd.execute(); + } + + @Test + public void testExecute() { + UpdateWebhookCmd cmd = new UpdateWebhookCmd(); + cmd.webhookApiService = webhookApiService; + WebhookResponse response = new WebhookResponse(); + Mockito.when(webhookApiService.updateWebhook(cmd)).thenReturn(response); + cmd.execute(); + Assert.assertEquals(cmd.getCommandName(), response.getResponseName()); + } +} diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java index f15f3f200016..a71ae26e670d 100644 --- a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java @@ -43,7 +43,6 @@ import com.cloud.storage.VMTemplateVO; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; @@ -89,7 +88,7 @@ private List findSuitableHosts(VirtualMachineProfile vmProfile, Deployment Long clusterId = plan.getClusterId(); ServiceOffering offering = vmProfile.getServiceOffering(); List hostsCopy = null; - List suitableHosts = new ArrayList(); + List suitableHosts = new ArrayList<>(); if (type == Host.Type.Storage) { return suitableHosts; @@ -107,7 +106,7 @@ private List findSuitableHosts(VirtualMachineProfile vmProfile, Deployment } if (hosts != null) { // retain all computing hosts, regardless of whether they support routing...it's random after all - hostsCopy = new ArrayList(hosts); + hostsCopy = new ArrayList<>(hosts); if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { hostsCopy.retainAll(listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag)); } else { @@ -124,14 +123,15 @@ private List findSuitableHosts(VirtualMachineProfile vmProfile, Deployment hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(offeringHostTag)); if (hostsCopy.isEmpty()) { - logger.error(String.format("No suitable host found for vm [%s] with tags [%s].", vmProfile, hostTag)); - throw new CloudRuntimeException(String.format("No suitable host found for vm [%s].", vmProfile)); + logger.info("No suitable host found for VM [{}] in {}.", vmProfile, hostTag); + return null; } - logger.debug("Random Allocator found " + hostsCopy.size() + " hosts"); - if (hostsCopy.size() == 0) { + logger.debug("Random Allocator found {} hosts", hostsCopy.size()); + if (hostsCopy.isEmpty()) { return suitableHosts; } + Collections.shuffle(hostsCopy); for (Host host : hostsCopy) { if (suitableHosts.size() == returnUpTo) { @@ -174,7 +174,7 @@ public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan pla if (logger.isDebugEnabled()) { logger.debug("Random Allocator found 0 hosts as given host list is empty"); } - return new ArrayList(); + return new ArrayList<>(); } return findSuitableHosts(vmProfile, plan, type, avoid, hosts, returnUpTo, considerReservedCapacity); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 5eed56806b83..dec7f70e62fd 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.hypervisor.kvm.resource; +import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION; import static com.cloud.host.Host.HOST_VOLUME_ENCRYPTION; import java.io.BufferedReader; @@ -306,6 +307,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String TUNGSTEN_PATH = "scripts/vm/network/tungsten"; + public static final String INSTANCE_CONVERSION_SUPPORTED_CHECK_CMD = "virt-v2v --version"; + // virt-v2v --version => sample output: virt-v2v 1.42.0rhel=8,release=22.module+el8.10.0+1590+a67ab969 + public static final String OVF_EXPORT_SUPPORTED_CHECK_CMD = "ovftool --version"; + // ovftool --version => sample output: VMware ovftool 4.6.0 (build-21452615) + public static final String OVF_EXPORT_TOOl_GET_VERSION_CMD = "ovftool --version | awk '{print $3}'"; + + public static final String WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "rpm -qa | grep -i virtio-win"; + public static final String UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "dpkg -l virtio-win"; + public static final String UBUNTU_NBDKIT_PKG_CHECK_CMD = "dpkg -l nbdkit"; + private String modifyVlanPath; private String versionStringPath; private String patchScriptPath; @@ -3646,6 +3657,8 @@ public StartupCommand[] initialize() { cmd.setGatewayIpAddress(localGateway); cmd.setIqn(getIqn()); cmd.getHostDetails().put(HOST_VOLUME_ENCRYPTION, String.valueOf(hostSupportsVolumeEncryption())); + cmd.setHostTags(getHostTags()); + cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(hostSupportsInstanceConversion())); HealthCheckResult healthCheckResult = getHostHealthCheckResult(); if (healthCheckResult != HealthCheckResult.IGNORE) { cmd.setHostHealthCheckResult(healthCheckResult == HealthCheckResult.SUCCESS); @@ -3674,6 +3687,19 @@ public StartupCommand[] initialize() { return startupCommandsArray; } + protected List getHostTags() { + List hostTagsList = new ArrayList<>(); + String hostTags = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_TAGS); + if (StringUtils.isNotBlank(hostTags)) { + for (String hostTag : hostTags.split(",")) { + if (!hostTagsList.contains(hostTag.trim())) { + hostTagsList.add(hostTag.trim()); + } + } + } + return hostTagsList; + } + /** * Calculates and sets the host CPU max capacity according to the cgroup version of the host. *
    @@ -3783,29 +3809,29 @@ public Pair getSourceHostPath(String diskPath) { } public List getAllVmNames(final Connect conn) { - final ArrayList la = new ArrayList(); + final ArrayList domainNames = new ArrayList(); try { final String names[] = conn.listDefinedDomains(); for (int i = 0; i < names.length; i++) { - la.add(names[i]); + domainNames.add(names[i]); } } catch (final LibvirtException e) { - LOGGER.warn("Failed to list Defined domains", e); + logger.warn("Failed to list defined domains", e); } int[] ids = null; try { ids = conn.listDomains(); } catch (final LibvirtException e) { - LOGGER.warn("Failed to list domains", e); - return la; + logger.warn("Failed to list domains", e); + return domainNames; } Domain dm = null; for (int i = 0; i < ids.length; i++) { try { dm = conn.domainLookupByID(ids[i]); - la.add(dm.getName()); + domainNames.add(dm.getName()); } catch (final LibvirtException e) { LOGGER.warn("Unable to get vms", e); } finally { @@ -3819,7 +3845,7 @@ public List getAllVmNames(final Connect conn) { } } - return la; + return domainNames; } private HashMap getHostVmStateReport() { @@ -5149,6 +5175,48 @@ public boolean isSecureMode(String bootMode) { return false; } + public boolean hostSupportsInstanceConversion() { + int exitValue = Script.runSimpleBashScriptForExitValue(INSTANCE_CONVERSION_SUPPORTED_CHECK_CMD); + if (isUbuntuHost() && exitValue == 0) { + exitValue = Script.runSimpleBashScriptForExitValue(UBUNTU_NBDKIT_PKG_CHECK_CMD); + } + return exitValue == 0; + } + + public boolean hostSupportsWindowsGuestConversion() { + if (isUbuntuHost()) { + int exitValue = Script.runSimpleBashScriptForExitValue(UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD); + return exitValue == 0; + } + int exitValue = Script.runSimpleBashScriptForExitValue(WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD); + return exitValue == 0; + } + + public boolean hostSupportsOvfExport() { + int exitValue = Script.runSimpleBashScriptForExitValue(OVF_EXPORT_SUPPORTED_CHECK_CMD); + return exitValue == 0; + } + + public boolean ovfExportToolSupportsParallelThreads() { + String ovfExportToolVersion = Script.runSimpleBashScript(OVF_EXPORT_TOOl_GET_VERSION_CMD); + if (StringUtils.isBlank(ovfExportToolVersion)) { + return false; + } + String[] ovfExportToolVersions = ovfExportToolVersion.trim().split("\\."); + if (ovfExportToolVersions.length > 1) { + try { + int majorVersion = Integer.parseInt(ovfExportToolVersions[0]); + int minorVersion = Integer.parseInt(ovfExportToolVersions[1]); + //ovftool version >= 4.4 supports parallel threads + if (majorVersion > 4 || (majorVersion == 4 && minorVersion >= 4)) { + return true; + } + } catch (NumberFormatException ignored) { + } + } + return false; + } + protected void setCpuTopology(CpuModeDef cmd, int vCpusInDef, Map details) { if (!enableManuallySettingCpuTopologyOnKvmVm) { LOGGER.debug(String.format("Skipping manually setting CPU topology on VM's XML due to it is disabled in agent.properties {\"property\": \"%s\", \"value\": %s}.", @@ -5365,20 +5433,31 @@ public void setInterfaceDefQueueSettings(Map details, Integer cp /* Scp volume from remote host to local directory */ - public String copyVolume(String srcIp, String username, String password, String localDir, String remoteFile, String tmpPath) { + public String copyVolume(String srcIp, String username, String password, String localDir, String remoteFile, String tmpPath, int timeoutInSecs) { + String outputFile = UUID.randomUUID().toString(); try { - String outputFile = UUID.randomUUID().toString(); StringBuilder command = new StringBuilder("qemu-img convert -O qcow2 "); command.append(remoteFile); - command.append(" "+tmpPath); + command.append(" " + tmpPath); command.append(outputFile); - logger.debug("Converting remoteFile: "+remoteFile); - SshHelper.sshExecute(srcIp, 22, username, null, password, command.toString()); - logger.debug("Copying remoteFile to: "+localDir); - SshHelper.scpFrom(srcIp, 22, username, null, password, localDir, tmpPath+outputFile); - logger.debug("Successfully copyied remoteFile to: "+localDir+"/"+outputFile); + logger.debug(String.format("Converting remote disk file: %s, output file: %s%s (timeout: %d secs)", remoteFile, tmpPath, outputFile, timeoutInSecs)); + SshHelper.sshExecute(srcIp, 22, username, null, password, command.toString(), timeoutInSecs * 1000); + logger.debug("Copying converted remote disk file " + outputFile + " to: " + localDir); + SshHelper.scpFrom(srcIp, 22, username, null, password, localDir, tmpPath + outputFile); + logger.debug("Successfully copied converted remote disk file to: " + localDir + "/" + outputFile); return outputFile; } catch (Exception e) { + try { + String deleteRemoteConvertedFileCmd = String.format("rm -f %s%s", tmpPath, outputFile); + SshHelper.sshExecute(srcIp, 22, username, null, password, deleteRemoteConvertedFileCmd); + } catch (Exception ignored) { + } + + try { + FileUtils.deleteQuietly(new File(localDir + "/" + outputFile)); + } catch (Exception ignored) { + } + throw new RuntimeException(e); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java index ff44c8df2fa6..09ee45d59089 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java @@ -16,6 +16,12 @@ // under the License. package com.cloud.hypervisor.kvm.resource; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.collections.CollectionUtils; + public class LibvirtStoragePoolDef { public enum PoolType { ISCSI("iscsi"), NETFS("netfs"), loggerICAL("logical"), DIR("dir"), RBD("rbd"), GLUSTERFS("glusterfs"), POWERFLEX("powerflex"); @@ -55,6 +61,7 @@ public String toString() { private String _authUsername; private AuthenticationType _authType; private String _secretUuid; + private Set _nfsMountOpts = new HashSet<>(); public LibvirtStoragePoolDef(PoolType type, String poolName, String uuid, String host, int port, String dir, String targetPath) { _poolType = type; @@ -75,6 +82,15 @@ public LibvirtStoragePoolDef(PoolType type, String poolName, String uuid, String _targetPath = targetPath; } + public LibvirtStoragePoolDef(PoolType type, String poolName, String uuid, String host, String dir, String targetPath, List nfsMountOpts) { + this(type, poolName, uuid, host, dir, targetPath); + if (CollectionUtils.isNotEmpty(nfsMountOpts)) { + for (String nfsMountOpt : nfsMountOpts) { + this._nfsMountOpts.add(nfsMountOpt); + } + } + } + public LibvirtStoragePoolDef(PoolType type, String poolName, String uuid, String sourceHost, int sourcePort, String dir, String authUsername, AuthenticationType authType, String secretUuid) { _poolType = type; @@ -124,69 +140,98 @@ public AuthenticationType getAuthType() { return _authType; } + public Set getNfsMountOpts() { + return _nfsMountOpts; + } + @Override public String toString() { StringBuilder storagePoolBuilder = new StringBuilder(); - if (_poolType == PoolType.GLUSTERFS) { - /* libvirt mounts a Gluster volume, similar to NFS */ - storagePoolBuilder.append("\n"); - } else { - storagePoolBuilder.append("\n"); + String poolTypeXML; + switch (_poolType) { + case NETFS: + if (_nfsMountOpts != null) { + poolTypeXML = "netfs' xmlns:fs='http://libvirt.org/schemas/storagepool/fs/1.0"; + } else { + poolTypeXML = _poolType.toString(); + } + break; + case GLUSTERFS: + /* libvirt mounts a Gluster volume, similar to NFS */ + poolTypeXML = "netfs"; + break; + default: + poolTypeXML = _poolType.toString(); } + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("" + _poolName + "\n"); if (_uuid != null) storagePoolBuilder.append("" + _uuid + "\n"); - if (_poolType == PoolType.NETFS) { - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - } - if (_poolType == PoolType.RBD) { - storagePoolBuilder.append("\n"); - for (String sourceHost : _sourceHost.split(",")) { + + switch (_poolType) { + case NETFS: + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + break; + + case RBD: + storagePoolBuilder.append("\n"); + for (String sourceHost : _sourceHost.split(",")) { + storagePoolBuilder.append("\n"); + } + + storagePoolBuilder.append("" + _sourceDir + "\n"); + if (_authUsername != null) { + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + } + storagePoolBuilder.append("\n"); + break; + + case GLUSTERFS: + storagePoolBuilder.append("\n"); storagePoolBuilder.append("\n"); - } - - storagePoolBuilder.append("" + _sourceDir + "\n"); - if (_authUsername != null) { - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - } - storagePoolBuilder.append("\n"); - } - if (_poolType == PoolType.GLUSTERFS) { - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); - storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + storagePoolBuilder.append("\n"); + break; } + if (_poolType != PoolType.RBD && _poolType != PoolType.POWERFLEX) { storagePoolBuilder.append("\n"); storagePoolBuilder.append("" + _targetPath + "\n"); storagePoolBuilder.append("\n"); } + if (_poolType == PoolType.NETFS && _nfsMountOpts != null) { + storagePoolBuilder.append("\n"); + for (String options : _nfsMountOpts) { + storagePoolBuilder.append("\n"); + } + storagePoolBuilder.append("\n"); + } storagePoolBuilder.append("\n"); return storagePoolBuilder.toString(); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java index 30616e047987..430e4ef851fc 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java @@ -38,6 +38,19 @@ public class LibvirtStoragePoolXMLParser { protected Logger logger = LogManager.getLogger(getClass()); + private List getNFSMountOptsFromRootElement(Element rootElement) { + List nfsMountOpts = new ArrayList<>(); + Element mountOpts = (Element) rootElement.getElementsByTagName("fs:mount_opts").item(0); + if (mountOpts != null) { + NodeList options = mountOpts.getElementsByTagName("fs:option"); + for (int i = 0; i < options.getLength(); i++) { + Element option = (Element) options.item(i); + nfsMountOpts.add(option.getAttribute("name")); + } + } + return nfsMountOpts; + } + public LibvirtStoragePoolDef parseStoragePoolXML(String poolXML) { DocumentBuilder builder; try { @@ -95,11 +108,15 @@ public LibvirtStoragePoolDef parseStoragePoolXML(String poolXML) { poolName, uuid, host, port, path, targetPath); } else { String path = getAttrValue("dir", "path", source); - Element target = (Element)rootElement.getElementsByTagName("target").item(0); String targetPath = getTagValue("path", target); - return new LibvirtStoragePoolDef(LibvirtStoragePoolDef.PoolType.valueOf(type.toUpperCase()), poolName, uuid, host, path, targetPath); + if (type.equalsIgnoreCase("netfs")) { + List nfsMountOpts = getNFSMountOptsFromRootElement(rootElement); + return new LibvirtStoragePoolDef(LibvirtStoragePoolDef.PoolType.valueOf(type.toUpperCase()), poolName, uuid, host, path, targetPath, nfsMountOpts); + } else { + return new LibvirtStoragePoolDef(LibvirtStoragePoolDef.PoolType.valueOf(type.toUpperCase()), poolName, uuid, host, path, targetPath); + } } } catch (ParserConfigurationException e) { logger.debug(e.toString()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java new file mode 100644 index 000000000000..d3ebb28b106a --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java @@ -0,0 +1,53 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckConvertInstanceAnswer; +import com.cloud.agent.api.CheckConvertInstanceCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; + +@ResourceWrapper(handles = CheckConvertInstanceCommand.class) +public class LibvirtCheckConvertInstanceCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(CheckConvertInstanceCommand cmd, LibvirtComputingResource serverResource) { + if (!serverResource.hostSupportsInstanceConversion()) { + String msg = String.format("Cannot convert the instance from VMware as the virt-v2v binary is not found on host %s. " + + "Please install virt-v2v%s on the host before attempting the instance conversion.", serverResource.getPrivateIp(), serverResource.isUbuntuHost()? ", nbdkit" : ""); + logger.info(msg); + return new CheckConvertInstanceAnswer(cmd, false, msg); + } + + if (cmd.getCheckWindowsGuestConversionSupport() && !serverResource.hostSupportsWindowsGuestConversion()) { + String msg = String.format("Cannot convert the instance from VMware as the virtio-win package is not found on host %s. " + + "Please install virtio-win package on the host before attempting the windows guest instance conversion.", serverResource.getPrivateIp()); + logger.info(msg); + return new CheckConvertInstanceAnswer(cmd, false, msg); + } + + if (serverResource.hostSupportsOvfExport()) { + return new CheckConvertInstanceAnswer(cmd, true, true, ""); + } + + return new CheckConvertInstanceAnswer(cmd, true, ""); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java index bd6634c83a4c..cc955e86d8a4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java @@ -18,6 +18,24 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + import com.cloud.agent.api.Answer; import com.cloud.agent.api.ConvertInstanceAnswer; import com.cloud.agent.api.ConvertInstanceCommand; @@ -34,27 +52,11 @@ import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.storage.Storage; +import com.cloud.utils.FileUtil; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URLEncoder; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.stream.Collectors; @ResourceWrapper(handles = ConvertInstanceCommand.class) public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper { @@ -62,8 +64,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper supportedInstanceConvertSourceHypervisors = List.of(Hypervisor.HypervisorType.VMware); - protected static final String checkIfConversionIsSupportedCommand = "which virt-v2v"; - @Override public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serverResource) { RemoteInstanceTO sourceInstance = cmd.getSourceInstance(); @@ -74,9 +74,9 @@ public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serve DataStoreTO conversionTemporaryLocation = cmd.getConversionTemporaryLocation(); long timeout = (long) cmd.getWait() * 1000; - if (!isInstanceConversionSupportedOnHost()) { + if (cmd.getCheckConversionSupport() && !serverResource.hostSupportsInstanceConversion()) { String msg = String.format("Cannot convert the instance %s from VMware as the virt-v2v binary is not found. " + - "Please install virt-v2v on the host before attempting the instance conversion", sourceInstanceName); + "Please install virt-v2v%s on the host before attempting the instance conversion.", sourceInstanceName, serverResource.isUbuntuHost()? ", nbdkit" : ""); logger.info(msg); return new ConvertInstanceAnswer(cmd, false, msg); } @@ -94,18 +94,48 @@ public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serve logger.info(String.format("Attempting to convert the instance %s from %s to KVM", sourceInstanceName, sourceHypervisorType)); - final String convertInstanceUrl = getConvertInstanceUrl(sourceInstance); - final String temporaryConvertUuid = UUID.randomUUID().toString(); - final String temporaryPasswordFilePath = createTemporaryPasswordFileAndRetrievePath(sourceInstance); final String temporaryConvertPath = temporaryStoragePool.getLocalPath(); + + String ovfTemplateDirOnConversionLocation; + String sourceOVFDirPath; + boolean ovfExported = false; + if (cmd.getExportOvfToConversionLocation()) { + String exportInstanceOVAUrl = getExportInstanceOVAUrl(sourceInstance); + if (StringUtils.isBlank(exportInstanceOVAUrl)) { + String err = String.format("Couldn't export OVA for the VM %s, due to empty url", sourceInstanceName); + logger.error(err); + return new ConvertInstanceAnswer(cmd, false, err); + } + + int noOfThreads = cmd.getThreadsCountToExportOvf(); + if (noOfThreads > 1 && !serverResource.ovfExportToolSupportsParallelThreads()) { + noOfThreads = 0; + } + ovfTemplateDirOnConversionLocation = UUID.randomUUID().toString(); + temporaryStoragePool.createFolder(ovfTemplateDirOnConversionLocation); + sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); + ovfExported = exportOVAFromVMOnVcenter(exportInstanceOVAUrl, sourceOVFDirPath, noOfThreads, timeout); + if (!ovfExported) { + String err = String.format("Export OVA for the VM %s failed", sourceInstanceName); + logger.error(err); + return new ConvertInstanceAnswer(cmd, false, err); + } + sourceOVFDirPath = String.format("%s%s/", sourceOVFDirPath, sourceInstanceName); + } else { + ovfTemplateDirOnConversionLocation = cmd.getTemplateDirOnConversionLocation(); + sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); + } + + logger.info(String.format("Attempting to convert the OVF %s of the instance %s from %s to KVM", ovfTemplateDirOnConversionLocation, sourceInstanceName, sourceHypervisorType)); + final String temporaryConvertUuid = UUID.randomUUID().toString(); boolean verboseModeEnabled = serverResource.isConvertInstanceVerboseModeEnabled(); try { - boolean result = performInstanceConversion(convertInstanceUrl, sourceInstanceName, temporaryPasswordFilePath, - temporaryConvertPath, temporaryConvertUuid, timeout, verboseModeEnabled); + boolean result = performInstanceConversion(sourceOVFDirPath, temporaryConvertPath, temporaryConvertUuid, + timeout, verboseModeEnabled); if (!result) { - String err = String.format("The virt-v2v conversion of the instance %s failed. " + - "Please check the agent logs for the virt-v2v output", sourceInstanceName); + String err = String.format("The virt-v2v conversion for the OVF %s failed. " + + "Please check the agent logs for the virt-v2v output", ovfTemplateDirOnConversionLocation); logger.error(err); return new ConvertInstanceAnswer(cmd, false, err); } @@ -130,8 +160,11 @@ public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serve logger.error(error, e); return new ConvertInstanceAnswer(cmd, false, error); } finally { - logger.debug("Cleaning up instance conversion temporary password file"); - Script.runSimpleBashScript(String.format("rm -rf %s", temporaryPasswordFilePath)); + if (ovfExported && StringUtils.isNotBlank(ovfTemplateDirOnConversionLocation)) { + String sourceOVFDir = String.format("%s/%s", temporaryConvertPath, ovfTemplateDirOnConversionLocation); + logger.debug("Cleaning up exported OVA at dir " + sourceOVFDir); + FileUtil.deletePath(sourceOVFDir); + } if (conversionTemporaryLocation instanceof NfsTO) { logger.debug("Cleaning up secondary storage temporary location"); storagePoolMgr.deleteStoragePool(temporaryStoragePool.getType(), temporaryStoragePool.getUuid()); @@ -155,6 +188,27 @@ protected boolean areSourceAndDestinationHypervisorsSupported(Hypervisor.Hypervi supportedInstanceConvertSourceHypervisors.contains(sourceHypervisorType); } + private String getExportInstanceOVAUrl(RemoteInstanceTO sourceInstance) { + String url = null; + if (sourceInstance.getHypervisorType() == Hypervisor.HypervisorType.VMware) { + url = getExportOVAUrlFromRemoteInstance(sourceInstance); + } + return url; + } + + private String getExportOVAUrlFromRemoteInstance(RemoteInstanceTO vmwareInstance) { + String vcenter = vmwareInstance.getVcenterHost(); + String username = vmwareInstance.getVcenterUsername(); + String password = vmwareInstance.getVcenterPassword(); + String datacenter = vmwareInstance.getDatacenterName(); + String vm = vmwareInstance.getInstanceName(); + + String encodedUsername = encodeUsername(username); + String encodedPassword = encodeUsername(password); + return String.format("vi://%s:%s@%s/%s/vm/%s", + encodedUsername, encodedPassword, vcenter, datacenter, vm); + } + protected List getTemporaryDisksFromParsedXml(KVMStoragePool pool, LibvirtDomainXMLParser xmlParser, String convertedBasePath) { List disksDefs = xmlParser.getDisks(); disksDefs = disksDefs.stream().filter(x -> x.getDiskType() == LibvirtVMDef.DiskDef.DiskType.FILE && @@ -201,12 +255,7 @@ private void cleanupDisksAndDomainFromTemporaryLocation(List di temporaryStoragePool.deletePhysicalDisk(disk.getName(), Storage.ImageFormat.QCOW2); } logger.info(String.format("Cleaning up temporary domain %s after conversion from temporary location", temporaryConvertUuid)); - Script.runSimpleBashScript(String.format("rm -f %s/%s*.xml", temporaryStoragePool.getLocalPath(), temporaryConvertUuid)); - } - - protected boolean isInstanceConversionSupportedOnHost() { - int exitValue = Script.runSimpleBashScriptForExitValue(checkIfConversionIsSupportedCommand); - return exitValue == 0; + FileUtil.deleteFiles(temporaryStoragePool.getLocalPath(), temporaryConvertUuid, ".xml"); } protected void sanitizeDisksPath(List disks) { @@ -234,6 +283,11 @@ protected List moveTemporaryDisksToDestination(List getUnmanagedInstanceDisks(List getNfsStoragePoolHostAndPath(KVMStoragePool storagePool) { String sourceHostIp = null; String sourcePath = null; - String storagePoolMountPoint = Script.runSimpleBashScript(String.format("mount | grep %s", storagePool.getLocalPath())); + List commands = new ArrayList<>(); + commands.add(new String[]{Script.getExecutableAbsolutePath("mount")}); + commands.add(new String[]{Script.getExecutableAbsolutePath("grep"), storagePool.getLocalPath()}); + String storagePoolMountPoint = Script.executePipedCommands(commands, 0).second(); + logger.debug(String.format("NFS Storage pool: %s - local path: %s, mount point: %s", storagePool.getUuid(), storagePool.getLocalPath(), storagePoolMountPoint)); if (StringUtils.isNotEmpty(storagePoolMountPoint)) { String[] res = storagePoolMountPoint.strip().split(" "); res = res[0].split(":"); - sourceHostIp = res[0].strip(); - sourcePath = res[1].strip(); + if (res.length > 1) { + sourceHostIp = res[0].strip(); + sourcePath = res[1].strip(); + } } return new Pair<>(sourceHostIp, sourcePath); } - protected boolean performInstanceConversion(String convertInstanceUrl, String sourceInstanceName, - String temporaryPasswordFilePath, - String temporaryConvertFolder, - String temporaryConvertUuid, - long timeout, boolean verboseModeEnabled) { + private boolean exportOVAFromVMOnVcenter(String vmExportUrl, + String targetOvfDir, + int noOfThreads, + long timeout) { + Script script = new Script("ovftool", timeout, logger); + script.add("--noSSLVerify"); + if (noOfThreads > 1) { + script.add(String.format("--parallelThreads=%s", noOfThreads)); + } + script.add(vmExportUrl); + script.add(targetOvfDir); + + String logPrefix = "export ovf"; + OutputInterpreter.LineByLineOutputLogger outputLogger = new OutputInterpreter.LineByLineOutputLogger(logger, logPrefix); + script.execute(outputLogger); + int exitValue = script.getExitValue(); + return exitValue == 0; + } + + protected boolean performInstanceConversion(String sourceOVFDirPath, + String temporaryConvertFolder, + String temporaryConvertUuid, + long timeout, boolean verboseModeEnabled) { Script script = new Script("virt-v2v", timeout, logger); script.add("--root", "first"); - script.add("-ic", convertInstanceUrl); - script.add(sourceInstanceName); - script.add("--password-file", temporaryPasswordFilePath); + script.add("-i", "ova"); + script.add(sourceOVFDirPath); script.add("-o", "local"); script.add("-os", temporaryConvertFolder); script.add("-of", "qcow2"); @@ -332,44 +409,13 @@ protected boolean performInstanceConversion(String convertInstanceUrl, String so script.add("-v"); } - String logPrefix = String.format("virt-v2v source: %s %s progress", convertInstanceUrl, sourceInstanceName); + String logPrefix = String.format("virt-v2v ovf source: %s progress", sourceOVFDirPath); OutputInterpreter.LineByLineOutputLogger outputLogger = new OutputInterpreter.LineByLineOutputLogger(logger, logPrefix); script.execute(outputLogger); int exitValue = script.getExitValue(); return exitValue == 0; } - private String createTemporaryPasswordFileAndRetrievePath(RemoteInstanceTO sourceInstance) { - String password = null; - if (sourceInstance.getHypervisorType() == Hypervisor.HypervisorType.VMware) { - password = sourceInstance.getVcenterPassword(); - } - String passwordFile = String.format("/tmp/vmw-%s", UUID.randomUUID()); - String msg = String.format("Creating a temporary password file for VMware instance %s conversion on: %s", sourceInstance.getInstanceName(), passwordFile); - logger.debug(msg); - Script.runSimpleBashScriptForExitValueAvoidLogging(String.format("echo \"%s\" > %s", password, passwordFile)); - return passwordFile; - } - - private String getConvertInstanceUrl(RemoteInstanceTO sourceInstance) { - String url = null; - if (sourceInstance.getHypervisorType() == Hypervisor.HypervisorType.VMware) { - url = getConvertInstanceUrlFromVmware(sourceInstance); - } - return url; - } - - private String getConvertInstanceUrlFromVmware(RemoteInstanceTO vmwareInstance) { - String vcenter = vmwareInstance.getVcenterHost(); - String datacenter = vmwareInstance.getDatacenterName(); - String username = vmwareInstance.getVcenterUsername(); - String host = vmwareInstance.getHostName(); - String cluster = vmwareInstance.getClusterName(); - - String encodedUsername = encodeUsername(username); - return String.format("vpx://%s@%s/%s/%s/%s?no_verify=1", - encodedUsername, vcenter, datacenter, cluster, host); - } protected LibvirtDomainXMLParser parseMigratedVMXmlDomain(String installPath) throws IOException { String xmlPath = String.format("%s.xml", installPath); if (!new File(xmlPath).exists()) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyRemoteVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyRemoteVolumeCommandWrapper.java index 025a5ed192cd..e6ec05fec23f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyRemoteVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyRemoteVolumeCommandWrapper.java @@ -43,7 +43,6 @@ public final class LibvirtCopyRemoteVolumeCommandWrapper extends CommandWrapper< @Override public Answer execute(final CopyRemoteVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { - String result = null; String srcIp = command.getRemoteIp(); String username = command.getUsername(); String password = command.getPassword(); @@ -53,23 +52,25 @@ public Answer execute(final CopyRemoteVolumeCommand command, final LibvirtComput KVMStoragePoolManager poolMgr = libvirtComputingResource.getStoragePoolMgr(); KVMStoragePool pool = poolMgr.getStoragePool(storageFilerTO.getType(), storageFilerTO.getUuid()); String dstPath = pool.getLocalPath(); + int timeoutInSecs = command.getWait(); try { if (storageFilerTO.getType() == Storage.StoragePoolType.Filesystem || storageFilerTO.getType() == Storage.StoragePoolType.NetworkFilesystem) { - String filename = libvirtComputingResource.copyVolume(srcIp, username, password, dstPath, srcFile, tmpPath); - logger.debug("Volume Copy Successful"); + String filename = libvirtComputingResource.copyVolume(srcIp, username, password, dstPath, srcFile, tmpPath, timeoutInSecs); + logger.debug("Volume " + srcFile + " copy successful, copied to file: " + filename); final KVMPhysicalDisk vol = pool.getPhysicalDisk(filename); final String path = vol.getPath(); long size = getVirtualSizeFromFile(path); - return new CopyRemoteVolumeAnswer(command, "", filename, size); + return new CopyRemoteVolumeAnswer(command, "", filename, size); } else { - return new Answer(command, false, "Unsupported Storage Pool"); + String msg = "Unsupported storage pool type: " + storageFilerTO.getType().toString() + ", only local and NFS pools are supported"; + return new Answer(command, false, msg); } - } catch (final Exception e) { - logger.error("Error while copying file from remote host: "+ e.getMessage()); - return new Answer(command, false, result); + logger.error("Error while copying volume file from remote host: " + e.getMessage(), e); + String msg = "Failed to copy volume due to: " + e.getMessage(); + return new Answer(command, false, msg); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java index 45b0c179938c..58a74d6e0f61 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java @@ -19,6 +19,9 @@ package com.cloud.hypervisor.kvm.resource.wrapper; +import java.util.ArrayList; +import java.util.List; + import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.libvirt.Connect; @@ -35,8 +38,8 @@ import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; -import com.cloud.storage.Volume; import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.Volume; import com.cloud.utils.script.Script; @ResourceWrapper(handles = DeleteVMSnapshotCommand.class) @@ -94,12 +97,20 @@ public Answer execute(final DeleteVMSnapshotCommand cmd, final LibvirtComputingR PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO) rootVolume.getDataStore(); KVMPhysicalDisk rootDisk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), rootVolume.getPath()); - String qemu_img_snapshot = Script.runSimpleBashScript("qemu-img snapshot -l " + rootDisk.getPath() + " | tail -n +3 | awk -F ' ' '{print $2}' | grep ^" + cmd.getTarget().getSnapshotName() + "$"); + String qemuImgPath = Script.getExecutableAbsolutePath("qemu-img"); + List commands = new ArrayList<>(); + commands.add(new String[]{qemuImgPath, "snapshot", "-l", sanitizeBashCommandArgument(rootDisk.getPath())}); + commands.add(new String[]{Script.getExecutableAbsolutePath("tail"), "-n", "+3"}); + commands.add(new String[]{Script.getExecutableAbsolutePath("awk"), "-F", " ", "{print $2}"}); + commands.add(new String[]{Script.getExecutableAbsolutePath("grep"), "^" + sanitizeBashCommandArgument(cmd.getTarget().getSnapshotName()) + "$"}); + String qemu_img_snapshot = Script.executePipedCommands(commands, 0).second(); if (qemu_img_snapshot == null) { logger.info("Cannot find snapshot " + cmd.getTarget().getSnapshotName() + " in file " + rootDisk.getPath() + ", return true"); return new DeleteVMSnapshotAnswer(cmd, cmd.getVolumeTOs()); } - int result = Script.runSimpleBashScriptForExitValue("qemu-img snapshot -d " + cmd.getTarget().getSnapshotName() + " " + rootDisk.getPath()); + int result = Script.executeCommandForExitValue(qemuImgPath, "snapshot", "-d", + sanitizeBashCommandArgument(cmd.getTarget().getSnapshotName()), + sanitizeBashCommandArgument(rootDisk.getPath())); if (result != 0) { return new DeleteVMSnapshotAnswer(cmd, false, "Delete VM Snapshot Failed due to can not remove snapshot from image file " + rootDisk.getPath() + " : " + result); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java index ead294ad05f7..114b27d3a5b7 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java @@ -47,37 +47,38 @@ public final class LibvirtGetRemoteVmsCommandWrapper extends CommandWrapper unmanagedInstances = new HashMap<>(); try { Connect conn = LibvirtConnection.getConnection(hypervisorURI); final List allVmNames = libvirtComputingResource.getAllVmNames(conn); + logger.info(String.format("Found %d VMs on the remote host %s", allVmNames.size(), remoteIp)); for (String name : allVmNames) { final Domain domain = libvirtComputingResource.getDomain(conn, name); - final DomainInfo.DomainState ps = domain.getInfo().state; - final VirtualMachine.PowerState state = libvirtComputingResource.convertToPowerState(ps); - logger.debug("VM " + domain.getName() + ": powerstate = " + ps + "; vm state=" + state.toString()); + logger.debug(String.format("Remote VM %s - powerstate: %s, state: %s", domain.getName(), ps.toString(), state.toString())); if (state == VirtualMachine.PowerState.PowerOff) { try { UnmanagedInstanceTO instance = getUnmanagedInstance(libvirtComputingResource, domain, conn); unmanagedInstances.put(instance.getName(), instance); } catch (Exception e) { - logger.error("Error while fetching instance details", e); + logger.error("Couldn't fetch remote VM " + domain.getName() + " details, due to: " + e.getMessage(), e); } } domain.free(); } - logger.debug("Found Vms: "+ unmanagedInstances.size()); - return new GetRemoteVmsAnswer(command, "", unmanagedInstances); + logger.debug("Found " + unmanagedInstances.size() + " stopped VMs on remote host " + remoteIp); + return new GetRemoteVmsAnswer(command, "", unmanagedInstances); } catch (final LibvirtException e) { - logger.error("Error while listing stopped Vms on remote host: "+ e.getMessage()); - return new Answer(command, false, result); + logger.error("Failed to list stopped VMs on remote host " + remoteIp + ", due to: " + e.getMessage(), e); + if (e.getMessage().toLowerCase().contains("connection refused")) { + return new Answer(command, false, "Unable to connect to remote host " + remoteIp + ", please check the libvirtd tcp connectivity and retry"); + } + return new Answer(command, false, "Unable to list stopped VMs on remote host " + remoteIp + ", due to: " + e.getMessage()); } } @@ -103,8 +104,8 @@ private UnmanagedInstanceTO getUnmanagedInstance(LibvirtComputingResource libvir return instance; } catch (Exception e) { - logger.debug("Unable to retrieve unmanaged instance info. ", e); - throw new CloudRuntimeException("Unable to retrieve unmanaged instance info. " + e.getMessage()); + logger.debug("Unable to retrieve remote unmanaged instance info, due to: " + e.getMessage(), e); + throw new CloudRuntimeException("Unable to retrieve remote unmanaged instance info, due to: " + e.getMessage()); } } @@ -116,7 +117,6 @@ private UnmanagedInstanceTO.PowerState getPowerState(VirtualMachine.PowerState v return UnmanagedInstanceTO.PowerState.PowerOff; default: return UnmanagedInstanceTO.PowerState.PowerUnknown; - } } @@ -163,7 +163,6 @@ private List getUnmanagedInstanceDisks(List sourceHostPath = getSourceHostPath(libvirtComputingResource, diskDef.getSourcePath()); if (sourceHostPath != null) { disk.setDatastoreHost(sourceHostPath.first()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVmIpAddressCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVmIpAddressCommandWrapper.java index 227e68872dac..61c20f96bacc 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVmIpAddressCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVmIpAddressCommandWrapper.java @@ -19,6 +19,9 @@ package com.cloud.hypervisor.kvm.resource.wrapper; +import java.util.ArrayList; +import java.util.List; + import com.cloud.agent.api.Answer; import com.cloud.agent.api.GetVmIpAddressCommand; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; @@ -35,31 +38,51 @@ public final class LibvirtGetVmIpAddressCommandWrapper extends CommandWrapper commands = new ArrayList<>(); + final String virt_ls_path = Script.getExecutableAbsolutePath("virt-ls"); + final String virt_cat_path = Script.getExecutableAbsolutePath("virt-cat"); + final String virt_win_reg_path = Script.getExecutableAbsolutePath("virt-win-reg"); + final String tail_path = Script.getExecutableAbsolutePath("tail"); + final String grep_path = Script.getExecutableAbsolutePath("grep"); + final String awk_path = Script.getExecutableAbsolutePath("awk"); + final String sed_path = Script.getExecutableAbsolutePath("sed"); if(!command.isWindows()) { //List all dhcp lease files inside guestVm - String leasesList = Script.runSimpleBashScript(new StringBuilder().append("virt-ls ").append(command.getVmName()) - .append(" /var/lib/dhclient/ | grep .*\\*.leases").toString()); + commands.add(new String[]{virt_ls_path, sanitizedVmName, "/var/lib/dhclient/"}); + commands.add(new String[]{grep_path, ".*\\*.leases"}); + String leasesList = Script.executePipedCommands(commands, 0).second(); if(leasesList != null) { String[] leasesFiles = leasesList.split("\n"); for(String leaseFile : leasesFiles){ - //Read from each dhclient lease file inside guest Vm using virt-cat libguestfs ulitiy - String ipAddr = Script.runSimpleBashScript(new StringBuilder().append("virt-cat ").append(command.getVmName()) - .append(" /var/lib/dhclient/" + leaseFile + " | tail -16 | grep 'fixed-address' | awk '{print $2}' | sed -e 's/;//'").toString()); + //Read from each dhclient lease file inside guest Vm using virt-cat libguestfs utility + commands = new ArrayList<>(); + commands.add(new String[]{virt_cat_path, sanitizedVmName, "/var/lib/dhclient/" + leaseFile}); + commands.add(new String[]{tail_path, "-16"}); + commands.add(new String[]{grep_path, "fixed-address"}); + commands.add(new String[]{awk_path, "{print $2}"}); + commands.add(new String[]{sed_path, "-e", "s/;//"}); + String ipAddr = Script.executePipedCommands(commands, 0).second(); // Check if the IP belongs to the network - if((ipAddr != null) && NetUtils.isIpWithInCidrRange(ipAddr, networkCidr)){ + if((ipAddr != null) && NetUtils.isIpWithInCidrRange(ipAddr, networkCidr)) { ip = ipAddr; break; } - logger.debug("GetVmIp: "+command.getVmName()+ " Ip: "+ipAddr+" does not belong to network "+networkCidr); + logger.debug("GetVmIp: "+ vmName + " Ip: "+ipAddr+" does not belong to network "+networkCidr); } } } else { // For windows, read from guest Vm registry using virt-win-reg libguestfs ulitiy. Registry Path: HKEY_LOCAL_MACHINE\SYSTEM\ControlSet001\Services\Tcpip\Parameters\Interfaces\\DhcpIPAddress - String ipList = Script.runSimpleBashScript(new StringBuilder().append("virt-win-reg --unsafe-printable-strings ").append(command.getVmName()) - .append(" 'HKEY_LOCAL_MACHINE\\SYSTEM\\ControlSet001\\Services\\Tcpip\\Parameters\\Interfaces' | grep DhcpIPAddress | awk -F : '{print $2}' | sed -e 's/^\"//' -e 's/\"$//'").toString()); + commands = new ArrayList<>(); + commands.add(new String[]{virt_win_reg_path, "--unsafe-printable-strings", sanitizedVmName, "HKEY_LOCAL_MACHINE\\SYSTEM\\ControlSet001\\Services\\Tcpip\\Parameters\\Interfaces"}); + commands.add(new String[]{grep_path, "DhcpIPAddress"}); + commands.add(new String[]{awk_path, "-F", ":", "{print $2}"}); + commands.add(new String[]{sed_path, "-e", "s/^\"//", "-e", "s/\"$//"}); + String ipList = Script.executePipedCommands(commands, 0).second(); if(ipList != null) { - logger.debug("GetVmIp: "+command.getVmName()+ "Ips: "+ipList); + logger.debug("GetVmIp: "+ vmName + "Ips: "+ipList); String[] ips = ipList.split("\n"); for (String ipAddr : ips){ // Check if the IP belongs to the network @@ -67,13 +90,13 @@ public Answer execute(final GetVmIpAddressCommand command, final LibvirtComputin ip = ipAddr; break; } - logger.debug("GetVmIp: "+command.getVmName()+ " Ip: "+ipAddr+" does not belong to network "+networkCidr); + logger.debug("GetVmIp: "+ vmName + " Ip: "+ipAddr+" does not belong to network "+networkCidr); } } } if(ip != null){ result = true; - logger.debug("GetVmIp: "+command.getVmName()+ " Found Ip: "+ip); + logger.debug("GetVmIp: "+ vmName + " Found Ip: "+ip); } return new Answer(command, result, ip); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumeStatCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumeStatCommandWrapper.java new file mode 100644 index 000000000000..5619fcd91390 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumeStatCommandWrapper.java @@ -0,0 +1,63 @@ +// +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.GetVolumeStatAnswer; +import com.cloud.agent.api.GetVolumeStatCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.exception.CloudRuntimeException; + +@ResourceWrapper(handles = GetVolumeStatCommand.class) +public final class LibvirtGetVolumeStatCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(final GetVolumeStatCommand cmd, final LibvirtComputingResource libvirtComputingResource) { + try { + String volumePath = cmd.getVolumePath(); + StoragePoolType poolType = cmd.getPoolType(); + String poolUuid = cmd.getPoolUuid(); + + KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); + KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(poolType, poolUuid); + if (primaryPool == null) { + String msg = "Can't get volume stats as pool details unavailable for volume: " + volumePath + " on the storage pool: " + poolUuid; + return new GetVolumeStatAnswer(cmd, false, msg); + } + + KVMPhysicalDisk disk = primaryPool.getPhysicalDisk(volumePath); + if (disk == null) { + String msg = "Can't get volume stats as disk details unavailable for volume: " + volumePath + " on the storage pool: " + poolUuid; + return new GetVolumeStatAnswer(cmd, false, msg); + } + + return new GetVolumeStatAnswer(cmd, disk.getSize(), disk.getVirtualSize()); + } catch (CloudRuntimeException e) { + logger.error("Can't get volume stats, due to: " + e.getMessage(), e); + return new GetVolumeStatAnswer(cmd, false, "Can't get volume stats, due to: " + e.getMessage()); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index b97cb666de09..e15a32876927 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -67,6 +67,7 @@ import com.cloud.agent.api.MigrateAnswer; import com.cloud.agent.api.MigrateCommand; import com.cloud.agent.api.MigrateCommand.MigrateDiskInfo; +import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.DpdkTO; import com.cloud.agent.api.to.VirtualMachineTO; @@ -90,6 +91,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper disks, String vmName) { + String oldIsoVolumePath = null; + for (DiskDef disk : disks) { + if (DiskDef.DeviceType.CDROM.equals(disk.getDeviceType()) + && CDROM_LABEL.equals(disk.getDiskLabel()) + && disk.getDiskPath() != null) { + oldIsoVolumePath = disk.getDiskPath(); + break; + } + } + return oldIsoVolumePath; + } + + private String getNewVolumePathForCdrom(LibvirtComputingResource libvirtComputingResource, Connect conn, VirtualMachineTO to) throws LibvirtException, URISyntaxException { + DiskTO newDisk = null; + for (DiskTO disk : to.getDisks()) { + DataTO data = disk.getData(); + if (disk.getDiskSeq() == 3 && data != null && data.getPath() != null) { + newDisk = disk; + break; + } + } + + String newIsoVolumePath = null; + if (newDisk != null) { + newIsoVolumePath = libvirtComputingResource.getVolumePath(conn, newDisk); + } + return newIsoVolumePath; + } + + protected String replaceCdromIsoPath(String xmlDesc, String vmName, String oldIsoVolumePath, String newIsoVolumePath) throws IOException, ParserConfigurationException, TransformerException, SAXException { + InputStream in = IOUtils.toInputStream(xmlDesc); + + DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory(); + DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + Document doc = docBuilder.parse(in); + + // Get the root element + Node domainNode = doc.getFirstChild(); + + NodeList domainChildNodes = domainNode.getChildNodes(); + + for (int i = 0; i < domainChildNodes.getLength(); i++) { + Node domainChildNode = domainChildNodes.item(i); + if ("devices".equals(domainChildNode.getNodeName())) { + NodeList devicesChildNodes = domainChildNode.getChildNodes(); + for (int x = 0; x < devicesChildNodes.getLength(); x++) { + Node deviceChildNode = devicesChildNodes.item(x); + if ("disk".equals(deviceChildNode.getNodeName())) { + Node diskNode = deviceChildNode; + NodeList diskChildNodes = diskNode.getChildNodes(); + for (int z = 0; z < diskChildNodes.getLength(); z++) { + Node diskChildNode = diskChildNodes.item(z); + if ("source".equals(diskChildNode.getNodeName())) { + NamedNodeMap sourceNodeAttributes = diskChildNode.getAttributes(); + Node sourceNodeAttribute = sourceNodeAttributes.getNamedItem("file"); + if (oldIsoVolumePath != null && sourceNodeAttribute != null + && oldIsoVolumePath.equals(sourceNodeAttribute.getNodeValue())) { + diskNode.removeChild(diskChildNode); + Element newChildSourceNode = doc.createElement("source"); + newChildSourceNode.setAttribute("file", newIsoVolumePath); + diskNode.appendChild(newChildSourceNode); + logger.debug(String.format("Replaced ISO path [%s] with [%s] in VM [%s] XML configuration.", oldIsoVolumePath, newIsoVolumePath, vmName)); + return getXml(doc); + } + } + } + } + } + } + } + + return getXml(doc); + } + private String getPathFromSourceText(Set paths, String sourceText) { if (paths != null && StringUtils.isNotBlank(sourceText)) { for (String path : paths) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 00f627d05285..0221496b79c2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -140,7 +140,6 @@ protected MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand logger.info(String.format("Block copy has started for the volume %s : %s ", destDiskLabel, srcPath)); return checkBlockJobStatus(command, dm, destDiskLabel, srcPath, destPath, libvirtComputingResource, conn, srcSecretUUID); - } catch (Exception e) { String msg = "Migrate volume failed due to " + e.toString(); logger.warn(msg, e); @@ -166,17 +165,27 @@ protected MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand protected MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath, LibvirtComputingResource libvirtComputingResource, Connect conn, String srcSecretUUID) throws LibvirtException { int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found int waitTimeInSec = command.getWait(); + double blockCopyProgress = 0; while (waitTimeInSec > 0) { DomainBlockJobInfo blockJobInfo = dm.getBlockJobInfo(diskLabel, 0); if (blockJobInfo != null) { - logger.debug(String.format("Volume %s : %s block copy progress: %s%% current value:%s end value:%s", diskLabel, srcPath, (blockJobInfo.end == 0)? 0 : 100*(blockJobInfo.cur / (double) blockJobInfo.end), blockJobInfo.cur, blockJobInfo.end)); + blockCopyProgress = (blockJobInfo.end == 0)? blockCopyProgress : 100 * (blockJobInfo.cur / (double) blockJobInfo.end); + logger.debug(String.format("Volume %s : %s, block copy progress: %s%%, current value: %s end value: %s, job info - type: %s, bandwidth: %s", + diskLabel, srcPath, blockCopyProgress, blockJobInfo.cur, blockJobInfo.end, blockJobInfo.type, blockJobInfo.bandwidth)); if (blockJobInfo.cur == blockJobInfo.end) { - logger.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath)); - dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT); - if (StringUtils.isNotEmpty(srcSecretUUID)) { - libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID); + if (blockJobInfo.end > 0) { + logger.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath)); + dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT); + if (StringUtils.isNotEmpty(srcSecretUUID)) { + libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID); + } + break; + } else { + // cur = 0, end = 0 - at this point, disk does not have an active block job (so, no need to abort job) + String msg = String.format("No active block copy job for the volume %s : %s - job stopped at %s progress", diskLabel, srcPath, blockCopyProgress); + logger.warn(msg); + return new MigrateVolumeAnswer(command, false, msg, null); } - break; } } else { logger.info("Failed to get the block copy status, trying to abort the job"); @@ -291,15 +300,27 @@ protected MigrateVolumeAnswer migrateRegularVolume(final MigrateVolumeCommand co (destVolumeObjectTO.getPath() != null ? destVolumeObjectTO.getPath() : UUID.randomUUID().toString()); try { - storagePoolManager.connectPhysicalDisk(srcPrimaryDataStore.getPoolType(), srcPrimaryDataStore.getUuid(), srcPath, srcDetails); + KVMStoragePool sourceStoragePool = storagePoolManager.getStoragePool(srcPrimaryDataStore.getPoolType(), srcPrimaryDataStore.getUuid()); + + if (!sourceStoragePool.connectPhysicalDisk(srcPath, srcDetails)) { + return new MigrateVolumeAnswer(command, false, "Unable to connect source volume on hypervisor", srcPath); + } KVMPhysicalDisk srcPhysicalDisk = storagePoolManager.getPhysicalDisk(srcPrimaryDataStore.getPoolType(), srcPrimaryDataStore.getUuid(), srcPath); + if (srcPhysicalDisk == null) { + return new MigrateVolumeAnswer(command, false, "Unable to get handle to source volume on hypervisor", srcPath); + } KVMStoragePool destPrimaryStorage = storagePoolManager.getStoragePool(destPrimaryDataStore.getPoolType(), destPrimaryDataStore.getUuid()); - storagePoolManager.connectPhysicalDisk(destPrimaryDataStore.getPoolType(), destPrimaryDataStore.getUuid(), destPath, destDetails); + if (!destPrimaryStorage.connectPhysicalDisk(destPath, destDetails)) { + return new MigrateVolumeAnswer(command, false, "Unable to connect destination volume on hypervisor", srcPath); + } - storagePoolManager.copyPhysicalDisk(srcPhysicalDisk, destPath, destPrimaryStorage, command.getWaitInMillSeconds()); + KVMPhysicalDisk newDiskCopy = storagePoolManager.copyPhysicalDisk(srcPhysicalDisk, destPath, destPrimaryStorage, command.getWaitInMillSeconds()); + if (newDiskCopy == null) { + return new MigrateVolumeAnswer(command, false, "Copy command failed to return handle to copied physical disk", destPath); + } } catch (Exception ex) { return new MigrateVolumeAnswer(command, false, ex.getMessage(), null); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtOvsFetchInterfaceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtOvsFetchInterfaceCommandWrapper.java index db07cc5291a0..923c44f48285 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtOvsFetchInterfaceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtOvsFetchInterfaceCommandWrapper.java @@ -19,7 +19,11 @@ package com.cloud.hypervisor.kvm.resource.wrapper; -import org.apache.commons.lang3.StringUtils; +import java.net.InetAddress; +import java.net.InterfaceAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.util.Enumeration; import com.cloud.agent.api.Answer; import com.cloud.agent.api.OvsFetchInterfaceAnswer; @@ -27,32 +31,72 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; -import com.cloud.utils.script.Script; +import com.cloud.utils.Ternary; @ResourceWrapper(handles = OvsFetchInterfaceCommand.class) public final class LibvirtOvsFetchInterfaceCommandWrapper extends CommandWrapper { + private String getSubnetMaskForAddress(NetworkInterface networkInterface, InetAddress inetAddress) { + for (InterfaceAddress address : networkInterface.getInterfaceAddresses()) { + if (!inetAddress.equals(address.getAddress())) { + continue; + } + int prefixLength = address.getNetworkPrefixLength(); + int mask = 0xffffffff << (32 - prefixLength); + return String.format("%d.%d.%d.%d", + (mask >>> 24) & 0xff, + (mask >>> 16) & 0xff, + (mask >>> 8) & 0xff, + mask & 0xff); + } + return ""; + } + + private String getMacAddress(NetworkInterface networkInterface) throws SocketException { + byte[] macBytes = networkInterface.getHardwareAddress(); + if (macBytes == null) { + return ""; + } + StringBuilder macAddress = new StringBuilder(); + for (byte b : macBytes) { + macAddress.append(String.format("%02X:", b)); + } + if (macAddress.length() > 0) { + macAddress.deleteCharAt(macAddress.length() - 1); // Remove trailing colon + } + return macAddress.toString(); + } + + public Ternary getInterfaceDetails(String interfaceName) throws SocketException { + NetworkInterface networkInterface = NetworkInterface.getByName(interfaceName); + if (networkInterface == null) { + logger.warn(String.format("Network interface: '%s' not found", interfaceName)); + return new Ternary<>(null, null, null); + } + Enumeration inetAddresses = networkInterface.getInetAddresses(); + while (inetAddresses.hasMoreElements()) { + InetAddress inetAddress = inetAddresses.nextElement(); + if (inetAddress instanceof java.net.Inet4Address) { + String ipAddress = inetAddress.getHostAddress(); + String subnetMask = getSubnetMaskForAddress(networkInterface, inetAddress); + String macAddress = getMacAddress(networkInterface); + return new Ternary<>(ipAddress, subnetMask, macAddress); + } + } + return new Ternary<>(null, null, null); + } + @Override public Answer execute(final OvsFetchInterfaceCommand command, final LibvirtComputingResource libvirtComputingResource) { - final String label = command.getLabel(); + final String label = "'" + command.getLabel() + "'"; logger.debug("Will look for network with name-label:" + label); try { - String ipadd = Script.runSimpleBashScript("ifconfig " + label + " | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'"); - if (StringUtils.isEmpty(ipadd)) { - ipadd = Script.runSimpleBashScript("ifconfig " + label + " | grep ' inet ' | awk '{ print $2}'"); - } - String mask = Script.runSimpleBashScript("ifconfig " + label + " | grep 'inet addr:' | cut -d: -f4"); - if (StringUtils.isEmpty(mask)) { - mask = Script.runSimpleBashScript("ifconfig " + label + " | grep ' inet ' | awk '{ print $4}'"); - } - String mac = Script.runSimpleBashScript("ifconfig " + label + " | grep HWaddr | awk -F \" \" '{print $5}'"); - if (StringUtils.isEmpty(mac)) { - mac = Script.runSimpleBashScript("ifconfig " + label + " | grep ' ether ' | awk '{ print $2}'"); - } + Ternary interfaceDetails = getInterfaceDetails(label); return new OvsFetchInterfaceAnswer(command, true, "Interface " + label - + " retrieved successfully", ipadd, mask, mac); + + " retrieved successfully", interfaceDetails.first(), interfaceDetails.second(), + interfaceDetails.third()); } catch (final Exception e) { logger.warn("Caught execption when fetching interface", e); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java index c8b205113465..535494877043 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java @@ -46,7 +46,6 @@ import com.cloud.resource.ResourceWrapper; import com.cloud.storage.Volume; import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.Script; @ResourceWrapper(handles = PrepareForMigrationCommand.class) public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapper { @@ -127,9 +126,7 @@ public Answer execute(final PrepareForMigrationCommand command, final LibvirtCom } catch (final LibvirtException | CloudRuntimeException | InternalErrorException | URISyntaxException e) { if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) { for (DpdkTO to : dpdkInterfaceMapping.values()) { - String cmd = String.format("ovs-vsctl del-port %s", to.getPort()); - logger.debug("Removing DPDK port: " + to.getPort()); - Script.runSimpleBashScript(cmd); + removeDpdkPort(to.getPort()); } } return new PrepareForMigrationAnswer(command, e.toString()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareStorageClientCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareStorageClientCommandWrapper.java new file mode 100644 index 000000000000..b5cab17ecb14 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareStorageClientCommandWrapper.java @@ -0,0 +1,48 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.util.Map; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.PrepareStorageClientAnswer; +import com.cloud.agent.api.PrepareStorageClientCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Ternary; + +@ResourceWrapper(handles = PrepareStorageClientCommand.class) +public class LibvirtPrepareStorageClientCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(PrepareStorageClientCommand cmd, LibvirtComputingResource libvirtComputingResource) { + final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); + Ternary, String> prepareStorageClientResult = storagePoolMgr.prepareStorageClient(cmd.getPoolType(), cmd.getPoolUuid(), cmd.getDetails()); + if (!prepareStorageClientResult.first()) { + String msg = prepareStorageClientResult.third(); + logger.debug("Unable to prepare storage client, due to: " + msg); + return new PrepareStorageClientAnswer(cmd, false, msg); + } + Map details = prepareStorageClientResult.second(); + return new PrepareStorageClientAnswer(cmd, true, details); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java index 0b0f69f3eed5..8f23e79e4a32 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java @@ -19,7 +19,9 @@ package com.cloud.hypervisor.kvm.resource.wrapper; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import com.cloud.agent.api.Answer; @@ -33,7 +35,6 @@ import com.cloud.resource.ResourceWrapper; import com.cloud.utils.script.Script; - @ResourceWrapper(handles = ReadyCommand.class) public final class LibvirtReadyCommandWrapper extends CommandWrapper { @@ -50,13 +51,18 @@ public Answer execute(final ReadyCommand command, final LibvirtComputingResource } private boolean hostSupportsUefi(boolean isUbuntuHost) { - String cmd = "rpm -qa | grep -i ovmf"; int timeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.AGENT_SCRIPT_TIMEOUT) * 1000; // Get property value & convert to milliseconds + int result; if (isUbuntuHost) { - cmd = "dpkg -l ovmf"; + logger.debug("Running command : [dpkg -l ovmf] with timeout : " + timeout + " ms"); + result = Script.executeCommandForExitValue(timeout, Script.getExecutableAbsolutePath("dpkg"), "-l", "ovmf"); + } else { + logger.debug("Running command : [rpm -qa | grep -i ovmf] with timeout : " + timeout + " ms"); + List commands = new ArrayList<>(); + commands.add(new String[]{Script.getExecutableAbsolutePath("rpm"), "-qa"}); + commands.add(new String[]{Script.getExecutableAbsolutePath("grep"), "-i", "ovmf"}); + result = Script.executePipedCommands(commands, timeout).first(); } - logger.debug("Running command : [" + cmd + "] with timeout : " + timeout + " ms"); - int result = Script.runSimpleBashScriptForExitValue(cmd, timeout, false); logger.debug("Got result : " + result); return result == 0; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevokeDirectDownloadCertificateWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevokeDirectDownloadCertificateWrapper.java index 348151557110..9919689cf3b4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevokeDirectDownloadCertificateWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevokeDirectDownloadCertificateWrapper.java @@ -19,6 +19,14 @@ package com.cloud.hypervisor.kvm.resource.wrapper; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; + +import org.apache.cloudstack.agent.directdownload.RevokeDirectDownloadCertificateCommand; +import org.apache.cloudstack.utils.security.KeyStoreUtils; +import org.apache.commons.lang3.StringUtils; + import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; @@ -26,13 +34,6 @@ import com.cloud.utils.PropertiesUtil; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; -import org.apache.cloudstack.agent.directdownload.RevokeDirectDownloadCertificateCommand; -import org.apache.cloudstack.utils.security.KeyStoreUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; @ResourceWrapper(handles = RevokeDirectDownloadCertificateCommand.class) public class LibvirtRevokeDirectDownloadCertificateWrapper extends CommandWrapper { @@ -82,17 +83,17 @@ public Answer execute(RevokeDirectDownloadCertificateCommand command, LibvirtCom } final String keyStoreFile = getKeyStoreFilePath(agentFile); - - String checkCmd = String.format("keytool -list -alias %s -keystore %s -storepass %s", - certificateAlias, keyStoreFile, privatePassword); - int existsCmdResult = Script.runSimpleBashScriptForExitValue(checkCmd); + String keyToolPath = Script.getExecutableAbsolutePath("keytool"); + int existsCmdResult = Script.executeCommandForExitValue(keyToolPath, "-list", "-alias", + sanitizeBashCommandArgument(certificateAlias), "-keystore", keyStoreFile, "-storepass", + privatePassword); if (existsCmdResult == 1) { logger.error("Certificate alias " + certificateAlias + " does not exist, no need to revoke it"); } else { - String revokeCmd = String.format("keytool -delete -alias %s -keystore %s -storepass %s", - certificateAlias, keyStoreFile, privatePassword); logger.debug("Revoking certificate alias " + certificateAlias + " from keystore " + keyStoreFile); - Script.runSimpleBashScriptForExitValue(revokeCmd); + Script.executeCommandForExitValue(keyToolPath, "-delete", "-alias", + sanitizeBashCommandArgument(certificateAlias), "-keystore", keyStoreFile, "-storepass", + privatePassword); } } catch (FileNotFoundException | CloudRuntimeException e) { logger.error("Error while setting up certificate " + certificateAlias, e); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSetupDirectDownloadCertificateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSetupDirectDownloadCertificateCommandWrapper.java index eb4e6be7609e..fcca16ba6186 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSetupDirectDownloadCertificateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSetupDirectDownloadCertificateCommandWrapper.java @@ -18,20 +18,25 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificateCommand; +import org.apache.cloudstack.utils.security.KeyStoreUtils; +import org.apache.commons.lang3.StringUtils; + import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.FileUtil; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; -import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificateCommand; -import org.apache.cloudstack.utils.security.KeyStoreUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; @ResourceWrapper(handles = SetupDirectDownloadCertificateCommand.class) public class LibvirtSetupDirectDownloadCertificateCommandWrapper extends CommandWrapper { @@ -77,9 +82,10 @@ private String getKeyStoreFilePath(File agentFile) { */ private void importCertificate(String tempCerFilePath, String keyStoreFile, String certificateName, String privatePassword) { logger.debug("Importing certificate from temporary file to keystore"); - String importCommandFormat = "keytool -importcert -file %s -keystore %s -alias '%s' -storepass '%s' -noprompt"; - String importCmd = String.format(importCommandFormat, tempCerFilePath, keyStoreFile, certificateName, privatePassword); - int result = Script.runSimpleBashScriptForExitValue(importCmd); + String keyToolPath = Script.getExecutableAbsolutePath("keytool"); + int result = Script.executeCommandForExitValue(keyToolPath, "-importcert", "file", tempCerFilePath, + "-keystore", keyStoreFile, "-alias", sanitizeBashCommandArgument(certificateName), "-storepass", + privatePassword, "-noprompt"); if (result != 0) { logger.debug("Certificate " + certificateName + " not imported as it already exist on keystore"); } @@ -92,8 +98,7 @@ private String createTemporaryFile(File agentFile, String certificateName, Strin String tempCerFilePath = String.format("%s/%s-%s", agentFile.getParent(), temporaryCertFilePrefix, certificateName); logger.debug("Creating temporary certificate file into: " + tempCerFilePath); - int result = Script.runSimpleBashScriptForExitValue(String.format("echo '%s' > %s", certificate, tempCerFilePath)); - if (result != 0) { + if (!FileUtil.writeToFile(tempCerFilePath, certificate)) { throw new CloudRuntimeException("Could not create the certificate file on path: " + tempCerFilePath); } return tempCerFilePath; @@ -102,9 +107,24 @@ private String createTemporaryFile(File agentFile, String certificateName, Strin /** * Remove temporary file */ - private void cleanupTemporaryFile(String temporaryFile) { + + protected void cleanupTemporaryFile(String temporaryFile) { logger.debug("Cleaning up temporary certificate file"); - Script.runSimpleBashScript("rm -f " + temporaryFile); + if (StringUtils.isBlank(temporaryFile)) { + logger.debug("Provided temporary certificate file path is empty"); + return; + } + try { + Path filePath = Paths.get(temporaryFile); + if (!Files.exists(filePath)) { + logger.debug("Temporary certificate file does not exist: " + temporaryFile); + return; + } + Files.delete(filePath); + } catch (IOException e) { + logger.warn(String.format("Error while cleaning up temporary file: %s", temporaryFile)); + logger.debug(String.format("Error while cleaning up temporary file: %s", temporaryFile), e); + } } @Override diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopCommandWrapper.java index 8b3942f5ebad..1be5a1e949e5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStopCommandWrapper.java @@ -23,27 +23,26 @@ import java.util.List; import java.util.Map; -import com.cloud.agent.api.to.DpdkTO; -import com.cloud.hypervisor.kvm.resource.LibvirtKvmAgentHook; -import com.cloud.utils.Pair; -import com.cloud.utils.script.Script; -import com.cloud.utils.ssh.SshHelper; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.DomainInfo.DomainState; +import org.libvirt.LibvirtException; import com.cloud.agent.api.Answer; import com.cloud.agent.api.StopAnswer; import com.cloud.agent.api.StopCommand; +import com.cloud.agent.api.to.DpdkTO; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtKvmAgentHook; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef; import com.cloud.hypervisor.kvm.resource.VifDriver; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; -import org.libvirt.LibvirtException; +import com.cloud.utils.Pair; +import com.cloud.utils.ssh.SshHelper; @ResourceWrapper(handles = StopCommand.class) public final class LibvirtStopCommandWrapper extends CommandWrapper { @@ -119,10 +118,7 @@ public Answer execute(final StopCommand command, final LibvirtComputingResource Map dpdkInterfaceMapping = command.getDpdkInterfaceMapping(); if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) { for (DpdkTO to : dpdkInterfaceMapping.values()) { - String portToRemove = to.getPort(); - String cmd = String.format("ovs-vsctl del-port %s", portToRemove); - logger.debug("Removing DPDK port: " + portToRemove); - Script.runSimpleBashScript(cmd); + removeDpdkPort(to.getPort()); } } } else { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUnprepareStorageClientCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUnprepareStorageClientCommandWrapper.java new file mode 100644 index 000000000000..2f23a934003f --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUnprepareStorageClientCommandWrapper.java @@ -0,0 +1,45 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.UnprepareStorageClientAnswer; +import com.cloud.agent.api.UnprepareStorageClientCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Pair; + +@ResourceWrapper(handles = UnprepareStorageClientCommand.class) +public class LibvirtUnprepareStorageClientCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(UnprepareStorageClientCommand cmd, LibvirtComputingResource libvirtComputingResource) { + final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); + Pair unprepareStorageClientResult = storagePoolMgr.unprepareStorageClient(cmd.getPoolType(), cmd.getPoolUuid()); + if (!unprepareStorageClientResult.first()) { + String msg = unprepareStorageClientResult.second(); + logger.debug("Couldn't unprepare storage client, due to: " + msg); + return new UnprepareStorageClientAnswer(cmd, false, msg); + } + return new UnprepareStorageClientAnswer(cmd, true); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/FiberChannelAdapter.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/FiberChannelAdapter.java index 83636b9a9c31..0a25b89d8c52 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/FiberChannelAdapter.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/FiberChannelAdapter.java @@ -16,12 +16,36 @@ // under the License. package com.cloud.hypervisor.kvm.storage; +import java.net.InetAddress; +import java.net.UnknownHostException; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; + import com.cloud.storage.Storage; import com.cloud.utils.exception.CloudRuntimeException; public class FiberChannelAdapter extends MultipathSCSIAdapterBase { + + private Logger LOGGER = LogManager.getLogger(getClass()); + + private String hostname = null; + private String hostnameFq = null; + public FiberChannelAdapter() { LOGGER.info("Loaded FiberChannelAdapter for StorageLayer"); + // get the hostname - we need this to compare to connid values + try { + InetAddress inetAddress = InetAddress.getLocalHost(); + hostname = inetAddress.getHostName(); // basic hostname + if (hostname.indexOf(".") > 0) { + hostname = hostname.substring(0, hostname.indexOf(".")); // strip off domain + } + hostnameFq = inetAddress.getCanonicalHostName(); // fully qualified hostname + LOGGER.info("Loaded FiberChannelAdapter for StorageLayer on host [" + hostname + "]"); + } catch (UnknownHostException e) { + LOGGER.error("Error getting hostname", e); + } } @Override @@ -76,6 +100,11 @@ public AddressInfo parseAndValidatePath(String inPath) { address = value; } else if (key.equals("connid")) { connectionId = value; + } else if (key.startsWith("connid.")) { + String inHostname = key.substring(7); + if (inHostname != null && (inHostname.equals(this.hostname) || inHostname.equals(this.hostnameFq))) { + connectionId = value; + } } } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java index 96a4da09686b..674799c0bbe3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java @@ -38,6 +38,9 @@ public interface KVMStoragePool { public static final long HeartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP); public static final long HeartBeatCheckerTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_CHECKER_TIMEOUT); + public default KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) { + return createPhysicalDisk(volumeUuid, format, provisioningType, size, passphrase); + } public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 27f70b71ab4f..3c8026c7ffd6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -45,6 +45,8 @@ import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageLayer; import com.cloud.storage.Volume; +import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; @@ -475,4 +477,13 @@ public KVMPhysicalDisk createPhysicalDiskFromDirectDownloadTemplate(String templ return adaptor.createTemplateFromDirectDownloadFile(templateFilePath, destTemplatePath, destPool, format, timeout); } + public Ternary, String> prepareStorageClient(StoragePoolType type, String uuid, Map details) { + StorageAdaptor adaptor = getStorageAdaptor(type); + return adaptor.prepareStorageClient(type, uuid, details); + } + + public Pair unprepareStorageClient(StoragePoolType type, String uuid) { + StorageAdaptor adaptor = getStorageAdaptor(type); + return adaptor.unprepareStorageClient(type, uuid); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 008768f25e96..f6242444d911 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -134,6 +134,10 @@ import com.cloud.utils.script.Script; import com.cloud.utils.storage.S3.S3Utils; import com.cloud.vm.VmDetailConstants; +import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.qemu.QemuImageOptions; +import org.apache.cloudstack.utils.qemu.QemuObject.EncryptFormat; +import java.util.ArrayList; public class KVMStorageProcessor implements StorageProcessor { protected Logger logger = LogManager.getLogger(getClass()); @@ -267,7 +271,7 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) { Map details = primaryStore.getDetails(); - String path = details != null ? details.get("managedStoreTarget") : null; + String path = derivePath(primaryStore, destData, details); if (!storagePoolMgr.connectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path, details)) { logger.warn("Failed to connect physical disk at path: " + path + ", in storage pool id: " + primaryStore.getUuid()); @@ -327,6 +331,16 @@ public Answer copyTemplateToPrimaryStorage(final CopyCommand cmd) { } } + private String derivePath(PrimaryDataStoreTO primaryStore, DataTO destData, Map details) { + String path = null; + if (primaryStore.getPoolType() == StoragePoolType.FiberChannel) { + path = destData.getPath(); + } else { + path = details != null ? details.get("managedStoreTarget") : null; + } + return path; + } + // this is much like PrimaryStorageDownloadCommand, but keeping it separate. copies template direct to root disk private KVMPhysicalDisk templateToPrimaryDownload(final String templateUrl, final KVMStoragePool primaryPool, final String volUuid, final Long size, final int timeout) { final int index = templateUrl.lastIndexOf("/"); @@ -406,7 +420,7 @@ public Answer cloneVolumeFromBaseTemplate(final CopyCommand cmd) { vol = templateToPrimaryDownload(templatePath, primaryPool, volume.getUuid(), volume.getSize(), cmd.getWaitInMillSeconds()); } if (storagePoolMgr.supportsPhysicalDiskCopy(primaryPool.getType())) { Map details = primaryStore.getDetails(); - String path = details != null ? details.get("managedStoreTarget") : null; + String path = derivePath(primaryStore, destData, details); if (!storagePoolMgr.connectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), templatePath, details)) { logger.warn("Failed to connect base template volume at path: " + templatePath + ", in storage pool id: " + primaryStore.getUuid()); @@ -1047,7 +1061,7 @@ public Answer backupSnapshot(final CopyCommand cmd) { srcVolume.clearPassphrase(); if (isCreatedFromVmSnapshot) { logger.debug("Ignoring removal of vm snapshot on primary as this snapshot is created from vm snapshot"); - } else if (primaryPool.getType() != StoragePoolType.RBD) { + } else if (primaryPool != null && primaryPool.getType() != StoragePoolType.RBD) { deleteSnapshotOnPrimary(cmd, snapshot, primaryPool); } @@ -1633,7 +1647,7 @@ public Answer createVolume(final CreateObjectCommand cmd) { } } else { vol = primaryPool.createPhysicalDisk(volume.getUuid(), format, - volume.getProvisioningType(), disksize, volume.getPassphrase()); + volume.getProvisioningType(), disksize, volume.getUsableSize(), volume.getPassphrase()); } final VolumeObjectTO newVol = new VolumeObjectTO(); @@ -1748,7 +1762,7 @@ public Answer createSnapshot(final CreateObjectCommand cmd) { snapshotPath = getSnapshotPathInPrimaryStorage(primaryPool.getLocalPath(), snapshotName); String diskLabel = takeVolumeSnapshot(resource.getDisks(conn, vmName), snapshotName, diskPath, vm); - String convertResult = convertBaseFileToSnapshotFileInPrimaryStorageDir(primaryPool, diskPath, snapshotPath, volume, cmd.getWait()); + String convertResult = convertBaseFileToSnapshotFileInPrimaryStorageDir(primaryPool, disk, snapshotPath, volume, cmd.getWait()); mergeSnapshotIntoBaseFile(vm, diskLabel, diskPath, snapshotName, volume, conn); @@ -1817,7 +1831,7 @@ public Answer createSnapshot(final CreateObjectCommand cmd) { } } else { snapshotPath = getSnapshotPathInPrimaryStorage(primaryPool.getLocalPath(), snapshotName); - String convertResult = convertBaseFileToSnapshotFileInPrimaryStorageDir(primaryPool, diskPath, snapshotPath, volume, cmd.getWait()); + String convertResult = convertBaseFileToSnapshotFileInPrimaryStorageDir(primaryPool, disk, snapshotPath, volume, cmd.getWait()); validateConvertResult(convertResult, snapshotPath); } } @@ -1940,26 +1954,43 @@ protected void manuallyDeleteUnusedSnapshotFile(boolean isLibvirtSupportingFlagD * @param snapshotPath Path to convert the base file; * @return null if the conversion occurs successfully or an error message that must be handled. */ - protected String convertBaseFileToSnapshotFileInPrimaryStorageDir(KVMStoragePool primaryPool, String baseFile, String snapshotPath, VolumeObjectTO volume, int wait) { - try { - logger.debug(String.format("Trying to convert volume [%s] (%s) to snapshot [%s].", volume, baseFile, snapshotPath)); + protected String convertBaseFileToSnapshotFileInPrimaryStorageDir(KVMStoragePool primaryPool, + KVMPhysicalDisk baseFile, String snapshotPath, VolumeObjectTO volume, int wait) { + try (KeyFile srcKey = new KeyFile(volume.getPassphrase())) { + logger.debug( + String.format("Trying to convert volume [%s] (%s) to snapshot [%s].", volume, baseFile, snapshotPath)); primaryPool.createFolder(TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR); + convertTheBaseFileToSnapshot(baseFile, snapshotPath, wait, srcKey); + } catch (QemuImgException | LibvirtException | IOException ex) { + return String.format("Failed to convert %s snapshot of volume [%s] to [%s] due to [%s].", volume, baseFile, + snapshotPath, ex.getMessage()); + } - QemuImgFile srcFile = new QemuImgFile(baseFile); - srcFile.setFormat(PhysicalDiskFormat.QCOW2); + logger.debug(String.format("Converted volume [%s] (from path \"%s\") to snapshot [%s].", volume, baseFile, + snapshotPath)); + return null; + } - QemuImgFile destFile = new QemuImgFile(snapshotPath); - destFile.setFormat(PhysicalDiskFormat.QCOW2); + private void convertTheBaseFileToSnapshot(KVMPhysicalDisk baseFile, String snapshotPath, int wait, KeyFile srcKey) + throws LibvirtException, QemuImgException { + List qemuObjects = new ArrayList<>(); + Map options = new HashMap<>(); + QemuImageOptions qemuImageOpts = new QemuImageOptions(baseFile.getPath()); + if (srcKey.isSet()) { + String srcKeyName = "sec0"; + qemuObjects.add(QemuObject.prepareSecretForQemuImg(baseFile.getFormat(), EncryptFormat.LUKS, + srcKey.toString(), srcKeyName, options)); + qemuImageOpts = new QemuImageOptions(baseFile.getFormat(), baseFile.getPath(), srcKeyName); + } + QemuImgFile srcFile = new QemuImgFile(baseFile.getPath()); + srcFile.setFormat(PhysicalDiskFormat.QCOW2); - QemuImg q = new QemuImg(wait); - q.convert(srcFile, destFile); + QemuImgFile destFile = new QemuImgFile(snapshotPath); + destFile.setFormat(PhysicalDiskFormat.QCOW2); - logger.debug(String.format("Converted volume [%s] (from path \"%s\") to snapshot [%s].", volume, baseFile, snapshotPath)); - return null; - } catch (QemuImgException | LibvirtException ex) { - return String.format("Failed to convert %s snapshot of volume [%s] to [%s] due to [%s].", volume, baseFile, snapshotPath, ex.getMessage()); - } + QemuImg q = new QemuImg(wait); + q.convert(srcFile, destFile, options, qemuObjects, qemuImageOpts, null, true); } /** @@ -2467,8 +2498,7 @@ public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { if (!storagePoolMgr.connectPhysicalDisk(destPrimaryStore.getPoolType(), destPrimaryStore.getUuid(), destVolumePath, destPrimaryStore.getDetails())) { logger.warn("Failed to connect dest volume at path: " + destVolumePath + ", in storage pool id: " + destPrimaryStore.getUuid()); } - String managedStoreTarget = destPrimaryStore.getDetails() != null ? destPrimaryStore.getDetails().get("managedStoreTarget") : null; - destVolumeName = managedStoreTarget != null ? managedStoreTarget : destVolumePath; + destVolumeName = derivePath(destPrimaryStore, destData, destPrimaryStore.getDetails()); } else { final String volumeName = UUID.randomUUID().toString(); destVolumeName = volumeName + "." + destFormat.getFileExtension(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 3002fea41d3d..6b9e52c3e267 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.UUID; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.utils.cryptsetup.KeyFile; import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; @@ -34,6 +35,7 @@ import org.apache.commons.codec.binary.Base64; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.commons.collections.CollectionUtils; import org.libvirt.Connect; import org.libvirt.LibvirtException; import org.libvirt.Secret; @@ -273,9 +275,19 @@ public void storagePoolRefresh(StoragePool pool) { } } - private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String uuid, String host, String path) throws LibvirtException { + private void checkNetfsStoragePoolMounted(String uuid) { String targetPath = _mountPoint + File.separator + uuid; - LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(fsType, uuid, uuid, host, path, targetPath); + int mountpointResult = Script.runSimpleBashScriptForExitValue("mountpoint -q " + targetPath); + if (mountpointResult != 0) { + String errMsg = String.format("libvirt failed to mount storage pool %s at %s", uuid, targetPath); + logger.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + } + + private StoragePool createNetfsStoragePool(PoolType fsType, Connect conn, String uuid, String host, String path, List nfsMountOpts) throws LibvirtException { + String targetPath = _mountPoint + File.separator + uuid; + LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(fsType, uuid, uuid, host, path, targetPath, nfsMountOpts); _storageLayer.mkdir(targetPath); StoragePool sp = null; try { @@ -364,6 +376,42 @@ private StoragePool createCLVMStoragePool(Connect conn, String uuid, String host } + private List getNFSMountOptsFromDetails(StoragePoolType type, Map details) { + List nfsMountOpts = null; + if (!type.equals(StoragePoolType.NetworkFilesystem) || details == null) { + return nfsMountOpts; + } + if (details.containsKey(ApiConstants.NFS_MOUNT_OPTIONS)) { + nfsMountOpts = Arrays.asList(details.get(ApiConstants.NFS_MOUNT_OPTIONS).replaceAll("\\s", "").split(",")); + } + return nfsMountOpts; + } + + private boolean destroyStoragePoolOnNFSMountOptionsChange(StoragePool sp, Connect conn, List nfsMountOpts) { + try { + LibvirtStoragePoolDef poolDef = getStoragePoolDef(conn, sp); + Set poolNfsMountOpts = poolDef.getNfsMountOpts(); + boolean mountOptsDiffer = false; + if (poolNfsMountOpts.size() != nfsMountOpts.size()) { + mountOptsDiffer = true; + } else { + for (String nfsMountOpt : nfsMountOpts) { + if (!poolNfsMountOpts.contains(nfsMountOpt)) { + mountOptsDiffer = true; + break; + } + } + } + if (mountOptsDiffer) { + sp.destroy(); + return true; + } + } catch (LibvirtException e) { + logger.error("Failure in destroying the pre-existing storage pool for changing the NFS mount options" + e); + } + return false; + } + private StoragePool createRBDStoragePool(Connect conn, String uuid, String host, int port, String userInfo, String path) { LibvirtStoragePoolDef spd; @@ -661,12 +709,21 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri } catch (LibvirtException e) { logger.error("Failure in attempting to see if an existing storage pool might be using the path of the pool to be created:" + e); } + } + + List nfsMountOpts = getNFSMountOptsFromDetails(type, details); + if (sp != null && CollectionUtils.isNotEmpty(nfsMountOpts) && + destroyStoragePoolOnNFSMountOptionsChange(sp, conn, nfsMountOpts)) { + sp = null; + } + + if (sp == null) { logger.debug("Attempting to create storage pool " + name); if (type == StoragePoolType.NetworkFilesystem) { try { - sp = createNetfsStoragePool(PoolType.NETFS, conn, name, host, path); + sp = createNetfsStoragePool(PoolType.NETFS, conn, name, host, path, nfsMountOpts); } catch (LibvirtException e) { logger.error("Failed to create netfs mount: " + host + ":" + path , e); logger.error(e.getStackTrace()); @@ -674,7 +731,7 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri } } else if (type == StoragePoolType.Gluster) { try { - sp = createNetfsStoragePool(PoolType.GLUSTERFS, conn, name, host, path); + sp = createNetfsStoragePool(PoolType.GLUSTERFS, conn, name, host, path, null); } catch (LibvirtException e) { logger.error("Failed to create glusterfs mount: " + host + ":" + path , e); logger.error(e.getStackTrace()); @@ -699,6 +756,10 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri sp.create(0); } + if (type == StoragePoolType.NetworkFilesystem) { + checkNetfsStoragePoolMounted(name); + } + return getStoragePool(name); } catch (LibvirtException e) { String error = e.toString(); @@ -762,11 +823,11 @@ public boolean deleteStoragePool(String uuid) { // handle ebusy error when pool is quickly destroyed if (e.toString().contains("exit status 16")) { String targetPath = _mountPoint + File.separator + uuid; - logger.error("deleteStoragePool removed pool from libvirt, but libvirt had trouble unmounting the pool. Trying umount location " + targetPath + - "again in a few seconds"); + logger.error("deleteStoragePool removed pool from libvirt, but libvirt had trouble unmounting the pool. Trying umount location " + targetPath + + " again in a few seconds"); String result = Script.runSimpleBashScript("sleep 5 && umount " + targetPath); if (result == null) { - logger.error("Succeeded in unmounting " + targetPath); + logger.info("Succeeded in unmounting " + targetPath); return true; } logger.error("Failed to unmount " + targetPath); @@ -848,7 +909,7 @@ private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool destFile.setFormat(format); destFile.setSize(size); Map options = new HashMap(); - if (pool.getType() == StoragePoolType.NetworkFilesystem){ + if (List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.Filesystem).contains(pool.getType())) { options.put("preallocation", QemuImg.PreallocationType.getPreallocationType(provisioningType).toString()); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java index 1625ecc171a4..03acfcc89adf 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java @@ -21,18 +21,15 @@ import java.io.File; import java.io.IOException; import java.io.InputStreamReader; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Timer; import java.util.TimerTask; -import java.util.UUID; import java.util.concurrent.TimeUnit; import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; -import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; import com.cloud.storage.Storage; @@ -44,7 +41,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.libvirt.LibvirtException; import org.joda.time.Duration; public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { @@ -56,6 +52,14 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { */ static byte[] CLEANUP_LOCK = new byte[0]; + /** + * List of supported OUI's (needed for path-based cleanup logic on disconnects after live migrations) + */ + static String[] SUPPORTED_OUI_LIST = { + "0002ac", // HPE Primera 3PAR + "24a937" // Pure Flasharray + }; + /** * Property keys and defaults */ @@ -83,6 +87,7 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { * Initialize static program-wide configurations and background jobs */ static { + long cleanupFrequency = CLEANUP_FREQUENCY_SECS.getFinalValue() * 1000; boolean cleanupEnabled = CLEANUP_ENABLED.getFinalValue(); @@ -97,16 +102,13 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { throw new Error("Unable to find the disconnectVolume.sh script"); } - resizeScript = Script.findScript(STORAGE_SCRIPTS_DIR.getFinalValue(), resizeScript); - if (resizeScript == null) { - throw new Error("Unable to find the resizeVolume.sh script"); - } - copyScript = Script.findScript(STORAGE_SCRIPTS_DIR.getFinalValue(), copyScript); if (copyScript == null) { throw new Error("Unable to find the copyVolume.sh script"); } + resizeScript = Script.findScript(STORAGE_SCRIPTS_DIR.getFinalValue(), resizeScript); + if (cleanupEnabled) { cleanupScript = Script.findScript(STORAGE_SCRIPTS_DIR.getFinalValue(), cleanupScript); if (cleanupScript == null) { @@ -138,9 +140,6 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { public abstract boolean isStoragePoolTypeSupported(Storage.StoragePoolType type); - /** - * We expect WWN values in the volumePath so need to convert it to an actual physical path - */ public abstract AddressInfo parseAndValidatePath(String path); @Override @@ -152,6 +151,7 @@ public KVMPhysicalDisk getPhysicalDisk(String volumePath, KVMStoragePool pool) { return null; } + // we expect WWN values in the volumePath so need to convert it to an actual physical path AddressInfo address = parseAndValidatePath(volumePath); return getPhysicalDisk(address, pool); } @@ -187,15 +187,23 @@ public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool, Map volumeToDisconnect) { - LOGGER.debug(String.format("disconnectPhysicalDiskByPath(volumeToDisconnect) called with arg bag [not implemented]:") + " " + volumeToDisconnect); + LOGGER.debug(String.format("disconnectPhysicalDisk(volumeToDisconnect) called with arg bag [not implemented]:") + " " + volumeToDisconnect); return false; } @Override public boolean disconnectPhysicalDiskByPath(String localPath) { - LOGGER.debug(String.format("disconnectPhysicalDiskByPath(localPath) called with args (%s) STARTED", localPath)); - ScriptResult result = runScript(disconnectScript, 60000L, localPath.replace("/dev/mapper/3", "")); - if (LOGGER.isDebugEnabled()) LOGGER.debug("multipath flush output: " + result.getResult()); - LOGGER.debug(String.format("disconnectPhysicalDiskByPath(localPath) called with args (%s) COMPLETE [rc=%s]", localPath, result.getExitCode())); return true; + if (localPath == null) { + return false; + } + if (LOGGER.isDebugEnabled()) LOGGER.debug(String.format("disconnectPhysicalDiskByPath(localPath) called with args (%s) START", localPath)); + if (localPath.startsWith("/dev/mapper/")) { + String multipathName = localPath.replace("/dev/mapper/3", ""); + // this ensures we only disconnect multipath devices supported by this driver + for (String oui: SUPPORTED_OUI_LIST) { + if (multipathName.length() > 1 && multipathName.substring(2).startsWith(oui)) { + ScriptResult result = runScript(disconnectScript, 60000L, multipathName); + if (result.getExitCode() != 0) { + LOGGER.warn(String.format("Disconnect failed for path [%s] with return code [%s]", multipathName, result.getExitCode())); + } + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("multipath flush output: " + result.getResult()); + LOGGER.debug(String.format("disconnectPhysicalDiskByPath(localPath) called with args (%s) COMPLETE [rc=%s]", localPath, result.getExitCode())); + } + return (result.getExitCode() == 0); + } + } + } + if (LOGGER.isDebugEnabled()) LOGGER.debug(String.format("disconnectPhysicalDiskByPath(localPath) returning FALSE, volume path is not a multipath volume: %s", localPath)); + return false; } @Override public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.ImageFormat format) { - LOGGER.info(String.format("deletePhysicalDisk(uuid,pool,format) called with args (%s, %s, %s) [not implemented]", uuid, pool.getUuid(), format.toString())); - return true; + return false; } @Override @@ -276,15 +315,9 @@ public boolean createFolder(String uuid, String path, String localPath) { return true; } - /** - * Validate inputs and return the source file for a template copy - * @param templateFilePath - * @param destTemplatePath - * @param destPool - * @param format - * @return - */ - File createTemplateFromDirectDownloadFileValidate(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format) { + + @Override + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { if (StringUtils.isAnyEmpty(templateFilePath, destTemplatePath) || destPool == null) { LOGGER.error("Unable to create template from direct download template file due to insufficient data"); throw new CloudRuntimeException("Unable to create template from direct download template file due to insufficient data"); @@ -297,57 +330,18 @@ File createTemplateFromDirectDownloadFileValidate(String templateFilePath, Strin throw new CloudRuntimeException("Direct download template file " + templateFilePath + " does not exist on this host"); } - if (destTemplatePath == null || destTemplatePath.isEmpty()) { - LOGGER.error("Failed to create template, target template disk path not provided"); - throw new CloudRuntimeException("Target template disk path not provided"); - } - - if (this.isStoragePoolTypeSupported(destPool.getType())) { - throw new CloudRuntimeException("Unsupported storage pool type: " + destPool.getType().toString()); - } - - if (Storage.ImageFormat.RAW.equals(format) && Storage.ImageFormat.QCOW2.equals(format)) { - LOGGER.error("Failed to create template, unsupported template format: " + format.toString()); - throw new CloudRuntimeException("Unsupported template format: " + format.toString()); - } - return sourceFile; - } - - String extractSourceTemplateIfNeeded(File sourceFile, String templateFilePath) { - String srcTemplateFilePath = templateFilePath; - if (isTemplateExtractable(templateFilePath)) { - srcTemplateFilePath = sourceFile.getParent() + "/" + UUID.randomUUID().toString(); - LOGGER.debug("Extract the downloaded template " + templateFilePath + " to " + srcTemplateFilePath); - String extractCommand = getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath); - Script.runSimpleBashScript(extractCommand); - Script.runSimpleBashScript("rm -f " + templateFilePath); - } - return srcTemplateFilePath; - } - - QemuImg.PhysicalDiskFormat deriveImgFileFormat(Storage.ImageFormat format) { - if (format == Storage.ImageFormat.RAW) { - return QemuImg.PhysicalDiskFormat.RAW; - } else if (format == Storage.ImageFormat.QCOW2) { - return QemuImg.PhysicalDiskFormat.QCOW2; - } else { - return QemuImg.PhysicalDiskFormat.RAW; - } - } - - @Override - public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { - File sourceFile = createTemplateFromDirectDownloadFileValidate(templateFilePath, destTemplatePath, destPool, format); - LOGGER.debug("Create template from direct download template - file path: " + templateFilePath + ", dest path: " + destTemplatePath + ", format: " + format.toString()); - KVMPhysicalDisk sourceDisk = destPool.getPhysicalDisk(sourceFile.getAbsolutePath()); + KVMPhysicalDisk sourceDisk = destPool.getPhysicalDisk(templateFilePath); return copyPhysicalDisk(sourceDisk, destTemplatePath, destPool, timeout, null, null, Storage.ProvisioningType.THIN); } @Override public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, byte[] srcPassphrase, byte[] dstPassphrase, Storage.ProvisioningType provisioningType) { + if (StringUtils.isEmpty(name) || disk == null || destPool == null) { + LOGGER.error("Unable to copy physical disk due to insufficient data"); + throw new CloudRuntimeException("Unable to copy physical disk due to insufficient data"); + } - validateForDiskCopy(disk, name, destPool); LOGGER.info("Copying FROM source physical disk " + disk.getPath() + ", size: " + disk.getSize() + ", virtualsize: " + disk.getVirtualSize()+ ", format: " + disk.getFormat()); KVMPhysicalDisk destDisk = destPool.getPhysicalDisk(name); @@ -367,60 +361,34 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt LOGGER.info("Copying TO destination physical disk " + destDisk.getPath() + ", size: " + destDisk.getSize() + ", virtualsize: " + destDisk.getVirtualSize()+ ", format: " + destDisk.getFormat()); QemuImgFile srcFile = new QemuImgFile(disk.getPath(), disk.getFormat()); QemuImgFile destFile = new QemuImgFile(destDisk.getPath(), destDisk.getFormat()); - LOGGER.debug("Starting COPY from source downloaded template " + srcFile.getFileName() + " to Primera volume: " + destDisk.getPath()); + + LOGGER.debug("Starting COPY from source path " + srcFile.getFileName() + " to target volume path: " + destDisk.getPath()); + ScriptResult result = runScript(copyScript, timeout, destDisk.getFormat().toString().toLowerCase(), srcFile.getFileName(), destFile.getFileName()); int rc = result.getExitCode(); if (rc != 0) { throw new CloudRuntimeException("Failed to convert from " + srcFile.getFileName() + " to " + destFile.getFileName() + " the error was: " + rc + " - " + result.getResult()); } - LOGGER.debug("Successfully converted source downloaded template " + srcFile.getFileName() + " to Primera volume: " + destDisk.getPath() + " " + result.getResult()); + LOGGER.debug("Successfully converted source volume at " + srcFile.getFileName() + " to destination volume: " + destDisk.getPath() + " " + result.getResult()); return destDisk; } - void validateForDiskCopy(KVMPhysicalDisk disk, String name, KVMStoragePool destPool) { - if (StringUtils.isEmpty(name) || disk == null || destPool == null) { - LOGGER.error("Unable to copy physical disk due to insufficient data"); - throw new CloudRuntimeException("Unable to copy physical disk due to insufficient data"); - } - } - - /** - * Copy a disk path to another disk path using QemuImg command - * @param disk - * @param destDisk - * @param name - * @param timeout - */ - void qemuCopy(KVMPhysicalDisk disk, KVMPhysicalDisk destDisk, String name, int timeout) { - QemuImg qemu; - try { - qemu = new QemuImg(timeout); - } catch (LibvirtException | QemuImgException e) { - throw new CloudRuntimeException (e); - } - QemuImgFile srcFile = null; - QemuImgFile destFile = null; - - try { - srcFile = new QemuImgFile(disk.getPath(), disk.getFormat()); - destFile = new QemuImgFile(destDisk.getPath(), destDisk.getFormat()); - - LOGGER.debug("Starting copy from source disk image " + srcFile.getFileName() + " to volume: " + destDisk.getPath()); - qemu.convert(srcFile, destFile, true); - LOGGER.debug("Successfully converted source disk image " + srcFile.getFileName() + " to volume: " + destDisk.getPath()); - } catch (QemuImgException | LibvirtException e) { - try { - Map srcInfo = qemu.info(srcFile); - LOGGER.debug("Source disk info: " + Arrays.asList(srcInfo)); - } catch (Exception ignored) { - LOGGER.warn("Unable to get info from source disk: " + disk.getName()); - } - - String errMsg = String.format("Unable to convert/copy from %s to %s, due to: %s", disk.getName(), name, ((StringUtils.isEmpty(e.getMessage())) ? "an unknown error" : e.getMessage())); - LOGGER.error(errMsg); - throw new CloudRuntimeException(errMsg, e); + private static final ScriptResult runScript(String script, long timeout, String...args) { + ScriptResult result = new ScriptResult(); + Script cmd = new Script(script, Duration.millis(timeout), LOGGER); + cmd.add(args); + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String output = cmd.execute(parser); + // its possible the process never launches which causes an NPE on getExitValue below + if (output != null && output.contains("Unable to execute the command")) { + result.setResult(output); + result.setExitCode(-1); + return result; } + result.setResult(output); + result.setExitCode(cmd.getExitValue()); + return result; } @Override @@ -461,25 +429,9 @@ String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String } } - private static final ScriptResult runScript(String script, long timeout, String...args) { - ScriptResult result = new ScriptResult(); - Script cmd = new Script(script, Duration.millis(timeout), LOGGER); - cmd.add(args); - OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); - String output = cmd.execute(parser); - // its possible the process never launches which causes an NPE on getExitValue below - if (output != null && output.contains("Unable to execute the command")) { - result.setResult(output); - result.setExitCode(-1); - return result; - } - result.setResult(output); - result.setExitCode(cmd.getExitValue()); - return result; - } - boolean waitForDiskToBecomeAvailable(AddressInfo address, KVMStoragePool pool, long waitTimeInSec) { LOGGER.debug("Waiting for the volume with id: " + address.getPath() + " of the storage pool: " + pool.getUuid() + " to become available for " + waitTimeInSec + " secs"); + long scriptTimeoutSecs = 30; // how long to wait for each script execution to run long maxTries = 10; // how many max retries to attempt the script long waitTimeInMillis = waitTimeInSec * 1000; // how long overall to wait @@ -557,40 +509,6 @@ boolean waitForDiskToBecomeAvailable(AddressInfo address, KVMStoragePool pool, l return false; } - void runConnectScript(String lun, AddressInfo address) { - try { - ProcessBuilder builder = new ProcessBuilder(connectScript, lun, address.getAddress()); - Process p = builder.start(); - int rc = p.waitFor(); - StringBuffer output = new StringBuffer(); - if (rc == 0) { - BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream())); - String line = null; - while ((line = input.readLine()) != null) { - output.append(line); - output.append(" "); - } - } else { - LOGGER.warn("Failure discovering LUN via " + connectScript); - BufferedReader error = new BufferedReader(new InputStreamReader(p.getErrorStream())); - String line = null; - while ((line = error.readLine()) != null) { - LOGGER.warn("error --> " + line); - } - } - } catch (IOException | InterruptedException e) { - throw new CloudRuntimeException("Problem performing scan on SCSI hosts", e); - } - } - - void sleep(long sleepTimeMs) { - try { - Thread.sleep(sleepTimeMs); - } catch (Exception ex) { - // don't do anything - } - } - long getPhysicalDiskSize(String diskPath) { if (StringUtils.isEmpty(diskPath)) { return 0; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index 9190510c7a72..b33f49404ad4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.UUID; +import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.utils.cryptsetup.CryptSetup; import org.apache.cloudstack.utils.cryptsetup.CryptSetupException; @@ -43,6 +44,8 @@ import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; +import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; @@ -154,6 +157,11 @@ public boolean deleteStoragePool(String uuid) { return MapStorageUuidToStoragePool.remove(uuid) != null; } + @Override + public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) { + return createPhysicalDisk(name, pool, format, provisioningType, size, null, passphrase); + } + /** * ScaleIO doesn't need to communicate with the hypervisor normally to create a volume. This is used only to prepare a ScaleIO data disk for encryption. * Thin encrypted volumes are provisioned in QCOW2 format, which insulates the guest from zeroes/unallocated blocks in the block device that would @@ -163,11 +171,12 @@ public boolean deleteStoragePool(String uuid) { * @param format disk format * @param provisioningType provisioning type * @param size disk size + * @param usableSize usage disk size * @param passphrase passphrase * @return the disk object */ @Override - public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) { + public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) { if (passphrase == null || passphrase.length == 0) { return null; } @@ -185,7 +194,12 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, Qemu QemuImg qemuImg = new QemuImg(0, true, false); Map options = new HashMap<>(); List qemuObjects = new ArrayList<>(); - long formattedSize = getUsableBytesFromRawBytes(disk.getSize()); + long formattedSize; + if (usableSize != null && usableSize > 0) { + formattedSize = usableSize; + } else { + formattedSize = getUsableBytesFromRawBytes(disk.getSize()); + } options.put("preallocation", QemuImg.PreallocationType.Metadata.toString()); qemuObjects.add(QemuObject.prepareSecretForQemuImg(disk.getFormat(), disk.getQemuEncryptFormat(), keyFile.toString(), "sec0", options)); @@ -553,6 +567,67 @@ public void resizeQcow2ToVolume(String volumePath, QemuImageOptions options, Lis qemu.resize(options, objects, usableSizeBytes); } + public Ternary, String> prepareStorageClient(Storage.StoragePoolType type, String uuid, Map details) { + if (!ScaleIOUtil.isSDCServiceInstalled()) { + logger.debug("SDC service not installed on host, preparing the SDC client not possible"); + return new Ternary<>(false, null, "SDC service not installed on host"); + } + + if (!ScaleIOUtil.isSDCServiceEnabled()) { + logger.debug("SDC service not enabled on host, enabling it"); + if (!ScaleIOUtil.enableSDCService()) { + return new Ternary<>(false, null, "SDC service not enabled on host"); + } + } + + if (!ScaleIOUtil.isSDCServiceActive()) { + if (!ScaleIOUtil.startSDCService()) { + return new Ternary<>(false, null, "Couldn't start SDC service on host"); + } + } else if (!ScaleIOUtil.restartSDCService()) { + return new Ternary<>(false, null, "Couldn't restart SDC service on host"); + } + + return new Ternary<>( true, getSDCDetails(details), "Prepared client successfully"); + } + + public Pair unprepareStorageClient(Storage.StoragePoolType type, String uuid) { + if (!ScaleIOUtil.isSDCServiceInstalled()) { + logger.debug("SDC service not installed on host, no need to unprepare the SDC client"); + return new Pair<>(true, "SDC service not installed on host, no need to unprepare the SDC client"); + } + + if (!ScaleIOUtil.isSDCServiceEnabled()) { + logger.debug("SDC service not enabled on host, no need to unprepare the SDC client"); + return new Pair<>(true, "SDC service not enabled on host, no need to unprepare the SDC client"); + } + + if (!ScaleIOUtil.stopSDCService()) { + return new Pair<>(false, "Couldn't stop SDC service on host"); + } + + return new Pair<>(true, "Unprepared SDC client successfully"); + } + + private Map getSDCDetails(Map details) { + Map sdcDetails = new HashMap(); + if (details == null || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)) { + return sdcDetails; + } + + String storageSystemId = details.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); + String sdcId = ScaleIOUtil.getSdcId(storageSystemId); + if (sdcId != null) { + sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId); + } else { + String sdcGuId = ScaleIOUtil.getSdcGuid(); + if (sdcGuId != null) { + sdcDetails.put(ScaleIOGatewayClient.SDC_GUID, sdcGuId); + } + } + return sdcDetails; + } + /** * Calculates usable size from raw size, assuming qcow2 requires 192k/1GB for metadata * We also remove 128MiB for encryption/fragmentation/safety factor. diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java index 293ff29f984f..77f21910da6c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java @@ -72,6 +72,11 @@ private void addSDCDetails() { } } + @Override + public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) { + return this.storageAdaptor.createPhysicalDisk(volumeUuid, this, format, provisioningType, size, usableSize, passphrase); + } + @Override public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) { return this.storageAdaptor.createPhysicalDisk(volumeUuid, this, format, provisioningType, size, passphrase); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java index 34bf08f4496a..9a27d44ff924 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.hypervisor.kvm.storage; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -23,6 +24,8 @@ import com.cloud.storage.Storage; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; public interface StorageAdaptor { @@ -41,6 +44,11 @@ public interface StorageAdaptor { public boolean deleteStoragePool(String uuid); + public default KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, + PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) { + return createPhysicalDisk(name, pool, format, provisioningType, size, passphrase); + } + public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); @@ -52,8 +60,17 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, public boolean disconnectPhysicalDisk(Map volumeToDisconnect); - // given local path to file/device (per Libvirt XML), 1) check that device is - // handled by your adaptor, return false if not. 2) clean up device, return true + /** + * Given local path to file/device (per Libvirt XML), + * 1) Make sure to check that device is handled by your adaptor, return false if not. + * 2) clean up device, return true + * 3) if clean up fails, then return false + * + * If the method wrongly returns true, then there are chances that disconnect will not reach the right storage adapter + * + * @param localPath path for the file/device from the disk definition per Libvirt XML. + * @return true if the operation is successful; false if the operation fails or the adapter fails to handle the path. + */ public boolean disconnectPhysicalDiskByPath(String localPath); public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.ImageFormat format); @@ -100,4 +117,25 @@ KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, default boolean supportsPhysicalDiskCopy(StoragePoolType type) { return StoragePoolType.PowerFlex == type; } + + /** + * Prepares the storage client. + * @param type type of the storage pool + * @param uuid uuid of the storage pool + * @param details any details of the storage pool that are required for client preparation + * @return status, client details, & message in case failed + */ + default Ternary, String> prepareStorageClient(StoragePoolType type, String uuid, Map details) { + return new Ternary<>(true, new HashMap<>(), ""); + } + + /** + * Unprepares the storage client. + * @param type type of the storage pool + * @param uuid uuid of the storage pool + * @return status, & message in case failed + */ + default Pair unprepareStorageClient(StoragePoolType type, String uuid) { + return new Pair<>(true, ""); + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index 19515ac83613..ecb34adc6eda 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -6295,4 +6295,40 @@ public void setMaxHostCpuSharesIfCGroupV2TestShouldNotCalculateMaxCpuCapacityIfH Assert.assertEquals(expectedShares, libvirtComputingResourceSpy.getHostCpuMaxCapacity()); } } + + @Test + public void testGetHostTags() throws ConfigurationException { + try (MockedStatic ignored = Mockito.mockStatic(AgentPropertiesFileHandler.class)) { + Mockito.when(AgentPropertiesFileHandler.getPropertyValue(Mockito.eq(AgentProperties.HOST_TAGS))) + .thenReturn("aa,bb,cc,dd"); + + List hostTagsList = libvirtComputingResourceSpy.getHostTags(); + Assert.assertEquals(4, hostTagsList.size()); + Assert.assertEquals("aa,bb,cc,dd", StringUtils.join(hostTagsList, ",")); + } + } + + @Test + public void testGetHostTagsWithSpace() throws ConfigurationException { + try (MockedStatic ignored = Mockito.mockStatic(AgentPropertiesFileHandler.class)) { + Mockito.when(AgentPropertiesFileHandler.getPropertyValue(Mockito.eq(AgentProperties.HOST_TAGS))) + .thenReturn(" aa, bb , cc , dd "); + + List hostTagsList = libvirtComputingResourceSpy.getHostTags(); + Assert.assertEquals(4, hostTagsList.size()); + Assert.assertEquals("aa,bb,cc,dd", StringUtils.join(hostTagsList, ",")); + } + } + + @Test + public void testGetHostTagsWithEmptyPropertyValue() throws ConfigurationException { + try (MockedStatic ignored = Mockito.mockStatic(AgentPropertiesFileHandler.class)) { + Mockito.when(AgentPropertiesFileHandler.getPropertyValue(Mockito.eq(AgentProperties.HOST_TAGS))) + .thenReturn(" "); + + List hostTagsList = libvirtComputingResourceSpy.getHostTags(); + Assert.assertEquals(0, hostTagsList.size()); + Assert.assertEquals("", StringUtils.join(hostTagsList, ",")); + } + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDefTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDefTest.java index 51a47e9a025f..712b38b0bb4b 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDefTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDefTest.java @@ -19,6 +19,9 @@ package com.cloud.hypervisor.kvm.resource; +import java.util.ArrayList; +import java.util.List; + import junit.framework.TestCase; import com.cloud.hypervisor.kvm.resource.LibvirtStoragePoolDef.PoolType; import com.cloud.hypervisor.kvm.resource.LibvirtStoragePoolDef.AuthenticationType; @@ -47,6 +50,14 @@ public void testSetGetStoragePool() { assertEquals(port, pool.getSourcePort()); assertEquals(dir, pool.getSourceDir()); assertEquals(targetPath, pool.getTargetPath()); + + List nfsMountOpts = new ArrayList<>(); + nfsMountOpts.add("vers=4.1"); + nfsMountOpts.add("nconnect=4"); + pool = new LibvirtStoragePoolDef(type, name, uuid, host, dir, targetPath, nfsMountOpts); + assertTrue(pool.getNfsMountOpts().contains("vers=4.1")); + assertTrue(pool.getNfsMountOpts().contains("nconnect=4")); + assertEquals(pool.getNfsMountOpts().size(), 2); } @Test @@ -57,12 +68,38 @@ public void testNfsStoragePool() { String host = "127.0.0.1"; String dir = "/export/primary"; String targetPath = "/mnt/" + uuid; + List nfsMountOpts = new ArrayList<>(); + nfsMountOpts.add("vers=4.1"); + nfsMountOpts.add("nconnect=4"); - LibvirtStoragePoolDef pool = new LibvirtStoragePoolDef(type, name, uuid, host, dir, targetPath); + LibvirtStoragePoolDef pool = new LibvirtStoragePoolDef(type, name, uuid, host, dir, targetPath, nfsMountOpts); - String expectedXml = "\n" + name + "\n" + uuid + "\n" + + String expectedXml = "\n" + + "" +name + "\n" + uuid + "\n" + "\n\n\n\n\n" + - "" + targetPath + "\n\n\n"; + "" + targetPath + "\n\n" + + "\n\n\n\n\n"; + + assertEquals(expectedXml, pool.toString()); + } + + @Test + public void testGlusterFSStoragePool() { + PoolType type = PoolType.GLUSTERFS; + String name = "myGFSPool"; + String uuid = "89a605bc-d470-4637-b3df-27388be452f5"; + String host = "127.0.0.1"; + String dir = "/export/primary"; + String targetPath = "/mnt/" + uuid; + List nfsMountOpts = new ArrayList<>(); + + LibvirtStoragePoolDef pool = new LibvirtStoragePoolDef(type, name, uuid, host, dir, targetPath, nfsMountOpts); + + String expectedXml = "\n" + + "" +name + "\n" + uuid + "\n" + + "\n\n\n" + + "\n\n\n" + + "" + targetPath + "\n\n\n"; assertEquals(expectedXml, pool.toString()); } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParserTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParserTest.java index 3637b7b1f9b1..5854c21186f9 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParserTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParserTest.java @@ -30,7 +30,7 @@ public class LibvirtStoragePoolXMLParserTest extends TestCase { @Test public void testParseNfsStoragePoolXML() { - String poolXML = "\n" + + String poolXML = "\n" + " feff06b5-84b2-3258-b5f9-1953217295de\n" + " feff06b5-84b2-3258-b5f9-1953217295de\n" + " 111111111\n" + @@ -49,12 +49,18 @@ public void testParseNfsStoragePoolXML() { " 0\n" + " \n" + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + ""; LibvirtStoragePoolXMLParser parser = new LibvirtStoragePoolXMLParser(); LibvirtStoragePoolDef pool = parser.parseStoragePoolXML(poolXML); - Assert.assertEquals("10.11.12.13", pool.getSourceHost()); + assertEquals("10.11.12.13", pool.getSourceHost()); + assertTrue(pool.getNfsMountOpts().contains("vers=4.1")); + assertTrue(pool.getNfsMountOpts().contains("nconnect=8")); } @Test diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java new file mode 100644 index 000000000000..3cad9c27a687 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java @@ -0,0 +1,67 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource.wrapper; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.commons.lang3.StringUtils; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckConvertInstanceCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtCheckConvertInstanceCommandWrapperTest { + + @Spy + private LibvirtCheckConvertInstanceCommandWrapper checkConvertInstanceCommandWrapper = Mockito.spy(LibvirtCheckConvertInstanceCommandWrapper.class); + + @Mock + private LibvirtComputingResource libvirtComputingResourceMock; + + @Mock + CheckConvertInstanceCommand checkConvertInstanceCommandMock; + + @Before + public void setUp() { + } + + @Test + public void testCheckInstanceCommand_success() { + Mockito.when(libvirtComputingResourceMock.hostSupportsInstanceConversion()).thenReturn(true); + Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock); + assertTrue(answer.getResult()); + } + + @Test + public void testCheckInstanceCommand_failure() { + Mockito.when(libvirtComputingResourceMock.hostSupportsInstanceConversion()).thenReturn(false); + Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock); + assertFalse(answer.getResult()); + assertTrue(StringUtils.isNotBlank(answer.getDetails())); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java index d70f5f088845..f0e94e59485d 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java @@ -18,6 +18,23 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + import com.cloud.agent.api.Answer; import com.cloud.agent.api.ConvertInstanceCommand; import com.cloud.agent.api.to.NfsTO; @@ -32,22 +49,6 @@ import com.cloud.storage.Storage; import com.cloud.utils.Pair; import com.cloud.utils.script.Script; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.MockedConstruction; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.Spy; -import org.mockito.junit.MockitoJUnitRunner; - -import java.util.Arrays; -import java.util.List; -import java.util.UUID; @RunWith(MockitoJUnitRunner.class) public class LibvirtConvertInstanceCommandWrapperTest { @@ -71,12 +72,6 @@ public class LibvirtConvertInstanceCommandWrapperTest { private static final String secondaryPoolUrl = "nfs://192.168.1.1/secondary"; private static final String vmName = "VmToImport"; - private static final String hostName = "VmwareHost1"; - private static final String vmwareVcenter = "192.168.1.2"; - private static final String vmwareDatacenter = "Datacenter"; - private static final String vmwareCluster = "Cluster"; - private static final String vmwareUsername = "administrator@vsphere.local"; - private static final String vmwarePassword = "password"; @Before public void setUp() { @@ -88,15 +83,6 @@ public void setUp() { Mockito.when(temporaryPool.listPhysicalDisks()).thenReturn(Arrays.asList(physicalDisk1, physicalDisk2)); } - @Test - public void testIsInstanceConversionSupportedOnHost() { - try (MockedStatic diff --git a/ui/src/components/page/GlobalFooter.vue b/ui/src/components/page/GlobalFooter.vue index 86c8948e5640..854cecc78ace 100644 --- a/ui/src/components/page/GlobalFooter.vue +++ b/ui/src/components/page/GlobalFooter.vue @@ -23,7 +23,7 @@
    CloudStack {{ $store.getters.features.cloudstackversion }} - + {{ $t('label.report.bug') }} diff --git a/ui/src/components/view/stats/FilterStats.vue b/ui/src/components/view/DateTimeFilter.vue similarity index 93% rename from ui/src/components/view/stats/FilterStats.vue rename to ui/src/components/view/DateTimeFilter.vue index 20cd63af873d..aca958f58f74 100644 --- a/ui/src/components/view/stats/FilterStats.vue +++ b/ui/src/components/view/DateTimeFilter.vue @@ -22,12 +22,12 @@ :model="form" @finish="handleSubmit" layout="vertical"> -
    +
    - +
    @@ -64,7 +64,7 @@ import { ref, reactive, toRaw } from 'vue' import moment from 'moment' export default { - name: 'FilterStats', + name: 'DateTimeFilter', emits: ['closeAction', 'onSubmit'], props: { startDateProp: { @@ -74,6 +74,14 @@ export default { endDateProp: { type: [Date, String, Number], required: false + }, + showAllDataOption: { + type: Boolean, + default: true + }, + allDataMessage: { + type: String, + value: '' } }, computed: { diff --git a/ui/src/components/view/DetailsTab.vue b/ui/src/components/view/DetailsTab.vue index f731766f8781..c9ab6b89ec8c 100644 --- a/ui/src/components/view/DetailsTab.vue +++ b/ui/src/components/view/DetailsTab.vue @@ -103,6 +103,12 @@
    {{ dataResource.serviceofferingdetails[item] }}
    +
    + {{ dataResource[item] }} +
    +
    + {{ JSON.stringify(JSON.parse(dataResource[item]), null, 4) || dataResource[item] }} +
    {{ dataResource[item] }}
    @@ -120,6 +126,13 @@
    {{ dataResource[item] }}
    + +
    + {{ $t('label.' + item.replace('date', '.date.and.time'))}} +
    +
    {{ $toLocaleDate(dataResource[item]) }}
    +
    +     @@ -174,7 +187,12 @@ export default { }, computed: { customDisplayItems () { - return ['ip6routes', 'privatemtu', 'publicmtu', 'provider'] + var items = ['ip6routes', 'privatemtu', 'publicmtu', 'provider'] + if (this.$route.meta.name === 'webhookdeliveries') { + items.push('startdate') + items.push('enddate') + } + return items }, vnfAccessMethods () { if (this.resource.templatetype === 'VNF' && ['vm', 'vnfapp'].includes(this.$route.meta.name)) { diff --git a/ui/src/components/view/InfoCard.vue b/ui/src/components/view/InfoCard.vue index 66c878da0f81..267a759b28f1 100644 --- a/ui/src/components/view/InfoCard.vue +++ b/ui/src/components/view/InfoCard.vue @@ -140,6 +140,12 @@
    +
    +
    {{ $t('label.success') }}
    +
    + +
    +
    {{ $t('label.id') }}
    @@ -479,11 +485,12 @@
    -
    {{ $t('label.publicip') }}
    +
    {{ $t('label.public.ip') }}
    {{ resource.publicip }} - + +
    @@ -520,15 +527,15 @@
    -
    {{ $t('label.templatename') }}
    +
    {{ resource.templateformat === 'ISO'? $t('label.iso') : $t('label.templatename') }}
    - {{ resource.templatedisplaytext || resource.templatename || resource.templateid }} + {{ resource.templatedisplaytext || resource.templatename || resource.templateid }}
    -
    {{ $t('label.iso') }}
    +
    {{ $t('label.isoname') }}
    @@ -672,6 +679,22 @@ {{ resource.domain || resource.domainid }}
    +
    +
    {{ $t('label.payloadurl') }}
    +
    + + {{ resource.payloadurl }} + {{ resource.payloadurl }} +
    +
    +
    +
    {{ $t('label.webhook') }}
    +
    + + {{ resource.webhookname || resource.webhookid }} + {{ resource.webhookname || resource.webhookid }} +
    +
    {{ $t('label.management.servers') }}
    diff --git a/ui/src/components/view/InstanceVolumesStoragePoolSelectListView.vue b/ui/src/components/view/InstanceVolumesStoragePoolSelectListView.vue index 77f1b495d71a..77f3e8f91f47 100644 --- a/ui/src/components/view/InstanceVolumesStoragePoolSelectListView.vue +++ b/ui/src/components/view/InstanceVolumesStoragePoolSelectListView.vue @@ -101,9 +101,14 @@ export default { key: 'size', title: this.$t('label.size') }, + { + key: 'storage', + title: this.$t('label.current.storage'), + dataIndex: 'storage' + }, { key: 'selectedstorage', - title: this.$t('label.storage') + title: this.$t('label.selected.storage') }, { key: 'select', diff --git a/ui/src/components/view/ListResourceTable.vue b/ui/src/components/view/ListResourceTable.vue index 16c1b388c5df..a7e805b54439 100644 --- a/ui/src/components/view/ListResourceTable.vue +++ b/ui/src/components/view/ListResourceTable.vue @@ -41,21 +41,20 @@ -
    - + - - + - -
    diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index cf3d9361638e..2b1036a293dc 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -23,7 +23,7 @@ :dataSource="items" :rowKey="(record, idx) => record.id || record.name || record.usageType || idx + '-' + Math.random()" :pagination="false" - :rowSelection=" enableGroupAction() || $route.name === 'event' ? {selectedRowKeys: selectedRowKeys, onChange: onSelectChange, columnWidth: 30} : null" + :rowSelection="explicitlyAllowRowSelection || enableGroupAction() || $route.name === 'event' ? {selectedRowKeys: selectedRowKeys, onChange: onSelectChange, columnWidth: 30} : null" :rowClassName="getRowClassName" style="overflow-y: auto" > @@ -364,12 +364,47 @@ {{ record.enabled ? 'Enabled' : 'Disabled' }} - @@ -64,11 +68,20 @@ export default { } }, inject: ['parentFetchData'], + computed: { + columns () { + if (this.volumes?.[0]) { + return this.allColumns.filter(col => col.dataIndex in this.volumes[0]) + } + return this.allColumns.filter(col => this.defaultColumns.includes(col.dataIndex)) + } + }, data () { return { vm: {}, volumes: [], - volumeColumns: [ + defaultColumns: ['name', 'state', 'type', 'size'], + allColumns: [ { key: 'name', title: this.$t('label.name'), @@ -87,6 +100,11 @@ export default { key: 'size', title: this.$t('label.size'), dataIndex: 'size' + }, + { + key: 'storage', + title: this.$t('label.storage'), + dataIndex: 'storage' } ] } diff --git a/ui/src/components/view/WebhookDeliveriesTab.vue b/ui/src/components/view/WebhookDeliveriesTab.vue new file mode 100644 index 000000000000..c13b0453254a --- /dev/null +++ b/ui/src/components/view/WebhookDeliveriesTab.vue @@ -0,0 +1,526 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/components/widgets/CopyLabel.vue b/ui/src/components/widgets/CopyLabel.vue index 650f678206eb..c5216cc256df 100644 --- a/ui/src/components/widgets/CopyLabel.vue +++ b/ui/src/components/widgets/CopyLabel.vue @@ -18,7 +18,7 @@ @@ -32,6 +32,10 @@ export default { type: String, default: '' }, + copyValue: { + type: String, + default: '' + }, tooltip: { type: String, default: '' @@ -39,6 +43,10 @@ export default { tooltipPlacement: { type: String, default: 'top' + }, + showIcon: { + type: Boolean, + default: false } } } diff --git a/ui/src/components/widgets/ResourceLabel.vue b/ui/src/components/widgets/ResourceLabel.vue index 6e44ce54a5df..0a10843ff86d 100644 --- a/ui/src/components/widgets/ResourceLabel.vue +++ b/ui/src/components/widgets/ResourceLabel.vue @@ -18,7 +18,13 @@ -
    -

    {{ $t('label.accounttype') }}

    - - {{ $t('label.account') }} - {{ $t('label.project') }} - -
    - -
    -

    *{{ $t('label.domain') }}

    - - - - - - {{ domain.path || domain.name || domain.description }} - - - -
    - - - - +

    {{ $t('label.network') }}

    @@ -146,6 +67,7 @@ diff --git a/ui/src/views/compute/wizard/TemplateIsoRadioGroup.vue b/ui/src/views/compute/wizard/TemplateIsoRadioGroup.vue index 00fa2ef9a253..1cd1fa5f2910 100644 --- a/ui/src/views/compute/wizard/TemplateIsoRadioGroup.vue +++ b/ui/src/views/compute/wizard/TemplateIsoRadioGroup.vue @@ -46,6 +46,9 @@ :os-name="item.osName" />   {{ item.displaytext }} + + | {{ item.project }} + diff --git a/ui/src/views/dashboard/CapacityDashboard.vue b/ui/src/views/dashboard/CapacityDashboard.vue index 2fc41b6c1bea..dae53cf0015b 100644 --- a/ui/src/views/dashboard/CapacityDashboard.vue +++ b/ui/src/views/dashboard/CapacityDashboard.vue @@ -184,7 +184,7 @@
    - + - + - + - + - + - + - + - + - +