diff --git a/go.mod b/go.mod index 1630ce9f6c9..b55a71ce277 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/BurntSushi/toml v1.0.0 github.com/Masterminds/semver/v3 v3.1.1 github.com/aliyun/alibaba-cloud-sdk-go v1.61.985 + github.com/bramvdbogaerde/go-scp v1.2.0 github.com/cavaliergopher/grab/v3 v3.0.1 github.com/containers/buildah v1.25.0 github.com/containers/common v0.47.5 diff --git a/go.sum b/go.sum index a3eb5ada469..17fee3153ce 100644 --- a/go.sum +++ b/go.sum @@ -266,6 +266,8 @@ github.com/bombsimon/wsl/v2 v2.2.0/go.mod h1:Azh8c3XGEJl9LyX0/sFC+CKMc7Ssgua0g+6 github.com/bombsimon/wsl/v3 v3.0.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/bramvdbogaerde/go-scp v1.2.0 h1:mNF1lCXQ6jQcxCBBuc2g/CQwVy/4QONaoD5Aqg9r+Zg= +github.com/bramvdbogaerde/go-scp v1.2.0/go.mod h1:s4ZldBoRAOgUg8IrRP2Urmq5qqd2yPXQTPshACY8vQ0= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= diff --git a/pkg/hostpool/host.go b/pkg/hostpool/host.go new file mode 100644 index 00000000000..2a03f21fcbc --- /dev/null +++ b/pkg/hostpool/host.go @@ -0,0 +1,115 @@ +// Copyright © 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostpool + +import ( + "fmt" + "net" + "strconv" + + goscp "github.com/bramvdbogaerde/go-scp" + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +// Host contains both static and dynamic information of a host machine. +// Static part: the host config +// dynamic part, including ssh client and sftp client. +type Host struct { + config HostConfig + + // sshClient is used to create ssh.Session. + // TODO: remove this and just make ssh.Session remain. + sshClient *ssh.Client + // sshSession is created by ssh.Client and used for command execution on specified host. + sshSession *ssh.Session + // sftpClient is used to file remote operation on specified host except scp operation. + sftpClient *sftp.Client + // scpClient is used to scp files between sealer node and all nodes. + scpClient *goscp.Client + + // isLocal identifies that whether the initialized host is the sealer binary located node. + isLocal bool +} + +// HostConfig is the host config, including IP, port, login credentials and so on. +type HostConfig struct { + // IP is the IP address of host. + // It supports both IPv4 and IPv6. + IP net.IP + + // Port is the port config used by ssh to connect host + // The connecting operation will use port 22 if port is not set. + Port int + + // Usually User will be root. If it is set a non-root user, + // then this non-root must has a sudo permission. + User string + Password string + + // Encrypted means the password is encrypted. + // Password above should be decrypted first before being called. + Encrypted bool + + // TODO: add PkFile support + // PkFile string + // PkPassword string +} + +// Initialize setups ssh and sftp clients. +func (host *Host) Initialize() error { + config := &ssh.ClientConfig{ + User: host.config.User, + Auth: []ssh.AuthMethod{ + ssh.Password(host.config.Password), + }, + HostKeyCallback: nil, + } + + hostAddr := host.config.IP.String() + port := strconv.Itoa(host.config.Port) + + // sshClient + sshClient, err := ssh.Dial("tcp", net.JoinHostPort(hostAddr, port), config) + if err != nil { + return fmt.Errorf("failed to create ssh client for host(%s): %v", hostAddr, err) + } + host.sshClient = sshClient + + // sshSession + sshSession, err := sshClient.NewSession() + if err != nil { + return fmt.Errorf("failed to create ssh session for host(%s): %v", hostAddr, err) + } + host.sshSession = sshSession + + // sftpClient + sftpClient, err := sftp.NewClient(sshClient, nil) + if err != nil { + return fmt.Errorf("failed to create sftp client for host(%s): %v", hostAddr, err) + } + host.sftpClient = sftpClient + + // scpClient + scpClient, err := goscp.NewClientBySSH(sshClient) + if err != nil { + return fmt.Errorf("failed to create scp client for host(%s): %v", hostAddr, err) + } + host.scpClient = &scpClient + + // TODO: set isLocal + + return nil +} diff --git a/pkg/hostpool/host_pool.go b/pkg/hostpool/host_pool.go new file mode 100644 index 00000000000..2b4e98568ed --- /dev/null +++ b/pkg/hostpool/host_pool.go @@ -0,0 +1,72 @@ +// Copyright © 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostpool + +import ( + "fmt" +) + +// HostPool is a host resource pool of sealer's cluster, including masters and nodes. +// While SEALER DEPLOYING NODE has no restrict relationship with masters nor nodes: +// 1. sealer deploying node could be a node which is no master nor node; +// 2. sealer deploying node could also be one of masters and nodes. +// Then deploying node is not included in HostPool. +type HostPool struct { + // host is a map: + // key has a type of string which is from net.Ip.String() + hosts map[string]*Host +} + +// New initializes a brand new HostPool instance. +func New(hostConfigs []*HostConfig) (*HostPool, error) { + if len(hostConfigs) == 0 { + return nil, fmt.Errorf("input HostConfigs cannot be empty") + } + var hostPool HostPool + for _, hostConfig := range hostConfigs { + if _, OK := hostPool.hosts[hostConfig.IP.String()]; OK { + return nil, fmt.Errorf("there must not be duplicated host IP(%s) in cluster hosts", hostConfig.IP.String()) + } + hostPool.hosts[hostConfig.IP.String()] = &Host{ + config: HostConfig{ + IP: hostConfig.IP, + Port: hostConfig.Port, + User: hostConfig.User, + Password: hostConfig.Password, + Encrypted: hostConfig.Encrypted, + }, + } + } + return &hostPool, nil +} + +// Initialize helps HostPool to setup all attributes for each host, +// like scpClient, sshClient and so on. +func (hp *HostPool) Initialize() error { + for _, host := range hp.hosts { + if err := host.Initialize(); err != nil { + return fmt.Errorf("failed to initialize host in HostPool: %v", err) + } + } + return nil +} + +// GetHost gets the detailed host connection instance via IP string as a key. +func (hp *HostPool) GetHost(ipStr string) (*Host, error) { + if host, exist := hp.hosts[ipStr]; exist { + return host, nil + } + return nil, fmt.Errorf("cannot get host connection in HostPool by key(%s)", ipStr) +} diff --git a/pkg/hostpool/scp.go b/pkg/hostpool/scp.go new file mode 100644 index 00000000000..28f1210ec49 --- /dev/null +++ b/pkg/hostpool/scp.go @@ -0,0 +1,93 @@ +// Copyright © 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostpool + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// CopyFile copies the contents of localFilePath to remote destination path. +// Both localFilePath and remotePath must be an absolute path. +// +// It must be executed in deploying node and towards the host instance. +func (host *Host) CopyToRemote(localFilePath string, remotePath string, permissions string) error { + if host.isLocal { + // TODO: add local file copy. + return fmt.Errorf("local file copy is not implemented") + } + + f, err := os.Open(filepath.Clean(localFilePath)) + if err != nil { + return err + } + return host.scpClient.CopyFromFile(context.Background(), *f, remotePath, permissions) +} + +// CopyFile copies the contents of remotePath to local destination path. +// Both localFilePath and remotePath must be an absolute path. +// +// It must be executed in deploying node and towards the host instance. +func (host *Host) CopyFromRemote(localFilePath string, remotePath string) error { + if host.isLocal { + // TODO: add local file copy. + return fmt.Errorf("local file copy is not implemented") + } + + f, err := os.Open(filepath.Clean(localFilePath)) + if err != nil { + return err + } + return host.scpClient.CopyFromRemote(context.Background(), f, remotePath) +} + +// CopyToRemoteDir copies the contents of local directory to remote destination directory. +// Both localFilePath and remotePath must be an absolute path. +// +// It must be executed in deploying node and towards the host instance. +func (host *Host) CopyToRemoteDir(localDir string, remoteDir string) error { + if host.isLocal { + // TODO: add local file copy. + return fmt.Errorf("local file copy is not implemented") + } + + // get the localDir Directory name + fInfo, err := os.Lstat(localDir) + if err != nil { + return err + } + if !fInfo.IsDir() { + return fmt.Errorf("input localDir(%s) is not a directory when copying directory content", localDir) + } + dirName := fInfo.Name() + + err = filepath.Walk(localDir, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + // Since localDir is an absolute path, then every passed path has a prefix of localDir, + // then the relative path is the input path trims localDir. + fileRelativePath := strings.TrimPrefix(path, localDir) + remotePath := filepath.Join(remoteDir, dirName, fileRelativePath) + + return host.CopyToRemote(path, remotePath, info.Mode().String()) + }) + + return err +} diff --git a/pkg/hostpool/session.go b/pkg/hostpool/session.go new file mode 100644 index 00000000000..6e15845f4b0 --- /dev/null +++ b/pkg/hostpool/session.go @@ -0,0 +1,71 @@ +// Copyright © 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostpool + +import ( + "bytes" + "fmt" + "os/exec" +) + +// Output runs cmd on the remote host and returns its standard output. +// It must be executed in deploying node and towards the host instance. +func (host *Host) Output(cmd string) ([]byte, error) { + if host.isLocal { + return exec.Command(cmd).Output() + } + return host.sshSession.Output(cmd) +} + +// CombinedOutput wraps the sshSession.CombinedOutput and does the same in both input and output. +// It must be executed in deploying node and towards the host instance. +func (host *Host) CombinedOutput(cmd string) ([]byte, error) { + if host.isLocal { + return exec.Command(cmd).CombinedOutput() + } + return host.sshSession.CombinedOutput(cmd) +} + +// RunAndStderr runs a specified command and output stderr content. +// If command returns a nil, then no matter if there is content in session's stderr, just ignore stderr; +// If command return a non-nil, construct and return a new error with stderr content +// which may contains the exact error message. +// +// TODO: there is a potential issue that if much content is in stdout or stderr, and +// it may eventually cause the remote command to block. +// +// It must be executed in deploying node and towards the host instance. +func (host *Host) RunAndStderr(cmd string) ([]byte, error) { + var stdout, stderr bytes.Buffer + if host.isLocal { + localCmd := exec.Command(cmd) + localCmd.Stdout = &stdout + localCmd.Stderr = &stderr + if err := localCmd.Run(); err != nil { + return nil, fmt.Errorf("failed to exec cmd(%s) on host(%s): %s", cmd, host.config.IP, stderr.String()) + } + return stdout.Bytes(), nil + } + + host.sshSession.Stdout = &stdout + host.sshSession.Stderr = &stderr + if err := host.sshSession.Run(cmd); err != nil { + return nil, fmt.Errorf("failed to exec cmd(%s) on host(%s): %s", cmd, host.config.IP, stderr.String()) + } + + return stdout.Bytes(), nil +} + +// TODO: Do we need asynchronously output stdout and stderr? diff --git a/pkg/hostpool/sftp.go b/pkg/hostpool/sftp.go new file mode 100644 index 00000000000..a6649a523ff --- /dev/null +++ b/pkg/hostpool/sftp.go @@ -0,0 +1,15 @@ +// Copyright © 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostpool diff --git a/vendor/github.com/bramvdbogaerde/go-scp/.gitignore b/vendor/github.com/bramvdbogaerde/go-scp/.gitignore new file mode 100644 index 00000000000..ab27d1e8232 --- /dev/null +++ b/vendor/github.com/bramvdbogaerde/go-scp/.gitignore @@ -0,0 +1,2 @@ +/.idea +/vendor \ No newline at end of file diff --git a/vendor/github.com/bramvdbogaerde/go-scp/LICENSE.txt b/vendor/github.com/bramvdbogaerde/go-scp/LICENSE.txt new file mode 100644 index 00000000000..14e2f777f6c --- /dev/null +++ b/vendor/github.com/bramvdbogaerde/go-scp/LICENSE.txt @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/vendor/github.com/bramvdbogaerde/go-scp/README.md b/vendor/github.com/bramvdbogaerde/go-scp/README.md new file mode 100644 index 00000000000..041988da87b --- /dev/null +++ b/vendor/github.com/bramvdbogaerde/go-scp/README.md @@ -0,0 +1,97 @@ +Copy files over SCP with Go +============================= +[![Go Report Card](https://goreportcard.com/badge/bramvdbogaerde/go-scp)](https://goreportcard.com/report/bramvdbogaerde/go-scp) [![](https://godoc.org/github.com/bramvdbogaerde/go-scp?status.svg)](https://godoc.org/github.com/bramvdbogaerde/go-scp) + +This package makes it very easy to copy files over scp in Go. +It uses the golang.org/x/crypto/ssh package to establish a secure connection to a remote server in order to copy the files via the SCP protocol. + +### Example usage + + +```go +package main + +import ( + "fmt" + scp "github.com/bramvdbogaerde/go-scp" + "github.com/bramvdbogaerde/go-scp/auth" + "golang.org/x/crypto/ssh" + "os" + "context" +) + +func main() { + // Use SSH key authentication from the auth package + // we ignore the host key in this example, please change this if you use this library + clientConfig, _ := auth.PrivateKey("username", "/path/to/rsa/key", ssh.InsecureIgnoreHostKey()) + + // For other authentication methods see ssh.ClientConfig and ssh.AuthMethod + + // Create a new SCP client + client := scp.NewClient("example.com:22", &clientConfig) + + // Connect to the remote server + err := client.Connect() + if err != nil { + fmt.Println("Couldn't establish a connection to the remote server ", err) + return + } + + // Open a file + f, _ := os.Open("/path/to/local/file") + + // Close client connection after the file has been copied + defer client.Close() + + // Close the file after it has been copied + defer f.Close() + + // Finaly, copy the file over + // Usage: CopyFromFile(context, file, remotePath, permission) + + // the context can be adjusted to provide time-outs or inherit from other contexts if this is embedded in a larger application. + err = client.CopyFromFile(context.Background(), *f, "/home/server/test.txt", "0655") + + if err != nil { + fmt.Println("Error while copying file ", err) + } +} +``` + +#### Using an existing SSH connection + +If you have an existing established SSH connection, you can use that instead. + +```go +func connectSSH() *ssh.Client { + // setup SSH connection +} + +func main() { + sshClient := connectSSH() + + // Create a new SCP client, note that this function might + // return an error, as a new SSH session is established using the existing connecton + + client, err := scp.NewClientBySSH(sshClient) + if err != nil { + fmt.Println("Error creating new SSH session from existing connection", err) + } + + /* .. same as above .. */ +} +``` + +#### Copying Files from Remote Server + +It is also possible to copy remote files using this library. +The usage is similar to the example at the top of this section, except that `CopyFromRemote` needsto be used instead. + +For a more comprehensive example, please consult the `TestDownloadFile` function in t he `tests/basic_test.go` file. + +### License + +This library is licensed under the Mozilla Public License 2.0. +A copy of the license is provided in the `LICENSE.txt` file. + +Copyright (c) 2020 Bram Vandenbogaerde diff --git a/vendor/github.com/bramvdbogaerde/go-scp/client.go b/vendor/github.com/bramvdbogaerde/go-scp/client.go new file mode 100644 index 00000000000..3aed6e77488 --- /dev/null +++ b/vendor/github.com/bramvdbogaerde/go-scp/client.go @@ -0,0 +1,331 @@ +/* Copyright (c) 2021 Bram Vandenbogaerde And Contributors + * You may use, distribute or modify this code under the + * terms of the Mozilla Public License 2.0, which is distributed + * along with the source code. + */ +package scp + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "sync" + "time" + + "golang.org/x/crypto/ssh" +) + +type PassThru func(r io.Reader, total int64) io.Reader + +type Client struct { + // the host to connect to + Host string + + // the client config to use + ClientConfig *ssh.ClientConfig + + // stores the SSH session while the connection is running + Session *ssh.Session + + // stores the SSH connection itself in order to close it after transfer + Conn ssh.Conn + + // the maximal amount of time to wait for a file transfer to complete + // Deprecated: use context.Context for each function instead. + Timeout time.Duration + + // the absolute path to the remote SCP binary + RemoteBinary string +} + +// Connects to the remote SSH server, returns error if it couldn't establish a session to the SSH server +func (a *Client) Connect() error { + if a.Session != nil { + return nil + } + + client, err := ssh.Dial("tcp", a.Host, a.ClientConfig) + if err != nil { + return err + } + + a.Conn = client.Conn + a.Session, err = client.NewSession() + if err != nil { + return err + } + return nil +} + +// Copies the contents of an os.File to a remote location, it will get the length of the file by looking it up from the filesystem +func (a *Client) CopyFromFile(ctx context.Context, file os.File, remotePath string, permissions string) error { + return a.CopyFromFilePassThru(ctx, file, remotePath, permissions, nil) +} + +// Copies the contents of an os.File to a remote location, it will get the length of the file by looking it up from the filesystem. +// Access copied bytes by providing a PassThru reader factory +func (a *Client) CopyFromFilePassThru(ctx context.Context, file os.File, remotePath string, permissions string, passThru PassThru) error { + stat, _ := file.Stat() + return a.CopyPassThru(ctx, &file, remotePath, permissions, stat.Size(), passThru) +} + +// Copies the contents of an io.Reader to a remote location, the length is determined by reading the io.Reader until EOF +// if the file length in know in advance please use "Copy" instead +func (a *Client) CopyFile(ctx context.Context, fileReader io.Reader, remotePath string, permissions string) error { + return a.CopyFilePassThru(ctx, fileReader, remotePath, permissions, nil) +} + +// Copies the contents of an io.Reader to a remote location, the length is determined by reading the io.Reader until EOF +// if the file length in know in advance please use "Copy" instead. +// Access copied bytes by providing a PassThru reader factory +func (a *Client) CopyFilePassThru(ctx context.Context, fileReader io.Reader, remotePath string, permissions string, passThru PassThru) error { + contents_bytes, _ := ioutil.ReadAll(fileReader) + bytes_reader := bytes.NewReader(contents_bytes) + + return a.CopyPassThru(ctx, bytes_reader, remotePath, permissions, int64(len(contents_bytes)), passThru) +} + +// waitTimeout waits for the waitgroup for the specified max timeout. +// Returns true if waiting timed out. +func wait(wg *sync.WaitGroup, ctx context.Context) error { + c := make(chan struct{}) + go func() { + defer close(c) + wg.Wait() + }() + + select { + case <-c: + return nil + + case <-ctx.Done(): + return ctx.Err() + } +} + +// Checks the response it reads from the remote, and will return a single error in case +// of failure +func checkResponse(r io.Reader) error { + response, err := ParseResponse(r) + if err != nil { + return err + } + + if response.IsFailure() { + return errors.New(response.GetMessage()) + } + + return nil + +} + +// Copies the contents of an io.Reader to a remote location +func (a *Client) Copy(ctx context.Context, r io.Reader, remotePath string, permissions string, size int64) error { + return a.CopyPassThru(ctx, r, remotePath, permissions, size, nil) +} + +// Copies the contents of an io.Reader to a remote location. +// Access copied bytes by providing a PassThru reader factory +func (a *Client) CopyPassThru(ctx context.Context, r io.Reader, remotePath string, permissions string, size int64, passThru PassThru) error { + stdout, err := a.Session.StdoutPipe() + if err != nil { + return err + } + + if passThru != nil { + r = passThru(r, size) + } + + filename := path.Base(remotePath) + + wg := sync.WaitGroup{} + wg.Add(2) + + errCh := make(chan error, 2) + + go func() { + defer wg.Done() + w, err := a.Session.StdinPipe() + if err != nil { + errCh <- err + return + } + + defer w.Close() + + _, err = fmt.Fprintln(w, "C"+permissions, size, filename) + if err != nil { + errCh <- err + return + } + + if err = checkResponse(stdout); err != nil { + errCh <- err + return + } + + _, err = io.Copy(w, r) + if err != nil { + errCh <- err + return + } + + _, err = fmt.Fprint(w, "\x00") + if err != nil { + errCh <- err + return + } + + if err = checkResponse(stdout); err != nil { + errCh <- err + return + } + }() + + go func() { + defer wg.Done() + err := a.Session.Run(fmt.Sprintf("%s -qt %q", a.RemoteBinary, remotePath)) + if err != nil { + errCh <- err + return + } + }() + + if a.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, a.Timeout) + defer cancel() + } + + if err := wait(&wg, ctx); err != nil { + return err + } + + close(errCh) + for err := range errCh { + if err != nil { + return err + } + } + return nil +} + +// Copy a file from the remote to the local file given by the `file` +// parameter. Use `CopyFromRemotePassThru` if a more generic writer +// is desired instead of writing directly to a file on the file system.? +func (a *Client) CopyFromRemote(ctx context.Context, file *os.File, remotePath string) error { + return a.CopyFromRemotePassThru(ctx, file, remotePath, nil) +} + +// Copy a file from the remote to the given writer. The passThru parameter can be used +// to keep track of progress and how many bytes that were download from the remote. +// `passThru` can be set to nil to disable this behaviour. +func (a *Client) CopyFromRemotePassThru(ctx context.Context, w io.Writer, remotePath string, passThru PassThru) error { + wg := sync.WaitGroup{} + errCh := make(chan error, 1) + + wg.Add(1) + go func() { + var err error + + defer func() { + if err != nil { + errCh <- err + } + errCh <- err + wg.Done() + }() + + r, err := a.Session.StdoutPipe() + if err != nil { + errCh <- err + return + } + + in, err := a.Session.StdinPipe() + if err != nil { + errCh <- err + return + } + defer in.Close() + + err = a.Session.Start(fmt.Sprintf("%s -f %q", a.RemoteBinary, remotePath)) + if err != nil { + errCh <- err + return + } + + err = Ack(in) + if err != nil { + errCh <- err + return + } + + res, err := ParseResponse(r) + if err != nil { + errCh <- err + return + } + + infos, err := res.ParseFileInfos() + if err != nil { + errCh <- err + return + } + + err = Ack(in) + if err != nil { + errCh <- err + return + } + + if passThru != nil { + r = passThru(r, infos.Size) + } + + _, err = CopyN(w, r, infos.Size) + if err != nil { + errCh <- err + return + } + + err = Ack(in) + if err != nil { + errCh <- err + return + } + + err = a.Session.Wait() + if err != nil { + errCh <- err + return + } + }() + + if a.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, a.Timeout) + defer cancel() + } + + if err := wait(&wg, ctx); err != nil { + return err + } + + close(errCh) + return <-errCh +} + +func (a *Client) Close() { + if a.Session != nil { + a.Session.Close() + } + if a.Conn != nil { + a.Conn.Close() + } +} diff --git a/vendor/github.com/bramvdbogaerde/go-scp/configurer.go b/vendor/github.com/bramvdbogaerde/go-scp/configurer.go new file mode 100644 index 00000000000..31f4e7e91e7 --- /dev/null +++ b/vendor/github.com/bramvdbogaerde/go-scp/configurer.go @@ -0,0 +1,81 @@ +/* Copyright (c) 2020 Bram Vandenbogaerde + * You may use, distribute or modify this code under the + * terms of the Mozilla Public License 2.0, which is distributed + * along with the source code. + */ + +package scp + +import ( + "golang.org/x/crypto/ssh" + "time" +) + +// A struct containing all the configuration options +// used by an scp client. +type ClientConfigurer struct { + host string + clientConfig *ssh.ClientConfig + session *ssh.Session + timeout time.Duration + remoteBinary string +} + +// Creates a new client configurer. +// It takes the required parameters: the host and the ssh.ClientConfig and +// returns a configurer populated with the default values for the optional +// parameters. +// +// These optional parameters can be set by using the methods provided on the +// ClientConfigurer struct. +func NewConfigurer(host string, config *ssh.ClientConfig) *ClientConfigurer { + return &ClientConfigurer{ + host: host, + clientConfig: config, + timeout: 0, // no timeout by default + remoteBinary: "scp", + } +} + +// Sets the path of the location of the remote scp binary +// Defaults to: /usr/bin/scp +func (c *ClientConfigurer) RemoteBinary(path string) *ClientConfigurer { + c.remoteBinary = path + return c +} + +// Alters the host of the client connects to +func (c *ClientConfigurer) Host(host string) *ClientConfigurer { + c.host = host + return c +} + +// Changes the connection timeout. +// Defaults to one minute +func (c *ClientConfigurer) Timeout(timeout time.Duration) *ClientConfigurer { + c.timeout = timeout + return c +} + +// Alters the ssh.ClientConfig +func (c *ClientConfigurer) ClientConfig(config *ssh.ClientConfig) *ClientConfigurer { + c.clientConfig = config + return c +} + +// Alters the ssh.Session +func (c *ClientConfigurer) Session(session *ssh.Session) *ClientConfigurer { + c.session = session + return c +} + +// Builds a client with the configuration stored within the ClientConfigurer +func (c *ClientConfigurer) Create() Client { + return Client{ + Host: c.host, + ClientConfig: c.clientConfig, + Timeout: c.timeout, + RemoteBinary: c.remoteBinary, + Session: c.session, + } +} diff --git a/vendor/github.com/bramvdbogaerde/go-scp/protocol.go b/vendor/github.com/bramvdbogaerde/go-scp/protocol.go new file mode 100644 index 00000000000..d825e3803a6 --- /dev/null +++ b/vendor/github.com/bramvdbogaerde/go-scp/protocol.go @@ -0,0 +1,125 @@ +/* Copyright (c) 2021 Bram Vandenbogaerde And Contributors + * You may use, distribute or modify this code under the + * terms of the Mozilla Public License 2.0, which is distributed + * along with the source code. + */ +package scp + +import ( + "bufio" + "errors" + "io" + "strconv" + "strings" +) + +type ResponseType = uint8 + +const ( + Ok ResponseType = 0 + Warning ResponseType = 1 + Error ResponseType = 2 +) + +const buffSize = 1024 * 256 + +// There are tree types of responses that the remote can send back: +// ok, warning and error +// +// The difference between warning and error is that the connection is not closed by the remote, +// however, a warning can indicate a file transfer failure (such as invalid destination directory) +// and such be handled as such. +// +// All responses except for the `Ok` type always have a message (although these can be empty) +// +// The remote sends a confirmation after every SCP command, because a failure can occur after every +// command, the response should be read and checked after sending them. +type Response struct { + Type ResponseType + Message string +} + +// Reads from the given reader (assuming it is the output of the remote) and parses it into a Response structure +func ParseResponse(reader io.Reader) (Response, error) { + buffer := make([]uint8, 1) + _, err := reader.Read(buffer) + if err != nil { + return Response{}, err + } + + response_type := buffer[0] + message := "" + if response_type > 0 { + buffered_reader := bufio.NewReader(reader) + message, err = buffered_reader.ReadString('\n') + if err != nil { + return Response{}, err + } + } + + return Response{response_type, message}, nil +} + +func (r *Response) IsOk() bool { + return r.Type == Ok +} + +func (r *Response) IsWarning() bool { + return r.Type == Warning +} + +// Returns true when the remote responded with an error +func (r *Response) IsError() bool { + return r.Type == Error +} + +// Returns true when the remote answered with a warning or an error +func (r *Response) IsFailure() bool { + return r.Type > 0 +} + +// Returns the message the remote sent back +func (r *Response) GetMessage() string { + return r.Message +} + +type FileInfos struct { + Message string + Filename string + Permissions string + Size int64 +} + +func (r *Response) ParseFileInfos() (*FileInfos, error) { + message := strings.ReplaceAll(r.Message, "\n", "") + parts := strings.Split(message, " ") + if len(parts) < 3 { + return nil, errors.New("Unable to parse message as file infos") + } + + size, err := strconv.Atoi(parts[1]) + if err != nil { + return nil, err + } + + return &FileInfos{ + Message: r.Message, + Permissions: parts[0], + Size: int64(size), + Filename: parts[2], + }, nil +} + +// Writes an `Ack` message to the remote, does not await its response, a seperate call to ParseResponse is +// therefore required to check if the acknowledgement succeeded +func Ack(writer io.Writer) error { + var msg = []byte{0} + n, err := writer.Write(msg) + if err != nil { + return err + } + if n < len(msg) { + return errors.New("Failed to write ack buffer") + } + return nil +} diff --git a/vendor/github.com/bramvdbogaerde/go-scp/scp.go b/vendor/github.com/bramvdbogaerde/go-scp/scp.go new file mode 100644 index 00000000000..14d907b188f --- /dev/null +++ b/vendor/github.com/bramvdbogaerde/go-scp/scp.go @@ -0,0 +1,45 @@ +/* Copyright (c) 2021 Bram Vandenbogaerde + * You may use, distribute or modify this code under the + * terms of the Mozilla Public License 2.0, which is distributed + * along with the source code. + */ + +// Simple scp package to copy files over SSH +package scp + +import ( + "time" + + "golang.org/x/crypto/ssh" +) + +// Returns a new scp.Client with provided host and ssh.clientConfig +// It has a default timeout of one minute. +func NewClient(host string, config *ssh.ClientConfig) Client { + return NewConfigurer(host, config).Create() +} + +// Returns a new scp.Client with provides host, ssh.ClientConfig and timeout +// Deprecated: provide meaningful context to each "Copy*" function instead. +func NewClientWithTimeout(host string, config *ssh.ClientConfig, timeout time.Duration) Client { + return NewConfigurer(host, config).Timeout(timeout).Create() +} + +// Returns a new scp.Client using an already existing established SSH connection +func NewClientBySSH(ssh *ssh.Client) (Client, error) { + session, err := ssh.NewSession() + if err != nil { + return Client{}, err + } + return NewConfigurer("", nil).Session(session).Create(), nil +} + +// Same as NewClientWithTimeout but uses an existing SSH client +// Deprecated: provide meaningful context to each "Copy*" function instead. +func NewClientBySSHWithTimeout(ssh *ssh.Client, timeout time.Duration) (Client, error) { + session, err := ssh.NewSession() + if err != nil { + return Client{}, err + } + return NewConfigurer("", nil).Session(session).Timeout(timeout).Create(), nil +} diff --git a/vendor/github.com/bramvdbogaerde/go-scp/utils.go b/vendor/github.com/bramvdbogaerde/go-scp/utils.go new file mode 100644 index 00000000000..35213f06e46 --- /dev/null +++ b/vendor/github.com/bramvdbogaerde/go-scp/utils.go @@ -0,0 +1,25 @@ +/* Copyright (c) 2021 Bram Vandenbogaerde And Contributors + * You may use, distribute or modify this code under the + * terms of the Mozilla Public License 2.0, which is distributed + * along with the source code. + */ + +package scp + +import "io" + +// An adaptation of io.CopyN that keeps reading if it did not return +// a sufficient amount of bytes. +func CopyN(writer io.Writer, src io.Reader, size int64) (int64, error) { + var total int64 + total = 0 + for total < size { + n, err := io.CopyN(writer, src, size) + if err != nil { + return 0, err + } + total += n + } + + return total, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 0f4a5b04341..34215075ffe 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -99,6 +99,9 @@ github.com/beorn7/perks/quantile # github.com/blang/semver v3.5.1+incompatible ## explicit github.com/blang/semver +# github.com/bramvdbogaerde/go-scp v1.2.0 +## explicit; go 1.13 +github.com/bramvdbogaerde/go-scp # github.com/cavaliergopher/grab/v3 v3.0.1 ## explicit; go 1.14 github.com/cavaliergopher/grab/v3