Skip to content

Commit

Permalink
Merge pull request pentaho#9023 from Mandakini111/BACKLOG-38619
Browse files Browse the repository at this point in the history
[BACKLOG-38619] Put File SFTP Core To Plugin Movement
  • Loading branch information
peterrinehart authored Sep 21, 2023
2 parents adf1d35 + 14eceeb commit aec8d4f
Show file tree
Hide file tree
Showing 27 changed files with 463 additions and 105 deletions.
13 changes: 13 additions & 0 deletions assemblies/plugins/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
<ldap-plugin.version>${project.version}</ldap-plugin.version>
<ivw-bulk-loader-plugin.version>${project.version}</ivw-bulk-loader-plugin.version>
<sftpput-plugin.version>${project.version}</sftpput-plugin.version>
<put-file-sftp-plugin.version>${project.version}</put-file-sftp-plugin.version>
<edi2xml-plugin.version>${project.version}</edi2xml-plugin.version>
<random-cc-number-generator-plugin.version>${project.version}</random-cc-number-generator-plugin.version>
<credit-card-validator-plugin.version>${project.version}</credit-card-validator-plugin.version>
Expand Down Expand Up @@ -354,6 +355,18 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.pentaho.di.plugins</groupId>
<artifactId>put-file-sftp-plugins</artifactId>
<version>${put-file-sftp-plugin.version}</version>
<type>zip</type>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.pentaho.di.plugins</groupId>
<artifactId>mondrianinput-plugin</artifactId>
Expand Down
1 change: 0 additions & 1 deletion engine/src/main/resources/kettle-job-entries.xml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
<job-entry id="FTPS_GET"> <description>i18n:org.pentaho.di.job.entry:JobEntry.FTPSGet.TypeDesc</description> <classname>org.pentaho.di.job.entries.ftpsget.JobEntryFTPSGet</classname> <category>i18n:org.pentaho.di.job:JobCategory.Category.FileTransfer</category> <tooltip>i18n:org.pentaho.di.job.entry:JobEntry.FTPSGet.Tooltip</tooltip> <iconfile>ui/images/GFTPS.svg</iconfile> <documentation_url>http://wiki.pentaho.com/display/EAI/Get+a+file+with+FTPS</documentation_url> <cases_url/> <forum_url/> </job-entry>
<job-entry id="FTPS_PUT"> <description>i18n:org.pentaho.di.job.entry:JobEntry.FTPSPut.TypeDesc</description> <classname>org.pentaho.di.job.entries.ftpsput.JobEntryFTPSPUT</classname> <category>i18n:org.pentaho.di.job:JobCategory.Category.FileTransfer</category> <tooltip>i18n:org.pentaho.di.job.entry:JobEntry.FTPSPut.Tooltip</tooltip> <iconfile>ui/images/PFTPS.svg</iconfile> <documentation_url>http://wiki.pentaho.com/display/EAI/Upload+files+to+FTPS</documentation_url> <cases_url/> <forum_url/> </job-entry>
<job-entry id="SFTP"> <description>i18n:org.pentaho.di.job.entry:JobEntry.SFTP.TypeDesc</description> <classname>org.pentaho.di.job.entries.sftp.JobEntrySFTP</classname> <category>i18n:org.pentaho.di.job:JobCategory.Category.FileTransfer</category> <tooltip>i18n:org.pentaho.di.job.entry:JobEntry.SFTP.Tooltip</tooltip> <iconfile>ui/images/GSFTP.svg</iconfile> <documentation_url>http://wiki.pentaho.com/display/EAI/Get+a+file+with+SFTP</documentation_url> <cases_url/> <forum_url/> </job-entry>
<job-entry id="SFTPPUT"> <description>i18n:org.pentaho.di.job.entry:JobEntry.SFTPPut.TypeDesc</description> <classname>org.pentaho.di.job.entries.sftpput.JobEntrySFTPPUT</classname> <category>i18n:org.pentaho.di.job:JobCategory.Category.FileTransfer</category> <tooltip>i18n:org.pentaho.di.job.entry:JobEntry.SFTPPut.Tooltip</tooltip> <iconfile>ui/images/PSFTP.svg</iconfile> <documentation_url>http://wiki.pentaho.com/display/EAI/Put+a+file+with+SFTP</documentation_url> <cases_url/> <forum_url/> </job-entry>
<job-entry id="TABLE_EXISTS"> <description>i18n:org.pentaho.di.job.entry:JobEntry.TableExists.TypeDesc</description> <classname>org.pentaho.di.job.entries.tableexists.JobEntryTableExists</classname> <category>i18n:org.pentaho.di.job:JobCategory.Category.Conditions</category> <tooltip>i18n:org.pentaho.di.job.entry:JobEntry.TableExists.Tooltip</tooltip> <iconfile>ui/images/TEX.svg</iconfile> <documentation_url>http://wiki.pentaho.com/display/EAI/Table+Exists+%28Job+Entry%29</documentation_url> <cases_url/> <forum_url/> </job-entry>
<job-entry id="FILE_EXISTS"> <description>i18n:org.pentaho.di.job.entry:JobEntry.FileExists.TypeDesc</description> <classname>org.pentaho.di.job.entries.fileexists.JobEntryFileExists</classname> <category>i18n:org.pentaho.di.job:JobCategory.Category.Conditions</category> <tooltip>i18n:org.pentaho.di.job.entry:JobEntry.FileExists.Tooltip</tooltip> <iconfile>ui/images/FEX.svg</iconfile> <documentation_url>Products/File_Exists_(Job_Entry)</documentation_url> <cases_url/> <forum_url/> </job-entry>
<job-entry id="EVAL"> <description>i18n:org.pentaho.di.job.entry:JobEntry.Evaluation.TypeDesc</description> <classname>org.pentaho.di.job.entries.eval.JobEntryEval</classname> <category>i18n:org.pentaho.di.job:JobCategory.Category.Scripting</category> <tooltip>i18n:org.pentaho.di.job.entry:JobEntry.Evaluation.Tooltip</tooltip> <iconfile>ui/images/SCR_mod.svg</iconfile> <documentation_url>http://wiki.pentaho.com/display/EAI/Evaluating+conditions+in+The+JavaScript+job+entry</documentation_url> <cases_url/> <forum_url/> </job-entry>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ JobEntry.GetPOP.Tooltip=Get mails (POP3/IMAP) server and save into a local folde
JobEntry.WriteToLog.TypeDesc=Write to log
JobEntry.ZipFile.TypeDesc=Zip file
JobEntry.DeleteFile.TypeDesc=Delete file
JobEntry.SFTPPut.TypeDesc=Put a file with SFTP
JobEntry.CopyFiles.Tooltip=Copy files
JobEntry.XAction.Tooltip=Runs an XAction Script
JobEntry.MysqlBulkLoad.Tooltip=Load data from a file into a Mysql table
Expand Down Expand Up @@ -62,7 +61,6 @@ JobEntry.HTTP.TypeDesc=HTTP
JobEntry.XSLT.TypeDesc=XSL transformation
JobEntry.FTP.TypeDesc=Get a file with FTP
JobEntry.Ping.TypeDesc=Ping a host
JobEntry.SFTPPut.Tooltip=Put files using SFTP (Secure File Transfer Protocol)
JobEntry.TableExists.TypeDesc=Table exists
JobEntry.Delay.TypeDesc=Wait for
JobEntry.XSLT.Tooltip=Make an XSL transformation
Expand Down
1 change: 1 addition & 0 deletions plugins/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ __List of the current core plugins:__
* monet-db-bulk-loader
* ms-access-bulk-loader
* postgresql-db-bulk-loader
* put-file-sftp
* infobright-bulk-loader
* excel
* ivw-bulk-loader
Expand Down
1 change: 1 addition & 0 deletions plugins/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
<module>avro-format</module>
<module>get-repository-names</module>
<module>sftpput</module>
<module>put-file-sftp</module>
<module>connected-to-repository</module>
<module>mail</module>
<module>mail-validator-job</module>
Expand Down
55 changes: 55 additions & 0 deletions plugins/put-file-sftp/assemblies/plugin/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.pentaho.di.plugins</groupId>
<artifactId>put-file-sftp-assemblies</artifactId>
<version>10.1.0.0-SNAPSHOT</version>
</parent>

<artifactId>put-file-sftp-plugins</artifactId>
<version>10.1.0.0-SNAPSHOT</version>
<packaging>pom</packaging>

<name>PDI Put File SFTP Plugins Distribution</name>
<description>Put File SFTP</description>

<dependencies>
<dependency>
<groupId>org.pentaho.di.plugins</groupId>
<artifactId>put-file-sftp-plugins-impl</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.pentaho.di.plugins</groupId>
<artifactId>put-file-sftp-plugins-ui</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>add-libs</id>
<phase>generate-resources</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/assembly/lib</outputDirectory>
<includeScope>compile</includeScope>
<excludeArtifactIds>put-file-sftp-plugins-impl,put-file-sftp-plugins-ui</excludeArtifactIds>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>
37 changes: 37 additions & 0 deletions plugins/put-file-sftp/assemblies/plugin/src/assembly/assembly.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
<id>put-file-sftp-plugin</id>
<formats>
<format>zip</format>
</formats>
<baseDirectory>put-file-sftp-plugins</baseDirectory>

<files>
<file>
<source>${project.basedir}/src/main/resources/version.xml</source>
<outputDirectory>.</outputDirectory>
<filtered>true</filtered>
</file>
</files>

<fileSets>
<!-- the staging dir -->
<fileSet>
<directory>${project.build.directory}/assembly</directory>
<outputDirectory>.</outputDirectory>
</fileSet>
</fileSets>

<dependencySets>
<dependencySet>
<outputDirectory>.</outputDirectory>
<includes>
<include>org.pentaho.di.plugins:put-file-sftp-plugins-impl:jar</include>
<include>org.pentaho.di.plugins:put-file-sftp-plugins-ui:jar</include>
</includes>
<useProjectArtifact>false</useProjectArtifact>
</dependencySet>
</dependencySets>
</assembly>


Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
<?xml version="1.0" encoding="UTF-8"?>
<version>${project.version}</version>
22 changes: 22 additions & 0 deletions plugins/put-file-sftp/assemblies/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.pentaho.di.plugins</groupId>
<artifactId>put-file-sftp</artifactId>
<version>10.1.0.0-SNAPSHOT</version>
</parent>

<artifactId>put-file-sftp-assemblies</artifactId>
<version>10.1.0.0-SNAPSHOT</version>
<packaging>pom</packaging>

<name>PDI Put File SFTP Plugins Assemblies</name>

<modules>
<module>plugin</module>
</modules>
</project>
52 changes: 52 additions & 0 deletions plugins/put-file-sftp/impl/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.pentaho.di.plugins</groupId>
<artifactId>put-file-sftp</artifactId>
<version>10.1.0.0-SNAPSHOT</version>
</parent>

<artifactId>put-file-sftp-plugins-impl</artifactId>
<name>PDI Put File SFTP Plugins Impl</name>

<dependencies>
<dependency>
<groupId>pentaho-kettle</groupId>
<artifactId>kettle-core</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>pentaho-kettle</groupId>
<artifactId>kettle-engine</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.healthmarketscience.jackcess</groupId>
<artifactId>jackcess</artifactId>
<scope>compile</scope>
</dependency>

<!-- Test dependencies -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>pentaho-kettle</groupId>
<artifactId>kettle-core</artifactId>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
<dependency>
<groupId>pentaho-kettle</groupId>
<artifactId>kettle-engine</artifactId>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com
* Copyright (C) 2002-2023 by Hitachi Vantara : http://www.pentaho.com
*
*******************************************************************************
*
Expand All @@ -22,30 +22,21 @@

package org.pentaho.di.job.entries.sftpput;

import org.pentaho.di.job.entry.validator.AndValidator;
import org.pentaho.di.job.entry.validator.JobEntryValidatorUtils;

import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileType;
import org.pentaho.di.cluster.SlaveServer;
import org.pentaho.di.core.CheckResultInterface;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.Result;
import org.pentaho.di.core.ResultFile;
import org.pentaho.di.core.RowMetaAndData;
import org.pentaho.di.core.annotations.JobEntry;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.encryption.Encr;
import org.pentaho.di.core.exception.KettleDatabaseException;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleXMLException;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.variables.VariableSpace;
import org.pentaho.di.core.vfs.KettleVFS;
import org.pentaho.di.core.xml.XMLHandler;
Expand All @@ -54,6 +45,8 @@
import org.pentaho.di.job.entries.sftp.SFTPClient;
import org.pentaho.di.job.entry.JobEntryBase;
import org.pentaho.di.job.entry.JobEntryInterface;
import org.pentaho.di.job.entry.validator.AndValidator;
import org.pentaho.di.job.entry.validator.JobEntryValidatorUtils;
import org.pentaho.di.repository.ObjectId;
import org.pentaho.di.repository.Repository;
import org.pentaho.di.resource.ResourceEntry;
Expand All @@ -62,13 +55,27 @@
import org.pentaho.metastore.api.IMetaStore;
import org.w3c.dom.Node;

import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
* This defines an SFTP put job entry.
*
* @author Matt
* @since 05-11-2003
*
*/

@JobEntry( id = "SFTPPUT", name = "JobEntry.SFTPPut.TypeDesc",
i18nPackageName = "org.pentaho.di.job.entries.sftpput",
description = "JobEntry.SFTPPut.Tooltip",
categoryDescription = "i18n:org.pentaho.di.job:JobCategory.Category.FileTransfer",
image = "PSFTP.svg",
documentationUrl = "http://wiki.pentaho.com/display/EAI/Put+a+file+with+SFTP" )
public class JobEntrySFTPPUT extends JobEntryBase implements Cloneable, JobEntryInterface {
private static Class<?> PKG = JobEntrySFTPPUT.class; // for i18n purposes, needed by Translator2!!

Expand Down Expand Up @@ -111,7 +118,7 @@ public class JobEntrySFTPPUT extends JobEntryBase implements Cloneable, JobEntry

private boolean successWhenNoFile;

public JobEntrySFTPPUT( String n ) {
public JobEntrySFTPPUT(String n ) {
super( n, "" );
serverName = null;
serverPort = "22";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ JobSFTPPUT.SuccessWhenNoFile.Tooltip=Check this option if the job entry has to s
JobSFTPPUT.Port.Label=Port
JobSFTPPUT.Filetype.All=All files
JobSFTPPUT.FolderExists.NOK=We can not find remote folder {0}
JobEntry.SFTPPut.Tooltip=Put files using SFTP (Secure File Transfer Protocol)
JobEntry.SFTPPut.TypeDesc=Put a file with SFTP
JobSFTPPUT.DestinationFolder.Label=Destination folder
JobSFTPPUT.CreateDestinationFolder.Label=Create destination folder
JobSFTPPUT.FolderExists.Title.Ok=Folder exists
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com
* Copyright (C) 2002-2023 by Hitachi Vantara : http://www.pentaho.com
*
*******************************************************************************
*
Expand All @@ -22,17 +22,17 @@

package org.pentaho.di.job.entries.sftpput;

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.junit.ClassRule;
import org.pentaho.di.job.entry.loadSave.JobEntryLoadSaveTestSupport;
import org.pentaho.di.junit.rules.RestorePDIEngineEnvironment;
import org.pentaho.di.trans.steps.loadsave.validator.FieldLoadSaveValidator;
import org.pentaho.di.trans.steps.loadsave.validator.IntLoadSaveValidator;

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class JobEntrySFTPPUTLoadSaveTest extends JobEntryLoadSaveTestSupport<JobEntrySFTPPUT> {
@ClassRule public static RestorePDIEngineEnvironment env = new RestorePDIEngineEnvironment();

Expand Down
Loading

0 comments on commit aec8d4f

Please sign in to comment.