From 88aaadc1677a7aa87e51a259d4b16f68395b13a9 Mon Sep 17 00:00:00 2001 From: Alan Bryant Date: Mon, 15 Apr 2024 15:17:13 -0400 Subject: [PATCH] [BACKLOG-40099] - VFS support for Bowls Major changes: - KettleVFS and ConnectionManager needed to be refactored to support having a Bowl context for most operations - KettleVFS has been refactored a bit, adding IKettleVFS and KettleVFSImpl as per-Bowl implementations. KettleVFS itself has been refactored to use an instance of an IKettleVFS for backwards compatibility. - Most KettleVFS static methods have been deprecated. Callers should now use KettleVFS.getInstance(Bowl) instead. - Some (sadly public) parts of ConnectionManager and KettleVFS need to be single-instance/global, such as the VFS FileManager or the ConnectionManager providers that get registered by plugins. These remain shared. - Because ConnectionManager instances do not share state, even with the same metastore, all ConnectionManagers should now be retrieved from a Bowl instance. - Backwards-compatible methods and default stub methods have been left in place to allow existing code, including possible customer-written code to continue working. Some code will not work with Bowls until they update, however. Step Changes: - Two Steps, and two Job Entries have been updated to work as examples. NO UI work is ready for these steps in this PR, users need to hand-edit the step config to refer to a project connection by name (e.g. in a pvfs path). - Text File Input and Text File Output steps have been updated - Create File and Delete File job entries have been updated. Knock-on effects: - ConnectionProvider API has new methods. All VFS providers that need to support Bowls will need to be updated. - Because VFS Filesystems need to be unique and to provide it to downstream usage, like by the Providers, the Bowl is now included in FileSystemOptions. Because of this, all Bowl implementations will need to implement equals() and hashcode() - There was a bug in Effective Java's double-checked locking idiom which blocked testing these changes. (MetaStoreConst, etc) - ResolvableResource seemed unused (in this or other repos), and was difficult to change, so I removed it from a couple places. --- .../di/connections/ConnectionManager.java | 32 +- .../di/connections/ConnectionProvider.java | 28 +- .../vfs/BaseVFSConnectionProvider.java | 20 +- .../pentaho/di/connections/vfs/VFSHelper.java | 33 +- .../vfs/provider/ConnectionFileSystem.java | 9 +- .../org/pentaho/di/core/bowl/BaseBowl.java | 42 ++ .../java/org/pentaho/di/core/bowl/Bowl.java | 18 +- .../org/pentaho/di/core/bowl/DefaultBowl.java | 33 +- .../di/core/fileinput/FileInputList.java | 116 ++++- .../org/pentaho/di/core/vfs/IKettleVFS.java | 148 ++++++ .../org/pentaho/di/core/vfs/KettleVFS.java | 403 ++++++---------- .../pentaho/di/core/vfs/KettleVFSImpl.java | 431 ++++++++++++++++++ .../IKettleFileSystemConfigBuilder.java | 16 +- .../KettleGenericFileSystemConfigBuilder.java | 22 +- .../pentaho/di/metastore/MetaStoreConst.java | 81 ++-- .../di/connections/ConnectionManagerTest.java | 7 +- .../common/bucket/TestConnectionProvider.java | 11 +- .../TestConnectionWithDomainProvider.java | 11 +- .../org/pentaho/di/base/AbstractMeta.java | 3 +- .../createfile/JobEntryCreateFile.java | 6 +- .../deletefile/JobEntryDeleteFile.java | 4 +- .../trans/steps/file/BaseFileInputFiles.java | 6 +- .../trans/steps/file/BaseFileInputMeta.java | 12 +- .../trans/steps/file/BaseFileInputStep.java | 7 +- .../fileinput/text/TextFileInputMeta.java | 34 +- .../textfileinput/InputFileMetaInterface.java | 16 +- .../steps/textfileinput/TextFileInput.java | 4 +- .../textfileinput/TextFileInputMeta.java | 13 +- .../steps/textfileoutput/TextFileOutput.java | 22 +- .../textfileoutput/TextFileOutputMeta.java | 19 +- .../fileinput/text/TextFileInputTest.java | 5 +- .../textfileoutput/TextFileOutputTest.java | 4 +- .../ui/TestConnectionProvider.java | 12 +- .../providers/vfs/VFSFileProvider.java | 167 ++++--- .../trans/steps/jsoninput/JsonInputTest.java | 5 +- .../fileinput/text/TextFileInputDialog.java | 11 +- .../textfileinput/TextFileInputDialog.java | 6 +- 37 files changed, 1318 insertions(+), 499 deletions(-) create mode 100644 core/src/main/java/org/pentaho/di/core/bowl/BaseBowl.java create mode 100644 core/src/main/java/org/pentaho/di/core/vfs/IKettleVFS.java create mode 100644 core/src/main/java/org/pentaho/di/core/vfs/KettleVFSImpl.java diff --git a/core/src/main/java/org/pentaho/di/connections/ConnectionManager.java b/core/src/main/java/org/pentaho/di/connections/ConnectionManager.java index c7e5d68b914a..0c8bf1e2d0c3 100644 --- a/core/src/main/java/org/pentaho/di/connections/ConnectionManager.java +++ b/core/src/main/java/org/pentaho/di/connections/ConnectionManager.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2023 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -23,6 +23,7 @@ package org.pentaho.di.connections; import org.pentaho.di.connections.utils.EncryptUtils; +import org.pentaho.di.core.bowl.Bowl; import org.pentaho.di.core.exception.KettleException; import org.pentaho.di.core.variables.VariableSpace; import org.pentaho.metastore.api.IMetaStore; @@ -94,10 +95,39 @@ public synchronized void reset() { initialized = false; } + /** + * This getter should not generally be used because it is limited to the global scope. It would be better to have + * almost all callers use Bowl.getConnectionManager(). + * + * This instance may still be used to register ConnectionProviders and Lookup Filters. + * + * @return ConnectionManager + */ public static ConnectionManager getInstance() { return instance; } + /** + * Construct a new instance of a ConnectionManager using the metastore from the supplier. + * + * Instances returned by this will not share in-memory state with any other instannces. If you need the + * ConnectionManager for a Bowl, use Bowl.getConnectionManager() instead. + * + * + * @param bowl + * + * @return ConnectionManager + */ + public static ConnectionManager getInstance( Supplier metastoreSupplier ) { + ConnectionManager newManager = new ConnectionManager(); + newManager.setMetastoreSupplier( metastoreSupplier ); + // share the same set of connection providers and lookup filters. Everyone already registers with the one + // from getInstance() + newManager.connectionProviders = instance.connectionProviders; + newManager.lookupFilters = instance.lookupFilters; + return newManager; + } + /** * Construct a meta store factory for a specific class using the default meta store supplier * diff --git a/core/src/main/java/org/pentaho/di/connections/ConnectionProvider.java b/core/src/main/java/org/pentaho/di/connections/ConnectionProvider.java index 1bac9f9bbaa4..13692776b16b 100644 --- a/core/src/main/java/org/pentaho/di/connections/ConnectionProvider.java +++ b/core/src/main/java/org/pentaho/di/connections/ConnectionProvider.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2019-2022 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2019-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -37,9 +37,31 @@ public interface ConnectionProvider { Class getClassType(); - List getNames(); + /** + * @deprecated use getNames( ConnectionManager ) + */ + @Deprecated + default List getNames() { + throw new UnsupportedOperationException( "Deprecated method" ); + } - List getConnectionDetails(); + /** + * @deprecated use getNames( ConnectionManager ) + */ + @Deprecated + default List getConnectionDetails() { + throw new UnsupportedOperationException( "Deprecated method" ); + } + + // Subclasses should implement this to work with Bowls. + default List getNames( ConnectionManager connectionManager ) { + return getNames(); + } + + // Subclasses should implement this to work with Bowls. + default List getConnectionDetails( ConnectionManager connectionManager ) { + return getConnectionDetails(); + } boolean test( T connectionDetails ) throws KettleException; diff --git a/core/src/main/java/org/pentaho/di/connections/vfs/BaseVFSConnectionProvider.java b/core/src/main/java/org/pentaho/di/connections/vfs/BaseVFSConnectionProvider.java index b5b6864791df..8ddfb2bedcb7 100644 --- a/core/src/main/java/org/pentaho/di/connections/vfs/BaseVFSConnectionProvider.java +++ b/core/src/main/java/org/pentaho/di/connections/vfs/BaseVFSConnectionProvider.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2019-2022 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2019-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -38,13 +38,25 @@ public abstract class BaseVFSConnectionProvider private Supplier connectionManagerSupplier = ConnectionManager::getInstance; - @Override public List getNames() { + @Override + public List getNames() { return connectionManagerSupplier.get().getNamesByType( getClass() ); } + @Override + public List getConnectionDetails() { + return getConnectionDetails( connectionManagerSupplier.get() ); + } + + @Override + public List getNames( ConnectionManager connectionManager ) { + return connectionManager.getNamesByType( getClass() ); + } + @SuppressWarnings( "unchecked" ) - @Override public List getConnectionDetails() { - return (List) connectionManagerSupplier.get().getConnectionDetailsByScheme( getKey() ); + @Override + public List getConnectionDetails( ConnectionManager connectionManager ) { + return (List) connectionManager.getConnectionDetailsByScheme( getKey() ); } @Override public T prepare( T connectionDetails ) throws KettleException { diff --git a/core/src/main/java/org/pentaho/di/connections/vfs/VFSHelper.java b/core/src/main/java/org/pentaho/di/connections/vfs/VFSHelper.java index 65b9d133f0a8..1fa49f67e231 100644 --- a/core/src/main/java/org/pentaho/di/connections/vfs/VFSHelper.java +++ b/core/src/main/java/org/pentaho/di/connections/vfs/VFSHelper.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2019-2022 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2019-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -24,8 +24,12 @@ import org.apache.commons.vfs2.FileSystemOptions; import org.pentaho.di.connections.ConnectionManager; +import org.pentaho.di.core.bowl.Bowl; +import org.pentaho.di.core.bowl.DefaultBowl; import org.pentaho.di.core.variables.VariableSpace; import org.pentaho.di.core.variables.Variables; +import org.pentaho.di.core.vfs.configuration.KettleGenericFileSystemConfigBuilder; +import org.pentaho.metastore.api.exceptions.MetaStoreException; import java.util.function.Supplier; @@ -33,15 +37,34 @@ * Created by bmorrise on 2/13/19. */ public class VFSHelper { - public static FileSystemOptions getOpts( String file, String connection, VariableSpace space ){ + + /** + * @deprecated, use the version with the Bowl + */ + @Deprecated + public static FileSystemOptions getOpts( String file, String connection, VariableSpace space ) { + try { + return getOpts( DefaultBowl.getInstance(), file, connection, space ); + } catch ( MetaStoreException ex ) { + // deprecated behavior was to ignore failures and return nulls. + return null; + } + } + + public static FileSystemOptions getOpts( Bowl bowl, String file, String connection, VariableSpace space ) + throws MetaStoreException { if ( connection != null ) { VFSConnectionDetails vfsConnectionDetails = - (VFSConnectionDetails) ConnectionManager.getInstance().getConnectionDetails( file, connection ); + (VFSConnectionDetails) bowl.getConnectionManager().getConnectionDetails( file, connection ); VFSConnectionProvider vfsConnectionProvider = - (VFSConnectionProvider) ConnectionManager.getInstance().getConnectionProvider( file ); + (VFSConnectionProvider) bowl.getConnectionManager().getConnectionProvider( file ); if ( vfsConnectionDetails != null && vfsConnectionProvider != null ) { vfsConnectionDetails.setSpace( space ); - return vfsConnectionProvider.getOpts( vfsConnectionDetails ); + FileSystemOptions opts = vfsConnectionProvider.getOpts( vfsConnectionDetails ); + + KettleGenericFileSystemConfigBuilder.getInstance().setBowl( opts, bowl ); + + return opts; } } return null; diff --git a/core/src/main/java/org/pentaho/di/connections/vfs/provider/ConnectionFileSystem.java b/core/src/main/java/org/pentaho/di/connections/vfs/provider/ConnectionFileSystem.java index 2bb0cefff92b..22c46b45e8f1 100644 --- a/core/src/main/java/org/pentaho/di/connections/vfs/provider/ConnectionFileSystem.java +++ b/core/src/main/java/org/pentaho/di/connections/vfs/provider/ConnectionFileSystem.java @@ -34,20 +34,20 @@ import org.pentaho.di.connections.ConnectionDetails; import org.pentaho.di.connections.ConnectionManager; import org.pentaho.di.connections.vfs.VFSConnectionDetails; +import org.pentaho.di.core.bowl.Bowl; import org.pentaho.di.core.variables.VariableSpace; import org.pentaho.di.core.variables.Variables; import org.pentaho.di.core.vfs.KettleVFS; import org.pentaho.di.core.vfs.configuration.IKettleFileSystemConfigBuilder; import org.pentaho.di.core.vfs.configuration.KettleFileSystemConfigBuilderFactory; +import org.pentaho.di.core.vfs.configuration.KettleGenericFileSystemConfigBuilder; import java.util.Collection; -import java.util.function.Supplier; public class ConnectionFileSystem extends AbstractFileSystem implements FileSystem { public static final String CONNECTION = "connection"; public static final String DOMAIN_ROOT = "[\\w]+://"; - private Supplier connectionManager = ConnectionManager::getInstance; public ConnectionFileSystem( FileName rootName, FileSystemOptions fileSystemOptions ) { super( rootName, null, fileSystemOptions ); @@ -86,8 +86,9 @@ public static String getUrl( AbstractFileName abstractFileName, ConnectionDetail protected FileObject createFile( AbstractFileName abstractFileName ) throws Exception { String connectionName = ( (ConnectionFileName) abstractFileName ).getConnection(); + Bowl bowl = KettleGenericFileSystemConfigBuilder.getInstance().getBowl( getFileSystemOptions() ); VFSConnectionDetails connectionDetails = - (VFSConnectionDetails) connectionManager.get().getConnectionDetails( connectionName ); + (VFSConnectionDetails) bowl.getConnectionManager().getConnectionDetails( connectionName ); FileSystemOptions opts = super.getFileSystemOptions(); IKettleFileSystemConfigBuilder configBuilder = KettleFileSystemConfigBuilderFactory.getConfigBuilder ( new Variables(), ConnectionFileProvider.SCHEME ); @@ -103,7 +104,7 @@ protected FileObject createFile( AbstractFileName abstractFileName ) throws Exce if ( url != null ) { domain = connectionDetails.getDomain(); varSpace.setVariable( CONNECTION, connectionName ); - fileObject = (AbstractFileObject) KettleVFS.getFileObject( url, varSpace ); + fileObject = (AbstractFileObject) KettleVFS.getInstance( bowl ).getFileObject( url, varSpace ); } return new ConnectionFileObject( abstractFileName, this, fileObject, domain ); diff --git a/core/src/main/java/org/pentaho/di/core/bowl/BaseBowl.java b/core/src/main/java/org/pentaho/di/core/bowl/BaseBowl.java new file mode 100644 index 000000000000..50a511e56cf8 --- /dev/null +++ b/core/src/main/java/org/pentaho/di/core/bowl/BaseBowl.java @@ -0,0 +1,42 @@ +/*! + * Copyright 2024 Hitachi Vantara. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.pentaho.di.core.bowl; + +import org.pentaho.di.connections.ConnectionManager; +import org.pentaho.metastore.api.exceptions.MetaStoreException; +import org.pentaho.metastore.api.IMetaStore; + +public abstract class BaseBowl implements Bowl { + + private volatile ConnectionManager connectionManager; + + @Override + public ConnectionManager getConnectionManager() throws MetaStoreException { + ConnectionManager result = connectionManager; + if ( result != null ) { + return result; + } + synchronized( this ) { + if ( connectionManager == null ) { + IMetaStore metastore = getMetastore(); + connectionManager = ConnectionManager.getInstance( () -> metastore ); + } + return connectionManager; + } + } + +} diff --git a/core/src/main/java/org/pentaho/di/core/bowl/Bowl.java b/core/src/main/java/org/pentaho/di/core/bowl/Bowl.java index 39a48a88b103..f654c7531abd 100644 --- a/core/src/main/java/org/pentaho/di/core/bowl/Bowl.java +++ b/core/src/main/java/org/pentaho/di/core/bowl/Bowl.java @@ -16,6 +16,7 @@ */ package org.pentaho.di.core.bowl; +import org.pentaho.di.connections.ConnectionManager; import org.pentaho.metastore.api.exceptions.MetaStoreException; import org.pentaho.metastore.api.IMetaStore; @@ -24,11 +25,14 @@ * A Bowl is a generic container/context/workspace concept. Different plugin implementations may implement this for * additional features. * + * All implementations of Bowl should implement equals() and hashcode() + * */ public interface Bowl { /** - * Gets a Metastore that handles any defaulting required for execution-time handling of metastores, for the Bowl. + * Gets a Read-Only Metastore that handles any defaulting required for execution-time handling of metastores, for the + * Bowl. * * @return IMetaStore A metastore for execution with the Bowl. Never null. */ @@ -42,4 +46,16 @@ public interface Bowl { */ IMetaStore getExplicitMetastore() throws MetaStoreException; + /** + * Gets a ConnectionManager for this Bowl. Uses a metastore from getMetastore(), so global connections will be + * returned as well. This ConnectionManager is effectively read-only. + * + * Since constructing and initializing ConnectionManagers can be expensive, and ConnectionManager instances don't + * share state, consumers should always use this method instead of ConnectionManager.getInstance() + * + * @return ConnectionManager, never null. + */ + ConnectionManager getConnectionManager() throws MetaStoreException; + + } diff --git a/core/src/main/java/org/pentaho/di/core/bowl/DefaultBowl.java b/core/src/main/java/org/pentaho/di/core/bowl/DefaultBowl.java index 4cf1103c7f16..57c5ced6dd11 100644 --- a/core/src/main/java/org/pentaho/di/core/bowl/DefaultBowl.java +++ b/core/src/main/java/org/pentaho/di/core/bowl/DefaultBowl.java @@ -16,17 +16,25 @@ */ package org.pentaho.di.core.bowl; +import org.pentaho.di.connections.ConnectionManager; import org.pentaho.di.metastore.MetaStoreConst; import org.pentaho.metastore.api.exceptions.MetaStoreException; import org.pentaho.metastore.api.IMetaStore; +import com.google.common.annotations.VisibleForTesting; +import java.util.function.Supplier; + /** * The default/global Bowl. A singleton for standard behavior when there is no custom Bowl. * */ -public class DefaultBowl implements Bowl { +public class DefaultBowl extends BaseBowl { private static final DefaultBowl INSTANCE = new DefaultBowl(); + // for testing + private Supplier metastoreSupplier = MetaStoreConst.getDefaultMetastoreSupplier(); + private boolean customSupplier = false; + private DefaultBowl() { } @@ -37,12 +45,31 @@ public static DefaultBowl getInstance() { @Override public IMetaStore getExplicitMetastore() throws MetaStoreException { - return MetaStoreConst.getDefaultMetastore(); + return metastoreSupplier.get(); } @Override public IMetaStore getMetastore() throws MetaStoreException { - return MetaStoreConst.getDefaultMetastore(); + return metastoreSupplier.get(); } + + @Override + public ConnectionManager getConnectionManager() throws MetaStoreException { + // need to override getConnectionManager so this instance of DefaultBowl shares the same ConnectionManager + // instance with ConnectionManager.getInstance() + if ( customSupplier ) { + return super.getConnectionManager(); + } else { + return ConnectionManager.getInstance(); + } + } + + @VisibleForTesting + public void setMetastoreSupplier( Supplier metastoreSupplier ) { + this.metastoreSupplier = metastoreSupplier; + this.customSupplier = true; + } + + } diff --git a/core/src/main/java/org/pentaho/di/core/fileinput/FileInputList.java b/core/src/main/java/org/pentaho/di/core/fileinput/FileInputList.java index fd5826ec0812..1095c14aafd7 100644 --- a/core/src/main/java/org/pentaho/di/core/fileinput/FileInputList.java +++ b/core/src/main/java/org/pentaho/di/core/fileinput/FileInputList.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2021 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -29,6 +29,8 @@ import org.apache.commons.vfs2.FileType; import org.apache.commons.vfs2.provider.compressed.CompressedFileFileObject; import org.apache.commons.vfs2.provider.local.LocalFile; +import org.pentaho.di.core.bowl.Bowl; +import org.pentaho.di.core.bowl.DefaultBowl; import org.pentaho.di.core.Const; import org.pentaho.di.core.logging.LogChannel; import org.pentaho.di.core.logging.LogChannelInterface; @@ -116,23 +118,54 @@ private static boolean[] includeSubdirsFalse( int iLength ) { return includeSubdirs; } + /** + * @deprecated, use the version with the Bowl + */ + @Deprecated public static String[] createFilePathList( VariableSpace space, String[] fileName, String[] fileMask, String[] excludeFileMask, String[] fileRequired ) { + return createFilePathList( DefaultBowl.getInstance(), space, fileName, fileMask, excludeFileMask, fileRequired ); + } + + public static String[] createFilePathList( Bowl bowl, VariableSpace space, String[] fileName, String[] fileMask, + String[] excludeFileMask, String[] fileRequired ) { boolean[] includeSubdirs = includeSubdirsFalse( fileName.length ); - return createFilePathList( space, fileName, fileMask, excludeFileMask, fileRequired, includeSubdirs, null ); + return createFilePathList( bowl, space, fileName, fileMask, excludeFileMask, fileRequired, null ); } + /** + * @deprecated, use the version with the Bowl + */ + @Deprecated public static String[] createFilePathList( VariableSpace space, String[] fileName, String[] fileMask, String[] excludeFileMask, String[] fileRequired, boolean[] includeSubdirs ) { - return createFilePathList( space, fileName, fileMask, excludeFileMask, fileRequired, includeSubdirs, null ); + return createFilePathList( DefaultBowl.getInstance(), space, fileName, fileMask, excludeFileMask, fileRequired, + includeSubdirs, null ); + } + + public static String[] createFilePathList( Bowl bowl, VariableSpace space, String[] fileName, String[] fileMask, + String[] excludeFileMask, String[] fileRequired, + boolean[] includeSubdirs ) { + return createFilePathList( bowl, space, fileName, fileMask, excludeFileMask, fileRequired, includeSubdirs, null ); } + /** + * @deprecated, use the version with the Bowl + */ + @Deprecated public static String[] createFilePathList( VariableSpace space, String[] fileName, String[] fileMask, String[] excludeFileMask, String[] fileRequired, boolean[] includeSubdirs, FileTypeFilter[] filters ) { + return createFilePathList( DefaultBowl.getInstance(), space, fileName, fileMask, excludeFileMask, fileRequired, + includeSubdirs, filters ); + } + + public static String[] createFilePathList( Bowl bowl, VariableSpace space, String[] fileName, String[] fileMask, + String[] excludeFileMask, String[] fileRequired, boolean[] includeSubdirs, + FileTypeFilter[] filters ) { List fileList = - createFileList( space, fileName, fileMask, excludeFileMask, fileRequired, includeSubdirs, filters ) + createFileList( bowl, space, fileName, fileMask, excludeFileMask, fileRequired, includeSubdirs, filters ) .getFiles(); String[] filePaths = new String[ fileList.size() ]; for ( int i = 0; i < filePaths.length; i++ ) { @@ -141,21 +174,52 @@ public static String[] createFilePathList( VariableSpace space, String[] fileNam return filePaths; } + /** + * @deprecated, use the version with the Bowl + */ + @Deprecated public static FileInputList createFileList( VariableSpace space, String[] fileName, String[] fileMask, String[] excludeFileMask, String[] fileRequired ) { + return createFileList( DefaultBowl.getInstance(), space, fileName, fileMask, excludeFileMask, fileRequired ); + } + + public static FileInputList createFileList( Bowl bowl, VariableSpace space, String[] fileName, String[] fileMask, + String[] excludeFileMask, String[] fileRequired ) { boolean[] includeSubdirs = includeSubdirsFalse( fileName.length ); - return createFileList( space, fileName, fileMask, excludeFileMask, fileRequired, includeSubdirs, null ); + return createFileList( bowl, space, fileName, fileMask, excludeFileMask, fileRequired, includeSubdirs, null ); } + /** + * @deprecated, use the version with the Bowl + */ + @Deprecated public static FileInputList createFileList( VariableSpace space, String[] fileName, String[] fileMask, String[] excludeFileMask, String[] fileRequired, boolean[] includeSubdirs ) { - return createFileList( space, fileName, fileMask, excludeFileMask, fileRequired, includeSubdirs, null ); + return createFileList( DefaultBowl.getInstance(), space, fileName, fileMask, excludeFileMask, fileRequired, + includeSubdirs ); + } + + public static FileInputList createFileList( Bowl bowl, VariableSpace space, String[] fileName, String[] fileMask, + String[] excludeFileMask, String[] fileRequired, + boolean[] includeSubdirs ) { + return createFileList( bowl, space, fileName, fileMask, excludeFileMask, fileRequired, includeSubdirs, null ); } + /** + * @deprecated, use the version with the Bowl + */ + @Deprecated public static FileInputList createFileList( VariableSpace space, String[] fileName, String[] fileMask, String[] excludeFileMask, String[] fileRequired, boolean[] includeSubdirs, FileTypeFilter[] fileTypeFilters ) { + return createFileList( DefaultBowl.getInstance(), space, fileName, fileMask, excludeFileMask, fileRequired, + includeSubdirs, fileTypeFilters ); + } + + public static FileInputList createFileList( Bowl bowl, VariableSpace space, String[] fileName, String[] fileMask, + String[] excludeFileMask, String[] fileRequired, boolean[] includeSubdirs, + FileTypeFilter[] fileTypeFilters ) { FileInputList fileInputList = new FileInputList(); // Replace possible environment variables... @@ -178,7 +242,7 @@ public static FileInputList createFileList( VariableSpace space, String[] fileNa } try { - FileObject directoryFileObject = KettleVFS.getFileObject( onefile, space ); + FileObject directoryFileObject = KettleVFS.getInstance( bowl ).getFileObject( onefile, space ); boolean processFolder = true; if ( onerequired ) { if ( !directoryFileObject.exists() ) { @@ -267,7 +331,7 @@ public boolean includeFile( FileSelectInfo info ) { } // We don't sort here, keep the order of the files in the archive. } else { - FileObject fileObject = KettleVFS.getFileObject( onefile, space ); + FileObject fileObject = KettleVFS.getInstance( bowl ).getFileObject( onefile, space ); if ( fileObject.exists() ) { if ( fileObject.isReadable() ) { fileInputList.addFile( fileObject ); @@ -294,7 +358,17 @@ public boolean includeFile( FileSelectInfo info ) { return fileInputList; } - public static FileInputList createFolderList( VariableSpace space, String[] folderName, String[] folderRequired ) { + /** + * @deprecated, use the version with the Bowl + */ + @Deprecated + public static FileInputList createFolderList( VariableSpace space, String[] folderName, + String[] folderRequired ) { + return createFolderList( DefaultBowl.getInstance(), space, folderName, folderRequired ); + } + + public static FileInputList createFolderList( Bowl bowl, VariableSpace space, String[] folderName, + String[] folderRequired ) { FileInputList fileInputList = new FileInputList(); // Replace possible environment variables... @@ -314,7 +388,7 @@ public static FileInputList createFolderList( VariableSpace space, String[] fold try { // Find all folder names in this directory // - directoryFileObject = KettleVFS.getFileObject( onefile, space ); + directoryFileObject = KettleVFS.getInstance( bowl ).getFileObject( onefile, space ); if ( directoryFileObject != null && directoryFileObject.getType() == FileType.FOLDER ) { // it's a directory FileObject[] fileObjects = directoryFileObject.findFiles( new AllFileSelector() { @Override @@ -452,16 +526,34 @@ public int nrOfMissingFiles() { return nonAccessibleFiles.size() + nonExistantFiles.size(); } + /** + * @deprecated, use the version with the Bowl + */ + @Deprecated public static FileInputList createFileList( VariableSpace space, String[] fileName, String[] fileMask, String[] fileRequired, boolean[] includeSubdirs ) { + return createFileList( DefaultBowl.getInstance(), space, fileName, fileMask, fileRequired, includeSubdirs ); + } + + public static FileInputList createFileList( Bowl bowl, VariableSpace space, String[] fileName, String[] fileMask, + String[] fileRequired, boolean[] includeSubdirs ) { return createFileList( - space, fileName, fileMask, new String[ fileName.length ], fileRequired, includeSubdirs, null ); + bowl, space, fileName, fileMask, new String[ fileName.length ], fileRequired, includeSubdirs, null ); } + /** + * @deprecated, use the version with the Bowl + */ + @Deprecated public static String[] createFilePathList( VariableSpace space, String[] fileName, String[] fileMask, String[] fileRequired ) { + return createFilePathList( DefaultBowl.getInstance(), space, fileName, fileMask, fileRequired ); + } + + public static String[] createFilePathList( Bowl bowl, VariableSpace space, String[] fileName, String[] fileMask, + String[] fileRequired ) { boolean[] includeSubdirs = includeSubdirsFalse( fileName.length ); return createFilePathList( - space, fileName, fileMask, new String[ fileName.length ], fileRequired, includeSubdirs, null ); + bowl, space, fileName, fileMask, new String[ fileName.length ], fileRequired, includeSubdirs, null ); } } diff --git a/core/src/main/java/org/pentaho/di/core/vfs/IKettleVFS.java b/core/src/main/java/org/pentaho/di/core/vfs/IKettleVFS.java new file mode 100644 index 000000000000..c757de6fe53a --- /dev/null +++ b/core/src/main/java/org/pentaho/di/core/vfs/IKettleVFS.java @@ -0,0 +1,148 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2024 by Hitachi Vantara : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.core.vfs; + +import org.pentaho.di.core.exception.KettleFileException; +import org.pentaho.di.core.variables.VariableSpace; +import org.pentaho.metastore.api.exceptions.MetaStoreException; + +import java.io.InputStream; +import java.io.IOException; +import java.io.OutputStream; + +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileSystemOptions; + +/** + * An interface to a bowl-specific KettleVFS interface. Most, but not all methods from KettleVFS have been moved to this + * interface. Methods that remain are truly static or use state that must remain global. Use + * KettleVFS.getInstance( Bowl ) to get an instance. + * + */ +public interface IKettleVFS { + + FileObject getFileObject( String vfsFilename ) throws KettleFileException; + + FileObject getFileObject( String vfsFilename, VariableSpace space ) throws KettleFileException; + + FileObject getFileObject( String vfsFilename, FileSystemOptions fsOptions ) throws KettleFileException; + + FileObject getFileObject( String vfsFilename, VariableSpace space, FileSystemOptions fsOptions ) + throws KettleFileException; + + // warning, was not public + FileSystemOptions getFileSystemOptions( String scheme, String vfsFilename, VariableSpace space, + FileSystemOptions fileSystemOptions ) + throws IOException, MetaStoreException; + + String getFriendlyURI( String filename ); + + String getFriendlyURI( String filename, VariableSpace space ); + + /** + * Read a text file (like an XML document). WARNING DO NOT USE FOR DATA FILES. + * + * @param vfsFilename the filename or URL to read from + * @param charSetName the character set of the string (UTF-8, ISO8859-1, etc) + * @return The content of the file as a String + * @deprecated use getInstance( Bowl ) + * @throws IOException + */ + String getTextFileContent( String vfsFilename, String charSetName ) throws KettleFileException; + + String getTextFileContent( String vfsFilename, VariableSpace space, String charSetName ) + throws KettleFileException; + + boolean fileExists( String vfsFilename ) throws KettleFileException; + + boolean fileExists( String vfsFilename, VariableSpace space ) throws KettleFileException; + + InputStream getInputStream( String vfsFilename ) throws KettleFileException; + + InputStream getInputStream( String vfsFilename, VariableSpace space ) throws KettleFileException; + + OutputStream getOutputStream( FileObject fileObject, boolean append ) throws IOException; + + OutputStream getOutputStream( String vfsFilename, boolean append ) throws KettleFileException; + + OutputStream getOutputStream( String vfsFilename, VariableSpace space, boolean append ) + throws KettleFileException; + + OutputStream getOutputStream( String vfsFilename, VariableSpace space, + FileSystemOptions fsOptions, boolean append ) throws KettleFileException; + + + /** + * Creates a file using "java.io.tmpdir" directory + * + * @param prefix - file name + * @param prefix - file extension + * @return FileObject + * @throws KettleFileException + */ + FileObject createTempFile( String prefix, KettleVFS.Suffix suffix ) throws KettleFileException; + + /** + * Creates a file using "java.io.tmpdir" directory + * + * @param prefix - file name + * @param suffix - file extension + * @param variableSpace is used to get system variables + * @return FileObject + * @throws KettleFileException + */ + FileObject createTempFile( String prefix, KettleVFS.Suffix suffix, VariableSpace variableSpace ) + throws KettleFileException; + + /** + * @param prefix - file name + * @param suffix - file extension + * @param directory - directory where file will be created + * @return FileObject + * @throws KettleFileException + */ + FileObject createTempFile( String prefix, KettleVFS.Suffix suffix, String directory ) throws KettleFileException; + + FileObject createTempFile( String prefix, String suffix, String directory ) throws KettleFileException; + + /** + * @param prefix - file name + * @param directory path to directory where file will be created + * @param space is used to get system variables + * @return FileObject + * @throws KettleFileException + */ + FileObject createTempFile( String prefix, KettleVFS.Suffix suffix, String directory, VariableSpace space ) + throws KettleFileException; + + FileObject createTempFile( String prefix, String suffix, String directory, VariableSpace space ) + throws KettleFileException; + + /** + * resets the VariableSpace + * + */ + void reset(); + +} diff --git a/core/src/main/java/org/pentaho/di/core/vfs/KettleVFS.java b/core/src/main/java/org/pentaho/di/core/vfs/KettleVFS.java index 479fe2121221..cb7352b2f369 100644 --- a/core/src/main/java/org/pentaho/di/core/vfs/KettleVFS.java +++ b/core/src/main/java/org/pentaho/di/core/vfs/KettleVFS.java @@ -34,6 +34,8 @@ import org.apache.commons.vfs2.impl.StandardFileSystemManager; import org.apache.commons.vfs2.provider.local.LocalFile; import org.pentaho.di.connections.vfs.VFSHelper; +import org.pentaho.di.core.bowl.Bowl; +import org.pentaho.di.core.bowl.DefaultBowl; import org.pentaho.di.core.Const; import org.pentaho.di.core.exception.KettleFileException; import org.pentaho.di.core.util.UUIDUtil; @@ -43,6 +45,7 @@ import org.pentaho.di.core.vfs.configuration.KettleFileSystemConfigBuilderFactory; import org.pentaho.di.core.vfs.configuration.KettleGenericFileSystemConfigBuilder; import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.metastore.api.exceptions.MetaStoreException; import java.io.File; import java.io.FileInputStream; @@ -55,29 +58,26 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +/** + * This class now serves two purposes: legacy (deprecated) backwards-compatible access to VFS methods for components + * that don't yet support Bowls, and singleton/static data and methods that need to still be shared. Most code trying to + * read and write files over VFS should use getInstance( Bowl ) and IKettleVFS. + * + */ public class KettleVFS { public static final String TEMP_DIR = System.getProperty( "java.io.tmpdir" ); - public static final String CONNECTION = "connection"; + public static final String SMB_SCHEME = "smb"; + public static final String SMB_SCHEME_COLON = SMB_SCHEME + ":"; private static Class PKG = KettleVFS.class; // for i18n purposes, needed by Translator2!! + // for global state private static final KettleVFS kettleVFS = new KettleVFS(); - private final DefaultFileSystemManager fsm; - private static final int TIMEOUT_LIMIT = 9000; - private static final int TIME_TO_SLEEP_STEP = 50; - private static final String PROVIDER_PATTERN_SCHEME = "^[\\w\\d]+://(.*)"; - private static final Pattern SMB_PATTERN = Pattern.compile( "^[Ss][Mm][Bb]://[/]?([^:/]+)(?::([^/]+))?.*$" ); - public static final String SMB_SCHEME = "smb"; - public static final String SMB_SCHEME_COLON = SMB_SCHEME + ":"; - - private static VariableSpace defaultVariableSpace; + // for passing along previously-static methods to the new implementation + private static final IKettleVFS ikettleVFS = new KettleVFSImpl( DefaultBowl.getInstance() ); - static { - // Create a new empty variable space... - // - defaultVariableSpace = new Variables(); - defaultVariableSpace.initializeVariablesFrom( null ); - } + private final DefaultFileSystemManager fsm; + static final String PROVIDER_PATTERN_SCHEME = "^[\\w\\d]+://(.*)"; private KettleVFS() { fsm = new ConcurrentFileSystemManager(); @@ -112,10 +112,30 @@ public FileSystemManager getFileSystemManager() { return fsm; } + /** + * Use only when the caller is positive that a Bowl is not in use. + */ public static KettleVFS getInstance() { return kettleVFS; } + /** + * Gets a handle on an IKettleVFS for a particular Bowl. It is important that all VFS Filesystems for a given Bowl are + * shared correctly between instances. VFS Filesystem instances are compared by the FileSystemOptions, and the Bowl is + * set as a parameter in those options. It is important that all Bowl implementations correctly implement equals() and + * hashcode() to make this work. + * + * It is also critical that there is only one ConnectionManager for a given Bowl. Anything that needs a + * ConnectionManager (especially including VFS code) should only use Bowl.getConnectionManager() to ensure there is a + * single instance per bowl. + * + * @param bowl the bowl for the current context + * @return IKettleVFS The API for file operations. + */ + public static IKettleVFS getInstance( Bowl bowl ) { + return new KettleVFSImpl( bowl ); + } + /** only use this method as a last resort if you don't yet have a variables object. Since VFS connections can be used * with variables for the dialog fields, it is important to pass the Variables to getFileObject. Failure to do this * means any settings for the job/ktr will not be available for variables substitution. @@ -123,75 +143,36 @@ public static KettleVFS getInstance() { * @param vfsFilename * @return * @throws KettleFileException + * @deprecated use getInstance( Bowl ) */ @Deprecated public static FileObject getFileObject( String vfsFilename ) throws KettleFileException { - return getFileObject( vfsFilename, defaultVariableSpace ); + return ikettleVFS.getFileObject( vfsFilename ); } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static FileObject getFileObject( String vfsFilename, VariableSpace space ) throws KettleFileException { - return getFileObject( vfsFilename, space, null ); + return ikettleVFS.getFileObject( vfsFilename, space ); } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static FileObject getFileObject( String vfsFilename, FileSystemOptions fsOptions ) throws KettleFileException { - return getFileObject( vfsFilename, defaultVariableSpace, fsOptions ); - } - - // IMPORTANT: - // We have one problem with VFS: if the file is in a subdirectory of the current one: somedir/somefile - // In that case, VFS doesn't parse the file correctly. - // We need to put file: in front of it to make it work. - // However, how are we going to verify this? - // - // We are going to see if the filename starts with one of the known protocols like file: zip: ram: smb: jar: etc. - // If not, we are going to assume it's a file, when no scheme found ( flag as null ), and it only changes if - // a scheme is provided. - // + return ikettleVFS.getFileObject( vfsFilename, fsOptions ); + } + + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static FileObject getFileObject( String vfsFilename, VariableSpace space, FileSystemOptions fsOptions ) throws KettleFileException { - - // Protect the code below from invalid input. - if ( vfsFilename == null ) { - throw new IllegalArgumentException( "Unexpected null VFS filename." ); - } - - try { - FileSystemManager fsManager = getInstance().getFileSystemManager(); - String[] schemes = fsManager.getSchemes(); - - String scheme = getScheme( schemes, vfsFilename ); - - //Waiting condition - PPP-4374: - //We have to check for hasScheme even if scheme is null because that scheme could not - //be available by getScheme at the time we validate our scheme flag ( Kitchen loading problem ) - //So we check if - even it has not a scheme - our vfsFilename has a possible scheme format (PROVIDER_PATTERN_SCHEME) - //If it does, then give it some time and tries to load. It stops when timeout is up or a scheme is found. - int timeOut = TIMEOUT_LIMIT; - if ( hasSchemePattern( vfsFilename, PROVIDER_PATTERN_SCHEME ) ) { - while (scheme == null && timeOut > 0) { - // ask again to refresh schemes list - schemes = fsManager.getSchemes(); - try { - Thread.sleep( TIME_TO_SLEEP_STEP ); - timeOut -= TIME_TO_SLEEP_STEP; - scheme = getScheme( schemes, vfsFilename ); - } catch ( InterruptedException e ) { - Thread.currentThread().interrupt(); - break; - } - } - } - - fsOptions = getFileSystemOptions( scheme, vfsFilename, space, fsOptions ); - - String filename = normalizePath( vfsFilename, scheme ); - - return fsOptions != null ? fsManager.resolveFile( filename, fsOptions ) : fsManager.resolveFile( filename ); - - } catch ( IOException e ) { - throw new KettleFileException( "Unable to get VFS File object for filename '" - + cleanseFilename( vfsFilename ) + "' : " + e.getMessage(), e ); - } + return ikettleVFS.getFileObject( vfsFilename, space, fsOptions ); } protected static String normalizePath( String path, String scheme ) { @@ -233,13 +214,14 @@ static String getScheme( String[] schemes, String fileName ) { return null; } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated static FileSystemOptions getFileSystemOptions( String scheme, String vfsFilename, VariableSpace space, - FileSystemOptions fileSystemOptions ) throws IOException { - if ( scheme == null ) { - return fileSystemOptions; - } - - return buildFsOptions( space, fileSystemOptions, vfsFilename, scheme ); + FileSystemOptions fileSystemOptions ) + throws IOException, MetaStoreException { + return ikettleVFS.getFileSystemOptions( scheme, vfsFilename, space, fileSystemOptions ); } /** @@ -252,111 +234,43 @@ public static String cleanseFilename( String vfsFilename ) { return vfsFilename.replaceAll( ":[^:@/]+@", ":@" ); } - private static FileSystemOptions buildFsOptions( VariableSpace parentVariableSpace, FileSystemOptions sourceOptions, - String vfsFilename, String scheme ) throws IOException { - VariableSpace varSpace = parentVariableSpace; - if ( vfsFilename == null ) { - return null; - } - if ( varSpace == null ) { - varSpace = defaultVariableSpace; - } - - IKettleFileSystemConfigBuilder configBuilder = - KettleFileSystemConfigBuilderFactory.getConfigBuilder( varSpace, scheme ); - - FileSystemOptions fsOptions = ( sourceOptions == null ) ? new FileSystemOptions() : sourceOptions; - - if ( scheme.equals( SMB_SCHEME ) ) { - Matcher matcher = SMB_PATTERN.matcher( vfsFilename ); - if ( matcher.matches() ) { - return VFSHelper.getOpts( vfsFilename, matcher.group( 1 ), varSpace ); - } - } - - String[] varList = varSpace.listVariables(); - - for (String var : varList) { - if ( var.equalsIgnoreCase( CONNECTION ) && varSpace.getVariable( var ) != null ) { - FileSystemOptions fileSystemOptions = VFSHelper.getOpts( vfsFilename, varSpace.getVariable( var ), varSpace ); - if ( fileSystemOptions != null ) { - return fileSystemOptions; - } - } - if ( var.startsWith( "vfs." ) ) { - String param = configBuilder.parseParameterName( var, scheme ); - String varScheme = KettleGenericFileSystemConfigBuilder.extractScheme( var ); - if ( param != null ) { - if ( varScheme == null || varScheme.equals( "sftp" ) || varScheme.equals( scheme ) ) { - configBuilder.setParameter( fsOptions, param, varSpace.getVariable( var ), var, vfsFilename ); - } - } else { - throw new IOException( "FileSystemConfigBuilder could not parse parameter: " + var ); - } - } - } - if ( scheme.equals( "pvfs" ) ) { - configBuilder.setParameter( fsOptions, "VariableSpace", varSpace, vfsFilename ); - } - return fsOptions; - } - /** * Read a text file (like an XML document). WARNING DO NOT USE FOR DATA FILES. * * @param vfsFilename the filename or URL to read from * @param charSetName the character set of the string (UTF-8, ISO8859-1, etc) * @return The content of the file as a String + * @deprecated use getInstance( Bowl ) * @throws IOException */ + @Deprecated public static String getTextFileContent( String vfsFilename, String charSetName ) throws KettleFileException { - return getTextFileContent( vfsFilename, null, charSetName ); + return ikettleVFS.getTextFileContent( vfsFilename, charSetName ); } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static String getTextFileContent( String vfsFilename, VariableSpace space, String charSetName ) throws KettleFileException { - try { - InputStream inputStream = null; - - if ( space == null ) { - inputStream = getInputStream( vfsFilename ); - } else { - inputStream = getInputStream( vfsFilename, space ); - } - InputStreamReader reader = new InputStreamReader( inputStream, charSetName ); - int c; - StringBuilder aBuffer = new StringBuilder(); - while (( c = reader.read() ) != -1) { - aBuffer.append( (char) c ); - } - reader.close(); - inputStream.close(); - - return aBuffer.toString(); - } catch ( IOException e ) { - throw new KettleFileException( e ); - } + return ikettleVFS.getTextFileContent( vfsFilename, space, charSetName ); } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static boolean fileExists( String vfsFilename ) throws KettleFileException { - return fileExists( vfsFilename, null ); + return ikettleVFS.fileExists( vfsFilename ); } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static boolean fileExists( String vfsFilename, VariableSpace space ) throws KettleFileException { - FileObject fileObject = null; - try { - fileObject = getFileObject( vfsFilename, space ); - return fileObject.exists(); - } catch ( IOException e ) { - throw new KettleFileException( e ); - } finally { - if ( fileObject != null ) { - try { - fileObject.close(); - } catch ( Exception e ) { /* Ignore */ - } - } - } + return ikettleVFS.fileExists( vfsFilename, space ); } public static InputStream getInputStream( FileObject fileObject ) throws FileSystemException { @@ -364,71 +278,54 @@ public static InputStream getInputStream( FileObject fileObject ) throws FileSys return content.getInputStream(); } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static InputStream getInputStream( String vfsFilename ) throws KettleFileException { - return getInputStream( vfsFilename, defaultVariableSpace ); + return ikettleVFS.getInputStream( vfsFilename ); } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static InputStream getInputStream( String vfsFilename, VariableSpace space ) throws KettleFileException { - try { - FileObject fileObject = getFileObject( vfsFilename, space ); - - return getInputStream( fileObject ); - } catch ( IOException e ) { - throw new KettleFileException( e ); - } + return ikettleVFS.getInputStream( vfsFilename, space ); } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static OutputStream getOutputStream( FileObject fileObject, boolean append ) throws IOException { - FileObject parent = fileObject.getParent(); - if ( parent != null ) { - if ( !parent.exists() ) { - throw new IOException( BaseMessages.getString( - PKG, "KettleVFS.Exception.ParentDirectoryDoesNotExist", getFriendlyURI( parent ) ) ); - } - } - try { - fileObject.createFile(); - FileContent content = fileObject.getContent(); - return content.getOutputStream( append ); - } catch ( FileSystemException e ) { - // Perhaps if it's a local file, we can retry using the standard - // File object. This is because on Windows there is a bug in VFS. - // - if ( fileObject instanceof LocalFile ) { - try { - String filename = getFilename( fileObject ); - return new FileOutputStream( new File( filename ), append ); - } catch ( Exception e2 ) { - throw e; // throw the original exception: hide the retry. - } - } else { - throw e; - } - } + return ikettleVFS.getOutputStream( fileObject, append ); } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static OutputStream getOutputStream( String vfsFilename, boolean append ) throws KettleFileException { - return getOutputStream( vfsFilename, defaultVariableSpace, append ); + return ikettleVFS.getOutputStream( vfsFilename, append ); } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static OutputStream getOutputStream( String vfsFilename, VariableSpace space, boolean append ) throws KettleFileException { - try { - FileObject fileObject = getFileObject( vfsFilename, space ); - return getOutputStream( fileObject, append ); - } catch ( IOException e ) { - throw new KettleFileException( e ); - } + return ikettleVFS.getOutputStream( vfsFilename, space, append ); } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static OutputStream getOutputStream( String vfsFilename, VariableSpace space, FileSystemOptions fsOptions, boolean append ) throws KettleFileException { - try { - FileObject fileObject = getFileObject( vfsFilename, space, fsOptions ); - return getOutputStream( fileObject, append ); - } catch ( IOException e ) { - throw new KettleFileException( e ); - } + return ikettleVFS.getOutputStream( vfsFilename, space, fsOptions, append ); } public static String getFilename( FileObject fileObject ) { @@ -454,23 +351,20 @@ public static String getFilename( FileObject fileObject ) { return fileString; } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static String getFriendlyURI( String filename ) { - return getFriendlyURI( filename, defaultVariableSpace ); + return ikettleVFS.getFriendlyURI( filename ); } - public static String getFriendlyURI( String filename, VariableSpace space ) { - if ( filename == null ) { - return null; - } - String friendlyName; - try { - friendlyName = getFriendlyURI( KettleVFS.getFileObject( filename, space ) ); - } catch ( Exception e ) { - // unable to get a friendly name from VFS object. - // Cleanse name of pwd before returning - friendlyName = cleanseFilename( filename ); - } - return friendlyName; + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated + public static String getFriendlyURI( String filename, VariableSpace space ) { + return ikettleVFS.getFriendlyURI( filename, space ); } public static String getFriendlyURI( FileObject fileObject ) { @@ -483,10 +377,12 @@ public static String getFriendlyURI( FileObject fileObject ) { * @param prefix - file name * @param prefix - file extension * @return FileObject + * @deprecated use getInstance( Bowl ) * @throws KettleFileException */ + @Deprecated public static FileObject createTempFile( String prefix, Suffix suffix ) throws KettleFileException { - return createTempFile( prefix, suffix, TEMP_DIR ); + return ikettleVFS.createTempFile( prefix, suffix ); } /** @@ -495,60 +391,58 @@ public static FileObject createTempFile( String prefix, Suffix suffix ) throws K * @param prefix - file name * @param suffix - file extension * @param variableSpace is used to get system variables + * @deprecated use getInstance( Bowl ) * @return FileObject * @throws KettleFileException */ + @Deprecated public static FileObject createTempFile( String prefix, Suffix suffix, VariableSpace variableSpace ) throws KettleFileException { - return createTempFile( prefix, suffix, TEMP_DIR, variableSpace ); + return ikettleVFS.createTempFile( prefix, suffix, variableSpace ); } /** * @param prefix - file name * @param suffix - file extension * @param directory - directory where file will be created + * @deprecated use getInstance( Bowl ) * @return FileObject * @throws KettleFileException */ + @Deprecated public static FileObject createTempFile( String prefix, Suffix suffix, String directory ) throws KettleFileException { - return createTempFile( prefix, suffix, directory, null ); + return ikettleVFS.createTempFile( prefix, suffix, directory ); } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static FileObject createTempFile( String prefix, String suffix, String directory ) throws KettleFileException { - return createTempFile( prefix, suffix, directory, null ); + return ikettleVFS.createTempFile( prefix, suffix, directory ); } /** * @param prefix - file name * @param directory path to directory where file will be created * @param space is used to get system variables + * @deprecated use getInstance( Bowl ) * @return FileObject * @throws KettleFileException */ + @Deprecated public static FileObject createTempFile( String prefix, Suffix suffix, String directory, VariableSpace space ) throws KettleFileException { - return createTempFile( prefix, suffix.ext, directory, space ); + return ikettleVFS.createTempFile( prefix, suffix, directory, space ); } + /** + * @deprecated use getInstance( Bowl ) + */ + @Deprecated public static FileObject createTempFile( String prefix, String suffix, String directory, VariableSpace space ) throws KettleFileException { - try { - FileObject fileObject; - do { - // Build temporary file name using UUID to ensure uniqueness. Old mechanism would fail using Sort Rows (for - // example) - // when there multiple nodes with multiple JVMs on each node. In this case, the temp file names would end up - // being - // duplicated which would cause the sort to fail. - String filename = - new StringBuilder( 50 ).append( directory ).append( '/' ).append( prefix ).append( '_' ).append( - UUIDUtil.getUUIDAsString() ).append( suffix ).toString(); - fileObject = getFileObject( filename, space ); - } while (fileObject.exists()); - return fileObject; - } catch ( IOException e ) { - throw new KettleFileException( e ); - } + return ikettleVFS.createTempFile( prefix, suffix, directory, space ); } public static Comparator getComparator() { @@ -612,16 +506,18 @@ public static void closeEmbeddedFileSystem( String embeddedMetastoreKey ) { } } + /** + * resets the FileSystemManager + * + */ public void reset() { - defaultVariableSpace = new Variables(); - defaultVariableSpace.initializeVariablesFrom( null ); + ikettleVFS.reset(); fsm.close(); try { fsm.setFilesCache( new WeakRefFilesCache() ); fsm.init(); } catch ( FileSystemException ignored ) { } - } /** @@ -639,6 +535,9 @@ public enum Suffix { Suffix( String ext ) { this.ext = ext; } + public String getExt() { + return ext; + } } private void serializeVariableSpace( VariableSpace space ) { diff --git a/core/src/main/java/org/pentaho/di/core/vfs/KettleVFSImpl.java b/core/src/main/java/org/pentaho/di/core/vfs/KettleVFSImpl.java new file mode 100644 index 000000000000..a827d6672079 --- /dev/null +++ b/core/src/main/java/org/pentaho/di/core/vfs/KettleVFSImpl.java @@ -0,0 +1,431 @@ +/*! ****************************************************************************** + * + * Pentaho Data Integration + * + * Copyright (C) 2024 by Hitachi Vantara : http://www.pentaho.com + * + ******************************************************************************* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package org.pentaho.di.core.vfs; + +import org.pentaho.di.connections.vfs.VFSHelper; +import org.pentaho.di.core.bowl.Bowl; +import org.pentaho.di.core.exception.KettleFileException; +import org.pentaho.di.core.util.UUIDUtil; +import org.pentaho.di.core.variables.Variables; +import org.pentaho.di.core.variables.VariableSpace; +import org.pentaho.di.core.vfs.configuration.IKettleFileSystemConfigBuilder; +import org.pentaho.di.core.vfs.configuration.KettleFileSystemConfigBuilderFactory; +import org.pentaho.di.core.vfs.configuration.KettleGenericFileSystemConfigBuilder; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.metastore.api.exceptions.MetaStoreException; + +import com.google.common.base.Preconditions; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.IOException; +import java.io.OutputStream; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.vfs2.cache.WeakRefFilesCache; +import org.apache.commons.vfs2.FileContent; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileSystemManager; +import org.apache.commons.vfs2.FileSystemOptions; +import org.apache.commons.vfs2.impl.DefaultFileSystemManager; +import org.apache.commons.vfs2.impl.StandardFileSystemManager; +import org.apache.commons.vfs2.provider.local.LocalFile; + +/** + * Implementation of IKettleVFS. Implementation of VFS file access for kettle. Use KettleVFS.getInstance( Bowl ) to get + * an instance. + * + */ +public class KettleVFSImpl implements IKettleVFS { + public static final String CONNECTION = "connection"; + + private static Class PKG = KettleVFS.class; // for i18n purposes, needed by Translator2!! + private static final int TIMEOUT_LIMIT = 9000; + private static final int TIME_TO_SLEEP_STEP = 50; + private static final String PROVIDER_PATTERN_SCHEME = "^[\\w\\d]+://(.*)"; + private static final Pattern SMB_PATTERN = Pattern.compile( "^[Ss][Mm][Bb]://[/]?([^:/]+)(?::([^/]+))?.*$" ); + + private final Bowl bowl; + + private static VariableSpace defaultVariableSpace; + + static { + // Create a new empty variable space... + // + defaultVariableSpace = new Variables(); + defaultVariableSpace.initializeVariablesFrom( null ); + } + + // non-public. Should only be created by the factory in KettleVFS + KettleVFSImpl( Bowl bowl ) { + this.bowl = Preconditions.checkNotNull( bowl ); + } + + @Override + public FileObject getFileObject( String vfsFilename ) throws KettleFileException { + return getFileObject( vfsFilename, defaultVariableSpace ); + } + + @Override + public FileObject getFileObject( String vfsFilename, VariableSpace space ) throws KettleFileException { + return getFileObject( vfsFilename, space, null ); + } + + @Override + public FileObject getFileObject( String vfsFilename, FileSystemOptions fsOptions ) throws KettleFileException { + return getFileObject( vfsFilename, defaultVariableSpace, fsOptions ); + } + + // IMPORTANT: + // We have one problem with VFS: if the file is in a subdirectory of the current one: somedir/somefile + // In that case, VFS doesn't parse the file correctly. + // We need to put file: in front of it to make it work. + // However, how are we going to verify this? + // + // We are going to see if the filename starts with one of the known protocols like file: zip: ram: smb: jar: etc. + // If not, we are going to assume it's a file, when no scheme found ( flag as null ), and it only changes if + // a scheme is provided. + // + @Override + public FileObject getFileObject( String vfsFilename, VariableSpace space, FileSystemOptions fsOptions ) + throws KettleFileException { + + // Protect the code below from invalid input. + if ( vfsFilename == null ) { + throw new IllegalArgumentException( "Unexpected null VFS filename." ); + } + + try { + FileSystemManager fsManager = KettleVFS.getInstance().getFileSystemManager(); + String[] schemes = fsManager.getSchemes(); + + String scheme = KettleVFS.getScheme( schemes, vfsFilename ); + + //Waiting condition - PPP-4374: + //We have to check for hasScheme even if scheme is null because that scheme could not + //be available by getScheme at the time we validate our scheme flag ( Kitchen loading problem ) + //So we check if - even it has not a scheme - our vfsFilename has a possible scheme format (PROVIDER_PATTERN_SCHEME) + //If it does, then give it some time and tries to load. It stops when timeout is up or a scheme is found. + int timeOut = TIMEOUT_LIMIT; + if ( KettleVFS.hasSchemePattern( vfsFilename, KettleVFS.PROVIDER_PATTERN_SCHEME ) ) { + while (scheme == null && timeOut > 0) { + // ask again to refresh schemes list + schemes = fsManager.getSchemes(); + try { + Thread.sleep( TIME_TO_SLEEP_STEP ); + timeOut -= TIME_TO_SLEEP_STEP; + scheme = KettleVFS.getScheme( schemes, vfsFilename ); + } catch ( InterruptedException e ) { + Thread.currentThread().interrupt(); + break; + } + } + } + + fsOptions = getFileSystemOptions( scheme, vfsFilename, space, fsOptions ); + + String filename = KettleVFS.normalizePath( vfsFilename, scheme ); + + return fsOptions != null ? fsManager.resolveFile( filename, fsOptions ) : fsManager.resolveFile( filename ); + + } catch ( IOException | MetaStoreException e ) { + throw new KettleFileException( "Unable to get VFS File object for filename '" + + KettleVFS.cleanseFilename( vfsFilename ) + "' : " + e.getMessage(), e ); + } + } + + @Override + public FileSystemOptions getFileSystemOptions( String scheme, String vfsFilename, VariableSpace space, + FileSystemOptions fileSystemOptions ) + throws IOException, MetaStoreException { + if ( scheme == null ) { + return fileSystemOptions; + } + + return buildFsOptions( space, fileSystemOptions, vfsFilename, scheme ); + } + + private FileSystemOptions buildFsOptions( VariableSpace parentVariableSpace, FileSystemOptions sourceOptions, + String vfsFilename, String scheme ) throws IOException, MetaStoreException { + VariableSpace varSpace = parentVariableSpace; + if ( vfsFilename == null ) { + return null; + } + if ( varSpace == null ) { + varSpace = defaultVariableSpace; + } + + IKettleFileSystemConfigBuilder configBuilder = + KettleFileSystemConfigBuilderFactory.getConfigBuilder( varSpace, scheme ); + + FileSystemOptions fsOptions = ( sourceOptions == null ) ? new FileSystemOptions() : sourceOptions; + configBuilder.setBowl( fsOptions, bowl ); + + if ( scheme.equals( KettleVFS.SMB_SCHEME ) ) { + Matcher matcher = SMB_PATTERN.matcher( vfsFilename ); + if ( matcher.matches() ) { + return VFSHelper.getOpts( bowl, vfsFilename, matcher.group( 1 ), varSpace ); + } + } + + String[] varList = varSpace.listVariables(); + + for (String var : varList) { + if ( var.equalsIgnoreCase( CONNECTION ) && varSpace.getVariable( var ) != null ) { + FileSystemOptions fileSystemOptions = VFSHelper.getOpts( bowl, vfsFilename, varSpace.getVariable( var ), + varSpace ); + if ( fileSystemOptions != null ) { + return fileSystemOptions; + } + } + if ( var.startsWith( "vfs." ) ) { + String param = configBuilder.parseParameterName( var, scheme ); + String varScheme = KettleGenericFileSystemConfigBuilder.extractScheme( var ); + if ( param != null ) { + if ( varScheme == null || varScheme.equals( "sftp" ) || varScheme.equals( scheme ) ) { + configBuilder.setParameter( fsOptions, param, varSpace.getVariable( var ), var, vfsFilename ); + } + } else { + throw new IOException( "FileSystemConfigBuilder could not parse parameter: " + var ); + } + } + } + if ( scheme.equals( "pvfs" ) ) { + configBuilder.setParameter( fsOptions, "VariableSpace", varSpace, vfsFilename ); + } + return fsOptions; + } + + @Override + public String getFriendlyURI( String filename ) { + return getFriendlyURI( filename, defaultVariableSpace ); + } + + @Override + public String getFriendlyURI( String filename, VariableSpace space ) { + if ( filename == null ) { + return null; + } + + String friendlyName; + try { + friendlyName = KettleVFS.getFriendlyURI( getFileObject( filename, space ) ); + } catch ( Exception e ) { + // unable to get a friendly name from VFS object. + // Cleanse name of pwd before returning + friendlyName = KettleVFS.cleanseFilename( filename ); + } + return friendlyName; + } + + + @Override + public String getTextFileContent( String vfsFilename, String charSetName ) throws KettleFileException { + return getTextFileContent( vfsFilename, null, charSetName ); + } + + @Override + public String getTextFileContent( String vfsFilename, VariableSpace space, String charSetName ) + throws KettleFileException { + try { + InputStream inputStream = null; + + if ( space == null ) { + inputStream = getInputStream( vfsFilename ); + } else { + inputStream = getInputStream( vfsFilename, space ); + } + InputStreamReader reader = new InputStreamReader( inputStream, charSetName ); + int c; + StringBuilder aBuffer = new StringBuilder(); + while (( c = reader.read() ) != -1) { + aBuffer.append( (char) c ); + } + reader.close(); + inputStream.close(); + + return aBuffer.toString(); + } catch ( IOException e ) { + throw new KettleFileException( e ); + } + } + + @Override + public boolean fileExists( String vfsFilename ) throws KettleFileException { + return fileExists( vfsFilename, null ); + } + + @Override + public boolean fileExists( String vfsFilename, VariableSpace space ) throws KettleFileException { + FileObject fileObject = null; + try { + fileObject = getFileObject( vfsFilename, space ); + return fileObject.exists(); + } catch ( IOException e ) { + throw new KettleFileException( e ); + } finally { + if ( fileObject != null ) { + try { + fileObject.close(); + } catch ( Exception e ) { /* Ignore */ + } + } + } + } + + + @Override + public InputStream getInputStream( String vfsFilename ) throws KettleFileException { + return getInputStream( vfsFilename, defaultVariableSpace ); + } + + @Override + public InputStream getInputStream( String vfsFilename, VariableSpace space ) throws KettleFileException { + try { + FileObject fileObject = getFileObject( vfsFilename, space ); + + return KettleVFS.getInputStream( fileObject ); + } catch ( IOException e ) { + throw new KettleFileException( e ); + } + } + + @Override + public OutputStream getOutputStream( FileObject fileObject, boolean append ) throws IOException { + FileObject parent = fileObject.getParent(); + if ( parent != null ) { + if ( !parent.exists() ) { + throw new IOException( BaseMessages.getString( + PKG, "KettleVFS.Exception.ParentDirectoryDoesNotExist", KettleVFS.getFriendlyURI( parent ) ) ); + } + } + try { + fileObject.createFile(); + FileContent content = fileObject.getContent(); + return content.getOutputStream( append ); + } catch ( FileSystemException e ) { + // Perhaps if it's a local file, we can retry using the standard + // File object. This is because on Windows there is a bug in VFS. + // + if ( fileObject instanceof LocalFile ) { + try { + String filename = KettleVFS.getFilename( fileObject ); + return new FileOutputStream( new File( filename ), append ); + } catch ( Exception e2 ) { + throw e; // throw the original exception: hide the retry. + } + } else { + throw e; + } + } + } + + @Override + public OutputStream getOutputStream( String vfsFilename, boolean append ) throws KettleFileException { + return getOutputStream( vfsFilename, defaultVariableSpace, append ); + } + + @Override + public OutputStream getOutputStream( String vfsFilename, VariableSpace space, boolean append ) + throws KettleFileException { + try { + FileObject fileObject = getFileObject( vfsFilename, space ); + return getOutputStream( fileObject, append ); + } catch ( IOException e ) { + throw new KettleFileException( e ); + } + } + + @Override + public OutputStream getOutputStream( String vfsFilename, VariableSpace space, + FileSystemOptions fsOptions, boolean append ) throws KettleFileException { + try { + FileObject fileObject = getFileObject( vfsFilename, space, fsOptions ); + return getOutputStream( fileObject, append ); + } catch ( IOException e ) { + throw new KettleFileException( e ); + } + } + + @Override + public FileObject createTempFile( String prefix, KettleVFS.Suffix suffix ) throws KettleFileException { + return createTempFile( prefix, suffix, KettleVFS.TEMP_DIR ); + } + + @Override + public FileObject createTempFile( String prefix, KettleVFS.Suffix suffix, VariableSpace variableSpace ) + throws KettleFileException { + return createTempFile( prefix, suffix, KettleVFS.TEMP_DIR, variableSpace ); + } + + @Override + public FileObject createTempFile( String prefix, KettleVFS.Suffix suffix, String directory ) + throws KettleFileException { + return createTempFile( prefix, suffix, directory, null ); + } + + @Override + public FileObject createTempFile( String prefix, String suffix, String directory ) throws KettleFileException { + return createTempFile( prefix, suffix, directory, null ); + } + + @Override + public FileObject createTempFile( String prefix, KettleVFS.Suffix suffix, String directory, VariableSpace space ) + throws KettleFileException { + return createTempFile( prefix, suffix.getExt(), directory, space ); + } + + @Override + public FileObject createTempFile( String prefix, String suffix, String directory, VariableSpace space ) + throws KettleFileException { + try { + FileObject fileObject; + do { + // Build temporary file name using UUID to ensure uniqueness. Old mechanism would fail using Sort Rows (for + // example) + // when there multiple nodes with multiple JVMs on each node. In this case, the temp file names would end up + // being + // duplicated which would cause the sort to fail. + String filename = + new StringBuilder( 50 ).append( directory ).append( '/' ).append( prefix ).append( '_' ).append( + UUIDUtil.getUUIDAsString() ).append( suffix ).toString(); + fileObject = getFileObject( filename, space ); + } while (fileObject.exists()); + return fileObject; + } catch ( IOException e ) { + throw new KettleFileException( e ); + } + } + + @Override + public void reset() { + defaultVariableSpace = new Variables(); + defaultVariableSpace.initializeVariablesFrom( null ); + } + + +} diff --git a/core/src/main/java/org/pentaho/di/core/vfs/configuration/IKettleFileSystemConfigBuilder.java b/core/src/main/java/org/pentaho/di/core/vfs/configuration/IKettleFileSystemConfigBuilder.java index 71072a7fb8ef..87ed5487ce15 100644 --- a/core/src/main/java/org/pentaho/di/core/vfs/configuration/IKettleFileSystemConfigBuilder.java +++ b/core/src/main/java/org/pentaho/di/core/vfs/configuration/IKettleFileSystemConfigBuilder.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2022 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -22,10 +22,13 @@ package org.pentaho.di.core.vfs.configuration; +import org.pentaho.di.core.bowl.Bowl; +import org.pentaho.di.core.bowl.DefaultBowl; +import org.pentaho.di.core.variables.VariableSpace; + import java.io.IOException; import org.apache.commons.vfs2.FileSystemOptions; -import org.pentaho.di.core.variables.VariableSpace; /** * @author cboyden @@ -53,4 +56,13 @@ default void setParameter( FileSystemOptions opts, String name, VariableSpace va default Object getVariableSpace( FileSystemOptions fileSystemOptions ) { return null; }; + + default void setBowl( FileSystemOptions opts, Bowl bowl ) { + // noop + } + + default Bowl getBowl( FileSystemOptions opts ) { + return DefaultBowl.getInstance(); + } } + diff --git a/core/src/main/java/org/pentaho/di/core/vfs/configuration/KettleGenericFileSystemConfigBuilder.java b/core/src/main/java/org/pentaho/di/core/vfs/configuration/KettleGenericFileSystemConfigBuilder.java index b8a29c266e1c..dd1399944096 100644 --- a/core/src/main/java/org/pentaho/di/core/vfs/configuration/KettleGenericFileSystemConfigBuilder.java +++ b/core/src/main/java/org/pentaho/di/core/vfs/configuration/KettleGenericFileSystemConfigBuilder.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2022 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -29,6 +29,7 @@ import org.apache.commons.vfs2.FileSystemOptions; import org.apache.commons.vfs2.FileSystem; import org.apache.commons.vfs2.util.DelegatingFileSystemOptionsBuilder; +import org.pentaho.di.core.bowl.Bowl; import org.pentaho.di.core.logging.LogChannel; import org.pentaho.di.core.logging.LogChannelInterface; import org.pentaho.di.core.variables.VariableSpace; @@ -47,6 +48,9 @@ public class KettleGenericFileSystemConfigBuilder extends FileSystemConfigBuilder implements IKettleFileSystemConfigBuilder { + private static final String BOWL_KEY = "kettle.bowl"; + private static final String VAR_SPACE_KEY = "kettle.VariableSpace"; + private static final KettleGenericFileSystemConfigBuilder builder = new KettleGenericFileSystemConfigBuilder(); private static final LogChannelInterface log = new LogChannel( "cfgbuilder" ); @@ -141,13 +145,21 @@ public void setParameter( FileSystemOptions opts, String name, String value, Str @Override public void setParameter( FileSystemOptions opts, String name, VariableSpace value, String vfsUrl ) { - DelegatingFileSystemOptionsBuilder delegateFSOptionsBuilder = - new DelegatingFileSystemOptionsBuilder( KettleVFS.getInstance().getFileSystemManager() ); - super.setParam( opts, "VariableSpace", value ); + super.setParam( opts, VAR_SPACE_KEY, value ); } @Override public Object getVariableSpace( FileSystemOptions fileSystemOptions ) { - return getParam( fileSystemOptions, "VariableSpace" ); + return getParam( fileSystemOptions, VAR_SPACE_KEY ); + } + + @Override + public void setBowl( FileSystemOptions opts, Bowl bowl ) { + super.setParam( opts, BOWL_KEY, bowl ); + } + + @Override + public Bowl getBowl( FileSystemOptions opts ) { + return super.getParam( opts, BOWL_KEY ); } } diff --git a/core/src/main/java/org/pentaho/di/metastore/MetaStoreConst.java b/core/src/main/java/org/pentaho/di/metastore/MetaStoreConst.java index cb6e5bbfdd09..d536a75c5b0e 100644 --- a/core/src/main/java/org/pentaho/di/metastore/MetaStoreConst.java +++ b/core/src/main/java/org/pentaho/di/metastore/MetaStoreConst.java @@ -79,20 +79,21 @@ public class MetaStoreConst { public MetastoreLocator get() { // double-checked idiom because this can fail until the plugin is loaded. MetastoreLocator result = metastoreLocator; - if ( result == null ) { - synchronized ( this ) { - try { - if ( metastoreLocator == null ) { - Collection metastoreLocators = - PluginServiceLoader.loadServices( MetastoreLocator.class ); - metastoreLocator = result = metastoreLocators.stream().findFirst().orElse( null ); - } - } catch ( KettlePluginException e ) { - logger.error( "Error getting metastore locator", e ); + if ( result != null ) { + return result; + } + synchronized ( this ) { + try { + if ( metastoreLocator == null ) { + Collection metastoreLocators = + PluginServiceLoader.loadServices( MetastoreLocator.class ); + metastoreLocator = result = metastoreLocators.stream().findFirst().orElse( null ); } + } catch ( KettlePluginException e ) { + logger.error( "Error getting metastore locator", e ); } + return metastoreLocator; } - return result; } }; @@ -111,24 +112,25 @@ public IMetaStore getMetastoreForDirectory(String rootFolder) { public MetastoreDirectoryProvider get() { // double-checked idiom because this can fail until the plugin is loaded. MetastoreDirectoryProvider result = metastoreDirectoryProvider; - if ( result == null ) { - synchronized ( this ) { - try { - if ( metastoreDirectoryProvider == null ) { - Collection metastoreDirectoryProviders = - PluginServiceLoader.loadServices( MetastoreDirectoryProvider.class ); - metastoreDirectoryProvider = result = metastoreDirectoryProviders.stream().findFirst().orElse( null ); - if ( result == null ) { - // cache the null so we don't constantly try to look this up. - metastoreDirectoryProvider = result = NULL_METASTORE_DIRECTORY_PROVIDER; - } + if ( result != null ) { + return result; + } + synchronized ( this ) { + try { + if ( metastoreDirectoryProvider == null ) { + Collection metastoreDirectoryProviders = + PluginServiceLoader.loadServices( MetastoreDirectoryProvider.class ); + metastoreDirectoryProvider = result = metastoreDirectoryProviders.stream().findFirst().orElse( null ); + if ( result == null ) { + // cache the null so we don't constantly try to look this up. + metastoreDirectoryProvider = result = NULL_METASTORE_DIRECTORY_PROVIDER; } - } catch ( KettlePluginException e ) { - logger.error( "Error getting metastore directory provider", e ); } + } catch ( KettlePluginException e ) { + logger.error( "Error getting metastore directory provider", e ); } + return metastoreDirectoryProvider; } - return result; } }; @@ -139,24 +141,25 @@ public MetastoreDirectoryProvider get() { public IMetaStore get() { // double-checked idiom because this can fail until the plugin is loaded. IMetaStore result = metaStore; - if ( result == null ) { - synchronized( this ) { - if ( metaStore == null ) { - MetastoreLocator locator = metastoreLocatorSupplier.get(); - if ( locator != null ) { - metaStore = result = new SuppliedMetaStore( () -> locator.getMetastore() ); - } else if ( defaultToLocalXml ) { - try { - // Do not store as metaStore - return openLocalPentahoMetaStore(); - } catch ( MetaStoreException e ) { - logger.error( "Error opening local XML metastore", e ); - } + if ( result != null ) { + return result; + } + synchronized( this ) { + if ( metaStore == null ) { + MetastoreLocator locator = metastoreLocatorSupplier.get(); + if ( locator != null ) { + metaStore = result = new SuppliedMetaStore( () -> locator.getMetastore() ); + } else if ( defaultToLocalXml ) { + try { + // Do not store as metaStore + return openLocalPentahoMetaStore(); + } catch ( MetaStoreException e ) { + logger.error( "Error opening local XML metastore", e ); } } } + return metaStore; } - return result; } }; diff --git a/core/src/test/java/org/pentaho/di/connections/ConnectionManagerTest.java b/core/src/test/java/org/pentaho/di/connections/ConnectionManagerTest.java index 5650e885ebeb..897b3b6c10e7 100644 --- a/core/src/test/java/org/pentaho/di/connections/ConnectionManagerTest.java +++ b/core/src/test/java/org/pentaho/di/connections/ConnectionManagerTest.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2019-2022 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2019-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -30,6 +30,7 @@ import org.pentaho.di.connections.common.bucket.TestConnectionProvider; import org.pentaho.di.connections.vfs.VFSHelper; import org.pentaho.di.connections.vfs.VFSLookupFilter; +import org.pentaho.di.core.bowl.DefaultBowl; import org.pentaho.di.core.KettleClientEnvironment; import org.pentaho.di.core.variables.VariableSpace; import org.pentaho.metastore.persist.MetaStoreFactory; @@ -230,8 +231,8 @@ public void testGetConnectionDetailsBySchemeEmpty() { } @Test - public void testNullConnectionName() { - FileSystemOptions fileSystemOptions = VFSHelper.getOpts( "file://fakefile.ktr", null, null ); + public void testNullConnectionName() throws Exception { + FileSystemOptions fileSystemOptions = VFSHelper.getOpts( DefaultBowl.getInstance(), "file://fakefile.ktr", null, null ); Assert.assertNull( fileSystemOptions ); } diff --git a/core/src/test/java/org/pentaho/di/connections/common/bucket/TestConnectionProvider.java b/core/src/test/java/org/pentaho/di/connections/common/bucket/TestConnectionProvider.java index 2bb0799c926e..02fb8e738325 100644 --- a/core/src/test/java/org/pentaho/di/connections/common/bucket/TestConnectionProvider.java +++ b/core/src/test/java/org/pentaho/di/connections/common/bucket/TestConnectionProvider.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2022 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -51,6 +51,10 @@ public TestConnectionProvider( ConnectionManager connectionManager ) { return TestConnectionDetails.class; } + @Override public List getNames( ConnectionManager connectionManager ) { + return connectionManager.getNamesByType( getClass() ); + } + @Override public List getNames() { return connectionManager.getNamesByType( getClass() ); } @@ -60,6 +64,11 @@ public TestConnectionProvider( ConnectionManager connectionManager ) { return (List) connectionManager.getConnectionDetailsByScheme( getKey() ); } + @SuppressWarnings( "unchecked" ) + @Override public List getConnectionDetails( ConnectionManager connectionManager ) { + return (List) connectionManager.getConnectionDetailsByScheme( getKey() ); + } + @Override public boolean test( TestConnectionDetails connectionDetails ) { return true; } diff --git a/core/src/test/java/org/pentaho/di/connections/common/domain/TestConnectionWithDomainProvider.java b/core/src/test/java/org/pentaho/di/connections/common/domain/TestConnectionWithDomainProvider.java index 690aec5e416e..cba00019b415 100644 --- a/core/src/test/java/org/pentaho/di/connections/common/domain/TestConnectionWithDomainProvider.java +++ b/core/src/test/java/org/pentaho/di/connections/common/domain/TestConnectionWithDomainProvider.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2019-2022 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2019-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -55,11 +55,20 @@ public TestConnectionWithDomainProvider( ConnectionManager connectionManager ) { return connectionManager.getNamesByType( getClass() ); } + @Override public List getNames( ConnectionManager connectionManager ) { + return connectionManager.getNamesByType( getClass() ); + } + @SuppressWarnings( "unchecked" ) @Override public List getConnectionDetails() { return (List) connectionManager.getConnectionDetailsByScheme( getKey() ); } + @SuppressWarnings( "unchecked" ) + @Override public List getConnectionDetails( ConnectionManager connectionManager ) { + return (List) connectionManager.getConnectionDetailsByScheme( getKey() ); + } + @Override public boolean test( TestConnectionWithDomainDetails connectionDetails ) { return true; } diff --git a/engine/src/main/java/org/pentaho/di/base/AbstractMeta.java b/engine/src/main/java/org/pentaho/di/base/AbstractMeta.java index de55f6ccbbd4..9bb9c426ed0d 100644 --- a/engine/src/main/java/org/pentaho/di/base/AbstractMeta.java +++ b/engine/src/main/java/org/pentaho/di/base/AbstractMeta.java @@ -34,6 +34,7 @@ import org.pentaho.di.core.Props; import org.pentaho.di.core.attributes.metastore.EmbeddedMetaStore; import org.pentaho.di.core.bowl.Bowl; +import org.pentaho.di.core.bowl.DefaultBowl; import org.pentaho.di.core.bowl.HasBowlInterface; import org.pentaho.di.core.changed.ChangedFlag; import org.pentaho.di.core.changed.ChangedFlagInterface; @@ -136,7 +137,7 @@ public abstract class AbstractMeta implements ChangedFlagInterface, UndoInterfac protected RepositoryDirectoryInterface directory; - protected Bowl bowl; + protected Bowl bowl = DefaultBowl.getInstance(); /** * The repository to reference in the one-off case that it is needed diff --git a/engine/src/main/java/org/pentaho/di/job/entries/createfile/JobEntryCreateFile.java b/engine/src/main/java/org/pentaho/di/job/entries/createfile/JobEntryCreateFile.java index 26853b74c03a..d7a43b97a4e0 100644 --- a/engine/src/main/java/org/pentaho/di/job/entries/createfile/JobEntryCreateFile.java +++ b/engine/src/main/java/org/pentaho/di/job/entries/createfile/JobEntryCreateFile.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -162,7 +162,7 @@ public Result execute( Result previousResult, int nr ) throws KettleException { String realFilename = getRealFilename(); FileObject fileObject = null; try { - fileObject = KettleVFS.getFileObject( realFilename, this ); + fileObject = KettleVFS.getInstance( getParentJobMeta().getBowl() ).getFileObject( realFilename, this ); if ( fileObject.exists() ) { if ( isFailIfFileExists() ) { @@ -213,7 +213,7 @@ public Result execute( Result previousResult, int nr ) throws KettleException { private void addFilenameToResult( String targetFilename, Result result, Job parentJob ) throws KettleException { FileObject targetFile = null; try { - targetFile = KettleVFS.getFileObject( targetFilename, this ); + targetFile = KettleVFS.getInstance( getParentJobMeta().getBowl() ).getFileObject( targetFilename, this ); // Add to the result files... ResultFile resultFile = diff --git a/engine/src/main/java/org/pentaho/di/job/entries/deletefile/JobEntryDeleteFile.java b/engine/src/main/java/org/pentaho/di/job/entries/deletefile/JobEntryDeleteFile.java index 2ccc4117a293..b6ca17097176 100644 --- a/engine/src/main/java/org/pentaho/di/job/entries/deletefile/JobEntryDeleteFile.java +++ b/engine/src/main/java/org/pentaho/di/job/entries/deletefile/JobEntryDeleteFile.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -158,7 +158,7 @@ public Result execute( Result previousResult, int nr ) { FileObject fileObject = null; try { - fileObject = KettleVFS.getFileObject( realFilename, this ); + fileObject = KettleVFS.getInstance( getParentJobMeta().getBowl() ).getFileObject( realFilename, this ); if ( !fileObject.exists() ) { if ( isFailIfFileNotExists() ) { diff --git a/engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputFiles.java b/engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputFiles.java index 9ec03b51839f..6613c8416258 100644 --- a/engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputFiles.java +++ b/engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputFiles.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2018 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2018-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -163,8 +163,8 @@ public List getResourceDependencies( TransMeta transMeta, Ste references.add( reference ); String[] textFiles = - FileInputList.createFilePathList( transMeta, fileName, fileMask, excludeFileMask, fileRequired, - includeSubFolderBoolean() ); + FileInputList.createFilePathList( transMeta.getBowl(), transMeta, fileName, fileMask, + excludeFileMask, fileRequired, includeSubFolderBoolean() ); if ( textFiles != null ) { for ( int i = 0; i < textFiles.length; i++ ) { reference.getEntries().add( new ResourceEntry( textFiles[i], ResourceType.FILE ) ); diff --git a/engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputMeta.java b/engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputMeta.java index 791821e36066..4ff13d5c3a21 100644 --- a/engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputMeta.java +++ b/engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputMeta.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -25,6 +25,8 @@ import java.util.List; import com.google.common.base.Preconditions; +import org.pentaho.di.core.bowl.Bowl; +import org.pentaho.di.core.bowl.DefaultBowl; import org.pentaho.di.core.fileinput.FileInputList; import org.pentaho.di.core.injection.InjectionDeep; import org.pentaho.di.core.variables.VariableSpace; @@ -112,8 +114,12 @@ public static String getRequiredFilesCode( String tt ) { } public FileInputList getFileInputList( VariableSpace space ) { + return getFileInputList( DefaultBowl.getInstance(), space ); + } + + public FileInputList getFileInputList( Bowl bowl, VariableSpace space ) { inputFiles.normalizeAllocation( inputFiles.fileName.length ); - return FileInputList.createFileList( space, inputFiles.fileName, inputFiles.fileMask, inputFiles.excludeFileMask, + return FileInputList.createFileList( bowl, space, inputFiles.fileName, inputFiles.fileMask, inputFiles.excludeFileMask, inputFiles.fileRequired, inputFiles.includeSubFolderBoolean() ); } @@ -143,7 +149,7 @@ public String[] getFilePaths( final boolean showSamples ) { if ( parentStepMeta != null ) { final TransMeta parentTransMeta = parentStepMeta.getParentTransMeta(); if ( parentTransMeta != null ) { - final FileInputList inputList = getFileInputList( parentTransMeta ); + final FileInputList inputList = getFileInputList( parentTransMeta.getBowl(), parentTransMeta ); if ( inputList != null ) { return inputList.getFileStrings(); } diff --git a/engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputStep.java b/engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputStep.java index 98af58bbd7f6..c5f9f58e9a7a 100644 --- a/engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputStep.java +++ b/engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputStep.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2021 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -106,7 +106,7 @@ public boolean init( StepMetaInterface smi, StepDataInterface sdi ) { initErrorHandling(); meta.additionalOutputFields.normalize(); - data.files = meta.getFileInputList( this ); + data.files = meta.getFileInputList( getTransMeta().getBowl(), this ); data.currentFileIndex = 0; // If there are missing files, @@ -320,7 +320,8 @@ private RowMetaInterface[] filesFromPreviousStep() throws KettleException { } String fileValue = prevInfoFields.getString( fileRow, idx ); try { - FileObject parentFileObject = KettleVFS.getFileObject( environmentSubstitute( fileValue ), getTransMeta() ); + FileObject parentFileObject = KettleVFS.getInstance( getTransMeta().getBowl() ) + .getFileObject( environmentSubstitute( fileValue ), getTransMeta() ); boolean isDir = ( parentFileObject != null && parentFileObject.getType() == FileType.FOLDER ); int startingIndex = data.files.nrOfFiles(); diff --git a/engine/src/main/java/org/pentaho/di/trans/steps/fileinput/text/TextFileInputMeta.java b/engine/src/main/java/org/pentaho/di/trans/steps/fileinput/text/TextFileInputMeta.java index a051d88e1720..f2012a69e10b 100644 --- a/engine/src/main/java/org/pentaho/di/trans/steps/fileinput/text/TextFileInputMeta.java +++ b/engine/src/main/java/org/pentaho/di/trans/steps/fileinput/text/TextFileInputMeta.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2022 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -26,6 +26,7 @@ import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.StringUtils; import org.apache.commons.vfs2.FileObject; +import org.pentaho.di.core.bowl.Bowl; import org.pentaho.di.core.CheckResult; import org.pentaho.di.core.CheckResultInterface; import org.pentaho.di.core.Const; @@ -67,7 +68,6 @@ import org.pentaho.di.trans.steps.file.BaseFileInputAdditionalField; import org.pentaho.di.trans.steps.file.BaseFileInputFiles; import org.pentaho.di.trans.steps.file.BaseFileInputMeta; -import org.pentaho.di.workarounds.ResolvableResource; import org.pentaho.metastore.api.IMetaStore; import org.w3c.dom.Node; @@ -78,7 +78,7 @@ @SuppressWarnings( "deprecation" ) @InjectionSupported( localizationPrefix = "TextFileInput.Injection.", groups = { "FILENAME_LINES", "FIELDS", "FILTERS" } ) public class TextFileInputMeta extends BaseFileInputMeta - implements StepMetaInterface, ResolvableResource, CsvInputAwareMeta { + implements StepMetaInterface, CsvInputAwareMeta { private static Class PKG = TextFileInputMeta.class; // for i18n purposes, needed by Translator2!! TODO: check i18n // for base @@ -1125,7 +1125,7 @@ public void check( List remarks, TransMeta transMeta, Step remarks.add( cr ); } - FileInputList textFileList = getFileInputList( transMeta ); + FileInputList textFileList = getFileInputList( transMeta.getBowl(), transMeta ); if ( textFileList.nrOfFiles() == 0 ) { if ( !inputFiles.acceptingFilenames ) { cr = @@ -1347,15 +1347,15 @@ public String getAcceptingField() { return inputFiles.acceptingField; } - public String[] getFilePaths( VariableSpace space ) { + public String[] getFilePaths( Bowl bowl, VariableSpace space ) { return FileInputList.createFilePathList( - space, inputFiles.fileName, inputFiles.fileMask, inputFiles.excludeFileMask, + bowl, space, inputFiles.fileName, inputFiles.fileMask, inputFiles.excludeFileMask, inputFiles.fileRequired, inputFiles.includeSubFolderBoolean() ); } - public FileInputList getTextFileList( VariableSpace space ) { + public FileInputList getTextFileList( Bowl bowl, VariableSpace space ) { return FileInputList.createFileList( - space, inputFiles.fileName, inputFiles.fileMask, inputFiles.excludeFileMask, + bowl, space, inputFiles.fileName, inputFiles.fileMask, inputFiles.excludeFileMask, inputFiles.fileRequired, inputFiles.includeSubFolderBoolean() ); } @@ -1366,22 +1366,6 @@ FileObject getFileObject( String vfsFileName, VariableSpace variableSpace ) thro return KettleVFS.getFileObject( variableSpace.environmentSubstitute( vfsFileName ), variableSpace ); } - @Override - public void resolve() { - for ( int i = 0; i < inputFiles.fileName.length; i++ ) { - if ( inputFiles.fileName[i] != null && !inputFiles.fileName[i].isEmpty() ) { - try { - FileObject fileObject = KettleVFS.getFileObject( getParentStepMeta().getParentTransMeta().environmentSubstitute( inputFiles.fileName[i] ) ); - if ( AliasedFileObject.isAliasedFile( fileObject ) ) { - inputFiles.fileName[i] = ( (AliasedFileObject) fileObject ).getAELSafeURIString(); - } - } catch ( KettleFileException e ) { - throw new RuntimeException( e ); - } - } - } - } - @Override public boolean hasHeader() { return content == null ? false : content.header; @@ -1404,7 +1388,7 @@ public String getEnclosure() { @Override public FileObject getHeaderFileObject( final TransMeta transMeta ) { - final FileInputList fileList = getFileInputList( transMeta ); + final FileInputList fileList = getFileInputList( transMeta.getBowl(), transMeta ); return fileList.nrOfFiles() == 0 ? null : fileList.getFile( 0 ); } } diff --git a/engine/src/main/java/org/pentaho/di/trans/steps/textfileinput/InputFileMetaInterface.java b/engine/src/main/java/org/pentaho/di/trans/steps/textfileinput/InputFileMetaInterface.java index a3c1294d7639..53f3fd0fc9ca 100644 --- a/engine/src/main/java/org/pentaho/di/trans/steps/textfileinput/InputFileMetaInterface.java +++ b/engine/src/main/java/org/pentaho/di/trans/steps/textfileinput/InputFileMetaInterface.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -22,6 +22,8 @@ package org.pentaho.di.trans.steps.textfileinput; +import org.pentaho.di.core.bowl.Bowl; +import org.pentaho.di.core.bowl.DefaultBowl; import org.pentaho.di.core.variables.VariableSpace; import org.pentaho.di.trans.step.StepMetaInterface; @@ -39,7 +41,17 @@ public interface InputFileMetaInterface extends StepMetaInterface { public int getNrHeaderLines(); - public String[] getFilePaths( VariableSpace space ); + /** + * @deprecated replaced by getFilePaths( Bowl bowl, VariableSpace space ) + */ + @Deprecated + default String[] getFilePaths( VariableSpace space ) { + throw new UnsupportedOperationException( "deprecated" ); + } + + default String[] getFilePaths( Bowl bowl, VariableSpace space ) { + return getFilePaths( space ); + } public boolean isErrorIgnored(); diff --git a/engine/src/main/java/org/pentaho/di/trans/steps/textfileinput/TextFileInput.java b/engine/src/main/java/org/pentaho/di/trans/steps/textfileinput/TextFileInput.java index 31f44ad1c9aa..c36f9fe639ef 100644 --- a/engine/src/main/java/org/pentaho/di/trans/steps/textfileinput/TextFileInput.java +++ b/engine/src/main/java/org/pentaho/di/trans/steps/textfileinput/TextFileInput.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2021 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -1532,7 +1532,7 @@ public boolean init( StepMetaInterface smi, StepDataInterface sdi ) { initErrorHandling(); initReplayFactory(); - data.setFiles( meta.getTextFileList( this ) ); + data.setFiles( meta.getTextFileList( getTransMeta().getBowl(), this ) ); data.filterProcessor = new TextFileFilterProcessor( meta.getFilter() ); // If there are missing files, diff --git a/engine/src/main/java/org/pentaho/di/trans/steps/textfileinput/TextFileInputMeta.java b/engine/src/main/java/org/pentaho/di/trans/steps/textfileinput/TextFileInputMeta.java index f76163da5fa9..8aae69de7c80 100644 --- a/engine/src/main/java/org/pentaho/di/trans/steps/textfileinput/TextFileInputMeta.java +++ b/engine/src/main/java/org/pentaho/di/trans/steps/textfileinput/TextFileInputMeta.java @@ -3,7 +3,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -30,6 +30,7 @@ import org.apache.commons.codec.binary.Base64; import org.apache.commons.vfs2.FileObject; +import org.pentaho.di.core.bowl.Bowl; import org.pentaho.di.core.CheckResult; import org.pentaho.di.core.CheckResultInterface; import org.pentaho.di.core.Const; @@ -1591,14 +1592,14 @@ public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transform } @Override - public String[] getFilePaths( VariableSpace space ) { + public String[] getFilePaths( Bowl bowl, VariableSpace space ) { return FileInputList.createFilePathList( - space, fileName, fileMask, excludeFileMask, fileRequired, includeSubFolderBoolean() ); + bowl, space, fileName, fileMask, excludeFileMask, fileRequired, includeSubFolderBoolean() ); } - public FileInputList getTextFileList( VariableSpace space ) { + public FileInputList getTextFileList( Bowl bowl, VariableSpace space ) { return FileInputList.createFileList( - space, fileName, fileMask, excludeFileMask, fileRequired, includeSubFolderBoolean() ); + bowl, space, fileName, fileMask, excludeFileMask, fileRequired, includeSubFolderBoolean() ); } private boolean[] includeSubFolderBoolean() { @@ -1636,7 +1637,7 @@ public void check( List remarks, TransMeta transMeta, Step remarks.add( cr ); } - FileInputList textFileList = getTextFileList( transMeta ); + FileInputList textFileList = getTextFileList( transMeta.getBowl(), transMeta ); if ( textFileList.nrOfFiles() == 0 ) { if ( !isAcceptingFilenames() ) { cr = diff --git a/engine/src/main/java/org/pentaho/di/trans/steps/textfileoutput/TextFileOutput.java b/engine/src/main/java/org/pentaho/di/trans/steps/textfileoutput/TextFileOutput.java index 6836352df006..90d103485381 100644 --- a/engine/src/main/java/org/pentaho/di/trans/steps/textfileoutput/TextFileOutput.java +++ b/engine/src/main/java/org/pentaho/di/trans/steps/textfileoutput/TextFileOutput.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2023 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -207,7 +207,8 @@ public void initFileStreamWriter( String filename ) throws KettleException { data.getFileStreamsCollection().add( filename, fileStreams ); if ( log.isDetailed() ) { - logDetailed( "Opened new file with name [" + KettleVFS.getFriendlyURI( filename ) + "]" ); + logDetailed( "Opened new file with name [" + KettleVFS.getInstance( getTransMeta().getBowl() ) + .getFriendlyURI( filename ) + "]" ); } } else if ( fileStreams.getBufferedOutputStream() == null ) { // File was previously opened and now needs to be reopened. int maxOpenFiles = getMaxOpenFiles(); @@ -227,7 +228,7 @@ public void initFileStreamWriter( String filename ) throws KettleException { } } catch ( Exception e ) { if ( !( e instanceof KettleException ) ) { - throw new KettleException( "Error opening new file : " + e.toString() ); + throw new KettleException( "Error opening new file : " + e.toString(), e ); } else { throw (KettleException) e; } @@ -828,10 +829,12 @@ public boolean init( StepMetaInterface smi, StepDataInterface sdi ) { } catch ( Exception e ) { if ( getParentVariableSpace() == null ) { logError( "Couldn't open file " - + KettleVFS.getFriendlyURI( meta.getFileName() ) + "." + meta.getExtension(), e ); + + KettleVFS.getInstance( getTransMeta().getBowl() ) + .getFriendlyURI( meta.getFileName() ) + "." + meta.getExtension(), e ); } else { logError( "Couldn't open file " - + KettleVFS.getFriendlyURI( getParentVariableSpace().environmentSubstitute( meta.getFileName() ) ) + + KettleVFS.getInstance( getTransMeta().getBowl() ) + .getFriendlyURI( getParentVariableSpace().environmentSubstitute( meta.getFileName() ) ) + "." + getParentVariableSpace().environmentSubstitute( meta.getExtension() ), e ); } setErrors( 1L ); @@ -1005,7 +1008,8 @@ private void createParentFolder( String filename ) throws Exception { } } else { throw new KettleException( BaseMessages.getString( PKG, "TextFileOutput.Log.ParentFolderNotExistCreateIt", - KettleVFS.getFriendlyURI( parentfolder ), KettleVFS.getFriendlyURI( filename ) ) ); + KettleVFS.getFriendlyURI( parentfolder ), + KettleVFS.getInstance( getTransMeta().getBowl() ).getFriendlyURI( filename ) ) ); } } } finally { @@ -1059,15 +1063,15 @@ boolean isEnclosureFixDisabledAndContainsSeparatorOrEnclosure( byte[] source ) { } protected FileObject getFileObject( String vfsFilename ) throws KettleFileException { - return KettleVFS.getFileObject( vfsFilename ); + return KettleVFS.getInstance( getTransMeta().getBowl() ).getFileObject( vfsFilename ); } protected FileObject getFileObject( String vfsFilename, VariableSpace space ) throws KettleFileException { - return KettleVFS.getFileObject( vfsFilename, space ); + return KettleVFS.getInstance( getTransMeta().getBowl() ).getFileObject( vfsFilename, space ); } protected OutputStream getOutputStream( String vfsFilename, VariableSpace space, boolean append ) throws KettleFileException { - return KettleVFS.getOutputStream( vfsFilename, space, append ); + return KettleVFS.getInstance( getTransMeta().getBowl() ).getOutputStream( vfsFilename, space, append ); } } diff --git a/engine/src/main/java/org/pentaho/di/trans/steps/textfileoutput/TextFileOutputMeta.java b/engine/src/main/java/org/pentaho/di/trans/steps/textfileoutput/TextFileOutputMeta.java index 42a75b9e4bf6..22eb78082ed6 100644 --- a/engine/src/main/java/org/pentaho/di/trans/steps/textfileoutput/TextFileOutputMeta.java +++ b/engine/src/main/java/org/pentaho/di/trans/steps/textfileoutput/TextFileOutputMeta.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2023 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -54,7 +54,6 @@ import org.pentaho.di.trans.step.StepMeta; import org.pentaho.di.trans.step.StepMetaInterface; import org.pentaho.di.trans.steps.file.BaseFileOutputMeta; -import org.pentaho.di.workarounds.ResolvableResource; import org.pentaho.metastore.api.IMetaStore; import org.w3c.dom.Node; @@ -67,7 +66,7 @@ * */ @InjectionSupported( localizationPrefix = "TextFileOutput.Injection.", groups = { "OUTPUT_FIELDS" } ) -public class TextFileOutputMeta extends BaseFileOutputMeta implements StepMetaInterface, ResolvableResource { +public class TextFileOutputMeta extends BaseFileOutputMeta implements StepMetaInterface { private static Class PKG = TextFileOutputMeta.class; // for i18n purposes, needed by Translator2!! // Strings used in XML @@ -1150,20 +1149,6 @@ public boolean passDataToServletOutput() { return servletOutput; } - @Override - public void resolve() { - if ( fileName != null && !fileName.isEmpty() ) { - try { - FileObject fileObject = KettleVFS.getFileObject( getParentStepMeta().getParentTransMeta().environmentSubstitute( fileName ) ); - if ( AliasedFileObject.isAliasedFile( fileObject ) ) { - fileName = ( (AliasedFileObject) fileObject ).getAELSafeURIString(); - } - } catch ( KettleFileException e ) { - throw new RuntimeException( e ); - } - } - } - /** *

Creates a copy of the meta information of the output fields, so that we don't make any changes to the * original meta information.

diff --git a/engine/src/test/java/org/pentaho/di/trans/steps/fileinput/text/TextFileInputTest.java b/engine/src/test/java/org/pentaho/di/trans/steps/fileinput/text/TextFileInputTest.java index 1f8cf2fbfb8b..ce2014729309 100644 --- a/engine/src/test/java/org/pentaho/di/trans/steps/fileinput/text/TextFileInputTest.java +++ b/engine/src/test/java/org/pentaho/di/trans/steps/fileinput/text/TextFileInputTest.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2021 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -30,6 +30,7 @@ import org.junit.Test; import org.mockito.Mockito; import org.mockito.stubbing.Answer; +import org.pentaho.di.core.bowl.DefaultBowl; import org.pentaho.di.core.KettleEnvironment; import org.pentaho.di.core.RowSet; import org.pentaho.di.core.exception.KettleFileException; @@ -298,6 +299,7 @@ public void test_PDI17117() throws Exception { Mockito.when( input.getRowFrom( rowset ) ).thenReturn( obj1, obj2, null ); Mockito.doReturn( rwi ).when( rowset ).getRowMeta(); Mockito.when( rwi.getString( obj2, 0 ) ).thenReturn( "filename1", "filename2" ); + Mockito.when( input.getTransMeta().getBowl() ).thenReturn( DefaultBowl.getInstance() ); List output = TransTestingUtil.execute( input, meta, data, 0, false ); List passThroughKeys = new ArrayList<>( data.passThruFields.keySet() ); @@ -353,6 +355,7 @@ public void testFolderFromPreviousStep() throws Exception { Mockito.when( input.getTransMeta().listVariables() ).thenReturn( space.listVariables() ); Mockito.when( input.getTransMeta().getVariable( anyString() ) ).thenAnswer( (Answer) invocation -> space.getVariable( (String) invocation.getArguments()[0] ) ); + Mockito.when( input.getTransMeta().getBowl() ).thenReturn( DefaultBowl.getInstance() ); Mockito.doReturn( rwi ).when( rowset ).getRowMeta(); Mockito.when( rwi.getString( obj2, 0 ) ).thenReturn( "ram:///." ); diff --git a/engine/src/test/java/org/pentaho/di/trans/steps/textfileoutput/TextFileOutputTest.java b/engine/src/test/java/org/pentaho/di/trans/steps/textfileoutput/TextFileOutputTest.java index 3b67a19a82d1..7b6312531dfb 100644 --- a/engine/src/test/java/org/pentaho/di/trans/steps/textfileoutput/TextFileOutputTest.java +++ b/engine/src/test/java/org/pentaho/di/trans/steps/textfileoutput/TextFileOutputTest.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2023 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -67,6 +67,7 @@ import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; +import org.pentaho.di.core.bowl.DefaultBowl; import org.pentaho.di.core.Const; import org.pentaho.di.core.RowSet; import org.pentaho.di.core.compress.CompressionOutputStream; @@ -225,6 +226,7 @@ public void setUp() throws Exception { when( stepMockHelper.processRowsStepMetaInterface.getEnclosure() ).thenReturn( "\"" ); when( stepMockHelper.processRowsStepMetaInterface.getNewline() ).thenReturn( "\n" ); when( stepMockHelper.transMeta.listVariables() ).thenReturn( new String[0] ); + when( stepMockHelper.transMeta.getBowl() ).thenReturn( DefaultBowl.getInstance() ); } @After diff --git a/plugins/connections/ui/src/test/java/org/pentaho/di/connections/ui/TestConnectionProvider.java b/plugins/connections/ui/src/test/java/org/pentaho/di/connections/ui/TestConnectionProvider.java index c028a3527c07..7c72b19e27d2 100644 --- a/plugins/connections/ui/src/test/java/org/pentaho/di/connections/ui/TestConnectionProvider.java +++ b/plugins/connections/ui/src/test/java/org/pentaho/di/connections/ui/TestConnectionProvider.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2020 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -51,11 +51,19 @@ public TestConnectionProvider( ConnectionManager connectionManager ) { } @Override public List getNames() { + return getNames( connectionManager ); + } + + @Override public List getNames( ConnectionManager connectionManager ) { return connectionManager.getNamesByType( getClass() ); } - @SuppressWarnings( "unchecked" ) @Override public List getConnectionDetails() { + return getConnectionDetails( connectionManager ); + } + + @SuppressWarnings( "unchecked" ) + @Override public List getConnectionDetails( ConnectionManager connectionManager ) { return (List) connectionManager.getConnectionDetailsByScheme( getKey() ); } diff --git a/plugins/file-open-save-new/core/src/main/java/org/pentaho/di/plugins/fileopensave/providers/vfs/VFSFileProvider.java b/plugins/file-open-save-new/core/src/main/java/org/pentaho/di/plugins/fileopensave/providers/vfs/VFSFileProvider.java index e25ab0b4650c..7b8ae3c70138 100644 --- a/plugins/file-open-save-new/core/src/main/java/org/pentaho/di/plugins/fileopensave/providers/vfs/VFSFileProvider.java +++ b/plugins/file-open-save-new/core/src/main/java/org/pentaho/di/plugins/fileopensave/providers/vfs/VFSFileProvider.java @@ -36,6 +36,8 @@ import org.pentaho.di.connections.vfs.VFSHelper; import org.pentaho.di.connections.vfs.VFSRoot; import org.pentaho.di.connections.vfs.provider.ConnectionFileProvider; +import org.pentaho.di.core.bowl.Bowl; +import org.pentaho.di.core.bowl.DefaultBowl; import org.pentaho.di.core.exception.KettleException; import org.pentaho.di.core.exception.KettleFileException; import org.pentaho.di.core.variables.VariableSpace; @@ -51,6 +53,7 @@ import org.pentaho.di.plugins.fileopensave.providers.vfs.model.VFSFile; import org.pentaho.di.plugins.fileopensave.providers.vfs.model.VFSLocation; import org.pentaho.di.plugins.fileopensave.providers.vfs.model.VFSTree; +import org.pentaho.metastore.api.exceptions.MetaStoreException; import java.io.IOException; import java.io.InputStream; @@ -59,7 +62,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Supplier; /** * Created by bmorrise on 2/14/19. @@ -70,9 +72,10 @@ public class VFSFileProvider extends BaseFileProvider { public static final String TYPE = "vfs"; public static final String DOMAIN_ROOT = "[\\w]+://"; - private Supplier connectionManagerSupplier = ConnectionManager::getInstance; + private Bowl bowl = DefaultBowl.getInstance(); private Map> roots = new HashMap<>(); + @Override public Class getFileClass() { return VFSFile.class; } @@ -109,36 +112,39 @@ public boolean isAvailable() { @Override public VFSTree getTree( List connectionTypes ) { VFSTree vfsTree = new VFSTree( NAME ); - List> providers = - connectionManagerSupplier.get().getProvidersByType( VFSConnectionProvider.class ); - - for ( ConnectionProvider provider : providers ) { - for ( ConnectionDetails connectionDetails : provider.getConnectionDetails() ) { - VFSConnectionDetails vfsConnectionDetails = (VFSConnectionDetails) connectionDetails; - VFSLocation vfsLocation = new VFSLocation(); - vfsLocation.setName( connectionDetails.getName() ); - vfsLocation.setRoot( NAME ); - vfsLocation.setHasChildren( true ); - vfsLocation.setCanDelete( false ); - String path = vfsConnectionDetails.getType() + "://"; - if ( KettleVFS.SMB_SCHEME.equals( vfsConnectionDetails.getType() ) ) { - path += vfsConnectionDetails.getName(); - } else { - path += vfsConnectionDetails.getDomain(); - } - vfsLocation.setPath( path ); - vfsLocation.setDomain( vfsConnectionDetails.getDomain() ); - vfsLocation.setConnection( connectionDetails.getName() ); - vfsLocation.setCanAddChildren( vfsConnectionDetails.hasBuckets() ); - if ( connectionDetails.getType().startsWith( "s3" ) || connectionDetails.getType().startsWith( "snw" ) ) { - vfsLocation.setHasBuckets( true ); - } - if ( connectionTypes.isEmpty() || connectionTypes.contains( connectionDetails.getType() ) ) { - vfsTree.addChild( vfsLocation ); + try { + List> providers = + bowl.getConnectionManager().getProvidersByType( VFSConnectionProvider.class ); + + for ( ConnectionProvider provider : providers ) { + for ( ConnectionDetails connectionDetails : provider.getConnectionDetails( bowl.getConnectionManager() ) ) { + VFSConnectionDetails vfsConnectionDetails = (VFSConnectionDetails) connectionDetails; + VFSLocation vfsLocation = new VFSLocation(); + vfsLocation.setName( connectionDetails.getName() ); + vfsLocation.setRoot( NAME ); + vfsLocation.setHasChildren( true ); + vfsLocation.setCanDelete( false ); + String path = vfsConnectionDetails.getType() + "://"; + if ( KettleVFS.SMB_SCHEME.equals( vfsConnectionDetails.getType() ) ) { + path += vfsConnectionDetails.getName(); + } else { + path += vfsConnectionDetails.getDomain(); + } + vfsLocation.setPath( path ); + vfsLocation.setDomain( vfsConnectionDetails.getDomain() ); + vfsLocation.setConnection( connectionDetails.getName() ); + vfsLocation.setCanAddChildren( vfsConnectionDetails.hasBuckets() ); + if ( connectionDetails.getType().startsWith( "s3" ) || connectionDetails.getType().startsWith( "snw" ) ) { + vfsLocation.setHasBuckets( true ); + } + if ( connectionTypes.isEmpty() || connectionTypes.contains( connectionDetails.getType() ) ) { + vfsTree.addChild( vfsLocation ); + } } } + } catch ( MetaStoreException e ) { + // ignored } - return vfsTree; } @@ -152,18 +158,21 @@ private List getRoot( VFSFile file ) throws FileException { } List files = new ArrayList<>(); - VFSConnectionDetails vfsConnectionDetails = - (VFSConnectionDetails) ConnectionManager.getInstance().getConnectionDetails( file.getConnection() ); - if( !vfsConnectionDetails.hasBuckets() ) { - return null; - } - @SuppressWarnings( "unchecked" ) - VFSConnectionProvider vfsConnectionProvider = - (VFSConnectionProvider) ConnectionManager.getInstance() - .getConnectionProvider( vfsConnectionDetails.getType() ); - List vfsRoots = new ArrayList<>(); + VFSConnectionDetails vfsConnectionDetails; + VFSConnectionProvider vfsConnectionProvider; try { + vfsConnectionDetails = (VFSConnectionDetails) bowl.getConnectionManager() + .getConnectionDetails( file.getConnection() ); + if( !vfsConnectionDetails.hasBuckets() ) { + return null; + } + @SuppressWarnings( "unchecked" ) + VFSConnectionProvider temp = + (VFSConnectionProvider) bowl + .getConnectionManager().getConnectionProvider( vfsConnectionDetails.getType() ); + vfsConnectionProvider = temp; + vfsRoots = vfsConnectionProvider.getLocations( vfsConnectionDetails ); } catch ( Exception e ) { throw new FileException( "Error getting VFS locations. Check your credentials and for connectivity." + e.getMessage(), e ); @@ -205,9 +214,10 @@ public List getFiles( VFSFile file, String filters, VariableSpace space } FileObject fileObject; try { - fileObject = KettleVFS - .getFileObject( file.getPath(), new Variables(), VFSHelper.getOpts( file.getPath(), file.getConnection(), space ) ); - } catch ( KettleFileException e ) { + fileObject = KettleVFS.getInstance( bowl ) + .getFileObject( file.getPath(), new Variables(), VFSHelper.getOpts( bowl, file.getPath(), file.getConnection(), + space ) ); + } catch ( KettleFileException | MetaStoreException e ) { throw new FileNotFoundException( file.getPath(), TYPE ); } return populateChildren( file, fileObject, filters ); @@ -269,7 +279,8 @@ private List populateChildren( VFSFile parent, FileObject fileObject, S @Override public VFSFile getFile( VFSFile file, VariableSpace space ) { try { FileObject fileObject = KettleVFS - .getFileObject( file.getPath(), new Variables(), VFSHelper.getOpts( file.getPath(), file.getConnection(), space ) ); + .getFileObject( file.getPath(), new Variables(), VFSHelper.getOpts( bowl, file.getPath(), file.getConnection(), + space ) ); if ( !fileObject.exists() ) { return null; } @@ -284,7 +295,7 @@ private List populateChildren( VFSFile parent, FileObject fileObject, S } else { return VFSFile.create( parent, fileObject, null, file.getDomain() ); } - } catch ( KettleFileException | FileSystemException e ) { + } catch ( KettleFileException | FileSystemException | MetaStoreException e ) { // File does not exist } return null; @@ -301,11 +312,12 @@ public List delete( List files, VariableSpace space ) { for ( VFSFile file : files ) { try { FileObject fileObject = KettleVFS - .getFileObject( file.getPath(), new Variables(), VFSHelper.getOpts( file.getPath(), file.getConnection(), space ) ); + .getFileObject( file.getPath(), new Variables(), VFSHelper.getOpts( bowl, file.getPath(), + file.getConnection(), space ) ); if ( fileObject.delete( getAllFileSelector() ) > 0 ) { deletedFiles.add( file ); } - } catch ( KettleFileException | FileSystemException kfe ) { + } catch ( KettleFileException | FileSystemException | MetaStoreException kfe ) { // Ignore don't add } } @@ -320,11 +332,11 @@ public List delete( List files, VariableSpace space ) { try { FileObject fileObject = KettleVFS .getFileObject( folder.getPath(), space, - VFSHelper.getOpts( folder.getPath(), folder.getConnection(), space ) ); + VFSHelper.getOpts( bowl, folder.getPath(), folder.getConnection(), space ) ); fileObject.createFolder(); String parent = folder.getPath().substring( 0, folder.getPath().length() - 1 ); return VFSDirectory.create( parent, fileObject, folder.getConnection(), folder.getDomain() ); - } catch ( KettleFileException | FileSystemException ignored ) { + } catch ( KettleFileException | FileSystemException | MetaStoreException ignored ) { // Ignored } return null; @@ -362,9 +374,11 @@ public VFSFile move( VFSFile file, String toPath, OverwriteStatus overwriteStatu private VFSFile doMove( VFSFile file, String newPath, OverwriteStatus overwriteStatus, VariableSpace space ) { try { FileObject fileObject = KettleVFS - .getFileObject( file.getPath(), new Variables(), VFSHelper.getOpts( file.getPath(), file.getConnection(), space ) ); + .getFileObject( file.getPath(), new Variables(), VFSHelper.getOpts( bowl, file.getPath(), file.getConnection(), + space ) ); FileObject renameObject = KettleVFS - .getFileObject( newPath, new Variables(), VFSHelper.getOpts( file.getPath(), file.getConnection(), space ) ); + .getFileObject( newPath, new Variables(), VFSHelper.getOpts( bowl, file.getPath(), file.getConnection(), + space ) ); if ( renameObject.exists() ) { overwriteStatus.promptOverwriteIfNecessary( file.getPath(), @@ -379,7 +393,8 @@ private VFSFile doMove( VFSFile file, String newPath, OverwriteStatus overwriteS file.getDomain() ); newPath = getNewName( vfsDir, newPath, space ); renameObject = KettleVFS - .getFileObject( newPath, new Variables(), VFSHelper.getOpts( file.getPath(), file.getConnection(), space ) ); + .getFileObject( newPath, new Variables(), VFSHelper.getOpts( bowl, file.getPath(), file.getConnection(), + space ) ); } } fileObject.moveTo( renameObject ); @@ -390,7 +405,7 @@ private VFSFile doMove( VFSFile file, String newPath, OverwriteStatus overwriteS return VFSFile.create( renameObject.getParent().getPublicURIString(), renameObject, file.getConnection(), file.getDomain() ); } - } catch ( KettleFileException | FileSystemException| FileException e ) { + } catch ( KettleFileException | FileSystemException | FileException | MetaStoreException e ) { return null; } } @@ -410,10 +425,10 @@ public VFSFile copy( VFSFile file, String toPath, OverwriteStatus overwriteStatu overwriteStatus.setCurrentFileInProgressDialog( file.getPath() ); FileObject fileObject = KettleVFS - .getFileObject( file.getPath(), space, VFSHelper.getOpts( file.getPath(), file.getConnection(), space ) ); + .getFileObject( file.getPath(), space, VFSHelper.getOpts( bowl, file.getPath(), file.getConnection(), space ) ); FileObject copyObject = KettleVFS.getFileObject( toPath, new Variables(), - VFSHelper.getOpts( file.getPath(), file.getConnection(), space ) ); + VFSHelper.getOpts( bowl, file.getPath(), file.getConnection(), space ) ); overwriteStatus.promptOverwriteIfNecessary( copyObject.exists(), toPath, file.getEntityType().isDirectory() ? "folder" : "file" , null, @@ -429,7 +444,7 @@ public VFSFile copy( VFSFile file, String toPath, OverwriteStatus overwriteStatu toDirectory = VFSDirectory.create( copyObject.getParent().getPublicURIString(), copyObject, file.getConnection(), file.getDomain() ); String newDestination = getNewName( toDirectory, copyObject.getName().toString(), space ); - copyObject = KettleVFS.getFileObject( newDestination, new Variables(), VFSHelper.getOpts( file.getPath(), + copyObject = KettleVFS.getFileObject( newDestination, new Variables(), VFSHelper.getOpts( bowl, file.getPath(), file.getConnection(), space ) ); } @@ -443,7 +458,7 @@ public VFSFile copy( VFSFile file, String toPath, OverwriteStatus overwriteStatu return VFSFile .create( copyObject.getParent().getPublicURIString(), fileObject, file.getConnection(), file.getDomain() ); } - } catch ( KettleFileException | FileSystemException e ) { + } catch ( KettleFileException | FileSystemException | MetaStoreException e ) { throw new FileException(); } } @@ -458,9 +473,9 @@ public VFSFile copy( VFSFile file, String toPath, OverwriteStatus overwriteStatu path = sanitizeName( dir, path ); try { FileObject fileObject = - KettleVFS.getFileObject( path, space, VFSHelper.getOpts( path, dir.getConnection(), space ) ); + KettleVFS.getFileObject( path, space, VFSHelper.getOpts( bowl, path, dir.getConnection(), space ) ); return fileObject.exists(); - } catch ( KettleFileException | FileSystemException e ) { + } catch ( KettleFileException | FileSystemException | MetaStoreException e ) { throw new FileException(); } } @@ -474,9 +489,10 @@ public VFSFile copy( VFSFile file, String toPath, OverwriteStatus overwriteStatu public InputStream readFile( VFSFile file, VariableSpace space ) { try { FileObject fileObject = KettleVFS - .getFileObject( file.getPath(), new Variables(), VFSHelper.getOpts( file.getPath(), file.getConnection(), space ) ); + .getFileObject( file.getPath(), new Variables(), VFSHelper.getOpts( bowl, file.getPath(), file.getConnection(), + space ) ); return fileObject.getContent().getInputStream(); - } catch ( KettleException | FileSystemException e ) { + } catch ( KettleException | FileSystemException | MetaStoreException e ) { return null; } } @@ -495,8 +511,9 @@ public InputStream readFile( VFSFile file, VariableSpace space ) { FileObject fileObject = null; try { fileObject = KettleVFS - .getFileObject( path, new Variables(), VFSHelper.getOpts( destDir.getPath(), destDir.getConnection(), space ) ); - } catch ( KettleException ke ) { + .getFileObject( path, new Variables(), VFSHelper.getOpts( bowl, destDir.getPath(), destDir.getConnection(), + space ) ); + } catch ( KettleException | MetaStoreException ke ) { throw new FileException(); } if ( fileObject != null ) { @@ -541,7 +558,7 @@ public boolean isSame( org.pentaho.di.plugins.fileopensave.api.providers.File fi String testName = sanitizeName( destDir, newPath ); try { while ( KettleVFS - .getFileObject( testName, new Variables(), VFSHelper.getOpts( testName, destDir.getConnection(), space ) ) + .getFileObject( testName, new Variables(), VFSHelper.getOpts( bowl, testName, destDir.getConnection(), space ) ) .exists() ) { if ( Utils.isValidExtension( extension ) ) { testName = sanitizeName( destDir, parent + name + "_" + i + "." + extension ); @@ -550,7 +567,7 @@ public boolean isSame( org.pentaho.di.plugins.fileopensave.api.providers.File fi } i++; } - } catch ( KettleFileException | FileSystemException e ) { + } catch ( KettleFileException | FileSystemException | MetaStoreException e ) { return testName; } return testName; @@ -571,15 +588,19 @@ public boolean isSame( org.pentaho.di.plugins.fileopensave.api.providers.File fi if ( newPath.startsWith( ConnectionFileProvider.SCHEME + "://" ) ) { return newPath; } - return getConnectionProvider( newPath ).sanitizeName( newPath ); + return getConnectionProvider( newPath ).sanitizeName( newPath ); } - private VFSConnectionProvider getConnectionProvider( String key ) { - @SuppressWarnings( "unchecked" ) - VFSConnectionProvider vfsConnectionProvider = - (VFSConnectionProvider) ConnectionManager.getInstance() - .getConnectionProvider( key ); - return vfsConnectionProvider; + private VFSConnectionProvider getConnectionProvider( String key ) { + try { + @SuppressWarnings( "unchecked" ) + VFSConnectionProvider vfsConnectionProvider = + (VFSConnectionProvider) bowl.getConnectionManager() + .getConnectionProvider( key ); + return vfsConnectionProvider; + } catch ( MetaStoreException ex ) { + return null; + } } public void clearProviderCache() { @@ -590,11 +611,11 @@ public void clearProviderCache() { try { FileObject fileObject = KettleVFS .getFileObject( file.getPath() + VFSFile.DELIMITER + newDirectoryName, new Variables(), - VFSHelper.getOpts( file.getPath(), file.getConnection(), new Variables() ) ); + VFSHelper.getOpts( bowl, file.getPath(), file.getConnection(), new Variables() ) ); fileObject.createFolder(); return VFSDirectory.create( parentPath, fileObject, file.getConnection(), file.getDomain() ); - } catch ( KettleFileException | FileSystemException ignored ) { + } catch ( KettleFileException | FileSystemException | MetaStoreException ignored ) { // Ignored } return null; diff --git a/plugins/json/core/src/test/java/org/pentaho/di/trans/steps/jsoninput/JsonInputTest.java b/plugins/json/core/src/test/java/org/pentaho/di/trans/steps/jsoninput/JsonInputTest.java index 07846f29d32a..66c86e9c0c99 100644 --- a/plugins/json/core/src/test/java/org/pentaho/di/trans/steps/jsoninput/JsonInputTest.java +++ b/plugins/json/core/src/test/java/org/pentaho/di/trans/steps/jsoninput/JsonInputTest.java @@ -2,7 +2,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2022 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -61,6 +61,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.mockito.internal.util.reflection.Whitebox; +import org.pentaho.di.core.bowl.Bowl; import org.pentaho.di.core.Const; import org.pentaho.di.core.KettleClientEnvironment; import org.pentaho.di.core.RowSet; @@ -1266,7 +1267,7 @@ protected void disposeJsonInput( JsonInput jsonInput ) { protected JsonInputMeta createFileListMeta( final List files ) { JsonInputMeta meta = new JsonInputMeta() { @Override - public FileInputList getFileInputList( VariableSpace space ) { + public FileInputList getFileInputList( Bowl bowl, VariableSpace space ) { return new FileInputList() { @Override public List getFiles() { diff --git a/ui/src/main/java/org/pentaho/di/ui/trans/steps/fileinput/text/TextFileInputDialog.java b/ui/src/main/java/org/pentaho/di/ui/trans/steps/fileinput/text/TextFileInputDialog.java index 12f09d0d4a7c..4fba4be2ab87 100644 --- a/ui/src/main/java/org/pentaho/di/ui/trans/steps/fileinput/text/TextFileInputDialog.java +++ b/ui/src/main/java/org/pentaho/di/ui/trans/steps/fileinput/text/TextFileInputDialog.java @@ -3,7 +3,7 @@ * * Pentaho Data Integration * - * Copyright (C) 2002-2021 by Hitachi Vantara : http://www.pentaho.com + * Copyright (C) 2002-2024 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * @@ -698,8 +698,9 @@ private void showFiles() { TextFileInputMeta tfii = new TextFileInputMeta(); getInfo( tfii, true ); String[] files = - FileInputList.createFilePathList( transMeta, tfii.inputFiles.fileName, tfii.inputFiles.fileMask, - tfii.inputFiles.excludeFileMask, tfii.inputFiles.fileRequired, tfii.inputFiles.includeSubFolderBoolean() ); + FileInputList.createFilePathList( transMeta.getBowl(), transMeta, tfii.inputFiles.fileName, + tfii.inputFiles.fileMask, tfii.inputFiles.excludeFileMask, tfii.inputFiles.fileRequired, + tfii.inputFiles.includeSubFolderBoolean() ); if ( files != null && files.length > 0 ) { EnterSelectionDialog esd = new EnterSelectionDialog( shell, files, "Files read", "Files read:" ); @@ -2698,7 +2699,7 @@ private void first( boolean skipHeaders ) { getInfo( info, true ); try { - if ( info.getFileInputList( transMeta ).nrOfFiles() > 0 ) { + if ( info.getFileInputList( transMeta.getBowl(), transMeta ).nrOfFiles() > 0 ) { String shellText = BaseMessages.getString( PKG, "TextFileInputDialog.LinesToView.DialogTitle" ); String lineText = BaseMessages.getString( PKG, "TextFileInputDialog.LinesToView.DialogMessage" ); EnterNumberDialog end = new EnterNumberDialog( shell, 100, shellText, lineText ); @@ -2741,7 +2742,7 @@ private void first( boolean skipHeaders ) { private List getFirst( int nrlines, boolean skipHeaders ) throws KettleException { TextFileInputMeta meta = new TextFileInputMeta(); getInfo( meta, true ); - FileInputList textFileList = meta.getFileInputList( transMeta ); + FileInputList textFileList = meta.getFileInputList( transMeta.getBowl(), transMeta ); InputStream fi; CompressionInputStream f = null; diff --git a/ui/src/main/java/org/pentaho/di/ui/trans/steps/textfileinput/TextFileInputDialog.java b/ui/src/main/java/org/pentaho/di/ui/trans/steps/textfileinput/TextFileInputDialog.java index 7c2a6d297e4b..873f31325b98 100644 --- a/ui/src/main/java/org/pentaho/di/ui/trans/steps/textfileinput/TextFileInputDialog.java +++ b/ui/src/main/java/org/pentaho/di/ui/trans/steps/textfileinput/TextFileInputDialog.java @@ -2593,7 +2593,7 @@ private void getCSV() { TextFileInputMeta meta = new TextFileInputMeta(); getInfo( meta ); TextFileInputMeta previousMeta = (TextFileInputMeta) meta.clone(); - FileInputList textFileList = meta.getTextFileList( transMeta ); + FileInputList textFileList = meta.getTextFileList( transMeta.getBowl(), transMeta ); InputStream fileInputStream; CompressionInputStream inputStream = null; StringBuilder lineStringBuilder = new StringBuilder( 256 ); @@ -2826,7 +2826,7 @@ private void first( boolean skipHeaders ) { getInfo( info ); try { - if ( info.getTextFileList( transMeta ).nrOfFiles() > 0 ) { + if ( info.getTextFileList( transMeta.getBowl(), transMeta ).nrOfFiles() > 0 ) { String shellText = BaseMessages.getString( PKG, "TextFileInputDialog.LinesToView.DialogTitle" ); String lineText = BaseMessages.getString( PKG, "TextFileInputDialog.LinesToView.DialogMessage" ); EnterNumberDialog end = new EnterNumberDialog( shell, 100, shellText, lineText ); @@ -2869,7 +2869,7 @@ private void first( boolean skipHeaders ) { private List getFirst( int nrlines, boolean skipHeaders ) throws KettleException { TextFileInputMeta meta = new TextFileInputMeta(); getInfo( meta ); - FileInputList textFileList = meta.getTextFileList( transMeta ); + FileInputList textFileList = meta.getTextFileList( transMeta.getBowl(), transMeta ); InputStream fi; CompressionInputStream f = null;