diff --git a/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGenerator.java b/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGenerator.java index d000fc8d0..b037748cd 100644 --- a/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGenerator.java +++ b/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGenerator.java @@ -12,19 +12,28 @@ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See the GNU Lesser General Public License for more details. * -* Copyright (c) 2002-2013 Pentaho Corporation.. All rights reserved. +* Copyright (c) 2002-2016 Pentaho Corporation.. All rights reserved. */ package org.pentaho.platform.dataaccess.datasource.wizard.service.agile; import java.io.File; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.pentaho.di.core.database.Database; import org.pentaho.di.core.database.DatabaseMeta; +import org.pentaho.di.core.exception.KettleDatabaseException; +import org.pentaho.di.core.exception.KettleTransException; +import org.pentaho.di.core.row.RowMetaInterface; import org.pentaho.di.core.row.ValueMeta; +import org.pentaho.di.core.row.ValueMetaInterface; import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.step.StepErrorMeta; import org.pentaho.di.trans.step.StepMeta; @@ -33,8 +42,8 @@ import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; import org.pentaho.platform.dataaccess.datasource.wizard.models.ColumnInfo; import org.pentaho.platform.dataaccess.datasource.wizard.models.CsvFileInfo; -import org.pentaho.platform.dataaccess.datasource.wizard.sources.csv.FileTransformStats; import org.pentaho.platform.dataaccess.datasource.wizard.models.ModelInfo; +import org.pentaho.platform.dataaccess.datasource.wizard.sources.csv.FileTransformStats; import org.pentaho.platform.engine.core.system.PentahoSystem; public class CsvTransformGenerator extends StagingTransformGenerator { @@ -45,6 +54,8 @@ public class CsvTransformGenerator extends StagingTransformGenerator { private static final String SELECT_VALUES = "select"; //$NON-NLS-1$ + private static final String CUT_LONG_NAMES = "cutLongNames"; //$NON-NLS-1$ + public static final String DEFAULT_RELATIVE_UPLOAD_FILE_PATH = File.separatorChar + "system" + File.separatorChar + "metadata" + File.separatorChar + "csvfiles" + File.separatorChar; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ @@ -106,6 +117,14 @@ protected StepMeta[] getSteps( TransMeta transMeta ) { createHop(steps.get(steps.size()-2), step, transMeta); } */ + + final int targetDatabaseMaxColumnNameLength = getMaxColumnNameLength(); + StepMeta cutLongNamesStep = createCutLongNamesStep( transMeta, steps, targetDatabaseMaxColumnNameLength, CUT_LONG_NAMES ); + if ( cutLongNamesStep != null ) { + steps.add( cutLongNamesStep ); + createHop( steps.get( steps.size() - 2 ), cutLongNamesStep, transMeta ); + } + return steps.toArray( new StepMeta[ steps.size() ] ); } @@ -229,4 +248,148 @@ public Log getLogger() { return log; } + /** + * The target database maxColumnNameLength value if available; + * 0 otherwise. + * @return + */ + protected int getMaxColumnNameLength() { + int maxLen = 0; + Database db = null; + try { + db = this.getDatabase( getTargetDatabaseMeta() ); + if ( db == null ) { + log.debug( "Cannot getMaxColumnNameLength (defaults to 0): database is not available." ); //$NON-NLS-1$ + return maxLen; + } + db.connect( null ); + final DatabaseMetaData databaseMetaData = db.getDatabaseMetaData(); + if ( databaseMetaData == null ) { + log.debug( "Cannot getMaxColumnNameLength (defaults to 0): database metadata are not available." ); //$NON-NLS-1$ + return maxLen; + } + maxLen = databaseMetaData.getMaxColumnNameLength(); + } catch ( KettleDatabaseException e ) { + log.debug( "Cannot getMaxColumnNameLength (defaults to 0): " + e.getMessage(), e ); //$NON-NLS-1$ + } catch ( SQLException e ) { + log.debug( "Cannot getMaxColumnNameLength (defaults to 0): " + e.getMessage(), e ); //$NON-NLS-1$ + } finally { + if ( db != null ) { + db.disconnect(); + } + } + return maxLen; + } + + /** + * This step scans output fields of the last step in steps, + * + * cut field names that longer than maxColumnNameLength, + * + * renames them if necessary to keep them unique. + *
+ * If maxColumnNameLength<=0 or all field names are short enough, the step is not created; + * @param transMeta + * @param steps + * @param maxColumnNameLength + * @param stepName + * @return created {@link StepMeta} or null + * @throws KettleTransException + */ + protected StepMeta createCutLongNamesStep( TransMeta transMeta, List steps, int maxColumnNameLength, String stepName ) { + if ( maxColumnNameLength <= 0 ) { + return null; + } + try { + StepMeta prevStepMeta = steps.get( steps.size() - 1 ); + RowMetaInterface fields = transMeta.getStepFields( prevStepMeta ); + StepMeta stepMeta = createCutLongNamesStep( fields, maxColumnNameLength, stepName ); + if ( stepMeta != null ) { + transMeta.addStep( stepMeta ); + } + return stepMeta; + } catch ( KettleTransException e ) { + log.warn( "Unable to createCutLongNamesStep. Skipping it.", e ); + } catch ( RuntimeException e ) { + log.warn( "Unable to createCutLongNamesStep. Skipping it.", e ); + } + return null; + } + + /** + * + * @param fields + * @param maxColumnNameLength + * @param stepName + * @return + * @throws KettleTransException + */ + protected StepMeta createCutLongNamesStep( RowMetaInterface fields, int maxColumnNameLength, String stepName ) throws KettleTransException { + final int fieldsCount = fields.size(); + + SelectValuesMeta meta = new SelectValuesMeta(); + List selectNameList = new ArrayList( fieldsCount ); + List selectRenameList = new ArrayList( fieldsCount ); + List selectLengthList = new ArrayList( fieldsCount ); + List selectPrecisionList = new ArrayList( fieldsCount ); + final Collection controlNames = new HashSet(); + boolean renameRequired = false; + for ( ValueMetaInterface valueMeta : fields.getValueMetaList() ) { + final String oldName = valueMeta.getName(); + selectNameList.add( oldName ); + String newName = oldName; + if ( newName.length() > maxColumnNameLength ) { + renameRequired = true; + newName = newName.substring( 0, maxColumnNameLength ); + } + if ( controlNames.contains( newName.toLowerCase() ) ) { + renameRequired = true; + newName = null; + String candidateName = null; + final int maxAppendableSuffixLength = maxColumnNameLength - oldName.length(); + for ( int j = 1; newName == null && j < Integer.MAX_VALUE; j++ ) { + String suffix = "_" + j; + if ( suffix.length() > maxColumnNameLength ) { + throw new KettleTransException( "Cannot cut field name. Maximum suffix length is exceeded" ); //$NON-NLS-1$ + } + if ( suffix.length() <= maxAppendableSuffixLength ) { + candidateName = oldName + suffix; + } else { + candidateName = oldName.substring( 0, maxColumnNameLength - suffix.length() ) + suffix; + } + if ( !controlNames.contains( candidateName.toLowerCase() ) ) { + newName = candidateName; + } + } + if ( newName == null ) { // This is fantastic but... let it be + throw new KettleTransException( "Cannot cut field name. Maximum trials number is reached." ); //$NON-NLS-1$ + } + } + controlNames.add( newName.toLowerCase() ); + selectRenameList.add( newName ); + selectLengthList.add( valueMeta.getLength() ); + selectPrecisionList.add( valueMeta.getPrecision() ); + } + if ( !renameRequired ) { + return null; + } + String[] selectName = selectNameList.toArray( new String[ selectNameList.size() ] ); + meta.setSelectName( selectName ); + String[] selectRename = selectRenameList.toArray( new String[ selectRenameList.size() ] ); + meta.setSelectRename( selectRename ); + + int[] selectLength = new int[ selectLengthList.size() ]; + int[] selectPrecision = new int[ selectPrecisionList.size() ]; + for ( int i = 0; i < selectLength.length; i++ ) { + selectLength[ i ] = selectLengthList.get( i ); + } + for ( int i = 0; i < selectPrecision.length; i++ ) { + selectPrecision[ i ] = selectPrecisionList.get( i ); + } + meta.setSelectLength( selectLength ); + meta.setSelectPrecision( selectPrecision ); + + StepMeta stepMeta = new StepMeta( stepName, stepName, meta ); + return stepMeta; + } } diff --git a/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/StagingTransformGenerator.java b/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/StagingTransformGenerator.java index 940b0e42f..44b37aca6 100644 --- a/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/StagingTransformGenerator.java +++ b/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/StagingTransformGenerator.java @@ -690,4 +690,8 @@ private boolean checkTableExists( String tableName ) throws CsvTransformGenerato db.disconnect(); } } + + DatabaseMeta getTargetDatabaseMeta() { + return targetDatabaseMeta; + } } diff --git a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java index 167b30c9f..940617bfa 100644 --- a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java +++ b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java @@ -17,10 +17,35 @@ package org.pentaho.platform.dataaccess.datasource.wizard.service.agile; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +import java.io.File; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Date; +import java.util.List; + import org.apache.commons.lang.ArrayUtils; +import org.junit.Assert; import org.mockito.Mockito; import org.pentaho.di.core.database.Database; import org.pentaho.di.core.database.DatabaseMeta; +import org.pentaho.di.core.exception.KettleDatabaseException; +import org.pentaho.di.core.exception.KettleStepException; +import org.pentaho.di.core.row.RowMeta; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMetaInterface; +import org.pentaho.di.core.row.value.ValueMetaBigNumber; +import org.pentaho.di.trans.TransMeta; +import org.pentaho.di.trans.step.StepMeta; +import org.pentaho.di.trans.step.StepMetaInterface; +import org.pentaho.di.trans.steps.selectvalues.SelectValuesMeta; import org.pentaho.metadata.model.concept.types.AggregationType; import org.pentaho.metadata.model.concept.types.DataType; import org.pentaho.platform.api.engine.IApplicationContext; @@ -37,12 +62,6 @@ import org.pentaho.platform.plugin.action.kettle.KettleSystemListener; import org.pentaho.test.platform.engine.core.BaseTest; -import java.io.File; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.Date; - @SuppressWarnings( { "all" } ) public class CsvTransformGeneratorIT extends BaseTest { @@ -56,6 +75,21 @@ public class CsvTransformGeneratorIT extends BaseTest { private static final String SYSTEM_FOLDER = "/system"; //$NON-NLS-1$ + static class CutLongNamesStepInputContext { + String[] fieldNames = new String[] {"a", "b", "A_1", "b_1", "LONGlonglong", "longlonglong_again", "a_2", }; + int[] fieldLengths = new int[] { 5, 6, 10, 15, 12, 7, 4 }; + int[] fieldPrecisions = new int[] { 0, 0, 1, 1, 2, 0, 2 }; + RowMetaInterface fields = new RowMeta(); + { + for ( int i = 0; i < fieldNames.length; i++ ) { + final ValueMetaInterface valueMeta = new ValueMetaBigNumber( fieldNames[i] ); + valueMeta.setLength( fieldLengths[i] ); + valueMeta.setPrecision( fieldPrecisions[i] ); + fields.addValueMeta( valueMeta ); + } + } + } + public String getSolutionPath() { File file = new File( SOLUTION_PATH + PENTAHO_XML_PATH ); if ( file.exists() ) { @@ -196,6 +230,54 @@ public void testCreateTable() throws Exception { assertEquals( (long) 0, rowCount ); } + public void testCreateTable_longColumnNames() throws Exception { + + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + + DatabaseMeta dbMeta = getDatabaseMeta(); + ModelInfo info = createModel(); + CsvTransformGenerator gen = spy( new CsvTransformGenerator( info, dbMeta ) ); + doReturn( 8 ).when( gen ).getMaxColumnNameLength(); + + String tableName = info.getStageTableName(); + + try { + gen.execSqlStatement( getDropTableStatement( tableName ), dbMeta, null ); + } catch ( CsvTransformGeneratorException e ) { + // it is OK if the table doesn't exist previously + } + gen.createOrModifyTable( session ); + + // check the results + long rowCount = this.getRowCount( tableName ); + assertEquals( (long) 0, rowCount ); + } + + public void testCreateTable_littleMaxColumnNameLength() throws Exception { + + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + + DatabaseMeta dbMeta = getDatabaseMeta(); + ModelInfo info = createModel(); + CsvTransformGenerator gen = spy( new CsvTransformGenerator( info, dbMeta ) ); + doReturn( 1 ).when( gen ).getMaxColumnNameLength(); + + String tableName = info.getStageTableName(); + + try { + gen.execSqlStatement( getDropTableStatement( tableName ), dbMeta, null ); + } catch ( CsvTransformGeneratorException e ) { + // it is OK if the table doesn't exist previously + } + gen.createOrModifyTable( session ); + + // check the results + long rowCount = this.getRowCount( tableName ); + assertEquals( (long) 0, rowCount ); + } + /** * Given a name of an existing table to drop. *
@@ -416,7 +498,7 @@ public void testModifyEmptyTable_AddColumn() throws Exception { // generate the database table initially gen.createOrModifyTable( session ); - // now, lets update it by changing the model info slightly.. add a column + // now, lets update it by changing the model info slightly.. add a column addColumnToModel( info ); gen.createOrModifyTable( session ); @@ -440,7 +522,7 @@ public void testModifyEmptyTable_RemoveColumn() throws Exception { String removedColumn = info.getColumns()[ info.getColumns().length - 1 ].getId(); - // now, lets update it by changing the model info slightly.. add a column + // now, lets update it by changing the model info slightly.. add a column removeColumnFromModel( info ); gen.createOrModifyTable( session ); @@ -542,7 +624,7 @@ private int loadTable( CsvTransformGenerator gen, ModelInfo info, boolean trunca FileTransformStats stats = gen.getTransformStats(); - // wait until it it done + // wait until it it done while ( !stats.isRowsFinished() ) { Thread.sleep( 100 ); } @@ -727,4 +809,174 @@ public static ModelInfo createModel() { return info; } + + private static DatabaseMeta getJndiDatabaseMeta( String jndi ) { + DatabaseMeta databaseMeta = new DatabaseMeta(); + databaseMeta.setAccessType( DatabaseMeta.TYPE_ACCESS_JNDI ); + databaseMeta.setDBName( jndi ); + databaseMeta.setName( jndi ); + return databaseMeta; + } + + public void testGetMaxColumnNameLength() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + ModelInfo info = createModel(); + + final String H2_JNDI = "pentaho_staging_H2"; // test-res/solution1/system/simple-jndi/jdbc.properties + final int H2_MAX_COLUMN_NAME_LENGTH = 0; // org.h2.jdbc.JdbcDatabaseMetaData: h2-1.0.78.jar + + final String HYPERSONIC_JNDI = "pentaho_staging_Hypersonic"; // test-res/solution1/system/simple-jndi/jdbc.properties + final int HYPERSONIC_MAX_COLUMN_NAME_LENGTH = 128; // org.hsqldb.jdbc.JDBCDatabaseMetaData: hsqldb-2.3.2.jar + + final String INVALID_JNDI = "some_invalid_jndi"; + final int DEFAULT_MAX_COLUMN_NAME_LENGTH = 0; + + CsvTransformGenerator genH2 = new CsvTransformGenerator( info, getJndiDatabaseMeta( H2_JNDI ) ); + assertEquals( H2_MAX_COLUMN_NAME_LENGTH, genH2.getMaxColumnNameLength() ); + + CsvTransformGenerator genHsqldb = new CsvTransformGenerator( info, getJndiDatabaseMeta( HYPERSONIC_JNDI ) ); + assertEquals( HYPERSONIC_MAX_COLUMN_NAME_LENGTH, genHsqldb.getMaxColumnNameLength() ); + + CsvTransformGenerator genInvalid = new CsvTransformGenerator( info, getJndiDatabaseMeta( INVALID_JNDI ) ); + assertEquals( DEFAULT_MAX_COLUMN_NAME_LENGTH, genInvalid.getMaxColumnNameLength() ); + } + + public void testGetMaxColumnNameLength_specialCases() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + ModelInfo info = createModel(); + DatabaseMeta dbMeta = getDatabaseMeta(); + + final int DEFAULT_MAX_COLUMN_NAME_LENGTH = 0; + + CsvTransformGenerator genNoDB = spy( new CsvTransformGenerator( info, dbMeta ) ); + doReturn( null ).when( genNoDB ).getDatabase( dbMeta ); + assertEquals( DEFAULT_MAX_COLUMN_NAME_LENGTH, genNoDB.getMaxColumnNameLength() ); + + CsvTransformGenerator genNoMetadata = spy( new CsvTransformGenerator( info, dbMeta ) ); + Database dbNoMetadata = mock( Database.class ); + doReturn( null ).when( dbNoMetadata ).getDatabaseMetaData(); + doReturn( dbNoMetadata ).when( genNoDB ).getDatabase( dbMeta ); + assertEquals( DEFAULT_MAX_COLUMN_NAME_LENGTH, genNoMetadata.getMaxColumnNameLength() ); + + CsvTransformGenerator genInvalidDb = spy( new CsvTransformGenerator( info, dbMeta ) ); + Database dbInvalid = mock( Database.class ); + doThrow( new KettleDatabaseException() ).when( dbInvalid ).connect(); + doReturn( dbInvalid ).when( genInvalidDb ).getDatabase( dbMeta ); + assertEquals( DEFAULT_MAX_COLUMN_NAME_LENGTH, genInvalidDb.getMaxColumnNameLength() ); + + CsvTransformGenerator genInvalidDbMetadata = spy( new CsvTransformGenerator( info, dbMeta ) ); + final DatabaseMetaData dbMetaDataInvalid = mock( DatabaseMetaData.class ); + Database dbInvalidMetadata = mock( Database.class ); + doThrow( new SQLException() ).when( dbMetaDataInvalid ).getMaxColumnNameLength(); + doReturn( dbMetaDataInvalid ).when( dbInvalidMetadata ).getDatabaseMetaData(); + doReturn( dbInvalidMetadata ).when( genInvalidDbMetadata ).getDatabase( dbMeta ); + assertEquals( DEFAULT_MAX_COLUMN_NAME_LENGTH, genInvalidDbMetadata.getMaxColumnNameLength() ); + + } + + public void testCreateCutLongNamesStep_longColumnNames() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + ModelInfo info = createModel(); + + final int DATABASE_MAX_COLUMN_NAME_LENGTH = 8; + final String STEP_NAME = "TEST_STEP_CutLongNames"; + CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); + String[] fieldRenames = new String[] {"a", "b", "A_1", "b_1", "LONGlong", "longlo_1", "a_2"}; + StepMeta prevStepMeta = mock( StepMeta.class ); // No functionality is required + List steps = java.util.Collections.singletonList( prevStepMeta ); + TransMeta transMeta = spy( new TransMeta() ); + doReturn( prev.fields ).when( transMeta ).getStepFields( prevStepMeta ); + + final DatabaseMeta databaseMeta = getDatabaseMeta(); + CsvTransformGenerator gen = new CsvTransformGenerator( info, databaseMeta ); + + StepMeta step = gen.createCutLongNamesStep( transMeta, steps, DATABASE_MAX_COLUMN_NAME_LENGTH, STEP_NAME ); + + Assert.assertNotNull( "step", step ); + Assert.assertEquals( "step name", STEP_NAME, step.getName() ); + + StepMetaInterface stepMetaIntegrface = step.getStepMetaInterface(); + Assert.assertNotNull( "stepMetaIntegrface", stepMetaIntegrface ); + Assert.assertTrue( "stepMetaIntegrface instanceof SelectValuesMeta", stepMetaIntegrface instanceof SelectValuesMeta ); + SelectValuesMeta svm = (SelectValuesMeta) stepMetaIntegrface; + + String[] selectName = svm.getSelectName(); + Assert.assertArrayEquals( "selectName", prev.fieldNames, selectName ); + String[] selectRename = svm.getSelectRename(); + Assert.assertArrayEquals( "selectRename", fieldRenames, selectRename ); + int[] selectLengths = svm.getSelectLength(); + Assert.assertArrayEquals( "selectLength", prev.fieldLengths, selectLengths ); + int[] selectPrecisions = svm.getSelectPrecision(); + Assert.assertArrayEquals( "selectPrecision", selectPrecisions, selectPrecisions ); + + Assert.assertEquals( step, transMeta.findStep( STEP_NAME ) ); + } + + public void testCreateCutLongNamesStep_shortColumnNames() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + ModelInfo info = createModel(); + + final int DATABASE_MAX_COLUMN_NAME_LENGTH = 18; + final String STEP_NAME = "TEST_STEP_CutLongNames"; + CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); + + StepMeta prevStepMeta = mock( StepMeta.class ); // No functionality is required + List steps = java.util.Collections.singletonList( prevStepMeta ); + TransMeta transMeta = spy( new TransMeta() ); + doReturn( prev.fields ).when( transMeta ).getStepFields( prevStepMeta ); + + final DatabaseMeta databaseMeta = getDatabaseMeta(); + CsvTransformGenerator gen = new CsvTransformGenerator( info, databaseMeta ); + + StepMeta step = gen.createCutLongNamesStep( transMeta, steps, DATABASE_MAX_COLUMN_NAME_LENGTH, STEP_NAME ); + + Assert.assertNull( "step", step ); + } + + public void testCreateCutLongNamesStep_littleMaxColumnNameLength() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + ModelInfo info = createModel(); + + final int DATABASE_MAX_COLUMN_NAME_LENGTH = 1; + final String STEP_NAME = "TEST_STEP_CutLongNames"; + CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); + + StepMeta prevStepMeta = mock( StepMeta.class ); // No functionality is required + List steps = java.util.Collections.singletonList( prevStepMeta ); + TransMeta transMeta = spy( new TransMeta() ); + doReturn( prev.fields ).when( transMeta ).getStepFields( prevStepMeta ); + + final DatabaseMeta databaseMeta = getDatabaseMeta(); + CsvTransformGenerator gen = new CsvTransformGenerator( info, databaseMeta ); + + StepMeta step = gen.createCutLongNamesStep( transMeta, steps, DATABASE_MAX_COLUMN_NAME_LENGTH, STEP_NAME ); + assertNull( step ); + } + + public void testCreateCutLongNamesStep_prevStepError() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + ModelInfo info = createModel(); + + final int DATABASE_MAX_COLUMN_NAME_LENGTH = 18; + final String STEP_NAME = "TEST_STEP_CutLongNames"; + + StepMeta prevStepMeta = mock( StepMeta.class ); // No functionality is required + List steps = java.util.Collections.singletonList( prevStepMeta ); + TransMeta transMeta = spy( new TransMeta() ); + doThrow( new KettleStepException() ).when( transMeta ).getStepFields( prevStepMeta ); + + final DatabaseMeta databaseMeta = getDatabaseMeta(); + CsvTransformGenerator gen = new CsvTransformGenerator( info, databaseMeta ); + + StepMeta step = gen.createCutLongNamesStep( transMeta, steps, DATABASE_MAX_COLUMN_NAME_LENGTH, STEP_NAME ); + + Assert.assertNull( "step", step ); + } + } diff --git a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java new file mode 100644 index 000000000..354dd6926 --- /dev/null +++ b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java @@ -0,0 +1,78 @@ +/*! +* This program is free software; you can redistribute it and/or modify it under the +* terms of the GNU Lesser General Public License, version 2.1 as published by the Free Software +* Foundation. +* +* You should have received a copy of the GNU Lesser General Public License along with this +* program; if not, you can obtain a copy at http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html +* or from the Free Software Foundation, Inc., +* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +* +* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +* See the GNU Lesser General Public License for more details. +* +* Copyright (c) 2002-2016 Pentaho Corporation.. All rights reserved. +*/ +package org.pentaho.platform.dataaccess.datasource.wizard.service.agile; + +import static org.junit.Assert.fail; + +import org.junit.Assert; +import org.junit.Test; +import org.pentaho.di.trans.step.StepMeta; +import org.pentaho.di.trans.step.StepMetaInterface; +import org.pentaho.di.trans.steps.selectvalues.SelectValuesMeta; +import org.pentaho.platform.dataaccess.datasource.wizard.models.ModelInfo; +import org.pentaho.platform.dataaccess.datasource.wizard.service.agile.CsvTransformGeneratorIT.CutLongNamesStepInputContext; + +public class CsvTransformGeneratorTest { + + @Test + public void testCreateCutLongNamesStep_shortColumnNames() throws Exception { + CsvTransformGenerator ctg = new CsvTransformGenerator( new ModelInfo(), null ); + int maxColumnNameLength = 18; + String stepName = "TEST_STEP_CutLongNames"; + CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); + StepMeta step = ctg.createCutLongNamesStep( prev.fields, maxColumnNameLength, stepName ); + Assert.assertNull( "step", step ); + } + + @Test + public void testCreateCutLongNamesStep_longColumnNames() throws Exception { + CsvTransformGenerator ctg = new CsvTransformGenerator( new ModelInfo(), null ); + int maxColumnNameLength = 8; + String stepName = "TEST_STEP_CutLongNames"; + CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); + String[] fieldRenames = new String[] {"a", "b", "A_1", "b_1", "LONGlong", "longlo_1", "a_2"}; + + StepMeta step = ctg.createCutLongNamesStep( prev.fields, maxColumnNameLength, stepName ); + Assert.assertNotNull( "step", step ); + Assert.assertEquals( "step name", stepName, step.getName() ); + StepMetaInterface stepMetaIntegrface = step.getStepMetaInterface(); + Assert.assertNotNull( "stepMetaIntegrface", stepMetaIntegrface ); + Assert.assertTrue( "stepMetaIntegrface instanceof SelectValuesMeta", stepMetaIntegrface instanceof SelectValuesMeta ); + SelectValuesMeta svm = (SelectValuesMeta) stepMetaIntegrface; + String[] selectName = svm.getSelectName(); + Assert.assertArrayEquals( "selectName", prev.fieldNames, selectName ); + String[] selectRename = svm.getSelectRename(); + Assert.assertArrayEquals( "selectName", fieldRenames, selectRename ); + } + + @Test + public void testCreateCutLongNamesStep_littleMaxColumnNameLength() throws Exception { + CsvTransformGenerator ctg = new CsvTransformGenerator( new ModelInfo(), null ); + int maxColumnNameLength = 1; + String stepName = "TEST_STEP_CutLongNames"; + CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); + + try { + StepMeta step = ctg.createCutLongNamesStep( prev.fields, maxColumnNameLength, stepName ); + fail( "Ex[pected exception: Cannot cut field name. Maximum suffix length is exceeded" ); + } catch ( Exception e ) { + // expected + } + } + + +}