From 90303fa7ebc351282c5707fd7b1a7f68c00f1c79 Mon Sep 17 00:00:00 2001 From: Jan Bob Date: Wed, 25 Sep 2019 14:37:09 +0200 Subject: [PATCH 01/22] updated poi-ooxml added eager datatype checker for first 1000 rows --- excel/pom.xml | 2 +- .../DefaultSpreadsheetReaderDelegate.java | 406 ++++++++++-------- .../metamodel/excel/ExcelDataContextTest.java | 25 +- 3 files changed, 248 insertions(+), 185 deletions(-) diff --git a/excel/pom.xml b/excel/pom.xml index 7f8e8f09f..a87f50f4c 100644 --- a/excel/pom.xml +++ b/excel/pom.xml @@ -40,7 +40,7 @@ under the License. org.apache.poi poi-ooxml - 4.0.1 + 4.1.0 commons-logging diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index 97ba50f6a..7040f3129 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -41,6 +41,7 @@ import org.apache.metamodel.util.FileHelper; import org.apache.metamodel.util.Resource; import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellType; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; @@ -53,175 +54,238 @@ */ final class DefaultSpreadsheetReaderDelegate implements SpreadsheetReaderDelegate { - private static final Logger logger = LoggerFactory.getLogger(DefaultSpreadsheetReaderDelegate.class); - - private final Resource _resource; - private final ExcelConfiguration _configuration; - - public DefaultSpreadsheetReaderDelegate(Resource resource, ExcelConfiguration configuration) { - _resource = resource; - _configuration = configuration; - } - - @Override - public Schema createSchema(String schemaName) { - final MutableSchema schema = new MutableSchema(schemaName); - final Workbook wb = ExcelUtils.readWorkbook(_resource, true); - try { - for (int i = 0; i < wb.getNumberOfSheets(); i++) { - final Sheet currentSheet = wb.getSheetAt(i); - final MutableTable table = createTable(wb, currentSheet); - table.setSchema(schema); - schema.addTable(table); - } - - return schema; - } finally { - FileHelper.safeClose(wb); - } - } - - @Override - public DataSet executeQuery(Table table, List columns, int maxRows) { - final Workbook wb = ExcelUtils.readWorkbook(_resource, true); - final Sheet sheet = wb.getSheet(table.getName()); - - if (sheet == null || sheet.getPhysicalNumberOfRows() == 0) { - return new EmptyDataSet(columns.stream().map(SelectItem::new).collect(Collectors.toList())); - } - - DataSet dataSet = ExcelUtils.getDataSet(wb, sheet, table, _configuration); - - if (maxRows > 0) { - dataSet = new MaxRowsDataSet(dataSet, maxRows); - } - return dataSet; - } - - @Override - public void notifyTablesModified() { - // do nothing - } - - private MutableTable createTable(final Workbook wb, final Sheet sheet) { - final MutableTable table = new MutableTable(sheet.getSheetName(), TableType.TABLE); - - if (sheet.getPhysicalNumberOfRows() <= 0) { - // no physical rows in sheet - return table; - } - - final Iterator rowIterator = ExcelUtils.getRowIterator(sheet, _configuration, false); - - if (!rowIterator.hasNext()) { - // no physical rows in sheet - return table; - } - - Row row = null; - - if (_configuration.isSkipEmptyLines()) { - while (row == null && rowIterator.hasNext()) { - row = rowIterator.next(); - } - } else { - row = rowIterator.next(); - } - - final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); - if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) { - - // get to the first non-empty line (no matter if lines are skipped - // or not we need to read ahead to figure out how many columns there - // are!) - while (row == null && rowIterator.hasNext()) { - row = rowIterator.next(); - } - - // build columns without any intrinsic column names - final ColumnNamingStrategy columnNamingStrategy = _configuration.getColumnNamingStrategy(); - try (final ColumnNamingSession columnNamingSession = columnNamingStrategy.startColumnNamingSession()) { - final int offset = getColumnOffset(row); - for (int i = 0; i < offset; i++) { - columnNamingSession.getNextColumnName(new ColumnNamingContextImpl(i)); - } - - for (int j = offset; j < row.getLastCellNum(); j++) { - final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); - final Column column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), - ColumnType.STRING, table, j, true); - table.addColumn(column); - } - } - - } else { - - boolean hasColumns = true; - - // iterate to the column name line number (if above 1) - for (int j = 1; j < columnNameLineNumber; j++) { - if (rowIterator.hasNext()) { - row = rowIterator.next(); - } else { - hasColumns = false; - break; - } - } - - if (hasColumns) { - createColumns(table, wb, row); - } - } - - return table; - } - - /** - * Builds columns based on row/cell values. - * - * @param table - * @param wb - * @param row - */ - private void createColumns(MutableTable table, Workbook wb, Row row) { - if (row == null) { - logger.warn("Cannot create columns based on null row!"); - return; - } - final short rowLength = row.getLastCellNum(); - - final int offset = getColumnOffset(row); - - // build columns based on cell values. - try (final ColumnNamingSession columnNamingSession = _configuration.getColumnNamingStrategy() - .startColumnNamingSession()) { - for (int j = offset; j < rowLength; j++) { - final Cell cell = row.getCell(j); - final String intrinsicColumnName = ExcelUtils.getCellValue(wb, cell); - final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName, - j); - final String columnName = columnNamingSession.getNextColumnName(columnNamingContext); - final Column column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true); - table.addColumn(column); - } - } - } - - /** - * Gets the column offset (first column to include). This is dependent on - * the row used for column processing and whether the skip empty columns - * property is set. - * - * @param row - * @return - */ - private int getColumnOffset(Row row) { - final int offset; - if (_configuration.isSkipEmptyColumns()) { - offset = row.getFirstCellNum(); - } else { - offset = 0; - } - return offset; - } + private static final Logger logger = LoggerFactory.getLogger(DefaultSpreadsheetReaderDelegate.class); + + private final Resource _resource; + private final ExcelConfiguration _configuration; + + public DefaultSpreadsheetReaderDelegate(Resource resource, ExcelConfiguration configuration) { + _resource = resource; + _configuration = configuration; + } + + @Override + public Schema createSchema(String schemaName) { + final MutableSchema schema = new MutableSchema(schemaName); + final Workbook wb = ExcelUtils.readWorkbook(_resource, true); + try { + for (int i = 0; i < wb.getNumberOfSheets(); i++) { + final Sheet currentSheet = wb.getSheetAt(i); + final MutableTable table = createTable(wb, currentSheet); + table.setSchema(schema); + schema.addTable(table); + } + + return schema; + } finally { + FileHelper.safeClose(wb); + } + } + + @Override + public DataSet executeQuery(Table table, List columns, int maxRows) { + final Workbook wb = ExcelUtils.readWorkbook(_resource, true); + final Sheet sheet = wb.getSheet(table.getName()); + + if (sheet == null || sheet.getPhysicalNumberOfRows() == 0) { + return new EmptyDataSet(columns.stream().map(SelectItem::new).collect(Collectors.toList())); + } + + DataSet dataSet = ExcelUtils.getDataSet(wb, sheet, table, _configuration); + + if (maxRows > 0) { + dataSet = new MaxRowsDataSet(dataSet, maxRows); + } + return dataSet; + } + + @Override + public void notifyTablesModified() { + // do nothing + } + + private MutableTable createTable(Workbook wb, Sheet sheet) { + final MutableTable table = new MutableTable(sheet.getSheetName(), TableType.TABLE); + if (sheet.getPhysicalNumberOfRows() <= 0) { + // no physical rows in sheet + return table; + } + + final Iterator rowIterator = ExcelUtils.getRowIterator(sheet, _configuration, false); + + if (!rowIterator.hasNext()) { + // no physical rows in sheet + return table; + } + Row row = null; + if (_configuration.isSkipEmptyLines()) { + while (row == null && rowIterator.hasNext()) { + row = rowIterator.next(); + } + } else { + row = rowIterator.next(); + } + + // Get first 1000 rows for the eager-read + final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); + int rowLength = row.getLastCellNum(); + ColumnType[] columnTypes = new ColumnType[rowLength]; + + setColumnType(data, rowLength, columnTypes); + final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); + if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) { + + // get to the first non-empty line (no matter if lines are skipped + // or not we need to read ahead to figure out how many columns there + // are!) + while (row == null && rowIterator.hasNext()) { + row = rowIterator.next(); + } + // build columns without any intrinsic column names + final ColumnNamingStrategy columnNamingStrategy = _configuration.getColumnNamingStrategy(); + try (final ColumnNamingSession columnNamingSession = columnNamingStrategy.startColumnNamingSession()) { + final int offset = getColumnOffset(row); + for (int i = 0; i < offset; i++) { + columnNamingSession.getNextColumnName(new ColumnNamingContextImpl(i)); + } + for (int j = offset; j < row.getLastCellNum(); j++) { + final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); + final Column column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), + columnTypes[j], table, j, true); + table.addColumn(column); + } + } + } else { + boolean hasColumns = true; + // iterate to the column name line number (if above 1) + for (int j = 1; j < columnNameLineNumber; j++) { + if (rowIterator.hasNext()) { + row = rowIterator.next(); + } else { + hasColumns = false; + break; + } + } + if (hasColumns) { + createColumns(table, wb, row, columnTypes); + } + } + return table; + } + + private void setColumnType(Iterator data, int rowLength, ColumnType[] columnTypes) { + while (data.hasNext()) { + Row row = data.next(); + for (int index = 0; index < rowLength; index++) { + if (row.getLastCellNum() == 0) { + continue; + } + if (row.getCell(index) == null) { + columnTypes = checkColumnType(ColumnType.STRING, columnTypes, index); + } else { + CellType cellType = row.getCell(index).getCellType(); + if (cellType.getCode() != 0 && cellType.getCode() <= 2) { + columnTypes = checkColumnType(ColumnType.STRING, columnTypes, index); + } else if (cellType.getCode() == 0) { + columnTypes = checkColumnType( + (row.getCell(index).getNumericCellValue() % 1 == 0) ? ColumnType.INTEGER + : ColumnType.DOUBLE, + columnTypes, index); + } else if (cellType.getCode() == 4) { + columnTypes = checkColumnType(ColumnType.BOOLEAN, columnTypes, index); + } + } + } + } + } + + private ColumnType[] checkColumnType(ColumnType columnType, ColumnType[] columnTypes, int index) { + if (columnTypes[index] != null) { + if (!columnTypes[index].equals(ColumnType.STRING) && !columnTypes[index].equals(columnType)) { + columnTypes[index] = ColumnType.STRING; + } + } else { + columnTypes[index] = columnType; + } + return columnTypes; + } + + private void determineColumnDatatype(Object[] datatypes, Row row) { + for (int index = 0; index < row.getLastCellNum(); index++) { + CellType type = ((Cell) row.getCell(index)).getCellType(); + + if (datatypes[index] instanceof Object) { + datatypes[index] = type; + } else if (datatypes[index] instanceof CellType) { + if (datatypes[index].equals(type)) { + continue; + } else { + datatypes[index] = CellType.STRING; + } + } + } + } + + /** + * Builds columns based on row/cell values. + * + * @param table + * @param wb + * @param row + */ + private void createColumns(MutableTable table, Workbook wb, Row row, ColumnType[] columTypes) { + if (row == null) { + logger.warn("Cannot create columns based on null row!"); + return; + } + final short rowLength = row.getLastCellNum(); + final int offset = getColumnOffset(row); + // build columns based on cell values. + try (final ColumnNamingSession columnNamingSession = _configuration.getColumnNamingStrategy() + .startColumnNamingSession()) { + for (int j = offset; j < rowLength; j++) { + final Cell cell = row.getCell(j); + final String intrinsicColumnName = ExcelUtils.getCellValue(wb, cell); + final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName, + j); + final String columnName = columnNamingSession.getNextColumnName(columnNamingContext); + Column column = null; + if (columTypes == null) { + column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true); + } else { + column = new MutableColumn(columnName, columTypes[j], table, j, true); + } + table.addColumn(column); + } + } + } + + /** + * Builds columns based on row/cell values. + * + * @param table + * @param wb + * @param row + */ + private void createColumns(MutableTable table, Workbook wb, Row row) { + createColumns(table, wb, row, null); + } + + /** + * Gets the column offset (first column to include). This is dependent on the + * row used for column processing and whether the skip empty columns property is + * set. + * + * @param row + * @return + */ + private int getColumnOffset(Row row) { + final int offset; + if (_configuration.isSkipEmptyColumns()) { + offset = row.getFirstCellNum(); + } else { + offset = 0; + } + return offset; + } } diff --git a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java index 13cfdec80..19773ad08 100644 --- a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java +++ b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java @@ -402,10 +402,10 @@ public void testMissingValues() throws Exception { assertEquals(2, schema.getTableCount()); Table table = schema.getTables().get(0); - assertEquals("[Column[name=a,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=b,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=c,columnNumber=2,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=d,columnNumber=3,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]]", + assertEquals("[Column[name=a,columnNumber=0,type=STRING,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=b,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=c,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=d,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]", Arrays.toString(table.getColumns().toArray())); Query q = new Query().select(table.getColumns()).from(table); @@ -426,10 +426,10 @@ public void testMissingColumnHeader() throws Exception { assertEquals(2, schema.getTableCount()); Table table = schema.getTables().get(0); - assertEquals("[Column[name=a,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=b,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=A,columnNumber=2,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=d,columnNumber=3,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]]", + assertEquals("[Column[name=a,columnNumber=0,type=STRING,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=b,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=A,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=d,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]", Arrays.toString(table.getColumns().toArray())); Query q = new Query().select(table.getColumns()).from(table); @@ -532,11 +532,10 @@ public void testTicket99defect() throws Exception { Table table = schema.getTableByName("Sheet1"); assertEquals( - - "[Column[name=Pkg No.,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=Description,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=Room,columnNumber=2,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=Level,columnNumber=3,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]]", + "[Column[name=Pkg No.,columnNumber=0,type=STRING,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=Description,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=Room,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=Level,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]", Arrays.toString(table.getColumns().toArray())); } From 999647cd1ed775aac72d49b09751e22e19c990a2 Mon Sep 17 00:00:00 2001 From: Jan Bob Date: Thu, 26 Sep 2019 09:31:28 +0200 Subject: [PATCH 02/22] refactored [tabs] to [spaces] --- .../DefaultSpreadsheetReaderDelegate.java | 423 +++++++++--------- 1 file changed, 211 insertions(+), 212 deletions(-) diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index 7040f3129..8c5cc2f7c 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -54,238 +54,237 @@ */ final class DefaultSpreadsheetReaderDelegate implements SpreadsheetReaderDelegate { - private static final Logger logger = LoggerFactory.getLogger(DefaultSpreadsheetReaderDelegate.class); + private static final Logger logger = LoggerFactory.getLogger(DefaultSpreadsheetReaderDelegate.class); - private final Resource _resource; - private final ExcelConfiguration _configuration; + private final Resource _resource; + private final ExcelConfiguration _configuration; - public DefaultSpreadsheetReaderDelegate(Resource resource, ExcelConfiguration configuration) { - _resource = resource; - _configuration = configuration; - } + public DefaultSpreadsheetReaderDelegate(Resource resource, ExcelConfiguration configuration) { + _resource = resource; + _configuration = configuration; + } - @Override - public Schema createSchema(String schemaName) { - final MutableSchema schema = new MutableSchema(schemaName); - final Workbook wb = ExcelUtils.readWorkbook(_resource, true); - try { - for (int i = 0; i < wb.getNumberOfSheets(); i++) { - final Sheet currentSheet = wb.getSheetAt(i); - final MutableTable table = createTable(wb, currentSheet); - table.setSchema(schema); - schema.addTable(table); - } + @Override + public Schema createSchema(String schemaName) { + final MutableSchema schema = new MutableSchema(schemaName); + final Workbook wb = ExcelUtils.readWorkbook(_resource, true); + try { + for (int i = 0; i < wb.getNumberOfSheets(); i++) { + final Sheet currentSheet = wb.getSheetAt(i); + final MutableTable table = createTable(wb, currentSheet); + table.setSchema(schema); + schema.addTable(table); + } - return schema; - } finally { - FileHelper.safeClose(wb); - } - } + return schema; + } finally { + FileHelper.safeClose(wb); + } + } - @Override - public DataSet executeQuery(Table table, List columns, int maxRows) { - final Workbook wb = ExcelUtils.readWorkbook(_resource, true); - final Sheet sheet = wb.getSheet(table.getName()); + @Override + public DataSet executeQuery(Table table, List columns, int maxRows) { + final Workbook wb = ExcelUtils.readWorkbook(_resource, true); + final Sheet sheet = wb.getSheet(table.getName()); - if (sheet == null || sheet.getPhysicalNumberOfRows() == 0) { - return new EmptyDataSet(columns.stream().map(SelectItem::new).collect(Collectors.toList())); - } + if (sheet == null || sheet.getPhysicalNumberOfRows() == 0) { + return new EmptyDataSet(columns.stream().map(SelectItem::new).collect(Collectors.toList())); + } - DataSet dataSet = ExcelUtils.getDataSet(wb, sheet, table, _configuration); + DataSet dataSet = ExcelUtils.getDataSet(wb, sheet, table, _configuration); - if (maxRows > 0) { - dataSet = new MaxRowsDataSet(dataSet, maxRows); - } - return dataSet; - } + if (maxRows > 0) { + dataSet = new MaxRowsDataSet(dataSet, maxRows); + } + return dataSet; + } - @Override - public void notifyTablesModified() { - // do nothing - } + @Override + public void notifyTablesModified() { + // do nothing + } - private MutableTable createTable(Workbook wb, Sheet sheet) { - final MutableTable table = new MutableTable(sheet.getSheetName(), TableType.TABLE); - if (sheet.getPhysicalNumberOfRows() <= 0) { - // no physical rows in sheet - return table; - } + private MutableTable createTable(Workbook wb, Sheet sheet) { + final MutableTable table = new MutableTable(sheet.getSheetName(), TableType.TABLE); + if (sheet.getPhysicalNumberOfRows() <= 0) { + // no physical rows in sheet + return table; + } - final Iterator rowIterator = ExcelUtils.getRowIterator(sheet, _configuration, false); + final Iterator rowIterator = ExcelUtils.getRowIterator(sheet, _configuration, false); - if (!rowIterator.hasNext()) { - // no physical rows in sheet - return table; - } - Row row = null; - if (_configuration.isSkipEmptyLines()) { - while (row == null && rowIterator.hasNext()) { - row = rowIterator.next(); - } - } else { - row = rowIterator.next(); - } + if (!rowIterator.hasNext()) { + // no physical rows in sheet + return table; + } + Row row = null; + if (_configuration.isSkipEmptyLines()) { + while (row == null && rowIterator.hasNext()) { + row = rowIterator.next(); + } + } else { + row = rowIterator.next(); + } - // Get first 1000 rows for the eager-read - final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); - int rowLength = row.getLastCellNum(); - ColumnType[] columnTypes = new ColumnType[rowLength]; + // Get first 1000 rows for the eager-read + final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); + int rowLength = row.getLastCellNum(); + ColumnType[] columnTypes = new ColumnType[rowLength]; - setColumnType(data, rowLength, columnTypes); - final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); - if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) { + setColumnType(data, rowLength, columnTypes); + final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); + if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) { - // get to the first non-empty line (no matter if lines are skipped - // or not we need to read ahead to figure out how many columns there - // are!) - while (row == null && rowIterator.hasNext()) { - row = rowIterator.next(); - } - // build columns without any intrinsic column names - final ColumnNamingStrategy columnNamingStrategy = _configuration.getColumnNamingStrategy(); - try (final ColumnNamingSession columnNamingSession = columnNamingStrategy.startColumnNamingSession()) { - final int offset = getColumnOffset(row); - for (int i = 0; i < offset; i++) { - columnNamingSession.getNextColumnName(new ColumnNamingContextImpl(i)); - } - for (int j = offset; j < row.getLastCellNum(); j++) { - final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); - final Column column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), - columnTypes[j], table, j, true); - table.addColumn(column); - } - } - } else { - boolean hasColumns = true; - // iterate to the column name line number (if above 1) - for (int j = 1; j < columnNameLineNumber; j++) { - if (rowIterator.hasNext()) { - row = rowIterator.next(); - } else { - hasColumns = false; - break; - } - } - if (hasColumns) { - createColumns(table, wb, row, columnTypes); - } - } - return table; - } + // get to the first non-empty line (no matter if lines are skipped + // or not we need to read ahead to figure out how many columns there + // are!) + while (row == null && rowIterator.hasNext()) { + row = rowIterator.next(); + } + // build columns without any intrinsic column names + final ColumnNamingStrategy columnNamingStrategy = _configuration.getColumnNamingStrategy(); + try (final ColumnNamingSession columnNamingSession = columnNamingStrategy.startColumnNamingSession()) { + final int offset = getColumnOffset(row); + for (int i = 0; i < offset; i++) { + columnNamingSession.getNextColumnName(new ColumnNamingContextImpl(i)); + } + for (int j = offset; j < row.getLastCellNum(); j++) { + final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); + final Column column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), + columnTypes[j], table, j, true); + table.addColumn(column); + } + } + } else { + boolean hasColumns = true; + // iterate to the column name line number (if above 1) + for (int j = 1; j < columnNameLineNumber; j++) { + if (rowIterator.hasNext()) { + row = rowIterator.next(); + } else { + hasColumns = false; + break; + } + } + if (hasColumns) { + createColumns(table, wb, row, columnTypes); + } + } + return table; + } - private void setColumnType(Iterator data, int rowLength, ColumnType[] columnTypes) { - while (data.hasNext()) { - Row row = data.next(); - for (int index = 0; index < rowLength; index++) { - if (row.getLastCellNum() == 0) { - continue; - } - if (row.getCell(index) == null) { - columnTypes = checkColumnType(ColumnType.STRING, columnTypes, index); - } else { - CellType cellType = row.getCell(index).getCellType(); - if (cellType.getCode() != 0 && cellType.getCode() <= 2) { - columnTypes = checkColumnType(ColumnType.STRING, columnTypes, index); - } else if (cellType.getCode() == 0) { - columnTypes = checkColumnType( - (row.getCell(index).getNumericCellValue() % 1 == 0) ? ColumnType.INTEGER - : ColumnType.DOUBLE, - columnTypes, index); - } else if (cellType.getCode() == 4) { - columnTypes = checkColumnType(ColumnType.BOOLEAN, columnTypes, index); - } - } - } - } - } + private void setColumnType(Iterator data, int rowLength, ColumnType[] columnTypes) { + while (data.hasNext()) { + Row row = data.next(); + for (int index = 0; index < rowLength; index++) { + if (row.getLastCellNum() == 0) { + continue; + } + if (row.getCell(index) == null) { + columnTypes = checkColumnType(ColumnType.STRING, columnTypes, index); + } else { + CellType cellType = row.getCell(index).getCellType(); + if (cellType.getCode() != 0 && cellType.getCode() <= 2) { + columnTypes = checkColumnType(ColumnType.STRING, columnTypes, index); + } else if (cellType.getCode() == 0) { + columnTypes = checkColumnType((row.getCell(index).getNumericCellValue() % 1 == 0) + ? ColumnType.INTEGER : ColumnType.DOUBLE, columnTypes, index); + } else if (cellType.getCode() == 4) { + columnTypes = checkColumnType(ColumnType.BOOLEAN, columnTypes, index); + } + } + } + } + } - private ColumnType[] checkColumnType(ColumnType columnType, ColumnType[] columnTypes, int index) { - if (columnTypes[index] != null) { - if (!columnTypes[index].equals(ColumnType.STRING) && !columnTypes[index].equals(columnType)) { - columnTypes[index] = ColumnType.STRING; - } - } else { - columnTypes[index] = columnType; - } - return columnTypes; - } + private ColumnType[] checkColumnType(ColumnType columnType, ColumnType[] columnTypes, int index) { + if (columnTypes[index] != null) { + if (!columnTypes[index].equals(ColumnType.STRING) && !columnTypes[index].equals(columnType)) { + columnTypes[index] = ColumnType.STRING; + } + } else { + columnTypes[index] = columnType; + } + return columnTypes; + } - private void determineColumnDatatype(Object[] datatypes, Row row) { - for (int index = 0; index < row.getLastCellNum(); index++) { - CellType type = ((Cell) row.getCell(index)).getCellType(); + private void determineColumnDatatype(Object[] datatypes, Row row) { + for (int index = 0; index < row.getLastCellNum(); index++) { + CellType type = ((Cell) row.getCell(index)).getCellType(); - if (datatypes[index] instanceof Object) { - datatypes[index] = type; - } else if (datatypes[index] instanceof CellType) { - if (datatypes[index].equals(type)) { - continue; - } else { - datatypes[index] = CellType.STRING; - } - } - } - } + if (datatypes[index] instanceof Object) { + datatypes[index] = type; + } else if (datatypes[index] instanceof CellType) { + if (datatypes[index].equals(type)) { + continue; + } else { + datatypes[index] = CellType.STRING; + } + } + } + } - /** - * Builds columns based on row/cell values. - * - * @param table - * @param wb - * @param row - */ - private void createColumns(MutableTable table, Workbook wb, Row row, ColumnType[] columTypes) { - if (row == null) { - logger.warn("Cannot create columns based on null row!"); - return; - } - final short rowLength = row.getLastCellNum(); - final int offset = getColumnOffset(row); - // build columns based on cell values. - try (final ColumnNamingSession columnNamingSession = _configuration.getColumnNamingStrategy() - .startColumnNamingSession()) { - for (int j = offset; j < rowLength; j++) { - final Cell cell = row.getCell(j); - final String intrinsicColumnName = ExcelUtils.getCellValue(wb, cell); - final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName, - j); - final String columnName = columnNamingSession.getNextColumnName(columnNamingContext); - Column column = null; - if (columTypes == null) { - column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true); - } else { - column = new MutableColumn(columnName, columTypes[j], table, j, true); - } - table.addColumn(column); - } - } - } + /** + * Builds columns based on row/cell values. + * + * @param table + * @param wb + * @param row + */ + private void createColumns(MutableTable table, Workbook wb, Row row, ColumnType[] columTypes) { + if (row == null) { + logger.warn("Cannot create columns based on null row!"); + return; + } + final short rowLength = row.getLastCellNum(); + final int offset = getColumnOffset(row); + // build columns based on cell values. + try (final ColumnNamingSession columnNamingSession = _configuration + .getColumnNamingStrategy() + .startColumnNamingSession()) { + for (int j = offset; j < rowLength; j++) { + final Cell cell = row.getCell(j); + final String intrinsicColumnName = ExcelUtils.getCellValue(wb, cell); + final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName, + j); + final String columnName = columnNamingSession.getNextColumnName(columnNamingContext); + Column column = null; + if (columTypes == null) { + column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true); + } else { + column = new MutableColumn(columnName, columTypes[j], table, j, true); + } + table.addColumn(column); + } + } + } - /** - * Builds columns based on row/cell values. - * - * @param table - * @param wb - * @param row - */ - private void createColumns(MutableTable table, Workbook wb, Row row) { - createColumns(table, wb, row, null); - } + /** + * Builds columns based on row/cell values. + * + * @param table + * @param wb + * @param row + */ + private void createColumns(MutableTable table, Workbook wb, Row row) { + createColumns(table, wb, row, null); + } - /** - * Gets the column offset (first column to include). This is dependent on the - * row used for column processing and whether the skip empty columns property is - * set. - * - * @param row - * @return - */ - private int getColumnOffset(Row row) { - final int offset; - if (_configuration.isSkipEmptyColumns()) { - offset = row.getFirstCellNum(); - } else { - offset = 0; - } - return offset; - } + /** + * Gets the column offset (first column to include). This is dependent on the + * row used for column processing and whether the skip empty columns property is + * set. + * + * @param row + * @return + */ + private int getColumnOffset(Row row) { + final int offset; + if (_configuration.isSkipEmptyColumns()) { + offset = row.getFirstCellNum(); + } else { + offset = 0; + } + return offset; + } } From 5a75486d81725add8741a96e517ff4dddd7a3f37 Mon Sep 17 00:00:00 2001 From: Jan Bob Date: Thu, 26 Sep 2019 13:53:14 +0200 Subject: [PATCH 03/22] resolved review comments --- .../DefaultSpreadsheetReaderDelegate.java | 92 +++++++++---------- 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index 8c5cc2f7c..4d08c520b 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -42,6 +42,7 @@ import org.apache.metamodel.util.Resource; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.CellType; +import org.apache.poi.ss.usermodel.DateUtil; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; @@ -104,8 +105,9 @@ public void notifyTablesModified() { // do nothing } - private MutableTable createTable(Workbook wb, Sheet sheet) { + private MutableTable createTable(final Workbook wb, final Sheet sheet) { final MutableTable table = new MutableTable(sheet.getSheetName(), TableType.TABLE); + if (sheet.getPhysicalNumberOfRows() <= 0) { // no physical rows in sheet return table; @@ -117,7 +119,9 @@ private MutableTable createTable(Workbook wb, Sheet sheet) { // no physical rows in sheet return table; } + Row row = null; + if (_configuration.isSkipEmptyLines()) { while (row == null && rowIterator.hasNext()) { row = rowIterator.next(); @@ -127,11 +131,8 @@ private MutableTable createTable(Workbook wb, Sheet sheet) { } // Get first 1000 rows for the eager-read - final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); - int rowLength = row.getLastCellNum(); - ColumnType[] columnTypes = new ColumnType[rowLength]; + final ColumnType[] columnTypes = getColumnTypes(sheet, row); - setColumnType(data, rowLength, columnTypes); final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) { @@ -173,31 +174,54 @@ private MutableTable createTable(Workbook wb, Sheet sheet) { return table; } - private void setColumnType(Iterator data, int rowLength, ColumnType[] columnTypes) { - while (data.hasNext()) { - Row row = data.next(); + private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) { + final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); + final int rowLength = row.getLastCellNum(); + int eagerness = 1000; + final ColumnType[] columnTypes = new ColumnType[rowLength]; + + while (data.hasNext() && eagerness-- > 0) { + Row currentRow = data.next(); for (int index = 0; index < rowLength; index++) { - if (row.getLastCellNum() == 0) { + if (currentRow.getLastCellNum() == 0) { continue; } - if (row.getCell(index) == null) { - columnTypes = checkColumnType(ColumnType.STRING, columnTypes, index); + if (currentRow.getCell(index) == null) { + checkColumnType(ColumnType.STRING, columnTypes, index); } else { - CellType cellType = row.getCell(index).getCellType(); - if (cellType.getCode() != 0 && cellType.getCode() <= 2) { - columnTypes = checkColumnType(ColumnType.STRING, columnTypes, index); - } else if (cellType.getCode() == 0) { - columnTypes = checkColumnType((row.getCell(index).getNumericCellValue() % 1 == 0) - ? ColumnType.INTEGER : ColumnType.DOUBLE, columnTypes, index); - } else if (cellType.getCode() == 4) { - columnTypes = checkColumnType(ColumnType.BOOLEAN, columnTypes, index); + CellType cellType = currentRow.getCell(index).getCellType(); + switch (cellType) { + case NUMERIC: + if (DateUtil.isCellDateFormatted(currentRow.getCell(index))) { + checkColumnType(ColumnType.DATE, columnTypes, index); + } else { + checkColumnType((currentRow.getCell(index).getNumericCellValue() % 1 == 0) + ? ColumnType.INTEGER : ColumnType.DOUBLE, columnTypes, index); + } + break; + case BOOLEAN: + checkColumnType(ColumnType.BOOLEAN, columnTypes, index); + break; + case ERROR: + // fall through + break; + case _NONE: + // fall through + case STRING: + // fall through + case FORMULA: + // fall through + case BLANK: + checkColumnType(ColumnType.STRING, columnTypes, index); + break; } } } } + return columnTypes; } - private ColumnType[] checkColumnType(ColumnType columnType, ColumnType[] columnTypes, int index) { + private void checkColumnType(final ColumnType columnType, final ColumnType[] columnTypes, int index) { if (columnTypes[index] != null) { if (!columnTypes[index].equals(ColumnType.STRING) && !columnTypes[index].equals(columnType)) { columnTypes[index] = ColumnType.STRING; @@ -205,23 +229,6 @@ private ColumnType[] checkColumnType(ColumnType columnType, ColumnType[] columnT } else { columnTypes[index] = columnType; } - return columnTypes; - } - - private void determineColumnDatatype(Object[] datatypes, Row row) { - for (int index = 0; index < row.getLastCellNum(); index++) { - CellType type = ((Cell) row.getCell(index)).getCellType(); - - if (datatypes[index] instanceof Object) { - datatypes[index] = type; - } else if (datatypes[index] instanceof CellType) { - if (datatypes[index].equals(type)) { - continue; - } else { - datatypes[index] = CellType.STRING; - } - } - } } /** @@ -259,17 +266,6 @@ private void createColumns(MutableTable table, Workbook wb, Row row, ColumnType[ } } - /** - * Builds columns based on row/cell values. - * - * @param table - * @param wb - * @param row - */ - private void createColumns(MutableTable table, Workbook wb, Row row) { - createColumns(table, wb, row, null); - } - /** * Gets the column offset (first column to include). This is dependent on the * row used for column processing and whether the skip empty columns property is From b82e7285e8d4993961bd51e619228b22a3a0f8c1 Mon Sep 17 00:00:00 2001 From: Pixel Date: Mon, 30 Sep 2019 13:52:07 +0200 Subject: [PATCH 04/22] resolved review comments part 1 --- .../DefaultSpreadsheetReaderDelegate.java | 128 +++++++++++------- 1 file changed, 76 insertions(+), 52 deletions(-) diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index 4d08c520b..6fd7afa3a 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -65,14 +65,13 @@ public DefaultSpreadsheetReaderDelegate(Resource resource, ExcelConfiguration co _configuration = configuration; } - @Override - public Schema createSchema(String schemaName) { + public Schema createSchema(String schemaName, boolean validateColumnTypes) { final MutableSchema schema = new MutableSchema(schemaName); final Workbook wb = ExcelUtils.readWorkbook(_resource, true); try { for (int i = 0; i < wb.getNumberOfSheets(); i++) { final Sheet currentSheet = wb.getSheetAt(i); - final MutableTable table = createTable(wb, currentSheet); + final MutableTable table = createTable(wb, currentSheet, validateColumnTypes); table.setSchema(schema); schema.addTable(table); } @@ -83,6 +82,11 @@ public Schema createSchema(String schemaName) { } } + @Override + public Schema createSchema(String schemaName) { + return createSchema(schemaName, false); + } + @Override public DataSet executeQuery(Table table, List columns, int maxRows) { final Workbook wb = ExcelUtils.readWorkbook(_resource, true); @@ -105,7 +109,7 @@ public void notifyTablesModified() { // do nothing } - private MutableTable createTable(final Workbook wb, final Sheet sheet) { + private MutableTable createTable(final Workbook wb, final Sheet sheet, boolean validateColumnTypes) { final MutableTable table = new MutableTable(sheet.getSheetName(), TableType.TABLE); if (sheet.getPhysicalNumberOfRows() <= 0) { @@ -126,14 +130,13 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) { while (row == null && rowIterator.hasNext()) { row = rowIterator.next(); } + } else { row = rowIterator.next(); } - // Get first 1000 rows for the eager-read - final ColumnType[] columnTypes = getColumnTypes(sheet, row); - final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); + if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) { // get to the first non-empty line (no matter if lines are skipped @@ -142,6 +145,7 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) { while (row == null && rowIterator.hasNext()) { row = rowIterator.next(); } + // build columns without any intrinsic column names final ColumnNamingStrategy columnNamingStrategy = _configuration.getColumnNamingStrategy(); try (final ColumnNamingSession columnNamingSession = columnNamingStrategy.startColumnNamingSession()) { @@ -149,15 +153,29 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) { for (int i = 0; i < offset; i++) { columnNamingSession.getNextColumnName(new ColumnNamingContextImpl(i)); } + for (int j = offset; j < row.getLastCellNum(); j++) { final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); - final Column column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), - columnTypes[j], table, j, true); + final Column column; + if (validateColumnTypes) { + + final ColumnType[] columnTypes = getColumnTypes(sheet, row); + column = + new MutableColumn(columnNamingSession.getNextColumnName(namingContext), columnTypes[j], + table, j, true); + } else { + column = + new MutableColumn(columnNamingSession.getNextColumnName(namingContext), + ColumnType.STRING, table, j, true); + } table.addColumn(column); } } + } else { + boolean hasColumns = true; + // iterate to the column name line number (if above 1) for (int j = 1; j < columnNameLineNumber; j++) { if (rowIterator.hasNext()) { @@ -167,10 +185,12 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) { break; } } + if (hasColumns) { createColumns(table, wb, row, columnTypes); } } + return table; } @@ -181,54 +201,57 @@ private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) { final ColumnType[] columnTypes = new ColumnType[rowLength]; while (data.hasNext() && eagerness-- > 0) { - Row currentRow = data.next(); + final Row currentRow = data.next(); for (int index = 0; index < rowLength; index++) { if (currentRow.getLastCellNum() == 0) { continue; } - if (currentRow.getCell(index) == null) { - checkColumnType(ColumnType.STRING, columnTypes, index); - } else { - CellType cellType = currentRow.getCell(index).getCellType(); - switch (cellType) { - case NUMERIC: - if (DateUtil.isCellDateFormatted(currentRow.getCell(index))) { - checkColumnType(ColumnType.DATE, columnTypes, index); - } else { - checkColumnType((currentRow.getCell(index).getNumericCellValue() % 1 == 0) - ? ColumnType.INTEGER : ColumnType.DOUBLE, columnTypes, index); - } - break; - case BOOLEAN: - checkColumnType(ColumnType.BOOLEAN, columnTypes, index); - break; - case ERROR: - // fall through - break; - case _NONE: - // fall through - case STRING: - // fall through - case FORMULA: - // fall through - case BLANK: - checkColumnType(ColumnType.STRING, columnTypes, index); - break; - } - } + columnTypes[index] = getColumnTypeFromRow(columnTypes[index], currentRow, index); } } return columnTypes; } - private void checkColumnType(final ColumnType columnType, final ColumnType[] columnTypes, int index) { - if (columnTypes[index] != null) { - if (!columnTypes[index].equals(ColumnType.STRING) && !columnTypes[index].equals(columnType)) { - columnTypes[index] = ColumnType.STRING; + private ColumnType getColumnTypeFromRow(final ColumnType columnType, final Row currentRow, int index) { + if (currentRow.getCell(index) == null) { + return checkColumnType(ColumnType.STRING, columnType); + } else { + CellType cellType = currentRow.getCell(index).getCellType(); + switch (cellType) { + case NUMERIC: + if (DateUtil.isCellDateFormatted(currentRow.getCell(index))) { + return checkColumnType(ColumnType.DATE, columnType); + } else { + return checkColumnType((currentRow.getCell(index).getNumericCellValue() % 1 == 0) + ? ColumnType.INTEGER : ColumnType.DOUBLE, columnType); + } + case BOOLEAN: + return checkColumnType(ColumnType.BOOLEAN, columnType); + case ERROR: + // fall through + case _NONE: + // fall through + case STRING: + // fall through + case FORMULA: + // fall through + case BLANK: + // fall through + default : + return checkColumnType(ColumnType.STRING, columnType); + } + } + } + + private ColumnType checkColumnType(final ColumnType expecetedColumnType, ColumnType columnType) { + if (columnType != null) { + if (!columnType.equals(ColumnType.STRING) && !columnType.equals(expecetedColumnType)) { + return ColumnType.STRING; } } else { - columnTypes[index] = columnType; + return expecetedColumnType; } + return columnType; } /** @@ -238,16 +261,17 @@ private void checkColumnType(final ColumnType columnType, final ColumnType[] col * @param wb * @param row */ - private void createColumns(MutableTable table, Workbook wb, Row row, ColumnType[] columTypes) { + private void createColumns(final MutableTable table, final Workbook wb, final Row row, final ColumnType[] columTypes) { if (row == null) { logger.warn("Cannot create columns based on null row!"); return; } final short rowLength = row.getLastCellNum(); + final int offset = getColumnOffset(row); + // build columns based on cell values. - try (final ColumnNamingSession columnNamingSession = _configuration - .getColumnNamingStrategy() + try (final ColumnNamingSession columnNamingSession = _configuration.getColumnNamingStrategy() .startColumnNamingSession()) { for (int j = offset; j < rowLength; j++) { final Cell cell = row.getCell(j); @@ -255,7 +279,7 @@ private void createColumns(MutableTable table, Workbook wb, Row row, ColumnType[ final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName, j); final String columnName = columnNamingSession.getNextColumnName(columnNamingContext); - Column column = null; + final Column column; if (columTypes == null) { column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true); } else { @@ -267,9 +291,9 @@ private void createColumns(MutableTable table, Workbook wb, Row row, ColumnType[ } /** - * Gets the column offset (first column to include). This is dependent on the - * row used for column processing and whether the skip empty columns property is - * set. + * Gets the column offset (first column to include). This is dependent on + * the row used for column processing and whether the skip empty columns + * property is set. * * @param row * @return From 8c21387d37d1eb55c3a830df10abed53cedd4678 Mon Sep 17 00:00:00 2001 From: Pixel Date: Mon, 30 Sep 2019 15:13:39 +0200 Subject: [PATCH 05/22] + minor fix which sneaked in when reverting few changes. * I do like the change from VARCHAR to STRING. For the sake of separating concerns I would do that in a separate PR though. - changed STRING back to VARCHAR * And potentially have the 1000 constant in here incapsulated - Added EAGER_READ to ExcelConfiguration including boolean validateColumnTypes --- .../metamodel/excel/DefaultSpreadsheetReaderDelegate.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index 6fd7afa3a..a55916dab 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -136,6 +136,7 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet, boolean v } final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); + final ColumnType[] columnTypes = getColumnTypes(sheet, row); if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) { @@ -159,7 +160,6 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet, boolean v final Column column; if (validateColumnTypes) { - final ColumnType[] columnTypes = getColumnTypes(sheet, row); column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), columnTypes[j], table, j, true); @@ -246,7 +246,7 @@ private ColumnType getColumnTypeFromRow(final ColumnType columnType, final Row c private ColumnType checkColumnType(final ColumnType expecetedColumnType, ColumnType columnType) { if (columnType != null) { if (!columnType.equals(ColumnType.STRING) && !columnType.equals(expecetedColumnType)) { - return ColumnType.STRING; + return ColumnType.VARCHAR; } } else { return expecetedColumnType; From 891990006df56b3cffecab4f3adfc228b5587182 Mon Sep 17 00:00:00 2001 From: Pixel Date: Mon, 30 Sep 2019 15:14:00 +0200 Subject: [PATCH 06/22] + minor fix which sneaked in when reverting few changes. * I do like the change from VARCHAR to STRING. For the sake of separating concerns I would do that in a separate PR though. - changed STRING back to VARCHAR * And potentially have the 1000 constant in here incapsulated - Added EAGER_READ to ExcelConfiguration including boolean validateColumnTypes --- .../DefaultSpreadsheetReaderDelegate.java | 11 +++------- .../metamodel/excel/ExcelConfiguration.java | 22 +++++++++++++++++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index a55916dab..d3baa9a37 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -65,13 +65,13 @@ public DefaultSpreadsheetReaderDelegate(Resource resource, ExcelConfiguration co _configuration = configuration; } - public Schema createSchema(String schemaName, boolean validateColumnTypes) { + public Schema createSchema(String schemaName) { final MutableSchema schema = new MutableSchema(schemaName); final Workbook wb = ExcelUtils.readWorkbook(_resource, true); try { for (int i = 0; i < wb.getNumberOfSheets(); i++) { final Sheet currentSheet = wb.getSheetAt(i); - final MutableTable table = createTable(wb, currentSheet, validateColumnTypes); + final MutableTable table = createTable(wb, currentSheet, _configuration.isValidateColumnTypes()); table.setSchema(schema); schema.addTable(table); } @@ -82,11 +82,6 @@ public Schema createSchema(String schemaName, boolean validateColumnTypes) { } } - @Override - public Schema createSchema(String schemaName) { - return createSchema(schemaName, false); - } - @Override public DataSet executeQuery(Table table, List columns, int maxRows) { final Workbook wb = ExcelUtils.readWorkbook(_resource, true); @@ -197,7 +192,7 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet, boolean v private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) { final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); final int rowLength = row.getLastCellNum(); - int eagerness = 1000; + int eagerness = ExcelConfiguration.EAGER_READ; final ColumnType[] columnTypes = new ColumnType[rowLength]; while (data.hasNext() && eagerness-- > 0) { diff --git a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java index 4779bb1e7..4036a4c8a 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java +++ b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java @@ -37,26 +37,34 @@ public final class ExcelConfiguration extends BaseObject implements public static final int NO_COLUMN_NAME_LINE = 0; public static final int DEFAULT_COLUMN_NAME_LINE = 1; + public static final int EAGER_READ = 1000; private final int columnNameLineNumber; private final ColumnNamingStrategy columnNamingStrategy; private final boolean skipEmptyLines; private final boolean skipEmptyColumns; + private final boolean validateColumnTypes; public ExcelConfiguration() { this(DEFAULT_COLUMN_NAME_LINE, true, false); } public ExcelConfiguration(int columnNameLineNumber, boolean skipEmptyLines, boolean skipEmptyColumns) { - this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns); + this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, false); + } + + public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnNamingStrategy, + Boolean skipEmptyLines, Boolean skipEmptyColumns) { + this(columnNameLineNumber, columnNamingStrategy, skipEmptyLines, skipEmptyColumns, false); } public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnNamingStrategy, - boolean skipEmptyLines, boolean skipEmptyColumns) { + boolean skipEmptyLines, boolean skipEmptyColumns, boolean validateColumnTypes) { this.columnNameLineNumber = columnNameLineNumber; this.skipEmptyLines = skipEmptyLines; this.skipEmptyColumns = skipEmptyColumns; this.columnNamingStrategy = columnNamingStrategy; + this.validateColumnTypes = validateColumnTypes; } /** @@ -102,6 +110,16 @@ public boolean isSkipEmptyColumns() { return skipEmptyColumns; } + /** + * Defines if columns in the excel spreadsheet should be validated on datatypes while + * reading the spreadsheet. + * + * @return a boolean indicating whether or not to validate column types. + */ + public boolean isValidateColumnTypes() { + return validateColumnTypes; + } + @Override protected void decorateIdentity(List identifiers) { identifiers.add(columnNameLineNumber); From aae088f98815650d6c8dc2f9c024540f5fff5cd8 Mon Sep 17 00:00:00 2001 From: Pixel Date: Mon, 30 Sep 2019 15:26:35 +0200 Subject: [PATCH 07/22] removed empty spaces in whitelines --- .../DefaultSpreadsheetReaderDelegate.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index d3baa9a37..dbf734e47 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -141,7 +141,7 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet, boolean v while (row == null && rowIterator.hasNext()) { row = rowIterator.next(); } - + // build columns without any intrinsic column names final ColumnNamingStrategy columnNamingStrategy = _configuration.getColumnNamingStrategy(); try (final ColumnNamingSession columnNamingSession = columnNamingStrategy.startColumnNamingSession()) { @@ -149,7 +149,7 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet, boolean v for (int i = 0; i < offset; i++) { columnNamingSession.getNextColumnName(new ColumnNamingContextImpl(i)); } - + for (int j = offset; j < row.getLastCellNum(); j++) { final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); final Column column; @@ -166,11 +166,11 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet, boolean v table.addColumn(column); } } - + } else { - + boolean hasColumns = true; - + // iterate to the column name line number (if above 1) for (int j = 1; j < columnNameLineNumber; j++) { if (rowIterator.hasNext()) { @@ -180,12 +180,12 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet, boolean v break; } } - + if (hasColumns) { createColumns(table, wb, row, columnTypes); } } - + return table; } @@ -262,9 +262,9 @@ private void createColumns(final MutableTable table, final Workbook wb, final Ro return; } final short rowLength = row.getLastCellNum(); - + final int offset = getColumnOffset(row); - + // build columns based on cell values. try (final ColumnNamingSession columnNamingSession = _configuration.getColumnNamingStrategy() .startColumnNamingSession()) { From 7ae1b26dc40117b8d56259042753362df0b717f3 Mon Sep 17 00:00:00 2001 From: Pixel Date: Mon, 30 Sep 2019 17:52:49 +0200 Subject: [PATCH 08/22] added a test, however datatypes are not found... --- .../metamodel/excel/ExcelConfiguration.java | 4 ++++ .../metamodel/excel/ExcelDataContextTest.java | 13 +++++++++++++ excel/src/test/resources/testDataTypes.xlsx | Bin 0 -> 32656 bytes 3 files changed, 17 insertions(+) create mode 100644 excel/src/test/resources/testDataTypes.xlsx diff --git a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java index 4036a4c8a..40b138f89 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java +++ b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java @@ -52,6 +52,10 @@ public ExcelConfiguration() { public ExcelConfiguration(int columnNameLineNumber, boolean skipEmptyLines, boolean skipEmptyColumns) { this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, false); } + + public ExcelConfiguration(int columnNameLineNumber, boolean skipEmptyLines, boolean skipEmptyColumns, boolean validateColumnTypes) { + this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, validateColumnTypes); + } public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnNamingStrategy, Boolean skipEmptyLines, Boolean skipEmptyColumns) { diff --git a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java index 19773ad08..08d080aea 100644 --- a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java +++ b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java @@ -419,6 +419,19 @@ public void testMissingValues() throws Exception { assertFalse(ds.next()); } + public void testDataTypes() throws Exception { + ExcelConfiguration conf = new ExcelConfiguration(ExcelConfiguration.DEFAULT_COLUMN_NAME_LINE, true, true, true); + ExcelDataContext dc = new ExcelDataContext(copyOf("src/test/resources/testDataTypes.xlsx"), conf); + Schema schema = dc.getMainSchema(); + assertEquals(1, schema.getTableCount()); + + Table table = schema.getTables().get(0); + assertEquals("[Column[name=INTEGER,columnNumber=0,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=STRING,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=FORMULA,columnNumber=2,type=FORMULA,nullable=true,nativeType=null,columnSize=null]]", + Arrays.toString(table.getColumns().toArray())); + } + public void testMissingColumnHeader() throws Exception { File file = copyOf("src/test/resources/xls_missing_column_header.xls"); DataContext dc = new ExcelDataContext(file); diff --git a/excel/src/test/resources/testDataTypes.xlsx b/excel/src/test/resources/testDataTypes.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..50c23b2d08eb8eba0a51f57a91f3781142f03e16 GIT binary patch literal 32656 zcmeFYby$>J_cxBBU=U){DrwP3g9rjjS}1~`Al==dqJ)wnHN+q((jrKQAU%w9m%t1q zHFW;=eb3K}DYUz)2z!qC-SPMCXZO zu9%n`?;|2Qc94kZ1ks`W>Qd%rHo9guTK63-bgiFoIGCE8i#xENJ(g%cc>n+X^MA1i z(mqruRdSJe&dL=_giP-Hg#a`{u!S!J{m_#9z7>*NwjPIhXr> zxf+`0-z;I_|HV`0>9FXdo)irO>4eU#Uz)YKkM@<(`_1IrTzlo`Xu{uRvv4S+D?D86 z_c-n)^9#vP6y?kGhjp)4WaO!QKA5MkkG*H4KT@y2{m3{#;B|#VvQ!c7TLt%Q$n6tf zUzwc0;X+0JY%Y2j$&n@zrG5r`Q<(k91H;4hb&67_*ng59J7WBzEBr$GO{T9~81WJ0 ztDt0EQ7$>>Qs%x&stI>uWpo48dxe1W)g}Ar3a4*{+VOAH1tsnLIw9p;#;!s?Alk3r z$_c`2XNQ0xeZa-E2%aba2p$C()w0qxvF7A}{{8>3`F}AW_iR1wy`nN6o10#> zEA9P($8N~mOUf0Uzwh?qRv%@qUpzBye;FIyvHLV0#Io)sZkzSJ{i1%8wda?A@uj^x zeo}-f&pzEVY~9k9^b)I8sElRWa21tZ`%?RIgzRlccDs^*OR4#vKFZe(GRd^gOJ^TL zb3CLud?Jy~^R!sR%g5DE0QdG(j%mEu=8r5^<{5 zT3f$6!=={b{L&iDgL?)-hFRJ%R>Bt?wVs=l{J z&CQLWRJMzmeITEKcmMDHC+&TMQ56^2&6PLX9?hkeJgzF9mU_*1!%ZXIURZn(RL~|> zcO9KIzAIc;9%%N3TkiV`gHKVfyM6j9Gb4@uspq$jyh^jv)iFLe`=bHHCsDSb z`OitTp7gr=QsTq6b8kNeH0>A{eKz6~6WMAG8asNLXYOb9LFt#Jtg3VBY(Z3`x*I6O zG9CIx@W!M1c~t1xFZ|S{Q4%ClRVQjvPK^qxy)Mu?A)}LuB)ew!N+>s!!|Iz6JDJm^ z!LFJ{bPl)qPDO1_7ghos{lCW0<4pF8e1K%%9v~v31aE*C+8aG-_Z!T5xsDZ|8J9R# zEaLR&3w_1iz%CA>Sax5&fOnjHxnB<5OSpYE*DrQwAmo0cq4lk1&8ozFx|zl+LS@1a z-n$U%yivVK&f#+Vx@GjyT;++aey?WV<5S`F5p75AI+4g{ep#8t$eL1!q*ET)_d(Hc zLL=HCW{l;2qvklx$&8ad_Zx@Xrp450pB7x*r@>P4bA?Uw$Llz;8`=Y}KEAtQ@yKZ4 zeyj$i_H`bwvEI9FcGpu^)rT(WQ7DT2D*Q~P&6Z$+rTFmvE!$K{i|23V7V(4gFMNkD zkKeY4Pv_`6c*D^(D07vh^vnUVF%!*0hbc>@U$-ymtL7%l-Db`)3pq;ac$-bIU3T@i zm*HfjjGP8}9Q$HQn3#GtH?!9({+9lahlxd5t4FxVdKH*$uhO6WL3ib=)u&WzeWe zyL#pl??B5^pt;sP#+^E`$ZzzU=Y|a2Q@WJuChmOFV7yb*eIT>SFok85``f$!2-h6MYfMw;@gF`?;aFUvKzQ!QlQ^5`_fE5(pGp}0l&c%? z1rbDDSHyi@<-N{F6-21!k&-2q9+KnZEL+BTiiY=U%YL$Yskg+-S_J*A;)ISH+n{tu)mAI+6RhMWj;0$VqyS{_s%aklQN$ zRJSzPB)P*&u0;e=*?G1k6qWD|BwIut;-Hn`>kmuHZkBph%PcXkTeyR7lf9r@6` z(lO^Dc}SBF{P(cyJ7t>OpU~IC9{euxEfbF9#F^KrSn7sp3V)Fdj5Uf|J1-j ziDY~zEvHH#dFu3gDTPNDu8lqRoyoExB08KP7g$C0jyfim>ba#iyO5kuj(ido^8gu( zuatG7D)s%zIjwC`+eFb1c6q95KB(GdHH`fy;E?Q|?AZ$BO6vlr8QZI%m&oFJ7nW z$_t-)S4*V%&*Glv?un%wZov_s70`Gw&(*V}Hmd8Z6l zHyCiJnD7cobGOr%zLS2=R8S@-E1ZEoDDCw2qV>8?V?jDP_ar9u;MNiEcVg4dHdK+d z_XDLOQBSJE{im}lxo*ri)<-Z0`ER{oyz%w|R`|rVRR-R{V~&)T*Xyb+1}}6STIs%F zal+<7@_lA+7arxS*~PvQYvLT^2c9MJoZIk3)G!O(>rwV8+)--F`E2vJP2^&B>0^=P z1eX&t66`}_e99R*4*DHOyF|9!k_TsYG|L1^8nBr?Z&W-y&!pPs+64%3^s~Kgam|i( z2up1j@mk|}ID^d9RJ$%yZ!&tmw@c}WByv9aNPE&PxTk7xH)^~w#KG6HyggkT|{w8<@)}rKXu*o)Tu`jLRphWMWtcU z2g;L<+z}16tfx2_+8k0L)G23f#oEqGboxXC8T*C)_Zk99%1s>dxevX+O5Ttc2!0*v zNkuavDzX$GjPtckJ(MD;T&TTv?Ia%!N`T7v;)Ud|FIfxk{;U!*E0XN@YR#rXm zP8K+p5I*?nHmUz%FXC7u42t{Z2~|I}6gPcv5V+^`-nHEH2J+R9Du&1dl3Q8#+~Rd4d0eh7j(< z*M8?yWg~C+C9XU`xRs5zm)uU7u%ArypY2R_Eqgp+YrOI?%bPu8y<0zQUzBX~&F%S{ zxtXpne8ObS&d6Nh;9Yu|nC6{I$<^M2q2!|2*IaO>jU?SEmG;BqrKZ@?r9#x*%VdR# zn#Zm=d1#IcsC#T=SSOTgCNnrmmZ=^2{Or-ftG5Y*CLuJM=^gVl(RzbZedsW5E}Vk6 zmDUUAQ<=ZZ{%sD6C(lD zV=niuzY(I4ad&f#V}3o(^s8;_uG){9ZspG?kYWm}S0k57;AN|feb4)KzkVN{J>tLJaM`^_?>O4); zc|X3(oyF#v4`tZTxr*Bkc9xNXia(VC|EtXSFX)A?NPVMeRL-I@a znXXnHiR~xPaZV|%?{l@PZ@MJk$mDd6j_>?(PJ)Vt;q&u6*VEeNE*R)rN(!nQVHz)8 zI6G)JS$nbP>CE&2;?AfRdr`Z0FQt5OF%L!$y|^HMPB_#k!sCVAMv*Cbr@Y*A($=1> zYNy1yYrelK_LY@z$%oz!QxWuXn0}!4-B9=CC%x2M_4Lof_92E`|X1&9$@jz=mI%@#}=2{GEF6FpEwrRhta(! zh9iDE7&RKCXIVatVqTItdIB>O*2XH?YlfeopGcqJfLCh&Ze#4xP^@KT( zxL=7VMf?v3zA|5&L%En6{R-DyWtQ`E`ZWi=h}ft)*NkOq8l&^K&TW4`?2bIVGskvr z|FVE@>#&DyQk_P!{4n$_L9y)S8xZj*JQ(@(XgWzRo-X`})FIXBelZi`zzP5v94f z`^J2yeOYaN0HcJf1McQFTG`!oeyvk^c3?AWN0!mejaytajZ?hXai;;N;}wSFadz39 zuURo4+*%vyEpvBRX$aE^R-Scp!`)nT-=5gYM4Eo1H0SLFlTDbr?<{EzxNXj*n#_l- zNR;g?pj}(BXz^m#HAeTXv5{l}_cb-m6$#HZlzOl8j>Pw6#(}b}#d%x}W0}k4-V(vr z)t#HGzvkaeFRr&!&W@eZFm+#^=noha+i`PPGq0>1S={_F^|0vX4tA?(P~yzaMt2yP z5(%tXS#Bs|9A#VD>|SD<3kzV}vAeoSxt2=Wy@=Da=wC$ja?ZLtuPn}Bk!8+qCY#f^ zonDdtp5pu^$F0e!oxGTx(w))mjbwd^;*ELKp!@cMg!uOM;6TaFl(JUB+65&Ia>unw ziJi?Eoch`=xefE0u}j;(YmmA}*KpllE1mMUqg-$SQga)OrJIZWxQ1`Ie3LNA`mvUE zZ2OZnEqBM2wT7~?GPCW?Zs5f2h55=Iv3k<}#V)h9-=jM_TaVl~?ZO#5v~&8utMA;} zAzs^Ak~lkB)?T_0R#%!+;%i%2yJNQOX89ZHqXGcHODPPQ-er!^2ROX+1 zSM9;P&HY-+N7L&KAI;Y>8xv)ln`32e67EdqYNm+`jF=G3cZw7D9Uam_Zl}IYj*n9* z+)>7)89C| zYk5hO`ahJJ$nE;T5z?ZQe*f_UQTkKKA)4;j43#r~ zB-x~YU@%T>`TT3*lVJMd`uTI|i;qoif4e=At$Hc_QjGuG*4vEKiJjW_oKp6@XcHLl zSeiy7RdGrunz%;$V%{ako+8(k&Em%#MOTWK2U{~==9JZJ8hRHz2oW``b1b-z_>DYd zW~q8}9BGq#%HXk;CdVmDsR+w?seMX9{X@kXm$05ZWV!zwenw=t$8&CsyLQD#ZD%k>|=cvYd}f zIs0lNLZiaeNoYdVS$kAudtUG=$}2U{H>h}6txWcAm_-{E_T1)&hA8k~`KFMYt&~fZ zjDUtDfgu!zI?+Zt+6p;Bzps&p@RDpZi-)FojuqCWGxRT)EFWIKR@;#i85(8(*Xa32 zrip*XC-po^?r~eGDpFV01^Y3JFnR?32j*mAKz$tr`)#l0X|EP;x5?#4!VdcEr~BE@ z_Va|1v0N*(+xxT)_i3l@x3+gAB(WrqDc%44yZxKG3iQV1cKi%UJnc!)3?J0dMv%`! zjP6Ak-Glap*z<;k)N!;?VFfsY&x*RA7607?{Hb^ocAp!o;!TLGzaKdXOi=y&_vimK zAOtM`ct#T~m&GrWyAXAF8J$v0%sUrd$h4xz?GIl$w5@ zut3B8NNA6D&BO>MpmaZ@hShqr!O_5B(XO|$nWn`51n*k;#2$^L9*zHv6`(B`790&R za$fp<$M(hde1sD zwf)Mo66s-K%9Y!VH=C9y&|24_WS3 zzB6Mv;0Jxr`b|mvi75*bU`w85P?dF1l?|DZ_g)zak@x18gUwS3!$64CeG94k)?SyK znm!7L$Y^iK_FmU?u3BjXm%M}aFuRV zIcn8QcDrJOBg92L!bLsW1;AbWst|M}>OyGKT^xMv!ynq^Q@h_l;N;N#{A6$BKdHZ8 zwD;7DKBU;~jS9Q1ywOh1?*IDA#e2W^2G(y7Xi(r?9qSm>!NUQX_3)r>W%7=#2SfpT zS4kcdznhZxaInUI00Fw6#rB^5i8;ynEtqHi(;LoL?)}c* z8(%sRMge<$!O(EuIc-mXEk8=?0Yp~#xIVosE*-`-{M9Pr+O|@pnZ@Iox$8fA?znAy zw%g3G!!-2BDtbM?)Z^*J(>r?b^=ug9RJu)$m@wbHwQHgu_ObPQV021OS?0-l>0KHH zFO4NQ|MV@ReCljtko4ZnfjwTSkQ7zpba~n&fC6>DkDO9>;MPhkHS$_EG)IJh=!cJB z4!uo_rseUU3I_-0R9KNVl!$rP+r9BpydiFtQq=A(Vg}VZ(Z8ENM!dZ~{3=A@mDk)4 zvnQPHq{feZep_nh5Qmn_e)Bjk`+;idn3YPWJZj(LS1ByNBfBb_sBTG^iPx9Cx_%*_ zImqMelUID!LXRrsZ&2vXv4m?upT-ZfQay{CN_R9bKXxJ^qCql%+lMZgtoA^#LZBW8 zsaV7FLqg#nBc4VbZfP8gB}GWb-6Ao36~QXU8|HC@rFo^$o4?LNk*n)~A;wCjjyXwa z^eKtis|on|TQ+Xgc~aijCwv9>A%v>yhR(A|)m9w9R4of4`$Zg?kse1b-9ZvhK9|c< zN=uT0X5`~W1^o&MC$$Cc(xABXrHaqXkxNpHnrr#M6EyhZ8~nvfacZs&c*S8BXvJ6V zRzpUS2gD%zS;E6HT$HTm*ol*3&w)9FDc;&->Zcw!tXs~05_*ADDriR1TAQIt6#ghH zO}+oPt{m5$G|Z+mz3feDy3;ocJH^?`*`Y(7DgC4qFSima#7(I!dL;jv-P2p*MfB%x zQXC;g`Oum0-u1Vxt>mkh9Nnr2q?2YCJNIQ1CD19c91p%7&Z)h7|GLV&*g@HQK}y59 zEHg;$*m?&R|3R@p(P{4!zp%qINRp)b>}Y-S_B8!eLyZ;S)iD3m3;iBwpN-_R>EM7I zCx$Lb#io)SE`Hl?v)VY8$do3xGGzU0`=V#sIb=bxO?b$83suhp20x61T`&tR*{TW7 zbJP+y>A1`et~SoAjxzGI&#oz*=R8q+s&}LqEpm`$M-%($a!bjwP{k8!E{B6_jVM2g z60jylT2p)d2z1ONnPmIQd6*AfQEjy}wXlx&$O3EPf`U<4@9@f3E*PtyL61BkzyIKQ zdiE$ZqB0w{O6UUX7ue~n4aZ`yNWH4p=BMuj=StBC@2P+)M=cN>qp)zpu{rPfoA@x;mr3ZT;;RiqgFvq_YGN-D~`_9#&p044GYtwdFq33@-_B@aQZ`0~0l{(la zbSMT#7r>c%nNsinrYrYCJuRtHpM#XX=w=})@&qn+27m^fxCM1+pvU3Y4KJuU#(RdW ze%{d_j1Gj2mntB8ImW~Mlmp(b(6|Ffi`Z&ZI4O4va3S{-5>i`ueA>I;mIK)6ukX<( zW$l7ya?ywL8eR-@)asit*f^Y-VBllJNGdo;A`0kie8bLP*YrFv2CU5>2lkBP?K!7c z0?v%i=4;sbA@F#Xu$l8(cylHh=%K-VVb92g!q1hFo!xo`kKV+Oej2%BwRPqp1WWpU z-A7Ustg9TvB*jE1x?x=l?=usc8&NNcyBM&U*yy77XvrE!LY#j(GYTESGS7jYhQRWY zM~Yt15&U{*Ln#g{CUcb(2Q-kyDK%6|c6ja`9A&XOWzcy7kgVbd6Y(C{-xzqkoG|!^ zFMe=8ASp2QLEZzW8v|cdfTxgglrpgQ&&hm$F%_Znos8lLugQ7R6OYcq0hq#q^zbKR z>NiH7FbUsR`3w#SQ7|Fm1nwMb6&yY?jmOS$OT9X-1Hy*{EO3$uf`sF-9BugR$BoA< zz~eLEF=TrnH{rbaN!Fm~ttQ(?$ID#Ht=c*aS;s&ajMIS!ziK>oj7RF#c^xSB*m*oE zYvI_FtA=7vjM3%_HU21lgZFh-Q#jP=eUSSy5y6$BhDAg@k*|^XEw~Z8kh4Py92;u39vMC^P!n%aP21!;H zgtzIwiZvNyR{7#bTIE(ATfb7)fhNOL-|6OA-xh7Nj!ze`K87TSX-RDMFHgp4})AAuIvG&vQ@r?Bd;*h6wqTR4qAO*D)X|g@~O7Yi_}7&p_slv z%8kT_$zPA5F!5K0PnE99g$9C)zcMcch*=i|70&R59{2=%&Q%F`GXU2~Qj<(gj|5?k zTHQ{8GIJy;{-X~x-p14dGG^qg6<0+vHCPVh#-H=x%;xsmI&v0MaR8~;hVY*P995GX zG*f|vZHEXA%ukHeQc6b6o3Us1^gF~Kr0%~0Pt>!Q@?5iSBbFJH&VY^9OdUaxJbW94^x z(ufnV*-{^}*;4DIwLTo_GmU5Sd39ci5F$OF7mDJxp`sT26Y&|&`%1{1sBZ37en z;31;p+&8}_$aeK6Co*ZvFsQFAaxCgAUT|2qj0m5^Cd@|DkFN54(XV(E0x6D#<2Xai zNYvsJR9_zf9w4i(@xeMnnU_Qy6+YDZNO8pelp&xMe28)j;PI2V2R{HHVFZ8_06>6X z&=eL&on7VA0honI*l_})1CTUn@#xw!*m^+!@9?3(8w1!wY4jYX^@XtxY5gcVpM7^LevN)Q}7r{NWd>ct-goF0bK*hMj#%LO#e{J|7>v7N(49q$nPLmy3jxsfdD0* zhC!-?Jm&#F2e$a73m}<*e!#CD#|6um|HgBUfLCEm!^aq(5!PuP5~7R;T75|KEF0-e zWB5=LKBI?R)Xqx zYAIRh?vO`4Km~)mm4vCNg9zoPFIEyRZaiB3Q`RP}qdoF8*EQO`(Xy&`P;Rh-d7~N^ z3z0(o?ilNzP=PcyGzyW9!9SnQzUu+nDTOYI4yF(X1~?4~2@1_Xy26s8hXKoP-4_9` zpDqL#3d&NrzfT5X8d6LkLD-YH$n>1&n$SlCk2ER3xTmY?qlT&nJnjKc&yc}6271iP z0#THu7&v)6CDuwdM#F)S&wFpENdXWXArJt`z!_+lqIL&E7!l!xtj%;)ve1W$MIg~& z<4*{E0!a~edQy3GgiVR9R-2J9pEVB9Bbbf1l+!7S+006=Vs?n)B@ipnx3JupX|z;w zqR|&(E0MVK-nzxAchY$+(mca`8k>-8S%FPRl%o(ok@-9PXn^zcmW#!CFfqvEq87ha zY9B($G9WO4Vf8VF{Sso2uVtPGdzZ)OsH(nC&U|hUgY-B!K^{8dg1FTwk4#kvxwj*m zWCy8}T^gB$obH0Cn>S1dmMAG8dxd`REe@&KudkiNY2&^tJx4>%K1#xlCFzEoY8mf@?CSt&@ zt9-#DtqL!d0kcDy8$RJ=I4Oyv68N#1e)OkS&dqBAuXo!*pP&jMvIAu<v z2z0!914p@MFh-DNvmduml2LzK>{g$3wXIkAv02)h;$*B9(0+H56jFqu&X`*3FGekn zEi4!gAWzo`O)NPnty=|Iw}drVDy=Vs3$m3E>w0(Ejeac23k*990mX01%@i5D8?Ebw z%8*P390pVZR5YBhW`d(_BN-%4fSiOW6r8wW3XwTUg2cZ7zmM9DeyDNqZ$!tR$9qKw z)GmS*PODD7I|6_YR0J91W@d2S1|rulT)DMF6p&H;X7#zlE;Q=qC_h!}^`3AL!*|RO ziOyp=>EB|Q9)tSz5~)7KbCSFH*dy2CUhDrsYpY{gd20@LY!}PmTcEfuKA8S!96m zdZ48FPelfW3qt5mkpWxtr^vwb=+7vS>XM;4`NOS2UL@dY3dp(rXL6wuz>!e&a8nXG z8(_LQL%k=Qt{Gnz>YyYh^rff;;Y!(e`No$$4N9tdASWY$O|S-mx;yr8U1q6|0U)+55rS}nPwAk_ z1g#IiXl3Tp$HF)DnQNjWz?vOPE!az!k@!yKF7tqL;3p*Uni_!)m~xu1c8vk>gR0S8 z=EAo!g=ZS+@_BPI!o(k{UF^gKg}{QO8#3m1P6jA*t@X>D`#}aTGZ&okEq+BnJ>DSv z+|~?#(;!u%Kp1F1tsKJC0$9B4fRw@R76=~1Z*j3;+dk@lWqt;%F7}#_L+tS@FWj2# z69wCg%UV_g-R7mT5gVXDVuDF#ZRzaw>>&9DT45E}>@lEJ@zWH)x^aR9sE|4#!(IJ> z)%_0ihxhO;#<6eK@^JQr_#h7>G~ozp0SS2_D2GV|z)~Z_nV>z+(DAWB1q~oeNO|lA zCkV7+0?tQSV449Rp+LhBQvR+Y2?`5WhEqPkStGnCUSB}MhLG}oA^9O7KwvF`8iCY4 zsW>VMq4{j~7%EghCeIYS>5yEcJJk3PHt31k_3Z9cX;< z0ag2OSq!~SoIwtZ5fMkxfrkUd?r&ka1obW;xdczhr{!@OG}!)_tXQ-3Z%Yuuj{YWK z{;FTBhguu|AzhDN0`7I}_8;H@cty3#Gnh@FQs32K|4m8s zzqY4Pv$XRKj_)y?`RhZQaYegMg_^v&)!`V>!NIT|mH z_ol@Ez$GA`LQDbS=D$?!6nzm;F|oJZu}2*Ir@^MkGyWph9|oIl9GLM<|AvuF75a_) zZmI8Y)q17>GSalf7N(GULivCRpq85`AzfI)&>8{h0#q;mN1oycmDoGjG1S;taGwNP zHEK{9)rkN*hYB@YEgQnhj!+Q>vmL(12p1t1XR_>z_+hCZS{|Lf)Zut%Z-WD_lJL4k zZ9iZGmTAY9r4_Pte2*S4UWB?ZO?23VuxBCGEub$d#O4Zw#)810#!eUd7H6d=`j$Ku z&UcNy%(V%hS|oB2)##BHQMQultnfv#FY%kHaG`RMzjLrM3wVami_nnrDFze{wHM!w zv|hoJ%pI7mL36_UdyoDT^CqARh%-QYB1cdhbsU0}Fut8(4Yw9S7BIut>_CU)91usn zCn$crtN|JuulS+nN5k0#o{D5OuD#7ZLMH{>XZ$sK4;tZ4B)Fq7JYAEl2A4lje-FA? z!&gY4`FIdwJA#Y>oC+<6m-H-ihM^F>HSJK$WQA{HVa)Fo$&>=kRf?$D?_nwjn z(m@2q2`x`nJN|cFP_=dMPXQx8)OrQNKiq=20x6(BEr?wq0>2KW{&#g{5`dC`|KthX zpg+wUe6JO>)BbY6-h$~bzS&*x-}g)Q`X01;{xntMEvHm$puh-m#ooI{b@ns zGwGkq3j$fM83=(n=oOHOVA&1?EOgg|?=m-Cc`1@J?u1+WBg+Ym)0R%NAf9~p(nRg(|!{Y>yc%cIKC?VE| zYM1;NsNWjFVxfN;Qdc`$M^Yztfr{5M1W6)4)O4i~v|Dhu)8Px(e~~eG<0q^I**pq( z$6Y;kJ06hQfPw)sHV5J@0c%+L;LU)HZv&H_X^h?nn&Wr^A+&M{PJ%WA_eG!phD{BH zD;2mVC2#@gwi24Dhrt{FU{=s8+%0p!KJkO`y;VYET?8a|;8{>_f^I3Ksj^Vf&7f-W z3R9li{wg;!D5iaVUS zeg-&CLb~J+4D+w$?^Q8yb^Qw--{r&$R)qWfAk?LwL2?m`C!Q5$pbg+Cp#>@vDggoJ z=M3Q(*j3p0KB`N9llwQ$K8bdSAR&0dhH0+T#V~Whxq!ciI96zp{ggA8cnFL{(pH|^Uk}%6_ zLKga}Ps~vsyUkUhm}Gn#@K0qxlh2!)0Sf_$4$^k%04_=LW9$zptgH1-=JOJ6w|2`E zxUB+p2Q_G6J#IYIlvX&<_*d@{Y6I6w_Q%6l;`VyveGv~jBZdBe3@*H(k8sKn$sw6( z{0z0h{@&;Wp$t`J0QG-8hpuzf@coJ@CtiFd1yo;j(8FVp|8|KHzanT3rn!7U;5u;> zZ|mF#EOxJj2%(tI3)>AiNrLe!xTGZXJ;m8(-Vv(Cz6gHEgMin{w4U+fugD2kOaC&5 zd)fEB)mJ{}XXmI+aMh#=3?{Y*9mc=9sW6hl5L2@c(*WyQ1ri(&-TDBZt~SEVmU#}o zq#zIvwNMSHK`@g2BmU?bp$XdnPDya5G8g_M_Z z1zk4bG(nFsk&fJU1$P z6NPQpo*<(HR}0~wsPOy5(U{}SAB{R*d{#U03G@pr5s~NVQ{iQOyw&)B8gfw4r|FSr zA^7*DW+NDq3#4tn(ik#EDfKD-_)0U*7v@dzFo#3~g(zuG8ys^InHrM=REmMbQz*nNV)dS;7#&W! zVu>5+NPDTC!oI7bQ6weK7B+|nbv#bQJU_wxZn5b16k3I35E0Tg4m3e$qm&xG&nklm@Bxf(Jk**7 zNxDUi);`~HqFraj!p>r9OM9u3!+L1fS$7D|Vlsj~ErX&t3p{;?;OTR_gA+)EPzd5Cx5W@nBLHl;2#}Yvw9h;nj1BlV$$rARAV_G*m?H`(! zL_|3^%*L#EW8Sx(HO_HP zzO+p-WM8RSC+3BDxIIi)`^=#Ma?)hQaL89hUOi@h?TI<-1=ayL3`b#v9;ae5hY^g^ zJK7kj*eRbpHM_2^h~~T?z$82l4q0gg>qqN3a4uPB5{3CB&{BFjI(DFofnI5r4B?V5 zI1RG>Mg?5!|>EW-e=uCqA;rhkt2g9FWtdS zRZ&v7I{-Cr9PtoLSw#AkM0D(cZDh-XmL&m~R2_gndX;zY->=`gw6O50tj?UDzRX)B zkag!}9k&JEKtt8GBb(n%ye^E9xK8k34if{?3OB0Wa#;=rCFrOKC<+&zqn4S=yEWo z2E6E8a^?HNYT?vhb=O1@wznzeye8VRvQ1Mt|0AM&fbME0W$EY69If#Y%geSw?ufiq zFQ!W<4OQVuQ1_itx^UUNEiwsHljvYK%WwTN5+!M#M^uu%q<4QHR^TdXLHWtanmgeA zyTTYhsPW5Xo=1yIN^#V9Yb_C?$H7j&0C&@3i_e=Q5Li#NhLZ3v)M;-4^;Zr${J#e! zD-!$4VhSpnL2QBFzEA}V3@Nti3msoxQkAzgwYWIpAfP_fZ*q9Rubl|VxiW~fUsl-O z>K-f6YwzS+*{!oh4ntR7BsBGO)uvz20YXxC(ZW z&U{G8B=n<8D%eHblmmwZ6p@UaKXCS$5yX9TxUJvihD3eP9b)l5v&RTVuM~%sOt?t7 z26q*|b(^_FN|df-lXD_DHl&iLN&+UFW_4&s2Wmp(t@}0{=on}9Mw>p?X-!eM=6qZI zy_9;!T&v8ZYS+TPLDY9$VI23-NwFlN((aQ+_txns;s(kE6j3~)gOR6%W>A=X0a#bp(yk<> ztj&LXT5=y;V+K(;GeKIh@vXqLqEwEH+|?bb)Z4rQ;E$D2r$2$ig|M_)j^8Gt9W<4K(w<#xP+Uv4I}9 zzZiRG)BIg&mO>3JdhyCo>@1GHyxx<12a5NNjn?-?*sWCVV;3A6jC%Hgy8z_m26hAXV zAi|brN+#&#rBV@ghHn?vR%KVJ-RM^Jb65l}EPpy+q!7{+w>= zJn$VAdJY1En%TSlhKlHGup5a4BEK8guVCj+`O*#n^jt`>nl14M(EdjCX4sx9$fJ;S zPAv;*KOitFeGJF41?mGd-;k+ENr;CkF<*m;tv*BXbUtD+*YZQ8-h}3w$y6c+VWz`x zisB(3XsBpVND(YsaYyTx4AaI)9Hmu>A^F(VlU7O zdiE<$NmeRuoXaa(xPm50_+E}C*0HqDc`$$a6V0z4cI&K7l=w^pWA9p~N(Y-_I*CE4 zCcXQnCR>1c!HORbX`*xfH46%ctHVoI&J095&VAW24ttH(rML|j?6UHo*0MKc{rbtA zyM_2)W1#nc`p&*%xEh4fJv-}~5qvbd=vw_OMjpJ59cn&q#K5^CE}f7`|K8+A zT;4!q1#Mc=lA}%p*4euLrc&047t!&XNkUnmXo~P~mLJD05lu(0_(fk0E{mc63~tCf zW%fU?ZbmFS_g|n~m~kEG%I1EJv10fD|8mDX0fEV#7wj)8v+S!4GO1D#}xY@`!~TM%uo*gtD2g%yd4n%OLv*Vxhc^uGbN@Bx=B4 zj$NB^ZEo=QJ}sBa&ILUHYb-2-Pq)K#gcn!aoRvAH`$W2$_fea6M<}@JxyM^ry4m^}{?~Dp}uIP?%ba1?Gp1o6&F|29&JVxWsp5jEm4s6QFyOE-6A*8Bh; z=pkEBLS^9p&yMFIS22J z%aEf!(hxJ`ckA90bj`|l-J?RZB@%3fF@B*BmrG#Py=d}v^YZD1_BoMR4-qX-2QdyN z;tb4tP{pDccgQd!;IZ?u75yLd^uPvcY6Jc=EX{np3o@x|@hbqO1tv4=jk*aQ?cU$# zV&3QLe|F|`4b8L%yyZGQ1UWwFt8TOub21##42SP+o!5aKcj~A>@SX=w2BAo5U1;I}@i_Mv> zvg$r&uV9BQ^PMp-Mt*A0c_oR`C!=SfMzAP_TW^9t$ih`R4Mw*tP`piu$sYPCT2W$9TRCD8oe_lCALH^BW zQ1Xi5+FD;E?*lrTMKh~-E`1s+1LmfuybdGPNBj+|)FxsHuipA4YBs}wVDKU_)~T7* zulzWSt#P?#>~-GDbHt@+RXG2Z4vnSi((da%;7f+?Ps!OC4=FEEkUI&iX?!A;qVTJt z)_>pcGBXkT>*p0ushpF7e0{7wym`}3gJ>}dRGc8+_!b>@zn7;ev-n7$&%@+2m2fBV z%g_YkXD|w2g3;=^sftr{t#f~w2G%O;>nn7A^BwF>*g%l)D7W5sq1HgDO3PB6oU%x? z82x=Z8V3Wb_*jqUOg^}+u_mT;m2jD;5DFR!)D7PVizdF&pK?)mzB}P;FAF9O=d?&x zepEX`M`IRTMg6X0K(mFxOYaWeLs?<)yEPVAq%$4jjUexRN*W4s3Cd22IvG0s!1X@< z7pIQ2){^sc8sQDet%MA5>zkQ*EFU11b4w5Eoe3x@nUEJ1I&&6z6c=~i z*mGplrKrGo7`%ET>TJD!C1MymTx?K_ph9Mzu}4b=@aM31kcskHHmUKNFg5vMO)XdK z(uTsSMLya5%+mX*?35SS%PcOjDaLo14s881DY>!U#Voyi#b3N;(Q~B&s&n#{`ng`! zS$d~`)xH)!`$dl#na$anO8QVxp;AB5!;?SfORr1j7jCW$>6{kvMv$9M zKi=bD2U)MTYqcTOH>HVeU~E+u8e{V8C4c3$<4AKivHVwBlOs9Ws91a6j#HBrf&7)T z=Q_gr3X5tGYThiBme+7`&<16PnldM~*0oHv&FRXmgsvw3xL4a)$& zWNFWTMmR*oGh|JqgjS~x>dISw{23a1npPFOj24?&A3EZeg7t(J9kKq%K&HlfTz}ef z(IyLCQ))K$8EOt^q4~jt+0sq4-*NlYW{Woze83{66CEDVvc3V#vl|7wHVQv_ipuh1 zZW@`K2?m?Y%AOE3gPcm)Nh;Mr_Jca|$mUW7Fp%{22eb@)0#w!1_lcq+7pt)Y>*p-B z1qFO5PCw%9Wze4846j<7$+XY>6ARG!;^G*LJ(pzy=C>$yp2#DdkqFlwBjLdA$NCX(& zG&`|D_GXdKYEh6+9oLV|ky*ffQXReKgFa)Qd0hOSQ-P(1i1aMmlflM`tyfsw=|0x^ zwdF-zL9U&5iJ+I|u@YJ&E>{W!(x%1Az;LZRXH!jX<%rpZCag=7%~T65=^8!w`>!G2 z{`i`Y-H6#WmyA5+ADGjC; z986m#nB(+hLVR@C-h*#Czm=uS0%G7Fw5}W~ z5-oZS7_^!1XzhsKt2iya2I`0YC4ht+_dE-{hXD{$3&N$KpvMktaU#`oe?7|PF9Ojp z;rPj;jdgj=-&RnW6mXYh6m`pocg+=ewk0;0_M-)xQ2{jP#(+FZM+1RZ_ozMxC9%D+ ziz>u^E-@GzabzdOHx6FKicZf|5BPmV{M}{&2{Fnk$SVjdQkfrq*2<)(yhzH*tAqxn zJZ=>(&0c6CkJ6a$_@4c#tAYFhU`&)AM>F)krJII^Sq;p z%COYe(z7KVHA44*?M;x~IrD3d1^FW~0aCmty)dhKt*aVCKePMT}v`bf!Sqh@cZ4BE7&D4b3e@4-RW;z@W*m^VH3>d&!HB zc=P*Ye?U6|boE|vL!+fx2L(kaEj@wiUUxWI&a`^~iWxv7Za~&ON6DED8Bt|2OlRwn zSZX`;aqY50{!AOKfKyt;N02+V=yrM4pSLz|AmST5F$sZLX8Ghi=0kg(id&KIaqzW1xr?{gi+SZ6ZaPov)4Pax6vw1O(2UkR2| zNsF*_$VtKHgR}Z4RcFL*#MdCY5$}>kKcEl5445S3G!H*qF3k5(Bb&x@V=qKz>lF1$vHiqI- zM7y}#{!ulKrsbN?~d-1oWfbMAAW`#$G<=eoY1N1M*&ts8cOoue0o zZ+|4%jh~io?rjwV2;sV~TY;5t_ndnFQ`#kc_ZX&b61(8=c3}0$*D4}pUPP8hLQ8J zXXLx+D3+>M#M1Vi^G44s`ooGhU?_+ai>Bru)i}VB1tHWnhUsYgaG)?&RsO10JGQMk2F~ z1VZ=tIUi!F6$*fZoGGP?L@0_(4xCZzWjN#-UC@Wv6y6@}52-&(49jV=_ zH!LiDMTdFYLXrKbvO-XaSeOhxMD|a^1a%m}+<DpS=zf2d$yg6gqN=Mm4k5d_cJ@l&2YA3GOlchrdEUL{wY<8v{9}Ny8^H^fo zQ^4GQn~zHh1Ru?rNM@IDeNZ(tM0%w0IS?L9OL;txeE4k;CJiN%gG|B0cr({>c$BT-&sBKb%MZNDmrYr6SpsMyj)#Q+5rrC=TUHm z+D4g}e!;y*MXK+VQnUismq55O(xA|4V~VZ7nOV$v3}?WsO7}7LYkCf;L0OpI96lxa z`&>r6m5N^$4uH3U^E>B^8@rvMo3`Ht^^M6#F)*Ii1Uu%^AB?FetM%?LnZhgozCdhyz-B>Pbwr*blt*c6A6an;-Q~rrUo4+0)o=b zxyob-9XLfUxs;v1hBF`5N)|t7FdUv5>eYJMS$9;vuaS<5GUvj?)l4G}P8$Sbz{n1+Zj69H1SVtQH&Yh6P|gf$3-$>;4t zSsWLq#H*)lSbFt~NJK)S;lY$kgYvQ0&Opt? z%Aa7uB%b*p%cYX^CFT1V&Yo0<^$=(CVE?+1im}AG@Uz-vVZ+bW#?5?p%=yx*qR2X-4sZYs* zGVjmV3DCmAn+-UQB`_>|S>n=@ZW3=hBQko2HL*t4(C7#o z#?V8hujh?TJL`S+icg=55(sPpi+cZlQX=!+Ood)4Ng? z@Qnapk$juBDRW+=&?myAuk$oPqebPuu{D#0hH(|LgwN{QJg1F;p|wvq*FuW)oGrD9 z)>flN0JSzn4lf5LcBNEu_{3rcvTx8jgcRw#9H77cfM*(n^JCpi`5;OS3LT8Fc~9+4 z2szuL!*r#UYh^s|f=Ga8T8F|QuN;x8sW?|hKhR>e>hEm{GcO+xjXOzp&FkhO`)ud_ zEnT`v-0O*2MiO6;MlL;j!vaTp|3tZ1W>N@wHyX+)#4T~Bic^nnZN)qC^$pQk@ATeD z&5f*L_s~qPLEhCyC&)_ctDv4T$dFHPaJZr794a)jo+l|vGMj$juy0jpINDqvRrk^- zq82vh)!je6VcPR~61{3iPDK0zt%H-(0Uo< zMv%wkMLm&aCf))(U|wddsP(ip8Z}kc?*~Rl`qBrbE<3;0-{$v}VUS6L*AL7t*%eX% zTz$s8kuC*Fo34HVBaPGWCtr#jr7Ob+D`f<;O(?+xtM6f%i*AJ3a%t`4Icx%$ypSfx zc#-=dSk{g77On@LZicd%oFGzQSV^9?<$C4rj!3TlVX=UbB4I;67Yw z58t-TZ@ZtI&CY&t_W=;};lz=^FnK~KwLWSMjG*#%Y4F?v#N`SijgRIW)(8sSQ9rg= z88nVd95l^(MWZ9@SaX63UU2=MO&_@#!b$WXbAE#1=X#dKmv1=FP`h$c7*+$H8$+41 zmB_{i3m`DTutlbzAd`@}dI-piw7_*utXGLt)w*U)&N?X@>O=E*arSAcTOtU-9$tw% zq?M7QpRGXpV(6q4U!cBLozen!+Q(GjFGu}KZ}T)JI6AHh$r(F+A;O~DTa(wmEIWkT z+z+ffeP98Fb{8XVt^>*XnewXQ@*Oy*MrSosg>eL&nfazDC?v{J_)uvn)$P@-ZWhiZFJ5+uSI!K)&qATN}#|X zaBzk9vkaFB_`9R%R4m*+4YpHfqbzO&Dz@LUU^%nkvL6Ss5F;SvUocw?s=l`76z84# zwbq8pU1r?}b$2FooW?63+<)p^O=7F+XHas}Sc;mcNjpvmi1eM7+RivCF`u2T3aNc5 zh7pnKl4?6E0%KyDJWCwH`urkP;^y zGV}?Ff+ULFuMud=?d(MK+VQ(XS6Q)O4s%nK`9a{svIbm>+l|b`&#=S1Pncpf!Kz4b z7CwSVV>s5gI?|szrkmIxKA$Q^U7K-@TW&mlY52`9Qn_L>BbyH?a9RvupV7zW*dfhrX}KR}k=TP?n;D^6K+vlT zy{p+g{!26YkrwEsfwHpI-kwDC%F+;|bafYh&wap*TVJIPS+qcccCM~Yw`3=-SFX=a zr$(-?%v@b#M^hpbA^rpFG}{Nd)|LDhTiVGX>x|$Pvg<#AB1Of+95lh}XC1*ym-hj@ z7aM2mi*C*?NHJ??x2xY^%l-rN0;VoJ+U$I#_@SV2`gLfv-ve7NB%>pPo(JFF?Z+{m z5~zIE2h7p%e1GS5M}nkS@$PDQlv~~TsYUZ<3cHb^-4)*GCqlF&LRwRvGMRc+LGAXm z_AUbiJ^kqb==9VQcx9)XV=u*?uG#P-PWf_#o8IF`^TDt$3ws5S=Umcwo$xU=A(>sU z$_0iFTJDsQ?dK=7#mO#=;X1mi$De#uZ18O_qT3mfv}4!1Gur#F`xkR({N`^yd^b?f zV9oMus$M6b$wYE1aX#zwEuBUIoo3R`#a(DKMFsaO{aS>VeF5)GUEYa^XFGyJB))$9 zk`Y%LC-W1JaA_o(Rj@Q&`Q%>qvGW?1o>wB6147Trd1ts8nCEs7XZOb>->vEUjqTb< zyv=XE(8dvyBOhB_SRE)KigEbUxW#_c9`W!mnN0@7{?x>gFKW;h8X_{a1d^f3@cfT$Tv!%R_L$#BJ z6Jte3Bk%s3EeS?bb^t0Z;bBVAmvJJ`+T=+8^D+n;wKlOhQVU(T_VJosm>HYaisa?lCKLXSzTBhnXIjZ@Fen}7uM#>mdCm}BFE9b>vNTj&+qGcc>|3# zYKz6%%+l;MaXvD!TWNiAS}amm46@iUB$n8X_FJUhPqcunPYq0t>XOHKc*@XA3*Pv9 z*^m<7^2+(15}(^thuQZas?pEek zZ-J()uOD9>z?PvTy(Cdnh~b{nXLs@s6x=B|P(C0CK607a1*I{m`cSk5oGRwz#^u+V|}bv7ShtP z!lX%EvCD1vEoIkvy?cTpIwtW2QjLZJ@ud$o$t5QsCo1Rhr6WW+5BD!qt+P3dHimCS zZ=3NGH9ca!NVd7>GF~-&CmLgRf@tJH_~OHj1X2=@K7%sEtOq zOB1m6J*5&?2~>Ob=Bx}eV`83%>*(k90Rj~PDII6cJMYlaDw=C{m008PX!LAHOx}6N zGLrFIQdINaOfKUycn|n!eR~oCiU`?YWI;B+zVnlfLf9B7_QyV7#v5VTuoRfI@ktHr zEIfzz!*{Lk&A*xXKV-rAoG zmcAx}oJVGznWC4-1#@h2OWzlQobQK!s+YfSq)^1ahbrA2;os<>>?6$c3na7DzFa-m zj`X6jPkyO=uk>8|(+j4a5R(4Ilur2Hq>&;S!X{ytNhJFIzF=2ZQbB>Wsi|a^(_x?5 zf`V(Nrc%xAP5%UC5=;>J%}$wJ#zcY#VzjyaU(R43BJ4>G8)xegDK+uUUvQf2>LZdo zv476!2pZdHNIW$C%b+htB@xz;dT90+TR)6dqJ8@t*F;#i#^1E@y_fmW@qIn~r5e7G z3*3{6eLhwH4hDV>K1yj%CTJico4GR;jeY)3278ePLRZ4nM7xEppFr|;l(00>Ze`=o zBZWIk+feUj{uQHyIs4HM?av8@i0o$b%o1at|33_~mTp`FZWu45C4w=);+k>81R;zF z#ukff$9>LAdnbbNz&`87ea=tYE&41N`>YT5xgbqT^w~WuZV<<+$et zv?Prww<(xtAhN;Z1VLYiGlky4)%1WC5XAt$faoe7Ckpxkq8hlGHPAX`Oo;=20g){p zCj3(<&Re=D;Jml+@Qdr z{@dqaw2E_w_@1C~x;dUbT}R}*PVWiKfN9FkyntjttUQ4Ry%y>>)%tTR07sCc{2kEyTL9oxo6!feXEC}xwIAGsz9iYXzjYL>mP;qqA zj;h|q#<}msevwPY=fd)W>ONi{DAziGycXI~d_TwTVbBA3;_00NV(+K0H|bQ;l1epG zl_id4c`xDH&V z8GdKqEqT^Nw$ic)R|nh4-9wHSdnC@fNx8T>ancI1u+{DJj8OQPZEG!A0xf=cAac)v z@tZHX7M`9O^6igVd}7!}Z9R4%;^f0;iJ}U9e(j_q(23%@>7@2NL1?X*i}AeOV<*&| zHfBC*^G(Gjazg&Ai3R3H=9c%*)fUq~c|NMW>$~`Vi3ddD>-ybBD*})c{~W#i_jvvO z{U=!F7tjA_;74fX-xW8$pMa$J1JLqT#UJ5j|EO3EkPQDi`0Q3YTOmikTVmSv69DN} z<*jg%-<6%fLlZEFpTH%zTG$G`_}#+IJ)0J`0x@niuod^}y8%rA<+w3~zo1}kRow~` z^j%e-?JutX1RAu}%8$>1zT2RoDRiKr`RS?9R`sp-`0obfoHqyY!>xX+_Ky|xyDkmQ e3hED?f2*e#&(njov4P>uNK* Date: Tue, 1 Oct 2019 12:59:39 +0200 Subject: [PATCH 09/22] added test case for datatypes corrected a few tests cause these where breaking on the datatypes --- .../DefaultSpreadsheetReaderDelegate.java | 3 ++ .../metamodel/excel/ExcelConfiguration.java | 8 ++++- .../metamodel/excel/ExcelDataContextTest.java | 30 +++++++++++++----- .../test/resources/different_datatypes.xls | Bin 0 -> 27136 bytes 4 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 excel/src/test/resources/different_datatypes.xls diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index dbf734e47..bfe7396d1 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -197,6 +197,9 @@ private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) { while (data.hasNext() && eagerness-- > 0) { final Row currentRow = data.next(); + if (currentRow.getRowNum() < _configuration.getColumnNameLineNumber()) { + continue; + } for (int index = 0; index < rowLength; index++) { if (currentRow.getLastCellNum() == 0) { continue; diff --git a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java index 4036a4c8a..1cd8286b6 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java +++ b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java @@ -57,6 +57,10 @@ public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnN Boolean skipEmptyLines, Boolean skipEmptyColumns) { this(columnNameLineNumber, columnNamingStrategy, skipEmptyLines, skipEmptyColumns, false); } + + public ExcelConfiguration(int columnNameLineNumber, boolean skipEmptyLines, boolean skipEmptyColumns, boolean validateColumnTypes) { + this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, validateColumnTypes); + } public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnNamingStrategy, boolean skipEmptyLines, boolean skipEmptyColumns, boolean validateColumnTypes) { @@ -125,12 +129,14 @@ protected void decorateIdentity(List identifiers) { identifiers.add(columnNameLineNumber); identifiers.add(skipEmptyLines); identifiers.add(skipEmptyColumns); + identifiers.add(validateColumnTypes); } @Override public String toString() { return "ExcelConfiguration[columnNameLineNumber=" + columnNameLineNumber + ", skipEmptyLines=" + skipEmptyLines - + ", skipEmptyColumns=" + skipEmptyColumns + "]"; + + ", skipEmptyColumns=" + skipEmptyColumns +", validateColumnTypes=" + + validateColumnTypes + "]"; } } diff --git a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java index 19773ad08..eecf3ce32 100644 --- a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java +++ b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java @@ -402,9 +402,9 @@ public void testMissingValues() throws Exception { assertEquals(2, schema.getTableCount()); Table table = schema.getTables().get(0); - assertEquals("[Column[name=a,columnNumber=0,type=STRING,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=b,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=c,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null], " + assertEquals("[Column[name=a,columnNumber=0,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=b,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=c,columnNumber=2,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " + "Column[name=d,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]", Arrays.toString(table.getColumns().toArray())); @@ -419,6 +419,20 @@ public void testMissingValues() throws Exception { assertFalse(ds.next()); } + public void testDifferentDataTypes() throws Exception { + DataContext dc = new ExcelDataContext(copyOf("src/test/resources/different_datatypes.xls"), + new ExcelConfiguration(ExcelConfiguration.DEFAULT_COLUMN_NAME_LINE, true, false, true)); + + Schema schema = dc.getDefaultSchema(); + assertEquals(2, schema.getTableCount()); + + Table table = schema.getTables().get(0); + assertEquals("[Column[name=INTEGER,columnNumber=0,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=TEXT,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=FORMULA,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null]]", + Arrays.toString(table.getColumns().toArray())); + } + public void testMissingColumnHeader() throws Exception { File file = copyOf("src/test/resources/xls_missing_column_header.xls"); DataContext dc = new ExcelDataContext(file); @@ -426,10 +440,10 @@ public void testMissingColumnHeader() throws Exception { assertEquals(2, schema.getTableCount()); Table table = schema.getTables().get(0); - assertEquals("[Column[name=a,columnNumber=0,type=STRING,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=b,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=A,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=d,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]", + assertEquals("[Column[name=a,columnNumber=0,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=b,columnNumber=1,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=A,columnNumber=2,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=d,columnNumber=3,type=INTEGER,nullable=true,nativeType=null,columnSize=null]]", Arrays.toString(table.getColumns().toArray())); Query q = new Query().select(table.getColumns()).from(table); @@ -532,7 +546,7 @@ public void testTicket99defect() throws Exception { Table table = schema.getTableByName("Sheet1"); assertEquals( - "[Column[name=Pkg No.,columnNumber=0,type=STRING,nullable=true,nativeType=null,columnSize=null], " + "[Column[name=Pkg No.,columnNumber=0,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " + "Column[name=Description,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], " + "Column[name=Room,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null], " + "Column[name=Level,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]", diff --git a/excel/src/test/resources/different_datatypes.xls b/excel/src/test/resources/different_datatypes.xls new file mode 100644 index 0000000000000000000000000000000000000000..684b4853e9256e1f5c48d7d740b71583850e6117 GIT binary patch literal 27136 zcmeHQ2Urxzv+rFNSd zDk_SRLs3*z&{NNV>GV)fJ(crL-B&fkvb(bZc<=wd_ult*_RZVL`{s&f@4?Vxh{e00s)>1e_h}hNXqU1kOefjP?2h?Sqa?sBsCYgWm%jJR{^Wq;X^_(I#Y2L?X{GDUt`j)A@L6O3O?e~;^o=30B!$x2 zp2dRPC()d8%Ef6m$FE>7#TaNky(XRHS9+D+-CcTJ=I*vGq zT`}c&L-6k;k8LaoC44f8j8_2QL)1xU*#M3pTak47x+!NM6x~8GMDv;m!BR;#;zd~D zU;$C7yGhssuk|9*IV5c(FP8;a(1PNoF?^|GQ;s9Cg+CI`>&CORx3qK)=k>zlaSq~V z77i8^-8YWL1kyKc5URrG*XfYx*q3RyQ4&sMMbw_80_96llz{f{e zgh#{>ypoWQ0v`2C@6Pa)bhO03-wrHr+@_69)8& z4!xU9x&dHP_#YbK8{1bKA-&1}X+KmK(rd_2I5ZQsN9g*}I5gczNY@b2d4yzkkfJl> zY6+l2;AXV@9X zaI)-UD5TdwCzteTvV*>mE>^BNTthMk4b-&tA&Dc$tx4xPkaP(;W?&O7i2o{Z#DXNe zWoSb`ge}d0Sz5(FcVb&10TpSL1Ws!iFr>Ch0@g{Z7|>~3#ekL4Dh71(Rxx1vY83+p zlvXid%V-q?HoaCcU|VVx0|viVG1#@1!M?Q&4y|QyY%N32)-v>xV8He#9t2qF+`f#c z0ewt}7s58pWMFd}!wEJ_9(3b*FpiN1!3n5lK?FXEL2!nvSr8XjNf2fbbOR8Pmq-;y zT6O0psTkcL9t*?`#^ylz@j}HPSD`~KulzO=Ao)a1Hi+PYA`nkWkWq3p5!C04X|jq@ zo$dglPE{vr?c*~;jwV$bn?TeXiCG~sxqjTjBGQ(HQh zsZ0RVaiN$}f2Yeaw=AXpeU@Uj zMh{SmZUbn2DW!zfBFJPlSxUNO-I_{<{*RkXmo6wMXaW)Xz`xJj%!cU=YS0Z6O9u5K z=>yWG@Pcf#(_ks2x--ItDG*dim7=(~SjuUT#&jB_F`Wi!AdlLlyG%|%7nK72K*F@? z+!2iS*c%u)P2p9l6lixWo_J6b54pnjC>^&ffJ+oD;Y7NUnZVK@*fAD?NNI;QRw4Xo zaqXl*pu65YNL7$xkWMWDVFrI@>#>voffLy_!9Jt>SdN3K7T82FNc)z6R0Syp>D&?! z7gxm~RuUj^BHJd|EtYQ+9GJAgCW=8|_|`l~Rghv3)0Tj^xGDzeA^`#?vTcGrfB81S z!EXy}q8P-uB_LHnia}uf*t{k#u8KjdB|zXrwoPzwBi|-Cux)`&6obI>MMe!3;y619(NiLeMabs~z1zN-f9xQYLLpVU{OQPV9PZFu8U}yjq(In~|e)OHX z_X1TsW>L(7E;u)VV)U3|T%^TVC=f%BK`1e*{OB)FZ^;&;&lKY-Eyh}b753LeD-HD`c|dl4b8)4R(CwGx4X|1d$j zFhIq9h7g=fpzi8SO}mId!CRQ1whT~l4}l_|A+kZOMWEmtOi%{~sJK56GB&SPHfUE7 zD0l)Bv^xV-y!Hv1x;Ic3D3{p4-Ie%7Ov>f9)#&uy81I*$3M;kjv#s zs-mg_MfV4jOo6`$P=FB3YoDnGd2JYZ)nE;#C@+tUg*(CcCLV`O4)9nrcsK$s@T9_u zwPdRA8%1U{aX4nG1BWAnLtUUL#UUxH9dUvlI0;Cbnh=o~k&=`=ixL}ol zKrmVo!&PxcT!;n2gjm|JY6V6R1Nae>;u2$cFjinFs-g>)9RLcU@5h0nF`yv*6awMY zp~3VEn@q!@E2@KLbRb;K)`mfb7&yc}Cb*sqa610U0T=jeJyr(=L2y@baEO6SaD5rz zbR3iePInF16u6r>IK)CmKu|G|0gmyIt}zL-tY)_%?og3rVB*wNaD`OKDq|Z$c?N=I zr-CnHrHh)s0OtLLm4*)o{)^3D7dl7~mxAIFs+y7vOG94f;;gZNiX#KcwO*{gq zGzN%%tHVeOPUNmg@J`0(NOUe9+n8p zn-5P^8XX>7Dw9BSbd}VVML^^b@bqLVKp<43^;vKT(YdL;cwu`Pl9a;FV z(seE#@-ae6)HxDFn$TxLP3SYBCiEEw%$}L5U^^>G6;xHBB8EbllbZ(qXfkxwUg)Y} zaVZI4t?rUsDmqX-Vc;DpP;qE4X>VsCL~jZrDFr=%G54TiOhKT*>DuQ|P9-tFfOZ$g z5xTU(GjuUM;thd^jEYE3#E?aYS0715*=-?Gj{*tF;K;(LB`d7rO`Nju3hrpKLTr*G z2TG?0e_;R;NtJdus62~03Wl;{FiZu%a{(*Hkr|W+iE@Z9D)t}~S@;lT0hDSkKJW$4 z7#_>#r*z-M?Lan($)w$uetNPE7MyPryM}zrz>Qh34(BY33yBc--`V5~XkbKPV&EVH zhlB+L284pUhXsrd1493h(BKh4eL;b~5wP=%gq?qs0J;HOQUDLQ!IC73YXk`;&>Q06 z#wP2yqcqovUC~;`CPN3Sa15agNH!-gALyJ0Ba%*lYXHAv{gVAHhd^Ob<)Oj zQRcwxO`!#|pHGbmD)wHHlWdl={l?#3?91*mtTVr6YF|km_G^dw>{VV*D*bOiQz<#n zd9?SNXP-A%JftjVl7PWnbhA4xuDkO1UGB@9}T;_zwKN4xo*vGJfls!)y_Zm z_qiNfH^cjM#=Ob}+jn;1j!m1oaNF#AN5W4Iy}qsYmF^}514^9N>gz1N6LKAFsw944IYFhYhvkgBQKk^8=`Ozr!w&%osD+?vv@Y9XW_ijIXGT-y+L&Ms< z!CGVN^VPMM@UJ*7KKgc!{Z5m@y8}A?Y@bkfq22SBX&ovLFE@BN@Z$7g>6W!Q3-%tl z&%1TRuGahy`}-4)TTS*}8aV33%?8`Y0rhs)ZP;G2l^=3r%Jpjyzz>)as&$E}O&6g% zs{Lxr;3N}I-BayfWQ=}Qc5qU?p7AP``gUCpCRm1sJ{>Ty#?j$Ta^KJyDR#NTomAUw z4{%cbSN#^7jz6(Fez;*4#p|2*ZL#+88;-Wa-l_CoWmH1^2dBG!@pA3u+H3Z2Zw@W~ zSJw?~&pj>;KRM#9cPE`Yi_81o&-nF^IWz7xWc7Q;9cJ{T)aU%U(z@hiRQE{ z@0^uCd+h#a%bLnh6$?9@bJ0IKdS29DQ>xOktj4{c7*&4V_QmoS713LU-dU`ho$l0M zuh6^G%1YhY=><0)jv^2O3*b%|00)yxC{2 zb^1yR^SVW);nf-6Dz5&Zb=bT8*5u`T%x_=Mn11~Dnb_i84NuSV?i~*6w!YhmjdQhr zN^W<1Uy0r8K|4ZJs;6?h1|R>WXHnjj8=c}aZ|v8;zjtzcQEZ*o*Rgi_GcW%hRp)L~ z8~)wdnLUFKY+Z2sC&TawXV;q?>3{jw{=T-Sis$V&nlZLucR~1kzd?^H+msI7m*C!h z=M`R5|Azdx-yN^nm3VdSZ#Ih;+LzWCMHyXAJ!rHgs-hPe>bt}$+q=i$w;i<1m*g$Z zWp#bfN5hYMlJj%WndFE=)wT&|Gd|kBOuD-**2L2PyLW8sNdCZg@5*yl_q+8~{ij+_ zUed#pJ!XIV`_<>97^eyO-U9xqqa*p1ueW$PKYJkf>*K?R8FgPj*6aKcK6iCu*|PT5 z-|k-vKlRVol{&i?8T_^*?Vs00Gxvv=eSK5plku;Y)u%3BKa7wzE8*_tfsqdXR(b1OMUK7V9tdSyCAHHr>jF<{4tSfAY5V70OZ%+B z&vVYt(%R{hrTJvW?DbbIOe(u6%U^Lqpha3ZnVd>fj(UGqE`{4CVKnM*t5I7L)+NZVc~~1 zSu|uH@GDuQQnD~)X33hgXqy$f;X6N*lZkgvYK^$Magff-sgZ0Q?%@nwry<)qwW*Ha zG<>%=(7br^`ck7eX+KQ6oYDUHg-54)m&A^Hbc$mdLAfz`r(*CjO7`t zto^&I&Z-$XJ{oX2Ex+{SzH{5~?R=kn5oHA8wX z4REgwvMcg+33pm^D0M?}rhe{mjs=f%-YfQ6;H8jVMeQHdUOhP^ zWWVc&Z(>E!pM_|(2< zjz?bD4uA2LcR0CY?LD)|X(Pht|C*fm)4`3lv9Er(_4U$7W54Ldxure}v%Ak(uD`hR zmLE%XwH_V0JoNpCg{KEa4?LFRV0EtMQO}p#@9-@2r?1Q{d{gu!1^}*_ZyGzzK>u9e z;C*Uw&!Q~VhaCN3*@dXyYj)gQ-j4tBgTBk0O?LUGC%-G2zt$>m-ig_DFKSGxZ8NfX z<(F!=x;$FUO`A3}Xm1aTx7PN9C(Zie&~C@9yYAruFFOpntrzU)w%;b&C8VzB+L)9N zBcfHFOr9Ic=^m;bU-ag)-=*aKH>%=ytQ`?nn-bGKeo=t& z=;CL!OHQ7h91(djE_y-8@P_0|dmWPRhi=Xpq2Zhs?GspA=G)h~@LArN8p9{GZhMkH z&8gkjd8x{e{;w?p&RzU_?Si{qPVn8&9q(*EVb^;wSk1W`Y1cvHON( z?XH7X=T|QY`<%JYYun0^Pc63X=zH-*^?-)d^q=14vF7}2RTil`yOtQQ8@fqw=&M)9 z>=-9Aqw(K{^__lafvU~cimFGJdAox+m&TTsmOpGNI*aW9%V*^EY9X80CZmr#bg(1< zvzsU4b_RQjb^4f$%x;s2(HQ!ZQvDDy^D~i{eHanaEm=rZkA{iCbOI5^9hle=*G7R$ z3>~bB3p`+)r%J{{>1%L7G|7VjPM+A%0xpncKwBn8!+e;Te*0Ee;_chEY(u z)^5G>Bk-62-&fUy{UoA89qYlKz^ojXeA)HxMsVa%pKyxajiTF8`*{>4lN!{}m|bA# zOo%<}d$@8I20VncgEXJYoki`fA$K&jJ@@G5ukkyc4#ThzLVIZrJm6rglYLcwz zjmX-8VpWq+7_tgkeN%u{U6K`@7Fjz{tT-hrQW&xdS#xFsD_4>geH&SM6f0LkVaO_E zCG&t)Ly{Gn0J2(8tQryuLslW{#f88M6)SEgY$?cUMX{pih!jR2z-zGamH}&9Nmgu7 zc-fj_Z7ZQLWEHYj72{=lid9=eVf+X5sG{mZx@ZM+cyKN zt|TkQ1ib7_vFb`F3|WP&WGk@3Jx5$Bh7M%yPO<7qD2%tKMya9OffYX3AkK;r1~0o% ztojlP_jrHu!;Hh0o4a z`q6iO(S`rnqi-m{mj{k>3!l{tPDlon80KxqM`1(X(0T0m(5r3I80P+CA~0i^~0|5*S> z#5iEaF*lCC>19v!m>Qqy<5(Y8;NV~y$Lu(M7p_vlF+MH;!Ds$B&KIsF#IZik2jG{s z@XJ#8B_;fF5WX#g9}~mDxfvuLBy&jk|6?rS*9uY>NY;?LLb8Em3kk2=!LL0e2S|>P zoFF+v!Z&vD`d3K!tK3>=flzqC58qkj!Asltl_Y#oyEU5;QKiI<%BA&7aPxN`_OfJK~%8Say70IhxW5 zDWXpo$iuY(`PhcA{j^lR1%hGYflU=wA|=7As`$z!{hg}T>ErdED1SE;+$jG)0Dm?L A82|tP literal 0 HcmV?d00001 From 84c6f0654a750b264ff3cf377adc348a3cc121ca Mon Sep 17 00:00:00 2001 From: Jan Bob Date: Tue, 1 Oct 2019 13:16:04 +0200 Subject: [PATCH 10/22] fixed another test that fel over --- .../java/org/apache/metamodel/excel/ExcelConfigurationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/excel/src/test/java/org/apache/metamodel/excel/ExcelConfigurationTest.java b/excel/src/test/java/org/apache/metamodel/excel/ExcelConfigurationTest.java index 6e7559cbf..647c11196 100644 --- a/excel/src/test/java/org/apache/metamodel/excel/ExcelConfigurationTest.java +++ b/excel/src/test/java/org/apache/metamodel/excel/ExcelConfigurationTest.java @@ -27,7 +27,7 @@ public class ExcelConfigurationTest extends TestCase { public void testToString() throws Exception { ExcelConfiguration conf = new ExcelConfiguration(1, true, false); assertEquals( - "ExcelConfiguration[columnNameLineNumber=1, skipEmptyLines=true, skipEmptyColumns=false]", + "ExcelConfiguration[columnNameLineNumber=1, skipEmptyLines=true, skipEmptyColumns=false, validateColumnTypes=false]", conf.toString()); } From b4d9a201a308fd51be221e8967f4308d0d252462 Mon Sep 17 00:00:00 2001 From: Jan Bob Date: Wed, 2 Oct 2019 16:33:04 +0200 Subject: [PATCH 11/22] commit part 1; did some indentation fixes switching devices --- .../metamodel/excel/DefaultSpreadsheetReaderDelegate.java | 7 ++++--- .../org/apache/metamodel/excel/ExcelConfiguration.java | 2 +- .../org/apache/metamodel/excel/ExcelDataContextTest.java | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index bfe7396d1..d9cb294ee 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -65,13 +65,14 @@ public DefaultSpreadsheetReaderDelegate(Resource resource, ExcelConfiguration co _configuration = configuration; } + @Override public Schema createSchema(String schemaName) { final MutableSchema schema = new MutableSchema(schemaName); final Workbook wb = ExcelUtils.readWorkbook(_resource, true); try { for (int i = 0; i < wb.getNumberOfSheets(); i++) { final Sheet currentSheet = wb.getSheetAt(i); - final MutableTable table = createTable(wb, currentSheet, _configuration.isValidateColumnTypes()); + final MutableTable table = createTable(wb, currentSheet); table.setSchema(schema); schema.addTable(table); } @@ -104,7 +105,7 @@ public void notifyTablesModified() { // do nothing } - private MutableTable createTable(final Workbook wb, final Sheet sheet, boolean validateColumnTypes) { + private MutableTable createTable(final Workbook wb, final Sheet sheet) { final MutableTable table = new MutableTable(sheet.getSheetName(), TableType.TABLE); if (sheet.getPhysicalNumberOfRows() <= 0) { @@ -153,7 +154,7 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet, boolean v for (int j = offset; j < row.getLastCellNum(); j++) { final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); final Column column; - if (validateColumnTypes) { + if (_configuration.isValidateColumnTypes()) { column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), columnTypes[j], diff --git a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java index 1cd8286b6..db9ff0218 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java +++ b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java @@ -129,7 +129,7 @@ protected void decorateIdentity(List identifiers) { identifiers.add(columnNameLineNumber); identifiers.add(skipEmptyLines); identifiers.add(skipEmptyColumns); - identifiers.add(validateColumnTypes); + identifiers.add(validateColumnTypes); } @Override diff --git a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java index eecf3ce32..a7cb8bc28 100644 --- a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java +++ b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java @@ -402,7 +402,7 @@ public void testMissingValues() throws Exception { assertEquals(2, schema.getTableCount()); Table table = schema.getTables().get(0); - assertEquals("[Column[name=a,columnNumber=0,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " + assertEquals("[Column[name=a,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " + "Column[name=b,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " + "Column[name=c,columnNumber=2,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " + "Column[name=d,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]", From 1a342856dd4762de450bc79bf731bd2346aebde9 Mon Sep 17 00:00:00 2001 From: Jan Bob Date: Wed, 2 Oct 2019 16:35:20 +0200 Subject: [PATCH 12/22] commit part 1.01; did some indentation fixes switching devices and apearantly autosave was off -_- --- .../org/apache/metamodel/excel/ExcelConfiguration.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java index db9ff0218..99f317736 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java +++ b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java @@ -43,7 +43,7 @@ public final class ExcelConfiguration extends BaseObject implements private final ColumnNamingStrategy columnNamingStrategy; private final boolean skipEmptyLines; private final boolean skipEmptyColumns; - private final boolean validateColumnTypes; + private final boolean detectColumnTypes; public ExcelConfiguration() { this(DEFAULT_COLUMN_NAME_LINE, true, false); @@ -68,7 +68,7 @@ public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnN this.skipEmptyLines = skipEmptyLines; this.skipEmptyColumns = skipEmptyColumns; this.columnNamingStrategy = columnNamingStrategy; - this.validateColumnTypes = validateColumnTypes; + this.detectColumnTypes = validateColumnTypes; } /** @@ -121,7 +121,7 @@ public boolean isSkipEmptyColumns() { * @return a boolean indicating whether or not to validate column types. */ public boolean isValidateColumnTypes() { - return validateColumnTypes; + return detectColumnTypes; } @Override @@ -129,7 +129,7 @@ protected void decorateIdentity(List identifiers) { identifiers.add(columnNameLineNumber); identifiers.add(skipEmptyLines); identifiers.add(skipEmptyColumns); - identifiers.add(validateColumnTypes); + identifiers.add(detectColumnTypes); } @Override @@ -137,6 +137,6 @@ public String toString() { return "ExcelConfiguration[columnNameLineNumber=" + columnNameLineNumber + ", skipEmptyLines=" + skipEmptyLines + ", skipEmptyColumns=" + skipEmptyColumns +", validateColumnTypes=" - + validateColumnTypes + "]"; + + detectColumnTypes + "]"; } } From 3785c91ff735d49c1eb00a421092957e75a6e0d3 Mon Sep 17 00:00:00 2001 From: Jan Bob Date: Thu, 3 Oct 2019 10:53:08 +0200 Subject: [PATCH 13/22] resolving review comments --- .../DefaultSpreadsheetReaderDelegate.java | 490 +++++++++--------- .../metamodel/excel/ExcelConfiguration.java | 15 +- .../metamodel/excel/ExcelDataContextTest.java | 20 +- 3 files changed, 264 insertions(+), 261 deletions(-) diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index d9cb294ee..2393dfb34 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -55,255 +55,253 @@ */ final class DefaultSpreadsheetReaderDelegate implements SpreadsheetReaderDelegate { - private static final Logger logger = LoggerFactory.getLogger(DefaultSpreadsheetReaderDelegate.class); - - private final Resource _resource; - private final ExcelConfiguration _configuration; - - public DefaultSpreadsheetReaderDelegate(Resource resource, ExcelConfiguration configuration) { - _resource = resource; - _configuration = configuration; - } - - @Override - public Schema createSchema(String schemaName) { - final MutableSchema schema = new MutableSchema(schemaName); - final Workbook wb = ExcelUtils.readWorkbook(_resource, true); - try { - for (int i = 0; i < wb.getNumberOfSheets(); i++) { - final Sheet currentSheet = wb.getSheetAt(i); - final MutableTable table = createTable(wb, currentSheet); - table.setSchema(schema); - schema.addTable(table); - } - - return schema; - } finally { - FileHelper.safeClose(wb); - } - } - - @Override - public DataSet executeQuery(Table table, List columns, int maxRows) { - final Workbook wb = ExcelUtils.readWorkbook(_resource, true); - final Sheet sheet = wb.getSheet(table.getName()); - - if (sheet == null || sheet.getPhysicalNumberOfRows() == 0) { - return new EmptyDataSet(columns.stream().map(SelectItem::new).collect(Collectors.toList())); - } - - DataSet dataSet = ExcelUtils.getDataSet(wb, sheet, table, _configuration); - - if (maxRows > 0) { - dataSet = new MaxRowsDataSet(dataSet, maxRows); - } - return dataSet; - } - - @Override - public void notifyTablesModified() { - // do nothing - } - - private MutableTable createTable(final Workbook wb, final Sheet sheet) { - final MutableTable table = new MutableTable(sheet.getSheetName(), TableType.TABLE); - - if (sheet.getPhysicalNumberOfRows() <= 0) { - // no physical rows in sheet - return table; - } - - final Iterator rowIterator = ExcelUtils.getRowIterator(sheet, _configuration, false); - - if (!rowIterator.hasNext()) { - // no physical rows in sheet - return table; - } - - Row row = null; - - if (_configuration.isSkipEmptyLines()) { - while (row == null && rowIterator.hasNext()) { - row = rowIterator.next(); - } - - } else { - row = rowIterator.next(); - } - - final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); - final ColumnType[] columnTypes = getColumnTypes(sheet, row); - - if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) { - - // get to the first non-empty line (no matter if lines are skipped - // or not we need to read ahead to figure out how many columns there - // are!) - while (row == null && rowIterator.hasNext()) { - row = rowIterator.next(); - } - - // build columns without any intrinsic column names - final ColumnNamingStrategy columnNamingStrategy = _configuration.getColumnNamingStrategy(); - try (final ColumnNamingSession columnNamingSession = columnNamingStrategy.startColumnNamingSession()) { - final int offset = getColumnOffset(row); - for (int i = 0; i < offset; i++) { - columnNamingSession.getNextColumnName(new ColumnNamingContextImpl(i)); - } - - for (int j = offset; j < row.getLastCellNum(); j++) { - final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); - final Column column; - if (_configuration.isValidateColumnTypes()) { - - column = - new MutableColumn(columnNamingSession.getNextColumnName(namingContext), columnTypes[j], - table, j, true); - } else { - column = - new MutableColumn(columnNamingSession.getNextColumnName(namingContext), - ColumnType.STRING, table, j, true); - } - table.addColumn(column); - } - } - - } else { - - boolean hasColumns = true; - - // iterate to the column name line number (if above 1) - for (int j = 1; j < columnNameLineNumber; j++) { - if (rowIterator.hasNext()) { - row = rowIterator.next(); - } else { - hasColumns = false; - break; - } - } - - if (hasColumns) { - createColumns(table, wb, row, columnTypes); - } - } - - return table; - } - - private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) { - final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); - final int rowLength = row.getLastCellNum(); - int eagerness = ExcelConfiguration.EAGER_READ; - final ColumnType[] columnTypes = new ColumnType[rowLength]; - - while (data.hasNext() && eagerness-- > 0) { - final Row currentRow = data.next(); - if (currentRow.getRowNum() < _configuration.getColumnNameLineNumber()) { - continue; - } - for (int index = 0; index < rowLength; index++) { - if (currentRow.getLastCellNum() == 0) { - continue; - } - columnTypes[index] = getColumnTypeFromRow(columnTypes[index], currentRow, index); - } - } - return columnTypes; - } - - private ColumnType getColumnTypeFromRow(final ColumnType columnType, final Row currentRow, int index) { + private static final Logger logger = LoggerFactory.getLogger(DefaultSpreadsheetReaderDelegate.class); + + private final Resource _resource; + private final ExcelConfiguration _configuration; + + public DefaultSpreadsheetReaderDelegate(Resource resource, ExcelConfiguration configuration) { + _resource = resource; + _configuration = configuration; + } + + @Override + public Schema createSchema(String schemaName) { + final MutableSchema schema = new MutableSchema(schemaName); + final Workbook wb = ExcelUtils.readWorkbook(_resource, true); + try { + for (int i = 0; i < wb.getNumberOfSheets(); i++) { + final Sheet currentSheet = wb.getSheetAt(i); + final MutableTable table = createTable(wb, currentSheet); + table.setSchema(schema); + schema.addTable(table); + } + + return schema; + } finally { + FileHelper.safeClose(wb); + } + } + + @Override + public DataSet executeQuery(Table table, List columns, int maxRows) { + final Workbook wb = ExcelUtils.readWorkbook(_resource, true); + final Sheet sheet = wb.getSheet(table.getName()); + + if (sheet == null || sheet.getPhysicalNumberOfRows() == 0) { + return new EmptyDataSet(columns.stream().map(SelectItem::new).collect(Collectors.toList())); + } + + DataSet dataSet = ExcelUtils.getDataSet(wb, sheet, table, _configuration); + + if (maxRows > 0) { + dataSet = new MaxRowsDataSet(dataSet, maxRows); + } + return dataSet; + } + + @Override + public void notifyTablesModified() { + // do nothing + } + + private MutableTable createTable(final Workbook wb, final Sheet sheet) { + final MutableTable table = new MutableTable(sheet.getSheetName(), TableType.TABLE); + + if (sheet.getPhysicalNumberOfRows() <= 0) { + // no physical rows in sheet + return table; + } + + final Iterator rowIterator = ExcelUtils.getRowIterator(sheet, _configuration, false); + + if (!rowIterator.hasNext()) { + // no physical rows in sheet + return table; + } + + Row row = null; + + if (_configuration.isSkipEmptyLines()) { + while (row == null && rowIterator.hasNext()) { + row = rowIterator.next(); + } + + } else { + row = rowIterator.next(); + } + + final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); + final ColumnType[] columnTypes = getColumnTypes(sheet, row); + + if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) { + + // get to the first non-empty line (no matter if lines are skipped + // or not we need to read ahead to figure out how many columns there + // are!) + while (row == null && rowIterator.hasNext()) { + row = rowIterator.next(); + } + + // build columns without any intrinsic column names + final ColumnNamingStrategy columnNamingStrategy = _configuration.getColumnNamingStrategy(); + try (final ColumnNamingSession columnNamingSession = columnNamingStrategy.startColumnNamingSession()) { + final int offset = getColumnOffset(row); + for (int i = 0; i < offset; i++) { + columnNamingSession.getNextColumnName(new ColumnNamingContextImpl(i)); + } + + for (int j = offset; j < row.getLastCellNum(); j++) { + final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); + final Column column; + if (_configuration.isValidateColumnTypes()) { + + column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), columnTypes[j], + table, j, true); + } else { + column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), + ColumnType.STRING, table, j, true); + } + table.addColumn(column); + } + } + + } else { + + boolean hasColumns = true; + + // iterate to the column name line number (if above 1) + for (int j = 1; j < columnNameLineNumber; j++) { + if (rowIterator.hasNext()) { + row = rowIterator.next(); + } else { + hasColumns = false; + break; + } + } + + if (hasColumns) { + createColumns(table, wb, row, columnTypes); + } + } + + return table; + } + + private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) { + final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); + final int rowLength = row.getLastCellNum(); + int eagerness = _configuration.getEagerness(); + final ColumnType[] columnTypes = new ColumnType[rowLength]; + + while (data.hasNext() && eagerness-- > 0) { + final Row currentRow = data.next(); + if (currentRow.getRowNum() < _configuration.getColumnNameLineNumber()) { + continue; + } + for (int index = 0; index < rowLength; index++) { + if (currentRow.getLastCellNum() == 0) { + continue; + } + + ColumnType columnType = columnTypes[index]; + ColumnType expecetedColumnType = getColumnTypeFromRow(currentRow, index); + if (columnType != null) { + if (!columnType.equals(ColumnType.STRING) && !columnType.equals(expecetedColumnType)) { + columnTypes[index] = ColumnType.VARCHAR; + } + } else { + columnTypes[index] = expecetedColumnType; + } + } + } + return columnTypes; + } + + private ColumnType getColumnTypeFromRow(final Row currentRow, int index) { if (currentRow.getCell(index) == null) { - return checkColumnType(ColumnType.STRING, columnType); + return ColumnType.STRING; } else { - CellType cellType = currentRow.getCell(index).getCellType(); - switch (cellType) { - case NUMERIC: - if (DateUtil.isCellDateFormatted(currentRow.getCell(index))) { - return checkColumnType(ColumnType.DATE, columnType); - } else { - return checkColumnType((currentRow.getCell(index).getNumericCellValue() % 1 == 0) - ? ColumnType.INTEGER : ColumnType.DOUBLE, columnType); - } - case BOOLEAN: - return checkColumnType(ColumnType.BOOLEAN, columnType); - case ERROR: - // fall through - case _NONE: - // fall through - case STRING: - // fall through - case FORMULA: - // fall through - case BLANK: - // fall through - default : - return checkColumnType(ColumnType.STRING, columnType); - } + CellType cellType = currentRow.getCell(index).getCellType(); + switch (cellType) { + case NUMERIC: + if (DateUtil.isCellDateFormatted(currentRow.getCell(index))) { + return ColumnType.DATE; + } else { + return (currentRow.getCell(index).getNumericCellValue() % 1 == 0) + ? ColumnType.INTEGER : ColumnType.DOUBLE; + } + case BOOLEAN: + return ColumnType.BOOLEAN; + case ERROR: + // fall through + case _NONE: + // fall through + case STRING: + // fall through + case FORMULA: + // fall through + case BLANK: + // fall through + default: + return ColumnType.STRING; + } } } - private ColumnType checkColumnType(final ColumnType expecetedColumnType, ColumnType columnType) { - if (columnType != null) { - if (!columnType.equals(ColumnType.STRING) && !columnType.equals(expecetedColumnType)) { - return ColumnType.VARCHAR; - } - } else { - return expecetedColumnType; - } - return columnType; - } - - /** - * Builds columns based on row/cell values. - * - * @param table - * @param wb - * @param row - */ - private void createColumns(final MutableTable table, final Workbook wb, final Row row, final ColumnType[] columTypes) { - if (row == null) { - logger.warn("Cannot create columns based on null row!"); - return; - } - final short rowLength = row.getLastCellNum(); - - final int offset = getColumnOffset(row); - - // build columns based on cell values. - try (final ColumnNamingSession columnNamingSession = _configuration.getColumnNamingStrategy() - .startColumnNamingSession()) { - for (int j = offset; j < rowLength; j++) { - final Cell cell = row.getCell(j); - final String intrinsicColumnName = ExcelUtils.getCellValue(wb, cell); - final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName, - j); - final String columnName = columnNamingSession.getNextColumnName(columnNamingContext); - final Column column; - if (columTypes == null) { - column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true); - } else { - column = new MutableColumn(columnName, columTypes[j], table, j, true); - } - table.addColumn(column); - } - } - } - - /** - * Gets the column offset (first column to include). This is dependent on - * the row used for column processing and whether the skip empty columns - * property is set. - * - * @param row - * @return - */ - private int getColumnOffset(Row row) { - final int offset; - if (_configuration.isSkipEmptyColumns()) { - offset = row.getFirstCellNum(); - } else { - offset = 0; - } - return offset; - } + /** + * Builds columns based on row/cell values. + * + * @param table + * @param wb + * @param row + */ + private void createColumns(final MutableTable table, final Workbook wb, final Row row, + final ColumnType[] columTypes) { + if (row == null) { + logger.warn("Cannot create columns based on null row!"); + return; + } + final short rowLength = row.getLastCellNum(); + + final int offset = getColumnOffset(row); + + // build columns based on cell values. + try (final ColumnNamingSession columnNamingSession = _configuration + .getColumnNamingStrategy() + .startColumnNamingSession()) { + for (int j = offset; j < rowLength; j++) { + final Cell cell = row.getCell(j); + final String intrinsicColumnName = ExcelUtils.getCellValue(wb, cell); + final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName, + j); + final String columnName = columnNamingSession.getNextColumnName(columnNamingContext); + final Column column; + if (!_configuration.isValidateColumnTypes()) { + column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true); + } else { + column = new MutableColumn(columnName, columTypes[j], table, j, true); + } + table.addColumn(column); + } + } + } + + /** + * Gets the column offset (first column to include). This is dependent on + * the row used for column processing and whether the skip empty columns + * property is set. + * + * @param row + * @return + */ + private int getColumnOffset(Row row) { + final int offset; + if (_configuration.isSkipEmptyColumns()) { + offset = row.getFirstCellNum(); + } else { + offset = 0; + } + return offset; + } } diff --git a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java index 99f317736..d94caac73 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java +++ b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java @@ -37,8 +37,8 @@ public final class ExcelConfiguration extends BaseObject implements public static final int NO_COLUMN_NAME_LINE = 0; public static final int DEFAULT_COLUMN_NAME_LINE = 1; - public static final int EAGER_READ = 1000; + private final int eagerReader; private final int columnNameLineNumber; private final ColumnNamingStrategy columnNamingStrategy; private final boolean skipEmptyLines; @@ -50,25 +50,26 @@ public ExcelConfiguration() { } public ExcelConfiguration(int columnNameLineNumber, boolean skipEmptyLines, boolean skipEmptyColumns) { - this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, false); + this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, false, 1000); } public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnNamingStrategy, Boolean skipEmptyLines, Boolean skipEmptyColumns) { - this(columnNameLineNumber, columnNamingStrategy, skipEmptyLines, skipEmptyColumns, false); + this(columnNameLineNumber, columnNamingStrategy, skipEmptyLines, skipEmptyColumns, false, 1000); } public ExcelConfiguration(int columnNameLineNumber, boolean skipEmptyLines, boolean skipEmptyColumns, boolean validateColumnTypes) { - this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, validateColumnTypes); + this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, validateColumnTypes, 1000); } public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnNamingStrategy, - boolean skipEmptyLines, boolean skipEmptyColumns, boolean validateColumnTypes) { + boolean skipEmptyLines, boolean skipEmptyColumns, boolean validateColumnTypes, int eagerness) { this.columnNameLineNumber = columnNameLineNumber; this.skipEmptyLines = skipEmptyLines; this.skipEmptyColumns = skipEmptyColumns; this.columnNamingStrategy = columnNamingStrategy; this.detectColumnTypes = validateColumnTypes; + this.eagerReader = eagerness; } /** @@ -139,4 +140,8 @@ public String toString() { + ", skipEmptyColumns=" + skipEmptyColumns +", validateColumnTypes=" + detectColumnTypes + "]"; } + + public int getEagerness() { + return eagerReader; + } } diff --git a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java index a7cb8bc28..cda4bc3b9 100644 --- a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java +++ b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java @@ -404,8 +404,8 @@ public void testMissingValues() throws Exception { Table table = schema.getTables().get(0); assertEquals("[Column[name=a,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " + "Column[name=b,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=c,columnNumber=2,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=d,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]", + + "Column[name=c,columnNumber=2,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=d,columnNumber=3,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]]", Arrays.toString(table.getColumns().toArray())); Query q = new Query().select(table.getColumns()).from(table); @@ -440,10 +440,10 @@ public void testMissingColumnHeader() throws Exception { assertEquals(2, schema.getTableCount()); Table table = schema.getTables().get(0); - assertEquals("[Column[name=a,columnNumber=0,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=b,columnNumber=1,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=A,columnNumber=2,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=d,columnNumber=3,type=INTEGER,nullable=true,nativeType=null,columnSize=null]]", + assertEquals("[Column[name=a,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=b,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=A,columnNumber=2,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=d,columnNumber=3,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]]", Arrays.toString(table.getColumns().toArray())); Query q = new Query().select(table.getColumns()).from(table); @@ -546,10 +546,10 @@ public void testTicket99defect() throws Exception { Table table = schema.getTableByName("Sheet1"); assertEquals( - "[Column[name=Pkg No.,columnNumber=0,type=INTEGER,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=Description,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=Room,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null], " - + "Column[name=Level,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]", + "[Column[name=Pkg No.,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=Description,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=Room,columnNumber=2,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " + + "Column[name=Level,columnNumber=3,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]]", Arrays.toString(table.getColumns().toArray())); } From a8ab4b6cff716c375ec0caf3e99516aef58c8cb7 Mon Sep 17 00:00:00 2001 From: Pixel Date: Fri, 4 Oct 2019 11:16:27 +0200 Subject: [PATCH 14/22] resolving indentation filler --- .../DefaultSpreadsheetReaderDelegate.java | 500 +++++++++--------- .../metamodel/excel/ExcelConfiguration.java | 4 - excel/src/test/resources/testDataTypes.xlsx | Bin 32656 -> 0 bytes 3 files changed, 251 insertions(+), 253 deletions(-) delete mode 100644 excel/src/test/resources/testDataTypes.xlsx diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index 2393dfb34..687acc672 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -55,253 +55,255 @@ */ final class DefaultSpreadsheetReaderDelegate implements SpreadsheetReaderDelegate { - private static final Logger logger = LoggerFactory.getLogger(DefaultSpreadsheetReaderDelegate.class); - - private final Resource _resource; - private final ExcelConfiguration _configuration; - - public DefaultSpreadsheetReaderDelegate(Resource resource, ExcelConfiguration configuration) { - _resource = resource; - _configuration = configuration; - } - - @Override - public Schema createSchema(String schemaName) { - final MutableSchema schema = new MutableSchema(schemaName); - final Workbook wb = ExcelUtils.readWorkbook(_resource, true); - try { - for (int i = 0; i < wb.getNumberOfSheets(); i++) { - final Sheet currentSheet = wb.getSheetAt(i); - final MutableTable table = createTable(wb, currentSheet); - table.setSchema(schema); - schema.addTable(table); - } - - return schema; - } finally { - FileHelper.safeClose(wb); - } - } - - @Override - public DataSet executeQuery(Table table, List columns, int maxRows) { - final Workbook wb = ExcelUtils.readWorkbook(_resource, true); - final Sheet sheet = wb.getSheet(table.getName()); - - if (sheet == null || sheet.getPhysicalNumberOfRows() == 0) { - return new EmptyDataSet(columns.stream().map(SelectItem::new).collect(Collectors.toList())); - } - - DataSet dataSet = ExcelUtils.getDataSet(wb, sheet, table, _configuration); - - if (maxRows > 0) { - dataSet = new MaxRowsDataSet(dataSet, maxRows); - } - return dataSet; - } - - @Override - public void notifyTablesModified() { - // do nothing - } - - private MutableTable createTable(final Workbook wb, final Sheet sheet) { - final MutableTable table = new MutableTable(sheet.getSheetName(), TableType.TABLE); - - if (sheet.getPhysicalNumberOfRows() <= 0) { - // no physical rows in sheet - return table; - } - - final Iterator rowIterator = ExcelUtils.getRowIterator(sheet, _configuration, false); - - if (!rowIterator.hasNext()) { - // no physical rows in sheet - return table; - } - - Row row = null; - - if (_configuration.isSkipEmptyLines()) { - while (row == null && rowIterator.hasNext()) { - row = rowIterator.next(); - } - - } else { - row = rowIterator.next(); - } - - final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); - final ColumnType[] columnTypes = getColumnTypes(sheet, row); - - if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) { - - // get to the first non-empty line (no matter if lines are skipped - // or not we need to read ahead to figure out how many columns there - // are!) - while (row == null && rowIterator.hasNext()) { - row = rowIterator.next(); - } - - // build columns without any intrinsic column names - final ColumnNamingStrategy columnNamingStrategy = _configuration.getColumnNamingStrategy(); - try (final ColumnNamingSession columnNamingSession = columnNamingStrategy.startColumnNamingSession()) { - final int offset = getColumnOffset(row); - for (int i = 0; i < offset; i++) { - columnNamingSession.getNextColumnName(new ColumnNamingContextImpl(i)); - } - - for (int j = offset; j < row.getLastCellNum(); j++) { - final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); - final Column column; - if (_configuration.isValidateColumnTypes()) { - - column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), columnTypes[j], - table, j, true); - } else { - column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), - ColumnType.STRING, table, j, true); - } - table.addColumn(column); - } - } - - } else { - - boolean hasColumns = true; - - // iterate to the column name line number (if above 1) - for (int j = 1; j < columnNameLineNumber; j++) { - if (rowIterator.hasNext()) { - row = rowIterator.next(); - } else { - hasColumns = false; - break; - } - } - - if (hasColumns) { - createColumns(table, wb, row, columnTypes); - } - } - - return table; - } - - private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) { - final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); - final int rowLength = row.getLastCellNum(); - int eagerness = _configuration.getEagerness(); - final ColumnType[] columnTypes = new ColumnType[rowLength]; - - while (data.hasNext() && eagerness-- > 0) { - final Row currentRow = data.next(); - if (currentRow.getRowNum() < _configuration.getColumnNameLineNumber()) { - continue; - } - for (int index = 0; index < rowLength; index++) { - if (currentRow.getLastCellNum() == 0) { - continue; - } - - ColumnType columnType = columnTypes[index]; - ColumnType expecetedColumnType = getColumnTypeFromRow(currentRow, index); - if (columnType != null) { - if (!columnType.equals(ColumnType.STRING) && !columnType.equals(expecetedColumnType)) { - columnTypes[index] = ColumnType.VARCHAR; - } - } else { - columnTypes[index] = expecetedColumnType; - } - } - } - return columnTypes; - } - - private ColumnType getColumnTypeFromRow(final Row currentRow, int index) { - if (currentRow.getCell(index) == null) { - return ColumnType.STRING; - } else { - CellType cellType = currentRow.getCell(index).getCellType(); - switch (cellType) { - case NUMERIC: - if (DateUtil.isCellDateFormatted(currentRow.getCell(index))) { - return ColumnType.DATE; - } else { - return (currentRow.getCell(index).getNumericCellValue() % 1 == 0) - ? ColumnType.INTEGER : ColumnType.DOUBLE; - } - case BOOLEAN: - return ColumnType.BOOLEAN; - case ERROR: - // fall through - case _NONE: - // fall through - case STRING: - // fall through - case FORMULA: - // fall through - case BLANK: - // fall through - default: - return ColumnType.STRING; - } - } - } - - /** - * Builds columns based on row/cell values. - * - * @param table - * @param wb - * @param row - */ - private void createColumns(final MutableTable table, final Workbook wb, final Row row, - final ColumnType[] columTypes) { - if (row == null) { - logger.warn("Cannot create columns based on null row!"); - return; - } - final short rowLength = row.getLastCellNum(); - - final int offset = getColumnOffset(row); - - // build columns based on cell values. - try (final ColumnNamingSession columnNamingSession = _configuration - .getColumnNamingStrategy() - .startColumnNamingSession()) { - for (int j = offset; j < rowLength; j++) { - final Cell cell = row.getCell(j); - final String intrinsicColumnName = ExcelUtils.getCellValue(wb, cell); - final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName, - j); - final String columnName = columnNamingSession.getNextColumnName(columnNamingContext); - final Column column; - if (!_configuration.isValidateColumnTypes()) { - column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true); - } else { - column = new MutableColumn(columnName, columTypes[j], table, j, true); - } - table.addColumn(column); - } - } - } - - /** - * Gets the column offset (first column to include). This is dependent on - * the row used for column processing and whether the skip empty columns - * property is set. - * - * @param row - * @return - */ - private int getColumnOffset(Row row) { - final int offset; - if (_configuration.isSkipEmptyColumns()) { - offset = row.getFirstCellNum(); - } else { - offset = 0; - } - return offset; - } + private static final Logger logger = LoggerFactory.getLogger(DefaultSpreadsheetReaderDelegate.class); + + private final Resource _resource; + private final ExcelConfiguration _configuration; + + public DefaultSpreadsheetReaderDelegate(Resource resource, ExcelConfiguration configuration) { + _resource = resource; + _configuration = configuration; + } + + @Override + public Schema createSchema(String schemaName) { + final MutableSchema schema = new MutableSchema(schemaName); + final Workbook wb = ExcelUtils.readWorkbook(_resource, true); + try { + for (int i = 0; i < wb.getNumberOfSheets(); i++) { + final Sheet currentSheet = wb.getSheetAt(i); + final MutableTable table = createTable(wb, currentSheet); + table.setSchema(schema); + schema.addTable(table); + } + + return schema; + } finally { + FileHelper.safeClose(wb); + } + } + + @Override + public DataSet executeQuery(Table table, List columns, int maxRows) { + final Workbook wb = ExcelUtils.readWorkbook(_resource, true); + final Sheet sheet = wb.getSheet(table.getName()); + + if (sheet == null || sheet.getPhysicalNumberOfRows() == 0) { + return new EmptyDataSet(columns.stream().map(SelectItem::new).collect(Collectors.toList())); + } + + DataSet dataSet = ExcelUtils.getDataSet(wb, sheet, table, _configuration); + + if (maxRows > 0) { + dataSet = new MaxRowsDataSet(dataSet, maxRows); + } + return dataSet; + } + + @Override + public void notifyTablesModified() { + // do nothing + } + + private MutableTable createTable(final Workbook wb, final Sheet sheet) { + final MutableTable table = new MutableTable(sheet.getSheetName(), TableType.TABLE); + + if (sheet.getPhysicalNumberOfRows() <= 0) { + // no physical rows in sheet + return table; + } + + final Iterator rowIterator = ExcelUtils.getRowIterator(sheet, _configuration, false); + + if (!rowIterator.hasNext()) { + // no physical rows in sheet + return table; + } + + Row row = null; + + if (_configuration.isSkipEmptyLines()) { + while (row == null && rowIterator.hasNext()) { + row = rowIterator.next(); + } + + } else { + row = rowIterator.next(); + } + + final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); + final ColumnType[] columnTypes = getColumnTypes(sheet, row); + + if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) { + + // get to the first non-empty line (no matter if lines are skipped + // or not we need to read ahead to figure out how many columns there + // are!) + while (row == null && rowIterator.hasNext()) { + row = rowIterator.next(); + } + + // build columns without any intrinsic column names + final ColumnNamingStrategy columnNamingStrategy = _configuration.getColumnNamingStrategy(); + try (final ColumnNamingSession columnNamingSession = columnNamingStrategy.startColumnNamingSession()) { + final int offset = getColumnOffset(row); + for (int i = 0; i < offset; i++) { + columnNamingSession.getNextColumnName(new ColumnNamingContextImpl(i)); + } + + for (int j = offset; j < row.getLastCellNum(); j++) { + final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); + final Column column; + if (_configuration.isValidateColumnTypes()) { + + column = + new MutableColumn(columnNamingSession.getNextColumnName(namingContext), columnTypes[j], + table, j, true); + } else { + column = + new MutableColumn(columnNamingSession.getNextColumnName(namingContext), + ColumnType.STRING, table, j, true); + } + table.addColumn(column); + } + } + + } else { + + boolean hasColumns = true; + + // iterate to the column name line number (if above 1) + for (int j = 1; j < columnNameLineNumber; j++) { + if (rowIterator.hasNext()) { + row = rowIterator.next(); + } else { + hasColumns = false; + break; + } + } + + if (hasColumns) { + createColumns(table, wb, row, columnTypes); + } + } + + return table; + } + + private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) { + final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); + final int rowLength = row.getLastCellNum(); + int eagerness = _configuration.getEagerness(); + final ColumnType[] columnTypes = new ColumnType[rowLength]; + + while (data.hasNext() && eagerness-- > 0) { + final Row currentRow = data.next(); + if (currentRow.getRowNum() < _configuration.getColumnNameLineNumber()) { + continue; + } + for (int index = 0; index < rowLength; index++) { + if (currentRow.getLastCellNum() == 0) { + continue; + } + + ColumnType columnType = columnTypes[index]; + ColumnType expecetedColumnType = getColumnTypeFromRow(currentRow, index); + if (columnType != null) { + if (!columnType.equals(ColumnType.STRING) && !columnType.equals(expecetedColumnType)) { + columnTypes[index] = ColumnType.VARCHAR; + } + } else { + columnTypes[index] = expecetedColumnType; + } + } + } + return columnTypes; + } + + private ColumnType getColumnTypeFromRow(final Row currentRow, int index) { + if (currentRow.getCell(index) == null) { + return ColumnType.STRING; + } else { + CellType cellType = currentRow.getCell(index).getCellType(); + switch (cellType) { + case NUMERIC: + if (DateUtil.isCellDateFormatted(currentRow.getCell(index))) { + return ColumnType.DATE; + } else { + return (currentRow.getCell(index).getNumericCellValue() % 1 == 0) ? ColumnType.INTEGER + : ColumnType.DOUBLE; + } + case BOOLEAN: + return ColumnType.BOOLEAN; + case ERROR: + // fall through + case _NONE: + // fall through + case STRING: + // fall through + case FORMULA: + // fall through + case BLANK: + // fall through + default: + return ColumnType.STRING; + } + } + } + + /** + * Builds columns based on row/cell values. + * + * @param table + * @param wb + * @param row + */ + private void createColumns(final MutableTable table, final Workbook wb, final Row row, + final ColumnType[] columTypes) { + if (row == null) { + logger.warn("Cannot create columns based on null row!"); + return; + } + final short rowLength = row.getLastCellNum(); + + final int offset = getColumnOffset(row); + + // build columns based on cell values. + try (final ColumnNamingSession columnNamingSession = + _configuration.getColumnNamingStrategy().startColumnNamingSession()) { + for (int j = offset; j < rowLength; j++) { + final Cell cell = row.getCell(j); + final String intrinsicColumnName = ExcelUtils.getCellValue(wb, cell); + final ColumnNamingContext columnNamingContext = + new ColumnNamingContextImpl(table, intrinsicColumnName, j); + final String columnName = + columnNamingSession.getNextColumnName(columnNamingContext); + final Column column; + if (!_configuration.isValidateColumnTypes()) { + column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true); + } else { + column = new MutableColumn(columnName, columTypes[j], table, j, true); + } + table.addColumn(column); + } + } + } + + /** + * Gets the column offset (first column to include). This is dependent on the + * row used for column processing and whether the skip empty columns property is + * set. + * + * @param row + * @return + */ + private int getColumnOffset(Row row) { + final int offset; + if (_configuration.isSkipEmptyColumns()) { + offset = row.getFirstCellNum(); + } else { + offset = 0; + } + return offset; + } } diff --git a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java index 8e4c9cdd7..d94caac73 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java +++ b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java @@ -52,10 +52,6 @@ public ExcelConfiguration() { public ExcelConfiguration(int columnNameLineNumber, boolean skipEmptyLines, boolean skipEmptyColumns) { this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, false, 1000); } - - public ExcelConfiguration(int columnNameLineNumber, boolean skipEmptyLines, boolean skipEmptyColumns, boolean validateColumnTypes) { - this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, validateColumnTypes); - } public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnNamingStrategy, Boolean skipEmptyLines, Boolean skipEmptyColumns) { diff --git a/excel/src/test/resources/testDataTypes.xlsx b/excel/src/test/resources/testDataTypes.xlsx deleted file mode 100644 index 50c23b2d08eb8eba0a51f57a91f3781142f03e16..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32656 zcmeFYby$>J_cxBBU=U){DrwP3g9rjjS}1~`Al==dqJ)wnHN+q((jrKQAU%w9m%t1q zHFW;=eb3K}DYUz)2z!qC-SPMCXZO zu9%n`?;|2Qc94kZ1ks`W>Qd%rHo9guTK63-bgiFoIGCE8i#xENJ(g%cc>n+X^MA1i z(mqruRdSJe&dL=_giP-Hg#a`{u!S!J{m_#9z7>*NwjPIhXr> zxf+`0-z;I_|HV`0>9FXdo)irO>4eU#Uz)YKkM@<(`_1IrTzlo`Xu{uRvv4S+D?D86 z_c-n)^9#vP6y?kGhjp)4WaO!QKA5MkkG*H4KT@y2{m3{#;B|#VvQ!c7TLt%Q$n6tf zUzwc0;X+0JY%Y2j$&n@zrG5r`Q<(k91H;4hb&67_*ng59J7WBzEBr$GO{T9~81WJ0 ztDt0EQ7$>>Qs%x&stI>uWpo48dxe1W)g}Ar3a4*{+VOAH1tsnLIw9p;#;!s?Alk3r z$_c`2XNQ0xeZa-E2%aba2p$C()w0qxvF7A}{{8>3`F}AW_iR1wy`nN6o10#> zEA9P($8N~mOUf0Uzwh?qRv%@qUpzBye;FIyvHLV0#Io)sZkzSJ{i1%8wda?A@uj^x zeo}-f&pzEVY~9k9^b)I8sElRWa21tZ`%?RIgzRlccDs^*OR4#vKFZe(GRd^gOJ^TL zb3CLud?Jy~^R!sR%g5DE0QdG(j%mEu=8r5^<{5 zT3f$6!=={b{L&iDgL?)-hFRJ%R>Bt?wVs=l{J z&CQLWRJMzmeITEKcmMDHC+&TMQ56^2&6PLX9?hkeJgzF9mU_*1!%ZXIURZn(RL~|> zcO9KIzAIc;9%%N3TkiV`gHKVfyM6j9Gb4@uspq$jyh^jv)iFLe`=bHHCsDSb z`OitTp7gr=QsTq6b8kNeH0>A{eKz6~6WMAG8asNLXYOb9LFt#Jtg3VBY(Z3`x*I6O zG9CIx@W!M1c~t1xFZ|S{Q4%ClRVQjvPK^qxy)Mu?A)}LuB)ew!N+>s!!|Iz6JDJm^ z!LFJ{bPl)qPDO1_7ghos{lCW0<4pF8e1K%%9v~v31aE*C+8aG-_Z!T5xsDZ|8J9R# zEaLR&3w_1iz%CA>Sax5&fOnjHxnB<5OSpYE*DrQwAmo0cq4lk1&8ozFx|zl+LS@1a z-n$U%yivVK&f#+Vx@GjyT;++aey?WV<5S`F5p75AI+4g{ep#8t$eL1!q*ET)_d(Hc zLL=HCW{l;2qvklx$&8ad_Zx@Xrp450pB7x*r@>P4bA?Uw$Llz;8`=Y}KEAtQ@yKZ4 zeyj$i_H`bwvEI9FcGpu^)rT(WQ7DT2D*Q~P&6Z$+rTFmvE!$K{i|23V7V(4gFMNkD zkKeY4Pv_`6c*D^(D07vh^vnUVF%!*0hbc>@U$-ymtL7%l-Db`)3pq;ac$-bIU3T@i zm*HfjjGP8}9Q$HQn3#GtH?!9({+9lahlxd5t4FxVdKH*$uhO6WL3ib=)u&WzeWe zyL#pl??B5^pt;sP#+^E`$ZzzU=Y|a2Q@WJuChmOFV7yb*eIT>SFok85``f$!2-h6MYfMw;@gF`?;aFUvKzQ!QlQ^5`_fE5(pGp}0l&c%? z1rbDDSHyi@<-N{F6-21!k&-2q9+KnZEL+BTiiY=U%YL$Yskg+-S_J*A;)ISH+n{tu)mAI+6RhMWj;0$VqyS{_s%aklQN$ zRJSzPB)P*&u0;e=*?G1k6qWD|BwIut;-Hn`>kmuHZkBph%PcXkTeyR7lf9r@6` z(lO^Dc}SBF{P(cyJ7t>OpU~IC9{euxEfbF9#F^KrSn7sp3V)Fdj5Uf|J1-j ziDY~zEvHH#dFu3gDTPNDu8lqRoyoExB08KP7g$C0jyfim>ba#iyO5kuj(ido^8gu( zuatG7D)s%zIjwC`+eFb1c6q95KB(GdHH`fy;E?Q|?AZ$BO6vlr8QZI%m&oFJ7nW z$_t-)S4*V%&*Glv?un%wZov_s70`Gw&(*V}Hmd8Z6l zHyCiJnD7cobGOr%zLS2=R8S@-E1ZEoDDCw2qV>8?V?jDP_ar9u;MNiEcVg4dHdK+d z_XDLOQBSJE{im}lxo*ri)<-Z0`ER{oyz%w|R`|rVRR-R{V~&)T*Xyb+1}}6STIs%F zal+<7@_lA+7arxS*~PvQYvLT^2c9MJoZIk3)G!O(>rwV8+)--F`E2vJP2^&B>0^=P z1eX&t66`}_e99R*4*DHOyF|9!k_TsYG|L1^8nBr?Z&W-y&!pPs+64%3^s~Kgam|i( z2up1j@mk|}ID^d9RJ$%yZ!&tmw@c}WByv9aNPE&PxTk7xH)^~w#KG6HyggkT|{w8<@)}rKXu*o)Tu`jLRphWMWtcU z2g;L<+z}16tfx2_+8k0L)G23f#oEqGboxXC8T*C)_Zk99%1s>dxevX+O5Ttc2!0*v zNkuavDzX$GjPtckJ(MD;T&TTv?Ia%!N`T7v;)Ud|FIfxk{;U!*E0XN@YR#rXm zP8K+p5I*?nHmUz%FXC7u42t{Z2~|I}6gPcv5V+^`-nHEH2J+R9Du&1dl3Q8#+~Rd4d0eh7j(< z*M8?yWg~C+C9XU`xRs5zm)uU7u%ArypY2R_Eqgp+YrOI?%bPu8y<0zQUzBX~&F%S{ zxtXpne8ObS&d6Nh;9Yu|nC6{I$<^M2q2!|2*IaO>jU?SEmG;BqrKZ@?r9#x*%VdR# zn#Zm=d1#IcsC#T=SSOTgCNnrmmZ=^2{Or-ftG5Y*CLuJM=^gVl(RzbZedsW5E}Vk6 zmDUUAQ<=ZZ{%sD6C(lD zV=niuzY(I4ad&f#V}3o(^s8;_uG){9ZspG?kYWm}S0k57;AN|feb4)KzkVN{J>tLJaM`^_?>O4); zc|X3(oyF#v4`tZTxr*Bkc9xNXia(VC|EtXSFX)A?NPVMeRL-I@a znXXnHiR~xPaZV|%?{l@PZ@MJk$mDd6j_>?(PJ)Vt;q&u6*VEeNE*R)rN(!nQVHz)8 zI6G)JS$nbP>CE&2;?AfRdr`Z0FQt5OF%L!$y|^HMPB_#k!sCVAMv*Cbr@Y*A($=1> zYNy1yYrelK_LY@z$%oz!QxWuXn0}!4-B9=CC%x2M_4Lof_92E`|X1&9$@jz=mI%@#}=2{GEF6FpEwrRhta(! zh9iDE7&RKCXIVatVqTItdIB>O*2XH?YlfeopGcqJfLCh&Ze#4xP^@KT( zxL=7VMf?v3zA|5&L%En6{R-DyWtQ`E`ZWi=h}ft)*NkOq8l&^K&TW4`?2bIVGskvr z|FVE@>#&DyQk_P!{4n$_L9y)S8xZj*JQ(@(XgWzRo-X`})FIXBelZi`zzP5v94f z`^J2yeOYaN0HcJf1McQFTG`!oeyvk^c3?AWN0!mejaytajZ?hXai;;N;}wSFadz39 zuURo4+*%vyEpvBRX$aE^R-Scp!`)nT-=5gYM4Eo1H0SLFlTDbr?<{EzxNXj*n#_l- zNR;g?pj}(BXz^m#HAeTXv5{l}_cb-m6$#HZlzOl8j>Pw6#(}b}#d%x}W0}k4-V(vr z)t#HGzvkaeFRr&!&W@eZFm+#^=noha+i`PPGq0>1S={_F^|0vX4tA?(P~yzaMt2yP z5(%tXS#Bs|9A#VD>|SD<3kzV}vAeoSxt2=Wy@=Da=wC$ja?ZLtuPn}Bk!8+qCY#f^ zonDdtp5pu^$F0e!oxGTx(w))mjbwd^;*ELKp!@cMg!uOM;6TaFl(JUB+65&Ia>unw ziJi?Eoch`=xefE0u}j;(YmmA}*KpllE1mMUqg-$SQga)OrJIZWxQ1`Ie3LNA`mvUE zZ2OZnEqBM2wT7~?GPCW?Zs5f2h55=Iv3k<}#V)h9-=jM_TaVl~?ZO#5v~&8utMA;} zAzs^Ak~lkB)?T_0R#%!+;%i%2yJNQOX89ZHqXGcHODPPQ-er!^2ROX+1 zSM9;P&HY-+N7L&KAI;Y>8xv)ln`32e67EdqYNm+`jF=G3cZw7D9Uam_Zl}IYj*n9* z+)>7)89C| zYk5hO`ahJJ$nE;T5z?ZQe*f_UQTkKKA)4;j43#r~ zB-x~YU@%T>`TT3*lVJMd`uTI|i;qoif4e=At$Hc_QjGuG*4vEKiJjW_oKp6@XcHLl zSeiy7RdGrunz%;$V%{ako+8(k&Em%#MOTWK2U{~==9JZJ8hRHz2oW``b1b-z_>DYd zW~q8}9BGq#%HXk;CdVmDsR+w?seMX9{X@kXm$05ZWV!zwenw=t$8&CsyLQD#ZD%k>|=cvYd}f zIs0lNLZiaeNoYdVS$kAudtUG=$}2U{H>h}6txWcAm_-{E_T1)&hA8k~`KFMYt&~fZ zjDUtDfgu!zI?+Zt+6p;Bzps&p@RDpZi-)FojuqCWGxRT)EFWIKR@;#i85(8(*Xa32 zrip*XC-po^?r~eGDpFV01^Y3JFnR?32j*mAKz$tr`)#l0X|EP;x5?#4!VdcEr~BE@ z_Va|1v0N*(+xxT)_i3l@x3+gAB(WrqDc%44yZxKG3iQV1cKi%UJnc!)3?J0dMv%`! zjP6Ak-Glap*z<;k)N!;?VFfsY&x*RA7607?{Hb^ocAp!o;!TLGzaKdXOi=y&_vimK zAOtM`ct#T~m&GrWyAXAF8J$v0%sUrd$h4xz?GIl$w5@ zut3B8NNA6D&BO>MpmaZ@hShqr!O_5B(XO|$nWn`51n*k;#2$^L9*zHv6`(B`790&R za$fp<$M(hde1sD zwf)Mo66s-K%9Y!VH=C9y&|24_WS3 zzB6Mv;0Jxr`b|mvi75*bU`w85P?dF1l?|DZ_g)zak@x18gUwS3!$64CeG94k)?SyK znm!7L$Y^iK_FmU?u3BjXm%M}aFuRV zIcn8QcDrJOBg92L!bLsW1;AbWst|M}>OyGKT^xMv!ynq^Q@h_l;N;N#{A6$BKdHZ8 zwD;7DKBU;~jS9Q1ywOh1?*IDA#e2W^2G(y7Xi(r?9qSm>!NUQX_3)r>W%7=#2SfpT zS4kcdznhZxaInUI00Fw6#rB^5i8;ynEtqHi(;LoL?)}c* z8(%sRMge<$!O(EuIc-mXEk8=?0Yp~#xIVosE*-`-{M9Pr+O|@pnZ@Iox$8fA?znAy zw%g3G!!-2BDtbM?)Z^*J(>r?b^=ug9RJu)$m@wbHwQHgu_ObPQV021OS?0-l>0KHH zFO4NQ|MV@ReCljtko4ZnfjwTSkQ7zpba~n&fC6>DkDO9>;MPhkHS$_EG)IJh=!cJB z4!uo_rseUU3I_-0R9KNVl!$rP+r9BpydiFtQq=A(Vg}VZ(Z8ENM!dZ~{3=A@mDk)4 zvnQPHq{feZep_nh5Qmn_e)Bjk`+;idn3YPWJZj(LS1ByNBfBb_sBTG^iPx9Cx_%*_ zImqMelUID!LXRrsZ&2vXv4m?upT-ZfQay{CN_R9bKXxJ^qCql%+lMZgtoA^#LZBW8 zsaV7FLqg#nBc4VbZfP8gB}GWb-6Ao36~QXU8|HC@rFo^$o4?LNk*n)~A;wCjjyXwa z^eKtis|on|TQ+Xgc~aijCwv9>A%v>yhR(A|)m9w9R4of4`$Zg?kse1b-9ZvhK9|c< zN=uT0X5`~W1^o&MC$$Cc(xABXrHaqXkxNpHnrr#M6EyhZ8~nvfacZs&c*S8BXvJ6V zRzpUS2gD%zS;E6HT$HTm*ol*3&w)9FDc;&->Zcw!tXs~05_*ADDriR1TAQIt6#ghH zO}+oPt{m5$G|Z+mz3feDy3;ocJH^?`*`Y(7DgC4qFSima#7(I!dL;jv-P2p*MfB%x zQXC;g`Oum0-u1Vxt>mkh9Nnr2q?2YCJNIQ1CD19c91p%7&Z)h7|GLV&*g@HQK}y59 zEHg;$*m?&R|3R@p(P{4!zp%qINRp)b>}Y-S_B8!eLyZ;S)iD3m3;iBwpN-_R>EM7I zCx$Lb#io)SE`Hl?v)VY8$do3xGGzU0`=V#sIb=bxO?b$83suhp20x61T`&tR*{TW7 zbJP+y>A1`et~SoAjxzGI&#oz*=R8q+s&}LqEpm`$M-%($a!bjwP{k8!E{B6_jVM2g z60jylT2p)d2z1ONnPmIQd6*AfQEjy}wXlx&$O3EPf`U<4@9@f3E*PtyL61BkzyIKQ zdiE$ZqB0w{O6UUX7ue~n4aZ`yNWH4p=BMuj=StBC@2P+)M=cN>qp)zpu{rPfoA@x;mr3ZT;;RiqgFvq_YGN-D~`_9#&p044GYtwdFq33@-_B@aQZ`0~0l{(la zbSMT#7r>c%nNsinrYrYCJuRtHpM#XX=w=})@&qn+27m^fxCM1+pvU3Y4KJuU#(RdW ze%{d_j1Gj2mntB8ImW~Mlmp(b(6|Ffi`Z&ZI4O4va3S{-5>i`ueA>I;mIK)6ukX<( zW$l7ya?ywL8eR-@)asit*f^Y-VBllJNGdo;A`0kie8bLP*YrFv2CU5>2lkBP?K!7c z0?v%i=4;sbA@F#Xu$l8(cylHh=%K-VVb92g!q1hFo!xo`kKV+Oej2%BwRPqp1WWpU z-A7Ustg9TvB*jE1x?x=l?=usc8&NNcyBM&U*yy77XvrE!LY#j(GYTESGS7jYhQRWY zM~Yt15&U{*Ln#g{CUcb(2Q-kyDK%6|c6ja`9A&XOWzcy7kgVbd6Y(C{-xzqkoG|!^ zFMe=8ASp2QLEZzW8v|cdfTxgglrpgQ&&hm$F%_Znos8lLugQ7R6OYcq0hq#q^zbKR z>NiH7FbUsR`3w#SQ7|Fm1nwMb6&yY?jmOS$OT9X-1Hy*{EO3$uf`sF-9BugR$BoA< zz~eLEF=TrnH{rbaN!Fm~ttQ(?$ID#Ht=c*aS;s&ajMIS!ziK>oj7RF#c^xSB*m*oE zYvI_FtA=7vjM3%_HU21lgZFh-Q#jP=eUSSy5y6$BhDAg@k*|^XEw~Z8kh4Py92;u39vMC^P!n%aP21!;H zgtzIwiZvNyR{7#bTIE(ATfb7)fhNOL-|6OA-xh7Nj!ze`K87TSX-RDMFHgp4})AAuIvG&vQ@r?Bd;*h6wqTR4qAO*D)X|g@~O7Yi_}7&p_slv z%8kT_$zPA5F!5K0PnE99g$9C)zcMcch*=i|70&R59{2=%&Q%F`GXU2~Qj<(gj|5?k zTHQ{8GIJy;{-X~x-p14dGG^qg6<0+vHCPVh#-H=x%;xsmI&v0MaR8~;hVY*P995GX zG*f|vZHEXA%ukHeQc6b6o3Us1^gF~Kr0%~0Pt>!Q@?5iSBbFJH&VY^9OdUaxJbW94^x z(ufnV*-{^}*;4DIwLTo_GmU5Sd39ci5F$OF7mDJxp`sT26Y&|&`%1{1sBZ37en z;31;p+&8}_$aeK6Co*ZvFsQFAaxCgAUT|2qj0m5^Cd@|DkFN54(XV(E0x6D#<2Xai zNYvsJR9_zf9w4i(@xeMnnU_Qy6+YDZNO8pelp&xMe28)j;PI2V2R{HHVFZ8_06>6X z&=eL&on7VA0honI*l_})1CTUn@#xw!*m^+!@9?3(8w1!wY4jYX^@XtxY5gcVpM7^LevN)Q}7r{NWd>ct-goF0bK*hMj#%LO#e{J|7>v7N(49q$nPLmy3jxsfdD0* zhC!-?Jm&#F2e$a73m}<*e!#CD#|6um|HgBUfLCEm!^aq(5!PuP5~7R;T75|KEF0-e zWB5=LKBI?R)Xqx zYAIRh?vO`4Km~)mm4vCNg9zoPFIEyRZaiB3Q`RP}qdoF8*EQO`(Xy&`P;Rh-d7~N^ z3z0(o?ilNzP=PcyGzyW9!9SnQzUu+nDTOYI4yF(X1~?4~2@1_Xy26s8hXKoP-4_9` zpDqL#3d&NrzfT5X8d6LkLD-YH$n>1&n$SlCk2ER3xTmY?qlT&nJnjKc&yc}6271iP z0#THu7&v)6CDuwdM#F)S&wFpENdXWXArJt`z!_+lqIL&E7!l!xtj%;)ve1W$MIg~& z<4*{E0!a~edQy3GgiVR9R-2J9pEVB9Bbbf1l+!7S+006=Vs?n)B@ipnx3JupX|z;w zqR|&(E0MVK-nzxAchY$+(mca`8k>-8S%FPRl%o(ok@-9PXn^zcmW#!CFfqvEq87ha zY9B($G9WO4Vf8VF{Sso2uVtPGdzZ)OsH(nC&U|hUgY-B!K^{8dg1FTwk4#kvxwj*m zWCy8}T^gB$obH0Cn>S1dmMAG8dxd`REe@&KudkiNY2&^tJx4>%K1#xlCFzEoY8mf@?CSt&@ zt9-#DtqL!d0kcDy8$RJ=I4Oyv68N#1e)OkS&dqBAuXo!*pP&jMvIAu<v z2z0!914p@MFh-DNvmduml2LzK>{g$3wXIkAv02)h;$*B9(0+H56jFqu&X`*3FGekn zEi4!gAWzo`O)NPnty=|Iw}drVDy=Vs3$m3E>w0(Ejeac23k*990mX01%@i5D8?Ebw z%8*P390pVZR5YBhW`d(_BN-%4fSiOW6r8wW3XwTUg2cZ7zmM9DeyDNqZ$!tR$9qKw z)GmS*PODD7I|6_YR0J91W@d2S1|rulT)DMF6p&H;X7#zlE;Q=qC_h!}^`3AL!*|RO ziOyp=>EB|Q9)tSz5~)7KbCSFH*dy2CUhDrsYpY{gd20@LY!}PmTcEfuKA8S!96m zdZ48FPelfW3qt5mkpWxtr^vwb=+7vS>XM;4`NOS2UL@dY3dp(rXL6wuz>!e&a8nXG z8(_LQL%k=Qt{Gnz>YyYh^rff;;Y!(e`No$$4N9tdASWY$O|S-mx;yr8U1q6|0U)+55rS}nPwAk_ z1g#IiXl3Tp$HF)DnQNjWz?vOPE!az!k@!yKF7tqL;3p*Uni_!)m~xu1c8vk>gR0S8 z=EAo!g=ZS+@_BPI!o(k{UF^gKg}{QO8#3m1P6jA*t@X>D`#}aTGZ&okEq+BnJ>DSv z+|~?#(;!u%Kp1F1tsKJC0$9B4fRw@R76=~1Z*j3;+dk@lWqt;%F7}#_L+tS@FWj2# z69wCg%UV_g-R7mT5gVXDVuDF#ZRzaw>>&9DT45E}>@lEJ@zWH)x^aR9sE|4#!(IJ> z)%_0ihxhO;#<6eK@^JQr_#h7>G~ozp0SS2_D2GV|z)~Z_nV>z+(DAWB1q~oeNO|lA zCkV7+0?tQSV449Rp+LhBQvR+Y2?`5WhEqPkStGnCUSB}MhLG}oA^9O7KwvF`8iCY4 zsW>VMq4{j~7%EghCeIYS>5yEcJJk3PHt31k_3Z9cX;< z0ag2OSq!~SoIwtZ5fMkxfrkUd?r&ka1obW;xdczhr{!@OG}!)_tXQ-3Z%Yuuj{YWK z{;FTBhguu|AzhDN0`7I}_8;H@cty3#Gnh@FQs32K|4m8s zzqY4Pv$XRKj_)y?`RhZQaYegMg_^v&)!`V>!NIT|mH z_ol@Ez$GA`LQDbS=D$?!6nzm;F|oJZu}2*Ir@^MkGyWph9|oIl9GLM<|AvuF75a_) zZmI8Y)q17>GSalf7N(GULivCRpq85`AzfI)&>8{h0#q;mN1oycmDoGjG1S;taGwNP zHEK{9)rkN*hYB@YEgQnhj!+Q>vmL(12p1t1XR_>z_+hCZS{|Lf)Zut%Z-WD_lJL4k zZ9iZGmTAY9r4_Pte2*S4UWB?ZO?23VuxBCGEub$d#O4Zw#)810#!eUd7H6d=`j$Ku z&UcNy%(V%hS|oB2)##BHQMQultnfv#FY%kHaG`RMzjLrM3wVami_nnrDFze{wHM!w zv|hoJ%pI7mL36_UdyoDT^CqARh%-QYB1cdhbsU0}Fut8(4Yw9S7BIut>_CU)91usn zCn$crtN|JuulS+nN5k0#o{D5OuD#7ZLMH{>XZ$sK4;tZ4B)Fq7JYAEl2A4lje-FA? z!&gY4`FIdwJA#Y>oC+<6m-H-ihM^F>HSJK$WQA{HVa)Fo$&>=kRf?$D?_nwjn z(m@2q2`x`nJN|cFP_=dMPXQx8)OrQNKiq=20x6(BEr?wq0>2KW{&#g{5`dC`|KthX zpg+wUe6JO>)BbY6-h$~bzS&*x-}g)Q`X01;{xntMEvHm$puh-m#ooI{b@ns zGwGkq3j$fM83=(n=oOHOVA&1?EOgg|?=m-Cc`1@J?u1+WBg+Ym)0R%NAf9~p(nRg(|!{Y>yc%cIKC?VE| zYM1;NsNWjFVxfN;Qdc`$M^Yztfr{5M1W6)4)O4i~v|Dhu)8Px(e~~eG<0q^I**pq( z$6Y;kJ06hQfPw)sHV5J@0c%+L;LU)HZv&H_X^h?nn&Wr^A+&M{PJ%WA_eG!phD{BH zD;2mVC2#@gwi24Dhrt{FU{=s8+%0p!KJkO`y;VYET?8a|;8{>_f^I3Ksj^Vf&7f-W z3R9li{wg;!D5iaVUS zeg-&CLb~J+4D+w$?^Q8yb^Qw--{r&$R)qWfAk?LwL2?m`C!Q5$pbg+Cp#>@vDggoJ z=M3Q(*j3p0KB`N9llwQ$K8bdSAR&0dhH0+T#V~Whxq!ciI96zp{ggA8cnFL{(pH|^Uk}%6_ zLKga}Ps~vsyUkUhm}Gn#@K0qxlh2!)0Sf_$4$^k%04_=LW9$zptgH1-=JOJ6w|2`E zxUB+p2Q_G6J#IYIlvX&<_*d@{Y6I6w_Q%6l;`VyveGv~jBZdBe3@*H(k8sKn$sw6( z{0z0h{@&;Wp$t`J0QG-8hpuzf@coJ@CtiFd1yo;j(8FVp|8|KHzanT3rn!7U;5u;> zZ|mF#EOxJj2%(tI3)>AiNrLe!xTGZXJ;m8(-Vv(Cz6gHEgMin{w4U+fugD2kOaC&5 zd)fEB)mJ{}XXmI+aMh#=3?{Y*9mc=9sW6hl5L2@c(*WyQ1ri(&-TDBZt~SEVmU#}o zq#zIvwNMSHK`@g2BmU?bp$XdnPDya5G8g_M_Z z1zk4bG(nFsk&fJU1$P z6NPQpo*<(HR}0~wsPOy5(U{}SAB{R*d{#U03G@pr5s~NVQ{iQOyw&)B8gfw4r|FSr zA^7*DW+NDq3#4tn(ik#EDfKD-_)0U*7v@dzFo#3~g(zuG8ys^InHrM=REmMbQz*nNV)dS;7#&W! zVu>5+NPDTC!oI7bQ6weK7B+|nbv#bQJU_wxZn5b16k3I35E0Tg4m3e$qm&xG&nklm@Bxf(Jk**7 zNxDUi);`~HqFraj!p>r9OM9u3!+L1fS$7D|Vlsj~ErX&t3p{;?;OTR_gA+)EPzd5Cx5W@nBLHl;2#}Yvw9h;nj1BlV$$rARAV_G*m?H`(! zL_|3^%*L#EW8Sx(HO_HP zzO+p-WM8RSC+3BDxIIi)`^=#Ma?)hQaL89hUOi@h?TI<-1=ayL3`b#v9;ae5hY^g^ zJK7kj*eRbpHM_2^h~~T?z$82l4q0gg>qqN3a4uPB5{3CB&{BFjI(DFofnI5r4B?V5 zI1RG>Mg?5!|>EW-e=uCqA;rhkt2g9FWtdS zRZ&v7I{-Cr9PtoLSw#AkM0D(cZDh-XmL&m~R2_gndX;zY->=`gw6O50tj?UDzRX)B zkag!}9k&JEKtt8GBb(n%ye^E9xK8k34if{?3OB0Wa#;=rCFrOKC<+&zqn4S=yEWo z2E6E8a^?HNYT?vhb=O1@wznzeye8VRvQ1Mt|0AM&fbME0W$EY69If#Y%geSw?ufiq zFQ!W<4OQVuQ1_itx^UUNEiwsHljvYK%WwTN5+!M#M^uu%q<4QHR^TdXLHWtanmgeA zyTTYhsPW5Xo=1yIN^#V9Yb_C?$H7j&0C&@3i_e=Q5Li#NhLZ3v)M;-4^;Zr${J#e! zD-!$4VhSpnL2QBFzEA}V3@Nti3msoxQkAzgwYWIpAfP_fZ*q9Rubl|VxiW~fUsl-O z>K-f6YwzS+*{!oh4ntR7BsBGO)uvz20YXxC(ZW z&U{G8B=n<8D%eHblmmwZ6p@UaKXCS$5yX9TxUJvihD3eP9b)l5v&RTVuM~%sOt?t7 z26q*|b(^_FN|df-lXD_DHl&iLN&+UFW_4&s2Wmp(t@}0{=on}9Mw>p?X-!eM=6qZI zy_9;!T&v8ZYS+TPLDY9$VI23-NwFlN((aQ+_txns;s(kE6j3~)gOR6%W>A=X0a#bp(yk<> ztj&LXT5=y;V+K(;GeKIh@vXqLqEwEH+|?bb)Z4rQ;E$D2r$2$ig|M_)j^8Gt9W<4K(w<#xP+Uv4I}9 zzZiRG)BIg&mO>3JdhyCo>@1GHyxx<12a5NNjn?-?*sWCVV;3A6jC%Hgy8z_m26hAXV zAi|brN+#&#rBV@ghHn?vR%KVJ-RM^Jb65l}EPpy+q!7{+w>= zJn$VAdJY1En%TSlhKlHGup5a4BEK8guVCj+`O*#n^jt`>nl14M(EdjCX4sx9$fJ;S zPAv;*KOitFeGJF41?mGd-;k+ENr;CkF<*m;tv*BXbUtD+*YZQ8-h}3w$y6c+VWz`x zisB(3XsBpVND(YsaYyTx4AaI)9Hmu>A^F(VlU7O zdiE<$NmeRuoXaa(xPm50_+E}C*0HqDc`$$a6V0z4cI&K7l=w^pWA9p~N(Y-_I*CE4 zCcXQnCR>1c!HORbX`*xfH46%ctHVoI&J095&VAW24ttH(rML|j?6UHo*0MKc{rbtA zyM_2)W1#nc`p&*%xEh4fJv-}~5qvbd=vw_OMjpJ59cn&q#K5^CE}f7`|K8+A zT;4!q1#Mc=lA}%p*4euLrc&047t!&XNkUnmXo~P~mLJD05lu(0_(fk0E{mc63~tCf zW%fU?ZbmFS_g|n~m~kEG%I1EJv10fD|8mDX0fEV#7wj)8v+S!4GO1D#}xY@`!~TM%uo*gtD2g%yd4n%OLv*Vxhc^uGbN@Bx=B4 zj$NB^ZEo=QJ}sBa&ILUHYb-2-Pq)K#gcn!aoRvAH`$W2$_fea6M<}@JxyM^ry4m^}{?~Dp}uIP?%ba1?Gp1o6&F|29&JVxWsp5jEm4s6QFyOE-6A*8Bh; z=pkEBLS^9p&yMFIS22J z%aEf!(hxJ`ckA90bj`|l-J?RZB@%3fF@B*BmrG#Py=d}v^YZD1_BoMR4-qX-2QdyN z;tb4tP{pDccgQd!;IZ?u75yLd^uPvcY6Jc=EX{np3o@x|@hbqO1tv4=jk*aQ?cU$# zV&3QLe|F|`4b8L%yyZGQ1UWwFt8TOub21##42SP+o!5aKcj~A>@SX=w2BAo5U1;I}@i_Mv> zvg$r&uV9BQ^PMp-Mt*A0c_oR`C!=SfMzAP_TW^9t$ih`R4Mw*tP`piu$sYPCT2W$9TRCD8oe_lCALH^BW zQ1Xi5+FD;E?*lrTMKh~-E`1s+1LmfuybdGPNBj+|)FxsHuipA4YBs}wVDKU_)~T7* zulzWSt#P?#>~-GDbHt@+RXG2Z4vnSi((da%;7f+?Ps!OC4=FEEkUI&iX?!A;qVTJt z)_>pcGBXkT>*p0ushpF7e0{7wym`}3gJ>}dRGc8+_!b>@zn7;ev-n7$&%@+2m2fBV z%g_YkXD|w2g3;=^sftr{t#f~w2G%O;>nn7A^BwF>*g%l)D7W5sq1HgDO3PB6oU%x? z82x=Z8V3Wb_*jqUOg^}+u_mT;m2jD;5DFR!)D7PVizdF&pK?)mzB}P;FAF9O=d?&x zepEX`M`IRTMg6X0K(mFxOYaWeLs?<)yEPVAq%$4jjUexRN*W4s3Cd22IvG0s!1X@< z7pIQ2){^sc8sQDet%MA5>zkQ*EFU11b4w5Eoe3x@nUEJ1I&&6z6c=~i z*mGplrKrGo7`%ET>TJD!C1MymTx?K_ph9Mzu}4b=@aM31kcskHHmUKNFg5vMO)XdK z(uTsSMLya5%+mX*?35SS%PcOjDaLo14s881DY>!U#Voyi#b3N;(Q~B&s&n#{`ng`! zS$d~`)xH)!`$dl#na$anO8QVxp;AB5!;?SfORr1j7jCW$>6{kvMv$9M zKi=bD2U)MTYqcTOH>HVeU~E+u8e{V8C4c3$<4AKivHVwBlOs9Ws91a6j#HBrf&7)T z=Q_gr3X5tGYThiBme+7`&<16PnldM~*0oHv&FRXmgsvw3xL4a)$& zWNFWTMmR*oGh|JqgjS~x>dISw{23a1npPFOj24?&A3EZeg7t(J9kKq%K&HlfTz}ef z(IyLCQ))K$8EOt^q4~jt+0sq4-*NlYW{Woze83{66CEDVvc3V#vl|7wHVQv_ipuh1 zZW@`K2?m?Y%AOE3gPcm)Nh;Mr_Jca|$mUW7Fp%{22eb@)0#w!1_lcq+7pt)Y>*p-B z1qFO5PCw%9Wze4846j<7$+XY>6ARG!;^G*LJ(pzy=C>$yp2#DdkqFlwBjLdA$NCX(& zG&`|D_GXdKYEh6+9oLV|ky*ffQXReKgFa)Qd0hOSQ-P(1i1aMmlflM`tyfsw=|0x^ zwdF-zL9U&5iJ+I|u@YJ&E>{W!(x%1Az;LZRXH!jX<%rpZCag=7%~T65=^8!w`>!G2 z{`i`Y-H6#WmyA5+ADGjC; z986m#nB(+hLVR@C-h*#Czm=uS0%G7Fw5}W~ z5-oZS7_^!1XzhsKt2iya2I`0YC4ht+_dE-{hXD{$3&N$KpvMktaU#`oe?7|PF9Ojp z;rPj;jdgj=-&RnW6mXYh6m`pocg+=ewk0;0_M-)xQ2{jP#(+FZM+1RZ_ozMxC9%D+ ziz>u^E-@GzabzdOHx6FKicZf|5BPmV{M}{&2{Fnk$SVjdQkfrq*2<)(yhzH*tAqxn zJZ=>(&0c6CkJ6a$_@4c#tAYFhU`&)AM>F)krJII^Sq;p z%COYe(z7KVHA44*?M;x~IrD3d1^FW~0aCmty)dhKt*aVCKePMT}v`bf!Sqh@cZ4BE7&D4b3e@4-RW;z@W*m^VH3>d&!HB zc=P*Ye?U6|boE|vL!+fx2L(kaEj@wiUUxWI&a`^~iWxv7Za~&ON6DED8Bt|2OlRwn zSZX`;aqY50{!AOKfKyt;N02+V=yrM4pSLz|AmST5F$sZLX8Ghi=0kg(id&KIaqzW1xr?{gi+SZ6ZaPov)4Pax6vw1O(2UkR2| zNsF*_$VtKHgR}Z4RcFL*#MdCY5$}>kKcEl5445S3G!H*qF3k5(Bb&x@V=qKz>lF1$vHiqI- zM7y}#{!ulKrsbN?~d-1oWfbMAAW`#$G<=eoY1N1M*&ts8cOoue0o zZ+|4%jh~io?rjwV2;sV~TY;5t_ndnFQ`#kc_ZX&b61(8=c3}0$*D4}pUPP8hLQ8J zXXLx+D3+>M#M1Vi^G44s`ooGhU?_+ai>Bru)i}VB1tHWnhUsYgaG)?&RsO10JGQMk2F~ z1VZ=tIUi!F6$*fZoGGP?L@0_(4xCZzWjN#-UC@Wv6y6@}52-&(49jV=_ zH!LiDMTdFYLXrKbvO-XaSeOhxMD|a^1a%m}+<DpS=zf2d$yg6gqN=Mm4k5d_cJ@l&2YA3GOlchrdEUL{wY<8v{9}Ny8^H^fo zQ^4GQn~zHh1Ru?rNM@IDeNZ(tM0%w0IS?L9OL;txeE4k;CJiN%gG|B0cr({>c$BT-&sBKb%MZNDmrYr6SpsMyj)#Q+5rrC=TUHm z+D4g}e!;y*MXK+VQnUismq55O(xA|4V~VZ7nOV$v3}?WsO7}7LYkCf;L0OpI96lxa z`&>r6m5N^$4uH3U^E>B^8@rvMo3`Ht^^M6#F)*Ii1Uu%^AB?FetM%?LnZhgozCdhyz-B>Pbwr*blt*c6A6an;-Q~rrUo4+0)o=b zxyob-9XLfUxs;v1hBF`5N)|t7FdUv5>eYJMS$9;vuaS<5GUvj?)l4G}P8$Sbz{n1+Zj69H1SVtQH&Yh6P|gf$3-$>;4t zSsWLq#H*)lSbFt~NJK)S;lY$kgYvQ0&Opt? z%Aa7uB%b*p%cYX^CFT1V&Yo0<^$=(CVE?+1im}AG@Uz-vVZ+bW#?5?p%=yx*qR2X-4sZYs* zGVjmV3DCmAn+-UQB`_>|S>n=@ZW3=hBQko2HL*t4(C7#o z#?V8hujh?TJL`S+icg=55(sPpi+cZlQX=!+Ood)4Ng? z@Qnapk$juBDRW+=&?myAuk$oPqebPuu{D#0hH(|LgwN{QJg1F;p|wvq*FuW)oGrD9 z)>flN0JSzn4lf5LcBNEu_{3rcvTx8jgcRw#9H77cfM*(n^JCpi`5;OS3LT8Fc~9+4 z2szuL!*r#UYh^s|f=Ga8T8F|QuN;x8sW?|hKhR>e>hEm{GcO+xjXOzp&FkhO`)ud_ zEnT`v-0O*2MiO6;MlL;j!vaTp|3tZ1W>N@wHyX+)#4T~Bic^nnZN)qC^$pQk@ATeD z&5f*L_s~qPLEhCyC&)_ctDv4T$dFHPaJZr794a)jo+l|vGMj$juy0jpINDqvRrk^- zq82vh)!je6VcPR~61{3iPDK0zt%H-(0Uo< zMv%wkMLm&aCf))(U|wddsP(ip8Z}kc?*~Rl`qBrbE<3;0-{$v}VUS6L*AL7t*%eX% zTz$s8kuC*Fo34HVBaPGWCtr#jr7Ob+D`f<;O(?+xtM6f%i*AJ3a%t`4Icx%$ypSfx zc#-=dSk{g77On@LZicd%oFGzQSV^9?<$C4rj!3TlVX=UbB4I;67Yw z58t-TZ@ZtI&CY&t_W=;};lz=^FnK~KwLWSMjG*#%Y4F?v#N`SijgRIW)(8sSQ9rg= z88nVd95l^(MWZ9@SaX63UU2=MO&_@#!b$WXbAE#1=X#dKmv1=FP`h$c7*+$H8$+41 zmB_{i3m`DTutlbzAd`@}dI-piw7_*utXGLt)w*U)&N?X@>O=E*arSAcTOtU-9$tw% zq?M7QpRGXpV(6q4U!cBLozen!+Q(GjFGu}KZ}T)JI6AHh$r(F+A;O~DTa(wmEIWkT z+z+ffeP98Fb{8XVt^>*XnewXQ@*Oy*MrSosg>eL&nfazDC?v{J_)uvn)$P@-ZWhiZFJ5+uSI!K)&qATN}#|X zaBzk9vkaFB_`9R%R4m*+4YpHfqbzO&Dz@LUU^%nkvL6Ss5F;SvUocw?s=l`76z84# zwbq8pU1r?}b$2FooW?63+<)p^O=7F+XHas}Sc;mcNjpvmi1eM7+RivCF`u2T3aNc5 zh7pnKl4?6E0%KyDJWCwH`urkP;^y zGV}?Ff+ULFuMud=?d(MK+VQ(XS6Q)O4s%nK`9a{svIbm>+l|b`&#=S1Pncpf!Kz4b z7CwSVV>s5gI?|szrkmIxKA$Q^U7K-@TW&mlY52`9Qn_L>BbyH?a9RvupV7zW*dfhrX}KR}k=TP?n;D^6K+vlT zy{p+g{!26YkrwEsfwHpI-kwDC%F+;|bafYh&wap*TVJIPS+qcccCM~Yw`3=-SFX=a zr$(-?%v@b#M^hpbA^rpFG}{Nd)|LDhTiVGX>x|$Pvg<#AB1Of+95lh}XC1*ym-hj@ z7aM2mi*C*?NHJ??x2xY^%l-rN0;VoJ+U$I#_@SV2`gLfv-ve7NB%>pPo(JFF?Z+{m z5~zIE2h7p%e1GS5M}nkS@$PDQlv~~TsYUZ<3cHb^-4)*GCqlF&LRwRvGMRc+LGAXm z_AUbiJ^kqb==9VQcx9)XV=u*?uG#P-PWf_#o8IF`^TDt$3ws5S=Umcwo$xU=A(>sU z$_0iFTJDsQ?dK=7#mO#=;X1mi$De#uZ18O_qT3mfv}4!1Gur#F`xkR({N`^yd^b?f zV9oMus$M6b$wYE1aX#zwEuBUIoo3R`#a(DKMFsaO{aS>VeF5)GUEYa^XFGyJB))$9 zk`Y%LC-W1JaA_o(Rj@Q&`Q%>qvGW?1o>wB6147Trd1ts8nCEs7XZOb>->vEUjqTb< zyv=XE(8dvyBOhB_SRE)KigEbUxW#_c9`W!mnN0@7{?x>gFKW;h8X_{a1d^f3@cfT$Tv!%R_L$#BJ z6Jte3Bk%s3EeS?bb^t0Z;bBVAmvJJ`+T=+8^D+n;wKlOhQVU(T_VJosm>HYaisa?lCKLXSzTBhnXIjZ@Fen}7uM#>mdCm}BFE9b>vNTj&+qGcc>|3# zYKz6%%+l;MaXvD!TWNiAS}amm46@iUB$n8X_FJUhPqcunPYq0t>XOHKc*@XA3*Pv9 z*^m<7^2+(15}(^thuQZas?pEek zZ-J()uOD9>z?PvTy(Cdnh~b{nXLs@s6x=B|P(C0CK607a1*I{m`cSk5oGRwz#^u+V|}bv7ShtP z!lX%EvCD1vEoIkvy?cTpIwtW2QjLZJ@ud$o$t5QsCo1Rhr6WW+5BD!qt+P3dHimCS zZ=3NGH9ca!NVd7>GF~-&CmLgRf@tJH_~OHj1X2=@K7%sEtOq zOB1m6J*5&?2~>Ob=Bx}eV`83%>*(k90Rj~PDII6cJMYlaDw=C{m008PX!LAHOx}6N zGLrFIQdINaOfKUycn|n!eR~oCiU`?YWI;B+zVnlfLf9B7_QyV7#v5VTuoRfI@ktHr zEIfzz!*{Lk&A*xXKV-rAoG zmcAx}oJVGznWC4-1#@h2OWzlQobQK!s+YfSq)^1ahbrA2;os<>>?6$c3na7DzFa-m zj`X6jPkyO=uk>8|(+j4a5R(4Ilur2Hq>&;S!X{ytNhJFIzF=2ZQbB>Wsi|a^(_x?5 zf`V(Nrc%xAP5%UC5=;>J%}$wJ#zcY#VzjyaU(R43BJ4>G8)xegDK+uUUvQf2>LZdo zv476!2pZdHNIW$C%b+htB@xz;dT90+TR)6dqJ8@t*F;#i#^1E@y_fmW@qIn~r5e7G z3*3{6eLhwH4hDV>K1yj%CTJico4GR;jeY)3278ePLRZ4nM7xEppFr|;l(00>Ze`=o zBZWIk+feUj{uQHyIs4HM?av8@i0o$b%o1at|33_~mTp`FZWu45C4w=);+k>81R;zF z#ukff$9>LAdnbbNz&`87ea=tYE&41N`>YT5xgbqT^w~WuZV<<+$et zv?Prww<(xtAhN;Z1VLYiGlky4)%1WC5XAt$faoe7Ckpxkq8hlGHPAX`Oo;=20g){p zCj3(<&Re=D;Jml+@Qdr z{@dqaw2E_w_@1C~x;dUbT}R}*PVWiKfN9FkyntjttUQ4Ry%y>>)%tTR07sCc{2kEyTL9oxo6!feXEC}xwIAGsz9iYXzjYL>mP;qqA zj;h|q#<}msevwPY=fd)W>ONi{DAziGycXI~d_TwTVbBA3;_00NV(+K0H|bQ;l1epG zl_id4c`xDH&V z8GdKqEqT^Nw$ic)R|nh4-9wHSdnC@fNx8T>ancI1u+{DJj8OQPZEG!A0xf=cAac)v z@tZHX7M`9O^6igVd}7!}Z9R4%;^f0;iJ}U9e(j_q(23%@>7@2NL1?X*i}AeOV<*&| zHfBC*^G(Gjazg&Ai3R3H=9c%*)fUq~c|NMW>$~`Vi3ddD>-ybBD*})c{~W#i_jvvO z{U=!F7tjA_;74fX-xW8$pMa$J1JLqT#UJ5j|EO3EkPQDi`0Q3YTOmikTVmSv69DN} z<*jg%-<6%fLlZEFpTH%zTG$G`_}#+IJ)0J`0x@niuod^}y8%rA<+w3~zo1}kRow~` z^j%e-?JutX1RAu}%8$>1zT2RoDRiKr`RS?9R`sp-`0obfoHqyY!>xX+_Ky|xyDkmQ e3hED?f2*e#&(njov4P>uNK* Date: Mon, 7 Oct 2019 16:28:55 +0200 Subject: [PATCH 15/22] revert all code style changes coudn't find any finals --- .../DefaultSpreadsheetReaderDelegate.java | 59 +++++++++---------- .../metamodel/excel/ExcelConfiguration.java | 12 ++-- .../metamodel/excel/ExcelDataContextTest.java | 1 + 3 files changed, 34 insertions(+), 38 deletions(-) diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index 687acc672..1f4a49b28 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -57,22 +57,22 @@ final class DefaultSpreadsheetReaderDelegate implements SpreadsheetReaderDelegat private static final Logger logger = LoggerFactory.getLogger(DefaultSpreadsheetReaderDelegate.class); - private final Resource _resource; + private final Resource _resource; private final ExcelConfiguration _configuration; public DefaultSpreadsheetReaderDelegate(Resource resource, ExcelConfiguration configuration) { - _resource = resource; + _resource = resource; _configuration = configuration; } @Override public Schema createSchema(String schemaName) { final MutableSchema schema = new MutableSchema(schemaName); - final Workbook wb = ExcelUtils.readWorkbook(_resource, true); + final Workbook wb = ExcelUtils.readWorkbook(_resource, true); try { for (int i = 0; i < wb.getNumberOfSheets(); i++) { - final Sheet currentSheet = wb.getSheetAt(i); - final MutableTable table = createTable(wb, currentSheet); + final Sheet currentSheet = wb.getSheetAt(i); + final MutableTable table = createTable(wb, currentSheet); table.setSchema(schema); schema.addTable(table); } @@ -85,8 +85,8 @@ public Schema createSchema(String schemaName) { @Override public DataSet executeQuery(Table table, List columns, int maxRows) { - final Workbook wb = ExcelUtils.readWorkbook(_resource, true); - final Sheet sheet = wb.getSheet(table.getName()); + final Workbook wb = ExcelUtils.readWorkbook(_resource, true); + final Sheet sheet = wb.getSheet(table.getName()); if (sheet == null || sheet.getPhysicalNumberOfRows() == 0) { return new EmptyDataSet(columns.stream().map(SelectItem::new).collect(Collectors.toList())); @@ -126,14 +126,12 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) { while (row == null && rowIterator.hasNext()) { row = rowIterator.next(); } - } else { row = rowIterator.next(); } final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); final ColumnType[] columnTypes = getColumnTypes(sheet, row); - if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) { // get to the first non-empty line (no matter if lines are skipped @@ -154,8 +152,7 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) { for (int j = offset; j < row.getLastCellNum(); j++) { final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); final Column column; - if (_configuration.isValidateColumnTypes()) { - + if (_configuration.isDetectColumnTypes()) { column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), columnTypes[j], table, j, true); @@ -191,10 +188,10 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) { } private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) { - final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); - final int rowLength = row.getLastCellNum(); - int eagerness = _configuration.getEagerness(); - final ColumnType[] columnTypes = new ColumnType[rowLength]; + final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); + final int rowLength = row.getLastCellNum(); + int eagerness = _configuration.getEagerness(); + final ColumnType[] columnTypes = new ColumnType[rowLength]; while (data.hasNext() && eagerness-- > 0) { final Row currentRow = data.next(); @@ -206,7 +203,7 @@ private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) { continue; } - ColumnType columnType = columnTypes[index]; + ColumnType columnType = columnTypes[index]; ColumnType expecetedColumnType = getColumnTypeFromRow(currentRow, index); if (columnType != null) { if (!columnType.equals(ColumnType.STRING) && !columnType.equals(expecetedColumnType)) { @@ -230,8 +227,8 @@ private ColumnType getColumnTypeFromRow(final Row currentRow, int index) { if (DateUtil.isCellDateFormatted(currentRow.getCell(index))) { return ColumnType.DATE; } else { - return (currentRow.getCell(index).getNumericCellValue() % 1 == 0) ? ColumnType.INTEGER - : ColumnType.DOUBLE; + return (currentRow.getCell(index).getNumericCellValue() % 1 == 0) + ? ColumnType.INTEGER : ColumnType.DOUBLE; } case BOOLEAN: return ColumnType.BOOLEAN; @@ -265,21 +262,19 @@ private void createColumns(final MutableTable table, final Workbook wb, final Ro return; } final short rowLength = row.getLastCellNum(); - final int offset = getColumnOffset(row); // build columns based on cell values. - try (final ColumnNamingSession columnNamingSession = - _configuration.getColumnNamingStrategy().startColumnNamingSession()) { + try (final ColumnNamingSession columnNamingSession = _configuration.getColumnNamingStrategy() + .startColumnNamingSession()) { for (int j = offset; j < rowLength; j++) { - final Cell cell = row.getCell(j); - final String intrinsicColumnName = ExcelUtils.getCellValue(wb, cell); - final ColumnNamingContext columnNamingContext = - new ColumnNamingContextImpl(table, intrinsicColumnName, j); - final String columnName = - columnNamingSession.getNextColumnName(columnNamingContext); - final Column column; - if (!_configuration.isValidateColumnTypes()) { + final Cell cell = row.getCell(j); + final String intrinsicColumnName = ExcelUtils.getCellValue(wb, cell); + final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName, + j); + final String columnName = columnNamingSession.getNextColumnName(columnNamingContext); + final Column column; + if (!_configuration.isDetectColumnTypes()) { column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true); } else { column = new MutableColumn(columnName, columTypes[j], table, j, true); @@ -290,9 +285,9 @@ private void createColumns(final MutableTable table, final Workbook wb, final Ro } /** - * Gets the column offset (first column to include). This is dependent on the - * row used for column processing and whether the skip empty columns property is - * set. + * Gets the column offset (first column to include). This is dependent on + * the row used for column processing and whether the skip empty columns + * property is set. * * @param row * @return diff --git a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java index d94caac73..c15908cbe 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java +++ b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java @@ -58,17 +58,17 @@ public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnN this(columnNameLineNumber, columnNamingStrategy, skipEmptyLines, skipEmptyColumns, false, 1000); } - public ExcelConfiguration(int columnNameLineNumber, boolean skipEmptyLines, boolean skipEmptyColumns, boolean validateColumnTypes) { - this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, validateColumnTypes, 1000); + public ExcelConfiguration(int columnNameLineNumber, boolean skipEmptyLines, boolean skipEmptyColumns, boolean detectColumnTypes) { + this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, detectColumnTypes, 1000); } public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnNamingStrategy, - boolean skipEmptyLines, boolean skipEmptyColumns, boolean validateColumnTypes, int eagerness) { + boolean skipEmptyLines, boolean skipEmptyColumns, boolean detectColumnTypes, int eagerness) { this.columnNameLineNumber = columnNameLineNumber; this.skipEmptyLines = skipEmptyLines; this.skipEmptyColumns = skipEmptyColumns; this.columnNamingStrategy = columnNamingStrategy; - this.detectColumnTypes = validateColumnTypes; + this.detectColumnTypes = detectColumnTypes; this.eagerReader = eagerness; } @@ -121,7 +121,7 @@ public boolean isSkipEmptyColumns() { * * @return a boolean indicating whether or not to validate column types. */ - public boolean isValidateColumnTypes() { + public boolean isDetectColumnTypes() { return detectColumnTypes; } @@ -137,7 +137,7 @@ protected void decorateIdentity(List identifiers) { public String toString() { return "ExcelConfiguration[columnNameLineNumber=" + columnNameLineNumber + ", skipEmptyLines=" + skipEmptyLines - + ", skipEmptyColumns=" + skipEmptyColumns +", validateColumnTypes=" + + ", skipEmptyColumns=" + skipEmptyColumns +", detectColumnTypes=" + detectColumnTypes + "]"; } diff --git a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java index cda4bc3b9..b398a0d32 100644 --- a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java +++ b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java @@ -546,6 +546,7 @@ public void testTicket99defect() throws Exception { Table table = schema.getTableByName("Sheet1"); assertEquals( + "[Column[name=Pkg No.,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " + "Column[name=Description,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " + "Column[name=Room,columnNumber=2,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], " From 138b1471a0818ebd10c804ffef923f4395e3462c Mon Sep 17 00:00:00 2001 From: Jan Bob Date: Tue, 8 Oct 2019 14:51:29 +0200 Subject: [PATCH 16/22] wrote some tests, added update check --- .../metamodel/DeleteAndInsertBuilder.java | 21 ++++ .../DefaultSpreadsheetReaderDelegate.java | 15 +-- .../metamodel/excel/ExcelInsertBuilder.java | 39 ++++++-- .../metamodel/excel/ExcelDataContextTest.java | 97 +++++++++++++++++++ 4 files changed, 159 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/apache/metamodel/DeleteAndInsertBuilder.java b/core/src/main/java/org/apache/metamodel/DeleteAndInsertBuilder.java index 172432dd1..670dbb75b 100644 --- a/core/src/main/java/org/apache/metamodel/DeleteAndInsertBuilder.java +++ b/core/src/main/java/org/apache/metamodel/DeleteAndInsertBuilder.java @@ -69,12 +69,33 @@ public void execute() throws MetaModelException { private List updateRows(List rows) { for (ListIterator it = rows.listIterator(); it.hasNext();) { final Row original = (Row) it.next(); + validateUpdateType(original); final Row updated = update(original); it.set(updated); } return rows; } + private void validateUpdateType(final Row original) { + for (int index = 0; index < this.getColumns().length; index++) { + switch(getColumns()[index].getType().getName()) { + case "INTEGER" : + try { + Integer.decode(getValues()[index].toString()); + } catch (NumberFormatException ex) { + throw new MetaModelException(original.getValue(index).toString() + " should be an Integer!"); + } + break; + case "STRING" : + // fall through + case "VARCHAR" : + // fall through + default : + break; + } + } + } + /** * Produces an updated row out of the original * diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index 1f4a49b28..f2b3135a2 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -130,8 +130,8 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) { row = rowIterator.next(); } - final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); - final ColumnType[] columnTypes = getColumnTypes(sheet, row); + final int columnNameLineNumber = _configuration.getColumnNameLineNumber(); + final ColumnType[] columnTypes = getColumnTypes(sheet, row); if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) { // get to the first non-empty line (no matter if lines are skipped @@ -151,7 +151,7 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) { for (int j = offset; j < row.getLastCellNum(); j++) { final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); - final Column column; + final Column column; if (_configuration.isDetectColumnTypes()) { column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), columnTypes[j], @@ -262,15 +262,16 @@ private void createColumns(final MutableTable table, final Workbook wb, final Ro return; } final short rowLength = row.getLastCellNum(); + final int offset = getColumnOffset(row); // build columns based on cell values. try (final ColumnNamingSession columnNamingSession = _configuration.getColumnNamingStrategy() .startColumnNamingSession()) { for (int j = offset; j < rowLength; j++) { - final Cell cell = row.getCell(j); + final Cell cell = row.getCell(j); final String intrinsicColumnName = ExcelUtils.getCellValue(wb, cell); - final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName, + final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName, j); final String columnName = columnNamingSession.getNextColumnName(columnNamingContext); final Column column; @@ -285,8 +286,8 @@ private void createColumns(final MutableTable table, final Workbook wb, final Ro } /** - * Gets the column offset (first column to include). This is dependent on - * the row used for column processing and whether the skip empty columns + * Gets the column offset (first column to include). This is dependent on + * the row used for column processing and whether the skip empty columns * property is set. * * @param row diff --git a/excel/src/main/java/org/apache/metamodel/excel/ExcelInsertBuilder.java b/excel/src/main/java/org/apache/metamodel/excel/ExcelInsertBuilder.java index b584e7693..80c8165f7 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/ExcelInsertBuilder.java +++ b/excel/src/main/java/org/apache/metamodel/excel/ExcelInsertBuilder.java @@ -20,12 +20,7 @@ import java.util.Date; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.CellStyle; -import org.apache.poi.ss.usermodel.FillPatternType; -import org.apache.poi.ss.usermodel.Font; -import org.apache.poi.ss.usermodel.HorizontalAlignment; -import org.apache.poi.ss.usermodel.Row; +import org.apache.metamodel.MetaModelException; import org.apache.metamodel.data.Style; import org.apache.metamodel.data.Style.Color; import org.apache.metamodel.data.Style.SizeUnit; @@ -33,8 +28,15 @@ import org.apache.metamodel.insert.AbstractRowInsertionBuilder; import org.apache.metamodel.insert.RowInsertionBuilder; import org.apache.metamodel.schema.Column; +import org.apache.metamodel.schema.ColumnType; import org.apache.metamodel.schema.Table; import org.apache.metamodel.util.LazyRef; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.FillPatternType; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.HorizontalAlignment; +import org.apache.poi.ss.usermodel.Row; /** * {@link RowInsertionBuilder} for excel spreadsheets. @@ -149,8 +151,33 @@ protected CellStyle fetch() { cell.setCellStyle(cellStyle.get()); } } + validateUpdateType(row); } } + + private void validateUpdateType(final Row original) { + for (int index = 0; index < this.getColumns().length; index++) { + final ColumnType columnType = getColumns()[index].getType(); + if (columnType != null && getValues()[index] != null) { + switch (columnType.getName()) { + case "INTEGER": + try { + Integer.decode(getValues()[index].toString()); + } catch (NumberFormatException ex) { + throw new MetaModelException(original.getCell(index) + + " should be an Integer!"); + } + break; + case "STRING": + // fall through + case "VARCHAR": + // fall through + default: + break; + } + } + } + } /** * Converts a percentage based font size to excel "pt" scale. diff --git a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java index b398a0d32..173b1df7b 100644 --- a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java +++ b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.metamodel.DataContext; +import org.apache.metamodel.MetaModelException; import org.apache.metamodel.MetaModelHelper; import org.apache.metamodel.UpdateCallback; import org.apache.metamodel.UpdateScript; @@ -432,6 +433,102 @@ public void testDifferentDataTypes() throws Exception { + "Column[name=FORMULA,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null]]", Arrays.toString(table.getColumns().toArray())); } + + public void testInsertingDifferentDataTypes() { + ExcelDataContext dc = new ExcelDataContext(copyOf("src/test/resources/different_datatypes.xls"), + new ExcelConfiguration(ExcelConfiguration.DEFAULT_COLUMN_NAME_LINE, true, false, true)); + Schema schema = dc.getDefaultSchema(); + assertEquals(2, schema.getTableCount()); + + Table table = schema.getTable(0); + + dc.executeUpdate(new UpdateScript() { + + @Override + public void run(UpdateCallback callback) { + try { + callback.update(table).value("INTEGER", "this is not an integer").execute(); + } catch (Exception ex) { + assert true; + } + } + + }); + + dc.executeUpdate(new UpdateScript() { + + @Override + public void run(UpdateCallback callback) { + try { + callback.update(table).value("INTEGER", 123).execute(); + } catch (Exception ex) { + assert false; + } + } + + }); + } + + public void testUpdateDifferentDataTypes() { + ExcelDataContext dc = new ExcelDataContext(copyOf("src/test/resources/different_datatypes.xls"), + new ExcelConfiguration(ExcelConfiguration.DEFAULT_COLUMN_NAME_LINE, true, false, true)); + Schema schema = dc.getDefaultSchema(); + assertEquals(2, schema.getTableCount()); + + Table table = schema.getTable(0); + + dc.executeUpdate(new UpdateScript() { + @Override + public void run(UpdateCallback callback) { + try { + callback.insertInto(table).value("INTEGER", "this is not an integer").execute(); + assert true; + } catch (MetaModelException mme) { + assert true; + } + } + + }); + + dc.executeUpdate(new UpdateScript() { + @Override + public void run(UpdateCallback callback) { + try { + callback.insertInto(table).value("TEXT", "this is not an integer").execute(); + assert true; + } catch (MetaModelException mme) { + assert false; + } + } + + }); + + dc.executeUpdate(new UpdateScript() { + @Override + public void run(UpdateCallback callback) { + try { + callback.insertInto(table).value("INTEGER", 1234).execute(); + assert true; + } catch (MetaModelException mme) { + assert false; + } + } + + }); + + dc.executeUpdate(new UpdateScript() { + @Override + public void run(UpdateCallback callback) { + try { + callback.insertInto(table).value("TEXT", 1234).execute(); + assert true; + } catch (MetaModelException mme) { + assert false; + } + } + + }); + } public void testMissingColumnHeader() throws Exception { File file = copyOf("src/test/resources/xls_missing_column_header.xls"); From 5088744296b3d559665496a44d4498b1f23bfec9 Mon Sep 17 00:00:00 2001 From: Pixel Date: Tue, 8 Oct 2019 21:02:10 +0200 Subject: [PATCH 17/22] fixed assert that was set wrong --- .../java/org/apache/metamodel/excel/ExcelDataContextTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java index 173b1df7b..205c3ddef 100644 --- a/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java +++ b/excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java @@ -482,7 +482,7 @@ public void testUpdateDifferentDataTypes() { public void run(UpdateCallback callback) { try { callback.insertInto(table).value("INTEGER", "this is not an integer").execute(); - assert true; + assert false; } catch (MetaModelException mme) { assert true; } From 4bfcc410bcd34f5bc63cd7c10c4e86e5bba792bf Mon Sep 17 00:00:00 2001 From: Jan Bob Date: Mon, 14 Oct 2019 17:00:48 +0200 Subject: [PATCH 18/22] resolving review comments 1 of many --- .../metamodel/DeleteAndInsertBuilder.java | 21 ------------------- .../DefaultSpreadsheetReaderDelegate.java | 11 +++++----- .../metamodel/excel/ExcelConfiguration.java | 6 +++--- .../excel/ExcelConfigurationTest.java | 2 +- 4 files changed, 9 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/org/apache/metamodel/DeleteAndInsertBuilder.java b/core/src/main/java/org/apache/metamodel/DeleteAndInsertBuilder.java index 670dbb75b..172432dd1 100644 --- a/core/src/main/java/org/apache/metamodel/DeleteAndInsertBuilder.java +++ b/core/src/main/java/org/apache/metamodel/DeleteAndInsertBuilder.java @@ -69,33 +69,12 @@ public void execute() throws MetaModelException { private List updateRows(List rows) { for (ListIterator it = rows.listIterator(); it.hasNext();) { final Row original = (Row) it.next(); - validateUpdateType(original); final Row updated = update(original); it.set(updated); } return rows; } - private void validateUpdateType(final Row original) { - for (int index = 0; index < this.getColumns().length; index++) { - switch(getColumns()[index].getType().getName()) { - case "INTEGER" : - try { - Integer.decode(getValues()[index].toString()); - } catch (NumberFormatException ex) { - throw new MetaModelException(original.getValue(index).toString() + " should be an Integer!"); - } - break; - case "STRING" : - // fall through - case "VARCHAR" : - // fall through - default : - break; - } - } - } - /** * Produces an updated row out of the original * diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index f2b3135a2..c2aa4eecd 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -203,14 +203,14 @@ private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) { continue; } - ColumnType columnType = columnTypes[index]; - ColumnType expecetedColumnType = getColumnTypeFromRow(currentRow, index); + final ColumnType columnType = columnTypes[index]; + final ColumnType expectedColumnType = getColumnTypeFromRow(currentRow, index); if (columnType != null) { - if (!columnType.equals(ColumnType.STRING) && !columnType.equals(expecetedColumnType)) { + if (!columnType.equals(ColumnType.STRING) && !columnType.equals(expectedColumnType)) { columnTypes[index] = ColumnType.VARCHAR; } } else { - columnTypes[index] = expecetedColumnType; + columnTypes[index] = expectedColumnType; } } } @@ -221,8 +221,7 @@ private ColumnType getColumnTypeFromRow(final Row currentRow, int index) { if (currentRow.getCell(index) == null) { return ColumnType.STRING; } else { - CellType cellType = currentRow.getCell(index).getCellType(); - switch (cellType) { + switch (currentRow.getCell(index).getCellType()) { case NUMERIC: if (DateUtil.isCellDateFormatted(currentRow.getCell(index))) { return ColumnType.DATE; diff --git a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java index c15908cbe..d70aefb17 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java +++ b/excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java @@ -38,7 +38,7 @@ public final class ExcelConfiguration extends BaseObject implements public static final int NO_COLUMN_NAME_LINE = 0; public static final int DEFAULT_COLUMN_NAME_LINE = 1; - private final int eagerReader; + private final int getNumberOfLinesToScan; private final int columnNameLineNumber; private final ColumnNamingStrategy columnNamingStrategy; private final boolean skipEmptyLines; @@ -69,7 +69,7 @@ public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnN this.skipEmptyColumns = skipEmptyColumns; this.columnNamingStrategy = columnNamingStrategy; this.detectColumnTypes = detectColumnTypes; - this.eagerReader = eagerness; + this.getNumberOfLinesToScan = eagerness; } /** @@ -142,6 +142,6 @@ public String toString() { } public int getEagerness() { - return eagerReader; + return getNumberOfLinesToScan; } } diff --git a/excel/src/test/java/org/apache/metamodel/excel/ExcelConfigurationTest.java b/excel/src/test/java/org/apache/metamodel/excel/ExcelConfigurationTest.java index 647c11196..a9a0bff64 100644 --- a/excel/src/test/java/org/apache/metamodel/excel/ExcelConfigurationTest.java +++ b/excel/src/test/java/org/apache/metamodel/excel/ExcelConfigurationTest.java @@ -27,7 +27,7 @@ public class ExcelConfigurationTest extends TestCase { public void testToString() throws Exception { ExcelConfiguration conf = new ExcelConfiguration(1, true, false); assertEquals( - "ExcelConfiguration[columnNameLineNumber=1, skipEmptyLines=true, skipEmptyColumns=false, validateColumnTypes=false]", + "ExcelConfiguration[columnNameLineNumber=1, skipEmptyLines=true, skipEmptyColumns=false, detectColumnTypes=false]", conf.toString()); } From 0701bab4d161632d59a2058e18f40d9e58c15433 Mon Sep 17 00:00:00 2001 From: Jan Bob Date: Wed, 16 Oct 2019 11:46:21 +0200 Subject: [PATCH 19/22] resolving review comments --- .../DefaultSpreadsheetReaderDelegate.java | 49 +++++++++---------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java index c2aa4eecd..7ff6f061a 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java @@ -18,6 +18,7 @@ */ package org.apache.metamodel.excel; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; @@ -41,7 +42,6 @@ import org.apache.metamodel.util.FileHelper; import org.apache.metamodel.util.Resource; import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.CellType; import org.apache.poi.ss.usermodel.DateUtil; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; @@ -151,16 +151,8 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) { for (int j = offset; j < row.getLastCellNum(); j++) { final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j); - final Column column; - if (_configuration.isDetectColumnTypes()) { - column = - new MutableColumn(columnNamingSession.getNextColumnName(namingContext), columnTypes[j], - table, j, true); - } else { - column = - new MutableColumn(columnNamingSession.getNextColumnName(namingContext), - ColumnType.STRING, table, j, true); - } + final Column column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext), + columnTypes[j], table, j, true); table.addColumn(column); } } @@ -190,29 +182,34 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) { private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) { final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); final int rowLength = row.getLastCellNum(); - int eagerness = _configuration.getEagerness(); final ColumnType[] columnTypes = new ColumnType[rowLength]; + if (_configuration.isDetectColumnTypes()) { - while (data.hasNext() && eagerness-- > 0) { - final Row currentRow = data.next(); - if (currentRow.getRowNum() < _configuration.getColumnNameLineNumber()) { - continue; - } - for (int index = 0; index < rowLength; index++) { - if (currentRow.getLastCellNum() == 0) { + int eagerness = _configuration.getEagerness(); + + while (data.hasNext() && eagerness-- > 0) { + final Row currentRow = data.next(); + if (currentRow.getRowNum() < _configuration.getColumnNameLineNumber()) { continue; } + for (int index = 0; index < rowLength; index++) { + if (currentRow.getLastCellNum() == 0) { + continue; + } - final ColumnType columnType = columnTypes[index]; - final ColumnType expectedColumnType = getColumnTypeFromRow(currentRow, index); - if (columnType != null) { - if (!columnType.equals(ColumnType.STRING) && !columnType.equals(expectedColumnType)) { - columnTypes[index] = ColumnType.VARCHAR; + final ColumnType columnType = columnTypes[index]; + final ColumnType expectedColumnType = getColumnTypeFromRow(currentRow, index); + if (columnType != null) { + if (!columnType.equals(ColumnType.STRING) && !columnType.equals(expectedColumnType)) { + columnTypes[index] = ColumnType.VARCHAR; + } + } else { + columnTypes[index] = expectedColumnType; } - } else { - columnTypes[index] = expectedColumnType; } } + } else { + Arrays.fill(columnTypes, ColumnType.STRING); } return columnTypes; } From 9c3976ee7ae2d312b7efcbe204fbb527ef8a2243 Mon Sep 17 00:00:00 2001 From: Jan Bob Date: Wed, 23 Oct 2019 16:40:47 +0200 Subject: [PATCH 20/22] resolving review comments, still there are a few thingies that could be better coded --- .../apache/metamodel/excel/ExcelUtils.java | 4 +- .../excel/XlsxSpreadsheetReaderDelegate.java | 155 +++++++++++++++++- .../metamodel/excel/ExcelDataContextTest.java | 12 +- .../test/resources/different_datatypes.xlsx | Bin 0 -> 9288 bytes 4 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 excel/src/test/resources/different_datatypes.xlsx diff --git a/excel/src/main/java/org/apache/metamodel/excel/ExcelUtils.java b/excel/src/main/java/org/apache/metamodel/excel/ExcelUtils.java index 2da6ef39b..352dcb4b6 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/ExcelUtils.java +++ b/excel/src/main/java/org/apache/metamodel/excel/ExcelUtils.java @@ -414,13 +414,13 @@ public static Iterator getRowIterator(Sheet sheet, ExcelConfiguration confi */ public static DefaultRow createRow(Workbook workbook, Row row, DataSetHeader header) { final int size = header.size(); - final String[] values = new String[size]; + final Object[] values = new Object[size]; final Style[] styles = new Style[size]; if (row != null) { for (int i = 0; i < size; i++) { final int columnNumber = header.getSelectItem(i).getColumn().getColumnNumber(); final Cell cell = row.getCell(columnNumber); - final String value = ExcelUtils.getCellValue(workbook, cell); + final Object value = ExcelUtils.getCellValue(workbook, cell); final Style style = ExcelUtils.getCellStyle(workbook, cell); values[i] = value; styles[i] = style; diff --git a/excel/src/main/java/org/apache/metamodel/excel/XlsxSpreadsheetReaderDelegate.java b/excel/src/main/java/org/apache/metamodel/excel/XlsxSpreadsheetReaderDelegate.java index 81214fe7b..815195d16 100644 --- a/excel/src/main/java/org/apache/metamodel/excel/XlsxSpreadsheetReaderDelegate.java +++ b/excel/src/main/java/org/apache/metamodel/excel/XlsxSpreadsheetReaderDelegate.java @@ -22,6 +22,8 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -39,6 +41,7 @@ import org.apache.metamodel.schema.MutableTable; import org.apache.metamodel.schema.Schema; import org.apache.metamodel.schema.Table; +import org.apache.metamodel.schema.naming.ColumnNamingContext; import org.apache.metamodel.schema.naming.ColumnNamingContextImpl; import org.apache.metamodel.schema.naming.ColumnNamingSession; import org.apache.metamodel.schema.naming.ColumnNamingStrategy; @@ -46,6 +49,11 @@ import org.apache.metamodel.util.FileResource; import org.apache.metamodel.util.Resource; import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.DateUtil; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.xssf.eventusermodel.XSSFReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -152,10 +160,150 @@ public void close() throws IOException { }); } + private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) { + final Iterator data = ExcelUtils.getRowIterator(sheet, _configuration, false); + final int rowLength = row.getLastCellNum(); + final ColumnType[] columnTypes = new ColumnType[rowLength]; + if (_configuration.isDetectColumnTypes()) { + + int eagerness = _configuration.getEagerness(); + + while (data.hasNext() && eagerness-- > 0) { + final Row currentRow = data.next(); + if (currentRow.getRowNum() < _configuration.getColumnNameLineNumber()) { + continue; + } + for (int index = 0; index < rowLength; index++) { + if (currentRow.getLastCellNum() == 0) { + continue; + } + + final ColumnType columnType = columnTypes[index]; + final ColumnType expectedColumnType = getColumnTypeFromRow(currentRow, index); + if (columnType != null) { + if (!columnType.equals(ColumnType.STRING) && !columnType.equals(expectedColumnType)) { + columnTypes[index] = ColumnType.VARCHAR; + } + } else { + columnTypes[index] = expectedColumnType; + } + } + } + } else { + Arrays.fill(columnTypes, ColumnType.STRING); + } + return columnTypes; + } + + private ColumnType getColumnTypeFromRow(final Row currentRow, int index) { + if (currentRow.getCell(index) == null) { + return ColumnType.STRING; + } else { + switch (currentRow.getCell(index).getCellType()) { + case NUMERIC: + if (DateUtil.isCellDateFormatted(currentRow.getCell(index))) { + return ColumnType.DATE; + } else { + return (currentRow.getCell(index).getNumericCellValue() % 1 == 0) ? ColumnType.INTEGER + : ColumnType.DOUBLE; + } + case BOOLEAN: + return ColumnType.BOOLEAN; + case ERROR: + // fall through + case _NONE: + // fall through + case STRING: + // fall through + case FORMULA: + // fall through + case BLANK: + // fall through + default: + return ColumnType.STRING; + } + } + } + + /** + * Builds columns based on row/cell values. + * + * @param table + * @param wb + * @param row + */ + private void createColumns(final MutableTable table, final Workbook wb, final Row row, + final ColumnType[] columTypes) { + if (row == null) { + logger.warn("Cannot create columns based on null row!"); + return; + } + final short rowLength = row.getLastCellNum(); + + final int offset = getColumnOffset(row); + + // build columns based on cell values. + try (final ColumnNamingSession columnNamingSession = _configuration + .getColumnNamingStrategy() + .startColumnNamingSession()) { + for (int j = offset; j < rowLength; j++) { + final Cell cell = row.getCell(j); + final String intrinsicColumnName = ExcelUtils.getCellValue(wb, cell); + final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName, + j); + final String columnName = columnNamingSession.getNextColumnName(columnNamingContext); + final Column column; + if (!_configuration.isDetectColumnTypes()) { + column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true); + } else { + column = new MutableColumn(columnName, columTypes[j], table, j, true); + } + table.addColumn(column); + } + } + } + + /** + * Gets the column offset (first column to include). This is dependent on + * the row used for column processing and whether the skip empty columns + * property is set. + * + * @param row + * @return + */ + private int getColumnOffset(Row row) { + final int offset; + if (_configuration.isSkipEmptyColumns()) { + offset = row.getFirstCellNum(); + } else { + offset = 0; + } + return offset; + } + private void buildColumns(final MutableTable table, final String relationshipId, final XSSFReader xssfReader) throws Exception { final InputStream sheetData = xssfReader.getSheet(relationshipId); + final Workbook wb = ExcelUtils.readWorkbook(_resource, true); + Sheet sheet = wb.getSheetAt(0); + final Iterator rowIterator = ExcelUtils.getRowIterator(sheet, _configuration, false); + + Row row = null; + if (!rowIterator.hasNext()) { + // no physical rows in sheet + return; + } + + if (_configuration.isSkipEmptyLines()) { + while (row == null && rowIterator.hasNext()) { + row = rowIterator.next(); + } + } else { + row = rowIterator.next(); + } + final Row currentRow = row; + final XlsxRowCallback rowCallback = new XlsxRowCallback() { @Override public boolean row(int rowNumber, List values, List