Skip to content

Commit

Permalink
feat: add GUI parameter to select date formatter for the Excel/CSV im…
Browse files Browse the repository at this point in the history 
…porter

Add a combo box with different patterns for the date formatter + a custom text. This pattern is then used when importing cells of type Date.

ING-4165
  • Loading branch information
@emanuelaepure10
emanuelaepure10 committed Feb 5, 2024
1 parent bb04e1d commit c3e9ecf
Show file tree
Hide file tree
Showing 13 changed files with 278 additions and 38 deletions.
3 changes: 3 additions & 0 deletions io/plugins/eu.esdihumboldt.hale.io.csv.test/src/data/test3.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Name Xcoord Ycoord date
test 12 16 1.02.2023
test3 45 2 22.03.2023
90 changes: 90 additions & 0 deletions ....esdihumboldt.hale.io.csv.test/src/eu/esdihumboldt/hale/io/csv/CSVInstanceReaderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,14 @@
package eu.esdihumboldt.hale.io.csv;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;

import javax.xml.namespace.QName;

import org.junit.BeforeClass;
Expand All @@ -35,6 +41,7 @@
import eu.esdihumboldt.hale.common.schema.model.Schema;
import eu.esdihumboldt.hale.common.schema.model.TypeDefinition;
import eu.esdihumboldt.hale.common.test.TestUtil;
import eu.esdihumboldt.hale.io.csv.reader.CSVConstants;
import eu.esdihumboldt.hale.io.csv.reader.CommonSchemaConstants;
import eu.esdihumboldt.hale.io.csv.reader.internal.CSVInstanceReader;
import eu.esdihumboldt.hale.io.csv.reader.internal.CSVSchemaReader;
Expand Down Expand Up @@ -238,6 +245,67 @@ public void testReadWithCommaDecimal() throws Exception {

}

/**
* Test - read a sample csv schema and data.
*
* @throws Exception , if an error occurs
*/
@Test
public void testReadSimpleWithDate() throws Exception {

String typeName = "location";
String[] properties = { "Name", "Xcoord", "Ycoord", "date" };
String[] dataFirstColumn = { "test", "12", "16", "1.02.2023" };
String dateFormatter = "dd.MM.yyyy";
String sourceLocation = "/data/test3.csv";
// read Schema ###
Schema schema = readCSVSchemaDate(sourceLocation, typeName,
"java.lang.String,java.lang.String,java.lang.String,java.lang.String",
"Name,Xcoord,Ycoord,date", null, null, null, null, dateFormatter);

// read Instances ###
InstanceCollection instances = readCSVInstances(sourceLocation, typeName, 1, schema, null,
null, null);

// Check the values of the date property in each instance
Iterator<Instance> instanceIt = instances.iterator();
while (instanceIt.hasNext()) {
Instance instance = instanceIt.next();
// Get the value of the date property
Object[] value = instance.getProperty(QName.valueOf(properties[properties.length - 1]));

// Ensure the value is not null
assertNotNull("Date property value is null", value);

// Ensure the value is an array with at least one element
assertTrue("Date property value is not an array or is empty", value.length > 0);

// Check the date string format
String dateString = (String) value[0];
assertTrue("Date string format is incorrect: " + dateString,
isStringDate(dateString, dateFormatter));
}
}

/**
* @param input String
* @param dateFormatter date formatter
* @return true is the input String is of type Date
*/
public boolean isStringDate(String input, String dateFormatter) {
// Define the date format you expect
SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormatter);
dateFormat.setLenient(false); // Disable lenient parsing

try {
// Try parsing the input string as a date
Date parsedDate = dateFormat.parse(input);
return true; // Parsing successful, input is a valid date
} catch (ParseException e) {
return false; // Parsing failed, input is not a valid date
}
}

private int collectionSize(InstanceCollection instances) {
if (instances.hasSize()) {
return instances.size();
Expand Down Expand Up @@ -280,6 +348,28 @@ private Schema readCSVSchema(String sourceLocation, String typeName, String para
return schemaReader.getSchema();
}

private Schema readCSVSchemaDate(String sourceLocation, String typeName,
String paramPropertyType, String propertyNames, String seperator, String quote,
String escape, String decimal, String dateFormatter) throws Exception {

CSVSchemaReader schemaReader = new CSVSchemaReader();
schemaReader.setSource(
new DefaultInputSupplier(getClass().getResource(sourceLocation).toURI()));
schemaReader.setParameter(CommonSchemaConstants.PARAM_TYPENAME, Value.of(typeName));
schemaReader.setParameter(CSVSchemaReader.PARAM_PROPERTY, Value.of(propertyNames));
schemaReader.setParameter(CSVSchemaReader.PARAM_PROPERTYTYPE, Value.of(paramPropertyType));
schemaReader.setParameter(CSVSchemaReader.PARAM_SEPARATOR, Value.of(seperator));
schemaReader.setParameter(CSVSchemaReader.PARAM_QUOTE, Value.of(quote));
schemaReader.setParameter(CSVSchemaReader.PARAM_ESCAPE, Value.of(escape));
schemaReader.setParameter(CSVSchemaReader.PARAM_DECIMAL, Value.of(decimal));
schemaReader.setParameter(CSVConstants.PARAMETER_DATE_FORMAT, Value.of(dateFormatter));

IOReport report = schemaReader.execute(new LogProgressIndicator());
assertTrue(report.isSuccess());

return schemaReader.getSchema();
}

private InstanceCollection readCSVInstances(String sourceLocation, String typeName, int skipN,
Schema sourceSchema, String seperator, String quote, String escape) throws Exception {

Expand Down
78 changes: 58 additions & 20 deletions .../eu.esdihumboldt.hale.io.csv.ui/src/eu/esdihumboldt/hale/io/csv/ui/TypeSelectionPage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@
import eu.esdihumboldt.hale.io.csv.reader.CSVConstants;
import eu.esdihumboldt.hale.io.csv.reader.CommonSchemaConstants;
import eu.esdihumboldt.hale.io.csv.reader.internal.CSVConfiguration;
import eu.esdihumboldt.hale.io.xls.reader.ReaderSettings;
import eu.esdihumboldt.hale.ui.common.definition.selector.TypeDefinitionSelector;
import eu.esdihumboldt.hale.ui.io.instance.InstanceReaderConfigurationPage;
import eu.esdihumboldt.hale.ui.service.schema.SchemaService;
Expand All @@ -59,15 +58,22 @@ public class TypeSelectionPage extends InstanceReaderConfigurationPage implement
/**
* Parameter for the the custom date label
*/
private static final String CUSTOM_FORMAT_LABEL = "Custom format";
public static final String CUSTOM_FORMAT_LABEL = "Custom format";
/**
* Parameter for combo box for an empty selection
*/
public static final String EMPTY_SELECTION = "";

private TypeDefinitionSelector sel;
private Spinner skipNlinesSpinner;
private Label setTypeLabel;
private Label skipNlinesLabels;
private ComboViewer dateFormatterCombo;
private Text customFormat;
private Label labelCustomFormat;
public Label labelCustomFormat;
private Label howToRepresentCustomFormatLabel;
private Label howToUseCustomFormatLabel;
protected Label dateFormatterLabel;

/**
* default constructor
Expand Down Expand Up @@ -140,8 +146,9 @@ public void selectionChanged(SelectionChangedEvent event) {
skipNlinesSpinner.setPageIncrement(10);

// create date formatter combo
Label dateFormatterLabel = new Label(page, SWT.NONE);
dateFormatterLabel.setText("Format for imported date values");
dateFormatterLabel = new Label(page, SWT.NONE);
dateFormatterLabel.setText("Reformatting of dates\r\n"
+ "(date values are tried to be detected automatically)");
dateFormatterCombo = new ComboViewer(page, SWT.READ_ONLY);
dateFormatterCombo.setContentProvider(ArrayContentProvider.getInstance());
List<String> list = createDatePatternsList();
Expand All @@ -153,26 +160,25 @@ public void selectionChanged(SelectionChangedEvent event) {
IStructuredSelection selection = (IStructuredSelection) event.getSelection();
if (selection.size() > 0 && labelCustomFormat != null && customFormat != null) {
String currentSelection = (String) selection.getFirstElement();
boolean isVisible = false;
if (currentSelection.equals(CUSTOM_FORMAT_LABEL)) {
// show the custom formatting label/text widget
labelCustomFormat.setVisible(true);
customFormat.setVisible(true);
}
else {
// hide the custom formatting label/text widget
labelCustomFormat.setVisible(false);
customFormat.setVisible(false);
isVisible = true;
}
labelCustomFormat.setVisible(isVisible);
customFormat.setVisible(isVisible);
howToRepresentCustomFormatLabel.setVisible(isVisible);
howToUseCustomFormatLabel.setVisible(isVisible);
}
}
});
dateFormatterCombo.setSelection(new StructuredSelection(list.get(0)));
dateFormatterCombo.setSelection(new StructuredSelection(createDatePatternsList().get(0)));

// add label and text area for a custom formatting
labelCustomFormat = new Label(page, SWT.NONE);
labelCustomFormat.setText("Custom date format:");
labelCustomFormat.setText("");
labelCustomFormat
.setLayoutData(GridDataFactory.swtDefaults().align(SWT.END, SWT.CENTER).create());
.setLayoutData(GridDataFactory.swtDefaults().align(SWT.LEFT, SWT.CENTER).create());

customFormat = new Text(page, SWT.BORDER | SWT.SINGLE);
customFormat.setLayoutData(GridDataFactory.swtDefaults().align(SWT.FILL, SWT.CENTER)
Expand All @@ -181,13 +187,30 @@ public void selectionChanged(SelectionChangedEvent event) {
labelCustomFormat.setVisible(false);
customFormat.setVisible(false);

howToRepresentCustomFormatLabel = new Label(page, SWT.NONE);
howToRepresentCustomFormatLabel.setText("How to represent");
howToRepresentCustomFormatLabel
.setLayoutData(GridDataFactory.swtDefaults().align(SWT.LEFT, SWT.CENTER).create());

howToUseCustomFormatLabel = new Label(page, SWT.NONE);
howToUseCustomFormatLabel.setText("dd: Day of the month\r\n" + "MM: Numerical month\r\n"
+ "yyyy: Year in four digits\r\n" + "hh: Hour in 24-hour format\r\n"
+ "mm: Minutes\r\n" + "ss: Seconds.");
howToUseCustomFormatLabel
.setLayoutData(GridDataFactory.swtDefaults().align(SWT.LEFT, SWT.CENTER).create());
howToRepresentCustomFormatLabel.setVisible(false);
howToUseCustomFormatLabel.setVisible(false);

page.pack();

setPageComplete(false);
}

private List<String> createDatePatternsList() {
return Arrays.asList(
/**
* @return list of formatting for date drop down
*/
protected List<String> createDatePatternsList() {
return Arrays.asList(EMPTY_SELECTION,
// Standard date formats
"yyyy-MM-dd", "yy-MM-dd", "dd-MM-yyyy", "MM-dd-yyyy", "yyyy/MM/dd", "dd/MM/yyyy",
"dd/MMM/yyyy", "MM/dd/yyyy", "yyyy.MM.dd", "dd.MM.yyyy", "MM.dd.yyyy", "yyyyMMdd",
Expand All @@ -202,12 +225,27 @@ public boolean updateConfiguration(InstanceReader provider) {
Value.of(skipNlinesSpinner.getSelection()));

if (customFormat.isVisible()) {
provider.setParameter(ReaderSettings.PARAMETER_DATE_FORMAT,

if (customFormat.getText().isBlank()) {
return false;
}

provider.setParameter(CSVConstants.PARAMETER_DATE_FORMAT,
Value.of(customFormat.getText()));
}
else {
provider.setParameter(ReaderSettings.PARAMETER_DATE_FORMAT,
Value.of(dateFormatterCombo.getSelection()));
// Get the selection from the combo viewer
String selectedValue = (String) ((IStructuredSelection) dateFormatterCombo
.getSelection()).getFirstElement();

if (selectedValue.equals("")) {
// The empty string is selected
provider.setParameter(CSVConstants.PARAMETER_DATE_FORMAT, null);
}
else {
provider.setParameter(CSVConstants.PARAMETER_DATE_FORMAT,
Value.of(dateFormatterCombo.getSelection()));
}
}

if (sel.getSelectedObject() != null) {
Expand Down
13 changes: 13 additions & 0 deletions io/plugins/eu.esdihumboldt.hale.io.csv/plugin.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,19 @@
sampleDescription="Any integer equal or larger than 0">
</valueDescriptor>
</providerParameter>
<providerParameter
description="Date Time Formatter specifying how values imported from Date cells should be imported"
label="Date Time Formatter"
name="dateTimeFormatterDefault"
optional="true">
<parameterBinding
class="java.lang.String">
</parameterBinding>
<valueDescriptor
default="dd.mm.yyyy"
defaultDescription="Default to dd.mm.yyyy">
</valueDescriptor>
</providerParameter>
<providerParameter
label="Type name"
name="typename"
Expand Down
6 changes: 6 additions & 0 deletions ...gins/eu.esdihumboldt.hale.io.csv/src/eu/esdihumboldt/hale/io/csv/reader/CSVConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,4 +64,10 @@ public interface CSVConstants {
*/
public static final char DEFAULT_DECIMAL = '.';

/**
* Parameter for the reader specifying how values imported from Date cells
* should be formatted.
*/
public static final String PARAMETER_DATE_FORMAT = "dateTimeFormatterDefault";

}
68 changes: 68 additions & 0 deletions ...dt.hale.io.csv/src/eu/esdihumboldt/hale/io/csv/reader/internal/CSVInstanceCollection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,16 @@
package eu.esdihumboldt.hale.io.csv.reader.internal;

import java.io.IOException;
import java.text.DateFormat;
import java.text.MessageFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.FormatStyle;
import java.util.Collections;
import java.util.Date;
import java.util.Map;
import java.util.NoSuchElementException;

Expand Down Expand Up @@ -174,12 +182,72 @@ public Instance next() {
return instance;
}

/**
* @param dateString String date
* @return Date
*/
public String parseDate(String dateString, String dateTime) {
DateFormat[] dateFormats = { DateFormat.getDateInstance(),
DateFormat.getDateTimeInstance(), new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"),
new SimpleDateFormat("MM/dd/yy HH:mm:ss"),
new SimpleDateFormat("MM/dd/yyyy HH:mm:ss"),
new SimpleDateFormat("MM-dd-yyyy HH:mm:ss"),
new SimpleDateFormat("MM-dd-yy HH:mm:ss"), new SimpleDateFormat("yyyy-MM-dd"),
new SimpleDateFormat("dd/MM/yyyy"), new SimpleDateFormat("dd/MMM/yyyy"),
new SimpleDateFormat("MM/dd/yy"), new SimpleDateFormat("MM/dd/yyyy"),
new SimpleDateFormat("yyyy/MM/dd"), new SimpleDateFormat("MM-dd-yyyy"),
new SimpleDateFormat("MM-dd-yy"), new SimpleDateFormat("yy-MM-dd"),
new SimpleDateFormat("dd-MM-yyyy"), new SimpleDateFormat("yyyy.MM.dd"),
new SimpleDateFormat("dd.MM.yyyy"), new SimpleDateFormat("MM.dd.yyyy"),
new SimpleDateFormat("yyyyMMdd"), new SimpleDateFormat("MMMM d, yyyy"),
new SimpleDateFormat("yy-MM"), new SimpleDateFormat("yyyy-MM"),
new SimpleDateFormat("MM-yy"), new SimpleDateFormat("MM-yyyy"),
// Add more date formats as needed
};

for (DateFormat dateFormat : dateFormats) {
try {
dateFormat.setLenient(false); // Disable lenient parsing
Date dateCellValue = dateFormat.parse(dateString);

// Convert java.util.Date to java.time.LocalDateTime
LocalDateTime localDateTime = dateCellValue.toInstant()
.atZone(ZoneId.systemDefault()).toLocalDateTime();

DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(dateTime);

// Define a DateTimeFormatter with a specific pattern
if (dateTimeFormatter == null) {
dateTimeFormatter = DateTimeFormatter.ofLocalizedDate(FormatStyle.SHORT);
}

// If parsing succeeds, break out of the loop
// Format LocalDateTime using DateTimeFormatter
return localDateTime.format(dateTimeFormatter);
} catch (ParseException e) {
// Parsing failed with this format, try the next one
}
}
return null;
}

private Object convertValue(String part, PropertyDefinition property) {
if (part == null || part.isEmpty()) {
// FIXME make this configurable?
return null;
}

try {
String dateTime = reader.getParameter(CSVUtil.PARAMETER_DATE_FORMAT)
.as(String.class);
if (dateTime != null && !part.isEmpty()) {
part = parseDate(part, dateTime);
}
} catch (Exception e) {
// Handle the exception appropriately, but no need for this
// test
}

Binding binding = property.getPropertyType().getConstraint(Binding.class);
try {
if (!binding.getBinding().equals(String.class)) {
Expand Down
Loading

0 comments on commit c3e9ecf

Please sign in to comment.