Merge branch 'emanuelaepure10-feat/ING-4165'

io/plugins/eu.esdihumboldt.hale.io.csv.test/src/data/test3.csv

@@ -0,0 +1,3 @@
+Name	Xcoord	Ycoord	date
+test	12	16	1.02.2023
+test3	45	2	22.03.2023

io/plugins/eu.esdihumboldt.hale.io.csv.test/src/eu/esdihumboldt/hale/io/csv/CSVInstanceReaderTest.java

@@ -16,8 +16,14 @@
 package eu.esdihumboldt.hale.io.csv;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Iterator;
+
 import javax.xml.namespace.QName;
 
 import org.junit.BeforeClass;
@@ -35,6 +41,7 @@
 import eu.esdihumboldt.hale.common.schema.model.Schema;
 import eu.esdihumboldt.hale.common.schema.model.TypeDefinition;
 import eu.esdihumboldt.hale.common.test.TestUtil;
+import eu.esdihumboldt.hale.io.csv.reader.CSVConstants;
 import eu.esdihumboldt.hale.io.csv.reader.CommonSchemaConstants;
 import eu.esdihumboldt.hale.io.csv.reader.internal.CSVInstanceReader;
 import eu.esdihumboldt.hale.io.csv.reader.internal.CSVSchemaReader;
@@ -238,6 +245,67 @@ public void testReadWithCommaDecimal() throws Exception {
 
 	}
 
+	/**
+	 * Test - read a sample csv schema and data.
+	 * 
+	 * @throws Exception , if an error occurs
+	 */
+	@Test
+	public void testReadSimpleWithDate() throws Exception {
+
+		String typeName = "location";
+		String[] properties = { "Name", "Xcoord", "Ycoord", "date" };
+		String[] dataFirstColumn = { "test", "12", "16", "1.02.2023" };
+		String dateFormatter = "dd.MM.yyyy";
+		String sourceLocation = "/data/test3.csv";
+		// read Schema ###
+		Schema schema = readCSVSchemaDate(sourceLocation, typeName,
+				"java.lang.String,java.lang.String,java.lang.String,java.lang.String",
+				"Name,Xcoord,Ycoord,date", null, null, null, null, dateFormatter);
+
+		// read Instances ###
+		InstanceCollection instances = readCSVInstances(sourceLocation, typeName, 1, schema, null,
+				null, null);
+
+		// Check the values of the date property in each instance
+		Iterator<Instance> instanceIt = instances.iterator();
+		while (instanceIt.hasNext()) {
+			Instance instance = instanceIt.next();
+			// Get the value of the date property
+			Object[] value = instance.getProperty(QName.valueOf(properties[properties.length - 1]));
+
+			// Ensure the value is not null
+			assertNotNull("Date property value is null", value);
+
+			// Ensure the value is an array with at least one element
+			assertTrue("Date property value is not an array or is empty", value.length > 0);
+
+			// Check the date string format
+			String dateString = (String) value[0];
+			assertTrue("Date string format is incorrect: " + dateString,
+					isStringDate(dateString, dateFormatter));
+		}
+	}
+
+	/**
+	 * @param input String
+	 * @param dateFormatter date formatter
+	 * @return true is the input String is of type Date
+	 */
+	public boolean isStringDate(String input, String dateFormatter) {
+		// Define the date format you expect
+		SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormatter);
+		dateFormat.setLenient(false); // Disable lenient parsing
+
+		try {
+			// Try parsing the input string as a date
+			Date parsedDate = dateFormat.parse(input);
+			return true; // Parsing successful, input is a valid date
+		} catch (ParseException e) {
+			return false; // Parsing failed, input is not a valid date
+		}
+	}
+
 	private int collectionSize(InstanceCollection instances) {
 		if (instances.hasSize()) {
 			return instances.size();
@@ -280,6 +348,28 @@ private Schema readCSVSchema(String sourceLocation, String typeName, String para
 		return schemaReader.getSchema();
 	}
 
+	private Schema readCSVSchemaDate(String sourceLocation, String typeName,
+			String paramPropertyType, String propertyNames, String seperator, String quote,
+			String escape, String decimal, String dateFormatter) throws Exception {
+
+		CSVSchemaReader schemaReader = new CSVSchemaReader();
+		schemaReader.setSource(
+				new DefaultInputSupplier(getClass().getResource(sourceLocation).toURI()));
+		schemaReader.setParameter(CommonSchemaConstants.PARAM_TYPENAME, Value.of(typeName));
+		schemaReader.setParameter(CSVSchemaReader.PARAM_PROPERTY, Value.of(propertyNames));
+		schemaReader.setParameter(CSVSchemaReader.PARAM_PROPERTYTYPE, Value.of(paramPropertyType));
+		schemaReader.setParameter(CSVSchemaReader.PARAM_SEPARATOR, Value.of(seperator));
+		schemaReader.setParameter(CSVSchemaReader.PARAM_QUOTE, Value.of(quote));
+		schemaReader.setParameter(CSVSchemaReader.PARAM_ESCAPE, Value.of(escape));
+		schemaReader.setParameter(CSVSchemaReader.PARAM_DECIMAL, Value.of(decimal));
+		schemaReader.setParameter(CSVConstants.PARAMETER_DATE_FORMAT, Value.of(dateFormatter));
+
+		IOReport report = schemaReader.execute(new LogProgressIndicator());
+		assertTrue(report.isSuccess());
+
+		return schemaReader.getSchema();
+	}
+
 	private InstanceCollection readCSVInstances(String sourceLocation, String typeName, int skipN,
 			Schema sourceSchema, String seperator, String quote, String escape) throws Exception {
 

io/plugins/eu.esdihumboldt.hale.io.csv.ui/src/eu/esdihumboldt/hale/io/csv/ui/TypeSelectionPage.java

@@ -44,7 +44,6 @@
 import eu.esdihumboldt.hale.io.csv.reader.CSVConstants;
 import eu.esdihumboldt.hale.io.csv.reader.CommonSchemaConstants;
 import eu.esdihumboldt.hale.io.csv.reader.internal.CSVConfiguration;
-import eu.esdihumboldt.hale.io.xls.reader.ReaderSettings;
 import eu.esdihumboldt.hale.ui.common.definition.selector.TypeDefinitionSelector;
 import eu.esdihumboldt.hale.ui.io.instance.InstanceReaderConfigurationPage;
 import eu.esdihumboldt.hale.ui.service.schema.SchemaService;
@@ -202,11 +201,11 @@ public boolean updateConfiguration(InstanceReader provider) {
 				Value.of(skipNlinesSpinner.getSelection()));
 
 		if (customFormat.isVisible()) {
-			provider.setParameter(ReaderSettings.PARAMETER_DATE_FORMAT,
-					Value.of(customFormat.getText()));
+			provider.setParameter(CSVConstants.PARAMETER_DATE_FORMAT, Value.of(customFormat
+					.getText().replace("mm", "MM").replace("DD", "dd").replace("YYYY", "yyyy")));
 		}
 		else {
-			provider.setParameter(ReaderSettings.PARAMETER_DATE_FORMAT,
+			provider.setParameter(CSVConstants.PARAMETER_DATE_FORMAT,
 					Value.of(dateFormatterCombo.getSelection()));
 		}
 

io/plugins/eu.esdihumboldt.hale.io.csv/plugin.xml

@@ -68,6 +68,19 @@
                   sampleDescription="Any integer equal or larger than 0">
             </valueDescriptor>
          </providerParameter>
+         <providerParameter
+               description="Date Time Formatter specifying how values imported from Date cells should be imported"
+               label="Date Time Formatter"
+               name="dateTimeFormatterDefault"
+               optional="true">
+            <parameterBinding
+                  class="java.lang.String">
+            </parameterBinding>
+            <valueDescriptor
+                  default="dd.mm.yyyy"
+                  defaultDescription="Default to dd.mm.yyyy">
+            </valueDescriptor>
+         </providerParameter>
          <providerParameter
                label="Type name"
                name="typename"

io/plugins/eu.esdihumboldt.hale.io.csv/src/eu/esdihumboldt/hale/io/csv/reader/CSVConstants.java

@@ -64,4 +64,10 @@ public interface CSVConstants {
 	 */
 	public static final char DEFAULT_DECIMAL = '.';
 
+	/**
+	 * Parameter for the reader specifying how values imported from Date cells
+	 * should be formatted.
+	 */
+	public static final String PARAMETER_DATE_FORMAT = "dateTimeFormatterDefault";
+
 }

io/plugins/eu.esdihumboldt.hale.io.csv/src/eu/esdihumboldt/hale/io/csv/reader/internal/CSVInstanceCollection.java

@@ -16,8 +16,16 @@
 package eu.esdihumboldt.hale.io.csv.reader.internal;
 
 import java.io.IOException;
+import java.text.DateFormat;
 import java.text.MessageFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.FormatStyle;
 import java.util.Collections;
+import java.util.Date;
 import java.util.Map;
 import java.util.NoSuchElementException;
 
@@ -174,12 +182,72 @@ public Instance next() {
 			return instance;
 		}
 
+		/**
+		 * @param dateString String date
+		 * @return Date
+		 */
+		public String parseDate(String dateString, String dateTime) {
+			DateFormat[] dateFormats = { DateFormat.getDateInstance(),
+					DateFormat.getDateTimeInstance(), new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"),
+					new SimpleDateFormat("MM/dd/yy HH:mm:ss"),
+					new SimpleDateFormat("MM/dd/yyyy HH:mm:ss"),
+					new SimpleDateFormat("MM-dd-yyyy HH:mm:ss"),
+					new SimpleDateFormat("MM-dd-yy HH:mm:ss"), new SimpleDateFormat("yyyy-MM-dd"),
+					new SimpleDateFormat("dd/MM/yyyy"), new SimpleDateFormat("dd/MMM/yyyy"),
+					new SimpleDateFormat("MM/dd/yy"), new SimpleDateFormat("MM/dd/yyyy"),
+					new SimpleDateFormat("yyyy/MM/dd"), new SimpleDateFormat("MM-dd-yyyy"),
+					new SimpleDateFormat("MM-dd-yy"), new SimpleDateFormat("yy-MM-dd"),
+					new SimpleDateFormat("dd-MM-yyyy"), new SimpleDateFormat("yyyy.MM.dd"),
+					new SimpleDateFormat("dd.MM.yyyy"), new SimpleDateFormat("MM.dd.yyyy"),
+					new SimpleDateFormat("yyyyMMdd"), new SimpleDateFormat("MMMM d, yyyy"),
+					new SimpleDateFormat("yy-MM"), new SimpleDateFormat("yyyy-MM"),
+					new SimpleDateFormat("MM-yy"), new SimpleDateFormat("MM-yyyy"),
+					// Add more date formats as needed
+			};
+
+			for (DateFormat dateFormat : dateFormats) {
+				try {
+					dateFormat.setLenient(false); // Disable lenient parsing
+					Date dateCellValue = dateFormat.parse(dateString);
+
+					// Convert java.util.Date to java.time.LocalDateTime
+					LocalDateTime localDateTime = dateCellValue.toInstant()
+							.atZone(ZoneId.systemDefault()).toLocalDateTime();
+
+					DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(dateTime);
+
+					// Define a DateTimeFormatter with a specific pattern
+					if (dateTimeFormatter == null) {
+						dateTimeFormatter = DateTimeFormatter.ofLocalizedDate(FormatStyle.SHORT);
+					}
+
+					// If parsing succeeds, break out of the loop
+					// Format LocalDateTime using DateTimeFormatter
+					return localDateTime.format(dateTimeFormatter);
+				} catch (ParseException e) {
+					// Parsing failed with this format, try the next one
+				}
+			}
+			return null;
+		}
+
 		private Object convertValue(String part, PropertyDefinition property) {
 			if (part == null || part.isEmpty()) {
 				// FIXME make this configurable?
 				return null;
 			}
 
+			try {
+				String dateTime = reader.getParameter(CSVUtil.PARAMETER_DATE_FORMAT)
+						.as(String.class);
+				if (dateTime != null && !part.isEmpty()) {
+					part = parseDate(part, dateTime);
+				}
+			} catch (Exception e) {
+				// Handle the exception appropriately, but no need for this
+				// trial
+			}
+
 			Binding binding = property.getPropertyType().getConstraint(Binding.class);
 			try {
 				if (!binding.getBinding().equals(String.class)) {

io/plugins/eu.esdihumboldt.hale.io.csv/src/eu/esdihumboldt/hale/io/csv/reader/internal/CSVInstanceReader.java

@@ -28,6 +28,7 @@
 import eu.esdihumboldt.hale.common.instance.io.impl.AbstractInstanceReader;
 import eu.esdihumboldt.hale.common.instance.model.InstanceCollection;
 import eu.esdihumboldt.hale.io.csv.CSVFileIO;
+import eu.esdihumboldt.hale.io.csv.reader.CSVConstants;
 
 /**
  * Reads instances from a CSVfile
@@ -38,6 +39,15 @@ public class CSVInstanceReader extends AbstractInstanceReader {
 
 	private InstanceCollection instances;
 
+	/**
+	 * Default constructor.
+	 */
+	public CSVInstanceReader() {
+		super();
+
+		addSupportedParameter(CSVConstants.PARAMETER_DATE_FORMAT);
+	}
+
 	/**
 	 * @see IOProvider#isCancelable()
 	 */

io/plugins/eu.esdihumboldt.hale.io.xls.test/src/eu/esdihumboldt/hale/io/xls/test/reader/XLSReaderTest.java

@@ -48,6 +48,7 @@
 import eu.esdihumboldt.hale.common.schema.model.constraint.type.Binding;
 import eu.esdihumboldt.hale.common.test.TestUtil;
 import eu.esdihumboldt.hale.io.csv.InstanceTableIOConstants;
+import eu.esdihumboldt.hale.io.csv.reader.CSVConstants;
 import eu.esdihumboldt.hale.io.csv.reader.CommonSchemaConstants;
 import eu.esdihumboldt.hale.io.csv.reader.internal.AbstractTableSchemaReader;
 import eu.esdihumboldt.hale.io.xls.reader.ReaderSettings;
@@ -674,7 +675,7 @@ private Schema readXLSSchemaDate(String sourceLocation, int sheetIndex, String t
 		schemaReader.setParameter(CommonSchemaConstants.PARAM_TYPENAME, Value.of(typeName));
 		schemaReader.setParameter(AbstractTableSchemaReader.PARAM_PROPERTYTYPE,
 				Value.of(paramPropertyType));
-		schemaReader.setParameter(ReaderSettings.PARAMETER_DATE_FORMAT, Value.of(dateFormatter));
+		schemaReader.setParameter(CSVConstants.PARAMETER_DATE_FORMAT, Value.of(dateFormatter));
 
 		IOReport report = schemaReader.execute(null);
 		assertTrue("Schema import was not successfull.", report.isSuccess());

io/plugins/eu.esdihumboldt.hale.io.xls.ui/src/eu/esdihumboldt/hale/io/xls/ui/XLSInstanceImportConfigurationPage.java

@@ -30,11 +30,9 @@
 import eu.esdihumboldt.hale.common.core.io.Value;
 import eu.esdihumboldt.hale.common.instance.io.InstanceReader;
 import eu.esdihumboldt.hale.io.csv.InstanceTableIOConstants;
+import eu.esdihumboldt.hale.io.csv.ui.TypeSelectionPage;
 import eu.esdihumboldt.hale.io.xls.AbstractAnalyseTable;
 import eu.esdihumboldt.hale.io.xls.reader.ReaderSettings;
-import eu.esdihumboldt.hale.ui.io.config.AbstractConfigurationPage;
-import eu.esdihumboldt.hale.ui.io.instance.InstanceImportWizard;
-import eu.esdihumboldt.hale.io.csv.ui.TypeSelectionPage;
 
 /**
  * Configuration page for the instance export provider of Excel files

io/plugins/eu.esdihumboldt.hale.io.xls/src/eu/esdihumboldt/hale/io/xls/reader/ReaderSettings.java

@@ -33,6 +33,7 @@
 import eu.esdihumboldt.hale.common.core.io.Value;
 import eu.esdihumboldt.hale.common.core.io.ValueList;
 import eu.esdihumboldt.hale.io.csv.InstanceTableIOConstants;
+import eu.esdihumboldt.hale.io.csv.reader.CSVConstants;
 import eu.esdihumboldt.hale.io.csv.reader.CommonSchemaConstants;
 import eu.esdihumboldt.hale.io.xls.AbstractAnalyseTable;
 
@@ -53,12 +54,6 @@ public class ReaderSettings {
 	 */
 	public static final String PARAMETER_SHEET_SETTINGS = "sheetSettings";
 
-	/**
-	 * Parameter for the reader specifying how values imported from Date cells
-	 * should be formatted.
-	 */
-	public static final String PARAMETER_DATE_FORMAT = "dateTimeFormatterDefault";
-
 	/**
 	 * Collect information and settings on a single sheet.
 	 */
@@ -201,7 +196,8 @@ else if (skipType) {
 		}
 
 		// read dateFormat
-		String dateFormatString = reader.getParameter(PARAMETER_DATE_FORMAT).as(String.class);
+		String dateFormatString = reader.getParameter(CSVConstants.PARAMETER_DATE_FORMAT)
+				.as(String.class);
 
 		// apply to all sheets as default
 		for (SheetInfo sheet : sheets) {

io/plugins/eu.esdihumboldt.hale.io.xls/src/eu/esdihumboldt/hale/io/xls/reader/XLSInstanceReader.java

@@ -42,6 +42,7 @@
 import eu.esdihumboldt.hale.common.schema.model.TypeDefinition;
 import eu.esdihumboldt.hale.common.schema.model.TypeIndex;
 import eu.esdihumboldt.hale.common.schema.model.constraint.type.Binding;
+import eu.esdihumboldt.hale.io.csv.reader.CSVConstants;
 import eu.esdihumboldt.hale.io.csv.reader.internal.CSVInstanceReader;
 import eu.esdihumboldt.hale.io.xls.AnalyseXLSSchemaTable;
 import eu.esdihumboldt.hale.io.xls.reader.ReaderSettings.SheetInfo;
@@ -62,7 +63,7 @@ public class XLSInstanceReader extends AbstractInstanceReader {
 	public XLSInstanceReader() {
 		super();
 
-		addSupportedParameter(ReaderSettings.PARAMETER_DATE_FORMAT);
+		addSupportedParameter(CSVConstants.PARAMETER_DATE_FORMAT);
 	}
 
 	/**

io/plugins/eu.esdihumboldt.hale.io.xls/src/eu/esdihumboldt/hale/io/xls/reader/XLSSchemaReader.java

@@ -39,6 +39,7 @@
 import eu.esdihumboldt.hale.io.csv.InstanceTableIOConstants;
 import eu.esdihumboldt.hale.io.csv.PropertyType;
 import eu.esdihumboldt.hale.io.csv.PropertyTypeExtension;
+import eu.esdihumboldt.hale.io.csv.reader.CSVConstants;
 import eu.esdihumboldt.hale.io.csv.reader.CommonSchemaConstants;
 import eu.esdihumboldt.hale.io.csv.reader.internal.AbstractTableSchemaReader;
 import eu.esdihumboldt.hale.io.csv.reader.internal.CSVConfiguration;
@@ -72,7 +73,7 @@ protected Schema loadFromSource(ProgressIndicator progress, IOReporter reporter)
 			throws IOProviderConfigurationException, IOException {
 
 		sheetNum = getParameter(InstanceTableIOConstants.SHEET_INDEX).as(int.class, 0);
-		String dateTime = getParameter(ReaderSettings.PARAMETER_DATE_FORMAT).as(String.class);
+		String dateTime = getParameter(CSVConstants.PARAMETER_DATE_FORMAT).as(String.class);
 
 		progress.begin("Load XLS/XLSX schema", ProgressIndicator.UNKNOWN);