diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69a838b4..12b098f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-13, windows-latest] - jdk: [8, 11, 17] + jdk: [11, 17, 21] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index 29d57714..c5357366 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ project/plugins/project/ # IntelliJ specific .idea/* *.iml +.bsp/ # MacOS specific **/.DS_Store diff --git a/README.md b/README.md index ca6f7bde..bb642c77 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ The Validation tool and APIs are written in Scala 2.13 and may be used as: * A library in your Scala project. -* A library in your Java project (We provide a Java 8 interface, to make things simple for Java programmers too). +* A library in your Java project (We provide a Java 11 interface, to make things simple for Java programmers too). -The Validation Tool and APIs can be used on any Java Virtual Machine which supports Java 8 or better (**NB Java 6 support was removed in version 1.1**). The source code is +The Validation Tool and APIs can be used on any Java Virtual Machine which supports Java 11 or better (**NB Java 6 support was removed in version 1.1**). The source code is built using the [Apache Maven](https://maven.apache.org/) build tool: 1. For use in other Java/Scala Applications, build by executing `mvn clean install`. diff --git a/csv-validator-cmd/src/main/scala/uk/gov/nationalarchives/csv/validator/cmd/CsvValidatorCmdApp.scala b/csv-validator-cmd/src/main/scala/uk/gov/nationalarchives/csv/validator/cmd/CsvValidatorCmdApp.scala index 63f94bba..915c8d2a 100644 --- a/csv-validator-cmd/src/main/scala/uk/gov/nationalarchives/csv/validator/cmd/CsvValidatorCmdApp.scala +++ b/csv-validator-cmd/src/main/scala/uk/gov/nationalarchives/csv/validator/cmd/CsvValidatorCmdApp.scala @@ -20,7 +20,7 @@ import java.nio.charset.Charset import java.nio.file.{Files, Path, Paths} import java.text.DecimalFormat import java.util.jar.{Attributes, Manifest} -import scala.util.Using +import scala.util.{Try, Using} object SystemExitCodes extends Enumeration { type ExitCode = Int @@ -140,6 +140,15 @@ object CsvValidatorCmdApp extends App { case _ => } + def getColumnFromCsv(csvFile: TextFile, csvSchemaFile: TextFile, columnName: String): List[String] = Try { + val validator = createValidator(true, Nil, false, false) + val csv = validator.loadCsvFile(csvFile, csvSchemaFile) + csv.headOption.map(_.indexOf("identifier")).map { identifierIdx => + csv.tail.map(arr => arr(identifierIdx)) + }.getOrElse(Nil) + }.getOrElse(Nil) + + def validate( csvFile: TextFile, schemaFile: TextFile, diff --git a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidator.scala b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidator.scala index 8aa41fdd..2bc9f6aa 100644 --- a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidator.scala +++ b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidator.scala @@ -104,13 +104,7 @@ trait MetaDataValidator { validateKnownRows(csv, schema, pf, rowCallback) } - def validateKnownRows( - csv: JReader, - schema: Schema, - progress: Option[ProgressFor], - rowCallback: MetaDataValidation[Any] => Unit - ): Boolean = { - + def createCsvParser(schema: Schema): CsvParser = { val separator: Char = schema.globalDirectives.collectFirst { case Separator(sep) => sep @@ -135,8 +129,20 @@ trait MetaDataValidator { //format.setLineSeparator(CSV_RFC1480_LINE_SEPARATOR) // CRLF //we need a better CSV Reader! + new CsvParser(settings) + } + + + def validateKnownRows( + csv: JReader, + schema: Schema, + progress: Option[ProgressFor], + rowCallback: MetaDataValidation[Any] => Unit + ): Boolean = { + + val parser = createCsvParser(schema) + val result : Try[Boolean] = Using { - val parser = new CsvParser(settings) parser.beginParsing(csv) parser } { diff --git a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidator.scala b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidator.scala index c49b2643..57479497 100644 --- a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidator.scala +++ b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidator.scala @@ -10,12 +10,15 @@ package uk.gov.nationalarchives.csv.validator.api import cats.data.{Chain, Validated, ValidatedNel} import cats.implicits._ +import com.univocity.parsers.csv.{CsvParser, CsvParserSettings} import uk.gov.nationalarchives.csv.validator._ -import uk.gov.nationalarchives.csv.validator.schema.{Schema, SchemaParser} +import uk.gov.nationalarchives.csv.validator.schema.{Quoted, Schema, SchemaParser, Separator} import java.io.{Reader => JReader} import java.nio.charset.{Charset => JCharset} import java.nio.file.Path +import scala.jdk.CollectionConverters._ +import scala.util.Try object CsvValidator { @@ -71,7 +74,19 @@ trait CsvValidator extends SchemaParser { case Some(errors) => Validated.invalid(errors) } } - + + + + def loadCsvFile(csvFile: TextFile, csvSchemaFile: TextFile): List[Array[String]] = { + parseSchema(csvSchemaFile) match { + case Validated.Valid(schema) => + withReader(csvFile) { reader => + createCsvParser(schema).parseAll(reader) + }.asScala.toList + case Validated.Invalid(_) => Nil + } + } + def validateCsvFile( csvFile: TextFile, csvSchema: Schema, diff --git a/csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/CsvValidatorUi.scala b/csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/CsvValidatorUi.scala index 8e3d9668..61965083 100644 --- a/csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/CsvValidatorUi.scala +++ b/csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/CsvValidatorUi.scala @@ -27,14 +27,17 @@ import java.nio.file.{Files, Path, Paths, StandardOpenOption} import java.util import java.util.Properties import java.util.jar.{Attributes, Manifest} +import javax.swing.SpringLayout.Constraints import javax.swing._ import javax.swing.filechooser.FileNameExtensionFilter import javax.swing.table.DefaultTableModel import scala.jdk.CollectionConverters.CollectionHasAsScala import scala.language.reflectiveCalls +import scala.swing.FileChooser.SelectionMode import scala.swing.GridBagPanel.Anchor import scala.swing.PopupMenuImplicits._ import scala.swing._ +import scala.swing.event.ButtonClicked import scala.util.{Failure, Success, Try, Using} /** @@ -53,6 +56,9 @@ object CsvValidatorUi extends SimpleSwingApplication { super.startup(args) } + private lazy val txtCsvFile = new JTextField(30) + private lazy val txtCsvSchemaFile = new JTextField(30) + def top: SJXFrame = new SJXFrame { title = "CSV Validator" @@ -233,8 +239,6 @@ object CsvValidatorUi extends SimpleSwingApplication { peer.setTransferHandler(fileHandler) private val layout = new DesignGridLayout(peer) - private val txtCsvFile = new JTextField(30) - private def showErrorDialog(message: String) = { JOptionPane.showMessageDialog(parentFrame.peer, message, "Error", JOptionPane.ERROR_MESSAGE) false @@ -302,7 +306,7 @@ object CsvValidatorUi extends SimpleSwingApplication { } private val lblCsvSchemaFile = new Label("CSV Schema file:") - private val txtCsvSchemaFile = new JTextField(30) + txtCsvSchemaFile.setTransferHandler(fileHandler) private val csvSchemaFileChooser = new FileChooser(loadSettings match { case Some(s) => @@ -480,6 +484,42 @@ object CsvValidatorUi extends SimpleSwingApplication { private val cbEnforceCaseSensitivePathChecks = new CheckBox("Enforce case-sensitive file path checks?") cbEnforceCaseSensitivePathChecks.tooltip = "Performs additional checks to ensure that the case of file-paths in the CSV file match those of the filesystem" + private def tablePathDialog(): Unit = { + val csvFile = TextFile(Paths.get(txtCsvFile.getText), csvEncoding, validateUtf8) + val schemaFile = TextFile(Paths.get(txtCsvSchemaFile.getText), csvSchemaEncoding) + val identifierRows = CsvValidatorCmdApp.getColumnFromCsv(csvFile, schemaFile, "identifier").sorted + val fromPath = identifierRows.headOption.getOrElse("") + + val fileTextField = new TextField(30) + val fromPathText = new TextField(fromPath, 30) + + def pathToUri(path: Path) = { + val uri = path.toUri.toString + if (uri.endsWith("/")) uri else s"$uri/" + } + + def updateFileText(path: Path): Option[IOException] = { + fileTextField.text = pathToUri(path) + None + } + + val okButton = new Button("OK") + val fileButton = new Button("...") + fileButton.reactions += { + case ev: ButtonClicked => + val startingDir = if(fileTextField.text.isEmpty) userDir.toFile else Path.of(fileTextField.text).toFile + val fileChooser = new FileChooser(startingDir) + fileChooser.fileSelectionMode = SelectionMode.FilesAndDirectories + chooseFile(fileChooser, f => updateFileText(f), fileButton, s"Select the ${fromPath.split("/").last} folder") + } + + val rows = List( + Row("From", List(fromPathText)), + Row("To", List(fileTextField, fileButton)) + ) + addToTableDialog(parentFrame, "Add path substitution...", rows, tblPathSubstitutions.addRow) + } + private val tblPathSubstitutions = new Table(0, 2) { preferredViewportSize = new Dimension(500, 70) model = new DefaultTableModel(Array[Object]("From", "To"), 0) @@ -505,7 +545,8 @@ object CsvValidatorUi extends SimpleSwingApplication { private val spTblPathSubstitutions = new ScrollPane(tblPathSubstitutions) private val btnAddPathSubstitution = new Button("Add Path Substitution...") - btnAddPathSubstitution.reactions += onClick(addToTableDialog(parentFrame, "Add Path Substitution...", tblPathSubstitutions, tblPathSubstitutions.addRow)) + + btnAddPathSubstitution.reactions += onClick(tablePathDialog()) private val settingsGroupLayout = new GridBagPanel { private val c = new Constraints diff --git a/csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/ScalaSwingHelpers.scala b/csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/ScalaSwingHelpers.scala index c8c45a6e..b7408434 100644 --- a/csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/ScalaSwingHelpers.scala +++ b/csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/ScalaSwingHelpers.scala @@ -42,8 +42,8 @@ object ScalaSwingHelpers { * @param result A function which takes the chosen file * @param locateOver A component over which the FileChooser dialog should be located */ - def chooseFile(fileChooser: FileChooser, result: Path => Option[IOException], locateOver: Component) : Unit = { - fileChooser.showSaveDialog(locateOver) match { + def chooseFile(fileChooser: FileChooser, result: Path => Option[IOException], locateOver: Component, dialogText: String = "Save") : Unit = { + fileChooser.showDialog(locateOver, dialogText) match { case Result.Approve => result(fileChooser.selectedFile.toPath) match { case Some(ioe) => @@ -64,28 +64,33 @@ object ScalaSwingHelpers { * @param table The table to create a dialog for * @param result A function which takes a row as the result of the dialog box */ - def addToTableDialog(owner: Window, title: String, table: Table, result: Array[String] => Unit) : Unit = { + case class Row(label: String, components: List[Component]) + val c = List() + def addToTableDialog(owner: Window, title: String, rows: List[Row], result: Array[String] => Unit) : Unit = { val btnOk = new Button("Ok") val optionLayout: GridBagPanel = new GridBagPanel { val c = new Constraints - - for(colIdx <- 0 to table.model.getColumnCount - 1) { - c.gridx = 0 - c.gridy = colIdx - c.anchor = Anchor.LineStart - layout(new Label(table.model.getColumnName(colIdx) + ":")) = c - - c.gridx = 1 - c.gridy = colIdx - c.anchor = Anchor.LineStart - layout(new TextField(30)) = c + rows.zipWithIndex.map { + case (row, colIdx) => + c.gridx = 0 + c.gridy = colIdx + c.anchor = Anchor.LineStart + layout(new Label(row.label + ":")) = c + + row.components.zipWithIndex.map { + case (component, rowIdx) => + c.gridx = rowIdx + 1 + c.gridy = colIdx + c.anchor = Anchor.LineStart + layout(component) = c + } } c.gridx = 0 - c.gridy = table.model.getColumnCount - c.gridwidth = 2 + c.gridy = rows.size + c.gridwidth = rows.size + 1 c.anchor = Anchor.LineEnd layout(btnOk) = c }